by_party_temp.close()
reg_by_party_dict, sessions = parse_excel_by_party('/tmp/temp.excel')
except Exception as e:
logger_to_db.error("No party excel file was found for ID %s due to %s"%(ID,str(e)))
problem_by_party = True
if problem_by_name or problem_by_party:
cur.execute("""INSERT INTO stenograms VALUES (%s, %s, %s, %s, %s)""",
(parser.date, parser.data_list, parser.votes_indices, True, original_url))
else:
try:
assert all([len(sessions) == len(mp_votes) for mp_votes in mp_vote_sessions]), "Not all sessions are recorded."
cur.execute("""INSERT INTO stenograms VALUES (%s, %s, %s, %s, %s)""",
(parser.date, parser.data_list, parser.votes_indices, False, original_url))
cur.executemany("""INSERT INTO party_reg VALUES (%s, %s, %s, %s)""",
((k, parser.date, v.present, v.expected) for k,v in reg_by_party_dict.items()))
cur.executemany("""INSERT INTO vote_sessions VALUES (%s, %s, %s, %s)""",
((parser.date, i, s.description, s.time) for i, s in enumerate(sessions)))
cur.executemany("""INSERT INTO party_votes VALUES (%s, %s, %s, %s, %s, %s, %s)""",
((party, parser.date, i, votes.yes, votes.no, votes.abstained, votes.total)
for i, s in enumerate(sessions)
for party, votes in s.votes_by_party_dict.items()))
cur.executemany("""INSERT INTO mp_reg VALUES (%s, %s, %s, %s)""",
((name, party, parser.date, reg) for name, party, reg in zip(mp_names, mp_parties, mp_reg_session)))
cur.executemany("""INSERT INTO mp_votes VALUES (%s, %s, %s, %s, %s)""",
((name, party, parser.date, i, v)
for name, party, votes in zip(mp_names, mp_parties, mp_vote_sessions)
for i, v in enumerate(votes)))
except Exception as e:
logger_to_db.error("Writting to db failed on stenogram %s due to %s" % (ID, str(e)))
db.commit()
)
.encode("UTF-8")
.upper()
.strip()
)
force = " ".join(r["schema"]["Profile"]["PoliticalForce"]["@value"].split(" ")[:-1])
force = canonical_party_name(force).encode("UTF-8")
mail = r["schema"]["Profile"]["E-mail"]["@value"].encode("UTF-8").replace(";", ",").replace(":", ",").strip()
except xml.parsers.expat.ExpatError:
logger_mps.warning("Parsing the xml file for MP %s failed. Trying csv." % i)
try:
csv_file = urlopen("http://www.parliament.bg/export.php/bg/csv/MP/%d" % i)
data = [l.strip().replace(""", '"').split(";")[:-1] for l in csv_file.readlines()]
name = " ".join([d.strip() for d in data[0]])
mail = ", ".join([d.strip() for d in data[9][1:]])
mail = mail.replace(";", ",").replace(":", ",")
force = " ".join(data[6][-1].decode("UTF-8").split(" ")[:-1])
force = canonical_party_name(force).encode("UTF-8")
except Exception, e:
logger_mps.error("The csv file for MP %s is unparsable as well due to %s. Skipping this id." % (i, str(e)))
continue
url_list.append(original_url)
names_list.append(name)
forces_list.append(force)
mails_list.append(mail)
cur.executemany("""INSERT INTO parties VALUES (%s)""", zip(list(set(forces_list)) + [u"независим"]))
cur.executemany("""INSERT INTO mps VALUES (%s, %s, %s, %s)""", zip(names_list, forces_list, mails_list, url_list))
db.commit()