def insert_meeting(region_id, obj, update=False):
with transaction() as session:
meeting = create_or_get_meeting(session, region_id, obj, update)
if not meeting:
return
session.add(meeting)
# 'meeting_attendee' table
attendee_names = get_attendee_names(obj)
attendees = list(get_attendees(meeting, attendee_names, session))
meeting.attendees = attendees
# clear the meeting's statements
for statement in meeting.statements:
session.delete(statement)
session.flush()
meeting.statements = []
# 'statement' table
statements = (stmt for stmt in obj['dialogue']
if stmt['type'] == 'statement')
for seq, statement in enumerate(statements):
item = create_statement(meeting, seq, statement, attendees)
session.add(item)
session.flush()
statement['person_id'] = item.person_id
statement['id'] = item.id
meeting.statements.append(item)
# Updated dialog field of meeting table
meeting.dialogue = obj['dialogue']
def insert_meeting(region_id, obj, update=False):
with transaction() as session:
meeting = create_or_get_meeting(session, region_id, obj, update)
if not meeting:
return
session.add(meeting)
# 'meeting_attendee' table
attendee_names = get_attendee_names(obj)
attendees = list(get_attendees(meeting, attendee_names, session))
meeting.attendees = attendees
# clear the meeting's statements
for statement in meeting.statements:
session.delete(statement)
session.flush()
meeting.statements = []
# 'statement' table
statements = (stmt for stmt in obj['dialogue']
if stmt['type'] == 'statement')
for seq, statement in enumerate(statements):
item = create_statement(meeting, seq, statement, attendees)
session.add(item)
session.flush()
statement['person_id'] = item.person_id
statement['id'] = item.id
meeting.statements.append(item)
# Updated dialog field of meeting table
meeting.dialogue = obj['dialogue']
def insert_bill_keywords(files):
hannanum = Hannanum()
with transaction() as session:
existing_bill_ids = set(r[0] for r in session.query(Bill.id))
keyword_store = KeywordStore(session)
for file_ in glob(files):
filename = basename(file_)
print 'processing %s' % filename
sys.stdout.flush()
bill_id = filename.split('.', 1)[0]
if bill_id not in existing_bill_ids:
continue
with open(file_, 'r') as f:
keywords = extract_keywords(f, hannanum)
keyword_ids = [
keyword_store.id(keyword[0]) for keyword in keywords
]
keyword_store.sync()
existing_keywords_for_bill = set(r[0]
for r in session.query(bill_keyword.c.keyword_id)\
.filter(bill_keyword.c.bill_id == bill_id)
)
new_bill_keywords = [
{
'bill_id': bill_id,
'keyword_id': keyword_id,
'weight': weight,
} for (_, weight), keyword_id in izip(keywords, keyword_ids)
if keyword_id not in existing_keywords_for_bill
]
if new_bill_keywords:
# TODO(lucypark, 2014-12-13): delete existing keywords before insert
session.execute(bill_keyword.insert(), new_bill_keywords)
def update_bills(source, files=None):
if source == 'redis':
queue = RedisQueue(REDIS_KEYS['insert_bills_db'], **REDIS_SETTINGS)
files = (bill_filepath(bill_id) for bill_id in queue)
elif source.startswith('db'):
with transaction() as session:
assembly_id = session.query(Election)\
.order_by(Election.assembly_id.desc())\
.first().assembly_id
# FIXME: filter finished bills out
bill_ids = session.query(Bill.id).filter_by(assembly_id=assembly_id)
# ranged query
m = re.match(r'db\[(\d*):(\d*)\]', source)
if m:
start, end = m.group(1), m.group(2)
offset = int(start) if start else 0
limit = int(end) - offset if end else None
if offset:
bill_ids = bill_ids.offset(offset)
if limit:
bill_ids = bill_ids.limit(limit)
bill_ids = (record[0] for record in bill_ids)
files = (bill_filepath(bill_id) for bill_id in bill_ids)
elif files:
files = [f for path in files for f in glob(path)]
update_bills_from_files(files)
def update_bills(source, files=None):
if source == 'redis':
queue = RedisQueue(REDIS_KEYS['insert_bills_db'], **REDIS_SETTINGS)
files = (bill_filepath(bill_id) for bill_id in queue)
elif source.startswith('db'):
with transaction() as session:
assembly_id = session.query(Election)\
.order_by(Election.assembly_id.desc())\
.first().assembly_id
# FIXME: filter finished bills out
bill_ids = session.query(Bill.id).filter_by(assembly_id=assembly_id)
# ranged query
m = re.match(r'db\[(\d*):(\d*)\]', source)
if m:
start, end = m.group(1), m.group(2)
offset = int(start) if start else 0
limit = int(end) - offset if end else None
if offset:
bill_ids = bill_ids.offset(offset)
if limit:
bill_ids = bill_ids.limit(limit)
bill_ids = (record[0] for record in bill_ids)
files = (bill_filepath(bill_id) for bill_id in bill_ids)
elif files:
files = [f for path in files for f in glob(path)]
update_bills_from_files(files)
def insert_candidacies(files, election_type, assembly_id, date):
with transaction() as session:
for file_ in glob(files):
with open(file_, 'r') as f:
list_ = json.load(f)
for record in list_:
person_id = insert_person(session, election_type, assembly_id, record)
insert_party(session, record)
insert_election(session, assembly_id, date)
insert_candidacy(session, record, person_id, date)
def insert_candidacies(files, election_type, assembly_id, date):
with transaction() as session:
for file_ in glob(files):
with open(file_, 'r') as f:
list_ = json.load(f)
for record in list_:
person_id = insert_person(session, election_type, assembly_id,
record)
insert_party(session, record)
insert_election(session, assembly_id, date)
insert_candidacy(session, record, person_id, date)
def update_people(fd, date):
if not fd:
return
connect_db(db_session)
with transaction() as session:
records = json.load(fd)
for record in records:
preprocess_record(record)
person_id = update_person(session, record)
if not person_id:
continue
party_id = insert_party(session, record['party'])
insert_party_affiliation(session, person_id, party_id, date)
def update_people(fd, date):
if not fd:
return
connect_db(db_session)
with transaction() as session:
records = json.load(fd)
for record in records:
preprocess_record(record)
person_id = update_person(session, record)
if not person_id:
continue
party_id = insert_party(session, record['party'])
insert_party_affiliation(session, person_id, party_id, date)
def insert_meeting(region_id, obj, update=False):
with transaction() as session:
meeting = create_or_get_meeting(session, region_id, obj, update)
if not meeting:
return
session.add(meeting)
# 'meeting_attendee' table
attendee_names = get_attendee_names(obj)
attendees = list(get_attendees(meeting, attendee_names, session))
meeting.attendees = attendees
# clear the meeting's statements
for statement in meeting.statements:
session.delete(statement)
session.flush()
meeting.statements = []
# 'statement' table
statements = (stmt for stmt in obj['dialogue']
if stmt['type'] == 'statement')
for seq, statement in enumerate(statements):
item = create_statement(meeting, seq, statement, attendees)
session.add(item)
session.flush()
statement['person_id'] = item.person_id
statement['id'] = item.id
meeting.statements.append(item)
# Updated dialog field of meeting table
meeting.dialogue = obj['dialogue']
# TODO: votes = obj['votes']
# The following code should be reviewed
meeting.votes = []
options = ['yea', 'nay', 'forfeit']
for option in options:
if option in stmt['votes']:
for person_name in stmt['votes'][option]:
person = guess_attendee(attendees, person_name)
statement = guess_statement(statements, stmt['name'])
if person and statement:
item = create_vote(meeting, statement, person, option)
session.add(item)
meeting.votes.append(item)
def update_bills_from_files(files):
if not files:
return
with transaction() as session:
bill_statuses.init(session)
for f in files:
try:
if not isinstance(f, file):
f = open(f, 'r')
with f:
record = json.load(f)
except Exception, e:
print >> sys.stderr, e
continue
insert_bill(session, record)
bill_statuses.insert_all()
def update_bills_from_files(files):
if not files:
return
with transaction() as session:
bill_statuses.init(session)
for f in files:
try:
if not isinstance(f, file):
f = open(f, 'r')
with f:
record = json.load(f)
except Exception, e:
print >> sys.stderr, e
continue
insert_bill(session, record)
bill_statuses.insert_all()
def insert_bill_keywords(files):
hannanum = Hannanum()
with transaction() as session:
existing_bill_ids = set(r[0] for r in session.query(Bill.id))
keyword_store = KeywordStore(session)
for file_ in glob(files):
filename = basename(file_)
print 'processing %s,' % filename,
bill_id = filename.split('.', 1)[0]
if bill_id not in existing_bill_ids:
continue
with open(file_, 'r') as f:
keywords = extract_keywords(f, hannanum)
keyword_ids = [keyword_store.id(keyword[0]) for keyword in keywords]
keyword_store.sync()
existing_keywords_for_bill = set(r[0]
for r in session.query(bill_keyword.c.keyword_id)\
.filter(bill_keyword.c.bill_id == bill_id)
)
new_bill_keywords = [
{
'bill_id': bill_id,
'keyword_id': keyword_id,
'weight': weight,
}
for (_, weight), keyword_id in izip(keywords, keyword_ids)
if keyword_id not in existing_keywords_for_bill
]
if new_bill_keywords:
# TODO(lucypark, 2014-12-13): delete existing keywords before insert
session.execute(bill_keyword.insert(), new_bill_keywords)
print 'done'
else:
print 'passed'
def log_query(query):
if not query:
return
log = QueryLog(unicode(query))
with transaction() as session:
session.add(log)
