def insert_meeting(region_id, obj, update=False): with transaction() as session: meeting = create_or_get_meeting(session, region_id, obj, update) if not meeting: return session.add(meeting) # 'meeting_attendee' table attendee_names = get_attendee_names(obj) attendees = list(get_attendees(meeting, attendee_names, session)) meeting.attendees = attendees # clear the meeting's statements for statement in meeting.statements: session.delete(statement) session.flush() meeting.statements = [] # 'statement' table statements = (stmt for stmt in obj['dialogue'] if stmt['type'] == 'statement') for seq, statement in enumerate(statements): item = create_statement(meeting, seq, statement, attendees) session.add(item) session.flush() statement['person_id'] = item.person_id statement['id'] = item.id meeting.statements.append(item) # Updated dialog field of meeting table meeting.dialogue = obj['dialogue']
def insert_bill_keywords(files): hannanum = Hannanum() with transaction() as session: existing_bill_ids = set(r[0] for r in session.query(Bill.id)) keyword_store = KeywordStore(session) for file_ in glob(files): filename = basename(file_) print 'processing %s' % filename sys.stdout.flush() bill_id = filename.split('.', 1)[0] if bill_id not in existing_bill_ids: continue with open(file_, 'r') as f: keywords = extract_keywords(f, hannanum) keyword_ids = [ keyword_store.id(keyword[0]) for keyword in keywords ] keyword_store.sync() existing_keywords_for_bill = set(r[0] for r in session.query(bill_keyword.c.keyword_id)\ .filter(bill_keyword.c.bill_id == bill_id) ) new_bill_keywords = [ { 'bill_id': bill_id, 'keyword_id': keyword_id, 'weight': weight, } for (_, weight), keyword_id in izip(keywords, keyword_ids) if keyword_id not in existing_keywords_for_bill ] if new_bill_keywords: # TODO(lucypark, 2014-12-13): delete existing keywords before insert session.execute(bill_keyword.insert(), new_bill_keywords)
def update_bills(source, files=None): if source == 'redis': queue = RedisQueue(REDIS_KEYS['insert_bills_db'], **REDIS_SETTINGS) files = (bill_filepath(bill_id) for bill_id in queue) elif source.startswith('db'): with transaction() as session: assembly_id = session.query(Election)\ .order_by(Election.assembly_id.desc())\ .first().assembly_id # FIXME: filter finished bills out bill_ids = session.query(Bill.id).filter_by(assembly_id=assembly_id) # ranged query m = re.match(r'db\[(\d*):(\d*)\]', source) if m: start, end = m.group(1), m.group(2) offset = int(start) if start else 0 limit = int(end) - offset if end else None if offset: bill_ids = bill_ids.offset(offset) if limit: bill_ids = bill_ids.limit(limit) bill_ids = (record[0] for record in bill_ids) files = (bill_filepath(bill_id) for bill_id in bill_ids) elif files: files = [f for path in files for f in glob(path)] update_bills_from_files(files)
def insert_candidacies(files, election_type, assembly_id, date): with transaction() as session: for file_ in glob(files): with open(file_, 'r') as f: list_ = json.load(f) for record in list_: person_id = insert_person(session, election_type, assembly_id, record) insert_party(session, record) insert_election(session, assembly_id, date) insert_candidacy(session, record, person_id, date)
def update_people(fd, date): if not fd: return connect_db(db_session) with transaction() as session: records = json.load(fd) for record in records: preprocess_record(record) person_id = update_person(session, record) if not person_id: continue party_id = insert_party(session, record['party']) insert_party_affiliation(session, person_id, party_id, date)
def insert_meeting(region_id, obj, update=False): with transaction() as session: meeting = create_or_get_meeting(session, region_id, obj, update) if not meeting: return session.add(meeting) # 'meeting_attendee' table attendee_names = get_attendee_names(obj) attendees = list(get_attendees(meeting, attendee_names, session)) meeting.attendees = attendees # clear the meeting's statements for statement in meeting.statements: session.delete(statement) session.flush() meeting.statements = [] # 'statement' table statements = (stmt for stmt in obj['dialogue'] if stmt['type'] == 'statement') for seq, statement in enumerate(statements): item = create_statement(meeting, seq, statement, attendees) session.add(item) session.flush() statement['person_id'] = item.person_id statement['id'] = item.id meeting.statements.append(item) # Updated dialog field of meeting table meeting.dialogue = obj['dialogue'] # TODO: votes = obj['votes'] # The following code should be reviewed meeting.votes = [] options = ['yea', 'nay', 'forfeit'] for option in options: if option in stmt['votes']: for person_name in stmt['votes'][option]: person = guess_attendee(attendees, person_name) statement = guess_statement(statements, stmt['name']) if person and statement: item = create_vote(meeting, statement, person, option) session.add(item) meeting.votes.append(item)
def update_bills_from_files(files): if not files: return with transaction() as session: bill_statuses.init(session) for f in files: try: if not isinstance(f, file): f = open(f, 'r') with f: record = json.load(f) except Exception, e: print >> sys.stderr, e continue insert_bill(session, record) bill_statuses.insert_all()
def insert_bill_keywords(files): hannanum = Hannanum() with transaction() as session: existing_bill_ids = set(r[0] for r in session.query(Bill.id)) keyword_store = KeywordStore(session) for file_ in glob(files): filename = basename(file_) print 'processing %s,' % filename, bill_id = filename.split('.', 1)[0] if bill_id not in existing_bill_ids: continue with open(file_, 'r') as f: keywords = extract_keywords(f, hannanum) keyword_ids = [keyword_store.id(keyword[0]) for keyword in keywords] keyword_store.sync() existing_keywords_for_bill = set(r[0] for r in session.query(bill_keyword.c.keyword_id)\ .filter(bill_keyword.c.bill_id == bill_id) ) new_bill_keywords = [ { 'bill_id': bill_id, 'keyword_id': keyword_id, 'weight': weight, } for (_, weight), keyword_id in izip(keywords, keyword_ids) if keyword_id not in existing_keywords_for_bill ] if new_bill_keywords: # TODO(lucypark, 2014-12-13): delete existing keywords before insert session.execute(bill_keyword.insert(), new_bill_keywords) print 'done' else: print 'passed'
def log_query(query): if not query: return log = QueryLog(unicode(query)) with transaction() as session: session.add(log)