def setUp(): drop_all_tables() # ensure everything is empty create_db() session = get_session(env='TST', echo=True) # load to satisfy fk constraints load_regions_and_stations(session) session.close()
def test_update_captures_username(self): ''' When a db record is updated, modified_by should be changed to show who it was''' # run a load (as bikeshare_tst) session = get_session(env='TST', echo=True) etl(Station_Information, session) conn, cur = get_connection_and_cursor() cur.execute('SELECT station_id FROM station_information;') row_id = cur.fetchone() # connect as different user conn, cur = get_connection_and_cursor(user='******', pw=environ['POSTGRES_PW_TST']) cur.execute( '''UPDATE station_information SET station_name = 'not an actual station' WHERE station_id = (%s);''', row_id) conn.commit() cur.execute('SELECT station_id, modified_by FROM station_information') db_data = cur.fetchall() correct = True i = 0 while i < len(db_data) and correct: row = db_data[i] if row[0] == row_id[0]: user = '******' else: user = environ['POSTGRES_USER_TST'] correct = (row[1] == user) if not correct: print(f'user should be {user} but is actually {row[1]}') i += 1 self.assertTrue(correct) empty_table_station_information(session) session.close()
def test_load_data_station_status(self): ''' Load data, then pull it from db and ensure match ''' session = get_session(env='TST', echo=False) data = get_data_from_api() print(f'rows pulled: {len(data["data"]["stations"])}') # tstmp needed for creating Station_Status obj last_updated = data['last_updated'] out = [] comp = {} for row in data['data']['stations']: row['last_updated'] = last_updated record = Station_Status(row) out.append(record) comp[record.station_id] = record load_db(out, session) # connect and get all data from db conn, cur = get_connection_and_cursor() cur.execute(select_all_stmt()) db_data = cur.fetchall() all_match = True i = 0 if len(db_data) != len(comp): all_match = False print('lens dont match.') print(f'db: {len(db_data)}') print(f'comp: {len(comp)}') while i < len(db_data) and all_match: row = db_data[i] if row != comp[row[2]].to_tuple(): all_match = False i += 1 self.assertTrue(all_match) empty_station_status_table(session) session.close()
def test_get_data_returns_dict(self): ''' ensure we get a dict of Station_Information back ''' session = get_session(env='TST', echo=True) metadata = create_metadata(Station_Information, session) data = get_data(Station_Information, metadata) self.assertIsInstance(data, dict) session.close()
def test_load_on_empty(self): ''' all records pulled down should be loaded into db ''' session = get_session(env='TST', echo=True) metadata = create_metadata(System_Region, session) data = get_data(System_Region, metadata) compare_data(data, System_Region, metadata, session) session.commit() # get data from db conn, cur = get_connection_and_cursor() cur.execute('''SELECT region_id, region_name, row_modified_tstmp, load_id, transtype, modified_by FROM system_regions''') db_data = cur.fetchall() correct = True # compare each val of each row from db to each row pulled from web i = 0 while i < len(db_data) and correct: row = db_data[i] orig = data[row[0]].to_tuple() correct = (row == orig) if not correct: print(f'row {row} didnt match orig {orig}') i += 1 self.assertTrue(correct) empty_db(session) session.close()
def setUp(): drop_all_tables() # ensure everything is empty create_db() session = get_session(env='TST', echo=True) load_regions(session) # load one dummy region with region_id 9999 load_single_region(create_dummy_region(), session) session.close()
def test_get_data_dict_key_is_id(self): ''' ensure dict key is row.id''' session = get_session(env='TST', echo=True) metadata = create_metadata(Station_Information, session) data = get_data(Station_Information, metadata) correct = True for row in data: if row != data[row].id: correct = False self.assertTrue(correct) session.close()
def test_get_latest_data(self): ''' Ensure get_latest_data_from_db brings back actual latest ''' session = get_session(env='TST', echo=True) # create a dummy record and load it d1 = create_dummy_data(time()) load_single_status_row(d1, session) # create another dummy with a later tstmp d2 = create_dummy_data(time() + 10) load_single_status_row(d2, session) # get latest (dict with station_id as key) latest = get_latest_from_db(session) self.assertEqual(d2, latest[d2.station_id]) empty_station_status_table(session) session.close()
def test_comparison(self): ''' Ensure only changed data is loaded.''' session = get_session(env='TST', echo=True) # load first dummy file on empty db data_d1 = get_json_from_file('tests/dummy1.json') last_updated = data_d1['last_updated'] d1_objs = [] for row in data_d1['data']['stations']: row['last_updated'] = last_updated session.add(Station_Status(row)) session.commit() # compare and load file 2 latest = get_latest_from_db(session) data_d2 = get_json_from_file('tests/dummy2.json') out, latest = get_changed_data(data_d2, latest) load_db(out, session) # compare and load file 3 data_d3 = get_json_from_file('tests/dummy3.json') out, latest = get_changed_data(data_d3, latest) load_db(out, session) # pull all data from db. # should match exactly data in desired file conn, cur = get_connection_and_cursor() cur.execute(select_all_stmt()) db_data = cur.fetchall() desired = get_json_from_file('tests/desired.json') all_match = True if len(db_data) != len(desired['stations']): all_match = False print(f"{db_data} should be same len as {desired['stations']}") print(f"{len(db_data)} != {len(desired['stations'])}") i = 0 while i < len(db_data) and all_match: row = db_data[i] should_be = Station_Status(desired['stations'][i]).to_tuple() # not worried about first and last vals (auto id and username) print(row) print(should_be) all_match = row[1:-1] == should_be[1:-1] if not all_match: print(f'row {row} should match row {should_be}') print('except first val which is auto incremented id') i += 1 self.assertTrue(all_match) empty_station_status_table(session) session.close()
def test_load_on_empty(self): ''' all records pulled down should be loaded into db ''' session = get_session(env='TST', echo=True) metadata = create_metadata(Station_Information, session) data = get_data(Station_Information, metadata) compare_data(data, Station_Information, metadata, session) session.commit() # get data from db conn, cur = get_connection_and_cursor() cur.execute('''SELECT station_id, short_name, station_name, lat, lon, capacity, region_id, eightd_has_key_dispenser, rental_method_key, rental_method_creditcard, rental_method_paypass, rental_method_applepay, rental_method_androidpay, rental_method_transitcard, rental_method_accountnumber, rental_method_phone, row_modified_tstmp, load_id, transtype, modified_by FROM station_information;''') db_data = cur.fetchall() correct = True # compare each val of each row from db to each row pulled from web i = 0 while i < len(db_data) and correct: row = db_data[i] orig = data[row[0]].to_tuple() correct = (row == orig) if not correct: print(f'row {row} didnt match orig {orig}') i += 1 self.assertTrue(correct) empty_table_station_information(session) session.close()
def test_get_session(): """Test the function get_session.""" session = get_session() assert session is not None
""" from src import sqla import csv f = utils.open_data_file("bug_summary.csv") f.readline() reader = csv.reader(f) for row in reader: br = sqla.BugRow(row) bug = session.query(Bug).filter_by(bzid=br.bug).scalar() bug.reported = br.reported session.commit() session = utils.get_session() ##### A bunch of silly little convenience methods intended to be run interactively (ideally only once) ###### def pop_bs(): from src.bugstate import populate_bugstates populate_bugstates(session, False, 15) def add_ids(): from src.bugs import Bug Bug.add_assignee_ids(session) def add_comments(interval=0): from src.bug_events import BugEvent BugEvent.scrape_comment_events(session, interval)
def test_get_session_returns_Session(self): self.assertIsInstance(get_session(env='TST'), Session)
def __init__(self, cls): self.cls = cls self.session = get_session()
def test_update_only_updates_that_record(self): ''' load, then load an update. ensure only that record was updated''' # first get data and load session = get_session(env='TST', echo=True) metadata = create_metadata(Station_Information, session) data = get_data(Station_Information, metadata) compare_data(data, Station_Information, metadata, session) session.commit() # get tuple copies of each record that was loaded originals = {} for row in data: originals[row] = data[row].to_tuple() # now get data again m2 = create_metadata(Station_Information, session) d2 = get_data(Station_Information, m2) # get one record from data and make a change u_record = d2[list(d2.keys())[0]] u_record.station_name = 'phoney balogna' u_data = {u_record.id: u_record} # load new record (should be update) compare_data(u_data, Station_Information, metadata, session) session.commit() # get all current data from db conn, cur = get_connection_and_cursor() cur.execute('''SELECT station_id, short_name, station_name, lat, lon, capacity, region_id, eightd_has_key_dispenser, rental_method_key, rental_method_creditcard, rental_method_paypass, rental_method_applepay, rental_method_androidpay, rental_method_transitcard, rental_method_accountnumber, rental_method_phone, row_modified_tstmp, load_id, transtype, modified_by FROM station_information;''') db_data = cur.fetchall() row_updated = True rows_match = True correct_trans = True # iterate through db data. break if any test fails i = 0 while i < len(db_data) and\ row_updated and\ rows_match and\ correct_trans: row = db_data[i] orig = originals[row[0]] if row[0] == u_record.id: trans = 'U' # update orig should not match row and ensure name was updated row_updated = (orig != row) and (row[2] == 'phoney balogna') else: # all other records row should match orig trans = 'I' rows_match = (orig == row) correct_trans = (row[-2] == trans) if not row_updated: print(f'row {row} shouldnt match orig {orig}') print('also, region_name in row should be phoney balogna') if not rows_match: print(f'row {row} didnt match orig {orig}') if not correct_trans: print(f'incorrect trans on row {row} -- should be {trans}') i += 1 self.assertTrue(row_updated and rows_match and correct_trans) empty_table_station_information(session) session.close()
def setUp(): drop_all_tables() # ensure everything is empty create_db() session = get_session(env='TST', echo=True) session.close()