def load_playgrounds(path='data/playgrounds.csv'): """ Load playground data from the CSV into sqlite. NOTE: THIS HAS NOT BEEN TESTED IN A VERY LONG TIME. """ features = copytext.Copy(app_config.COPY_PATH)['feature_list'] with open(path) as f: rows = CSVKitDictReader(f) for row in rows: if row['Duplicate'] == 'TRUE': #print 'Skipping duplicate: %s' % row['NAME'] continue playground = Playground.create( nprid=row['NPRID'], name=row['NAME'], facility=row['FACILITY'], facility_type=row['FACILITY_TYPE'], address=row['ADDRESS'], city=row['CITY'], state=row['STATE'], zip_code=row['ZIP'], longitude=float(row['LONGITUDE']) if row['LONGITUDE'] else None, latitude=float(row['LATITUDE']) if row['LATITUDE'] else None, agency=row['Agency'], agency_type=row['AgencyType'], owner=row['OWNER'], owner_type=row['OWNER_TYPE'], remarks=row['REMARKS'], public_remarks=row['PubRermarks'], url=row['url'], entry=row['Entry'], source=row['Source'] ) for feature in features: slug = feature['key'] if row[slug] == 'TRUE': PlaygroundFeature.create( slug=slug, playground=playground ) Revision.create( timestamp=datetime.datetime.now(pytz.utc), action='insert', playground=playground, log=json.dumps([]), headers='', cookies='', revision_group=1 )
def load_playgrounds(path='data/playgrounds.csv'): """ Load playground data from the CSV into sqlite. NOTE: THIS HAS NOT BEEN TESTED IN A VERY LONG TIME. """ features = copytext.Copy(app_config.COPY_PATH)['feature_list'] with open(path) as f: rows = CSVKitDictReader(f) for row in rows: if row['Duplicate'] == 'TRUE': #print 'Skipping duplicate: %s' % row['NAME'] continue playground = Playground.create( nprid=row['NPRID'], name=row['NAME'], facility=row['FACILITY'], facility_type=row['FACILITY_TYPE'], address=row['ADDRESS'], city=row['CITY'], state=row['STATE'], zip_code=row['ZIP'], longitude=float(row['LONGITUDE']) if row['LONGITUDE'] else None, latitude=float(row['LATITUDE']) if row['LATITUDE'] else None, agency=row['Agency'], agency_type=row['AgencyType'], owner=row['OWNER'], owner_type=row['OWNER_TYPE'], remarks=row['REMARKS'], public_remarks=row['PubRermarks'], url=row['url'], entry=row['Entry'], source=row['Source']) for feature in features: slug = feature['key'] if row[slug] == 'TRUE': PlaygroundFeature.create(slug=slug, playground=playground) Revision.create(timestamp=datetime.datetime.now(pytz.utc), action='insert', playground=playground, log=json.dumps([]), headers='', cookies='', revision_group=1)
def process_changes(path='changes-in-progress.json'): """ Iterate over changes-in-progress.json and process its contents. """ revision_group = time.mktime((datetime.datetime.now(pytz.utc)).timetuple()) with open(path) as f: changes = json.load(f) changed_playgrounds = [] ACTIONS = { 'update': process_update, 'insert': process_insert, 'delete-request': process_delete } for record in changes: action = ACTIONS[record['action']] playground, revisions = action(record) changed_playgrounds.append(playground) timestamp = datetime.datetime.fromtimestamp( record['timestamp']).replace(tzinfo=pytz.utc) # Assign the request headers to a variable. # Need to modify the headers to add the remote IP address which is # on the request object but not in the headers area. # Why? Because we don't want to add an additional field to the # Revisions model because we don't have DB migrations. headers = record['request']['headers'] headers['remote_ip_address'] = record['request'].get( 'ip_address', None) Revision.create(timestamp=timestamp, action=record['action'], playground=playground, log=json.dumps(revisions), headers=json.dumps(headers), cookies=json.dumps(record['request']['cookies']), revision_group=revision_group) # Ensure all changes have been written to disk database.commit() return (changed_playgrounds, revision_group)
def process_changes(path='changes-in-progress.json'): """ Iterate over changes-in-progress.json and process its contents. """ revision_group = time.mktime((datetime.datetime.now(pytz.utc)).timetuple()) with open(path) as f: changes = json.load(f) changed_playgrounds = [] ACTIONS = { 'update': process_update, 'insert': process_insert, 'delete-request': process_delete } for record in changes: action = ACTIONS[record['action']] playground, revisions = action(record) changed_playgrounds.append(playground) timestamp = datetime.datetime.fromtimestamp(record['timestamp']).replace(tzinfo=pytz.utc) # Assign the request headers to a variable. # Need to modify the headers to add the remote IP address which is # on the request object but not in the headers area. # Why? Because we don't want to add an additional field to the # Revisions model because we don't have DB migrations. headers = record['request']['headers'] headers['remote_ip_address'] = record['request'].get('ip_address', None) Revision.create( timestamp=timestamp, action=record['action'], playground=playground, log=json.dumps(revisions), headers=json.dumps(headers), cookies=json.dumps(record['request']['cookies']), revision_group=revision_group ) # Ensure all changes have been written to disk database.commit() return (changed_playgrounds, revision_group)