def load_playgrounds(path='data/playgrounds.csv'):
"""
Load playground data from the CSV into sqlite.
NOTE: THIS HAS NOT BEEN TESTED IN A VERY LONG TIME.
"""
features = copytext.Copy(app_config.COPY_PATH)['feature_list']
with open(path) as f:
rows = CSVKitDictReader(f)
for row in rows:
if row['Duplicate'] == 'TRUE':
#print 'Skipping duplicate: %s' % row['NAME']
continue
playground = Playground.create(
nprid=row['NPRID'],
name=row['NAME'],
facility=row['FACILITY'],
facility_type=row['FACILITY_TYPE'],
address=row['ADDRESS'],
city=row['CITY'],
state=row['STATE'],
zip_code=row['ZIP'],
longitude=float(row['LONGITUDE']) if row['LONGITUDE'] else None,
latitude=float(row['LATITUDE']) if row['LATITUDE'] else None,
agency=row['Agency'],
agency_type=row['AgencyType'],
owner=row['OWNER'],
owner_type=row['OWNER_TYPE'],
remarks=row['REMARKS'],
public_remarks=row['PubRermarks'],
url=row['url'],
entry=row['Entry'],
source=row['Source']
)
for feature in features:
slug = feature['key']
if row[slug] == 'TRUE':
PlaygroundFeature.create(
slug=slug,
playground=playground
)
Revision.create(
timestamp=datetime.datetime.now(pytz.utc),
action='insert',
playground=playground,
log=json.dumps([]),
headers='',
cookies='',
revision_group=1
)
def load_playgrounds(path='data/playgrounds.csv'):
"""
Load playground data from the CSV into sqlite.
NOTE: THIS HAS NOT BEEN TESTED IN A VERY LONG TIME.
"""
features = copytext.Copy(app_config.COPY_PATH)['feature_list']
with open(path) as f:
rows = CSVKitDictReader(f)
for row in rows:
if row['Duplicate'] == 'TRUE':
#print 'Skipping duplicate: %s' % row['NAME']
continue
playground = Playground.create(
nprid=row['NPRID'],
name=row['NAME'],
facility=row['FACILITY'],
facility_type=row['FACILITY_TYPE'],
address=row['ADDRESS'],
city=row['CITY'],
state=row['STATE'],
zip_code=row['ZIP'],
longitude=float(row['LONGITUDE'])
if row['LONGITUDE'] else None,
latitude=float(row['LATITUDE']) if row['LATITUDE'] else None,
agency=row['Agency'],
agency_type=row['AgencyType'],
owner=row['OWNER'],
owner_type=row['OWNER_TYPE'],
remarks=row['REMARKS'],
public_remarks=row['PubRermarks'],
url=row['url'],
entry=row['Entry'],
source=row['Source'])
for feature in features:
slug = feature['key']
if row[slug] == 'TRUE':
PlaygroundFeature.create(slug=slug, playground=playground)
Revision.create(timestamp=datetime.datetime.now(pytz.utc),
action='insert',
playground=playground,
log=json.dumps([]),
headers='',
cookies='',
revision_group=1)
def process_changes(path='changes-in-progress.json'):
"""
Iterate over changes-in-progress.json and process its contents.
"""
revision_group = time.mktime((datetime.datetime.now(pytz.utc)).timetuple())
with open(path) as f:
changes = json.load(f)
changed_playgrounds = []
ACTIONS = {
'update': process_update,
'insert': process_insert,
'delete-request': process_delete
}
for record in changes:
action = ACTIONS[record['action']]
playground, revisions = action(record)
changed_playgrounds.append(playground)
timestamp = datetime.datetime.fromtimestamp(
record['timestamp']).replace(tzinfo=pytz.utc)
# Assign the request headers to a variable.
# Need to modify the headers to add the remote IP address which is
# on the request object but not in the headers area.
# Why? Because we don't want to add an additional field to the
# Revisions model because we don't have DB migrations.
headers = record['request']['headers']
headers['remote_ip_address'] = record['request'].get(
'ip_address', None)
Revision.create(timestamp=timestamp,
action=record['action'],
playground=playground,
log=json.dumps(revisions),
headers=json.dumps(headers),
cookies=json.dumps(record['request']['cookies']),
revision_group=revision_group)
# Ensure all changes have been written to disk
database.commit()
return (changed_playgrounds, revision_group)
def process_changes(path='changes-in-progress.json'):
"""
Iterate over changes-in-progress.json and process its contents.
"""
revision_group = time.mktime((datetime.datetime.now(pytz.utc)).timetuple())
with open(path) as f:
changes = json.load(f)
changed_playgrounds = []
ACTIONS = {
'update': process_update,
'insert': process_insert,
'delete-request': process_delete
}
for record in changes:
action = ACTIONS[record['action']]
playground, revisions = action(record)
changed_playgrounds.append(playground)
timestamp = datetime.datetime.fromtimestamp(record['timestamp']).replace(tzinfo=pytz.utc)
# Assign the request headers to a variable.
# Need to modify the headers to add the remote IP address which is
# on the request object but not in the headers area.
# Why? Because we don't want to add an additional field to the
# Revisions model because we don't have DB migrations.
headers = record['request']['headers']
headers['remote_ip_address'] = record['request'].get('ip_address', None)
Revision.create(
timestamp=timestamp,
action=record['action'],
playground=playground,
log=json.dumps(revisions),
headers=json.dumps(headers),
cookies=json.dumps(record['request']['cookies']),
revision_group=revision_group
)
# Ensure all changes have been written to disk
database.commit()
return (changed_playgrounds, revision_group)
