def main(date_time):
try:
count = 0
for obj in KNACK_PARAMS['REFERENCE_OBJECTS']:
# get field metadata
fields = knack_helpers.get_all_fields(obj, KNACK_PARAMS)
# assign field metadata to 'raw' field name
field_list = {}
for field in fields:
field_list[field['key'] + '_raw'] = field
# update knack params with list of all field names
KNACK_PARAMS['FIELD_NAMES'] = knack_helpers.create_label_list(
fields)
# get knack object data
data = knack_helpers.get_object_data(obj, KNACK_PARAMS)
# parse data
parsed = knack_helpers.parse_data(data,
field_list,
KNACK_PARAMS,
convert_to_unix=True,
include_ids=True)
today = date_time.format('YYYY_MM_DD')
file_name = '{}/{}_{}.csv'.format(BACKUP_DIRECTORY, obj, today)
try:
data_helpers.write_csv(parsed, file_name=file_name)
except Exception as e:
print(e)
body = 'Data backup of failed when writing csv'
email_helpers.send_email(secrets['ALERTS_DISTRIBUTION'],
'data backup exception', body)
raise e
count += 1
return count
except Exception as e:
print('Failed to process data for {}'.format(date_time))
print(e)
body = 'Data backup of failed'
email_helpers.send_email(secrets['ALERTS_DISTRIBUTION'],
'data backup exception', body)
raise e
def main(date_time):
print('starting stuff now')
try:
field_list = knack_helpers.get_fields(KNACK_PARAMS)
knack_data = knack_helpers.get_data(KNACK_PARAMS)
knack_data = knack_helpers.parse_data(knack_data,
field_list,
KNACK_PARAMS,
require_locations=True,
convert_to_unix=True)
knack_data = data_helpers.stringify_key_values(knack_data)
knack_data = data_helpers.remove_linebreaks(knack_data,
['LOCATION_NAME'])
knack_data_mills = data_helpers.unix_to_mills(knack_data)
token = agol_helpers.get_token(secrets.AGOL_CREDENTIALS)
agol_payload = agol_helpers.build_payload(knack_data_mills)
del_response = agol_helpers.delete_features(SERVICE_URL, token)
add_response = agol_helpers.add_features(SERVICE_URL, token,
agol_payload)
# write to csv
knack_data = data_helpers.mills_to_unix(knack_data)
knack_data = data_helpers.unix_to_iso(knack_data)
file_name = '{}/{}.csv'.format(CSV_DESTINATION, DATASET_NAME)
data_helpers.write_csv(knack_data, file_name=file_name)
return add_response
except Exception as e:
print('Failed to process data for {}'.format(date_time))
print(e)
raise e
def main(date_time):
print('starting stuff now')
try:
field_list = knack_helpers.get_fields(KNACK_PARAMS)
knack_data = knack_helpers.get_data(KNACK_PARAMS)
knack_data_parsed = knack_helpers.parse_data(knack_data,
field_list,
KNACK_PARAMS,
require_locations=True,
convert_to_unix=True)
kits_query = kits_helpers.generate_status_id_query(
knack_data_parsed, 'ATD_SIGNAL_ID')
kits_data = kits_helpers.data_as_dict(secrets.KITS_CREDENTIALS,
kits_query)
kits_data = data_helpers.stringify_key_values(kits_data)
stale = kits_helpers.check_for_stale(kits_data,
'OPERATION_STATE_DATETIME', 15)
if stale['stale']:
email_helpers.send_stale_email(stale['delta_minutes'],
secrets.ALERTS_DISTRIBUTION)
response_obj = {
'Errors': 1,
'message': 'WARNING: stale data detected',
'Rows Updated': 0,
'Rows Created': 0,
'Rows Deleted': 0
}
stale_data_log = socrata_helpers.prep_pub_log(
date_time, 'signal_status_update', response_obj)
socrata_helpers.upsert_data(secrets.SOCRATA_CREDENTIALS,
stale_data_log, SOCRATA_PUB_LOG_ID)
sys.exit()
kits_data = data_helpers.filter_by_key(
kits_data, 'OPERATION_STATE',
FLASH_STATUSES) # filter by flash statuses
if kits_data:
new_data = data_helpers.merge_dicts(
knack_data_parsed, kits_data, 'ATD_SIGNAL_ID',
['OPERATION_STATE_DATETIME', 'OPERATION_STATE', 'PLAN_ID'])
new_data = data_helpers.iso_to_unix(new_data, replace_tz=True)
new_data = data_helpers.stringify_key_values(new_data)
else:
new_data = []
socrata_data = socrata_helpers.get_public_data(SOCRATA_SIGNAL_STATUS)
socrata_data = socrata_helpers.strip_geocoding(socrata_data)
socrata_data = data_helpers.upper_case_keys(socrata_data)
socrata_data = data_helpers.iso_to_unix(socrata_data)
socrata_data = data_helpers.stringify_key_values(socrata_data)
cd_results = data_helpers.detect_changes(socrata_data,
new_data,
'ATD_SIGNAL_ID',
keys=['OPERATION_STATE'])
for thing in cd_results:
print('{} : {}'.format(thing, len(cd_results[thing])))
if cd_results['new'] or cd_results['change'] or cd_results['delete']:
socrata_payload = socrata_helpers.create_payload(
cd_results, 'ATD_SIGNAL_ID')
socrata_payload = socrata_helpers.create_location_fields(
socrata_payload)
socrata_payload = data_helpers.lower_case_keys(socrata_payload)
socrata_payload = data_helpers.unix_to_iso(socrata_payload)
status_upsert_response = socrata_helpers.upsert_data(
secrets.SOCRATA_CREDENTIALS, socrata_payload,
SOCRATA_SIGNAL_STATUS)
else:
status_upsert_response = {
'Errors': 0,
'message': 'No signal status change detected',
'Rows Updated': 0,
'Rows Created': 0,
'Rows Deleted': 0
}
log_payload = socrata_helpers.prep_pub_log(date_time,
'signal_status_update',
status_upsert_response)
pub_log_response = socrata_helpers.upsert_data(
secrets.SOCRATA_CREDENTIALS, log_payload, SOCRATA_PUB_LOG_ID)
if 'error' in status_upsert_response:
email_helpers.send_socrata_alert(secrets.ALERTS_DISTRIBUTION,
SOCRATA_SIGNAL_STATUS,
status_upsert_response)
elif status_upsert_response['Errors']:
email_helpers.send_socrata_alert(secrets.ALERTS_DISTRIBUTION,
SOCRATA_SIGNAL_STATUS,
status_upsert_response)
if cd_results['delete']:
historical_payload = data_helpers.lower_case_keys(
cd_results['delete'])
historical_payload = socrata_helpers.add_hist_fields(
historical_payload)
status_upsert_historical_response = socrata_helpers.upsert_data(
secrets.SOCRATA_CREDENTIALS, historical_payload,
SOCRATA_SIGNAL_STATUS_HISTORICAL)
historical_log_payload = socrata_helpers.prep_pub_log(
date_time, 'signal_status_historical_update',
status_upsert_historical_response)
pub_log_historical_response = socrata_helpers.upsert_data(
secrets.SOCRATA_CREDENTIALS, historical_log_payload,
SOCRATA_PUB_LOG_ID)
if 'error' in status_upsert_historical_response:
email_helpers.send_socrata_alert(
secrets.ALERTS_DISTRIBUTION, SOCRATA_SIGNAL_STATUS,
status_upsert_historical_response)
elif status_upsert_historical_response['Errors']:
email_helpers.send_socrata_alert(
secrets.ALERTS_DISTRIBUTION, SOCRATA_SIGNAL_STATUS,
status_upsert_historical_response)
else:
print('no new historical status data to upload')
status_upsert_historical_response = None
return {
'res': status_upsert_response,
'res_historical': status_upsert_historical_response,
}
except Exception as e:
print('Failed to process data for {}'.format(date_time))
print(e)
email_helpers.send_email(
secrets.ALERTS_DISTRIBUTION,
'DATA PROCESSING ALERT: Signal Status Update Failure', str(e))
raise e
def main(date_time):
print('starting stuff now')
try:
field_list = knack_helpers.get_fields(KNACK_PARAMS)
knack_data = knack_helpers.get_data(KNACK_PARAMS)
knack_data = knack_helpers.parse_data(knack_data,
field_list,
KNACK_PARAMS,
require_locations=True,
convert_to_unix=True)
knack_data = data_helpers.stringify_key_values(knack_data)
knack_data = data_helpers.remove_linebreaks(knack_data,
['LOCATION_NAME'])
knack_data_mills = data_helpers.unix_to_mills(deepcopy(knack_data))
# token = agol_helpers.get_token(secrets.AGOL_CREDENTIALS)
# agol_payload = agol_helpers.build_payload(knack_data_mills)
# del_response = agol_helpers.delete_features(SERVICE_URL, token)
# add_response = agol_helpers.add_features(SERVICE_URL, token, agol_payload)
socrata_data = socrata_helpers.get_private_data(
secrets.SOCRATA_CREDENTIALS, SOCRATA_RESOURCE_ID)
socrata_data = data_helpers.upper_case_keys(socrata_data)
socrata_data = data_helpers.stringify_key_values(socrata_data)
socrata_data = data_helpers.iso_to_unix(socrata_data, replace_tz=True)
cd_results = data_helpers.detect_changes(
socrata_data,
knack_data,
PRIMARY_KEY,
keys=KNACK_PARAMS['FIELD_NAMES'] + ['LATITUDE', 'LONGITUDE'])
if cd_results['new'] or cd_results['change'] or cd_results['delete']:
socrata_payload = socrata_helpers.create_payload(
cd_results, PRIMARY_KEY)
socrata_payload = socrata_helpers.create_location_fields(
socrata_payload)
else:
socrata_payload = []
socrata_payload = data_helpers.lower_case_keys(socrata_payload)
socrata_payload = data_helpers.unix_to_iso(socrata_payload)
upsert_response = socrata_helpers.upsert_data(
secrets.SOCRATA_CREDENTIALS, socrata_payload, SOCRATA_RESOURCE_ID)
if 'error' in upsert_response:
email_helpers.send_socrata_alert(secrets.ALERTS_DISTRIBUTION,
SOCRATA_RESOURCE_ID,
upsert_response)
elif upsert_response['Errors']:
email_helpers.send_socrata_alert(secrets.ALERTS_DISTRIBUTION,
SOCRATA_RESOURCE_ID,
upsert_response)
log_payload = socrata_helpers.prep_pub_log(date_time, 'sensors_update',
upsert_response)
pub_log_response = socrata_helpers.upsert_data(
secrets.SOCRATA_CREDENTIALS, log_payload, SOCRATA_PUB_LOG_ID)
return log_payload
except Exception as e:
print('Failed to process data for {}'.format(date_time))
print(e)
raise e
def main(date_time):
print('starting stuff now')
try:
url = 'https://raw.githubusercontent.com/cityofaustin/transportation/gh-pages/components/data/quote_of_the_week.csv'
# get new quote data from Knack database
# this is where the user maintains the quotes
for obj in KNACK_PARAMS['REFERENCE_OBJECTS']:
# get field metadata
fields = knack_helpers.get_all_fields(obj, KNACK_PARAMS)
# assign field metadata to 'raw' field name
field_list = {}
for field in fields:
field_list[field['key'] + '_raw'] = field
# update knack params with list of all field names
KNACK_PARAMS['FIELD_NAMES'] = knack_helpers.create_label_list(
fields)
# get knack object data
data = knack_helpers.get_object_data(obj, KNACK_PARAMS)
# parse data
data = knack_helpers.parse_data(data,
field_list,
KNACK_PARAMS,
convert_to_unix=True)
# prepare dates for the internet
data = data_helpers.unix_to_mills(data)
payload = data_helpers.write_csv(data, in_memory=True)
git_auth = github_helpers.create_auth_tuple(
secrets.GITHUB_CREDENTIALS)
repo_data = github_helpers.get_file(GIT_PARAMS['REPO_URL'],
GIT_PARAMS['PATH'], 'gh-pages',
git_auth)
GIT_PARAMS['sha'] = repo_data['sha']
git_response = github_helpers.commit_file(GIT_PARAMS['REPO_URL'],
GIT_PARAMS['PATH'],
GIT_PARAMS['BRANCH'],
payload,
'update_quote_of_week',
GIT_PARAMS['sha'],
git_auth,
existing_file=repo_data)
return git_response
except Exception as e:
print('Failed to process data for {}'.format(date_time))
print(e)
raise e
def main(date_time):
try:
field_dict = knack_helpers.get_fields(KNACK_PARAMS)
field_lookup = knack_helpers.create_field_lookup(field_dict,
parse_raw=True)
knack_data = knack_helpers.get_object_data(REFERENCE_OBJECT,
KNACK_PARAMS)
knack_data = knack_helpers.parse_data(knack_data,
field_dict,
KNACK_PARAMS,
include_ids=True)
knack_data = data_helpers.filter_by_key(
knack_data, STATUS_KEY, ['NEW', 'IN PROGRESS', 'COMPLETED'])
knack_data = data_helpers.add_missing_keys(knack_data, [SCORE_KEY],
['0'])
knack_data = data_helpers.concat_key_values(knack_data, CONCAT_KEYS,
GROUP_KEY, '_')
knack_data_exclude = [
record for record in knack_data
if record['EXCLUDE_FROM_RANKING'] == True
]
knack_data_include = [
record for record in knack_data
if record['EXCLUDE_FROM_RANKING'] == False
]
# create list of scores grouped by group key
# scores are converted to integers
score_dict = {}
for row in knack_data_include:
key = row[GROUP_KEY]
score = int(row[SCORE_KEY])
if key not in score_dict:
score_dict[key] = []
score_dict[key].append(score)
# reverse sort lists of scores
for key in score_dict:
score_dict[key].sort()
score_dict[key].reverse()
# get score rank and append record to payload
payload = []
for record in knack_data_include:
score = int(record[SCORE_KEY])
key = record[GROUP_KEY]
rank = data_helpers.min_index(
score_dict[key], score
) + 1 # add one to score index, because list indices start at 0
if RANK_KEY in record:
if record[RANK_KEY] != rank:
record[RANK_KEY] = rank
payload.append(record)
else:
record[RANK_KEY] = rank
payload.append(record)
# assign null ranks to records flagged as exclude from ranking
for record in knack_data_exclude:
if RANK_KEY in record:
# updated excluded records if rank found
if record[RANK_KEY] != '':
record[RANK_KEY] = ''
payload.append(record)
# parse data to core fields
payload = data_helpers.reduce_dicts(payload, [RANK_KEY, 'KNACK_ID'])
# replace data keys with knack field names
payload = data_helpers.replace_keys(payload, field_lookup)
update_response = []
# update knack records
count = 0
for record in payload:
count += 1
print('updating record {} of {}'.format(count, len(payload)))
response_json = knack_helpers.update_record(record, KNACK_PARAMS)
update_response.append(response_json)
return update_response
except Exception as e:
print('Failed to process data for {}'.format(date_time))
print(e)
raise e
def main(date_time):
print('starting stuff now')
try:
# get and parse phb eval data
field_list = knack_helpers.get_fields(KNACK_PARAMS_SYNC_SIGNALS)
knack_data_sync_signals = knack_helpers.get_data(
KNACK_PARAMS_SYNC_SIGNALS)
knack_data_sync_signals = knack_helpers.parse_data(
knack_data_sync_signals,
field_list,
KNACK_PARAMS_SYNC_SIGNALS,
convert_to_unix=True)
knack_data_sync_signals = data_helpers.stringify_key_values(
knack_data_sync_signals)
knack_data_sync_signals_mills = data_helpers.unix_to_mills(
deepcopy(knack_data_sync_signals))
# get and parse location info
field_list = knack_helpers.get_fields(KNACK_PARAMS_LOCATIONS)
knack_data_loc = knack_helpers.get_data(KNACK_PARAMS_LOCATIONS)
knack_data_loc = knack_helpers.parse_data(knack_data_loc,
field_list,
KNACK_PARAMS_LOCATIONS,
convert_to_unix=True)
knack_data_loc = data_helpers.stringify_key_values(knack_data_loc)
# append location info to eval data dicts
knack_data_master = data_helpers.merge_dicts(
knack_data_sync_signals_mills, knack_data_loc, 'ATD_SIGNAL_ID',
['LATITUDE', 'LONGITUDE'])
print(knack_data_master[0:3])
# # get published request data from Socrata and compare to Knack database
socrata_data = socrata_helpers.get_private_data(
secrets.SOCRATA_CREDENTIALS, SOCRATA_RESOURCE_ID)
socrata_data = data_helpers.upper_case_keys(socrata_data)
socrata_data = data_helpers.stringify_key_values(socrata_data)
socrata_data = data_helpers.iso_to_unix(socrata_data, replace_tz=True)
cd_results = data_helpers.detect_changes(
socrata_data,
knack_data_master,
PRIMARY_KEY,
keys=KNACK_PARAMS_SYNC_SIGNALS['FIELD_NAMES'] +
['LATITUDE', 'LONGITUDE'])
if cd_results['new'] or cd_results['change'] or cd_results['delete']:
socrata_payload = socrata_helpers.create_payload(
cd_results, PRIMARY_KEY)
# socrata_payload = socrata_helpers.create_location_fields(socrata_payload)
else:
socrata_payload = []
socrata_payload = data_helpers.lower_case_keys(socrata_payload)
socrata_payload = data_helpers.unix_to_iso(socrata_payload)
upsert_response = socrata_helpers.upsert_data(
secrets.SOCRATA_CREDENTIALS, socrata_payload, SOCRATA_RESOURCE_ID)
if 'error' in upsert_response:
email_helpers.send_socrata_alert(secrets.ALERTS_DISTRIBUTION,
SOCRATA_RESOURCE_ID,
upsert_response)
elif upsert_response['Errors']:
email_helpers.send_socrata_alert(secrets.ALERTS_DISTRIBUTION,
SOCRATA_RESOURCE_ID,
upsert_response)
log_payload = socrata_helpers.prep_pub_log(date_time,
'sync_signal_corridors',
upsert_response)
pub_log_response = socrata_helpers.upsert_data(
secrets.SOCRATA_CREDENTIALS, log_payload, SOCRATA_PUB_LOG_ID)
return upsert_response
except Exception as e:
print('Failed to process data for {}'.format(date_time))
print(e)
raise e