def main():
curr_full_path = FileUtils.getCurrentDirFullPath()
config_fn = 'portal_activity_job_config.yaml'
cI = ConfigUtils(curr_full_path + "/configs/", config_fn)
configItems = cI.getConfigs()
configItems[
'config_dir'] = curr_full_path + "/" + configItems['config_dir']
configItems['curr_full_path'] = curr_full_path
db_ini = configItems['config_dir'] + configItems['database_config']
conn_alq, meta_alq = PostgresStuff.connect_alq(db_ini)
conn = PostgresStuff.connect(db_ini)
db_tbl = configItems['activity_table']
first_run = getFirstRun(conn)
if first_run == 0:
print("****First RUN! No new created datasets in the past " +
configItems['activity']['create']['time_interval'] + "*****")
exit(0)
insert_created = updateCreatedDatasets(
conn, configItems['activity']['create']['time_interval'])
#print insert_created
created_datasets = MonitorPortal.generateActivityReport(
conn_alq, configItems, 'create')
if (not (created_datasets)):
print("**** No new created datasets in the past " +
configItems['activity']['create']['time_interval'] + "*****")
exit(0)
datasetid_notified = MonitorPortal.generateEmail(conn_alq, configItems,
'create',
created_datasets)
updted_notified_cnt = MonitorPortal.updateNotifiedDatasetIds(
conn, configItems, 'create', datasetid_notified)
print("******Notfied that " + str(updted_notified_cnt) +
" datasets were created****")
print("******Updated " + str(updted_notified_cnt) +
" rows in the created_dataset table****")
def main():
fieldConfigFile, config_inputdir, jobType = parse_opts()
cI = ConfigUtils(config_inputdir, fieldConfigFile)
configItems = cI.getConfigs()
configItems['dataset_name'] = jobType
lg = pyLogger(configItems)
logger = lg.setConfig()
logger.info("****************JOB START******************")
sc = SocrataClient(config_inputdir, configItems, logger)
client = sc.connectToSocrata()
clientItems = sc.connectToSocrataConfigItems()
scrud = SocrataCRUD(client, clientItems, configItems, logger)
sQobj = SocrataQueries(clientItems, configItems, logger)
mmdd_fbf = configItems['dd']['master_dd']['fbf']
field_profiles_fbf = configItems['dd']['field_profiles']['fbf']
base_url = configItems['baseUrl']
field_type_fbf = configItems['dd']['field_type']['fbf']
load_mm_dd = ProfileFields.getBaseDatasetJson(sQobj, configItems, mmdd_fbf)
#print load_mm_dd
#load_mm_dd = True
current_field_profiles = ProfileFields.getCurrentFieldProfiles(
sQobj, base_url, field_profiles_fbf)
if load_mm_dd:
master_dfList = ProfileFields.get_dataset_as_dfList(
configItems['pickle_data_dir'], configItems['mm_dd_json_fn'],
base_url)
ProfileFields.removeDeletedFields(scrud, field_profiles_fbf,
master_dfList,
current_field_profiles)
current_field_profiles = ProfileFields.getCurrentFieldProfiles(
sQobj, base_url, field_profiles_fbf)
dataset_info = ProfileFields.buildInsertFieldProfiles(
sQobj, scrud, configItems, master_dfList, current_field_profiles)
print dataset_info
dsse = JobStatusEmailerComposer(configItems, logger, jobType)
if dataset_info['DatasetRecordsCnt'] > 1:
dsse.sendJobStatusEmail([dataset_info])
else:
dataset_info = {
'Socrata Dataset Name': configItems['dataset_name'],
'SrcRecordsCnt': 0,
'DatasetRecordsCnt': 0,
'fourXFour': "Nothing to Insert"
}
dataset_info['isLoaded'] = 'success'
dsse.sendJobStatusEmail([dataset_info])
def runWebTask(configItems):
cItemsWebtask = ConfigUtils(configItems['inputConfigDir'], configItems['webtask_config_fn'] )
configItemsWebTasks = cItemsWebtask.getConfigs()
url = configItemsWebTasks['webtask_url']
attempts = 0
while attempts < 3:
try:
response = urllib2.urlopen(url, timeout = 15)
content = response.read()
if(content):
f = open( configItems['log_dir'] + "algolia-sync-results.json", 'w' )
f.write( content )
f.close()
return True
break
except urllib2.URLError as e:
attempts += 1
print type(e)
return False
return False
def main():
curr_full_path = FileUtils.getCurrentDirFullPath()
config_fn = 'portal_activity_job_config.yaml'
cI = ConfigUtils(curr_full_path + "/configs/", config_fn)
configItems = cI.getConfigs()
configItems[
'config_dir'] = curr_full_path + "/" + configItems['config_dir']
configItems['curr_full_path'] = curr_full_path
db_ini = configItems['config_dir'] + configItems['database_config']
conn_alq, meta_alq = PostgresStuff.connect_alq(db_ini)
conn = PostgresStuff.connect(db_ini)
db_tbl = configItems['activity_table']
rotate_items = rotateActivityData(conn)
activity = "rotate_portal_activity"
subject_line = configItems['activity'][activity]['email_msg'][
'subject_line']
msg_body = configItems['email_msg_template']['header'] + configItems[
'activity'][activity]['email_msg']['msg']
msg_body = msg_body + configItems['email_msg_template']['footer']
em = Emailer(configItems)
em.sendEmails(subject_line, msg_body.encode('utf-8').strip())
def main():
curr_full_path = FileUtils.getCurrentDirFullPath()
config_fn = 'portal_activity_job_config.yaml'
cI = ConfigUtils(curr_full_path + "/configs/", config_fn)
configItems = cI.getConfigs()
configItems[
'config_dir'] = curr_full_path + "/" + configItems['config_dir']
configItems['curr_full_path'] = curr_full_path
db_ini = configItems['config_dir'] + configItems['database_config']
conn_alq, meta_alq = PostgresStuff.connect_alq(db_ini)
conn = PostgresStuff.connect(db_ini)
db_tbl = configItems['activity_table']
digest_items = digestStaleDelayedDatasets(conn_alq, configItems,
'stale_delayed_digest')
if (not (digest_items)):
print("**** No digest items " +
configItems['activity']['update']['time_interval'] + "*****")
exit(0)
datasetid_notified = MonitorPortal.generateEmail(conn_alq, configItems,
'stale_delayed_digest',
digest_items)
def main():
curr_full_path = FileUtils.getCurrentDirFullPath()
config_fn = 'portal_activity_job_config.yaml'
cI = ConfigUtils(curr_full_path+ "/configs/" , config_fn)
configItems = cI.getConfigs()
configItems['config_dir'] = curr_full_path+ "/" + configItems['config_dir']
configItems['curr_full_path'] = curr_full_path
db_ini = configItems['config_dir'] + configItems['database_config']
conn_alq, meta_alq =PostgresStuff.connect_alq(db_ini)
conn = PostgresStuff.connect(db_ini)
db_tbl = configItems['activity_table']
insert_late_updated = updateStaleDelayedDatasets(conn, configItems['activity']['update']['time_interval'])
print (insert_late_updated)
stale_late_datasets = MonitorPortal.generateActivityReport(conn_alq, configItems, 'update')
print ( stale_late_datasets)
if (not (stale_late_datasets)):
print ("**** No changes for stale or deleyed datasets " + configItems['activity']['update']['time_interval'] + "*****")
exit (0)
datasetid_notified = MonitorPortal.generateEmail(conn_alq, configItems, 'update', stale_late_datasets)
updted_notified_cnt = MonitorPortal.updateNotifiedDatasetIds(conn, configItems, 'update', datasetid_notified)
print ("******Notfied that " +str(updted_notified_cnt) + " datasets are late or stale****" )
print ("******Updated" + str(updted_notified_cnt) + " rows in the late_updated_dataset table****" )
def main():
fieldConfigFile, config_inputdir, jobType, hourly = parse_opts()
cI = ConfigUtils(config_inputdir, fieldConfigFile)
configItems = cI.getConfigs()
configItems['dataset_name'] = jobType
lg = pyLogger(configItems)
logger = lg.setConfig()
logger.info("****************JOB START******************")
sc = SocrataClient(config_inputdir, configItems, logger)
client = sc.connectToSocrata()
clientItems = sc.connectToSocrataConfigItems()
scrud = SocrataCRUD(client, clientItems, configItems, logger)
sQobj = SocrataQueries(clientItems, configItems, logger)
mmdd_fbf = configItems['dd']['master_dd']['fbf']
ds_profiles_fbf = configItems['dd']['ds_profiles']['fbf']
base_url = configItems['baseUrl']
field_type_fbf = configItems['dd']['field_type']['fbf']
asset_inventory_fbf = configItems['dd']['asset_inventory']['fbf']
ds_profiles = ProfileDatasets.getCurrentDatasetProfiles(
sQobj, base_url, ds_profiles_fbf)
update_counter = 0
updated_datasets = []
#if int(hourly) == 1:
# print "****hourly update****"
for datasetid, last_updt in ds_profiles.iteritems():
mm_profiles_to_updt = ProfileDatasets.getViewsLastUpdatedAt(
datasetid, last_updt, clientItems)
#print datasetid
#print mm_profiles_to_updt
if ('cols' in mm_profiles_to_updt.keys()):
dataset_info_mm = {
'Socrata Dataset Name': configItems['dataset_name'],
'SrcRecordsCnt': 0,
'DatasetRecordsCnt': 0,
'fourXFour': mmdd_fbf,
'row_id': 'columnid'
}
dataset_info_mm['DatasetRecordsCnt'] = 0
dataset_info_mm['SrcRecordsCnt'] = len(mm_profiles_to_updt['cols'])
#print mm_profiles_to_updt.keys()
print "***************"
print "Updating " + mm_profiles_to_updt['dataset_name']
print "*************"
print "**************"
dataset_info_mm = scrud.postDataToSocrata(
dataset_info_mm, mm_profiles_to_updt['cols'])
update_counter = update_counter + 1
updated_datasets.append(mm_profiles_to_updt['dataset_name'])
datasets = ProfileDatasets.getBaseDatasets(sQobj, base_url, mmdd_fbf)
asset_inventory_dict = ProfileDatasets.getAssetInventoryInfo(
sQobj, base_url, asset_inventory_fbf)
#delete datasets from the profile that no longer exist
ProfileDatasets.removeDeletedDatasets(scrud, ds_profiles_fbf,
asset_inventory_dict, ds_profiles,
datasets)
ds_profiles = ProfileDatasets.getCurrentDatasetProfiles(
sQobj, base_url, ds_profiles_fbf)
field_types = ProfileDatasets.getFieldTypes(sQobj, base_url,
field_type_fbf)
dataset_info = ProfileDatasets.buildInsertDatasetProfiles(
sQobj, scrud, configItems, datasets, ds_profiles, field_types,
asset_inventory_dict)
## triggers a webtask to update the agolia index
updated_algolia = False
if (WebTasks.runWebTask(configItems)):
updated_algolia = True
print "******Ran Updated *** agolia script****" + str(updated_algolia)
print dataset_info
dsse = JobStatusEmailerComposer(configItems, logger, jobType)
if dataset_info['DatasetRecordsCnt'] > 1 and updated_algolia:
print "update complete"
dsse.sendJobStatusEmail([dataset_info])
else:
dataset_info = {
'Socrata Dataset Name': configItems['dataset_name'],
'SrcRecordsCnt': 0,
'DatasetRecordsCnt': 0,
'fourXFour': "Nothing to Insert"
}
dataset_info['isLoaded'] = 'success'
dsse.sendJobStatusEmail([dataset_info])
