def control(label_to_titles, s):
utils.log(str(len(label_to_titles)) + " titles loaded")
label_to_titles = add_image_thumbs(label_to_titles, s)
utils.write_to_s3(label_to_titles,
'batch_titles_fd_' + utils.file_date() + '.json',
directory=s["s3dir"])
return label_to_titles
def control(filtered_label_dict, s):
user_data, filtered_label_dict = update_user_info(filtered_label_dict, s)
filtered_label_dict = update_dict_with_user_info(filtered_label_dict,
user_data)
utils.write_to_s3(
filtered_label_dict,
utils.file_name(prefix='_batch_filt_label_dict_enhanced_'),
directory=s["s3dir"])
return filtered_label_dict
def control(s):
processed_list = make_processed_list(s)
exist_min_id, exist_max_id = get_min_max(processed_list)
new_status_batch = get_new(exist_max_id, s)
deduped_status_batch = dedup(new_status_batch, processed_list)
updated_processed_list = update_processed_list(processed_list,
deduped_status_batch)
utils.write_to_s3(updated_processed_list,
"processed_list",
directory=s["s3dir"])
date_filtered_batch = filter_on_day(deduped_status_batch)
# utils.write_to_s3(date_filtered_batch, utils.file_name(sufix="_batch"), directory=s["s3dir"])
return date_filtered_batch
def control(batch_enhanced, s):
batch_enhanced_full = utils.read_from_s3(
utils.file_name(sufix="_batch_enhanced_full"),
seed=[],
directory=s["s3dir"])
print("Length of enhanced pre batch: " + str(len(batch_enhanced_full)))
enhanced_w_batch = add_batch(batch_enhanced_full, batch_enhanced)
utils.write_to_s3(enhanced_w_batch,
utils.file_name(sufix="_batch_enhanced_full"),
directory=s["s3dir"])
filtered_label_dict = make_filtered_label_dict(batch_enhanced_full)
return filtered_label_dict
def control(filt_label_dict, label_to_titles, s):
utils.log(str(len(filt_label_dict)) + " loaded, " + str(len(label_to_titles)) + " titles loaded")
label_data = process_label_dict(filt_label_dict, label_to_titles)
sorted_label_data_list = label_data_to_list(label_data)
meta = make_meta(label_data)
final_data = {"label_data":sorted_label_data_list, "meta_data": meta}
# name = "d3-" + utils.file_date() + "-label_format" + ".json"
# utils.write_to_s3(
# json.dumps(final_data),
# name,
# directory=s["s3dir"] + 'production/',
# public=True)
normalized_name = s["name"] + ".json"
utils.write_to_s3(
json.dumps(final_data),
normalized_name,
directory='data-aws/shared_data/production/'+utils.file_date() + "/",
public=True)
return None
def control(batch_enhanced,s):
filtered_label_dict = utils.read_from_s3(utils.file_name( prefix = "_batch_filt_label_dict_enhanced_fld"), directory=s["s3dir"])
filtered_label_dict, processed_list = make_filtered_label_dict(batch_enhanced, filtered_label_dict)
utils.write_to_s3(filtered_label_dict, utils.file_name( prefix = "_batch_filt_label_dict_enhanced_fld"), directory=s["s3dir"])
utils.write_to_s3(processed_list, utils.file_name( sufix = "_processed_list_fld"), directory=s["s3dir"])
return filtered_label_dict