def do_work(jid): update_job_status(jid, 'in progress') locations = get_locations() menus = get_menus() pricing = get_price() rd_jobs.hset(_generate_job_key(jid), 'location number', len(locations)) max_price = max(pricing) columns1 = ['id', 'address', 'categories', 'menus', 'pricing'] output = pd.DataFrame(columns = columns1) index = 0 counter = 0 subset1 = subset_.append(get_locations.loc[(get_locations['address'] == st\ atus) & (start_id <= get_locations['id']) & (end_id >= get_locations['id']), [' id', 'address', 'category']]) for index in range(len(subset1)): location = subset1.iloc[index]['address'] menu = subset1.iloc[index]['menu'] pricing = subset1.iloc[index]['pricing'] if (index ==0): counter = 0 elif(pricing != subset1.iloc[index-1]['pricing']): counter = 0) new_table = formula(id, address, categories, menus_amountmax, menus_amountmin) update_job_status(jid, 'complete')
def execute_job(jid): update_job_status(jid, 'in progress') start, end = job.hmget(generate_job_key(jid), 'start', 'end') start = int(start) end = int(end) years = list(range(start, end + 1)) numpoints = (end + 1 - start) total = [0] * numpoints count = [0] * numpoints for key in rd.keys(): year = int(rd.hget(key, 'year')) if year >= start and year <= end: index = year - start lettertotal = len(str(rd.hget(key, 'entity'))) total[index] = total[index] + lettertotal count[index] = count[index] + 1 avg = [i / j for i, j in zip(total, count)] title = "Average length of names in Oscar films between " + str( start) + " and " + str(end) plt.plot(years, avg, 'g-o', label="Nominees") plt.xlabel("Years") plt.ylabel("Average Num. Letters") plt.title(title) plt.savefig('/oscarlength.png') with open('/oscarlength.png', 'rb') as f: img = f.read() job.hset(generate_job_key(jid), 'image', img) update_job_status(jid, 'complete')
def execute_job(jid): ''' Define function that generates a plot of time-series data for a particular , and randomly-generated, temperature field: ''' def generate_plot(): temp_num = rd_temp.dbsize( ) # Get the number of temperature dictionaries in database ''' Use the "random" module to randomly-generate a temp field to develop a time-series line plot of from the list: "GALT", "GMAXLT", and "GMINLT": ''' temp_field_list = [ "GALT (Celsius)", "GMAXLT (Celsius)", "GMINLT(Celsius)" ] temp_field = temp_field_list[random.randint(0, 2)] # Use matplolib to generate plot of time-series temperature data: time_array = numpy.array([]) # Initialize array to store time data temperature_array = numpy.array( []) # Initialize array to store temperature data # Use for loop to append values to array storing time and temperature field values: for i in range(0, temp_num): temp_key = i # Assign keyname to temperature time_array = numpy.append(time_array, i) temperature_array = numpy.append( temperature_array, float(rd_temp.hget(temp_key, temp_field))) # Use "Matplotlib" to generate a line plot of the time-series dataset: plt.plot(time_array, temperature_array) plt.title("Plot of " + temp_field + " vs. Time (Months) Starting from 01/01/1930 ") plt.xlabel("Time (Months)") plt.ylabel(temp_field) plt.savefig('/Time-Series-Line-Plot.png') # Open image and add to redis database containing job information: with open('/Time-Series-Line-Plot.png', 'rb') as f: img = f.read() rd_jobs.hset(f'job.{jid}', "image", img) # Set the image in redis database return_status = "Plot Generated Successfully" return return_status # Invoke "generate_plot()" method to generate and store time-series line plot: generate_plot() # Invoke "update_job_status()" function to designate job as "complete": jobs.update_job_status(jid, "complete")
def execute_job(jid): # update job status update_job_status(jid, 'in_progress') # generate a graph from the data execute(jid) update_job_status(jid, 'completed')
def execute_job(jid): update_job_status(jid, 'in progress') # Create figure count = int(rd.get('count')) adoptions_dict = get_data() age_x = [] adoptions_y = [] for i in range(0, count): if (adoptions_dict[i]["Outcome Type"] == "Adoption"): adoption_age = adoptions_dict[i]["Age upon Outcome"] if adoption_age not in age_x: age_x.append(adoption_age) adoptions_y.append(1) else: for index, item in enumerate(age_x): if item == adoption_age: adoptions_y[index] = adoption_y[index] + 1
def execute_job(jid): data = jobs.get_job_data(jid) # call matplotlib to make a plot of something # plot stuff here... start = data['start'] end = data['end'] # pplot the counts of each type of animal adopted in between a date range animal_types = ['Bird', 'Cat', 'Dog', 'Livestock', 'Other'] animal_counts = [0, 0, 0, 0, 0] for key in rd1.keys(): if (start <= key[DateTime].decode('utf-8') <= end): this_animal_type = key['Animal_Type'] if this_animal_type == 'Bird': animal_counts[0] += 1 else if this_animal_type == 'Cat': animal_counts[1] += 1 else if this_animal_type == 'Dog': animal_counts[2] += 1 else if this_animal_type == 'Livestock': animal_counts[3] += 1 else if this_animal_type == 'Other': animal_counts[4] += 1 plt.clf() plt.bar(animal_types, animal_counts, color='green') plt.xlabel('Animal Type') plt.ylabel('Frequency') #plt.title('Amino Acid Frequency') #plt.xticks(aas_pos, aas) plt.savefig('/output_image.png') with open('/output_image.png', 'rb') as f: img = f.read() rd.hset(f'job.{jid}', 'result', img) jobs.update_job_status(jid, "completed")
def execute_job(jid): update_job_status(jid, 'in progress') restaurant = rd_plot.hget(f'job.{jid}', 'restaurant').decode('utf-8') dates = [] scores = [] for key in rd.keys(): if rd.hget(key, 'Restaurant Name').decode('utf-8') == restaurant: date = rd.hget(key, 'Date of Inspection').decode('utf-8') score = rd.hget(key, 'Inspection Score').decode('utf-8') dates.append(datetime.datetime.strptime(date, "%m/%d/%Y")) scores.append(int(score)) x = dates y = scores plt.figure(figsize=(10, 10)) plt.plot_date(x, y) restaurant = restaurant.replace(' ', '-') plt.savefig(str(restaurant) + '.png') file_bytes = open(str(restaurant) + '.png', 'rb').read() rd_plot.set(str(restaurant) + '_plot', file_bytes) print('Plot Key: ' + str(restaurant) + '_plot' + "\n") update_job_status(jid, 'complete')
def execute_job(jobid): update_job_status(jobid, 'in progress') time.sleep(15) df = pd.read_csv("./Austin_COVID-19_Complaint_Cases.csv") def df_query(x): return lambda dfx: dfx['TYPEOFCOMPLAINT'] == x # complaints related to face masks df1 = valid_df[df_query('Face Covering Non-Compliance - Business')] df2 = valid_df[df_query('Face Covering')] df3 = valid_df[df_query('Face-Covering')] # complaints related to social distancing df4 = valid_df[df_query('Social Distancing')] df5 = valid_df[df_query('Over Occupancy')] # complaints related to eviction df6 = valid_df[df_query('Eviction')] # complaints of other categories df7 = valid_df[df_query('Other')] plt.figure(figsize=(10, 10)) plt.scatter(df1.LATITUDE, df1.LONGITUDE, 1.0, 'b') plt.scatter(df2.LATITUDE, df2.LONGITUDE, 1.0, 'b') plt.scatter(df3.LATITUDE, df3.LONGITUDE, 1.0, 'b') plt.scatter(df4.LATITUDE, df4.LONGITUDE, 1.0, 'r') plt.scatter(df5.LATITUDE, df5.LONGITUDE, 1.0, 'r') plt.scatter(df6.LATITUDE, df6.LONGITUDE, 1.0, 'g') plt.scatter(df7.LATITUDE, df7.LONGITUDE, 1.0, 'y') plt.savefig('Austin_COVID_Complaints_Mapped.png') file_bytes = open('/tmp/Austin_COVID_Complaints_Mapped.png', 'rb').read() # set the file bytes as a key in Redis rd.set('key', file_bytes) update_job_status(jobid, 'complete')
def execute_job(jid): update_job_status(jid, 'in progress') animals = json.loads(rd.get('key')) outputType = animals['output_type'] job_id = 'job.{}'.format(jid) euthanasia = 0 returnedToOwner = 0 adoption = 0 transfer = 0 for x in outputType: if x == 'Euthanasia': euthanasia = euthanasia + 1 if x == 'Return to Owner': returnedToOwner = returnedToOwner + 1 if x == 'Adoption': adoption = adoption + 1 if x == 'Transfer': transfer = transfer + 1 x = ['Euthanasia', 'Return to Owner', 'Adoption', 'Transfer'] y = [euthanasia, returnedToOwner, adoption, transfer] plt.figure() plt.bar(y) plt.set(gca, 'XTickLabel', {'EU', 'RTO', 'ADP', 'TR'}) plt.savefig('/outputtype.png') with open('/outputtype.png', 'rb') as f: img = f.read() rd_imgs.hset(jid, 'image', img) update_job_status(jid, 'complete')
from matplotlib import pyplot as plt @q.worker def execute_job(jid): update_job_status(jid, 'in progress') # Create figure count = int(rd.get('count')) adoptions_dict = get_data() age_x = [] adoptions_y = [] for i in range(0, count): if (adoptions_dict[i]["Outcome Type"] == "Adoption"): adoption_age = adoptions_dict[i]["Age upon Outcome"] if adoption_age not in age_x: age_x.append(adoption_age) adoptions_y.append(1) else: for index, item in enumerate(age_x): if item == adoption_age: adoptions_y[index] = adoption_y[index] + 1 plt.bar(age_x, adoptions_y) plt.xlabel('Adoption Age') plt.ylabel('# of Adoptions') plt.savefig('adoption_ages.png') update_job_status(jid, 'complete') if __name__ == '__main__': execute_job()
def do_work(jid): update_job_status(jid, 'in progress') time.sleep(15) update_job_status(jid, 'complete')
def execute_job(jid): data = jobs.get_job_data(jid) job_type = jobs.get_job_type(jid) #analysis: plot total number of outcomes for each day in date range if (job_type == 'dates'): start = jobs.get_job_start(jid) end = jobs.get_job_end(jid) start_date = datetime.datetime.strptime(start, "%m-%d-%Y") end_date = datetime.datetime.strptime(end, '%m-%d-%Y') consecutive_dates = [ start_date + datetime.timedelta(days=x) for x in range((end_date - start_date).days + 1) ] list_of_all_dates = [] for key in rd1.keys(): key_time_temp = rd1.hget( key, 'Date_of_Entry').decode('utf-8').split()[0] key_time = datetime.datetime.strptime(key_time_temp, '%m-%d-%Y') list_of_all_dates.append(key_time) list_of_outcomes_of_the_day = [0] * len(consecutive_dates) x_labeler = [] for i in range(len(consecutive_dates)): list_of_outcomes_of_the_day[i] = list_of_all_dates.count( consecutive_dates[i]) x_labeler.append(i) #consecutive_date_strings = [] #for item in consecutive_dates: # consecutive_date_strings.append(item.strftime("%m-%d-%Y")) plt.clf() plt.bar(x_labeler, list_of_outcomes_of_the_day, color='green') plt.xlabel('Days since Start') plt.ylabel('# Outcomes per Day') plt.title('Outcome Occurences by Day \n' + start + ' - ' + end) plt.savefig('/outcomes_by_animal_type.png') with open('/outcomes_by_animal_type.png', 'rb') as f: img = f.read() rd.hset(f'job.{jid}', 'result', img) jobs.update_job_status(jid, 'complete') #analysis: plot total # of outcomes by type of animal in date range if (job_type == 'animal_type'): #jobs.update_job_status(jid, 'it has entered the for loop') animal_types = ['Bird', 'Cat', 'Dog', 'Livestock', 'Other'] animal_counts = [0, 0, 0, 0, 0] for key in rd1.keys(): #jobs.update_job_status(jid, str(key))#'it has entered the for loop') this_animal_type = str(rd1.hget(key, 'Animal_Type'))[1:] #jobs.update_job_status(jid, str(this_animal_type)) if this_animal_type == "'Bird'": animal_counts[0] += 1 elif this_animal_type == "'Cat'": animal_counts[1] += 1 elif this_animal_type == "'Dog'": animal_counts[2] += 1 elif this_animal_type == "'Livestock'": animal_counts[3] += 1 elif this_animal_type == "'Other'": animal_counts[4] += 1 #jobs.update_job_status(jid, str(animal_counts)) plt.clf() plt.bar(animal_types, animal_counts, color='green') plt.xlabel('Animal Type') plt.ylabel('Frequency') plt.title('Outcomes by Animal Type') plt.savefig('/outcomes_by_animal_type.png') with open('/outcomes_by_animal_type.png', 'rb') as f: img = f.read() rd.hset(f'job.{jid}', 'result', img) jobs.update_job_status(jid, 'complete')
def execute_job(jid): update_job_status(jid, 'in progress') add_worker_ip(jid, worker_ip) time.sleep(15) update_job_status(jid, 'complete')
def execute_job(jid): workerIP = os.environ.get('WORKER_IP') update_job_status(jid, 'in progress', workerIP) time.sleep(15) update_job_status(jid, 'complete', workerIP)
def execute_job(jid): data = jobs.get_job_data(jid) job_type = jobs.get_job_type(jid) #jobs.update_job_status(jid, job_type)#'in progress') #analysis: plot total number of outcomes for each day in date range #first job type: dates if (job_type == 'dates'): #jobs.update_job_status(jid, 'entered first if statement') start = jobs.get_job_start(jid) end = jobs.get_job_end(jid) #set inital outcomes count animal_outcomes_of_day = 0 #x values: list of dates x_values_to_plot = [] #y values: list of integer numbers of outcomes per date y_values_to_plot = [] start_date = datetime.datetime.strptime(start, "%m-%d-%Y") end_date = datetime.datetime.strptime(end, '%m-%d-%Y') #format to check for full day for key in rd1.keys(): #jobs.update_job_status(jid, 'you have entered first for loop') #decode time to a string without hours and minutes key_time_temp = rd1.hget( key, 'Date_of_Entry').decode('utf-8').split()[0] #string to datetime key_time = datetime.datetime.strptime(key_time_temp, '%m-%d-%Y') #jobs.update_job_status(jid, str(key_time)) #check for keys in date range if (start_date <= key_time and end_date >= key_time): #jobs.update_job_status(jid, 'this is entering 2nd if statement') #set specific date x = key_time_temp #jobs.update_job_status(jid, x) #check if date is alread in x_values_to_plot if x not in x_values_to_plot: #jobs.update_job_status(jid, 'this is entering 3rd if statement') #if new date: add to list of x_values_to_plot x_values_to_plot.append(x) #check through db for each animal with matching Date_of_Entry for i in range(rd1.dbsize()): #jobs.update_job_status(jid, 'this is entering 2nd for loop') #issue with formatting? if (x == rd1.hget( i, 'Date_of_Entry').decode('utf-8').split()[0]): #jobs.update_job_status(jid, 'this is entering 4th if statement') #increment animal_outcomes_of_day count animal_outcomes_of_day = animal_outcomes_of_day + 1 #finalize count for the day y = animal_outcomes_of_day #add total count to list of y_values_to_plot y_values_to_plot.append(y) #reset animal_outcomes_of_day count before moving to next day animal_outcomes_of_day = 0 jobs.update_job_status(jid, 'we have exited first loop') #plot scatter plot plt.scatter(x_values_to_plot, y_values_to_plot) plt.savefig('/outcomes_by_date.png') with open('/outcomes_by_date.png', 'rb') as f: img = f.read() rd.hset(f'job.{jid}', 'result', img) jobs.update_job_status(jid, 'complete') #analysis: plot total # of outcomes by type of animal in date range if (job_type == 'animal_type'): #jobs.update_job_status(jid, 'it has entered the for loop') animal_types = ['Bird', 'Cat', 'Dog', 'Livestock', 'Other'] animal_counts = [0, 0, 0, 0, 0] for key in rd1.keys(): #jobs.update_job_status(jid, str(key))#'it has entered the for loop') this_animal_type = str(rd1.hget(key, 'Animal_Type'))[1:] #jobs.update_job_status(jid, str(this_animal_type)) if this_animal_type == "'Bird'": animal_counts[0] += 1 elif this_animal_type == "'Cat'": animal_counts[1] += 1 elif this_animal_type == "'Dog'": animal_counts[2] += 1 elif this_animal_type == "'Livestock'": animal_counts[3] += 1 elif this_animal_type == "'Other'": animal_counts[4] += 1 #jobs.update_job_status(jid, str(animal_counts)) plt.clf() plt.bar(animal_types, animal_counts, color='green') plt.xlabel('Animal Type') plt.ylabel('Frequency') plt.title('Outcomes by Animal Type') plt.savefig('/outcomes_by_animal_type.png') with open('/outcomes_by_animal_type.png', 'rb') as f: img = f.read() rd.hset(f'job.{jid}', 'result', img) jobs.update_job_status(jid, 'complete')
def execute_job(jid): update_job_status(jid, "in progress") ## Added for section B add_worker_IP(jid, worker_ip) time.sleep(15) update_job_status(jid, "complete")
def execute_job(jid): update_job_status(jid, 'in progress') stock_and_sales_df = get_stock_and_sales() products_df = get_products() rd_jobs.hset(_generate_job_key(jid), 'stock rows', len(stock_and_sales_df)) #finds the maximum shelf life a product can have products_df = get_products() shelf_life_column = products_df["shelf_life"] max_shelf_life = shelf_life_column.max() - 2 output_columns = ['Date', 'Store', 'Product', 'Stock', 'Sales'] #create columns based on maximum shelf life for index in range(max_shelf_life): start_index = "start" + str(index + 1) end_index = "end" + str(index + 1) output_columns.extend([start_index, end_index]) output_columns.extend(['total start', 'total end', 'surplus', 'gapout']) output_df = pd.DataFrame(columns=output_columns) store_input = (rd_jobs.hget(_generate_job_key(jid), 'store_input')).decode("utf-8") start_date = (rd_jobs.hget(_generate_job_key(jid), 'start_date')).decode("utf-8") end_date = (rd_jobs.hget(_generate_job_key(jid), 'end_date')).decode("utf-8") subset_df = stock_and_sales_df.loc[ (stock_and_sales_df['Store'] == store_input) & (start_date <= stock_and_sales_df['Date']) & (stock_and_sales_df['Date'] <= end_date), ['Date', 'Product', 'Store', 'Stock', 'Sales']] rd_jobs.hset(_generate_job_key(jid), 'subset rows', len(subset_df)) #initializes index and counter variables to know where the method is at in the for loop index = 0 counter = 0 #iterates through every row and runs surplus / gapout formulas for index in range(len(subset_df)): #creates local variables for values that carry over from subset dataframe to output dataframe date = subset_df.iloc[index]['Date'] product = subset_df.iloc[index]['Product'] store = subset_df.iloc[index]['Store'] stock = subset_df.iloc[index]['Stock'] sales = subset_df.iloc[index]['Sales'] shelf_life = products_df.loc[product]['wf_shelf_life'] #counter resets to 0 if a new product is being started if (index == 0): counter = 0 elif (product != subset_df.iloc[index - 1]['Product']): counter = 0 new_row = formulas_dynamic(index, counter, date, store, product, stock, sales, shelf_life, output_df) output_df = output_df.append(new_row, ignore_index=True) counter += 1 index += 1 products_df = products_df.reset_index() output_json = output_df.to_json() summary_df = summary(output_df, products_df) summary_json = summary_df.to_json() rd_jobs.hset(_generate_job_key(jid), 'output json', output_json) rd_jobs.hset(_generate_job_key(jid), 'summary json', summary_json) update_job_status(jid, 'complete')
def execute_job(jid): update_job_status(jid, 'in progress') time.sleep(15) update_job_status(jid, 'complete')
def execute_job(jid): data = jobs.get_job_data(jid) job_type = jobs.get_job_type(jid) #jobs.update_job_status(jid, job_type)#'in progress') #analysis: plot total number of outcomes for each day in date range #first job type: dates if (job_type == 'dates'): #jobs.update_job_status(jid, str(data)) #jobs.update_job_status(jid, 'You have entered the if statement') #format of how dates come in??????????? should be a string start = jobs.get_job_start(jid) end = jobs.get_job_end(jid) #jobs.update_job_status(jid, str(start)) #set inital outcomes count animal_outcomes_of_day = 0 #x values: list of dates x_values_to_plot = [] #y values: list of integer numbers of outcomes per date y_values_to_plot = [] start_date = datetime.datetime.strptime(start, "%m-%d-%Y") end_date = datetime.datetime.strptime(end, '%m-%d-%Y') #format to check for full day, rather than specific time?????????? for key in rd1.keys(): #jobs.update_job_status(jid, 'you have entered the for loop') key_time_temp = str(rd1.hget(key, 'Date_of_Entry'))[1:] #jobs.update_job_status(jid, str(key_time_temp)) #key_time_temp = key_time_temp.replace("/","-") #jobs.update_job_status(jid, str(key_time_temp)) key_time = datetime.datetime.strptime(key_time_temp, "'%m-%d-%Y'") #jobs.update_job_status(jid, str(key_time_temp)) # ------------------- #check for keys in date range if (start_date <= key_time and end_date >= key_time): #jobs.update_job_status(jid, 'this is entering 2nd if statement') #set specific date x = str(key_time) jobs.update_job_status(jid, str(x)) # ----------- #check if date is alread in x_values_to_plot if x not in x_values_to_plot: jobs.update_job_status( jid, 'this is entering 3rd if statement') #if new date: add to list of x_values_to_plot x_values_to_plot.append(x) #check through db for each animal with matching Date_of_Entry for i in range(rd1.dbsize()): #will there be an issue with b formatting? should we use [1:]? if (x == rd1.hget(i, 'Date_of_Entry')): #increment animal_outcomes_of_day count animal_outcomes_of_day = animal_outcomes_of_day + 1 #finalize count for the day y = animal_outcomes_of_day #add total count to list of y_values_to_plot y_values_to_plot.append(y) #reset animal_outcomes_of_day count before moving to next day animal_outcomes_of_day = 0 #plot line graph plt.scatter(x_values_to_plot, y_values_to_plot) #plt.show() plt.savefig('/outcomes_by_date.png') with open('/outcomes_by_date.png', 'rb') as f: img = f.read() rd.hset(f'job.{jid}', 'result', img) #rd.hset(jobid, 'image', img) #jobs.update_job_status(jid, 'complete') #analysis: plot total # of outcomes by type of animal in date range if (job_type == 'animal_type'): #jobs.update_job_status(jid, 'it has entered the for loop') animal_types = ['Bird', 'Cat', 'Dog', 'Livestock', 'Other'] animal_counts = [0, 0, 0, 0, 0] for key in rd1.keys(): #jobs.update_job_status(jid, str(key))#'it has entered the for loop') this_animal_type = str(rd1.hget(key, 'Animal_Type'))[1:] #jobs.update_job_status(jid, str(this_animal_type)) if this_animal_type == "'Bird'": animal_counts[0] += 1 elif this_animal_type == "'Cat'": animal_counts[1] += 1 elif this_animal_type == "'Dog'": animal_counts[2] += 1 elif this_animal_type == "'Livestock'": animal_counts[3] += 1 elif this_animal_type == "'Other'": animal_counts[4] += 1 #jobs.update_job_status(jid, str(animal_counts)) plt.clf() plt.bar(animal_types, animal_counts, color='green') plt.xlabel('Animal Type') plt.ylabel('Frequency') plt.title('Outcomes by Animal Type') #plt.show() plt.savefig('/outcomes_by_animal_type.png') with open('/outcomes_by_animal_type.png', 'rb') as f: img = f.read() #rd.hset("job.{}".format(jid), 'image' img) rd.hset(f'job.{jid}', 'result', img) jobs.update_job_status(jid, 'complete')