def generate_schedule_item(self):
'''Generate a default schedule_item. This method should normally be called every few
hours.
'''
if self.should_generate_new_item():
default_profile = PROFILES[0]()
schedule_item = ScheduleItem(run_name='{0}-{1}'.format(
strftime("%Y-%b-%d-%H:%M:%S", localtime()), DEFAULT_RUN_NAME),
query_profile=default_profile,
time_limit_sec=RUN_TIME_LIMIT)
schedule_item.save_pickle()
self.time_last_generated = time.time()
LOG.info('Generated Schedule Item')
sleep(2)
def generate_schedule_item(self):
'''Generate a default schedule_item. This method should normally be called every few
hours.
'''
if self.should_generate_new_item():
profile = ImpalaNestedTypesProfile() if NESTED_TYPES_MODE else DefaultProfile()
schedule_item = ScheduleItem(
run_name = '{0}-{1}'.format(strftime(
"%Y-%b-%d-%H:%M:%S", localtime()), DEFAULT_RUN_NAME),
query_profile=profile,
time_limit_sec=RUN_TIME_LIMIT)
schedule_item.save_pickle()
self.time_last_generated = time.time()
LOG.info('Generated Schedule Item')
sleep(2)
def start_run():
'''Method that receives POST requests and gernerates a schedule item.'''
if request.method != 'POST': return 'fail'
if 'time_limit' in request.form:
# This is a custom run because time_limit item is present only in the custom_run form.
# Values will be extracted from the form and a new profile will be generated.
new_profile = DefaultProfile()
# Bounds
new_profile._bounds['MAX_NESTED_QUERY_COUNT'] = (
int(request.form['max_nested_query_count_from']),
int(request.form['max_nested_query_count_to']))
new_profile._bounds['MAX_NESTED_EXPR_COUNT'] = (
int(request.form['max_nested_expr_count_from']),
int(request.form['max_nested_expr_count_to']))
new_profile._bounds['SELECT_ITEM_COUNT'] = (
int(request.form['select_item_count_from']),
int(request.form['select_item_count_to']))
new_profile._bounds['WITH_TABLE_COUNT'] = (
int(request.form['with_table_count_from']),
int(request.form['with_table_count_to']))
new_profile._bounds['TABLE_COUNT'] = (
int(request.form['table_count_from']),
int(request.form['table_count_to']))
new_profile._bounds['ANALYTIC_LEAD_LAG_OFFSET'] = (
int(request.form['analytic_lead_lag_offset_from']),
int(request.form['analytic_lead_lag_offset_to']))
new_profile._bounds['ANALYTIC_WINDOW_OFFSET'] = (
int(request.form['analytic_window_offset_from']),
int(request.form['analytic_window_offset_to']))
# Select Item Category
new_profile._weights['SELECT_ITEM_CATEGORY']['AGG'] = int(
request.form['select_agg'])
new_profile._weights['SELECT_ITEM_CATEGORY']['ANALYTIC'] = int(
request.form['select_analytic'])
new_profile._weights['SELECT_ITEM_CATEGORY']['BASIC'] = int(
request.form['select_basic'])
# Types
new_profile._weights['TYPES'][Boolean] = int(
request.form['types_boolean'])
new_profile._weights['TYPES'][Char] = int(request.form['types_char'])
new_profile._weights['TYPES'][Decimal] = int(
request.form['types_decimal'])
new_profile._weights['TYPES'][Float] = int(request.form['types_float'])
new_profile._weights['TYPES'][Int] = int(request.form['types_int'])
new_profile._weights['TYPES'][Timestamp] = int(
request.form['types_timestamp'])
# Join
new_profile._weights['JOIN']['INNER'] = int(request.form['join_inner'])
new_profile._weights['JOIN']['LEFT'] = int(request.form['join_left'])
new_profile._weights['JOIN']['RIGHT'] = int(request.form['join_right'])
new_profile._weights['JOIN']['FULL_OUTER'] = int(
request.form['join_full_outer'])
new_profile._weights['JOIN']['CROSS'] = int(request.form['join_cross'])
# Optional Query Clauses Probabilities
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['WITH'] = float(
request.form['optional_with'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['FROM'] = float(
request.form['optional_from'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['WHERE'] = float(
request.form['optional_where'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES'][
'GROUP_BY'] = float(request.form['optional_group_by'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['HAVING'] = float(
request.form['optional_having'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['UNION'] = float(
request.form['optional_union'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES'][
'ORDER_BY'] = float(request.form['optional_order_by'])
# Optional Analytic Clauses Probabilities
new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES'][
'PARTITION_BY'] = float(
request.form['optional_analytic_partition_by'])
new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES'][
'ORDER_BY'] = float(request.form['optional_analytic_order_by'])
new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES'][
'WINDOW'] = float(request.form['optional_analytic_window'])
# Misc Probabilities
new_profile._probabilities['MISC']['INLINE_VIEW'] = float(
request.form['misc_inline_view'])
new_profile._probabilities['MISC']['SELECT_DISTINCT'] = float(
request.form['misc_select_distinct'])
new_profile._probabilities['MISC']['SCALAR_SUBQUERY'] = float(
request.form['misc_scalar_subquery'])
new_profile._probabilities['MISC']['UNION_ALL'] = float(
request.form['misc_union_all'])
# Analytic Designs
new_profile._flags['ANALYTIC_DESIGNS']['TOP_LEVEL_QUERY_WITHOUT_LIMIT'] = \
'analytic_designs_top_level_no_limit' in request.form
new_profile._flags['ANALYTIC_DESIGNS']['DETERMINISTIC_ORDER_BY'] = \
'analytic_designs_deterministic_order_by' in request.form
new_profile._flags['ANALYTIC_DESIGNS']['NO_ORDER_BY'] = \
'analytic_designs_no_order_by' in request.form
new_profile._flags['ANALYTIC_DESIGNS']['ONLY_SELECT_ITEM'] = \
'analytic_designs_only_select_item' in request.form
new_profile._flags['ANALYTIC_DESIGNS']['UNBOUNDED_WINDOW'] = \
'analytic_designs_unbounded_window' in request.form
new_profile._flags['ANALYTIC_DESIGNS']['RANK_FUNC'] = \
'analytic_designs_rank_func' in request.form
schedule_item = ScheduleItem(run_name=request.form['run_name'],
query_profile=new_profile,
time_limit_sec=int(
request.form['time_limit']),
git_command=request.form['git_command'],
parent_job='')
else:
# Run based on previous run
schedule_item = ScheduleItem(
run_name=request.form['run_name'],
query_profile=DefaultProfile(),
time_limit_sec=24 * 3600, # Default time limit is 24 hours
git_command=request.form['git_command'],
parent_job=request.form['report_id'])
schedule_item.save_pickle()
return 'success'
def start_run():
'''Method that receives POST requests and gernerates a schedule item.'''
if request.method != 'POST': return 'fail'
if 'time_limit' in request.form:
# This is a custom run because time_limit item is present only in the custom_run form.
# Values will be extracted from the form and a new profile will be generated.
new_profile = DefaultProfile()
# Bounds
new_profile._bounds['MAX_NESTED_QUERY_COUNT'] = (
int(request.form['max_nested_query_count_from']),
int(request.form['max_nested_query_count_to']))
new_profile._bounds['MAX_NESTED_EXPR_COUNT'] = (
int(request.form['max_nested_expr_count_from']),
int(request.form['max_nested_expr_count_to']))
new_profile._bounds['SELECT_ITEM_COUNT'] = (
int(request.form['select_item_count_from']),
int(request.form['select_item_count_to']))
new_profile._bounds['WITH_TABLE_COUNT'] = (
int(request.form['with_table_count_from']),
int(request.form['with_table_count_to']))
new_profile._bounds['TABLE_COUNT'] = (
int(request.form['table_count_from']),
int(request.form['table_count_to']))
new_profile._bounds['ANALYTIC_LEAD_LAG_OFFSET'] = (
int(request.form['analytic_lead_lag_offset_from']),
int(request.form['analytic_lead_lag_offset_to']))
new_profile._bounds['ANALYTIC_WINDOW_OFFSET'] = (
int(request.form['analytic_window_offset_from']),
int(request.form['analytic_window_offset_to']))
# Select Item Category
new_profile._weights['SELECT_ITEM_CATEGORY']['AGG'] = int(
request.form['select_agg'])
new_profile._weights['SELECT_ITEM_CATEGORY']['ANALYTIC'] = int(
request.form['select_analytic'])
new_profile._weights['SELECT_ITEM_CATEGORY']['BASIC'] = int(
request.form['select_basic'])
# Types
new_profile._weights['TYPES'][Boolean] = int(request.form['types_boolean'])
new_profile._weights['TYPES'][Char] = int(request.form['types_char'])
new_profile._weights['TYPES'][Decimal] = int(request.form['types_decimal'])
new_profile._weights['TYPES'][Float] = int(request.form['types_float'])
new_profile._weights['TYPES'][Int] = int(request.form['types_int'])
new_profile._weights['TYPES'][Timestamp] = int(request.form['types_timestamp'])
# Join
new_profile._weights['JOIN']['INNER'] = int(request.form['join_inner'])
new_profile._weights['JOIN']['LEFT'] = int(request.form['join_left'])
new_profile._weights['JOIN']['RIGHT'] = int(request.form['join_right'])
new_profile._weights['JOIN']['FULL_OUTER'] = int(request.form['join_full_outer'])
new_profile._weights['JOIN']['CROSS'] = int(request.form['join_cross'])
# Optional Query Clauses Probabilities
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['WITH'] = float(
request.form['optional_with'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['FROM'] = float(
request.form['optional_from'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['WHERE'] = float(
request.form['optional_where'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['GROUP_BY'] = float(
request.form['optional_group_by'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['HAVING'] = float(
request.form['optional_having'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['UNION'] = float(
request.form['optional_union'])
new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['ORDER_BY'] = float(
request.form['optional_order_by'])
# Optional Analytic Clauses Probabilities
new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES']['PARTITION_BY'] = float(
request.form['optional_analytic_partition_by'])
new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES']['ORDER_BY'] = float(
request.form['optional_analytic_order_by'])
new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES']['WINDOW'] = float(
request.form['optional_analytic_window'])
# Misc Probabilities
new_profile._probabilities['MISC']['INLINE_VIEW'] = float(
request.form['misc_inline_view'])
new_profile._probabilities['MISC']['SELECT_DISTINCT'] = float(
request.form['misc_select_distinct'])
new_profile._probabilities['MISC']['SCALAR_SUBQUERY'] = float(
request.form['misc_scalar_subquery'])
new_profile._probabilities['MISC']['UNION_ALL'] = float(
request.form['misc_union_all'])
# Analytic Designs
new_profile._flags['ANALYTIC_DESIGNS']['TOP_LEVEL_QUERY_WITHOUT_LIMIT'] = \
'analytic_designs_top_level_no_limit' in request.form
new_profile._flags['ANALYTIC_DESIGNS']['DETERMINISTIC_ORDER_BY'] = \
'analytic_designs_deterministic_order_by' in request.form
new_profile._flags['ANALYTIC_DESIGNS']['NO_ORDER_BY'] = \
'analytic_designs_no_order_by' in request.form
new_profile._flags['ANALYTIC_DESIGNS']['ONLY_SELECT_ITEM'] = \
'analytic_designs_only_select_item' in request.form
new_profile._flags['ANALYTIC_DESIGNS']['UNBOUNDED_WINDOW'] = \
'analytic_designs_unbounded_window' in request.form
new_profile._flags['ANALYTIC_DESIGNS']['RANK_FUNC'] = \
'analytic_designs_rank_func' in request.form
schedule_item = ScheduleItem(
run_name = request.form['run_name'],
query_profile = new_profile,
time_limit_sec = int(request.form['time_limit']),
git_command = request.form['git_command'],
parent_job = '')
else:
# Run based on previous run
schedule_item = ScheduleItem(
run_name = request.form['run_name'],
query_profile = DefaultProfile(),
time_limit_sec = 24 * 3600, # Default time limit is 24 hours
git_command = request.form['git_command'],
parent_job = request.form['report_id'])
schedule_item.save_pickle()
return 'success'
