Example #1
0
 def generate_schedule_item(self):
     '''Generate a default schedule_item. This method should normally be called every few
 hours.
 '''
     if self.should_generate_new_item():
         default_profile = PROFILES[0]()
         schedule_item = ScheduleItem(run_name='{0}-{1}'.format(
             strftime("%Y-%b-%d-%H:%M:%S", localtime()), DEFAULT_RUN_NAME),
                                      query_profile=default_profile,
                                      time_limit_sec=RUN_TIME_LIMIT)
         schedule_item.save_pickle()
         self.time_last_generated = time.time()
         LOG.info('Generated Schedule Item')
     sleep(2)
 def generate_schedule_item(self):
   '''Generate a default schedule_item. This method should normally be called every few
   hours.
   '''
   if self.should_generate_new_item():
     profile = ImpalaNestedTypesProfile() if NESTED_TYPES_MODE else DefaultProfile()
     schedule_item = ScheduleItem(
         run_name = '{0}-{1}'.format(strftime(
             "%Y-%b-%d-%H:%M:%S", localtime()), DEFAULT_RUN_NAME),
         query_profile=profile,
         time_limit_sec=RUN_TIME_LIMIT)
     schedule_item.save_pickle()
     self.time_last_generated = time.time()
     LOG.info('Generated Schedule Item')
   sleep(2)
Example #3
0
def start_run():
    '''Method that receives POST requests and gernerates a schedule item.'''

    if request.method != 'POST': return 'fail'

    if 'time_limit' in request.form:
        # This is a custom run because time_limit item is present only in the custom_run form.
        # Values will be extracted from the form and a new profile will be generated.

        new_profile = DefaultProfile()

        # Bounds
        new_profile._bounds['MAX_NESTED_QUERY_COUNT'] = (
            int(request.form['max_nested_query_count_from']),
            int(request.form['max_nested_query_count_to']))
        new_profile._bounds['MAX_NESTED_EXPR_COUNT'] = (
            int(request.form['max_nested_expr_count_from']),
            int(request.form['max_nested_expr_count_to']))
        new_profile._bounds['SELECT_ITEM_COUNT'] = (
            int(request.form['select_item_count_from']),
            int(request.form['select_item_count_to']))
        new_profile._bounds['WITH_TABLE_COUNT'] = (
            int(request.form['with_table_count_from']),
            int(request.form['with_table_count_to']))
        new_profile._bounds['TABLE_COUNT'] = (
            int(request.form['table_count_from']),
            int(request.form['table_count_to']))
        new_profile._bounds['ANALYTIC_LEAD_LAG_OFFSET'] = (
            int(request.form['analytic_lead_lag_offset_from']),
            int(request.form['analytic_lead_lag_offset_to']))
        new_profile._bounds['ANALYTIC_WINDOW_OFFSET'] = (
            int(request.form['analytic_window_offset_from']),
            int(request.form['analytic_window_offset_to']))

        # Select Item Category
        new_profile._weights['SELECT_ITEM_CATEGORY']['AGG'] = int(
            request.form['select_agg'])
        new_profile._weights['SELECT_ITEM_CATEGORY']['ANALYTIC'] = int(
            request.form['select_analytic'])
        new_profile._weights['SELECT_ITEM_CATEGORY']['BASIC'] = int(
            request.form['select_basic'])

        # Types
        new_profile._weights['TYPES'][Boolean] = int(
            request.form['types_boolean'])
        new_profile._weights['TYPES'][Char] = int(request.form['types_char'])
        new_profile._weights['TYPES'][Decimal] = int(
            request.form['types_decimal'])
        new_profile._weights['TYPES'][Float] = int(request.form['types_float'])
        new_profile._weights['TYPES'][Int] = int(request.form['types_int'])
        new_profile._weights['TYPES'][Timestamp] = int(
            request.form['types_timestamp'])

        # Join
        new_profile._weights['JOIN']['INNER'] = int(request.form['join_inner'])
        new_profile._weights['JOIN']['LEFT'] = int(request.form['join_left'])
        new_profile._weights['JOIN']['RIGHT'] = int(request.form['join_right'])
        new_profile._weights['JOIN']['FULL_OUTER'] = int(
            request.form['join_full_outer'])
        new_profile._weights['JOIN']['CROSS'] = int(request.form['join_cross'])

        # Optional Query Clauses Probabilities
        new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['WITH'] = float(
            request.form['optional_with'])
        new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['FROM'] = float(
            request.form['optional_from'])
        new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['WHERE'] = float(
            request.form['optional_where'])
        new_profile._probabilities['OPTIONAL_QUERY_CLAUSES'][
            'GROUP_BY'] = float(request.form['optional_group_by'])
        new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['HAVING'] = float(
            request.form['optional_having'])
        new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['UNION'] = float(
            request.form['optional_union'])
        new_profile._probabilities['OPTIONAL_QUERY_CLAUSES'][
            'ORDER_BY'] = float(request.form['optional_order_by'])

        # Optional Analytic Clauses Probabilities
        new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES'][
            'PARTITION_BY'] = float(
                request.form['optional_analytic_partition_by'])
        new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES'][
            'ORDER_BY'] = float(request.form['optional_analytic_order_by'])
        new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES'][
            'WINDOW'] = float(request.form['optional_analytic_window'])

        # Misc Probabilities
        new_profile._probabilities['MISC']['INLINE_VIEW'] = float(
            request.form['misc_inline_view'])
        new_profile._probabilities['MISC']['SELECT_DISTINCT'] = float(
            request.form['misc_select_distinct'])
        new_profile._probabilities['MISC']['SCALAR_SUBQUERY'] = float(
            request.form['misc_scalar_subquery'])
        new_profile._probabilities['MISC']['UNION_ALL'] = float(
            request.form['misc_union_all'])

        # Analytic Designs
        new_profile._flags['ANALYTIC_DESIGNS']['TOP_LEVEL_QUERY_WITHOUT_LIMIT'] = \
            'analytic_designs_top_level_no_limit' in request.form
        new_profile._flags['ANALYTIC_DESIGNS']['DETERMINISTIC_ORDER_BY'] = \
            'analytic_designs_deterministic_order_by' in request.form
        new_profile._flags['ANALYTIC_DESIGNS']['NO_ORDER_BY'] = \
            'analytic_designs_no_order_by' in request.form
        new_profile._flags['ANALYTIC_DESIGNS']['ONLY_SELECT_ITEM'] = \
            'analytic_designs_only_select_item' in request.form
        new_profile._flags['ANALYTIC_DESIGNS']['UNBOUNDED_WINDOW'] = \
            'analytic_designs_unbounded_window' in request.form
        new_profile._flags['ANALYTIC_DESIGNS']['RANK_FUNC'] = \
            'analytic_designs_rank_func' in request.form

        schedule_item = ScheduleItem(run_name=request.form['run_name'],
                                     query_profile=new_profile,
                                     time_limit_sec=int(
                                         request.form['time_limit']),
                                     git_command=request.form['git_command'],
                                     parent_job='')
    else:
        # Run based on previous run
        schedule_item = ScheduleItem(
            run_name=request.form['run_name'],
            query_profile=DefaultProfile(),
            time_limit_sec=24 * 3600,  # Default time limit is 24 hours
            git_command=request.form['git_command'],
            parent_job=request.form['report_id'])

    schedule_item.save_pickle()

    return 'success'
Example #4
0
def start_run():
  '''Method that receives POST requests and gernerates a schedule item.'''

  if request.method != 'POST': return 'fail'

  if 'time_limit' in request.form:
    # This is a custom run because time_limit item is present only in the custom_run form.
    # Values will be extracted from the form and a new profile will be generated.

    new_profile = DefaultProfile()

    # Bounds
    new_profile._bounds['MAX_NESTED_QUERY_COUNT'] = (
        int(request.form['max_nested_query_count_from']),
        int(request.form['max_nested_query_count_to']))
    new_profile._bounds['MAX_NESTED_EXPR_COUNT'] = (
        int(request.form['max_nested_expr_count_from']),
        int(request.form['max_nested_expr_count_to']))
    new_profile._bounds['SELECT_ITEM_COUNT'] = (
        int(request.form['select_item_count_from']),
        int(request.form['select_item_count_to']))
    new_profile._bounds['WITH_TABLE_COUNT'] = (
        int(request.form['with_table_count_from']),
        int(request.form['with_table_count_to']))
    new_profile._bounds['TABLE_COUNT'] = (
        int(request.form['table_count_from']),
        int(request.form['table_count_to']))
    new_profile._bounds['ANALYTIC_LEAD_LAG_OFFSET'] = (
        int(request.form['analytic_lead_lag_offset_from']),
        int(request.form['analytic_lead_lag_offset_to']))
    new_profile._bounds['ANALYTIC_WINDOW_OFFSET'] = (
        int(request.form['analytic_window_offset_from']),
        int(request.form['analytic_window_offset_to']))

    # Select Item Category
    new_profile._weights['SELECT_ITEM_CATEGORY']['AGG'] = int(
        request.form['select_agg'])
    new_profile._weights['SELECT_ITEM_CATEGORY']['ANALYTIC'] = int(
        request.form['select_analytic'])
    new_profile._weights['SELECT_ITEM_CATEGORY']['BASIC'] = int(
        request.form['select_basic'])

    # Types
    new_profile._weights['TYPES'][Boolean] = int(request.form['types_boolean'])
    new_profile._weights['TYPES'][Char] = int(request.form['types_char'])
    new_profile._weights['TYPES'][Decimal] = int(request.form['types_decimal'])
    new_profile._weights['TYPES'][Float] = int(request.form['types_float'])
    new_profile._weights['TYPES'][Int] = int(request.form['types_int'])
    new_profile._weights['TYPES'][Timestamp] = int(request.form['types_timestamp'])

    # Join
    new_profile._weights['JOIN']['INNER'] = int(request.form['join_inner'])
    new_profile._weights['JOIN']['LEFT'] = int(request.form['join_left'])
    new_profile._weights['JOIN']['RIGHT'] = int(request.form['join_right'])
    new_profile._weights['JOIN']['FULL_OUTER'] = int(request.form['join_full_outer'])
    new_profile._weights['JOIN']['CROSS'] = int(request.form['join_cross'])

    # Optional Query Clauses Probabilities
    new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['WITH'] = float(
        request.form['optional_with'])
    new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['FROM'] = float(
        request.form['optional_from'])
    new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['WHERE'] = float(
        request.form['optional_where'])
    new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['GROUP_BY'] = float(
        request.form['optional_group_by'])
    new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['HAVING'] = float(
        request.form['optional_having'])
    new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['UNION'] = float(
        request.form['optional_union'])
    new_profile._probabilities['OPTIONAL_QUERY_CLAUSES']['ORDER_BY'] = float(
        request.form['optional_order_by'])

    # Optional Analytic Clauses Probabilities
    new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES']['PARTITION_BY'] = float(
        request.form['optional_analytic_partition_by'])
    new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES']['ORDER_BY'] = float(
        request.form['optional_analytic_order_by'])
    new_profile._probabilities['OPTIONAL_ANALYTIC_CLAUSES']['WINDOW'] = float(
        request.form['optional_analytic_window'])

    # Misc Probabilities
    new_profile._probabilities['MISC']['INLINE_VIEW'] = float(
        request.form['misc_inline_view'])
    new_profile._probabilities['MISC']['SELECT_DISTINCT'] = float(
        request.form['misc_select_distinct'])
    new_profile._probabilities['MISC']['SCALAR_SUBQUERY'] = float(
        request.form['misc_scalar_subquery'])
    new_profile._probabilities['MISC']['UNION_ALL'] = float(
        request.form['misc_union_all'])

    # Analytic Designs
    new_profile._flags['ANALYTIC_DESIGNS']['TOP_LEVEL_QUERY_WITHOUT_LIMIT'] = \
        'analytic_designs_top_level_no_limit' in request.form
    new_profile._flags['ANALYTIC_DESIGNS']['DETERMINISTIC_ORDER_BY'] = \
        'analytic_designs_deterministic_order_by' in request.form
    new_profile._flags['ANALYTIC_DESIGNS']['NO_ORDER_BY'] = \
        'analytic_designs_no_order_by' in request.form
    new_profile._flags['ANALYTIC_DESIGNS']['ONLY_SELECT_ITEM'] = \
        'analytic_designs_only_select_item' in request.form
    new_profile._flags['ANALYTIC_DESIGNS']['UNBOUNDED_WINDOW'] = \
        'analytic_designs_unbounded_window' in request.form
    new_profile._flags['ANALYTIC_DESIGNS']['RANK_FUNC'] = \
        'analytic_designs_rank_func' in request.form

    schedule_item = ScheduleItem(
        run_name = request.form['run_name'],
        query_profile = new_profile,
        time_limit_sec = int(request.form['time_limit']),
        git_command = request.form['git_command'],
        parent_job = '')
  else:
    # Run based on previous run
    schedule_item = ScheduleItem(
        run_name = request.form['run_name'],
        query_profile = DefaultProfile(),
        time_limit_sec = 24 * 3600, # Default time limit is 24 hours
        git_command = request.form['git_command'],
        parent_job = request.form['report_id'])

  schedule_item.save_pickle()

  return 'success'