def calculateOnFlowForFilters(pairs, filters, tasks, report_file_name, threads, examples_limit=20, overwrite=True, log_level=2): pairs.dumpFiltersIndexes(filters, overwrite=overwrite) report = Report(report_file_name, overwrite=overwrite) result = report.get() pairs.setReturnNames(True) compiled_filters = list(filter(lambda f: len(pairs.filterBy(f)) > 0, compileFilters(filters))) if log_level >= 1: base_progress_bar_desc = 'Doing tasks' progress_bar = tqdm(total = len(compiled_filters) * len(tasks), desc = base_progress_bar_desc) for f in compiled_filters: pairs.filterBy(f) filter_as_string = json.dumps(f) if not filter_as_string in result: result[filter_as_string] = {} for task in tasks: if log_level >= 1: progress_bar.set_description(base_progress_bar_desc + ' ("' + task['name'] + '" for filter "' + filter_as_string + '")') if (task['name'] in result[filter_as_string]) and not ('overwrite' in task): progress_bar.set_description(base_progress_bar_desc) progress_bar.update(1) continue pairs.shuffle() initial_metric_values = getPairsMetricsWithPairs(iter(pairs), task['initial_metric'], threads=threads, log=(log_level > 1)) initial_metric_values_batches = task['batcher'](initial_metric_values, batch_size = task['batch_size']) if 'values_to_load' in task: can_not_load_values_names = [] fake_globals = globals() for v in task['values_to_load']: load_result = loadOnFlowMetric(v, result, report_file_name, filter_as_string) if load_result == None: can_not_load_values_names.append(v) else: fake_globals[v] = load_result if len(can_not_load_values_names) > 0: if log_level > 1: print('\n\nCan not load values', ', '.join(can_not_load_values_names), 'from filter', filter_as_string, '\n\n') progress_bar.update(1) continue else: # print('\n\nloaded metrics', fake_globals, '\n\n') task['preprocessing'] = map(lambda f: FunctionType(f.__code__, fake_globals), task['preprocessing']) preprocessed_batches = map(\ lambda batch:\ (reduce(lambda value, f: f(value), task['preprocessing'], batch[0]), batch[1]),\ initial_metric_values_batches\ ) metrics_on_flow = calculateMetricsOnFlow(preprocessed_batches, copy.deepcopy(task['on_flow_metrics']), examples_limit, log = (log_level > 1)) result[filter_as_string][task['name']] = metrics_on_flow report.write(filter_as_string, result[filter_as_string], dump_now=False) if log_level >= 1: progress_bar.set_description(base_progress_bar_desc) progress_bar.update(1) report.dump() progress_bar.close() pairs.setReturnNames(False)
def loop(): global CURRENT_DAY global CURRENT_MON global CURRENT_HOUR global CROP global SYSTEM_CONTROLLER global SOIL_HUMIDITY global CURRENT_TEMPERATURE global IS_RAINING global CURRENT_ILLUMINATION global READINGS_DELAY global DEBUG global DEBUG_DAY global DEBUG_MONTH global DEBUG_HOUR global SYSTEM_OVERRIDE global CURRENT_CROP_REPORT global CURRENT_SYSTEM_REPORT global UPDATE_SYSTEM while True: if read_rain(): UPDATE_SYSTEM = True if read_temperature(): UPDATE_SYSTEM = True if read_soil_humiture(): UPDATE_SYSTEM = True if read_illumination(): UPDATE_SYSTEM = True if UPDATE_SYSTEM: print_line('UPDATING SYSTEM') CURRENT_DAY = datetime.datetime.today().day CURRENT_MON = datetime.datetime.today().month CURRENT_HOUR = datetime.datetime.today().hour # Overriding date/time if debugging if DEBUG: override_debug_values() if DEBUG_DAY is not False: CURRENT_DAY = DEBUG_DAY print_line('DEBUGGING WITH DAY AS: ' + str(CURRENT_DAY)) if DEBUG_MONTH is not False: CURRENT_MON = DEBUG_MONTH print_line('DEBUGGING WITH MONTH AS: ' + str(CURRENT_MON)) if DEBUG_HOUR is not False: CURRENT_HOUR = DEBUG_HOUR print_line('DEBUGGING WITH HOUR AS: ' + str(CURRENT_HOUR)) report = Report(SOIL_HUMIDITY, CURRENT_TEMPERATURE, IS_RAINING, CURRENT_ILLUMINATION) report.build_crop_status(CROP, CURRENT_MON, CURRENT_DAY, CURRENT_HOUR) crop_report = report.get() update_status = SYSTEM_CONTROLLER.update_status(crop_report, SYSTEM_OVERRIDE) CURRENT_CROP_REPORT = build_crop_report(crop_report) CURRENT_SYSTEM_REPORT = '' for status in update_status: print_line(' .' + status) CURRENT_SYSTEM_REPORT += ' .' + status + '\n' UPDATE_SYSTEM = False time.sleep(READINGS_DELAY)