def do_sim(config, scripts): logger.info("###################### Running pipeline config '%s' "\ "######################", config['name']) logger.debug("Producing directory '%s'", config['name']) os.mkdir(config['name']) os.chdir(config['name']) logger.debug("CWD: %s", os.getcwd()) run_pipeline(config['parameters'], config['options'], scripts=scripts) os.chdir('..')
def run_test_changing_bitrate(video, parts, bitrate): file_to_transcode = video task_def = utils.create_bitrate_change_params(file_to_transcode, parts, bitrate) tests_dir = utils.build_test_directory_path(file_to_transcode, "change-bitrate") pipeline.run_pipeline(task_def, tests_dir, utils.DOCKER_IMAGE) # This intentionally won't happen if tests fails. User can check content of test directory. pipeline.clean_step(tests_dir)
def change_codec_test(video, target_codec, parts): file_to_transcode = video task_def = utils.create_codec_change_params(file_to_transcode, target_codec, parts) tests_dir = utils.build_test_directory_path(file_to_transcode, "change-codec/" + target_codec) pipeline.run_pipeline(task_def, tests_dir, utils.DOCKER_IMAGE) # This intentionally won't happen if tests fails. User can check content of test directory. pipeline.clean_step(tests_dir)
def main(): args = parse_command_line_args() setup_logging(args.verbose) citations = run_pipeline(args.input, args.keyword_file, find_authors=args.find_authors, resolve_repetitions=args.resolve_repetitions) save_citations(citations, args.output) create_export_bundle(args.output, args.zip_output)
def kfold_run_pipeline(model_module, model_params, X, y, nb_features, nb_classes, k, perm_indices, interpret_model, out_directory, id_string): losses, accs, runtimes = [], [], [] list_contrib_sums_D, list_contrib_sums_D2, list_contrib_sums = \ [], [], [] print('Starting trials ... k = {}'.format(k)) print() pairs = None for k_idx in range(k): print('Partition k = {}'.format(k_idx)) print() data_partition_dict = emr.get_k_fold_partition( X, y, k_idx=k_idx, k=k, perm_indices=perm_indices) sub_out_directory = '{}/{}_idx_partition'.format(out_directory, k_idx) if not os.path.exists(sub_out_directory): os.makedirs(sub_out_directory) (loss, acc, runtime), (sums_D, sums_D2, sums_contribs, curr_pairs) = \ run_pipeline(model_module, model_params[k_idx], data_partition_dict, nb_features=nb_features, nb_classes=nb_classes, interpret_model=interpret_model, out_directory=sub_out_directory) losses.append(loss) accs.append(acc) runtimes.append(runtime) list_contrib_sums_D.append(sums_D) list_contrib_sums_D2.append(sums_D2) list_contrib_sums.append(sums_contribs) if pairs is None: pairs = curr_pairs else: assert pairs == curr_pairs print('Finished with partition k = {}'.format(k_idx)) print('=' * 72) print() return (losses, accs, runtimes), (list_contrib_sums_D, list_contrib_sums_D2, list_contrib_sums), pairs
def kfold_run_pipeline(model_module, model_params, X, y, nb_features, nb_classes, k, perm_indices, interpret_model, out_directory, id_string): losses, accs, runtimes = [], [], [] list_contrib_sums_D, list_contrib_sums_D2, list_contrib_sums = \ [], [], [] print('Starting trials ... k = {}'.format(k)) print() pairs = None for k_idx in range(k): print('Partition k = {}'.format(k_idx)) print() data_partition_dict = emr.get_k_fold_partition(X, y, k_idx=k_idx, k=k, perm_indices=perm_indices) sub_out_directory = '{}/{}_idx_partition'.format(out_directory, k_idx) if not os.path.exists(sub_out_directory): os.makedirs(sub_out_directory) (loss, acc, runtime), (sums_D, sums_D2, sums_contribs, curr_pairs) = \ run_pipeline(model_module, model_params[k_idx], data_partition_dict, nb_features=nb_features, nb_classes=nb_classes, interpret_model=interpret_model, out_directory=sub_out_directory) losses.append(loss) accs.append(acc) runtimes.append(runtime) list_contrib_sums_D.append(sums_D) list_contrib_sums_D2.append(sums_D2) list_contrib_sums.append(sums_contribs) if pairs is None: pairs = curr_pairs else: assert pairs == curr_pairs print('Finished with partition k = {}'.format(k_idx)) print('=' * 72) print() return (losses, accs, runtimes), (list_contrib_sums_D, list_contrib_sums_D2, list_contrib_sums), pairs
def run(self): pipeline.run_pipeline(self.options, self.in_seq_files, False, self.trace_stream) print self.options.gene_name, "processing completed" self.trace_stream.close()
for index, row in tqdm(user_data.iterrows(), desc="Setting other phases"): if data.loc[index, 'phase'] == "": # Set phase to indicate the day of training data.loc[index, 'phase'] = self.get_experiment_day(row.DateTime, first_day, id_=id_) # # check for ambiguity on april 5th # if first_day in ['05', 5]: # if data.loc[index, 'phase'] == "day5": # data.loc[index, 'phase'] = key original_max_columns = pd.options.display.max_columns original_max_colwidth = pd.options.display.max_colwidth pd.options.display.max_columns = 30 # pd.options.display.max_rows = 400 pd.options.display.max_colwidth = 300 pd.options.display.width = 120 # print(data.loc[data.UserId == user]) pd.options.display.max_columns = original_max_columns pd.options.display.max_colwidth = original_max_colwidth return data if __name__ == "__main__": import pipeline pipeline.run_pipeline(False)
def run(self): pipeline.run_pipeline(self.options, self.in_seq_files, False, self.trace_stream) print self.options.gene_name, "processing completed" self.trace_stream.close()