def iterate_graph_cache_datasets( self) -> typing.Generator[typing.Tuple, None, None]: for graph_dataset in dataset_helper.get_all_cached_graph_datasets(): dataset_name = filename_utils.get_dataset_from_filename( graph_dataset) with self.subTest(graph_dataset=graph_dataset, dataset_name=dataset_name): yield graph_dataset, dataset_name
def process_dataset(cache_file, label_lookup_file, args): dataset = filename_utils.get_dataset_from_filename(cache_file) cache_filename = filename_utils.get_filename_only(cache_file, with_extension=False) threshold, topn = filename_utils.get_topn_threshold_from_lookupfilename( label_lookup_file) result_file = cache_file.replace( dataset, 'relabeled_threshold_{}_topn_{}_{}'.format(threshold, topn, dataset)) if not args.force and os.path.exists(result_file): return with open(label_lookup_file, 'rb') as f: label_lookup = pickle.load(f) X, Y = dataset_helper.get_dataset_cached(cache_file) X = graph_helper.get_graphs_only(X) # Get label to be renamed node_labels = list(chain.from_iterable([x.nodes() for x in X])) unique_labels = set(node_labels) counter = collections.Counter(node_labels) node_labels_to_be_renamed = set([ label for label, occurrences in counter.items() if occurrences <= args.max_occurrence ]) lookup_ = { label: new_label for label, new_label in label_lookup.items() if label in node_labels_to_be_renamed } new_labels = set(lookup_.values()) lookup__ = collections.defaultdict(list) for label, new_label in label_lookup.items(): if new_label in new_labels: lookup__[label].append(new_label) lookup_ = dict(lookup_, **lookup__) LOGGER.info( '{:80} topn={:4} threshold={:4}\n\t\t#relabeled labels: {}\n\t\t#unique labels: {}\n\t\t#nodes: {}' .format(cache_filename, topn, threshold, len(lookup_), len(unique_labels), len(node_labels))) relabel_trans = transformers.RelabelGraphsTransformer(lookup_) X = relabel_trans.transform(X) with open(result_file, 'wb') as f: pickle.dump((X, Y), f)
def get_dataset_names_with_concept_map(limit_datasets: typing.Iterable = None): all_graph_datasets = [ filename_utils.get_dataset_from_filename(x) for x in get_all_cached_graph_datasets(graph_type=TYPE_CONCEPT_MAP) ] return [ x for x in get_all_available_dataset_names(limit_datasets=limit_datasets) if x in all_graph_datasets ]
def graph_dataset_filter(cache_file): filename = filename_utils.get_filename_only(cache_file) is_graph_dataset = filename.startswith('dataset_graph') is_not_relabeled = '_relabeled' not in filename is_not_gram_or_phi = 'gram' not in filename and 'phi' not in filename is_in_dataset = not dataset_name or dataset_name == filename_utils.get_dataset_from_filename( filename) is_right_graph_type = not graph_type or graph_type == graph_helper.get_graph_type_from_filename( cache_file) return np.all([ is_graph_dataset, is_not_relabeled, is_not_gram_or_phi, is_in_dataset, is_right_graph_type ])
def get_filtered_tasks(task_type=None, dataset=None, graph_type_filter: str = None, task_name_filter: str = None, tasks=None) -> typing.List[ExperimentTask]: task_type = _ensure_is_container(task_type) dataset = _ensure_is_container(dataset) if tasks is None: tasks = get_all_tasks() return [ t for t in tasks if (not task_type or t.type in task_type) and ( not dataset or filename_utils.get_dataset_from_filename(t.name) in dataset) and (not task_name_filter or task_name_filter in t.name) and (not graph_type_filter or graph_type_filter == graph_helper.get_graph_type_from_filename(t.name)) ]
def file_should_be_processed(file : str, include_filter : str, exclude_filter : str, limit_dataset : list): """Returns true, if file is included AND not excluded AND in the limited datasets. Args: file (str): the file to be processed include_filter (str): string that has to be in `file` (can be None) exclude_filter (str): string that must not be in `file` (can be None) dataset (str): the dataset of the file limit_dataset (list(str)): the datasets that have been limited (= are allowed) Returns: bool: Whether the file should be processed """ dataset = filename_utils.get_dataset_from_filename(file) is_in_limited_datasets = (not limit_dataset or dataset in limit_dataset) is_included = (not include_filter or include_filter in file) is_excluded = (exclude_filter and exclude_filter in file) return is_in_limited_datasets and is_included and not is_excluded
def get_combined_text_graph_dataset(graph_cache_file, use_ana=False) -> typing.Tuple[typing.List[typing.Tuple], typing.List]: dataset_name = filename_utils.get_dataset_from_filename(graph_cache_file) X_text, Y_text = dataset_helper.get_dataset(dataset_name + ('-ana' if use_ana else '')) X_graph, Y_graph = dataset_helper.get_dataset_cached(graph_cache_file) # Same length but has ID if len(X_graph) == len(X_text) and (not isinstance(X_graph[0], tuple) or not isinstance(X_graph[0][1], str)): return list(zip(X_graph, X_text, [None] * len(X_graph))), Y_graph # Get class to class ids mapping class_2_id = collections.defaultdict(lambda: []) for x, y in zip(X_text, Y_text): class_2_id[y].append(x) X_combined, Y_combined = [], Y_graph for (x_graph, y_id), y_graph in zip(X_graph, Y_graph): y_id = int(y_id) X_combined.append((x_graph, class_2_id[y_graph][y_id], y_id)) return X_combined, Y_combined
def should_process_task(task: ExperimentTask): # Dataset filter is_filtered_by_dataset = limit_dataset and filename_utils.get_dataset_from_filename( task.name) not in limit_dataset # Task type filters is_filtered_by_include_filter = (args.task_type_include_filter and task.type not in args.task_type_include_filter) is_filtered_by_exclude_filter = (args.task_type_exclude_filter and task.type in args.task_type_exclude_filter) is_filtered_by_name_filter = (args.task_name_filter and args.task_name_filter not in task.name) is_filtered_by_param_options = (filtered_task_types and task.type not in filtered_task_types) is_filtered_by_graph_type = ( limit_graph_type and graph_helper.get_graph_type_from_filename( task.name) not in [None] + limit_graph_type) # Do not process tasks that have already been calculated (unless args.force == True) created_files = [ '{}/{}'.format( args.results_folder, filename_utils.get_result_filename_for_task( task, experiment_config, cfo=classification_options)) ] is_filtered_by_file_exists = (not args.force and np.any( [os.path.exists(file) for file in created_files])) should_process = not np.any([ is_filtered_by_graph_type, is_filtered_by_dataset, is_filtered_by_include_filter, is_filtered_by_name_filter, is_filtered_by_file_exists, is_filtered_by_exclude_filter, is_filtered_by_param_options ]) return should_process
def get_task_graphs(graph_cache_file: str) -> ExperimentTask: def process() -> tuple: X, Y = dataset_helper.get_dataset_cached(graph_cache_file) X = graph_helper.get_graphs_only(X) estimator, params = task_helper.get_graph_estimator_and_params(X, Y) return ClassificationData(X, Y, estimator, params) tasks = list() tasks.append(ExperimentTask('graph', get_filename_only(graph_cache_file), process)) dataset = get_dataset_from_filename(graph_cache_file) def process_relabeled(): X, Y = dataset_helper.get_dataset_cached(graph_cache_file) X = graph_helper.get_graphs_only(X) estimator, params = task_helper.get_graph_estimator_and_params(X, Y) params['graph_preprocessing'] = [transformers.RelabelGraphsTransformer()] params['graph_preprocessing__dataset'] = [dataset] params['graph_preprocessing__threshold'] = [0.99] params['graph_preprocessing__topn'] = [10] return ClassificationData(X, Y, estimator, params) tasks.append(ExperimentTask('graph_relabeled', get_filename_only(graph_cache_file), process_relabeled)) return tasks
def get_tasks() -> typing.List[ExperimentTask]: graph_cache_files = dataset_helper.get_all_cached_graph_datasets() gram_cache_files = dataset_helper.get_all_gram_datasets() datasets = dataset_helper.get_all_available_dataset_names() cmap_cache_files = dataset_helper.get_all_cached_graph_datasets(graph_type=constants.TYPE_CONCEPT_MAP) cmap_datasets = [get_dataset_from_filename(x) for x in cmap_cache_files] datasets = [x for x in datasets if x in cmap_datasets] graph_task_fns = [ get_task_graph_content_only, get_task_graph_structure_only, get_task_combined, get_task_graphs, get_task_graph_node_weights ] tasks = [] tasks += task_helper.get_tasks(graph_task_fns, graph_cache_files) tasks += task_helper.get_tasks([get_task_dummy, get_task_text], datasets) tasks += task_helper.get_tasks([get_gram_task], gram_cache_files) tasks += task_helper.get_tasks([get_task_extra_graphs], graph_helper.get_all_graph_benchmark_dataset_names()) return tasks
def get_results(folder=None, results_directory=constants.RESULTS_FOLDER, log_progress=tqdm.tqdm_notebook, exclude_filter=None, include_filter=None, remove_split_cols=True, remove_rank_cols=True, remove_fit_time_cols=True, filter_out_experiment=None, ignore_experiments=True, only_load_dataset=None, fetch_predictions=False): ''' Retrieves results from result folder. Note: This function _seriously_ has to be refactored! Args: folder: specify the results folder. If not specified, defaults to the most recent results folder results_directory: the base folder log_progress: function to log the progess. Takes an iterable and yields the item exclude_filter: which files to exclude include_filter: which files to include remove_split_cols: whether to keep the individual results for each split in CV remove_rank_cols: whether to keep the rank information in the CV results remove_fit_time_cols: keep the fit time filter_out_experiment: string thats gets filtered out ignore_experiments: only_load_dataset: filter the dataset fetch_predictions: whether to also retrieve the predictions and calculate the results on them Returns: pd.DataFrame: the results ''' result_folders = get_result_folders(results_directory) folder = 'data/results/{}'.format(folder) if folder else result_folders[-1] result_files = get_result_filenames_from_folder(folder) if filter_out_experiment: result_files = [ x for x in result_files if _get_experiment_name_from_filename(x) == filter_out_experiment ] if ignore_experiments and not filter_out_experiment: result_files = [x for x in result_files if 'experiment_' not in x] if only_load_dataset is not None: result_files = [ x for x in result_files if filename_utils.get_dataset_from_filename(x) in only_load_dataset ] data_ = [] for result_file in log_progress( result_files) if log_progress else result_files: if include_filter and include_filter not in result_file: continue if exclude_filter and exclude_filter in result_file: continue if '_nested_' in result_file: LOGGER.warning( 'Encountered nested CV result file. Currently not implemented. File: {}' .format(result_file)) continue dataset_name = filename_utils.get_dataset_from_filename(result_file) with open(result_file, 'rb') as f: result_data = pickle.load(f) remove_transformer_classes(result_data) result_file = filename_utils.get_filename_only(result_file) result = result_data if 'params' in result_data else result_data[ 'results'] assert 'params' in result result = clean_result_keys(result) for idx, el in enumerate(result['params']): result['params'][idx] = clean_result_keys(el) prediction_file = '{}/predictions/{}'.format( folder, filename_utils.get_filename_only(result_file)) predictions_exist = os.path.exists(prediction_file) num_results = len(result['params']) result['prediction_file_exists'] = [predictions_exist] * num_results if fetch_predictions and not predictions_exist: LOGGER.warning( 'fetch_predictions=True but could not find prediction: {}'. format(prediction_file)) # Fetch predictions and check whether the git commits are the same. # Also, calculate the prediction scores if fetch_predictions and predictions_exist: with open(prediction_file, 'rb') as f: r = pickle.load(f) result_git_commit = result_data['meta_data']['git_commit'] git_commit = r['meta_data']['git_commit'] if not git_commit == result_git_commit: LOGGER.warning( 'Unmatching git commit for prediction/result file! Prediction: {}, Result: {}' .format(git_commit, result_git_commit)) else: prediction = r['results'] Y_real, Y_pred, X_test = prediction['Y_real'], prediction[ 'Y_pred'], prediction['X_test'] scores = calculate_scores(Y_real, Y_pred) for name, val in scores.items(): result['prediction_score_{}'.format( name)] = [val] * num_results result['prediction_file'] = [prediction_file] * num_results def is_graph_dataset(): graph_file_types = [ constants.TYPE_CONCEPT_MAP, constants.TYPE_COOCCURRENCE, 'graph_extra' ] is_graph_dataset_ = False for x in graph_file_types: if '_{}_'.format(x) in result_file: is_graph_dataset_ = True break return is_graph_dataset_ result['combined'] = np.any([ 'graph_combined__dataset_' in result_file, 'graph_text_combined__dataset_' in result_file ]) # TEXT if is_graph_dataset(): is_cooccurrence_dataset = constants.TYPE_COOCCURRENCE in result_file result[ 'type'] = constants.TYPE_COOCCURRENCE if is_cooccurrence_dataset else constants.TYPE_CONCEPT_MAP result['lemmatized'] = '_lemmatized_' in result_file result['kernel'] = get_kernel_from_filename(result_file) # Co-Occurrence if is_cooccurrence_dataset: parts = re.findall(r'cooccurrence_(.+?)_(.+?)_', result_file)[0] assert len(parts) == 2 result['window_size'], result['words'] = parts # Concept Maps else: result['words'] = 'concepts' # DUMMY elif 'dummy' in result_file: result['type'] = 'dummy' result['words'] = 'dummy' # TEXT else: result['type'] = 'text' result['words'] = ['all'] * num_results if 'time_checkpoints' in result_data: timestamps = result_data['time_checkpoints'] timestamps = sorted(timestamps.items(), key=lambda x: x[1]) start = timestamps[0][1] end = timestamps[-1][1] result['timestamps'] = [timestamps] * num_results result['time'] = [end - start] * num_results result['filename'] = result_file result['dataset'] = dataset_name # Add meta data info = {} if 'results' in result_data: info = { 'info__' + k: v for k, v in result_data.get('meta_data', result_data).items() if k != 'results' } result = dict(result, **{k: [v] * num_results for k, v in info.items()}) data_.append(result) df_all = None for d in data_: result_df = pd.DataFrame(d) df_all = result_df if df_all is None else df_all.append(result_df) if df_all is None or not len(df_all): LOGGER.warning('Did not retrieve results! Aborting') return None # Remove cols df_all = df_all[[ x for x in df_all.columns.tolist() if (not remove_split_cols or not re.match(r'^split\d', x)) and ( not remove_fit_time_cols or not re.match(r'_time$', x)) and ( not remove_rank_cols or not re.match(r'rank_', x)) ]] # Change the column order prio_columns = ['dataset', 'type', 'combined'] low_prio_columns = ['params', 'filename'] + [ c for c in df_all.columns if c.startswith('std_') or c.startswith('mean_') ] columns = df_all.columns.tolist() for c in prio_columns + low_prio_columns: columns.remove(c) return df_all.reset_index(drop=True)[prio_columns + columns + low_prio_columns]
def process_graph_cache_file(graph_cache_file, args): graph_cache_filename = graph_cache_file.split('/')[-1].rsplit('.')[0] dataset = filename_utils.get_dataset_from_filename(graph_cache_file) if '.phi.' in graph_cache_filename or not filter_utils.file_should_be_processed( graph_cache_filename, args.include_filter, args.exclude_filter, args.limit_dataset): return LOGGER.info('{:15} starting ({})'.format(dataset, graph_cache_filename)) fast_wl_trans = FastWLGraphKernelTransformer( h=args.wl_h, use_early_stopping=False, truncate_to_highest_label=False) try: phi_graph_cache_file = graph_cache_file.replace('.npy', '.phi.npy') X_graphs, Y = dataset_helper.get_dataset_cached(graph_cache_file) X_graphs = graph_helper.get_graphs_only(X_graphs) # Kernel: WL if args.use_wl: used_phi_graph_cache_file = phi_graph_cache_file splitted_phi_graph_cache_file = phi_graph_cache_file.replace( '.phi', '.splitted.phi') phi_same_label_graph_cache_file = phi_graph_cache_file.replace( dataset, '{}_same-label'.format(dataset)).replace( '.phi', '.splitted.phi') # Stop here if all files have already been created if not args.force and np.all([ os.path.exists(x) for x in [ splitted_phi_graph_cache_file, used_phi_graph_cache_file, phi_same_label_graph_cache_file ] ]): return X_, Y_ = np.array(np.copy(X_graphs)), np.array(np.copy(Y)) if args.wl_sort_classes: X_, Y_ = sort(X_, Y_, by=Y_) num_vertices = len(graph_helper.get_all_node_labels(X_)) fast_wl_trans.set_params(phi_dim=num_vertices) X_train, X_test, Y_train, Y_test = sklearn.model_selection.train_test_split( np.copy(X_), np.copy(Y_), stratify=Y_, test_size=args.wl_test_size) X_train, Y_train = sort(X_train, Y_train, by=Y_train) X_test, Y_test = sort(X_test, Y_test, by=Y_test) # Splitted version if args.force or not os.path.exists(splitted_phi_graph_cache_file): t = sklearn.base.clone(fast_wl_trans).set_params( same_label=True) phi_train = t.fit_transform(np.copy(X_train)) phi_test = t.transform(np.copy(X_test)) with open(splitted_phi_graph_cache_file, 'wb') as f: pickle.dump((phi_train, phi_test, X_train, X_test, Y_train, Y_test), f) # Splitted, same label if args.force or not os.path.exists( phi_same_label_graph_cache_file): t = sklearn.base.clone(fast_wl_trans) phi_train = t.fit_transform(X_train) phi_test = t.transform(X_test) with open(phi_same_label_graph_cache_file, 'wb') as f: pickle.dump((phi_train, phi_test, X_train, X_test, Y_train, Y_test), f) # Whole dataset if args.force or not os.path.exists(used_phi_graph_cache_file): t = sklearn.base.clone(fast_wl_trans) with open(used_phi_graph_cache_file, 'wb') as f: pickle.dump((t.fit_transform(X_), Y_), f) # Kernel: spgk if args.use_spgk: for depth in args.spgk_depth: spgk_graph_cache_file = graph_cache_file.replace( '.npy', '.spgk-{}.gram.npy'.format(depth)) if args.force or not os.path.exists(spgk_graph_cache_file): K = spgk.transform(X_graphs, depth=depth) with open(spgk_graph_cache_file, 'wb') as f: pickle.dump((K, Y), f) except Exception as e: LOGGER.exception(e) LOGGER.info('{:15} finished ({})'.format(dataset, graph_cache_filename))
def test(self): for test_item in FILENAMES: self.assertEqual( filename_utils.get_dataset_from_filename( test_item.filename, test_item.ignore_subtype), test_item.expected_dataset)
def get_all_gram_datasets(dataset_name=None, cache_path=CACHE_PATH): gram_files = glob('{}/*gram*.npy'.format(cache_path)) return [ x for x in gram_files if not dataset_name or filename_utils.get_dataset_from_filename(x) == dataset_name ]
def print_tasks(tasks: typing.List[ExperimentTask]): for task in tasks: print('\t{t.type:26} {dataset:18} {t.name}'.format( t=task, dataset=filename_utils.get_dataset_from_filename(task.name))) print('\n')
def start_tasks(args, all_tasks: typing.List[ExperimentTask], classification_options: ClassificationOptions, experiment_config: dict): filtered_task_types = experiment_config['params_per_type'].keys( ) if experiment_config else None if experiment_config.get('limit_dataset', None) is not None: limit_dataset = experiment_config['limit_dataset'] else: limit_dataset = args.limit_dataset limit_graph_type = experiment_config.get('limit_graph_type', None) def should_process_task(task: ExperimentTask): # Dataset filter is_filtered_by_dataset = limit_dataset and filename_utils.get_dataset_from_filename( task.name) not in limit_dataset # Task type filters is_filtered_by_include_filter = (args.task_type_include_filter and task.type not in args.task_type_include_filter) is_filtered_by_exclude_filter = (args.task_type_exclude_filter and task.type in args.task_type_exclude_filter) is_filtered_by_name_filter = (args.task_name_filter and args.task_name_filter not in task.name) is_filtered_by_param_options = (filtered_task_types and task.type not in filtered_task_types) is_filtered_by_graph_type = ( limit_graph_type and graph_helper.get_graph_type_from_filename( task.name) not in [None] + limit_graph_type) # Do not process tasks that have already been calculated (unless args.force == True) created_files = [ '{}/{}'.format( args.results_folder, filename_utils.get_result_filename_for_task( task, experiment_config, cfo=classification_options)) ] is_filtered_by_file_exists = (not args.force and np.any( [os.path.exists(file) for file in created_files])) should_process = not np.any([ is_filtered_by_graph_type, is_filtered_by_dataset, is_filtered_by_include_filter, is_filtered_by_name_filter, is_filtered_by_file_exists, is_filtered_by_exclude_filter, is_filtered_by_param_options ]) return should_process def print_tasks(tasks: typing.List[ExperimentTask]): for task in tasks: print('\t{t.type:26} {dataset:18} {t.name}'.format( t=task, dataset=filename_utils.get_dataset_from_filename(task.name))) print('\n') # Filter out tasks tasks = sorted( [task for task in all_tasks if should_process_task(task)], key=lambda x: filename_utils.get_dataset_from_filename(x.name)) if args.dry_run: print('All tasks:') print_tasks(all_tasks) print('Filtered tasks:') print_tasks(tasks) print('# tasks per type (filtered/unfiltered)') task_type_counter_unfiltered = collections.Counter( [t.type for t in all_tasks]) task_type_counter_filtered = collections.Counter([t.type for t in tasks]) for task_type, unfiltered_count in task_type_counter_unfiltered.items(): print('\t{:25} {:2}/{:2}'.format( task_type, task_type_counter_filtered.get(task_type, 0), unfiltered_count)) print('\n') if args.dry_run: print('Only doing a dry-run. Exiting.') return num_tasks = len(tasks) for task_idx, t in enumerate(tasks): def print_task(msg=''): LOGGER.info( 'Task {idx:>2}/{num_tasks}: {t.type:30} - {t.name:40} - {msg}'. format(idx=task_idx + 1, num_tasks=num_tasks, t=t, msg=msg)) start_time = time() print_task('Started') try: task_runner.run_classification_task(t, classification_options, experiment_config) gc.collect() except Exception as e: print_task('Error: {}'.format(e)) LOGGER.exception(e) elapsed_seconds = time() - start_time print_task('Finished (time={})'.format( time_utils.seconds_to_human_readable(elapsed_seconds))) gc.collect() LOGGER.info('Finished!')