def main(): args = get_args() helper.print_script_args_and_info(args) datasets = dataset_helper.get_dataset_names_with_concept_map( limit_datasets=args.limit_dataset) graph_files = [] for dataset in datasets: label_lookup_files = glob('{}/{}.*.label-lookup.npy'.format( args.lookup_path, dataset)) for label_lookup_file in label_lookup_files: if not os.path.exists(label_lookup_file): print('No lookup file for dataset found: {}'.format(dataset)) continue graph_files += [ (cache_file, label_lookup_file) for cache_file in dataset_helper.get_all_cached_graph_datasets( dataset_name=dataset) ] print('# Num tasks: {}'.format(len(graph_files))) Parallel(n_jobs=args.n_jobs)( delayed(process_dataset)(cache_file, label_lookup_file, args) for cache_file, label_lookup_file in graph_files) LOGGER.info('Finished')
def main(): args = get_args() start_time = time() helper.print_script_args_and_info(args) Parallel(n_jobs=args.n_jobs)( delayed(process_dataset)(dataset_name, args) for dataset_name in dataset_helper.get_dataset_names_with_concept_map( limit_datasets=args.limit_dataset)) print('Finished (time={})'.format( time_utils.seconds_to_human_readable(time() - start_time)))
def main(): args = get_args() helper.print_script_args_and_info(args) os.makedirs(args.embeddings_result_folder, exist_ok=True) LOGGER.info('Loading pre-trained embedding') LOGGER.info('Starting to process datasets') Parallel(n_jobs=args.n_jobs)( delayed(process_dataset)(dataset_name, args) for dataset_name in dataset_helper.get_dataset_names_with_concept_map( limit_datasets=args.limit_dataset)) LOGGER.info('Finished')
def main(): args = get_args() helper.print_script_args_and_info(args) limited_datasets = args.limit_dataset os.makedirs(args.embedding_save_path, exist_ok=True) datasets = dataset_helper.get_dataset_names_with_concept_map( limit_datasets=limited_datasets) Parallel(n_jobs=args.n_jobs)(delayed(process_dataset)(dataset_name, args) for dataset_name in datasets) LOGGER.info('Finished')
def main(): args = get_args() helper.print_script_args_and_info(args) for graph_folder in glob(args.graphs_folder + '/*'): if graph_folder.rsplit('/', 1)[1].startswith('_'): continue if not os.path.isdir(graph_folder): continue print('Processing: {}'.format(graph_folder)) graph_dataset_name = graph_folder.split('/')[-1] try: X, Y = dataset_helper.get_gml_graph_dataset( dataset_name=graph_dataset_name, graphs_folder=args.graphs_folder, use_cached=args.use_cached, suffix=args.suffix ) except Exception as e: traceback.print_exc() print('Error:', e)
def main(): args = get_args() helper.print_script_args_and_info(args) if args.experiment_config: experiment_config = experiment_helper.get_experiment_config( args.experiment_config) else: experiment_config = {} create_results_dir(args) classification_options: ClassificationOptions = ClassificationOptions.from_argparse_options( args) tasks: typing.List[ExperimentTask] = experiments.get_all_tasks() start_tasks(args, tasks, classification_options, experiment_config)
def main(): args = get_args() helper.print_script_args_and_info(args) Parallel(n_jobs=args.n_jobs)( delayed(process_graph_cache_file)(graph_cache_file, args) for graph_cache_file in dataset_helper.get_all_cached_graph_datasets())