def _install(args, use_cache, debug): """Install datasets for retriever.""" engine = choose_engine(args) engine.use_cache = use_cache script_list = SCRIPT_LIST() if not (script_list or os.listdir(SCRIPT_WRITE_PATH)): check_for_updates() script_list = SCRIPT_LIST() data_sets_scripts = name_matches(script_list, args['dataset']) if data_sets_scripts: for data_sets_script in data_sets_scripts: try: engine.script_table_registry = OrderedDict() data_sets_script.download(engine, debug=debug) data_sets_script.engine.final_cleanup() except Exception as e: print(e) if debug: raise else: message = "Run retriever.datasets()to list the currently available " \ "datasets." raise ValueError(message) return engine
def _install(args, use_cache, debug): """Install datasets for retriever.""" engine = choose_engine(args) engine.use_cache = use_cache script_list = SCRIPT_LIST() if not script_list or not os.listdir(SCRIPT_WRITE_PATH): check_for_updates() script_list = SCRIPT_LIST(force_compile=False) data_sets_scripts = name_matches(script_list, args['dataset']) if data_sets_scripts: for data_sets_script in data_sets_scripts: print("=> Installing", data_sets_script.name) try: data_sets_script.download(engine, debug=debug) data_sets_script.engine.final_cleanup() except Exception as e: print(e) if debug: raise else: message = "The dataset \"{}\" isn't available in the Retriever. " \ "Run retriever.datasets()to list the currently available " \ "datasets".format(args['dataset']) raise ValueError(message)
def download(dataset, path='./', quiet=False, sub_dir='', debug=False, use_cache=True): """Download scripts for retriever.""" args = { 'dataset': dataset, 'command': 'download', 'path': path, 'sub_dir': sub_dir, 'quiet': quiet } engine = choose_engine(args) engine.use_cache = use_cache script_list = SCRIPT_LIST() if not script_list or not os.listdir(SCRIPT_WRITE_PATH): check_for_updates() script_list = SCRIPT_LIST() scripts = name_matches(script_list, args['dataset']) if scripts: for script in scripts: print("=> Downloading", script.name) try: script.download(engine, debug=debug) script.engine.final_cleanup() except Exception as e: print(e) if debug: raise else: message = "Run retriever.datasets() to see the list of currently " \ "available datasets." raise ValueError(message) return engine
def download(dataset, path='./', quiet=False, subdir=False, debug=False): """Download scripts for retriever.""" args = { 'dataset': dataset, 'command': 'download', 'path': path, 'subdir': subdir, 'quiet': quiet } engine = choose_engine(args) script_list = SCRIPT_LIST() if not script_list or not os.listdir(SCRIPT_WRITE_PATH): check_for_updates() script_list = SCRIPT_LIST() scripts = name_matches(script_list, args['dataset']) if scripts: for script in scripts: print("=> Downloading", script.name) try: script.download(engine, debug=debug) script.engine.final_cleanup() except Exception as e: print(e) if debug: raise else: message = "Run retriever.datasets() to see a list of currently " \ "available datasets." raise ValueError(message)
def main(): """This function launches the Data Retriever.""" if len(sys.argv) == 1: # if no command line args are passed, show the help options parser.parse_args(['-h']) else: # otherwise, parse them args = parser.parse_args() if args.command not in ['reset', 'update'] \ and not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) \ and not [f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1]) if os.path.exists(SCRIPT_SEARCH_PATHS[-1])]: check_for_updates() reload_scripts() script_list = SCRIPT_LIST() if args.command == "install" and not args.engine: parser.parse_args(['install', '-h']) if args.quiet: sys.stdout = open(os.devnull, 'w') if args.command == 'help': parser.parse_args(['-h']) if hasattr(args, 'compile') and args.compile: script_list = reload_scripts() if args.command == 'defaults': for engine_item in engine_list: print("Default options for engine ", engine_item.name) for default_opts in engine_item.required_opts: print(default_opts[0], " ", default_opts[2]) print() return if args.command == 'update': check_for_updates() reload_scripts() return elif args.command == 'citation': if args.dataset is None: print("\nCitation for retriever:\n") print(CITATION) else: scripts = name_matches(script_list, args.dataset) for dataset in scripts: print("\nDataset: {}".format(dataset.name)) print("Citation: {}".format(dataset.citation)) print("Description: {}\n".format(dataset.description)) return elif args.command == 'license': if args.dataset is None: print(LICENSE) else: dataset_license = license(args.dataset) if dataset_license: print(dataset_license) else: print("There is no license information for {}".format( args.dataset)) return elif args.command == 'new': f = open(args.filename, 'w') f.write(sample_script) f.close() return elif args.command == 'reset': reset_retriever(args.scope) return elif args.command == 'new_json': # create new JSON script create_json() return elif args.command == 'edit_json': # edit existing JSON script json_file = get_script_filename(args.dataset.lower()) edit_json(json_file) return elif args.command == 'autocreate': if sum([args.f, args.d]) == 1: file_flag = True if args.f else False create_package(args.path, args.dt, file_flag, args.o, args.skip_lines) else: print('Please use one and only one of the flags -f -d') return elif args.command == 'delete_json': # delete existing JSON script from home directory and or script directory if exists in current dir confirm = input("Really remove " + args.dataset.lower() + " and all its contents? (y/N): ") if confirm.lower().strip() in ['y', 'yes']: json_file = get_script_filename(args.dataset.lower()) delete_json(json_file) return if args.command == 'ls': # scripts should never be empty because check_for_updates is run on SCRIPT_LIST init if not (args.l or args.k or isinstance(args.v, list)): all_scripts = dataset_names() print("Available datasets : {}\n".format(len(all_scripts))) from retriever import lscolumns lscolumns.printls(all_scripts) elif isinstance(args.v, list): if args.v: try: all_scripts = [ get_script(dataset) for dataset in args.v ] except KeyError: all_scripts = [] print("Dataset(s) is not found.") else: all_scripts = datasets() count = 1 for script in all_scripts: print("{count}. {title}\n {name}\n" "{keywords}\n{description}\n" "{licenses}\n{citation}\n" "".format( count=count, title=script.title, name=script.name, keywords=script.keywords, description=script.description, licenses=str(script.licenses[0]['name']), citation=script.citation, )) count += 1 else: param_licenses = args.l if args.l else None keywords = args.k if args.k else None # search searched_scripts = datasets(keywords, param_licenses) if not searched_scripts: print("No available datasets found") else: print("Available datasets : {}\n".format( len(searched_scripts))) count = 1 for script in searched_scripts: print("{count}. {title}\n{name}\n" "{keywords}\n{licenses}\n".format( count=count, title=script.title, name=script.name, keywords=script.keywords, licenses=str(script.licenses[0]['name']), )) count += 1 return engine = choose_engine(args.__dict__) if hasattr(args, 'debug') and args.debug: debug = True else: debug = False sys.tracebacklimit = 0 if hasattr(args, 'debug') and args.not_cached: engine.use_cache = False else: engine.use_cache = True if args.dataset is not None: scripts = name_matches(script_list, args.dataset) else: raise Exception("no dataset specified.") if scripts: for dataset in scripts: print("=> Installing", dataset.name) try: dataset.download(engine, debug=debug) dataset.engine.final_cleanup() except KeyboardInterrupt: pass except Exception as e: print(e) if debug: raise print("Done!") else: print( "Run 'retriever ls' to see a list of currently available datasets." )
def main(): """This function launches the Data Retriever.""" if len(sys.argv) == 1: # if no command line args are passed, show the help options parser.parse_args(['-h']) else: # otherwise, parse them args = parser.parse_args() if args.command not in ['reset', 'update'] \ and not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) \ and not [f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1]) if os.path.exists(SCRIPT_SEARCH_PATHS[-1])]: check_for_updates() reload_scripts() script_list = SCRIPT_LIST() if args.command == "install" and not args.engine: parser.parse_args(['install', '-h']) if args.quiet: sys.stdout = open(os.devnull, 'w') if args.command == 'help': parser.parse_args(['-h']) if hasattr(args, 'compile') and args.compile: script_list = reload_scripts() if args.command == 'defaults': for engine_item in engine_list: print("Default options for engine ", engine_item.name) for default_opts in engine_item.required_opts: print(default_opts[0], " ", default_opts[2]) print() return if args.command == 'update': check_for_updates() reload_scripts() return elif args.command == 'citation': if args.dataset is None: print("\nCitation for retriever:\n") print(CITATION) else: scripts = name_matches(script_list, args.dataset) for dataset in scripts: print("\nDataset: {}".format(dataset.name)) print("Citation: {}".format(dataset.citation)) print("Description: {}\n".format(dataset.description)) return elif args.command == 'license': if args.dataset is None: print(LICENSE) else: dataset_license = license(args.dataset) if dataset_license: print(dataset_license) else: print("There is no license information for {}".format(args.dataset)) return elif args.command == 'new': f = open(args.filename, 'w') f.write(sample_script) f.close() return elif args.command == 'reset': reset_retriever(args.scope) return elif args.command == 'new_json': # create new JSON script create_json() return elif args.command == 'edit_json': # edit existing JSON script json_file = get_script_filename(args.dataset.lower()) edit_json(json_file) return elif args.command == 'autocreate': if sum([args.f, args.d]) == 1: file_flag = True if args.f else False create_package(args.path, args.dt, file_flag, args.o, args.skip_lines) else: print('Please use one and only one of the flags -f -d') return elif args.command == 'delete_json': # delete existing JSON script from home directory and or script directory if exists in current dir confirm = input("Really remove " + args.dataset.lower() + " and all its contents? (y/N): ") if confirm.lower().strip() in ['y', 'yes']: json_file = get_script_filename(args.dataset.lower()) delete_json(json_file) return if args.command == 'ls': # scripts should never be empty because check_for_updates is run on SCRIPT_LIST init if not (args.l or args.k or isinstance(args.v, list)): all_scripts = dataset_names() print("Available datasets : {}\n".format(len(all_scripts))) from retriever import lscolumns lscolumns.printls(all_scripts) elif isinstance(args.v, list): if args.v: try: all_scripts = [get_script(dataset) for dataset in args.v] except KeyError: all_scripts = [] print("Dataset(s) is not found.") else: all_scripts = datasets() count = 1 for script in all_scripts: print( "{count}. {title}\n {name}\n" "{keywords}\n{description}\n" "{licenses}\n{citation}\n" "".format( count=count, title=script.title, name=script.name, keywords=script.keywords, description=script.description, licenses=str(script.licenses[0]['name']), citation=script.citation, ) ) count += 1 else: param_licenses = args.l if args.l else None keywords = args.k if args.k else None # search searched_scripts = datasets(keywords, param_licenses) if not searched_scripts: print("No available datasets found") else: print("Available datasets : {}\n".format(len(searched_scripts))) count = 1 for script in searched_scripts: print( "{count}. {title}\n{name}\n" "{keywords}\n{licenses}\n".format( count=count, title=script.title, name=script.name, keywords=script.keywords, licenses=str(script.licenses[0]['name']), ) ) count += 1 return engine = choose_engine(args.__dict__) if hasattr(args, 'debug') and args.debug: debug = True else: debug = False sys.tracebacklimit = 0 if hasattr(args, 'debug') and args.not_cached: engine.use_cache = False else: engine.use_cache = True if args.dataset is not None: scripts = name_matches(script_list, args.dataset) else: raise Exception("no dataset specified.") if scripts: for dataset in scripts: print("=> Installing", dataset.name) try: dataset.download(engine, debug=debug) dataset.engine.final_cleanup() except KeyboardInterrupt: pass except Exception as e: print(e) if debug: raise print("Done!") else: print("Run 'retriever ls' to see a list of currently available datasets.")
def main(): """This function launches the Data Retriever.""" sys.argv[1:] = [arg.lower() for arg in sys.argv[1:]] if len(sys.argv) == 1: # if no command line args are passed, show the help options parser.parse_args(['-h']) else: # otherwise, parse them if not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) and not \ [f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1]) if os.path.exists(SCRIPT_SEARCH_PATHS[-1])]: check_for_updates() script_list = SCRIPT_LIST() args = parser.parse_args() if args.command == "install" and not args.engine: parser.parse_args(['install', '-h']) if args.quiet: sys.stdout = open(os.devnull, 'w') if args.command == 'help': parser.parse_args(['-h']) if hasattr(args, 'compile') and args.compile: script_list = SCRIPT_LIST(force_compile=True) if args.command == 'defaults': for engine_item in engine_list: print("Default options for engine ", engine_item.name) for default_opts in engine_item.required_opts: print(default_opts[0], " ", default_opts[2]) print() return if args.command == 'update': check_for_updates(False) script_list = SCRIPT_LIST() return elif args.command == 'citation': if args.dataset is None: print("\nCitation for retriever:\n") print(CITATION) else: scripts = name_matches(script_list, args.dataset) for dataset in scripts: print("\nDataset: {}".format(dataset.name)) print("Citation: {}".format(dataset.citation)) print("Description: {}\n".format(dataset.description)) return elif args.command == 'license': dataset_license = license(args.dataset) if dataset_license: print(dataset_license) else: print("There is no license information for {}".format(args.dataset)) return elif args.command == 'new': f = open(args.filename, 'w') f.write(sample_script) f.close() return elif args.command == 'reset': reset_retriever(args.scope) return elif args.command == 'new_json': # create new JSON script create_json() return elif args.command == 'edit_json': # edit existing JSON script json_file = get_script_filename(args.dataset.lower()) edit_json(json_file) return elif args.command == 'delete_json': # delete existing JSON script from home directory and or script directory if exists in current dir confirm = input("Really remove " + args.dataset.lower() + " and all its contents? (y/N): ") if confirm.lower().strip() in ['y', 'yes']: json_file = get_script_filename(args.dataset.lower()) delete_json(json_file) return if args.command == 'ls': # If scripts have never been downloaded there is nothing to list if not script_list: print("No scripts are currently available. Updating scripts now...") check_for_updates(False) print("\n\nScripts downloaded.\n") if args.l is None: all_scripts = datasets() print("Available datasets : {}\n".format(len(all_scripts))) from retriever import lscolumns lscolumns.printls(dataset_names()) else: all_scripts = datasets(args.l[0]) print("Available datasets : {}\n".format(len(all_scripts))) count = 1 for script in all_scripts: print("{}. {}".format(count, script.title)) print(script.name) print(script.keywords) print() count += 1 return engine = choose_engine(args.__dict__) if hasattr(args, 'debug') and args.debug: debug = True else: debug = False sys.tracebacklimit = 0 if hasattr(args, 'debug') and args.not_cached: engine.use_cache = False else: engine.use_cache = True if args.dataset is not None: scripts = name_matches(script_list, args.dataset) else: raise Exception("no dataset specified.") if scripts: for dataset in scripts: print("=> Installing", dataset.name) try: dataset.download(engine, debug=debug) dataset.engine.final_cleanup() except KeyboardInterrupt: pass except Exception as e: print(e) if debug: raise print("Done!") else: print("The dataset {} isn't currently available in the Retriever".format( args.dataset)) print("Run 'retriever ls to see a list of currently available datasets")