def create_socrata_dataset(engine, name, resource, script_path=None): """Downloads raw data and creates a script for the socrata dataset""" if script_path is None: script_path = SOCRATA_SCRIPT_WRITE_PATH filename = resource["id"] + '.csv' engine.script = BasicTextTemplate(**{"name": name}) url = 'https://' + resource["domain"] + '/resource/' + filename if not engine.find_file(filename) or not engine.use_cache: path = engine.format_filename(filename) engine.create_raw_data_dir() progbar = tqdm( unit='B', unit_scale=True, unit_divisor=1024, miniters=1, desc='Downloading {}'.format(filename), ) result = engine.download_from_socrata(url, path, progbar) if not result: print("Unable to download the dataset using Socrata API") elif engine.find_file(filename): path = engine.format_filename(filename) engine.script = None if engine.opts["command"] == "download": engine.final_cleanup() return else: if not os.path.exists(script_path): os.makedirs(script_path) file = name.replace("-", "_") if str(file + ".json") in [ file_i for file_i in os.listdir(script_path) if file_i.endswith(".json") ]: print( f"Dataset {name} already exists as {file}.json in {script_path}" ) else: create_package(path, "tabular", True, script_path) print("Updating script name to {}".format(name + ".json")) print("Updating the contents of script {}".format(name)) update_socrata_script(name, filename, url, resource, script_path) reload_scripts()
def create_rdataset(engine, package, dataset_name, script_path=None): """Download files for RDatasets to the raw data directory""" if script_path is None: script_path = RDATASET_SCRIPT_WRITE_PATH update_rdataset_catalog() with open(RDATASET_PATH, 'r') as f: rdatasets = json.load(f) f.close() try: rpackage = rdatasets[package] try: data_obj = rpackage[dataset_name] except KeyError: print("Dataset '{}' not found in package '{}' RDatasets".format( dataset_name, package)) except KeyError: print("Package '{}' not found in RDatasets".format(package)) script_name = f"rdataset-{package}-{dataset_name}" filename = data_obj['csv'].split('/')[-1] engine.script = BasicTextTemplate(**{"name": script_name}) if not engine.find_file(filename) or not engine.use_cache: engine.download_file(data_obj['csv'], filename) path = engine.format_filename(filename) engine.script = None if engine.opts["command"] == "download": engine.final_cleanup() return else: create_package(path, 'tabular', True, script_path) print("Updating script name to {}".format(script_name + ".json")) print("Updating the contents of script {}".format(script_name)) update_rdataset_script(data_obj, dataset_name, package, script_path) print("Updated the script {}".format(script_name)) reload_scripts()
def to_str(object, object_encoding=encoding): return str(object).encode('UTF-8').decode(encoding) # Create the .rst file for the available datasets datasetfile = open_fw("datasets_list.rst") datasetfile_title = """================== Datasets Available ================== """ check_for_updates() reload_scripts() script_list = SCRIPT_LIST() # write the title of dataset rst file # ref:http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html datasetfile.write(datasetfile_title) # get info from the scripts using specified encoding for script_num, script in enumerate(script_list, start=1): reference_link = '' if script.ref.strip(): reference_link = script.ref elif hasattr(script, 'homepage'): reference_link = script.homepage elif not reference_link.strip(): if bool(script.urls.values()):
def compile(): print("Compiling retriever scripts...") reload_scripts() print("done.")
def main(): """This function launches the Data Retriever.""" if len(sys.argv) == 1: # if no command line args are passed, show the help options parser.parse_args(['-h']) else: # otherwise, parse them args = parser.parse_args() if args.command not in ['reset', 'update'] \ and not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) \ and not [f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1]) if os.path.exists(SCRIPT_SEARCH_PATHS[-1])]: check_for_updates() reload_scripts() script_list = SCRIPT_LIST() if args.command == "install" and not args.engine: parser.parse_args(['install', '-h']) if args.quiet: sys.stdout = open(os.devnull, 'w') if args.command == 'help': parser.parse_args(['-h']) if hasattr(args, 'compile') and args.compile: script_list = reload_scripts() if args.command == 'defaults': for engine_item in engine_list: print("Default options for engine ", engine_item.name) for default_opts in engine_item.required_opts: print(default_opts[0], " ", default_opts[2]) print() return if args.command == 'update': check_for_updates() reload_scripts() return elif args.command == 'citation': if args.dataset is None: print("\nCitation for retriever:\n") print(CITATION) else: scripts = name_matches(script_list, args.dataset) for dataset in scripts: print("\nDataset: {}".format(dataset.name)) print("Citation: {}".format(dataset.citation)) print("Description: {}\n".format(dataset.description)) return elif args.command == 'license': if args.dataset is None: print(LICENSE) else: dataset_license = license(args.dataset) if dataset_license: print(dataset_license) else: print("There is no license information for {}".format( args.dataset)) return elif args.command == 'new': f = open(args.filename, 'w') f.write(sample_script) f.close() return elif args.command == 'reset': reset_retriever(args.scope) return elif args.command == 'new_json': # create new JSON script create_json() return elif args.command == 'edit_json': # edit existing JSON script json_file = get_script_filename(args.dataset.lower()) edit_json(json_file) return elif args.command == 'autocreate': if sum([args.f, args.d]) == 1: file_flag = True if args.f else False create_package(args.path, args.dt, file_flag, args.o, args.skip_lines) else: print('Please use one and only one of the flags -f -d') return elif args.command == 'delete_json': # delete existing JSON script from home directory and or script directory if exists in current dir confirm = input("Really remove " + args.dataset.lower() + " and all its contents? (y/N): ") if confirm.lower().strip() in ['y', 'yes']: json_file = get_script_filename(args.dataset.lower()) delete_json(json_file) return if args.command == 'ls': # scripts should never be empty because check_for_updates is run on SCRIPT_LIST init if not (args.l or args.k or isinstance(args.v, list)): all_scripts = dataset_names() print("Available datasets : {}\n".format(len(all_scripts))) from retriever import lscolumns lscolumns.printls(all_scripts) elif isinstance(args.v, list): if args.v: try: all_scripts = [ get_script(dataset) for dataset in args.v ] except KeyError: all_scripts = [] print("Dataset(s) is not found.") else: all_scripts = datasets() count = 1 for script in all_scripts: print("{count}. {title}\n {name}\n" "{keywords}\n{description}\n" "{licenses}\n{citation}\n" "".format( count=count, title=script.title, name=script.name, keywords=script.keywords, description=script.description, licenses=str(script.licenses[0]['name']), citation=script.citation, )) count += 1 else: param_licenses = args.l if args.l else None keywords = args.k if args.k else None # search searched_scripts = datasets(keywords, param_licenses) if not searched_scripts: print("No available datasets found") else: print("Available datasets : {}\n".format( len(searched_scripts))) count = 1 for script in searched_scripts: print("{count}. {title}\n{name}\n" "{keywords}\n{licenses}\n".format( count=count, title=script.title, name=script.name, keywords=script.keywords, licenses=str(script.licenses[0]['name']), )) count += 1 return engine = choose_engine(args.__dict__) if hasattr(args, 'debug') and args.debug: debug = True else: debug = False sys.tracebacklimit = 0 if hasattr(args, 'debug') and args.not_cached: engine.use_cache = False else: engine.use_cache = True if args.dataset is not None: scripts = name_matches(script_list, args.dataset) else: raise Exception("no dataset specified.") if scripts: for dataset in scripts: print("=> Installing", dataset.name) try: dataset.download(engine, debug=debug) dataset.engine.final_cleanup() except KeyboardInterrupt: pass except Exception as e: print(e) if debug: raise print("Done!") else: print( "Run 'retriever ls' to see a list of currently available datasets." )
def main(): """This function launches the Data Retriever.""" if len(sys.argv) == 1: # if no command line args are passed, show the help options parser.parse_args(['-h']) else: # otherwise, parse them args = parser.parse_args() reset_or_update = args.command in ["reset", "update"] if (not reset_or_update and not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) and not [ f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1]) if os.path.exists(SCRIPT_SEARCH_PATHS[-1]) ]): check_for_updates() reload_scripts() script_list = SCRIPT_LIST() if args.command == "install" and not args.engine: parser.parse_args(["install", "-h"]) if args.quiet: sys.stdout = open(os.devnull, "w") if args.command == "help": parser.parse_args(["-h"]) if hasattr(args, "compile") and args.compile: script_list = reload_scripts() if args.command == "defaults": for engine_item in engine_list: print("Default options for engine ", engine_item.name) for default_opts in engine_item.required_opts: print(default_opts[0], " ", default_opts[2]) print() return if args.command == "update": check_for_updates() reload_scripts() return if args.command == "citation": if args.dataset is None: print("\nCitation for retriever:\n") print(CITATION) else: citations = get_script_citation(args.dataset) for citation in citations: print("Citation: {}".format(citation)) return if args.command == 'license': if args.dataset is None: print(LICENSE) else: dataset_license = license(args.dataset) if dataset_license: print(dataset_license) else: print("There is no license information for {}".format( args.dataset)) return if args.command == 'new': f = open(args.filename, 'w') f.write(sample_script) f.close() return if args.command == 'reset': reset_retriever(args.scope) return if args.command == 'autocreate': if args.c: url = args.path script_list = SCRIPT_LIST() flag = 0 for script in script_list: for dataset in script.tables: if script.tables[dataset].url == url: flag = 1 break if flag == 1: print("File already exist in dataset " + str(script.name)) else: print("Dataset is not avaliable, Please download") return if sum([args.f, args.d]) == 1: file_flag = bool(args.f) create_package(args.path, args.dt, file_flag, args.o, args.skip_lines, args.e) else: print('Please use one and only one of the flags -f -d') return if args.command == 'ls': # scripts should never be empty because check_for_updates is run on SCRIPT_LIST init if not any([args.l, args.k, args.v, args.s, args.rdataset]): all_scripts = dataset_names() from retriever import lscolumns all_scripts_combined = [] for dataset in all_scripts['offline']: all_scripts_combined.append((dataset, True)) for dataset in all_scripts['online']: if dataset in all_scripts['offline']: continue all_scripts_combined.append((dataset, False)) all_scripts_combined = sorted(all_scripts_combined, key=lambda x: x[0]) print("Available datasets : {}\n".format( len(all_scripts_combined))) lscolumns.printls(all_scripts_combined) print("\nThe symbol * denotes the online datasets.") print( "To see the full list of available online datasets, visit\n" "https://github.com/weecology/retriever-recipes.") elif isinstance(args.s, list): try: theme = INQUIRER_THEME except NameError: print("To use retriever ls -s, install inquirer") exit() name_list = socrata_autocomplete_search(args.s) print("Autocomplete suggestions : Total {} results\n".format( len(name_list))) if len(name_list): question = [ inquirer.List('dataset name', message='Select the dataset name', choices=name_list) ] answer = inquirer.prompt(question, theme=INQUIRER_THEME, raise_keyboard_interrupt=True) dataset_name = answer['dataset name'] metadata = socrata_dataset_info(dataset_name) print( "Dataset Information of {}: Total {} results\n".format( dataset_name, len(metadata))) for i in range(len(metadata)): print("{}. {}\n \tID : {}\n" "\tType : {}\n" "\tDescription : {}\n" "\tDomain : {}\n \tLink : {}\n".format( i + 1, metadata[i]["name"], metadata[i]["id"], metadata[i]["type"], metadata[i]["description"][:50] + "...", metadata[i]["domain"], metadata[i]["link"])) elif args.rdataset: if not isinstance(args.p, list) and not args.all: display_all_rdataset_names() elif not isinstance(args.p, list) and args.all: display_all_rdataset_names(package_name='all') else: display_all_rdataset_names(package_name=args.p) elif isinstance(args.v, list): dataset_verbose_list(args.v) else: param_licenses = args.l if args.l else None keywords = args.k if args.k else None # search searched_scripts = datasets(keywords, param_licenses) offline_mesg = "Available offline datasets : {}\n" online_mesg = "Available online datasets : {}\n" if not searched_scripts: print("No available datasets found") else: print(offline_mesg.format(len( searched_scripts['offline']))) count = 1 for script in searched_scripts['offline']: print("{count}. {title}\n{name}\n" "{keywords}\n{licenses}\n".format( count=count, title=script.title, name=script.name, keywords=script.keywords, licenses=str(script.licenses[0]['name']) if script.licenses and len(script.licenses) else str('N/A'), )) count += 1 count = 1 searched_scripts_offline = [ script.name for script in searched_scripts["offline"] ] searched_scripts_online = [] for script in searched_scripts['online']: if script in searched_scripts_offline: continue searched_scripts_online.append(script) print(online_mesg.format(len(searched_scripts_online))) for script in searched_scripts_online: print("{count}. {name}".format(count=count, name=script)) count += 1 return if args.command == 'commit': commit( dataset=args.dataset, path=os.path.normpath(args.path) if args.path else None, commit_message=args.message, ) return if args.command == 'log': commit_log(dataset=args.dataset) return engine = choose_engine(args.__dict__) if hasattr(args, 'debug') and args.debug: debug = True else: debug = False sys.tracebacklimit = 0 if hasattr(args, 'debug') and args.not_cached: use_cache = False else: use_cache = True engine.use_cache = use_cache if args.dataset is not None: if args.dataset.startswith(('socrata', 'rdataset')): scripts = True else: scripts = name_matches(script_list, args.dataset) else: raise Exception("no dataset specified.") if scripts: _install(vars(args), debug=debug, use_cache=use_cache) print("Done!") else: print( "Run 'retriever ls' to see a list of currently available datasets." )
def main(): """This function launches the Data Retriever.""" if len(sys.argv) == 1: # if no command line args are passed, show the help options parser.parse_args(['-h']) else: # otherwise, parse them args = parser.parse_args() reset_or_update = args.command in ["reset", "update"] if (not reset_or_update and not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) and not [ f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1]) if os.path.exists(SCRIPT_SEARCH_PATHS[-1]) ]): check_for_updates() reload_scripts() script_list = SCRIPT_LIST() if args.command == "install" and not args.engine: parser.parse_args(["install", "-h"]) if args.quiet: sys.stdout = open(os.devnull, "w") if args.command == "help": parser.parse_args(["-h"]) if hasattr(args, "compile") and args.compile: script_list = reload_scripts() if args.command == "defaults": for engine_item in engine_list: print("Default options for engine ", engine_item.name) for default_opts in engine_item.required_opts: print(default_opts[0], " ", default_opts[2]) print() return if args.command == "update": check_for_updates() reload_scripts() return if args.command == "citation": if args.dataset is None: print("\nCitation for retriever:\n") print(CITATION) else: citations = get_script_citation(args.dataset) for citation in citations: print("Citation: {}".format(citation)) return if args.command == 'license': if args.dataset is None: print(LICENSE) else: dataset_license = license(args.dataset) if dataset_license: print(dataset_license) else: print("There is no license information for {}".format( args.dataset)) return if args.command == 'new': f = open(args.filename, 'w') f.write(sample_script) f.close() return if args.command == 'reset': reset_retriever(args.scope) return if args.command == 'autocreate': if sum([args.f, args.d]) == 1: file_flag = bool(args.f) create_package(args.path, args.dt, file_flag, args.o, args.skip_lines, args.e) else: print('Please use one and only one of the flags -f -d') return if args.command == 'ls': # scripts should never be empty because check_for_updates is run on SCRIPT_LIST init if not (args.l or args.k or isinstance(args.v, list)): all_scripts = dataset_names() from retriever import lscolumns all_scripts_combined = [] for dataset in all_scripts['offline']: all_scripts_combined.append((dataset, True)) for dataset in all_scripts['online']: if dataset in all_scripts['offline']: continue all_scripts_combined.append((dataset, False)) all_scripts_combined = sorted(all_scripts_combined, key=lambda x: x[0]) print("Available datasets : {}\n".format( len(all_scripts_combined))) lscolumns.printls(all_scripts_combined) print("\nThe symbol * denotes the online datasets.") print( "To see the full list of available online datasets, visit\n" "https://github.com/weecology/retriever-recipes.") elif isinstance(args.v, list): online_scripts = [] if args.v: try: all_scripts = [ get_script(dataset) for dataset in args.v ] except KeyError: all_scripts = [] print("Dataset(s) is not found.") else: scripts = datasets() all_scripts = scripts['offline'] online_scripts = scripts['online'] count = 1 if not args.v: print("Offline datasets : {}\n".format(len(all_scripts))) for script in all_scripts: print("{count}. {title}\n {name}\n" "{keywords}\n{description}\n" "{licenses}\n{citation}\n" "".format( count=count, title=script.title, name=script.name, keywords=script.keywords, description=script.description, licenses=str(script.licenses[0]['name']), citation=script.citation, )) count += 1 count = 1 offline_scripts = [script.name for script in all_scripts] set_online_scripts = [] for script in online_scripts: if script in offline_scripts: continue set_online_scripts.append(script) if not args.v: print("Online datasets : {}\n".format( len(set_online_scripts))) for script in set_online_scripts: print("{count}. {name}".format(count=count, name=script)) count += 1 else: param_licenses = args.l if args.l else None keywords = args.k if args.k else None # search searched_scripts = datasets(keywords, param_licenses) offline_mesg = "Available offline datasets : {}\n" online_mesg = "Available online datasets : {}\n" if not searched_scripts: print("No available datasets found") else: print(offline_mesg.format(len( searched_scripts['offline']))) count = 1 for script in searched_scripts['offline']: print("{count}. {title}\n{name}\n" "{keywords}\n{licenses}\n".format( count=count, title=script.title, name=script.name, keywords=script.keywords, licenses=str(script.licenses[0]['name']), )) count += 1 count = 1 searched_scripts_offline = [ script.name for script in searched_scripts["offline"] ] searched_scripts_online = [] for script in searched_scripts['online']: if script in searched_scripts_offline: continue searched_scripts_online.append(script) print(online_mesg.format(len(searched_scripts_online))) for script in searched_scripts_online: print("{count}. {name}".format(count=count, name=script)) count += 1 return if args.command == 'commit': commit( dataset=args.dataset, path=os.path.normpath(args.path) if args.path else None, commit_message=args.message, ) return if args.command == 'log': commit_log(dataset=args.dataset) return engine = choose_engine(args.__dict__) if hasattr(args, 'debug') and args.debug: debug = True else: debug = False sys.tracebacklimit = 0 if hasattr(args, 'debug') and args.not_cached: use_cache = False else: use_cache = True engine.use_cache = use_cache if args.dataset is not None: scripts = name_matches(script_list, args.dataset) else: raise Exception("no dataset specified.") if scripts: if args.dataset.endswith('.zip') or hasattr(args, 'hash_value'): _install(vars(args), debug=debug, use_cache=use_cache) return for dataset in scripts: print("=> Installing", dataset.name) try: dataset.download(engine, debug=debug) dataset.engine.final_cleanup() except KeyboardInterrupt: pass except Exception as e: print(e) if debug: raise print("Done!") else: print( "Run 'retriever ls' to see a list of currently available datasets." )
def compile(): """Reload scripts for CLI""" print("Compiling retriever scripts...") reload_scripts() print("done.")
def to_str(object, object_encoding=encoding): if sys.version_info >= (3, 0, 0): return str(object).encode('UTF-8').decode(encoding) return object # Create the .rst file for the available datasets datasetfile = open_fw("datasets_list.rst") datasetfile_title = """================== Datasets Available ================== """ check_for_updates() reload_scripts() script_list = SCRIPT_LIST() # write the title of dataset rst file # ref:http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html datasetfile.write(datasetfile_title) # get info from the scripts using specified encoding for script_num, script in enumerate(script_list, start=1): reference_link = '' if script.ref.strip(): reference_link = script.ref elif hasattr(script, 'homepage'): reference_link = script.homepage elif not reference_link.strip(): if bool(script.urls.values()):
def main(): """This function launches the Data Retriever.""" if len(sys.argv) == 1: # if no command line args are passed, show the help options parser.parse_args(['-h']) else: # otherwise, parse them args = parser.parse_args() if args.command not in ['reset', 'update'] \ and not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) \ and not [f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1]) if os.path.exists(SCRIPT_SEARCH_PATHS[-1])]: check_for_updates() reload_scripts() script_list = SCRIPT_LIST() if args.command == "install" and not args.engine: parser.parse_args(['install', '-h']) if args.quiet: sys.stdout = open(os.devnull, 'w') if args.command == 'help': parser.parse_args(['-h']) if hasattr(args, 'compile') and args.compile: script_list = reload_scripts() if args.command == 'defaults': for engine_item in engine_list: print("Default options for engine ", engine_item.name) for default_opts in engine_item.required_opts: print(default_opts[0], " ", default_opts[2]) print() return if args.command == 'update': check_for_updates() reload_scripts() return elif args.command == 'citation': if args.dataset is None: print("\nCitation for retriever:\n") print(CITATION) else: scripts = name_matches(script_list, args.dataset) for dataset in scripts: print("\nDataset: {}".format(dataset.name)) print("Citation: {}".format(dataset.citation)) print("Description: {}\n".format(dataset.description)) return elif args.command == 'license': if args.dataset is None: print(LICENSE) else: dataset_license = license(args.dataset) if dataset_license: print(dataset_license) else: print("There is no license information for {}".format(args.dataset)) return elif args.command == 'new': f = open(args.filename, 'w') f.write(sample_script) f.close() return elif args.command == 'reset': reset_retriever(args.scope) return elif args.command == 'new_json': # create new JSON script create_json() return elif args.command == 'edit_json': # edit existing JSON script json_file = get_script_filename(args.dataset.lower()) edit_json(json_file) return elif args.command == 'autocreate': if sum([args.f, args.d]) == 1: file_flag = True if args.f else False create_package(args.path, args.dt, file_flag, args.o, args.skip_lines) else: print('Please use one and only one of the flags -f -d') return elif args.command == 'delete_json': # delete existing JSON script from home directory and or script directory if exists in current dir confirm = input("Really remove " + args.dataset.lower() + " and all its contents? (y/N): ") if confirm.lower().strip() in ['y', 'yes']: json_file = get_script_filename(args.dataset.lower()) delete_json(json_file) return if args.command == 'ls': # scripts should never be empty because check_for_updates is run on SCRIPT_LIST init if not (args.l or args.k or isinstance(args.v, list)): all_scripts = dataset_names() print("Available datasets : {}\n".format(len(all_scripts))) from retriever import lscolumns lscolumns.printls(all_scripts) elif isinstance(args.v, list): if args.v: try: all_scripts = [get_script(dataset) for dataset in args.v] except KeyError: all_scripts = [] print("Dataset(s) is not found.") else: all_scripts = datasets() count = 1 for script in all_scripts: print( "{count}. {title}\n {name}\n" "{keywords}\n{description}\n" "{licenses}\n{citation}\n" "".format( count=count, title=script.title, name=script.name, keywords=script.keywords, description=script.description, licenses=str(script.licenses[0]['name']), citation=script.citation, ) ) count += 1 else: param_licenses = args.l if args.l else None keywords = args.k if args.k else None # search searched_scripts = datasets(keywords, param_licenses) if not searched_scripts: print("No available datasets found") else: print("Available datasets : {}\n".format(len(searched_scripts))) count = 1 for script in searched_scripts: print( "{count}. {title}\n{name}\n" "{keywords}\n{licenses}\n".format( count=count, title=script.title, name=script.name, keywords=script.keywords, licenses=str(script.licenses[0]['name']), ) ) count += 1 return engine = choose_engine(args.__dict__) if hasattr(args, 'debug') and args.debug: debug = True else: debug = False sys.tracebacklimit = 0 if hasattr(args, 'debug') and args.not_cached: engine.use_cache = False else: engine.use_cache = True if args.dataset is not None: scripts = name_matches(script_list, args.dataset) else: raise Exception("no dataset specified.") if scripts: for dataset in scripts: print("=> Installing", dataset.name) try: dataset.download(engine, debug=debug) dataset.engine.final_cleanup() except KeyboardInterrupt: pass except Exception as e: print(e) if debug: raise print("Done!") else: print("Run 'retriever ls' to see a list of currently available datasets.")