def create_socrata_dataset(engine, name, resource, script_path=None):
"""Downloads raw data and creates a script for the socrata dataset"""
if script_path is None:
script_path = SOCRATA_SCRIPT_WRITE_PATH
filename = resource["id"] + '.csv'
engine.script = BasicTextTemplate(**{"name": name})
url = 'https://' + resource["domain"] + '/resource/' + filename
if not engine.find_file(filename) or not engine.use_cache:
path = engine.format_filename(filename)
engine.create_raw_data_dir()
progbar = tqdm(
unit='B',
unit_scale=True,
unit_divisor=1024,
miniters=1,
desc='Downloading {}'.format(filename),
)
result = engine.download_from_socrata(url, path, progbar)
if not result:
print("Unable to download the dataset using Socrata API")
elif engine.find_file(filename):
path = engine.format_filename(filename)
engine.script = None
if engine.opts["command"] == "download":
engine.final_cleanup()
return
else:
if not os.path.exists(script_path):
os.makedirs(script_path)
file = name.replace("-", "_")
if str(file + ".json") in [
file_i for file_i in os.listdir(script_path)
if file_i.endswith(".json")
]:
print(
f"Dataset {name} already exists as {file}.json in {script_path}"
)
else:
create_package(path, "tabular", True, script_path)
print("Updating script name to {}".format(name + ".json"))
print("Updating the contents of script {}".format(name))
update_socrata_script(name, filename, url, resource, script_path)
reload_scripts()
def create_rdataset(engine, package, dataset_name, script_path=None):
"""Download files for RDatasets to the raw data directory"""
if script_path is None:
script_path = RDATASET_SCRIPT_WRITE_PATH
update_rdataset_catalog()
with open(RDATASET_PATH, 'r') as f:
rdatasets = json.load(f)
f.close()
try:
rpackage = rdatasets[package]
try:
data_obj = rpackage[dataset_name]
except KeyError:
print("Dataset '{}' not found in package '{}' RDatasets".format(
dataset_name, package))
except KeyError:
print("Package '{}' not found in RDatasets".format(package))
script_name = f"rdataset-{package}-{dataset_name}"
filename = data_obj['csv'].split('/')[-1]
engine.script = BasicTextTemplate(**{"name": script_name})
if not engine.find_file(filename) or not engine.use_cache:
engine.download_file(data_obj['csv'], filename)
path = engine.format_filename(filename)
engine.script = None
if engine.opts["command"] == "download":
engine.final_cleanup()
return
else:
create_package(path, 'tabular', True, script_path)
print("Updating script name to {}".format(script_name + ".json"))
print("Updating the contents of script {}".format(script_name))
update_rdataset_script(data_obj, dataset_name, package, script_path)
print("Updated the script {}".format(script_name))
reload_scripts()
def main():
"""This function launches the Data Retriever."""
if len(sys.argv) == 1:
# if no command line args are passed, show the help options
parser.parse_args(['-h'])
else:
# otherwise, parse them
args = parser.parse_args()
if args.command not in ['reset', 'update'] \
and not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) \
and not [f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1])
if os.path.exists(SCRIPT_SEARCH_PATHS[-1])]:
check_for_updates()
reload_scripts()
script_list = SCRIPT_LIST()
if args.command == "install" and not args.engine:
parser.parse_args(['install', '-h'])
if args.quiet:
sys.stdout = open(os.devnull, 'w')
if args.command == 'help':
parser.parse_args(['-h'])
if hasattr(args, 'compile') and args.compile:
script_list = reload_scripts()
if args.command == 'defaults':
for engine_item in engine_list:
print("Default options for engine ", engine_item.name)
for default_opts in engine_item.required_opts:
print(default_opts[0], " ", default_opts[2])
print()
return
if args.command == 'update':
check_for_updates()
reload_scripts()
return
elif args.command == 'citation':
if args.dataset is None:
print("\nCitation for retriever:\n")
print(CITATION)
else:
scripts = name_matches(script_list, args.dataset)
for dataset in scripts:
print("\nDataset: {}".format(dataset.name))
print("Citation: {}".format(dataset.citation))
print("Description: {}\n".format(dataset.description))
return
elif args.command == 'license':
if args.dataset is None:
print(LICENSE)
else:
dataset_license = license(args.dataset)
if dataset_license:
print(dataset_license)
else:
print("There is no license information for {}".format(
args.dataset))
return
elif args.command == 'new':
f = open(args.filename, 'w')
f.write(sample_script)
f.close()
return
elif args.command == 'reset':
reset_retriever(args.scope)
return
elif args.command == 'new_json':
# create new JSON script
create_json()
return
elif args.command == 'edit_json':
# edit existing JSON script
json_file = get_script_filename(args.dataset.lower())
edit_json(json_file)
return
elif args.command == 'autocreate':
if sum([args.f, args.d]) == 1:
file_flag = True if args.f else False
create_package(args.path, args.dt, file_flag, args.o,
args.skip_lines)
else:
print('Please use one and only one of the flags -f -d')
return
elif args.command == 'delete_json':
# delete existing JSON script from home directory and or script directory if exists in current dir
confirm = input("Really remove " + args.dataset.lower() +
" and all its contents? (y/N): ")
if confirm.lower().strip() in ['y', 'yes']:
json_file = get_script_filename(args.dataset.lower())
delete_json(json_file)
return
if args.command == 'ls':
# scripts should never be empty because check_for_updates is run on SCRIPT_LIST init
if not (args.l or args.k or isinstance(args.v, list)):
all_scripts = dataset_names()
print("Available datasets : {}\n".format(len(all_scripts)))
from retriever import lscolumns
lscolumns.printls(all_scripts)
elif isinstance(args.v, list):
if args.v:
try:
all_scripts = [
get_script(dataset) for dataset in args.v
]
except KeyError:
all_scripts = []
print("Dataset(s) is not found.")
else:
all_scripts = datasets()
count = 1
for script in all_scripts:
print("{count}. {title}\n {name}\n"
"{keywords}\n{description}\n"
"{licenses}\n{citation}\n"
"".format(
count=count,
title=script.title,
name=script.name,
keywords=script.keywords,
description=script.description,
licenses=str(script.licenses[0]['name']),
citation=script.citation,
))
count += 1
else:
param_licenses = args.l if args.l else None
keywords = args.k if args.k else None
# search
searched_scripts = datasets(keywords, param_licenses)
if not searched_scripts:
print("No available datasets found")
else:
print("Available datasets : {}\n".format(
len(searched_scripts)))
count = 1
for script in searched_scripts:
print("{count}. {title}\n{name}\n"
"{keywords}\n{licenses}\n".format(
count=count,
title=script.title,
name=script.name,
keywords=script.keywords,
licenses=str(script.licenses[0]['name']),
))
count += 1
return
engine = choose_engine(args.__dict__)
if hasattr(args, 'debug') and args.debug:
debug = True
else:
debug = False
sys.tracebacklimit = 0
if hasattr(args, 'debug') and args.not_cached:
engine.use_cache = False
else:
engine.use_cache = True
if args.dataset is not None:
scripts = name_matches(script_list, args.dataset)
else:
raise Exception("no dataset specified.")
if scripts:
for dataset in scripts:
print("=> Installing", dataset.name)
try:
dataset.download(engine, debug=debug)
dataset.engine.final_cleanup()
except KeyboardInterrupt:
pass
except Exception as e:
print(e)
if debug:
raise
print("Done!")
else:
print(
"Run 'retriever ls' to see a list of currently available datasets."
)
def main():
"""This function launches the Data Retriever."""
if len(sys.argv) == 1:
# if no command line args are passed, show the help options
parser.parse_args(['-h'])
else:
# otherwise, parse them
args = parser.parse_args()
reset_or_update = args.command in ["reset", "update"]
if (not reset_or_update and not os.path.isdir(SCRIPT_SEARCH_PATHS[1])
and not [
f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1])
if os.path.exists(SCRIPT_SEARCH_PATHS[-1])
]):
check_for_updates()
reload_scripts()
script_list = SCRIPT_LIST()
if args.command == "install" and not args.engine:
parser.parse_args(["install", "-h"])
if args.quiet:
sys.stdout = open(os.devnull, "w")
if args.command == "help":
parser.parse_args(["-h"])
if hasattr(args, "compile") and args.compile:
script_list = reload_scripts()
if args.command == "defaults":
for engine_item in engine_list:
print("Default options for engine ", engine_item.name)
for default_opts in engine_item.required_opts:
print(default_opts[0], " ", default_opts[2])
print()
return
if args.command == "update":
check_for_updates()
reload_scripts()
return
if args.command == "citation":
if args.dataset is None:
print("\nCitation for retriever:\n")
print(CITATION)
else:
citations = get_script_citation(args.dataset)
for citation in citations:
print("Citation: {}".format(citation))
return
if args.command == 'license':
if args.dataset is None:
print(LICENSE)
else:
dataset_license = license(args.dataset)
if dataset_license:
print(dataset_license)
else:
print("There is no license information for {}".format(
args.dataset))
return
if args.command == 'new':
f = open(args.filename, 'w')
f.write(sample_script)
f.close()
return
if args.command == 'reset':
reset_retriever(args.scope)
return
if args.command == 'autocreate':
if args.c:
url = args.path
script_list = SCRIPT_LIST()
flag = 0
for script in script_list:
for dataset in script.tables:
if script.tables[dataset].url == url:
flag = 1
break
if flag == 1:
print("File already exist in dataset " + str(script.name))
else:
print("Dataset is not avaliable, Please download")
return
if sum([args.f, args.d]) == 1:
file_flag = bool(args.f)
create_package(args.path, args.dt, file_flag, args.o,
args.skip_lines, args.e)
else:
print('Please use one and only one of the flags -f -d')
return
if args.command == 'ls':
# scripts should never be empty because check_for_updates is run on SCRIPT_LIST init
if not any([args.l, args.k, args.v, args.s, args.rdataset]):
all_scripts = dataset_names()
from retriever import lscolumns
all_scripts_combined = []
for dataset in all_scripts['offline']:
all_scripts_combined.append((dataset, True))
for dataset in all_scripts['online']:
if dataset in all_scripts['offline']:
continue
all_scripts_combined.append((dataset, False))
all_scripts_combined = sorted(all_scripts_combined,
key=lambda x: x[0])
print("Available datasets : {}\n".format(
len(all_scripts_combined)))
lscolumns.printls(all_scripts_combined)
print("\nThe symbol * denotes the online datasets.")
print(
"To see the full list of available online datasets, visit\n"
"https://github.com/weecology/retriever-recipes.")
elif isinstance(args.s, list):
try:
theme = INQUIRER_THEME
except NameError:
print("To use retriever ls -s, install inquirer")
exit()
name_list = socrata_autocomplete_search(args.s)
print("Autocomplete suggestions : Total {} results\n".format(
len(name_list)))
if len(name_list):
question = [
inquirer.List('dataset name',
message='Select the dataset name',
choices=name_list)
]
answer = inquirer.prompt(question,
theme=INQUIRER_THEME,
raise_keyboard_interrupt=True)
dataset_name = answer['dataset name']
metadata = socrata_dataset_info(dataset_name)
print(
"Dataset Information of {}: Total {} results\n".format(
dataset_name, len(metadata)))
for i in range(len(metadata)):
print("{}. {}\n \tID : {}\n"
"\tType : {}\n"
"\tDescription : {}\n"
"\tDomain : {}\n \tLink : {}\n".format(
i + 1, metadata[i]["name"],
metadata[i]["id"], metadata[i]["type"],
metadata[i]["description"][:50] + "...",
metadata[i]["domain"], metadata[i]["link"]))
elif args.rdataset:
if not isinstance(args.p, list) and not args.all:
display_all_rdataset_names()
elif not isinstance(args.p, list) and args.all:
display_all_rdataset_names(package_name='all')
else:
display_all_rdataset_names(package_name=args.p)
elif isinstance(args.v, list):
dataset_verbose_list(args.v)
else:
param_licenses = args.l if args.l else None
keywords = args.k if args.k else None
# search
searched_scripts = datasets(keywords, param_licenses)
offline_mesg = "Available offline datasets : {}\n"
online_mesg = "Available online datasets : {}\n"
if not searched_scripts:
print("No available datasets found")
else:
print(offline_mesg.format(len(
searched_scripts['offline'])))
count = 1
for script in searched_scripts['offline']:
print("{count}. {title}\n{name}\n"
"{keywords}\n{licenses}\n".format(
count=count,
title=script.title,
name=script.name,
keywords=script.keywords,
licenses=str(script.licenses[0]['name'])
if script.licenses and len(script.licenses)
else str('N/A'),
))
count += 1
count = 1
searched_scripts_offline = [
script.name for script in searched_scripts["offline"]
]
searched_scripts_online = []
for script in searched_scripts['online']:
if script in searched_scripts_offline:
continue
searched_scripts_online.append(script)
print(online_mesg.format(len(searched_scripts_online)))
for script in searched_scripts_online:
print("{count}. {name}".format(count=count,
name=script))
count += 1
return
if args.command == 'commit':
commit(
dataset=args.dataset,
path=os.path.normpath(args.path) if args.path else None,
commit_message=args.message,
)
return
if args.command == 'log':
commit_log(dataset=args.dataset)
return
engine = choose_engine(args.__dict__)
if hasattr(args, 'debug') and args.debug:
debug = True
else:
debug = False
sys.tracebacklimit = 0
if hasattr(args, 'debug') and args.not_cached:
use_cache = False
else:
use_cache = True
engine.use_cache = use_cache
if args.dataset is not None:
if args.dataset.startswith(('socrata', 'rdataset')):
scripts = True
else:
scripts = name_matches(script_list, args.dataset)
else:
raise Exception("no dataset specified.")
if scripts:
_install(vars(args), debug=debug, use_cache=use_cache)
print("Done!")
else:
print(
"Run 'retriever ls' to see a list of currently available datasets."
)
def main():
"""This function launches the Data Retriever."""
if len(sys.argv) == 1:
# if no command line args are passed, show the help options
parser.parse_args(['-h'])
else:
# otherwise, parse them
args = parser.parse_args()
reset_or_update = args.command in ["reset", "update"]
if (not reset_or_update and not os.path.isdir(SCRIPT_SEARCH_PATHS[1])
and not [
f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1])
if os.path.exists(SCRIPT_SEARCH_PATHS[-1])
]):
check_for_updates()
reload_scripts()
script_list = SCRIPT_LIST()
if args.command == "install" and not args.engine:
parser.parse_args(["install", "-h"])
if args.quiet:
sys.stdout = open(os.devnull, "w")
if args.command == "help":
parser.parse_args(["-h"])
if hasattr(args, "compile") and args.compile:
script_list = reload_scripts()
if args.command == "defaults":
for engine_item in engine_list:
print("Default options for engine ", engine_item.name)
for default_opts in engine_item.required_opts:
print(default_opts[0], " ", default_opts[2])
print()
return
if args.command == "update":
check_for_updates()
reload_scripts()
return
if args.command == "citation":
if args.dataset is None:
print("\nCitation for retriever:\n")
print(CITATION)
else:
citations = get_script_citation(args.dataset)
for citation in citations:
print("Citation: {}".format(citation))
return
if args.command == 'license':
if args.dataset is None:
print(LICENSE)
else:
dataset_license = license(args.dataset)
if dataset_license:
print(dataset_license)
else:
print("There is no license information for {}".format(
args.dataset))
return
if args.command == 'new':
f = open(args.filename, 'w')
f.write(sample_script)
f.close()
return
if args.command == 'reset':
reset_retriever(args.scope)
return
if args.command == 'autocreate':
if sum([args.f, args.d]) == 1:
file_flag = bool(args.f)
create_package(args.path, args.dt, file_flag, args.o,
args.skip_lines, args.e)
else:
print('Please use one and only one of the flags -f -d')
return
if args.command == 'ls':
# scripts should never be empty because check_for_updates is run on SCRIPT_LIST init
if not (args.l or args.k or isinstance(args.v, list)):
all_scripts = dataset_names()
from retriever import lscolumns
all_scripts_combined = []
for dataset in all_scripts['offline']:
all_scripts_combined.append((dataset, True))
for dataset in all_scripts['online']:
if dataset in all_scripts['offline']:
continue
all_scripts_combined.append((dataset, False))
all_scripts_combined = sorted(all_scripts_combined,
key=lambda x: x[0])
print("Available datasets : {}\n".format(
len(all_scripts_combined)))
lscolumns.printls(all_scripts_combined)
print("\nThe symbol * denotes the online datasets.")
print(
"To see the full list of available online datasets, visit\n"
"https://github.com/weecology/retriever-recipes.")
elif isinstance(args.v, list):
online_scripts = []
if args.v:
try:
all_scripts = [
get_script(dataset) for dataset in args.v
]
except KeyError:
all_scripts = []
print("Dataset(s) is not found.")
else:
scripts = datasets()
all_scripts = scripts['offline']
online_scripts = scripts['online']
count = 1
if not args.v:
print("Offline datasets : {}\n".format(len(all_scripts)))
for script in all_scripts:
print("{count}. {title}\n {name}\n"
"{keywords}\n{description}\n"
"{licenses}\n{citation}\n"
"".format(
count=count,
title=script.title,
name=script.name,
keywords=script.keywords,
description=script.description,
licenses=str(script.licenses[0]['name']),
citation=script.citation,
))
count += 1
count = 1
offline_scripts = [script.name for script in all_scripts]
set_online_scripts = []
for script in online_scripts:
if script in offline_scripts:
continue
set_online_scripts.append(script)
if not args.v:
print("Online datasets : {}\n".format(
len(set_online_scripts)))
for script in set_online_scripts:
print("{count}. {name}".format(count=count, name=script))
count += 1
else:
param_licenses = args.l if args.l else None
keywords = args.k if args.k else None
# search
searched_scripts = datasets(keywords, param_licenses)
offline_mesg = "Available offline datasets : {}\n"
online_mesg = "Available online datasets : {}\n"
if not searched_scripts:
print("No available datasets found")
else:
print(offline_mesg.format(len(
searched_scripts['offline'])))
count = 1
for script in searched_scripts['offline']:
print("{count}. {title}\n{name}\n"
"{keywords}\n{licenses}\n".format(
count=count,
title=script.title,
name=script.name,
keywords=script.keywords,
licenses=str(script.licenses[0]['name']),
))
count += 1
count = 1
searched_scripts_offline = [
script.name for script in searched_scripts["offline"]
]
searched_scripts_online = []
for script in searched_scripts['online']:
if script in searched_scripts_offline:
continue
searched_scripts_online.append(script)
print(online_mesg.format(len(searched_scripts_online)))
for script in searched_scripts_online:
print("{count}. {name}".format(count=count,
name=script))
count += 1
return
if args.command == 'commit':
commit(
dataset=args.dataset,
path=os.path.normpath(args.path) if args.path else None,
commit_message=args.message,
)
return
if args.command == 'log':
commit_log(dataset=args.dataset)
return
engine = choose_engine(args.__dict__)
if hasattr(args, 'debug') and args.debug:
debug = True
else:
debug = False
sys.tracebacklimit = 0
if hasattr(args, 'debug') and args.not_cached:
use_cache = False
else:
use_cache = True
engine.use_cache = use_cache
if args.dataset is not None:
scripts = name_matches(script_list, args.dataset)
else:
raise Exception("no dataset specified.")
if scripts:
if args.dataset.endswith('.zip') or hasattr(args, 'hash_value'):
_install(vars(args), debug=debug, use_cache=use_cache)
return
for dataset in scripts:
print("=> Installing", dataset.name)
try:
dataset.download(engine, debug=debug)
dataset.engine.final_cleanup()
except KeyboardInterrupt:
pass
except Exception as e:
print(e)
if debug:
raise
print("Done!")
else:
print(
"Run 'retriever ls' to see a list of currently available datasets."
)
def main():
"""This function launches the Data Retriever."""
if len(sys.argv) == 1:
# if no command line args are passed, show the help options
parser.parse_args(['-h'])
else:
# otherwise, parse them
args = parser.parse_args()
if args.command not in ['reset', 'update'] \
and not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) \
and not [f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1])
if os.path.exists(SCRIPT_SEARCH_PATHS[-1])]:
check_for_updates()
reload_scripts()
script_list = SCRIPT_LIST()
if args.command == "install" and not args.engine:
parser.parse_args(['install', '-h'])
if args.quiet:
sys.stdout = open(os.devnull, 'w')
if args.command == 'help':
parser.parse_args(['-h'])
if hasattr(args, 'compile') and args.compile:
script_list = reload_scripts()
if args.command == 'defaults':
for engine_item in engine_list:
print("Default options for engine ", engine_item.name)
for default_opts in engine_item.required_opts:
print(default_opts[0], " ", default_opts[2])
print()
return
if args.command == 'update':
check_for_updates()
reload_scripts()
return
elif args.command == 'citation':
if args.dataset is None:
print("\nCitation for retriever:\n")
print(CITATION)
else:
scripts = name_matches(script_list, args.dataset)
for dataset in scripts:
print("\nDataset: {}".format(dataset.name))
print("Citation: {}".format(dataset.citation))
print("Description: {}\n".format(dataset.description))
return
elif args.command == 'license':
if args.dataset is None:
print(LICENSE)
else:
dataset_license = license(args.dataset)
if dataset_license:
print(dataset_license)
else:
print("There is no license information for {}".format(args.dataset))
return
elif args.command == 'new':
f = open(args.filename, 'w')
f.write(sample_script)
f.close()
return
elif args.command == 'reset':
reset_retriever(args.scope)
return
elif args.command == 'new_json':
# create new JSON script
create_json()
return
elif args.command == 'edit_json':
# edit existing JSON script
json_file = get_script_filename(args.dataset.lower())
edit_json(json_file)
return
elif args.command == 'autocreate':
if sum([args.f, args.d]) == 1:
file_flag = True if args.f else False
create_package(args.path, args.dt, file_flag, args.o, args.skip_lines)
else:
print('Please use one and only one of the flags -f -d')
return
elif args.command == 'delete_json':
# delete existing JSON script from home directory and or script directory if exists in current dir
confirm = input("Really remove " + args.dataset.lower() +
" and all its contents? (y/N): ")
if confirm.lower().strip() in ['y', 'yes']:
json_file = get_script_filename(args.dataset.lower())
delete_json(json_file)
return
if args.command == 'ls':
# scripts should never be empty because check_for_updates is run on SCRIPT_LIST init
if not (args.l or args.k or isinstance(args.v, list)):
all_scripts = dataset_names()
print("Available datasets : {}\n".format(len(all_scripts)))
from retriever import lscolumns
lscolumns.printls(all_scripts)
elif isinstance(args.v, list):
if args.v:
try:
all_scripts = [get_script(dataset) for dataset in args.v]
except KeyError:
all_scripts = []
print("Dataset(s) is not found.")
else:
all_scripts = datasets()
count = 1
for script in all_scripts:
print(
"{count}. {title}\n {name}\n"
"{keywords}\n{description}\n"
"{licenses}\n{citation}\n"
"".format(
count=count,
title=script.title,
name=script.name,
keywords=script.keywords,
description=script.description,
licenses=str(script.licenses[0]['name']),
citation=script.citation,
)
)
count += 1
else:
param_licenses = args.l if args.l else None
keywords = args.k if args.k else None
# search
searched_scripts = datasets(keywords, param_licenses)
if not searched_scripts:
print("No available datasets found")
else:
print("Available datasets : {}\n".format(len(searched_scripts)))
count = 1
for script in searched_scripts:
print(
"{count}. {title}\n{name}\n"
"{keywords}\n{licenses}\n".format(
count=count,
title=script.title,
name=script.name,
keywords=script.keywords,
licenses=str(script.licenses[0]['name']),
)
)
count += 1
return
engine = choose_engine(args.__dict__)
if hasattr(args, 'debug') and args.debug:
debug = True
else:
debug = False
sys.tracebacklimit = 0
if hasattr(args, 'debug') and args.not_cached:
engine.use_cache = False
else:
engine.use_cache = True
if args.dataset is not None:
scripts = name_matches(script_list, args.dataset)
else:
raise Exception("no dataset specified.")
if scripts:
for dataset in scripts:
print("=> Installing", dataset.name)
try:
dataset.download(engine, debug=debug)
dataset.engine.final_cleanup()
except KeyboardInterrupt:
pass
except Exception as e:
print(e)
if debug:
raise
print("Done!")
else:
print("Run 'retriever ls' to see a list of currently available datasets.")