def _get_source_folder(source_path_or_url, temporary_folder, file_extensions):
if isdir(source_path_or_url):
return source_path_or_url
if source_path_or_url.startswith(
'http://') or source_path_or_url.startswith('https://'):
source_name = basename(source_path_or_url).strip()
source_path = join(temporary_folder, source_name.lower())
urlretrieve(source_path_or_url, source_path)
else:
source_path = source_path_or_url
source_extension = splitext(source_path)[1]
target_folder = make_unique_folder(temporary_folder)
if source_extension not in file_extensions:
try:
return uncompress(source_path, target_folder)
except BadFormat:
pass
for extension in ARCHIVE_EXTENSIONS:
try:
link_path = link_safely(
make_unique_path(temporary_folder, extension), source_path)
return uncompress(link_path, target_folder)
except BadArchive:
pass
raise GeoTableError(source_path)
def receive_result_request(endpoint_url, queue_token, parent_folder):
response = requests.get(endpoint_url,
headers={'Authorization': 'Bearer ' + queue_token})
if response.status_code == 200:
pass
elif response.status_code == 204:
raise HTTPNoContent
elif response.status_code == 401:
raise HTTPUnauthorized
else:
raise HTTPBadRequest
archive_path = make_unique_path(parent_folder, '.zip', length=16)
open(archive_path, 'wb').write(response.content)
result_folder = uncompress(archive_path)
return result_folder
def receive_result_request(endpoint_url, queue_token, parent_folder):
response = requests.get(endpoint_url, headers={
'Authorization': 'Bearer ' + queue_token})
if response.status_code == 200:
pass
elif response.status_code == 204:
raise HTTPNoContent
elif response.status_code == 401:
raise HTTPUnauthorized
else:
raise HTTPBadRequest
archive_path = make_unique_path(parent_folder, '.zip', length=16)
open(archive_path, 'wb').write(response.content)
result_folder = uncompress(archive_path)
return result_folder
def prepare_script_folder(target_folder, notebook, notebook_name):
tool_arguments = load_tool_arguments(notebook)
# Prepare paths
for k, v in tool_arguments.items():
if not k.endswith('_path'):
continue
path = make_unique_path(target_folder, get_file_extension(v))
shutil.copy(v, path)
tool_arguments[k] = basename(path)
# Prepare command-line script
script_lines = []
script_lines.append('from sys import argv')
for i, arg in zip(range(1, len(tool_arguments) + 1), tool_arguments):
if arg.endswith('_integer'):
script_lines.append('%s = int(argv[%s])' % (arg, i))
else:
script_lines.append('%s = argv[%s]' % (arg, i))
#script_lines.append('%s = argv[1:]' % ', '.join(tool_arguments))
notebook.cells[0]['source'] = '\n'.join(script_lines)
script_content, script_info = nbconvert.export_script(notebook)
script_name = 'run' + script_info['output_extension']
if script_name.endswith('.py'):
command_name = 'python'
else:
raise CrossComputeError
# Save script
script_path = join(target_folder, script_name)
codecs.open(script_path, 'w', encoding='utf-8').write(script_content)
# Save configuration
configuration_path = join(target_folder, 'cc.ini')
configuration_lines = []
configuration_lines.append('[crosscompute %s]' % notebook_name)
configuration_lines.append(
'command_template = %s %s %s' %
(command_name, script_name, ' '.join('{%s}' % x
for x in tool_arguments).strip()))
for k, v in tool_arguments.items():
if k in RESERVED_ARGUMENT_NAMES:
continue
configuration_lines.append('%s = %s' % (k, v))
codecs.open(configuration_path, 'w',
encoding='utf-8').write('\n'.join(configuration_lines).strip())
return target_folder
def prepare_script_folder(target_folder, notebook, notebook_name):
tool_arguments = load_tool_arguments(notebook)
# Prepare paths
for k, v in tool_arguments.items():
if not k.endswith("_path"):
continue
path = make_unique_path(target_folder, get_file_extension(v))
shutil.copy(v, path)
tool_arguments[k] = basename(path)
# Prepare command-line script
script_lines = []
script_lines.append("from sys import argv")
script_lines.append("%s = argv[1:]" % ", ".join(tool_arguments))
notebook.cells[0]["source"] = "\n".join(script_lines)
script_content, script_info = nbconvert.export_script(notebook)
script_name = "run" + script_info["output_extension"]
if script_name.endswith(".py"):
command_name = "python"
else:
raise CrossComputeError
# Save script
script_path = join(target_folder, script_name)
codecs.open(script_path, "w", encoding="utf-8").write(script_content)
# Save configuration
configuration_path = join(target_folder, "cc.ini")
configuration_lines = []
configuration_lines.append("[crosscompute %s]" % notebook_name)
configuration_lines.append(
"command_template = %s %s %s"
% (command_name, script_name, " ".join("{%s}" % x for x in tool_arguments).strip())
)
for k, v in tool_arguments.items():
if k in RESERVED_ARGUMENT_NAMES:
continue
configuration_lines.append("%s = %s" % (k, v))
codecs.open(configuration_path, "w", encoding="utf-8").write("\n".join(configuration_lines).strip())
return target_folder
