def _get_source_folder(source_path_or_url, temporary_folder, file_extensions): if isdir(source_path_or_url): return source_path_or_url if source_path_or_url.startswith( 'http://') or source_path_or_url.startswith('https://'): source_name = basename(source_path_or_url).strip() source_path = join(temporary_folder, source_name.lower()) urlretrieve(source_path_or_url, source_path) else: source_path = source_path_or_url source_extension = splitext(source_path)[1] target_folder = make_unique_folder(temporary_folder) if source_extension not in file_extensions: try: return uncompress(source_path, target_folder) except BadFormat: pass for extension in ARCHIVE_EXTENSIONS: try: link_path = link_safely( make_unique_path(temporary_folder, extension), source_path) return uncompress(link_path, target_folder) except BadArchive: pass raise GeoTableError(source_path)
def receive_result_request(endpoint_url, queue_token, parent_folder): response = requests.get(endpoint_url, headers={'Authorization': 'Bearer ' + queue_token}) if response.status_code == 200: pass elif response.status_code == 204: raise HTTPNoContent elif response.status_code == 401: raise HTTPUnauthorized else: raise HTTPBadRequest archive_path = make_unique_path(parent_folder, '.zip', length=16) open(archive_path, 'wb').write(response.content) result_folder = uncompress(archive_path) return result_folder
def receive_result_request(endpoint_url, queue_token, parent_folder): response = requests.get(endpoint_url, headers={ 'Authorization': 'Bearer ' + queue_token}) if response.status_code == 200: pass elif response.status_code == 204: raise HTTPNoContent elif response.status_code == 401: raise HTTPUnauthorized else: raise HTTPBadRequest archive_path = make_unique_path(parent_folder, '.zip', length=16) open(archive_path, 'wb').write(response.content) result_folder = uncompress(archive_path) return result_folder
def prepare_script_folder(target_folder, notebook, notebook_name): tool_arguments = load_tool_arguments(notebook) # Prepare paths for k, v in tool_arguments.items(): if not k.endswith('_path'): continue path = make_unique_path(target_folder, get_file_extension(v)) shutil.copy(v, path) tool_arguments[k] = basename(path) # Prepare command-line script script_lines = [] script_lines.append('from sys import argv') for i, arg in zip(range(1, len(tool_arguments) + 1), tool_arguments): if arg.endswith('_integer'): script_lines.append('%s = int(argv[%s])' % (arg, i)) else: script_lines.append('%s = argv[%s]' % (arg, i)) #script_lines.append('%s = argv[1:]' % ', '.join(tool_arguments)) notebook.cells[0]['source'] = '\n'.join(script_lines) script_content, script_info = nbconvert.export_script(notebook) script_name = 'run' + script_info['output_extension'] if script_name.endswith('.py'): command_name = 'python' else: raise CrossComputeError # Save script script_path = join(target_folder, script_name) codecs.open(script_path, 'w', encoding='utf-8').write(script_content) # Save configuration configuration_path = join(target_folder, 'cc.ini') configuration_lines = [] configuration_lines.append('[crosscompute %s]' % notebook_name) configuration_lines.append( 'command_template = %s %s %s' % (command_name, script_name, ' '.join('{%s}' % x for x in tool_arguments).strip())) for k, v in tool_arguments.items(): if k in RESERVED_ARGUMENT_NAMES: continue configuration_lines.append('%s = %s' % (k, v)) codecs.open(configuration_path, 'w', encoding='utf-8').write('\n'.join(configuration_lines).strip()) return target_folder
def prepare_script_folder(target_folder, notebook, notebook_name): tool_arguments = load_tool_arguments(notebook) # Prepare paths for k, v in tool_arguments.items(): if not k.endswith("_path"): continue path = make_unique_path(target_folder, get_file_extension(v)) shutil.copy(v, path) tool_arguments[k] = basename(path) # Prepare command-line script script_lines = [] script_lines.append("from sys import argv") script_lines.append("%s = argv[1:]" % ", ".join(tool_arguments)) notebook.cells[0]["source"] = "\n".join(script_lines) script_content, script_info = nbconvert.export_script(notebook) script_name = "run" + script_info["output_extension"] if script_name.endswith(".py"): command_name = "python" else: raise CrossComputeError # Save script script_path = join(target_folder, script_name) codecs.open(script_path, "w", encoding="utf-8").write(script_content) # Save configuration configuration_path = join(target_folder, "cc.ini") configuration_lines = [] configuration_lines.append("[crosscompute %s]" % notebook_name) configuration_lines.append( "command_template = %s %s %s" % (command_name, script_name, " ".join("{%s}" % x for x in tool_arguments).strip()) ) for k, v in tool_arguments.items(): if k in RESERVED_ARGUMENT_NAMES: continue configuration_lines.append("%s = %s" % (k, v)) codecs.open(configuration_path, "w", encoding="utf-8").write("\n".join(configuration_lines).strip()) return target_folder