def prepare_argument_path(self, argument_noun, raw_arguments, draft_folder, default_path): data_type = get_data_type(argument_noun) # If client sent direct content (x_table_csv), save it for file_format in data_type.formats: raw_argument_name = '%s_%s' % (argument_noun, file_format) if raw_argument_name not in raw_arguments: continue source_text = raw_arguments[raw_argument_name] target_name = '%s.%s' % (argument_noun, file_format) return copy_text(join(draft_folder, target_name), source_text) # Raise KeyError if client did not specify noun (x_table) v = raw_arguments[argument_noun] # If client sent multipart content, save it if hasattr(v, 'file'): target_name = argument_noun + get_file_extension(v.filename) return copy_file(join(draft_folder, target_name), v.file) # If client sent empty content, use default if v == '': if not default_path: raise KeyError target_name = argument_noun + get_file_extension(default_path) return link_path(join(draft_folder, target_name), default_path) # If client sent a relative path (x_table=11/x/y.csv), find it if '/' in v: source_path = self.get_file_path(*parse_result_relative_path(v)) target_name = argument_noun + get_file_extension(source_path) return link_path(join(draft_folder, target_name), source_path) # If client sent an upload id (x_table=x), find it upload = get_upload(self, upload_id=v) source_path = realpath(join(upload.folder, data_type.get_file_name())) target_name = argument_noun + get_file_extension(source_path) target_path = move_path(join(draft_folder, target_name), source_path) remove_safely(upload.folder) return target_path
def prepare_argument_path( self, argument_noun, raw_arguments, draft_folder, default_path): data_type = get_data_type(argument_noun) # If client sent direct content (x_table_csv), save it for file_format in data_type.formats: raw_argument_name = '%s_%s' % (argument_noun, file_format) if raw_argument_name not in raw_arguments: continue source_text = raw_arguments[raw_argument_name] target_name = '%s.%s' % (argument_noun, file_format) return copy_text(join(draft_folder, target_name), source_text) # Raise KeyError if client did not specify noun (x_table) v = raw_arguments[argument_noun] # If client sent multipart content, save it if hasattr(v, 'file'): target_name = argument_noun + get_file_extension(v.filename) return copy_file(join(draft_folder, target_name), v.file) # If client sent empty content, use default if v == '': if not default_path: raise KeyError target_name = argument_noun + get_file_extension(default_path) return link_path(join(draft_folder, target_name), default_path) # If client sent a relative path (x_table=11/x/y.csv), find it if '/' in v: source_path = self.get_file_path(*parse_result_relative_path(v)) target_name = argument_noun + get_file_extension(source_path) return link_path(join(draft_folder, target_name), source_path) # If client sent an upload id (x_table=x), find it upload = get_upload(self, upload_id=v) source_path = join(upload.folder, data_type.get_file_name()) target_name = argument_noun + get_file_extension(source_path) target_path = move_path(join(draft_folder, target_name), source_path) remove_safely(upload.folder) return target_path
def save(Class, path, table): if path.endswith(".csv"): table.to_csv(path, encoding="utf-8", index=False) elif path.endswith(".msg"): table.to_msgpack(path, compress="blosc") elif path.endswith(".json"): table.to_json(path) elif path.endswith(".xls") or path.endswith(".xlsx"): table.to_excel(path) else: raise DataTypeError("file format not supported (%s)" % get_file_extension(path))
def save(Class, data_folder, owner_id, id_length, source_name, source_x): source_extension = get_file_extension(source_name) source_file = prepare_file(source_x) instance = Class.spawn(data_folder, id_length, owner_id) instance.name = source_name instance.path = join(instance.folder, 'raw' + source_extension) # Save name open(join(instance.folder, 'name.txt'), 'wt').write(source_name) # Save path temporary_path = join(instance.folder, 'temporary.bin') with open(temporary_path, 'wb') as temporary_file: copyfileobj(source_file, temporary_file) rename(temporary_path, instance.path) return instance
def load(Class, path): if not exists(path): raise IOError if path.endswith(".csv"): try: table = pandas.read_csv(path, encoding="utf-8", skipinitialspace=True) except UnicodeDecodeError: encoding = _get_encoding(open(path).read()) table = pandas.read_csv(path, encoding=encoding, skipinitialspace=True) elif path.endswith(".msg"): table = pandas.read_msgpack(path) elif path.endswith(".json"): table = pandas.read_json(path, orient="split") elif path.endswith(".xls") or path.endswith(".xlsx"): table = pandas.read_excel(path) else: raise DataTypeError("file format not supported (%s)" % get_file_extension(path)) return table
def prepare_script_folder(target_folder, notebook, notebook_name): tool_arguments = load_tool_arguments(notebook) # Prepare paths for k, v in tool_arguments.items(): if not k.endswith('_path'): continue path = make_unique_path(target_folder, get_file_extension(v)) shutil.copy(v, path) tool_arguments[k] = basename(path) # Prepare command-line script script_lines = [] script_lines.append('from sys import argv') for i, arg in zip(range(1, len(tool_arguments) + 1), tool_arguments): if arg.endswith('_integer'): script_lines.append('%s = int(argv[%s])' % (arg, i)) else: script_lines.append('%s = argv[%s]' % (arg, i)) #script_lines.append('%s = argv[1:]' % ', '.join(tool_arguments)) notebook.cells[0]['source'] = '\n'.join(script_lines) script_content, script_info = nbconvert.export_script(notebook) script_name = 'run' + script_info['output_extension'] if script_name.endswith('.py'): command_name = 'python' else: raise CrossComputeError # Save script script_path = join(target_folder, script_name) codecs.open(script_path, 'w', encoding='utf-8').write(script_content) # Save configuration configuration_path = join(target_folder, 'cc.ini') configuration_lines = [] configuration_lines.append('[crosscompute %s]' % notebook_name) configuration_lines.append( 'command_template = %s %s %s' % (command_name, script_name, ' '.join('{%s}' % x for x in tool_arguments).strip())) for k, v in tool_arguments.items(): if k in RESERVED_ARGUMENT_NAMES: continue configuration_lines.append('%s = %s' % (k, v)) codecs.open(configuration_path, 'w', encoding='utf-8').write('\n'.join(configuration_lines).strip()) return target_folder
def prepare_script_folder(target_folder, notebook, notebook_name): tool_arguments = load_tool_arguments(notebook) # Prepare paths for k, v in tool_arguments.items(): if not k.endswith("_path"): continue path = make_unique_path(target_folder, get_file_extension(v)) shutil.copy(v, path) tool_arguments[k] = basename(path) # Prepare command-line script script_lines = [] script_lines.append("from sys import argv") script_lines.append("%s = argv[1:]" % ", ".join(tool_arguments)) notebook.cells[0]["source"] = "\n".join(script_lines) script_content, script_info = nbconvert.export_script(notebook) script_name = "run" + script_info["output_extension"] if script_name.endswith(".py"): command_name = "python" else: raise CrossComputeError # Save script script_path = join(target_folder, script_name) codecs.open(script_path, "w", encoding="utf-8").write(script_content) # Save configuration configuration_path = join(target_folder, "cc.ini") configuration_lines = [] configuration_lines.append("[crosscompute %s]" % notebook_name) configuration_lines.append( "command_template = %s %s %s" % (command_name, script_name, " ".join("{%s}" % x for x in tool_arguments).strip()) ) for k, v in tool_arguments.items(): if k in RESERVED_ARGUMENT_NAMES: continue configuration_lines.append("%s = %s" % (k, v)) codecs.open(configuration_path, "w", encoding="utf-8").write("\n".join(configuration_lines).strip()) return target_folder
def test_get_file_extension(): assert get_file_extension('file.txt.zip') == '.txt.zip'
def test_get_file_extension(): assert get_file_extension('file.txt.zip') == '.txt.zip'