def convert(shock_url, shock_id, handle_url, handle_id, input_filename, output_filename, level=logging.INFO, logger=None): """ Converts FASTA file to KBaseAssembly.SingleEndLibrary json string. Args: shock_url: A url for the KBase SHOCK service. handle_url: A url for the KBase Handle Service. shock_id: A KBase SHOCK node id. handle_id: A KBase Handle id. input_filename: A file name for the input FASTA data. output_filename: A file name where the output JSON string should be stored. level: Logging level, defaults to logging.INFO. """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of FASTA to KBaseAssembly.SingleEndLibrary.") token = os.environ.get('KB_AUTH_TOKEN') logger.info("Gathering information.") handles = script_utils.getHandles(logger, shock_url, handle_url, [shock_id], [handle_id], token) assert len(handles) != 0 objectString = json.dumps({"handle" : handles[0]}, sort_keys=True, indent=4) logger.info("Writing out JSON.") with open(args.output_filename, "w") as outFile: outFile.write(objectString) logger.info("Conversion completed.")
def PluginManager(directory=None, logger=script_utils.stderrlogger(__file__)): if directory is None: raise Exception( "Must provide a directory to read plugin configs from!") manager = PlugIns(directory, logger) return manager
def transform(workspace_service_url=None, shock_service_url=None, handle_service_url=None, workspace_name=None, object_name=None, object_id=None, object_version_number=None, working_directory=None, output_file_name=None, level=logging.INFO, logger=None): """ Converts KBaseAssembly.SingleEndLibrary to a Fasta file of assembledDNA. Args: workspace_service_url: A url for the KBase Workspace service shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. workspace_name: Name of the workspace object_name: Name of the object in the workspace object_id: Id of the object in the workspace, mutually exclusive to object_name object_version_number: Version number of workspace object (ContigSet), defaults to most recent version working_directory: The working directory where the output file should be stored. output_file_name: The desired file name of the result file. level: Logging level, defaults to logging.INFO. Returns: A FASTA file containing assembled sequences from a KBase ContigSet object. Authors: Jason Baumohl, Matt Henderson """ def insert_newlines(s, every): lines = [] for i in xrange(0, len(s), every): lines.append(s[i:i+every]) return "\n".join(lines)+"\n" if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of KBaseGenomes.ContigSet to FASTA.DNA.Assembly") token = os.environ.get("KB_AUTH_TOKEN") if not os.path.isdir(args.working_directory): raise Exception("The working directory does not exist {0} does not exist".format(working_directory)) logger.info("Grabbing Data.") try: ws_client = biokbase.workspace.client.Workspace(workspace_service_url) if object_version_number and object_name: contig_set = ws_client.get_objects([{"workspace":workspace_name,"name":object_name, "ver":object_version_number}])[0] elif object_name: contig_set = ws_client.get_objects([{"workspace":workspace_name,"name":object_name}])[0] elif object_version_number and object_id: contig_set = ws_client.get_objects([{"workspace":workspace_name,"objid":object_id, "ver":object_version_number}])[0] else: contig_set = ws_client.get_objects([{"workspace":workspace_name,"objid":object_id}])[0] except Exception, e: logger.exception("Unable to retrieve workspace object from {0}:{1}.".format(workspace_service_url,workspace_name)) logger.exception(e) raise
def main(): script_details = script_utils.parse_docs(transform.__doc__) parser = argparse.ArgumentParser(prog=__file__, description=script_details["Description"], epilog=script_details["Authors"]) parser.add_argument('--workspace_service_url', help=script_details["Args"]["workspace_service_url"], action='store', type=str, nargs='?', required=True) parser.add_argument('--workspace_name', help=script_details["Args"]["workspace_name"], action='store', type=str, nargs='?', required=True) parser.add_argument("--object_name", help=script_details["Args"]["object_name"], action='store', type=str, nargs='?', required=True) parser.add_argument('--output_file_name', help=script_details["Args"]["output_file_name"], action='store', type=str, nargs='?', default=None, required=False) parser.add_argument('--input_directory', help=script_details["Args"]["input_directory"], action='store', type=str, nargs='?', required=True) parser.add_argument("--working_directory", help=script_details["Args"]["working_directory"], action='store', type=str, nargs='?', required=True) parser.add_argument("--has_replicates", help=script_details["Args"]["has_replicates"], action='store', type=str, nargs='?', required=True) parser.add_argument('--input_mapping', help=script_details["Args"]["input_mapping"], action='store', type=unicode, nargs='?', default=None, required=False) # custom arguments specific to this uploader parser.add_argument('--format_type', help=script_details["Args"]["format_type"], action='store', type=str, required=False) args, unknown = parser.parse_known_args() logger = script_utils.stderrlogger(__file__) logger.debug(args) try: transform(workspace_service_url=args.workspace_service_url, workspace_name=args.workspace_name, object_name=args.object_name, output_file_name=args.output_file_name, input_directory=args.input_directory, working_directory=args.working_directory, has_replicates=args.has_replicates, input_mapping=args.input_mapping, format_type=args.format_type, logger=logger) except Exception as e: logger.exception(e) sys.exit(1)
def __init__(self, plugins_directory=None, logger=script_utils.stderrlogger(__file__)): self.scripts_config = {"external_types": list(), "kbase_types": list(), "validate": dict(), "upload": dict(), "download": dict(), "convert": dict()} self.logger = logger plugins = sorted(os.listdir(plugins_directory)) for p in plugins: try: f = open(os.path.join(plugins_directory, p), 'r') pconfig = simplejson.loads(f.read()) f.close() id = None if pconfig["script_type"] == "validate": if pconfig["external_type"] not in self.scripts_config["external_types"]: self.scripts_config["external_types"].append(pconfig["external_type"]) id = pconfig["external_type"] elif pconfig["script_type"] == "upload": if pconfig["external_type"] not in self.scripts_config["external_types"]: self.scripts_config["external_types"].append(pconfig["external_type"]) if pconfig["kbase_type"] not in self.scripts_config["kbase_types"]: self.scripts_config["kbase_types"].append(pconfig["kbase_type"]) id = "{0}=>{1}".format(pconfig["external_type"],pconfig["kbase_type"]) elif pconfig["script_type"] == "download": if pconfig["external_type"] not in self.scripts_config["external_types"]: self.scripts_config["external_types"].append(pconfig["external_type"]) if pconfig["kbase_type"] not in self.scripts_config["kbase_types"]: self.scripts_config["kbase_types"].append(pconfig["kbase_type"]) id = "{0}=>{1}".format(pconfig["kbase_type"],pconfig["external_type"]) elif pconfig["script_type"] == "convert": if pconfig["source_kbase_type"] not in self.scripts_config["kbase_types"]: self.scripts_config["kbase_types"].append(pconfig["source_kbase_type"]) if pconfig["destination_kbase_type"] not in self.scripts_config["kbase_types"]: self.scripts_config["kbase_types"].append(pconfig["destination_kbase_type"]) id = "{0}=>{1}".format(pconfig["source_kbase_type"],pconfig["destination_kbase_type"]) self.scripts_config[pconfig["script_type"]][id] = pconfig self.logger.info("Successfully added plugin {0}".format(p)) except Exception, e: self.logger.warning("Unable to read plugin {0}: {1}".format(p,e.message))
def __init__(self, plugins_directory=None, logger=script_utils.stderrlogger(__file__)): self.scripts_config = {"external_types": list(), "kbase_types": list(), "validate": dict(), "upload": dict(), "download": dict(), "convert": dict()} self.logger = logger plugins = sorted(os.listdir(plugins_directory)) for p in plugins: try: f = open(os.path.join(plugins_directory, p), 'r') pconfig = simplejson.loads(f.read()) f.close() id = None if pconfig["script_type"] == "validate": if pconfig["external_type"] not in self.scripts_config["external_types"]: self.scripts_config["external_types"].append(pconfig["external_type"]) id = pconfig["external_type"] elif pconfig["script_type"] == "upload": if pconfig["external_type"] not in self.scripts_config["external_types"]: self.scripts_config["external_types"].append(pconfig["external_type"]) if pconfig["kbase_type"] not in self.scripts_config["kbase_types"]: self.scripts_config["kbase_types"].append(pconfig["kbase_type"]) id = "{0}=>{1}".format(pconfig["external_type"],pconfig["kbase_type"]) elif pconfig["script_type"] == "download": if pconfig["external_type"] not in self.scripts_config["external_types"]: self.scripts_config["external_types"].append(pconfig["external_type"]) if pconfig["kbase_type"] not in self.scripts_config["kbase_types"]: self.scripts_config["kbase_types"].append(pconfig["kbase_type"]) id = "{0}=>{1}".format(pconfig["kbase_type"],pconfig["external_type"]) elif pconfig["script_type"] == "convert": if pconfig["source_kbase_type"] not in self.scripts_config["kbase_types"]: self.scripts_config["kbase_types"].append(pconfig["source_kbase_type"]) if pconfig["destination_kbase_type"] not in self.scripts_config["kbase_types"]: self.scripts_config["kbase_types"].append(pconfig["destination_kbase_type"]) id = "{0}=>{1}".format(pconfig["source_kbase_type"],pconfig["destination_kbase_type"]) self.scripts_config[pconfig["script_type"]][id] = pconfig self.logger.info("Successfully added plugin {0}".format(p)) except Exception, e: self.logger.warning("Unable to read plugin {0}: {1}".format(p,e.message))
def run_task(logger, arguments, debug=False): """ A factory function to abstract the implementation details of how tasks are run. """ if logger is None: logger = script_utils.stderrlogger(__file__) h = TaskRunner(logger) out = h.run(arguments, debug) return out
def __init__(self, logger=None, callback=None): #logger_stdout = script_utils.getStdoutLogger() if logger is None: self.logger = script_utils.stderrlogger(__file__) else: self.logger = logger if callback is None: self.callback = lambda x: self.logger.info(x) else: self.callback = callback
def __init__(self, logger=None, callback=None): #logger_stdout = script_utils.getStdoutLogger() if logger is None: self.logger = script_utils.stderrlogger(__file__) else: self.logger = logger if callback is None: self.callback = lambda x: self.logger.info(x) else: self.callback = callback
def run_task(logger, arguments, debug=False, callback=None): """ A factory function to abstract the implementation details of how tasks are run. """ if logger is None: logger = script_utils.stderrlogger(__file__) h = TaskRunner(logger, callback=callback) out = h.run(arguments, debug) return out
def validate(input_directory, working_directory, level=logging.INFO, logger=None): """ Validates a FASTA file of nucleotide sequences. Args: input_directory: A directory containing one or more FASTA files. working_directory: A directory where any output files produced by validation can be written. level: Logging level, defaults to logging.INFO. Returns: Currently writes to stderr with a Java Exception trace on error, otherwise no output. Authors: Srividya Ramikrishnan, Matt Henderson """ if logger is None: logger = script_utils.stderrlogger(__file__) extensions = [".fa",".fasta",".fna"] validated = False for input_file_name in os.listdir(input_directory): logger.info("Checking for FASTA file : {0}".format(input_file_name)) filePath = os.path.join(os.path.abspath(input_directory), input_file_name) if not os.path.isfile(filePath): logger.warning("Skipping directory {0}".format(input_file_name)) continue elif os.path.splitext(input_file_name)[-1] not in extensions: logger.warning("Unrecognized file type, skipping.") continue logger.info("Starting FASTA validation of {0}".format(input_file_name)) # TODO This needs to be changed, this is really just a demo program for this library and not a serious tool java_classpath = os.path.join(os.environ.get("KB_TOP"), "lib/jars/FastaValidator/FastaValidator-1.0.jar") arguments = ["java", "-classpath", java_classpath, "FVTester", filePath] tool_process = subprocess.Popen(arguments, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if len(stderr) > 0: logger.error("Validation failed on {0}".format(input_file_name)) else: logger.info("Validation passed on {0}".format(input_file_name)) validated = True if not validated: raise Exception("Validation failed!") else: logger.info("Validation passed.")
def transform(input_file=None, level=logging.INFO, logger=None): """ Validate Genbank file. Args: input_directory: An genbank input file Returns: Any validation errors or success. Authors: Shinjae Yoo, Matt Henderson, Marcin Joachimiak """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting Genbank validation") token = os.environ.get("KB_AUTH_TOKEN") classpath = "/kb/dev_container/modules/transform/lib/jars/kbase/transform/GenBankTransform.jar:$KB_TOP/lib/jars/kbase/genomes/kbase-genomes-20140411.jar:$KB_TOP/lib/jars/kbase/common/kbase-common-0.0.6.jar:$KB_TOP/lib/jars/jackson/jackson-annotations-2.2.3.jar:$KB_TOP/lib/jars/jackson/jackson-core-2.2.3.jar:$KB_TOP/lib/jars/jackson/jackson-databind-2.2.3.jar:$KB_TOP/lib/jars/kbase/transform/GenBankTransform.jar:$KB_TOP/lib/jars/kbase/auth/kbase-auth-1398468950-3552bb2.jar:$KB_TOP/lib/jars/kbase/workspace/WorkspaceClient-0.2.0.jar" mc = 'us.kbase.genbank.ValidateGBK' java_classpath = os.path.join( os.environ.get("KB_TOP"), classpath.replace('$KB_TOP', os.environ.get("KB_TOP"))) argslist = "{0}".format("--input_file {0}".format(input_file)) arguments = [ "java", "-classpath", java_classpath, "us.kbase.genbank.ConvertGBK", argslist ] print arguments tool_process = subprocess.Popen(arguments, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if len(stderr) > 0: logger.error( "Validation of Genbank.Genome failed on {0}".format(input_file)) sys.exit(1) else: logger.info("Validation of Genbank.Genome completed.") sys.exit(0)
def transform(input_file=None, level=logging.INFO, logger=None): """ Validate Genbank file. Args: input_directory: An genbank input file Returns: Any validation errors or success. Authors: Shinjae Yoo, Matt Henderson, Marcin Joachimiak """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting Genbank validation") token = os.environ.get("KB_AUTH_TOKEN") classpath = "/kb/dev_container/modules/transform/lib/jars/kbase/transform/GenBankTransform.jar:$KB_TOP/lib/jars/kbase/genomes/kbase-genomes-20140411.jar:$KB_TOP/lib/jars/kbase/common/kbase-common-0.0.6.jar:$KB_TOP/lib/jars/jackson/jackson-annotations-2.2.3.jar:$KB_TOP/lib/jars/jackson/jackson-core-2.2.3.jar:$KB_TOP/lib/jars/jackson/jackson-databind-2.2.3.jar:$KB_TOP/lib/jars/kbase/transform/GenBankTransform.jar:$KB_TOP/lib/jars/kbase/auth/kbase-auth-1398468950-3552bb2.jar:$KB_TOP/lib/jars/kbase/workspace/WorkspaceClient-0.2.0.jar" mc = 'us.kbase.genbank.ValidateGBK' java_classpath = os.path.join(os.environ.get("KB_TOP"), classpath.replace('$KB_TOP', os.environ.get("KB_TOP"))) argslist = "{0}".format("--input_file {0}".format(input_file)) arguments = ["java", "-classpath", java_classpath, "us.kbase.genbank.ConvertGBK", argslist] print arguments tool_process = subprocess.Popen(arguments, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if len(stderr) > 0: logger.error("Validation of Genbank.Genome failed on {0}".format(input_file)) sys.exit(1) else: logger.info("Validation of Genbank.Genome completed.") sys.exit(0)
def filter_genes(self, ctx, args): # ctx is the context object # return variables are: result #BEGIN filter_genes try: os.makedirs(self.RAWEXPR_DIR) except: pass try: os.makedirs(self.FLTRD_DIR) except: pass try: os.makedirs(self.FINAL_DIR) except: pass if self.logger is None: self.logger = script_utils.stderrlogger(__file__) result = {} self.logger.info("Starting conversion of KBaseFeatureValues.ExpressionMatrix to TSV") token = ctx['token'] eenv = os.environ.copy() eenv['KB_AUTH_TOKEN'] = token param = args from biokbase.workspace.client import Workspace ws = Workspace(url=self.__WS_URL, token=token) expr = ws.get_objects([{'workspace': param['workspace_name'], 'name' : param['object_name']}])[0]['data'] cmd_dowload_cvt_tsv = [self.FVE_2_TSV, '--workspace_service_url', self.__WS_URL, '--workspace_name', param['workspace_name'], '--object_name', param['object_name'], '--working_directory', self.RAWEXPR_DIR, '--output_file_name', self.EXPRESS_FN ] # need shell in this case because the java code is depending on finding the KBase token in the environment # -- copied from FVE_2_TSV tool_process = subprocess.Popen(" ".join(cmd_dowload_cvt_tsv), stderr=subprocess.PIPE, shell=True, env=eenv) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: self.logger.info(stdout) if stderr is not None and len(stderr) > 0: self.logger.info(stderr) self.logger.info("Identifying differentially expressed genes") ## Prepare sample file # detect num of columns with open("{0}/{1}".format(self.RAWEXPR_DIR, self.EXPRESS_FN), 'r') as f: fl = f.readline() ncol = len(fl.split('\t')) # force to use ANOVA if the number of sample is two if(ncol == 3): param['method'] = 'anova' with open("{0}/{1}".format(self.RAWEXPR_DIR, self.SAMPLE_FN), 'wt') as s: s.write("0") for j in range(1,ncol-1): s.write("\t{0}".format(j)) s.write("\n") ## Run coex_filter cmd_coex_filter = [self.COEX_FILTER, '-i', "{0}/{1}".format(self.RAWEXPR_DIR, self.EXPRESS_FN), '-o', "{0}/{1}".format(self.FLTRD_DIR, self.FLTRD_FN), '-m', param['method'], '-s', "{0}/{1}".format(self.RAWEXPR_DIR, self.SAMPLE_FN), '-x', "{0}/{1}".format(self.RAWEXPR_DIR, self.GENELST_FN), '-t', 'y'] if 'num_features' in param: cmd_coex_filter.append("-n") cmd_coex_filter.append(str(param['num_features'])) if 'p_value' in param: cmd_coex_filter.append("-p") cmd_coex_filter.append(str(param['p_value'])) if 'p_value' not in param and 'num_features' not in param: self.logger.error("One of p_value or num_features must be defined"); return empty_results("One of p_value or num_features must be defined", expr,self.__WS_URL, param, self.logger, ws) #sys.exit(2) #TODO: No error handling in narrative so we do graceful termination #if 'p_value' in param and 'num_features' in param: # self.logger.error("Both of p_value and num_features cannot be defined together"); # sys.exit(3) tool_process = subprocess.Popen(cmd_coex_filter, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: self.logger.info(stdout) if stderr is not None and len(stderr) > 0: self.logger.info(stderr) ## Header correction try: with open("{0}/{1}".format(self.FLTRD_DIR, self.FLTRD_FN), 'r') as ff: fe = ff.readlines() with open("{0}/{1}".format(self.FLTRD_DIR, self.FLTRD_FN), 'w') as ff: ff.write(fl) # use original first line that has correct header information fe.pop(0) ff.writelines(fe) except: self.logger.error("Output was not found"); return empty_results("Increase p_value or specify num_features", expr,self.__WS_URL, param, self.logger, ws) ## checking genelist with open("{0}/{1}".format(self.RAWEXPR_DIR, self.GENELST_FN),'r') as glh: gl = glh.readlines() gl = [x.strip('\n') for x in gl] if(len(gl) < 1) : self.logger.error("No genes are selected") return empty_results("Increase p_value or specify num_features", expr,self.__WS_URL, param, self.logger, ws) #sys.exit(4) ## Upload FVE # change workspace to be the referenced object's workspace_name because it may not be in the same working ws due to referencing # Updates: change missing genome handling strategy by copying reference to working workspace cmd_upload_expr = [self.TSV_2_FVE, '--workspace_service_url', self.__WS_URL, '--object_name', param['out_expr_object_name'], '--working_directory', self.FINAL_DIR, '--input_directory', self.FLTRD_DIR, '--output_file_name', self.FINAL_FN ] tmp_ws = param['workspace_name'] if 'genome_ref' in expr: obj_infos = ws.get_object_info_new({"objects": [{'ref':expr['genome_ref']}]})[0] if len(obj_infos) < 1: self.logger.error("Couldn't find {0} from the workspace".format(expr['genome_ref'])) raise Exception("Couldn't find {0} from the workspace".format(expr['genome_ref'])) #tmp_ws = "{0}".format(obj_infos[7]) self.logger.info("{0} => {1} / {2}".format(expr['genome_ref'], obj_infos[7], obj_infos[1])) if obj_infos[7] != param['workspace_name']: #we need to copy it from the other workspace try: self.logger.info("trying to copy the referenced genome object : {0}".format(expr['genome_ref'])) ws.copy_object({'from' : {'ref' : expr['genome_ref']},'to' : {'workspace': param['workspace_name'], 'name' : obj_infos[1]}}) # add genome_object_name only after successful copy cmd_upload_expr.append('--genome_object_name') cmd_upload_expr.append(obj_infos[1]) except: # no permission or any issues... then, give up providing genome reference self.logger.info("".join(traceback.format_exc())) pass else: # it is local... we can simply add reference without copying genome cmd_upload_expr.append('--genome_object_name') cmd_upload_expr.append(obj_infos[1]) # updated ws name cmd_upload_expr.append('--workspace_name') cmd_upload_expr.append(tmp_ws) self.logger.info(" ".join(cmd_upload_expr)) tool_process = subprocess.Popen(" ".join(cmd_upload_expr), stderr=subprocess.PIPE, shell=True, env=eenv) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: self.logger.info(stdout) if stderr is not None and len(stderr) > 0: self.logger.info(stderr) with open("{0}/{1}".format(self.FINAL_DIR,self.FINAL_FN),'r') as et: eo = json.load(et) if 'description' not in expr: expr['description'] = "Filtered Expression Matrix" expr['description'] += " : Filtered by '{1}' method ".format(expr['description'], param['method']) if 'feature_mapping' in expr and 'feature_mapping' in eo: expr['feature_mapping'] = eo['feature_mapping'] expr['data'] = eo['data'] ws.save_objects({'workspace' : param['workspace_name'], 'objects' : [{'type' : 'KBaseFeatureValues.ExpressionMatrix', 'data' : expr, 'name' : (param['out_expr_object_name'])}]}) ## Upload FeatureSet fs ={'elements': {}} fs['description'] = "FeatureSet identified by filtering method '{0}' ".format(param['method']) fs['description'] += "from {0}/{1}".format(param['workspace_name'], param['object_name']) for g in gl: if 'genome_ref' in expr: fs['elements'][g] = [expr['genome_ref']] else: fs['elements'][g] = [] ws.save_objects({'workspace' : param['workspace_name'], 'objects' : [{'type' : 'KBaseCollections.FeatureSet', 'data' : fs, 'name' : (param['out_fs_object_name'])}]}) result = {'workspace_name' : param['workspace_name'], 'out_expr_object_name' : param['out_expr_object_name'], 'out_fs_object_name' : param['out_fs_object_name']} #END filter_genes # At some point might do deeper type checking... if not isinstance(result, dict): raise ValueError('Method filter_genes return value ' + 'result is not type dict as required.') # return the results return [result]
Args: shock_service_url: If you have shock references you need to make. handle_service_url: In case your type has at least one handle reference. working_directory: A directory where you can do work. Returns: JSON representing a KBase object. Authors: Your name here """ # there are utility functions for things you need to do, like log messages if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Python KBase Upload template transform script") # here is how you get the user token to access services token = os.environ.get("KB_AUTH_TOKEN") # stuff happens in here, transform the data # now make your JSON object objectString = json.dumps("{}", sort_keys=True, indent=4) # write it to disk logger.info("Transform completed.")
def transform(workspace_service_url=None, shock_service_url=None, handle_service_url=None, workspace_name=None, object_name=None, object_id=None, object_version_number=None, working_directory=None, output_file_name=None, level=logging.INFO, logger=None): """ Converts KBaseAssembly.SingleEndLibrary to a Fasta file of assembledDNA. Args: workspace_service_url: A url for the KBase Workspace service shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. workspace_name: Name of the workspace object_name: Name of the object in the workspace object_id: Id of the object in the workspace, mutually exclusive to object_name object_version_number: Version number of workspace object (ContigSet), defaults to most recent version working_directory: The working directory where the output file should be stored. output_file_name: The desired file name of the result file. level: Logging level, defaults to logging.INFO. Returns: A FASTA file containing assembled sequences from a KBase ContigSet object. Authors: Jason Baumohl, Matt Henderson """ def insert_newlines(s, every): lines = [] for i in xrange(0, len(s), every): lines.append(s[i:i + every]) return "\n".join(lines) + "\n" if logger is None: logger = script_utils.stderrlogger(__file__) logger.info( "Starting conversion of KBaseGenomes.ContigSet to FASTA.DNA.Assembly") token = os.environ.get("KB_AUTH_TOKEN") if not os.path.isdir(args.working_directory): raise Exception( "The working directory does not exist {0} does not exist".format( working_directory)) logger.info("Grabbing Data.") try: ws_client = biokbase.workspace.client.Workspace(workspace_service_url) if object_version_number and object_name: contig_set = ws_client.get_objects([{ "workspace": workspace_name, "name": object_name, "ver": object_version_number }])[0] elif object_name: contig_set = ws_client.get_objects([{ "workspace": workspace_name, "name": object_name }])[0] elif object_version_number and object_id: contig_set = ws_client.get_objects([{ "workspace": workspace_name, "objid": object_id, "ver": object_version_number }])[0] else: contig_set = ws_client.get_objects([{ "workspace": workspace_name, "objid": object_id }])[0] except Exception, e: logger.exception( "Unable to retrieve workspace object from {0}:{1}.".format( workspace_service_url, workspace_name)) logger.exception(e) raise
def transform(shock_service_url=None, handle_service_url=None, output_file_name=None, input_directory=None, working_directory=None, level=logging.INFO, logger=None): """ Converts a FASTQ file to a KBaseAssembly.SingleEndLibrary json string. Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. output_file_name: A file name where the output JSON string should be stored. input_directory: The directory containing the file. working_directory: The directory the resulting json file will be written to. level: Logging level, defaults to logging.INFO. Returns: JSON file on disk that can be saved as a KBase workspace object. Authors: """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Scanning for FASTQ files.") valid_extensions = [".fq", ".fastq", ".fnq"] files = os.listdir(working_directory) fastq_files = [ x for x in files if os.path.splitext(x)[-1] in valid_extensions ] assert len(fastq_files) != 0 logger.info("Found {0}".format(str(fastq_files))) input_file_name = files[0] if len(fastq_files) > 1: logger.warning( "Not sure how to handle multiple FASTQ files in this context. Using {0}" .format(input_file_name)) kb_token = os.environ.get('KB_AUTH_TOKEN') script_utils.upload_file_to_shock(logger=logger, shock_service_url=shock_service_url, filePath=os.path.join( input_directory, input_file_name), token=kb_token) handles = script_utils.getHandles(logger=logger, shock_service_url=shock_service_url, handle_service_url=handle_service_url, token=kb_token) assert len(handles) != 0 objectString = simplejson.dumps({"handle": handles[0]}, sort_keys=True, indent=4) if output_file_name is None: output_file_name = input_file_name with open(os.path.join(output_directory, output_file_name), "w") as f: f.write(objectString)
def transform(shock_service_url=None, workspace_service_url=None, workspace_name=None, object_name=None, contigset_object_name=None, input_directory=None, working_directory=None, level=logging.INFO, logger=None): """ Transforms Genbank file to KBaseGenomes.Genome and KBaseGenomes.ContigSet objects. Args: shock_service_url: If you have shock references you need to make. workspace_service_url: KBase Workspace URL workspace_name: Name of the workspace to save the data to object_name: Name of the genome object to save contigset_object_name: Name of the ContigSet object that is created with this Genome input_directory: A directory of either a genbank file or a directory of partial genome files to merge working_directory: A directory where you can do work Returns: Workspace objects saved to the user's workspace. Authors: Shinjae Yoo, Marcin Joachimiak, Matt Henderson """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting transformation of Genbank to KBaseGenomes.Genome") classpath = [ "$KB_TOP/lib/jars/kbase/transform/GenBankTransform.jar", "$KB_TOP/lib/jars/kbase/genomes/kbase-genomes-20140411.jar", "$KB_TOP/lib/jars/kbase/common/kbase-common-0.0.6.jar", "$KB_TOP/lib/jars/jackson/jackson-annotations-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-core-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-databind-2.2.3.jar", "$KB_TOP/lib/jars/kbase/transform/GenBankTransform.jar", "$KB_TOP/lib/jars/kbase/auth/kbase-auth-1398468950-3552bb2.jar", "$KB_TOP/lib/jars/kbase/workspace/WorkspaceClient-0.2.0.jar" ] mc = "us.kbase.genbank.ConvertGBK" argslist = [ "--shock_url {0}".format(shock_service_url), "--workspace_service_url {0}".format(workspace_service_url), "--workspace_name {0}".format(workspace_name), "--object_name {0}".format(object_name), "--working_directory {0}".format(working_directory), "--input_directory {0}".format(input_directory) ] if contigset_object_name is not None: argslist.append( "--contigset_object_name {0}".format(contigset_object_name)) arguments = [ "java", "-classpath", ":".join(classpath), "us.kbase.genbank.ConvertGBK", " ".join(argslist) ] logger.debug(arguments) # need shell in this case because the java code is depending on finding the KBase token in the environment tool_process = subprocess.Popen(" ".join(arguments), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.error( "Transformation from Genbank.Genome to KBaseGenomes.Genome failed on {0}" .format(input_directory)) logger.error(stderr) sys.exit(1) logger.info( "Transformation from Genbank.Genome to KBaseGenomes.Genome completed.") sys.exit(0)
def run_filter_genes(workspace_service_url=None, param_file = None, level=logging.INFO, logger = None): """ Narrative Job Wrapper script to execute coex_filter Args: workspace_service_url: A url for the KBase Workspace service param_file: parameter file object_name: Name of the object in the workspace level: Logging level, defaults to logging.INFO. Returns: Output is written back in WS Authors: Shinjae Yoo """ try: os.makedirs(RAWEXPR_DIR) except: pass try: os.makedirs(FLTRD_DIR) except: pass try: os.makedirs(FINAL_DIR) except: pass if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of KBaseFeatureValues.ExpressionMatrix to TSV") token = os.environ.get("KB_AUTH_TOKEN") with open(param_file) as paramh: param = json.load(paramh) cmd_dowload_cvt_tsv = [FVE_2_TSV, '--workspace_service_url', workspace_service_url, '--workspace_name', param['workspace_name'], '--object_name', param['object_name'], '--working_directory', RAWEXPR_DIR, '--output_file_name', EXPRESS_FN ] # need shell in this case because the java code is depending on finding the KBase token in the environment # -- copied from FVE_2_TSV tool_process = subprocess.Popen(" ".join(cmd_dowload_cvt_tsv), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.info(stderr) logger.info("Identifying differentially expressed genes") ## Prepare sample file # detect num of columns with open("{0}/{1}".format(RAWEXPR_DIR, EXPRESS_FN), 'r') as f: fl = f.readline() ncol = len(fl.split('\t')) with open("{0}/{1}".format(RAWEXPR_DIR, SAMPLE_FN), 'wt') as s: s.write("0") for j in range(1,ncol-1): s.write("\t{0}".format(j)) s.write("\n") ## Run coex_filter cmd_coex_filter = [COEX_FILTER, '-i', "{0}/{1}".format(RAWEXPR_DIR, EXPRESS_FN), '-o', "{0}/{1}".format(FLTRD_DIR, FLTRD_FN), '-m', param['method'], '-s', "{0}/{1}".format(RAWEXPR_DIR, SAMPLE_FN), '-x', "{0}/{1}".format(RAWEXPR_DIR, GENELST_FN), '-t', 'y'] if 'num_features' in param: cmd_coex_filter.append("-n") cmd_coex_filter.append(param['num_features']) if 'num_features' not in param and 'p_value' in param: cmd_coex_filter.append("-p") cmd_coex_filter.append(param['p_value']) if 'p_value' not in param and 'num_features' not in param: logger.error("One of p_value or num_features must be defined"); sys.exit(2) #if 'p_value' in param and 'num_features' in param: # logger.error("Both of p_value and num_features cannot be defined together"); # sys.exit(3) tool_process = subprocess.Popen(cmd_coex_filter, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.info(stderr) ## Header correction with open("{0}/{1}".format(FLTRD_DIR, FLTRD_FN), 'r') as ff: fe = ff.readlines() with open("{0}/{1}".format(FLTRD_DIR, FLTRD_FN), 'w') as ff: ff.write(fl) # use original first line that has correct header information fe.pop(0) ff.writelines(fe) ## Upload FVE from biokbase.workspace.client import Workspace ws = Workspace(url=workspace_service_url, token=os.environ['KB_AUTH_TOKEN']) expr = ws.get_objects([{'workspace': param['workspace_name'], 'name' : param['object_name']}])[0]['data'] # change workspace to be the referenced object's workspace_name because it may not be in the same working ws due to referencing cmd_upload_expr = [TSV_2_FVE, '--workspace_service_url', workspace_service_url, '--object_name', param['out_expr_object_name'], '--working_directory', FINAL_DIR, '--input_directory', FLTRD_DIR, '--output_file_name', FINAL_FN ] tmp_ws = param['workspace_name'] if 'genome_ref' in expr: cmd_upload_expr.append('--genome_object_name') obj_infos = ws.get_object_info_new({"objects": [{'ref':expr['genome_ref']}]})[0] if len(obj_infos) < 1: logger.error("Couldn't find {0} from the workspace".format(expr['genome_ref'])) raise Exception("Couldn't find {0} from the workspace".format(expr['genome_ref'])) cmd_upload_expr.append(obj_infos[1]) tmp_ws = obj_infos[7] logger.info("{0} => {1} / {2}".format(expr['genome_ref'], tmp_ws, obj_infos[1])) # updated ws name cmd_upload_expr.append('--workspace_name') cmd_upload_expr.append(tmp_ws) tool_process = subprocess.Popen(" ".join(cmd_upload_expr), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.info(stderr) with open("{0}/{1}".format(FINAL_DIR,FINAL_FN),'r') as et: eo = json.load(et) if 'description' in expr: expr['description'] = "{0}, coex_filter by {1}".format(expr['description'], " ".join(cmd_coex_filter)) if 'feature_mapping' in expr: expr['feature_mapping'] = eo['feature_mapping'] expr['data'] = eo['data'] ws.save_objects({'workspace' : param['workspace_name'], 'objects' : [{'type' : 'KBaseFeatureValues.ExpressionMatrix', 'data' : expr, 'name' : (param['out_expr_object_name'])}]}) ## Upload FeatureSet fs ={'description':'Differentially expressed genes generated by {0}'.format(" ".join(cmd_coex_filter)), 'elements': {}} with open("{0}/{1}".format(RAWEXPR_DIR, GENELST_FN),'r') as glh: gl = glh.readlines() gl = [x.strip('\n') for x in gl] for g in gl: if 'genome_ref' in expr: fs['elements'][g] = [expr['genome_ref']] else: fs['elements'][g] = [] ws.save_objects({'workspace' : param['workspace_name'], 'objects' : [{'type' : 'KBaseCollections.FeatureSet', 'data' : fs, 'name' : (param['out_fs_object_name'])}]})
def convert_to_contigs(shock_service_url, handle_service_url, input_file_name, contigset_id, working_directory, shock_id, handle_id, fasta_reference_only, source, level=logging.INFO, logger=None): """ Converts KBaseFile.AssemblyFile to KBaseGenomes.ContigSet and saves to WS. Note the MD5 for the contig is generated by uppercasing the sequence. The ContigSet MD5 is generated by taking the MD5 of joining the sorted list of individual contig's MD5s with a comma separator Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. input_file_name: A file name for the input FASTA data. contigset_id: The id of the ContigSet. If not specified the name will default to the name of the input file appended with "_contig_set"' working_directory: The directory the resulting json file will be written to. shock_id: Shock id for the fasta file if it already exists in shock handle_id: Handle id for the fasta file if it already exists as a handle fasta_reference_only: Creates a reference to the fasta file in Shock, but does not store the sequences in the workspace object. Not recommended unless the fasta file is larger than 1GB. This is the default behavior for files that large. level: Logging level, defaults to logging.INFO. """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of FASTA to KBaseGenomes.ContigSet") logger.info("Building Object.") if not os.path.isfile(input_file_name): raise Exception( "The input file name {0} is not a file!".format(input_file_name)) # default if not too large contig_set_has_sequences = True if fasta_reference_only: contig_set_has_sequences = False fasta_filesize = os.stat(input_file_name).st_size if fasta_filesize > 1000000000: # Fasta file too large to save sequences into the ContigSet object. contigset_warn = 'The FASTA input file seems to be too large. A ' +\ 'ContigSet object will be created without sequences, but will ' +\ 'contain a reference to the file.' logger.warning(contigset_warn) contig_set_has_sequences = False input_file_handle = open(input_file_name, 'r') fasta_header = None sequence_list = [] fasta_dict = dict() first_header_found = False contig_set_md5_list = [] # Pattern for replacing white space pattern = re.compile(r'\s+') for current_line in input_file_handle: if (current_line[0] == ">"): # found a header line # Wrap up previous fasta sequence if (not sequence_list) and first_header_found: raise Exception( "There is no sequence related to FASTA record: {0}".format( fasta_header)) if not first_header_found: first_header_found = True else: # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence: raise Exception( "There is no sequence related to FASTA record: " + fasta_header) contig_dict = dict() contig_dict["id"] = fasta_header contig_dict["length"] = len(total_sequence) contig_dict["name"] = fasta_header contig_dict["description"] = "Note MD5 is generated from " +\ "uppercasing the sequence" contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"] = contig_md5 contig_set_md5_list.append(contig_md5) if contig_set_has_sequences: contig_dict["sequence"] = total_sequence else: contig_dict["sequence"] = "" fasta_dict[fasta_header] = contig_dict # get set up for next fasta sequence sequence_list = [] fasta_header = current_line.replace('>', '').strip() else: sequence_list.append(current_line) input_file_handle.close() # wrap up last fasta sequence if (not sequence_list) and first_header_found: raise Exception( "There is no sequence related to FASTA record: {0}".format( fasta_header)) elif not first_header_found: raise Exception("There are no contigs in this file") else: # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence: raise Exception("There is no sequence related to FASTA record: " + fasta_header) contig_dict = dict() contig_dict["id"] = fasta_header contig_dict["length"] = len(total_sequence) contig_dict["name"] = fasta_header contig_dict["description"] = "Note MD5 is generated from " +\ "uppercasing the sequence" contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"] = contig_md5 contig_set_md5_list.append(contig_md5) if contig_set_has_sequences: contig_dict["sequence"] = total_sequence else: contig_dict["sequence"] = "" fasta_dict[fasta_header] = contig_dict contig_set_dict = dict() contig_set_dict["md5"] = hashlib.md5(",".join( sorted(contig_set_md5_list))).hexdigest() contig_set_dict["id"] = contigset_id contig_set_dict["name"] = contigset_id s = 'unknown' if source and source['source']: s = source['source'] contig_set_dict["source"] = s sid = os.path.basename(input_file_name) if source and source['source_id']: sid = source['source_id'] contig_set_dict["source_id"] = sid contig_set_dict["contigs"] = [ fasta_dict[x] for x in sorted(fasta_dict.keys()) ] contig_set_dict["fasta_ref"] = shock_id logger.info("Conversion completed.") return contig_set_dict
def const_coex_net_clust(self, ctx, args): # ctx is the context object # return variables are: result #BEGIN const_coex_net_clust try: os.makedirs(self.RAWEXPR_DIR) except: pass try: os.makedirs(self.CLSTR_DIR) except: pass try: os.makedirs(self.FINAL_DIR) except: pass if self.logger is None: self.logger = script_utils.stderrlogger(__file__) result = {} self.logger.info("Starting conversion of KBaseFeatureValues.ExpressionMatrix to TSV") token = ctx['token'] param = args auth_client = _KBaseAuth(self.__AUTH_SERVICE_URL) user_id = auth_client.get_user(token) workspace_name_t = Template(param['workspace_name']) workspace_name = workspace_name_t.substitute(user_id=user_id) provenance = [{}] if 'provenance' in ctx: provenance = ctx['provenance'] provenance[0]['input_ws_objects']=[workspace_name+'/'+param['object_name']] from biokbase.workspace.client import Workspace ws = Workspace(url=self.__WS_URL, token=token) expr = ws.get_objects([{'workspace': workspace_name, 'name' : param['object_name']}])[0]['data'] eenv = os.environ.copy() eenv['KB_AUTH_TOKEN'] = token self._dumpExp2File(expr, self.RAWEXPR_DIR, self.EXPRESS_FN) self.logger.info("Identifying differentially expressed genes") ## Prepare sample file # detect num of columns ncol = len(expr['data']['col_ids']) # grouping information with open("{0}/{1}".format(self.RAWEXPR_DIR, self.SAMPLE_FN), 'wt') as s: s.write("0") for j in range(1,ncol): s.write("\t{0}".format(j)) s.write("\n") ## Run coex_cluster cmd_coex_cluster = [self.COEX_CLUSTER, '-t', 'y', '-i', "{0}/{1}".format(self.RAWEXPR_DIR, self.EXPRESS_FN), '-o', "{0}/{1}".format(self.CLSTR_DIR, self.CLSTR_FN), '-m', "{0}/{1}".format(self.CLSTR_DIR, self.CSTAT_FN) ] for p in ['net_method', 'minRsq', 'maxmediank', 'maxpower', 'clust_method', 'minModuleSize', 'detectCutHeight']: if p in param: cmd_coex_cluster.append("--{0}".format(p)) cmd_coex_cluster.append(str(param[p])) #sys.exit(2) #TODO: No error handling in narrative so we do graceful termination #if 'p_value' in param and 'num_features' in param: # self.logger.error("Both of p_value and num_features cannot be defined together"); # sys.exit(3) tool_process = subprocess.Popen(cmd_coex_cluster, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: self.logger.info(stdout) if stderr is not None and len(stderr) > 0: if re.search(r'^There were \d+ warnings \(use warnings\(\) to see them\)', stderr): self.logger.info(stderr) else: self.logger.error(stderr) raise Exception(stderr) # build index for gene list pos_index ={expr['data']['row_ids'][i]: i for i in range(0, len(expr['data']['row_ids']))} # parse clustering results cid2genelist = {} cid2stat = {} with open("{0}/{1}".format(self.CLSTR_DIR, self.CSTAT_FN),'r') as glh: glh.readline() # skip header for line in glh: cluster, mcor, msec = line.rstrip().replace('"','').split("\t") cid2stat[cluster]= [mcor, msec] with open("{0}/{1}".format(self.CLSTR_DIR, self.CLSTR_FN),'r') as glh: glh.readline() # skip header for line in glh: gene, cluster = line.rstrip().replace('"','').split("\t") if cluster not in cid2genelist: cid2genelist[cluster] = [] cid2genelist[cluster].append(gene) if(len(cid2genelist) < 1) : self.logger.error("Clustering failed") return error_report("Error: No cluster output", expr,self.__WS_URL, workspace_name, provenance, ws) #sys.exit(4) self.logger.info("Uploading the results onto WS") feature_clusters = [] for cluster in cid2genelist: feature_clusters.append( {"meancor": float(cid2stat[cluster][0]), "msec": float(cid2stat[cluster][0]), "id_to_pos" : { gene : pos_index[gene] for gene in cid2genelist[cluster]}}) ## Upload Clusters feature_clusters ={"original_data": "{0}/{1}".format(workspace_name,param['object_name']), "feature_clusters": feature_clusters} cl_info = ws.save_objects({'workspace' : workspace_name, 'objects' : [{'type' : 'KBaseFeatureValues.FeatureClusters', 'data' : feature_clusters, 'name' : (param['out_object_name'])}]})[0] ## Create report object: report = "Clustering expression matrix using WGCNA on {0}".format(param['object_name']) reportObj = { 'objects_created':[ { 'ref':"{0}/{1}/{2}".format(cl_info[6], cl_info[0], cl_info[4]), 'description':'WGCNA FeatureClusters' }], 'text_message':report } # generate a unique name for the Method report reportName = 'WGCNA_Clusters_'+str(hex(uuid.getnode())) report_info = ws.save_objects({ 'id':cl_info[6], 'objects':[ { 'type':'KBaseReport.Report', 'data':reportObj, 'name':reportName, 'meta':{}, 'hidden':1, 'provenance':provenance } ] })[0] result = { "report_name" : reportName,"report_ref" : "{0}/{1}/{2}".format(report_info[6],report_info[0],report_info[4]) } #result = {'workspace_name' : workspace_name, 'out_object_name' : param['out_object_name']} #result = {'workspace' : workspace_name, 'output' : param['out_object_name']} #END const_coex_net_clust # At some point might do deeper type checking... if not isinstance(result, dict): raise ValueError('Method const_coex_net_clust return value ' + 'result is not type dict as required.') # return the results return [result]
def filter_genes(self, ctx, args): # ctx is the context object # return variables are: result #BEGIN filter_genes try: os.makedirs(self.RAWEXPR_DIR) except: pass try: os.makedirs(self.FLTRD_DIR) except: pass try: os.makedirs(self.FINAL_DIR) except: pass if self.logger is None: self.logger = script_utils.stderrlogger(__file__) result = {} self.logger.info("Starting conversion of KBaseFeatureValues.ExpressionMatrix to TSV") token = ctx['token'] eenv = os.environ.copy() eenv['KB_AUTH_TOKEN'] = token param = args auth_client = _KBaseAuth(self.__AUTH_SERVICE_URL) user_id = auth_client.get_user(token) workspace_name_t = Template(param['workspace_name']) workspace_name = workspace_name_t.substitute(user_id=user_id) provenance = [{}] if 'provenance' in ctx: provenance = ctx['provenance'] provenance[0]['input_ws_objects']=[workspace_name+'/'+param['object_name']] from biokbase.workspace.client import Workspace ws = Workspace(url=self.__WS_URL, token=token) expr = ws.get_objects([{'workspace': workspace_name, 'name' : param['object_name']}])[0]['data'] self._dumpExp2File(expr, self.RAWEXPR_DIR, self.EXPRESS_FN) self.logger.info("Identifying differentially expressed genes") ## Prepare sample file # detect num of columns ncol = len(expr['data']['col_ids']) # force to use ANOVA if the number of sample is two if(ncol == 3): param['method'] = 'anova' with open("{0}/{1}".format(self.RAWEXPR_DIR, self.SAMPLE_FN), 'wt') as s: s.write("0") for j in range(1,ncol): s.write("\t{0}".format(j)) s.write("\n") ## Run coex_filter cmd_coex_filter = [self.COEX_FILTER, '-i', "{0}/{1}".format(self.RAWEXPR_DIR, self.EXPRESS_FN), '-o', "{0}/{1}".format(self.FLTRD_DIR, self.FLTRD_FN), '-m', param['method'], '-s', "{0}/{1}".format(self.RAWEXPR_DIR, self.SAMPLE_FN), '-x', "{0}/{1}".format(self.RAWEXPR_DIR, self.GENELST_FN), '-t', 'y'] if 'num_features' in param: cmd_coex_filter.append("-n") cmd_coex_filter.append(str(param['num_features'])) if 'p_value' in param: cmd_coex_filter.append("-p") cmd_coex_filter.append(str(param['p_value'])) if 'p_value' not in param and 'num_features' not in param: self.logger.error("One of p_value or num_features must be defined"); return error_report("One of p_value or num_features must be defined", expr,self.__WS_URL, workspace_name, provenance, ws) #sys.exit(2) #TODO: No error handling in narrative so we do graceful termination #if 'p_value' in param and 'num_features' in param: # self.logger.error("Both of p_value and num_features cannot be defined together"); # sys.exit(3) tool_process = subprocess.Popen(cmd_coex_filter, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: self.logger.info(stdout) if stderr is not None and len(stderr) > 0: self.logger.info(stderr) ## checking genelist with open("{0}/{1}".format(self.RAWEXPR_DIR, self.GENELST_FN),'r') as glh: gl = glh.readlines() gl = [x.strip('\n') for x in gl] if(len(gl) < 1) : self.logger.error("No genes are selected") return error_report("Increase p_value or specify num_features", expr,self.__WS_URL, workspace_name, provenance, ws) #sys.exit(4) ## Upload FVE if 'description' not in expr: expr['description'] = "Filtered Expression Matrix" expr['description'] += " : Filtered by '{1}' method ".format(expr['description'], param['method']) expr = self._subselectExp(expr, gl) ex_info = ws.save_objects({'workspace' : workspace_name, 'objects' : [{'type' : 'KBaseFeatureValues.ExpressionMatrix', 'data' : expr, 'name' : (param['out_expr_object_name'])}]})[0] ## Upload FeatureSet fs ={'elements': {}} fs['description'] = "FeatureSet identified by filtering method '{0}' ".format(param['method']) fs['description'] += "from {0}/{1}".format(workspace_name, param['object_name']) for g in gl: if 'genome_ref' in expr: fs['elements'][g] = [expr['genome_ref']] else: fs['elements'][g] = [] fs_info = ws.save_objects({'workspace' : workspace_name, 'objects' : [{'type' : 'KBaseCollections.FeatureSet', 'data' : fs, 'name' : (param['out_fs_object_name'])}]})[0] ## Create report object: report = "Filtering expression matrix using {0} on {1}".format(param['method'],param['object_name']) reportObj = { 'objects_created':[{ 'ref':"{0}/{1}/{2}".format(fs_info[6], fs_info[0], fs_info[4]), 'description':'Filtered FeatureSet' }, { 'ref':"{0}/{1}/{2}".format(ex_info[6], ex_info[0], ex_info[4]), 'description':'Filetered ExpressionMatrix' }], 'text_message':report } # generate a unique name for the Method report reportName = 'FilterExpression_'+str(hex(uuid.getnode())) report_info = ws.save_objects({ 'id':ex_info[6], 'objects':[ { 'type':'KBaseReport.Report', 'data':reportObj, 'name':reportName, 'meta':{}, 'hidden':1, 'provenance':provenance } ] })[0] result = { "report_name" : reportName,"report_ref" : "{0}/{1}/{2}".format(report_info[6],report_info[0],report_info[4]) } #result = {'workspace_name' : workspace_name, 'out_expr_object_name' : param['out_expr_object_name'], 'out_fs_object_name' : param['out_fs_object_name']} #END filter_genes # At some point might do deeper type checking... if not isinstance(result, dict): raise ValueError('Method filter_genes return value ' + 'result is not type dict as required.') # return the results return [result]
def diff_p_distribution(self, ctx, args): # ctx is the context object # return variables are: result #BEGIN diff_p_distribution try: os.makedirs(self.RAWEXPR_DIR) except: pass try: os.makedirs(self.FLTRD_DIR) except: pass try: os.makedirs(self.FINAL_DIR) except: pass if self.logger is None: self.logger = script_utils.stderrlogger(__file__) result = {} self.logger.info("Starting conversion of KBaseFeatureValues.ExpressionMatrix to TSV") token = ctx['token'] eenv = os.environ.copy() eenv['KB_AUTH_TOKEN'] = token param = args auth_client = _KBaseAuth(self.__AUTH_SERVICE_URL) user_id = auth_client.get_user(token) workspace_name_t = Template(param['workspace_name']) workspace_name = workspace_name_t.substitute(user_id=user_id) from biokbase.workspace.client import Workspace ws = Workspace(url=self.__WS_URL, token=token) expr = ws.get_objects([{'workspace': workspace_name, 'name' : param['object_name']}])[0]['data'] self._dumpExp2File(expr, self.RAWEXPR_DIR, self.EXPRESS_FN) self.logger.info("Identifying differentially expressed genes") ## Prepare sample file # detect num of columns ncol = len(expr['data']['col_ids']) # force to use ANOVA if the number of sample is two if(ncol == 3): param['method'] = 'anova' with open("{0}/{1}".format(self.RAWEXPR_DIR, self.SAMPLE_FN), 'wt') as s: s.write("0") for j in range(1,ncol): s.write("\t{0}".format(j)) s.write("\n") ## Run coex_filter cmd_coex_filter = [self.COEX_FILTER, '-i', "{0}/{1}".format(self.RAWEXPR_DIR, self.EXPRESS_FN), '-o', "{0}/{1}".format(self.FLTRD_DIR, self.FLTRD_FN), '-m', param['method'], '-n', '10', '-s', "{0}/{1}".format(self.RAWEXPR_DIR, self.SAMPLE_FN), '-x', "{0}/{1}".format(self.RAWEXPR_DIR, self.GENELST_FN), '-t', 'y', '-j', self.PVFDT_FN] if 'num_features' in param: cmd_coex_filter.append("-n") cmd_coex_filter.append(str(param['num_features'])) if 'p_value' in param: cmd_coex_filter.append("-p") cmd_coex_filter.append(str(param['p_value'])) tool_process = subprocess.Popen(cmd_coex_filter, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: self.logger.info(stdout) if stderr is not None and len(stderr) > 0: self.logger.info(stderr) ## loading pvalue distribution FDT pvfdt = {'row_labels' :[], 'column_labels' : [], "data" : [[]]}; pvfdt = OrderedDict(pvfdt) with open(self.PVFDT_FN, 'r') as myfile: pvfdt = json.load(myfile) data_obj_name = "{0}.fdt".format(param['out_figure_object_name']) pvfdt['id'] = data_obj_name fig_properties = {"xlabel" : "-log2(p-value)", "ylabel" : "Number of features", "xlog_mode" : "-log2", "ylog_mode" : "none", "title" : "Histogram of P-values", "plot_type" : "histogram"} sstatus = ws.save_objects({'workspace' : workspace_name, 'objects' : [{'type' : 'MAK.FloatDataTable', 'data' : pvfdt, 'name' : data_obj_name}]}) data_ref = "{0}/{1}/{2}".format(sstatus[0][6], sstatus[0][0], sstatus[0][4]) fig_properties['data_ref'] = data_ref sstatus = ws.save_objects({'workspace' : workspace_name, 'objects' : [{'type' : 'CoExpression.FigureProperties', 'data' : fig_properties, 'name' : (param['out_figure_object_name'])}]}) result = fig_properties #END diff_p_distribution # At some point might do deeper type checking... if not isinstance(result, dict): raise ValueError('Method diff_p_distribution return value ' + 'result is not type dict as required.') # return the results return [result]
error_object["status"] = "ERROR : {0}".format(e.message)[:handler_utils.UJS_STATUS_MAX] error_object["error_message"] = traceback.format_exc() handler_utils.report_exception(logger, error_object, cleanup_details) ujs.complete_job(ujs_job_id, kb_token, "Download from {0} failed.".format(workspace_name), traceback.format_exc(), None) sys.exit(1) if __name__ == "__main__": logger = script_utils.stderrlogger(__file__, level=logging.DEBUG) script_details = script_utils.parse_docs(download_taskrunner.__doc__) parser = script_utils.ArgumentParser(description=script_details["Description"], epilog=script_details["Authors"]) # provided by service config parser.add_argument('--workspace_service_url', help=script_details["Args"]["workspace_service_url"], action='store', required=True) parser.add_argument('--ujs_service_url', help=script_details["Args"]["ujs_service_url"], action='store', required=True)
def convert_to_contigs(shock_service_url, handle_service_url, input_file_name, contigset_id, working_directory, shock_id, handle_id, fasta_reference_only, source, level=logging.INFO, logger=None): """ Converts KBaseFile.AssemblyFile to KBaseGenomes.ContigSet and saves to WS. Note the MD5 for the contig is generated by uppercasing the sequence. The ContigSet MD5 is generated by taking the MD5 of joining the sorted list of individual contig's MD5s with a comma separator Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. input_file_name: A file name for the input FASTA data. contigset_id: The id of the ContigSet. If not specified the name will default to the name of the input file appended with "_contig_set"' working_directory: The directory the resulting json file will be written to. shock_id: Shock id for the fasta file if it already exists in shock handle_id: Handle id for the fasta file if it already exists as a handle fasta_reference_only: Creates a reference to the fasta file in Shock, but does not store the sequences in the workspace object. Not recommended unless the fasta file is larger than 1GB. This is the default behavior for files that large. level: Logging level, defaults to logging.INFO. """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of FASTA to KBaseGenomes.ContigSet") logger.info("Building Object.") if not os.path.isfile(input_file_name): raise Exception("The input file name {0} is not a file!".format( input_file_name)) # default if not too large contig_set_has_sequences = True if fasta_reference_only: contig_set_has_sequences = False fasta_filesize = os.stat(input_file_name).st_size if fasta_filesize > 1000000000: # Fasta file too large to save sequences into the ContigSet object. contigset_warn = 'The FASTA input file seems to be too large. A ' +\ 'ContigSet object will be created without sequences, but will ' +\ 'contain a reference to the file.' logger.warning(contigset_warn) contig_set_has_sequences = False input_file_handle = open(input_file_name, 'r') fasta_header = None sequence_list = [] fasta_dict = dict() first_header_found = False contig_set_md5_list = [] # Pattern for replacing white space pattern = re.compile(r'\s+') for current_line in input_file_handle: if (current_line[0] == ">"): # found a header line # Wrap up previous fasta sequence if (not sequence_list) and first_header_found: raise Exception( "There is no sequence related to FASTA record: {0}".format( fasta_header)) if not first_header_found: first_header_found = True else: # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence: raise Exception( "There is no sequence related to FASTA record: " + fasta_header) contig_dict = dict() contig_dict["id"] = fasta_header contig_dict["length"] = len(total_sequence) contig_dict["name"] = fasta_header contig_dict["description"] = "Note MD5 is generated from " +\ "uppercasing the sequence" contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"] = contig_md5 contig_set_md5_list.append(contig_md5) if contig_set_has_sequences: contig_dict["sequence"] = total_sequence else: contig_dict["sequence"] = "" fasta_dict[fasta_header] = contig_dict # get set up for next fasta sequence sequence_list = [] fasta_header = current_line.replace('>', '').strip() else: sequence_list.append(current_line) input_file_handle.close() # wrap up last fasta sequence if (not sequence_list) and first_header_found: raise Exception( "There is no sequence related to FASTA record: {0}".format( fasta_header)) elif not first_header_found: raise Exception("There are no contigs in this file") else: # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence: raise Exception( "There is no sequence related to FASTA record: " + fasta_header) contig_dict = dict() contig_dict["id"] = fasta_header contig_dict["length"] = len(total_sequence) contig_dict["name"] = fasta_header contig_dict["description"] = "Note MD5 is generated from " +\ "uppercasing the sequence" contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"] = contig_md5 contig_set_md5_list.append(contig_md5) if contig_set_has_sequences: contig_dict["sequence"] = total_sequence else: contig_dict["sequence"] = "" fasta_dict[fasta_header] = contig_dict contig_set_dict = dict() contig_set_dict["md5"] = hashlib.md5(",".join(sorted( contig_set_md5_list))).hexdigest() contig_set_dict["id"] = contigset_id contig_set_dict["name"] = contigset_id s = 'unknown' if source and source['source']: s = source['source'] contig_set_dict["source"] = s sid = os.path.basename(input_file_name) if source and source['source_id']: sid = source['source_id'] contig_set_dict["source_id"] = sid contig_set_dict["contigs"] = [fasta_dict[x] for x in sorted( fasta_dict.keys())] contig_set_dict["fasta_ref"] = shock_id logger.info("Conversion completed.") return contig_set_dict
def main(): parser = script_utils.ArgumentParser( prog=SCRIPT_NAME, description='Converts KBaseFile.AssemblyFile to ' + 'KBaseGenomes.ContigSet.', epilog='Authors: Jason Baumohl, Matt Henderson, Gavin Price') # The following 7 arguments should be standard to all uploaders parser.add_argument( '--working_directory', help='Directory for temporary files', action='store', type=str, required=True) # Example of a custom argument specific to this uploader parser.add_argument('--workspace_service_url', help='workspace service url', action='store', type=str, required=True) parser.add_argument( '--source_workspace_name', help='name of the source workspace', action='store', type=str, required=True) parser.add_argument( '--destination_workspace_name', help='name of the target workspace', action='store', type=str, required=True) parser.add_argument( '--source_object_name', help='name of the workspace object to convert', action='store', type=str, required=True) parser.add_argument( '--destination_object_name', help='name for the produced ContigSet.', action='store', type=str, required=True) parser.add_argument( '--fasta_reference_only', help='Creates a reference to the fasta file in Shock, but does not ' + 'store the sequences in the workspace object. Not recommended ' + 'unless the fasta file is larger than 1GB. This is the default ' + 'behavior for files that large.', action='store_true', required=False) # ignore unknown arguments for now args, _ = parser.parse_known_args() logger = script_utils.stderrlogger(__file__) try: # make there's at least something for a token if not TOKEN: raise Exception("Unable to retrieve KBase Authentication token!") shock_url, shock_id, ref, source = download_workspace_data( args.workspace_service_url, args.source_workspace_name, args.source_object_name, args.working_directory, logger) inputfile = os.path.join(args.working_directory, args.source_object_name) cs = convert_to_contigs( None, None, inputfile, args.destination_object_name, args.working_directory, shock_id, None, args.fasta_reference_only, source, logger=logger) upload_workspace_data( cs, args.workspace_service_url, ref, args.destination_workspace_name, args.destination_object_name) except Exception, e: logger.exception(e) sys.exit(1)
def validate(input_directory, working_directory, level=logging.INFO, logger=None): """ Validates a FASTA file of nucleotide sequences. Args: input_directory: A directory containing one or more FASTA files. working_directory: A directory where any output files produced by validation can be written. level: Logging level, defaults to logging.INFO. Returns: Currently writes to stderr with a Java Exception trace on error, otherwise no output. Authors: Srividya Ramikrishnan, Matt Henderson """ if logger is None: logger = script_utils.stderrlogger(__file__) extensions = [".fa", ".fasta", ".fna"] validated = False for input_file_name in os.listdir(input_directory): logger.info("Checking for FASTA file : {0}".format(input_file_name)) filePath = os.path.join(os.path.abspath(input_directory), input_file_name) if not os.path.isfile(filePath): logger.warning("Skipping directory {0}".format(input_file_name)) continue elif os.path.splitext(input_file_name)[-1] not in extensions: logger.warning("Unrecognized file type, skipping.") continue logger.info("Starting FASTA validation of {0}".format(input_file_name)) # TODO This needs to be changed, this is really just a demo program for this library and not a serious tool java_classpath = os.path.join( os.environ.get("KB_TOP"), "lib/jars/FastaValidator/FastaValidator-1.0.jar") arguments = [ "java", "-classpath", java_classpath, "FVTester", filePath ] tool_process = subprocess.Popen(arguments, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if len(stderr) > 0: logger.error("Validation failed on {0}".format(input_file_name)) else: logger.info("Validation passed on {0}".format(input_file_name)) validated = True if not validated: raise Exception("Validation failed!") else: logger.info("Validation passed.")
def view_heatmap(self, ctx, args): # ctx is the context object # return variables are: result #BEGIN view_heatmap try: os.makedirs(self.RAWEXPR_DIR) except: pass try: os.makedirs(self.FLTRD_DIR) except: pass try: os.makedirs(self.FINAL_DIR) except: pass if self.logger is None: self.logger = script_utils.stderrlogger(__file__) result = {} self.logger.info("Loading data") token = ctx['token'] eenv = os.environ.copy() eenv['KB_AUTH_TOKEN'] = token param = args from biokbase.workspace.client import Workspace ws = Workspace(url=self.__WS_URL, token=token) fc = ws.get_objects([{'workspace': param['workspace_name'], 'name' : param['object_name']}])[0]['data'] if 'original_data' not in fc: raise Exception("FeatureCluster object does not have information for the original ExpressionMatrix") oexpr = ws.get_objects([{ 'ref' : fc['original_data']}])[0] df2 = pd.DataFrame(oexpr['data']['data']['values'], index=oexpr['data']['data']['row_ids'], columns=oexpr['data']['data']['col_ids']) # cmd_dowload_cvt_tsv = [self.FVE_2_TSV, '--workspace_service_url', self.__WS_URL, # '--workspace_name', oexpr['info'][7], # '--object_name', oexpr['info'][1], # '--working_directory', self.RAWEXPR_DIR, # '--output_file_name', self.EXPRESS_FN # ] # # # need shell in this case because the java code is depending on finding the KBase token in the environment # # -- copied from FVE_2_TSV # tool_process = subprocess.Popen(" ".join(cmd_dowload_cvt_tsv), stderr=subprocess.PIPE, shell=True, env=eenv) # stdout, stderr = tool_process.communicate() # # if stdout is not None and len(stdout) > 0: # self.logger.info(stdout) # # if stderr is not None and len(stderr) > 0: # self.logger.info(stderr) # # df = pd.read_csv("{0}/{1}".format(self.RAWEXPR_DIR,self.EXPRESS_FN), sep='\t') # df2 = df[df.columns[1:]] # rn = df[df.columns[0]] # df2.index = rn # L2 normalization df3 = df2.div(df2.pow(2).sum(axis=1).pow(0.5), axis=0) # type - ? level, ratio, log-ratio <---> "untransformed" # scale - ? probably: raw, ln, log2, log10 self.logger.info("Expression matrix type: {0}, scale: {1}".format(oexpr['data']['type'],oexpr['data']['scale'] )) if oexpr['data']['type'] == 'level' or oexpr['data']['type'] == 'untransformed': # need to compute fold changes if 'scale' not in oexpr['data'] or oexpr['data']['scale'] == 'raw' or oexpr['data']['scale'] == "1.0": factor = 0.125 fc_df = df2 + df2[df2 !=0].abs().min().min() * factor if param['control_condition'] in fc_df.columns: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[param['control_condition']]], axis=0)).apply(np.log2) else: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[0]], axis=0)).apply(np.log2) else: fc_df = df2 if param['control_condition'] in fc_df.columns: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[param['control_condition']]], axis=0)) else: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[0]], axis=0)) if oexpr['data']['scale'] == "log10": fc_df = fc_df/np.log10(2) elif oexpr['data']['scale'] == "ln": fc_df = fc_df/np.log(2) else: pass elif oexpr['data']['type'] == 'ratio': fc_cf = df2.apply(np.log2) elif oexpr['data']['type'] == 'log-ratio': fc_cf = df2 if oexpr['data']['scale'] == "log10": fc_df = fc_df/np.log10(2) elif oexpr['data']['scale'] == "ln": fc_df = fc_df/np.log(2) else: pass else: # do the same thing with simple level or untransformed if 'scale' not in oexpr['data'] or oexpr['data']['scale'] == 'raw' or oexpr['data']['scale'] == "1.0": factor = 0.125 fc_df = df2 + df2[df2 !=0].abs().min().min() * factor if param['control_condition'] in fc_df.columns: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[param['control_condition']]], axis=0)).apply(np.log2) else: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[0]], axis=0)).apply(np.log2) else: fc_df = df2 if param['control_condition'] in fc_df.columns: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[param['control_condition']]], axis=0)) else: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[0]], axis=0)) if oexpr['data']['scale'] == "log10": fc_df = fc_df/np.log10(2) elif oexpr['data']['scale'] == "ln": fc_df = fc_df/np.log(2) else: pass self.logger.info("Compute cluster statistics") cl = {} afs = []; cid = 1; c_stat = pd.DataFrame() for cluster in fc['feature_clusters']: try: fs = cluster['id_to_pos'].keys() except: continue # couldn't find feature_set fsn = "Cluster_{0}".format(cid) cid +=1 c_stat.loc[fsn,'size'] = len(fs) if 'meancor' in cluster: c_stat.loc[fsn,'mcor'] = cluster['meancor'] else: pass # TODO: Add mean cor calculation later #raise Exception("Mean correlation is not included in FeatureCluster object") # now it is NaN if 'quantile' in param: c_stat.loc[fsn,'stdstat'] = fc_df.loc[fs,].std(axis=1).quantile(float(param['quantile'])) else: c_stat.loc[fsn,'stdstat'] = fc_df.loc[fs,].std(axis=1).quantile(0.75) c1 = df3.loc[fs,].sum(axis=0) if df3.loc[fs,].shape[0] < 1: # empty continue cl[fsn] = fs #afs.extend(fs) #c1 = df3.loc[fs,].sum(axis=0) #c1 = c1 / np.sqrt(c1.pow(2).sum()) #if(len(cl.keys()) == 1): # centroids = c1.to_frame(fsn).T #else: # centroids.loc[fsn] = c1 # now we have centroids and statistics # let's subselect clusters min_features = 200 if 'min_features' in param : min_features = param['min_features'] c_stat.loc[:,'nmcor'] = c_stat.loc[:,'mcor'] / c_stat.loc[:,'mcor'].max() c_stat.loc[:,'nstdstat'] = c_stat.loc[:,'stdstat'] / c_stat.loc[:,'stdstat'].max() if 'use_norm_weight' in param and param['use_norm_weight'] != 0: if 'quantile_weight' in param: c_stat.loc[:,'weight'] = c_stat.loc[:,'nmcor'] + float(param['quantile_weight']) * c_stat.loc[:,'nstdstat'] else: c_stat.loc[:,'weight'] = c_stat.loc[:,'nmcor'] + 1.0 * c_stat.loc[:,'nstdstat'] else: if 'quantile_weight' in param: c_stat.loc[:,'weight'] = c_stat.loc[:,'mcor'] + float(param['quantile_weight']) * c_stat.loc[:,'stdstat'] else: c_stat.loc[:,'weight'] = c_stat.loc[:,'mcor'] + 0.1 * c_stat.loc[:,'stdstat'] c_stat.sort_values('weight', inplace=True, ascending=False) pprint(c_stat) centroids = pd.DataFrame() for i in range(c_stat.shape[0]): fsn = c_stat.index[i] fs = cl[fsn] if i != 0 and len(afs) + len(fs) > min_features : break; afs.extend(fs) c1 = df3.loc[fs,].sum(axis=0) c1 = c1 / np.sqrt(c1.pow(2).sum()) if(centroids.shape[0] < 1): centroids = c1.to_frame(fsn).T else: centroids.loc[fsn] = c1 pprint(centroids) if len(cl.keys()) == 0: raise Exception("No feature ids were mapped to dataset or no clusters were selected") # dataset centroid dc = df3.loc[afs,].sum(axis=0) dc = dc / np.sqrt(dc.pow(2).sum()) self.logger.info("Ordering Centroids and Data") # the most far away cluster centroid from dataset centroid fc = (centroids * dc).sum(axis=1).idxmin() # the most far away centroid centroid from fc ffc = (centroids * centroids.loc[fc,]).sum(axis=1).idxmin() # major direction to order on unit ball space md = centroids.loc[ffc,] - centroids.loc[fc,] # unnormalized component of projection to the major direction (ignored md quantities because it is the same to all) corder = (centroids * md).sum(axis=1).sort_values() # cluster order coidx = corder.index dorder =(df3.loc[afs,] * md).sum(axis=1).sort_values() # data order # get first fs table fig_properties = {"xlabel" : "Conditions", "ylabel" : "Features", "xlog_mode" : "none", "ylog_mode" : "none", "title" : "Log Fold Changes", "plot_type" : "heatmap", 'ygroup': []} fig_properties['ygtick_labels'] = coidx.tolist() if 'fold_change' in param and param['fold_change'] == 1: frange = 2 if 'fold_change_range' in param: frange = float(param['fold_change_range']) final=fc_df.loc[dorder.loc[cl[coidx[0]],].index,] fig_properties['ygroup'].append(final.shape[0]) for i in range(1,len(coidx)): tf = fc_df.loc[dorder.loc[cl[coidx[i]],].index,] fig_properties['ygroup'].append(tf.shape[0]) final = final.append(tf) if 'fold_cutoff' in param and param['fold_cutoff'] == 1: final[final > frange] = frange final[final < - frange] = - frange else: fc_df0b = final.sub(final.min(axis=1), axis=0) final = (fc_df0b.div(fc_df0b.max(axis=1), axis=0) - 0.5) * 2 * frange else: final=df2.loc[dorder.loc[cl[coidx[0]],].index,] fig_properties['ygroup'].append(final.shape[0]) for i in range(1,len(coidx)): tf = df2.loc[dorder.loc[cl[coidx[i]],].index,] fig_properties['ygroup'].append(tf.shape[0]) final = final.append(tf) ## loading pvalue distribution FDT fdt = {'row_labels' :[], 'column_labels' : [], "data" : [[]]}; #fdt = OrderedDict(fdt) fdt['data'] = final.T.as_matrix().tolist() # make sure Transpose fdt['row_labels'] = final.columns.tolist() fdt['column_labels'] = final.index.tolist() # TODO: Add group label later fdt['id'] = "{0}.fdt".format(param['out_figure_object_name']) self.logger.info("Saving the results") sstatus = ws.save_objects({'workspace' : param['workspace_name'], 'objects' : [{'type' : 'MAK.FloatDataTable', 'data' : fdt, 'name' : "{0}.fdt".format(param['out_figure_object_name'])}]}) data_ref = "{0}/{1}/{2}".format(sstatus[0][6], sstatus[0][0], sstatus[0][4]) fig_properties['data_ref'] = data_ref sstatus = ws.save_objects({'workspace' : param['workspace_name'], 'objects' : [{'type' : 'CoExpression.FigureProperties', 'data' : fig_properties, 'name' : (param['out_figure_object_name'])}]}) result = fig_properties #END view_heatmap # At some point might do deeper type checking... if not isinstance(result, dict): raise ValueError('Method view_heatmap return value ' + 'result is not type dict as required.') # return the results return [result]
def transform(shock_service_url=None, handle_service_url=None, output_file_name=None, input_directory=None, working_directory=None, shock_id=None, handle_id=None, input_mapping=None, fasta_reference_only=False, level=logging.INFO, logger=None): """ Converts FASTA file to KBaseGenomes.ContigSet json string. Note the MD5 for the contig is generated by uppercasing the sequence. The ContigSet MD5 is generated by taking the MD5 of joining the sorted list of individual contig's MD5s with a comma separator. Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. output_file_name: A file name where the output JSON string should be stored. If the output file name is not specified the name will default to the name of the input file appended with '_contig_set' input_directory: The directory where files will be read from. working_directory: The directory the resulting json file will be written to. shock_id: Shock id for the fasta file if it already exists in shock handle_id: Handle id for the fasta file if it already exists as a handle input_mapping: JSON string mapping of input files to expected types. If you don't get this you need to scan the input directory and look for your files. fasta_reference_only: Creates a reference to the fasta file in Shock, but does not store the sequences in the workspace object. Not recommended unless the fasta file is larger than 1GB. This is the default behavior for files that large. level: Logging level, defaults to logging.INFO. Returns: JSON file on disk that can be saved as a KBase workspace object. Authors: Jason Baumohl, Matt Henderson """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of FASTA to KBaseGenomes.ContigSet") token = os.environ.get('KB_AUTH_TOKEN') if input_mapping is None: logger.info("Scanning for FASTA files.") valid_extensions = [".fa",".fasta",".fna"] files = os.listdir(input_directory) fasta_files = [x for x in files if os.path.splitext(x)[-1] in valid_extensions] assert len(fasta_files) != 0 logger.info("Found {0}".format(str(fasta_files))) input_file_name = os.path.join(input_directory,files[0]) if len(fasta_files) > 1: logger.warning("Not sure how to handle multiple FASTA files in this context. Using {0}".format(input_file_name)) else: input_file_name = os.path.join(os.path.join(input_directory, "FASTA.DNA.Assembly"), simplejson.loads(input_mapping)["FASTA.DNA.Assembly"]) logger.info("Building Object.") if not os.path.isfile(input_file_name): raise Exception("The input file name {0} is not a file!".format(input_file_name)) if not os.path.isdir(args.working_directory): raise Exception("The working directory {0} is not a valid directory!".format(working_directory)) logger.debug(fasta_reference_only) # default if not too large contig_set_has_sequences = True if fasta_reference_only: contig_set_has_sequences = False fasta_filesize = os.stat(input_file_name).st_size if fasta_filesize > 1000000000: # Fasta file too large to save sequences into the ContigSet object. contigset_warn = """The FASTA input file seems to be too large. A ContigSet object will be created without sequences, but will contain a reference to the file.""" logger.warning(contigset_warn) contig_set_has_sequences = False input_file_handle = open(input_file_name, 'r') fasta_header = None sequence_list = [] fasta_dict = dict() first_header_found = False contig_set_md5_list = [] # Pattern for replacing white space pattern = re.compile(r'\s+') sequence_exists = False valid_chars = "-AaCcGgTtUuWwSsMmKkRrYyBbDdHhVvNn" amino_acid_specific_characters = "PpLlIiFfQqEe" for current_line in input_file_handle: if (current_line[0] == ">"): # found a header line # Wrap up previous fasta sequence if (not sequence_exists) and first_header_found: logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) if not first_header_found: first_header_found = True else: # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence : logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) for character in total_sequence: if character not in valid_chars: if character in amino_acid_specific_characters: raise Exception("This fasta file may have amino acids in it instead of the required nucleotides.") raise Exception("This FASTA file has non nucleic acid characters : {0}".format(character)) # fasta_key = fasta_header.strip() try: fasta_key , fasta_description = fasta_header.strip().split(' ',1) except: fasta_key = fasta_header.strip() fasta_description = None contig_dict = dict() contig_dict["id"] = fasta_key contig_dict["length"] = len(total_sequence) contig_dict["name"] = fasta_key if fasta_description is None: contig_dict["description"] = "Note MD5 is generated from uppercasing the sequence" else: contig_dict["description"] = "%s. Note MD5 is generated from uppercasing the sequence" % (fasta_description) contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"] = contig_md5 contig_set_md5_list.append(contig_md5) if contig_set_has_sequences: contig_dict["sequence"]= total_sequence else: contig_dict["sequence"]= "" fasta_dict[fasta_key] = contig_dict # get set up for next fasta sequence sequence_list = [] sequence_exists = False fasta_header = current_line.replace('>','') else: sequence_list.append(current_line) sequence_exists = True input_file_handle.close() # wrap up last fasta sequence if (not sequence_exists) and first_header_found: logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) elif not first_header_found : logger.error("There are no contigs in this file") raise Exception("There are no contigs in this file") else: # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence : logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) for character in total_sequence: if character not in valid_chars: if character in amino_acid_specific_characters: raise Exception("This fasta file may have amino acids in it instead of the required nucleotides.") raise Exception("This FASTA file has non nucleic acid characters : {0}".format(character)) # fasta_key = fasta_header.strip() try: fasta_key , fasta_description = fasta_header.strip().split(' ',1) except: fasta_key = fasta_header.strip() fasta_description = None contig_dict = dict() contig_dict["id"] = fasta_key contig_dict["length"] = len(total_sequence) contig_dict["name"] = fasta_key if fasta_description is None: contig_dict["description"] = "Note MD5 is generated from uppercasing the sequence" else: contig_dict["description"] = "%s. Note MD5 is generated from uppercasing the sequence" % (fasta_description) contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"]= contig_md5 contig_set_md5_list.append(contig_md5) if contig_set_has_sequences: contig_dict["sequence"] = total_sequence else: contig_dict["sequence"]= "" fasta_dict[fasta_key] = contig_dict if output_file_name is None: # default to input file name minus file extenstion adding "_contig_set" to the end base = os.path.basename(input_file_name) output_file_name = "{0}_contig_set.json".format(os.path.splitext(base)[0]) contig_set_dict = dict() contig_set_dict["md5"] = hashlib.md5(",".join(sorted(contig_set_md5_list))).hexdigest() contig_set_dict["id"] = output_file_name contig_set_dict["name"] = output_file_name contig_set_dict["source"] = "KBase" contig_set_dict["source_id"] = os.path.basename(input_file_name) contig_set_dict["contigs"] = [fasta_dict[x] for x in sorted(fasta_dict.keys())] if shock_id is None: shock_info = script_utils.upload_file_to_shock(logger, shock_service_url, input_file_name, token=token) shock_id = shock_info["id"] contig_set_dict["fasta_ref"] = shock_id # For future development if the type is updated to the handle_reference instead of a shock_reference # This generates the json for the object objectString = simplejson.dumps(contig_set_dict, sort_keys=True, indent=4) logger.info("ContigSet data structure creation completed. Writing out JSON.") output_file_path = os.path.join(working_directory,output_file_name) with open(output_file_path, "w") as outFile: outFile.write(objectString) logger.info("Conversion completed.")
def view_heatmap(self, ctx, args): # ctx is the context object # return variables are: result #BEGIN view_heatmap try: os.makedirs(self.RAWEXPR_DIR) except: pass try: os.makedirs(self.FLTRD_DIR) except: pass try: os.makedirs(self.FINAL_DIR) except: pass if self.logger is None: self.logger = script_utils.stderrlogger(__file__) result = {} self.logger.info("Loading data") token = ctx['token'] eenv = os.environ.copy() eenv['KB_AUTH_TOKEN'] = token param = args auth_client = _KBaseAuth(self.__AUTH_SERVICE_URL) user_id = auth_client.get_user(token) workspace_name_t = Template(param['workspace_name']) workspace_name = workspace_name_t.substitute(user_id=user_id) from biokbase.workspace.client import Workspace ws = Workspace(url=self.__WS_URL, token=token) fc = ws.get_objects([{'workspace': workspace_name, 'name' : param['object_name']}])[0]['data'] if 'original_data' not in fc: raise Exception("FeatureCluster object does not have information for the original ExpressionMatrix") oexpr = ws.get_objects([{ 'ref' : fc['original_data']}])[0] df2 = pd.DataFrame(oexpr['data']['data']['values'], index=oexpr['data']['data']['row_ids'], columns=oexpr['data']['data']['col_ids']) # L2 normalization df3 = df2.div(df2.pow(2).sum(axis=1).pow(0.5), axis=0) # type - ? level, ratio, log-ratio <---> "untransformed" # scale - ? probably: raw, ln, log2, log10 self.logger.info("Expression matrix type: {0}, scale: {1}".format(oexpr['data']['type'],oexpr['data']['scale'] )) # do default behavior factor = 0.125 fc_df = df2 + df2[df2 !=0].abs().min().min() * factor if param['control_condition'] in fc_df.columns: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[param['control_condition']]], axis=0)).apply(np.log2) else: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[0]], axis=0)).apply(np.log2) # now fc_df will be reset if oexpr['data']['type'] == 'level' or oexpr['data']['type'] == 'untransformed': # need to compute fold changes if 'scale' not in oexpr['data'] or oexpr['data']['scale'] == 'raw' or oexpr['data']['scale'] == "1.0": factor = 0.125 fc_df = df2 + df2[df2 !=0].abs().min().min() * factor if param['control_condition'] in fc_df.columns: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[param['control_condition']]], axis=0)).apply(np.log2) else: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[0]], axis=0)).apply(np.log2) else: fc_df = df2 if param['control_condition'] in fc_df.columns: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[param['control_condition']]], axis=0)) else: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[0]], axis=0)) if oexpr['data']['scale'] == "log10": fc_df = fc_df/np.log10(2) elif oexpr['data']['scale'] == "ln": fc_df = fc_df/np.log(2) else: pass elif oexpr['data']['type'] == 'ratio': fc_df = df2.apply(np.log2) elif oexpr['data']['type'] == 'log-ratio': fc_df = df2 if oexpr['data']['scale'] == "log10": fc_df = fc_df/np.log10(2) elif oexpr['data']['scale'] == "ln": fc_df = fc_df/np.log(2) else: pass else: # do the same thing with simple level or untransformed if 'scale' not in oexpr['data'] or oexpr['data']['scale'] == 'raw' or oexpr['data']['scale'] == "1.0": factor = 0.125 fc_df = df2 + df2[df2 !=0].abs().min().min() * factor if param['control_condition'] in fc_df.columns: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[param['control_condition']]], axis=0)).apply(np.log2) else: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[0]], axis=0)).apply(np.log2) else: fc_df = df2 if param['control_condition'] in fc_df.columns: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[param['control_condition']]], axis=0)) else: fc_df = (fc_df.div(fc_df.loc[:,fc_df.columns[0]], axis=0)) if oexpr['data']['scale'] == "log10": fc_df = fc_df/np.log10(2) elif oexpr['data']['scale'] == "ln": fc_df = fc_df/np.log(2) else: pass self.logger.info("Compute cluster statistics") cl = {} afs = []; cid = 1; c_stat = pd.DataFrame() for cluster in fc['feature_clusters']: try: fs = cluster['id_to_pos'].keys() except: continue # couldn't find feature_set fsn = "Cluster_{0}".format(cid) cid +=1 c_stat.loc[fsn,'size'] = len(fs) if 'meancor' in cluster: c_stat.loc[fsn,'mcor'] = cluster['meancor'] else: pass # TODO: Add mean cor calculation later #raise Exception("Mean correlation is not included in FeatureCluster object") # now it is NaN if 'quantile' in param: # enforcing quantile to be in [0 .. 1] rnage qt = float(param['quantile']) if qt > 1.0: qt = 1.0 if qt < 0.0: qt = 0.0 c_stat.loc[fsn,'stdstat'] = fc_df.loc[fs,].std(axis=1).quantile(qt) else: c_stat.loc[fsn,'stdstat'] = fc_df.loc[fs,].std(axis=1).quantile(0.75) c1 = df3.loc[fs,].sum(axis=0) if df3.loc[fs,].shape[0] < 1: # empty continue cl[fsn] = fs #afs.extend(fs) #c1 = df3.loc[fs,].sum(axis=0) #c1 = c1 / np.sqrt(c1.pow(2).sum()) #if(len(cl.keys()) == 1): # centroids = c1.to_frame(fsn).T #else: # centroids.loc[fsn] = c1 # now we have centroids and statistics # let's subselect clusters min_features = 200 if 'min_features' in param : min_features = param['min_features'] c_stat.loc[:,'nmcor'] = c_stat.loc[:,'mcor'] / c_stat.loc[:,'mcor'].max() c_stat.loc[:,'nstdstat'] = c_stat.loc[:,'stdstat'] / c_stat.loc[:,'stdstat'].max() if 'use_norm_weight' in param and param['use_norm_weight'] != 0: if 'quantile_weight' in param: c_stat.loc[:,'weight'] = c_stat.loc[:,'nmcor'] + float(param['quantile_weight']) * c_stat.loc[:,'nstdstat'] else: c_stat.loc[:,'weight'] = c_stat.loc[:,'nmcor'] + 1.0 * c_stat.loc[:,'nstdstat'] else: if 'quantile_weight' in param: c_stat.loc[:,'weight'] = c_stat.loc[:,'mcor'] + float(param['quantile_weight']) * c_stat.loc[:,'stdstat'] else: c_stat.loc[:,'weight'] = c_stat.loc[:,'mcor'] + 0.1 * c_stat.loc[:,'stdstat'] c_stat.sort_values('weight', inplace=True, ascending=False) pprint(c_stat) centroids = pd.DataFrame() for i in range(c_stat.shape[0]): fsn = c_stat.index[i] fs = cl[fsn] if i != 0 and len(afs) + len(fs) > min_features : break; afs.extend(fs) c1 = df3.loc[fs,].sum(axis=0) c1 = c1 / np.sqrt(c1.pow(2).sum()) if(centroids.shape[0] < 1): centroids = c1.to_frame(fsn).T else: centroids.loc[fsn] = c1 pprint(centroids) if len(cl.keys()) == 0: raise Exception("No feature ids were mapped to dataset or no clusters were selected") # dataset centroid dc = df3.loc[afs,].sum(axis=0) dc = dc / np.sqrt(dc.pow(2).sum()) self.logger.info("Ordering Centroids and Data") # the most far away cluster centroid from dataset centroid fc = (centroids * dc).sum(axis=1).idxmin() # the most far away centroid centroid from fc ffc = (centroids * centroids.loc[fc,]).sum(axis=1).idxmin() # major direction to order on unit ball space md = centroids.loc[ffc,] - centroids.loc[fc,] # unnormalized component of projection to the major direction (ignored md quantities because it is the same to all) corder = (centroids * md).sum(axis=1).sort_values() # cluster order coidx = corder.index dorder =(df3.loc[afs,] * md).sum(axis=1).sort_values() # data order # get first fs table fig_properties = {"xlabel" : "Conditions", "ylabel" : "Features", "xlog_mode" : "none", "ylog_mode" : "none", "title" : "Log Fold Changes", "plot_type" : "heatmap", 'ygroup': []} fig_properties['ygtick_labels'] = coidx.tolist() if 'fold_change' in param and param['fold_change'] == 1: frange = 2 if 'fold_change_range' in param: frange = float(param['fold_change_range']) final=fc_df.loc[dorder.loc[cl[coidx[0]],].index,] fig_properties['ygroup'].append(final.shape[0]) for i in range(1,len(coidx)): tf = fc_df.loc[dorder.loc[cl[coidx[i]],].index,] fig_properties['ygroup'].append(tf.shape[0]) final = final.append(tf) if 'fold_cutoff' in param and param['fold_cutoff'] == 1: final[final > frange] = frange final[final < - frange] = - frange else: fc_df0b = final.sub(final.min(axis=1), axis=0) final = (fc_df0b.div(fc_df0b.max(axis=1), axis=0) - 0.5) * 2 * frange else: final=df2.loc[dorder.loc[cl[coidx[0]],].index,] fig_properties['ygroup'].append(final.shape[0]) for i in range(1,len(coidx)): tf = df2.loc[dorder.loc[cl[coidx[i]],].index,] fig_properties['ygroup'].append(tf.shape[0]) final = final.append(tf) ## loading pvalue distribution FDT fdt = {'row_labels' :[], 'column_labels' : [], "data" : [[]]}; #fdt = OrderedDict(fdt) # Nan to None final = final.where(pd.notnull(final),None) fdt['data'] = final.T.as_matrix().tolist() # make sure Transpose fdt['row_labels'] = final.columns.tolist() fdt['column_labels'] = final.index.tolist() # TODO: Add group label later fdt['id'] = "{0}.fdt".format(param['out_figure_object_name']) self.logger.info("Saving the results") sstatus = ws.save_objects({'workspace' : workspace_name, 'objects' : [{'type' : 'MAK.FloatDataTable', 'data' : fdt, 'hidden':1, 'name' : "{0}.fdt".format(param['out_figure_object_name'])}]}) data_ref = "{0}/{1}/{2}".format(sstatus[0][6], sstatus[0][0], sstatus[0][4]) fig_properties['data_ref'] = data_ref sstatus = ws.save_objects({'workspace' : workspace_name, 'objects' : [{'type' : 'CoExpression.FigureProperties', 'data' : fig_properties, #'hidden':1, 'name' : "{0}".format(param['out_figure_object_name'])}]}) #'name' : "{0}.fp".format(param['out_figure_object_name'])}]}) #mchp = {} #mchp['figure_obj'] = "{0}/{1}/{2}".format(sstatus[0][6], sstatus[0][0], sstatus[0][4]) #sstatus = ws.save_objects({'workspace' : workspace_name, 'objects' : [{'type' : 'CoExpression.MulticlusterHeatmapPlot', # 'data' : mchp, # 'name' : (param['out_figure_object_name'])}]}) result = fig_properties #END view_heatmap # At some point might do deeper type checking... if not isinstance(result, dict): raise ValueError('Method view_heatmap return value ' + 'result is not type dict as required.') # return the results return [result]
def main(): script_details = script_utils.parse_docs(transform.__doc__) parser = argparse.ArgumentParser(prog=__file__, description=script_details["Description"], epilog=script_details["Authors"]) parser.add_argument('--workspace_service_url', help=script_details["Args"]["workspace_service_url"], action='store', type=str, nargs='?', required=True) parser.add_argument('--workspace_name', help=script_details["Args"]["workspace_name"], action='store', type=str, nargs='?', required=True) parser.add_argument("--object_name", help=script_details["Args"]["object_name"], action='store', type=str, nargs='?', required=True) parser.add_argument('--output_file_name', help=script_details["Args"]["output_file_name"], action='store', type=str, nargs='?', default=None, required=False) parser.add_argument('--input_directory', help=script_details["Args"]["input_directory"], action='store', type=str, nargs='?', required=True) parser.add_argument("--working_directory", help=script_details["Args"]["working_directory"], action='store', type=str, nargs='?', required=True) parser.add_argument('--input_mapping', help=script_details["Args"]["input_mapping"], action='store', type=unicode, nargs='?', default=None, required=False) # custom arguments specific to this uploader parser.add_argument('--format_type', help=script_details["Args"]["format_type"], action='store', type=str, required=False) parser.add_argument('--genome_object_name', help=script_details["Args"]["genome_object_name"], action='store', type=str, required=False) parser.add_argument('--fill_missing_values', help=script_details["Args"]["fill_missing_values"], action='store', type=int, required=False) parser.add_argument('--data_type', help=script_details["Args"]["data_type"], action='store', type=str, required=False) parser.add_argument('--data_scale', help=script_details["Args"]["data_scale"], action='store', type=str, required=False) args, unknown = parser.parse_known_args() logger = script_utils.stderrlogger(__file__) logger.debug(args) try: transform(workspace_service_url=args.workspace_service_url, workspace_name=args.workspace_name, object_name=args.object_name, output_file_name=args.output_file_name, input_directory=args.input_directory, working_directory=args.working_directory, input_mapping=args.input_mapping, format_type=args.format_type, genome_object_name=args.genome_object_name, fill_missing_values=args.fill_missing_values, data_type=args.data_type, data_scale=args.data_scale, logger=logger) except Exception as e: logger.exception(e) sys.exit(1)
def convert(shock_service_url, handle_service_url, input_directory, object_name, level=logging.INFO, logger=None): """ Converts FASTQ file to KBaseAssembly.PairedEndLibrary json string. Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. input_directory: Where the FASTQ file can be found. object_name: A name to use when storing the JSON string. mean_insert: The average insert size. std_dev: standard deviation of the inserts interleaved: Are the reads interleaved? read_orientation: Do the reads have an outward orientation? level: Logging level, defaults to logging.INFO. """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of FASTQ to KBaseAssembly.PairedEndLibrary.") token = os.environ.get('KB_AUTH_TOKEN') # scan the directory for files logger.info("Scanning for FASTQ files.") valid_extensions = [".fq",".fastq",".fnq"] files = os.listdir(working_directory) fastq_files = [x for x in files if os.path.splitext(x)[-1] in valid_extensions] assert len(fastq_files) != 0 # put the files in shock, get handles shock_ids = list() for x in fastq_files: shock_info = script_utils.upload_file_to_shock(logger, shock_service_url, input_file_name, token=token) shock_ids.append(shock_info["id"]) logger.info("Gathering information.") handles = script_utils.getHandles(logger, shock_service_url, handle_service_url, shock_ids, [handle_id], token) assert len(handles) != 0 # fill out the object details resultObject = dict() resultObject["handle_1"] = handles[0] if len(handles) == 2: resultObject["handle_2"] = handles[1] if mean_insert is not None : resultObject["insert_size_mean"] = mean_insert if std_dev is not None: resultObject["insert_size_std_dev"] = std_dev if interleaved: resultObject["interleaved"] = 1 if read_orientation: resultObject["read_orientation_outward"] = 1 objectString = json.dumps(resultObject, sort_keys=True, indent=4) logger.info("Writing out JSON.") with open(args.output_filename, "w") as outFile: outFile.write(objectString) logger.info("Conversion completed.")
def transform( shock_service_url=None, handle_service_url=None, output_file_name=None, input_directory=None, working_directory=None, level=logging.INFO, logger=None, ): """ Converts a FASTQ file to a KBaseAssembly.SingleEndLibrary json string. Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. output_file_name: A file name where the output JSON string should be stored. input_directory: The directory containing the file. working_directory: The directory the resulting json file will be written to. level: Logging level, defaults to logging.INFO. Returns: JSON file on disk that can be saved as a KBase workspace object. Authors: """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Scanning for FASTQ files.") valid_extensions = [".fq", ".fastq", ".fnq"] files = os.listdir(working_directory) fastq_files = [x for x in files if os.path.splitext(x)[-1] in valid_extensions] assert len(fastq_files) != 0 logger.info("Found {0}".format(str(fastq_files))) input_file_name = files[0] if len(fastq_files) > 1: logger.warning("Not sure how to handle multiple FASTQ files in this context. Using {0}".format(input_file_name)) kb_token = os.environ.get("KB_AUTH_TOKEN") script_utils.upload_file_to_shock( logger=logger, shock_service_url=shock_service_url, filePath=os.path.join(input_directory, input_file_name), token=kb_token, ) handles = script_utils.getHandles( logger=logger, shock_service_url=shock_service_url, handle_service_url=handle_service_url, token=kb_token ) assert len(handles) != 0 objectString = simplejson.dumps({"handle": handles[0]}, sort_keys=True, indent=4) if output_file_name is None: output_file_name = input_file_name with open(os.path.join(output_directory, output_file_name), "w") as f: f.write(objectString)
def transform(workspace_service_url=None, workspace_name=None, object_name=None, version=None, working_directory=None, output_file_name=None, level=logging.INFO, logger=None): """ Converts KBaseEnigmaMetals.SamplePropertyMatrix to TSV-formatted file. Args: workspace_service_url: A url for the KBase Workspace service workspace_name: Name of the workspace object_name: Name of the object in the workspace version: Version number of workspace object, defaults to most recent version working_directory: The working directory where the output file should be stored. output_file_name: The desired file name of the result file. level: Logging level, defaults to logging.INFO. Returns: TSV-formatted file containing data from SamplePropertyMatrix object. Authors: Roman Sutormin, Alexey Kazakov """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info( "Starting conversion of KBaseEnigmaMetals.SamplePropertyMatrix to TSV.SampleProperty" ) token = os.environ.get("KB_AUTH_TOKEN") if not working_directory or not os.path.isdir(args.working_directory): raise Exception("The working directory {0} does not exist".format( working_directory)) logger.info("Grabbing Data.") classpath = [ "$KB_TOP/lib/jars/kbase/transform/kbase_transform_deps.jar", "$KB_TOP/lib/jars/apache_commons/commons-cli-1.2.jar", "$KB_TOP/lib/jars/ini4j/ini4j-0.5.2.jar", "$KB_TOP/lib/jars/jackson/jackson-annotations-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-core-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-databind-2.2.3.jar", "$KB_TOP/lib/jars/jetty/jetty-all-7.0.0.jar", "$KB_TOP/lib/jars/jna/jna-3.4.0.jar", "$KB_TOP/lib/jars/kbase/auth/kbase-auth-0.3.1.jar", "$KB_TOP/lib/jars/kbase/common/kbase-common-0.0.10.jar", "$KB_TOP/lib/jars/servlet/servlet-api-2.5.jar", "$KB_TOP/lib/jars/syslog4j/syslog4j-0.9.46.jar", "$KB_TOP/lib/jars/kbase/workspace/WorkspaceClient-0.2.0.jar" ] mc = "us.kbase.kbaseenigmametals.SamplePropertyMatrixDownloader" argslist = [ "--workspace_service_url {0}".format(workspace_service_url), "--workspace_name {0}".format(workspace_name), "--object_name {0}".format(object_name), "--working_directory {0}".format(working_directory) ] if output_file_name: argslist.append("--output_file_name {0}".format(output_file_name)) if version: argslist.append("--version {0}".format(version)) arguments = [ "java", "-classpath", ":".join(classpath), mc, " ".join(argslist) ] logger.debug(arguments) # need shell in this case because the java code is depending on finding the KBase token in the environment tool_process = subprocess.Popen(" ".join(arguments), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.error( "Transformation from KBaseEnigmaMetals.SamplePropertyMatrix to TSV.SampleProperty failed" ) logger.error(stderr) sys.exit(1) logger.info("Conversion completed.")
def transform(workspace_service_url=None, workspace_name=None, object_name=None, output_file_name=None, input_directory=None, working_directory=None, has_replicates=None, input_mapping=None, format_type=None, level=logging.INFO, logger=None): """ Converts SampleProperty TSV file to json string of KBaseEnigmaMetals.SamplePropertyMatrix type. Args: workspace_service_url: URL for a KBase Workspace service where KBase objects. are stored. workspace_name: The name of the destination workspace. object_name: The destination object name. output_file_name: A file name where the output JSON string should be stored. If the output file name is not specified the name will default to the name of the input file appended with '_output.json'. input_directory: The directory where files will be read from. working_directory: The directory the resulting json file will be written to. has_replicates: 0 if the input file contains marked series of replicates, 1 if the input file contains non-marked series of replicates, 2 if the input file contains no replicates. input_mapping: JSON string mapping of input files to expected types. If you don't get this you need to scan the input directory and look for your files. format_type: Mannually defined type of TSV file format. Returns: JSON files on disk that can be saved as a KBase workspace objects. Authors: Roman Sutormin, Alexey Kazakov """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of SampleProperty TSV to KBaseEnigmaMetals.SamplePropertyMatrix") # token = os.environ.get('KB_AUTH_TOKEN') if not working_directory or not os.path.isdir(working_directory): raise Exception("The working directory {0} is not a valid directory!" .format(working_directory)) classpath = ["$KB_TOP/lib/jars/kbase/transform/kbase_transform_deps.jar", "$KB_TOP/lib/jars/apache_commons/commons-cli-1.2.jar", "$KB_TOP/lib/jars/apache_commons/commons-lang3-3.1.jar", "$KB_TOP/lib/jars/ini4j/ini4j-0.5.2.jar", "$KB_TOP/lib/jars/jackson/jackson-annotations-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-core-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-databind-2.2.3.jar", "$KB_TOP/lib/jars/jetty/jetty-all-7.0.0.jar", "$KB_TOP/lib/jars/jna/jna-3.4.0.jar", "$KB_TOP/lib/jars/kbase/auth/kbase-auth-0.3.1.jar", "$KB_TOP/lib/jars/kbase/common/kbase-common-0.0.10.jar", "$KB_TOP/lib/jars/servlet/servlet-api-2.5.jar", "$KB_TOP/lib/jars/syslog4j/syslog4j-0.9.46.jar", "$KB_TOP/lib/jars/kbase/workspace/WorkspaceClient-0.2.0.jar"] mc = "us.kbase.kbaseenigmametals.SamplePropertyMatrixUploader" argslist = ["--workspace_service_url {0}".format(workspace_service_url), "--workspace_name {0}".format(workspace_name), "--object_name {0}".format(object_name), "--input_directory {0}".format(input_directory), "--has_replicates {0}".format(has_replicates), "--working_directory {0}".format(working_directory)] if output_file_name: argslist.append("--output_file_name {0}".format(output_file_name)) if input_mapping: argslist.append("--input_mapping {0}".format(input_mapping)) argslist.append("--format_type {0}".format(format_type)) arguments = ["java", "-classpath", ":".join(classpath), mc, " ".join(argslist)] logger.info(arguments) # need shell in this case because the java code is depending on finding the KBase token in the environment tool_process = subprocess.Popen(" ".join(arguments), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.error(stderr) if tool_process.returncode: logger.error("Transformation from TSV.SampleProperty to KBaseEnigmaMetals.SamplePropertyMatrix failed on {0}".format(input_directory)) sys.exit(1) logger.info("Conversion completed.")
def main(): """ KBase Convert task manager for converting between KBase objects. Step 1 - Run a converter to pull the source object and save the destination object. Args: workspace_service_url: URL for a KBase Workspace service where KBase objects are stored. ujs_service_url: URL for a User and Job State service to report task progress back to the user. shock_service_url: URL for a KBase SHOCK data store service for storing files and large reference data. handle_service_url: URL for a KBase Handle service that maps permissions from the Workspace to SHOCK for KBase types that specify a Handle reference instead of a SHOCK reference. source_workspace_name: The name of the source workspace. destination_workspace_name: The name of the destination workspace. source_object_name: The source object name. destination_object_name: The destination object name. source_kbase_type: The KBase Workspace type string that indicates the module and type of the object being created. destination_kbase_type: The KBase Workspace type string that indicates the module and type of the object being created. optional_arguments: This is a JSON string containing optional parameters that can be passed in for custom behavior per conversion. ujs_job_id: The job id from the User and Job State service that can be used to report status on task progress back to the user. job_details: This is a JSON string that passes in the script specific command line options for a given conversion type. The service pulls these config settings from a script config created by the developer of the conversion script and passes that into the AWE job that calls this script. working_directory: The working directory on disk where files can be created and will be cleaned when the job ends with success or failure. keep_working_directory: A flag to tell the script not to delete the working directory, which is mainly for debugging purposes. debug: Run the taskrunner in debug mode for local execution in a virtualenv. Returns: Literal return value is 0 for success and 1 for failure. Actual data output is one or more Workspace objects saved to a user's workspace. Authors: Matt Henderson, Gavin Price """ logger = script_utils.stderrlogger(__file__, level=logging.DEBUG) logger.info("Executing KBase Convert tasks") script_details = script_utils.parse_docs(main.__doc__) logger.debug(script_details["Args"]) parser = script_utils.ArgumentParser( description=script_details["Description"], epilog=script_details["Authors"]) # provided by service config parser.add_argument('--workspace_service_url', help=script_details["Args"]["workspace_service_url"], action='store', required=True) parser.add_argument('--ujs_service_url', help=script_details["Args"]["ujs_service_url"], action='store', required=True) # optional because not all KBase Workspace types contain a SHOCK or Handle reference parser.add_argument('--shock_service_url', help=script_details["Args"]["shock_service_url"], action='store', default=None) parser.add_argument('--handle_service_url', help=script_details["Args"]["handle_service_url"], action='store', default=None) # workspace info for pulling the data parser.add_argument('--source_workspace_name', help=script_details["Args"]["source_workspace_name"], action='store', required=True) parser.add_argument('--source_object_name', help=script_details["Args"]["source_object_name"], action='store', required=True) # workspace info for saving the data parser.add_argument( '--destination_workspace_name', help=script_details["Args"]["destination_workspace_name"], action='store', required=True) parser.add_argument('--destination_object_name', help=script_details["Args"]["destination_object_name"], action='store', required=True) # the types that we are transforming between, currently assumed one to one parser.add_argument('--source_kbase_type', help=script_details["Args"]["source_kbase_type"], action='store', required=True) parser.add_argument('--destination_kbase_type', help=script_details["Args"]["destination_kbase_type"], action='store', required=True) # any user options provided, encoded as a jason string parser.add_argument('--optional_arguments', help=script_details["Args"]["optional_arguments"], action='store', default='{}') # Used if you are restarting a previously executed job? parser.add_argument('--ujs_job_id', help=script_details["Args"]["ujs_job_id"], action='store', default=None, required=False) # config information for running the validate and transform scripts parser.add_argument('--job_details', help=script_details["Args"]["job_details"], action='store', default=None) # the working directory is where all the files for this job will be written, # and normal operation cleans it after the job ends (success or fail) parser.add_argument('--working_directory', help=script_details["Args"]["working_directory"], action='store', default=None, required=True) parser.add_argument('--keep_working_directory', help=script_details["Args"]["keep_working_directory"], action='store_true') # turn on debugging options for script developers running locally parser.add_argument('--debug', help=script_details["Args"]["debug"], action='store_true') args = None try: args = parser.parse_args() except Exception, e: logger.debug("Caught exception parsing arguments!") logger.exception(e) sys.exit(1)
def transform(workspace_service_url=None, workspace_name=None, object_name=None, output_file_name=None, input_directory=None, working_directory=None, input_mapping=None, format_type=None, genome_object_name=None, fill_missing_values=None, data_type=None, data_scale=None, level=logging.INFO, logger=None): """ Converts Expression TSV file to json string of KBaseFeatureValues.ExpressionMatrix type. Args: workspace_service_url: URL for a KBase Workspace service where KBase objects. are stored. workspace_name: The name of the destination workspace. object_name: The destination object name. output_file_name: A file name where the output JSON string should be stored. If the output file name is not specified the name will default to the name of the input file appended with '_output.json'. input_directory: The directory where files will be read from. working_directory: The directory the resulting json file will be written to. input_mapping: JSON string mapping of input files to expected types. If you don't get this you need to scan the input directory and look for your files. format_type: Mannually defined type of TSV file format. genome_object_name: Optional reference to a Genome object that will be used. for mapping feature IDs to. fill_missing_values: Flag for filling in missing values in matrix (0-false, 1-true). data_type: Data type (default value is 'log-ratio'). data_scale: Data scale (default value is '1.0'). Returns: JSON files on disk that can be saved as a KBase workspace objects. Authors: Roman Sutormin """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of Expression TSV to KBaseFeatureValues.ExpressionMatrix") # token = os.environ.get('KB_AUTH_TOKEN') if not working_directory or not os.path.isdir(working_directory): raise Exception("The working directory {0} is not a valid directory!" .format(working_directory)) classpath = ["$KB_TOP/lib/jars/kbase/feature_values/kbase-feature-values-0.8.jar", "$KB_TOP/lib/jars/kohsuke/args4j-2.0.21.jar", "$KB_TOP/lib/jars/kbase/common/kbase-common-0.0.10.jar", "$KB_TOP/lib/jars/jackson/jackson-annotations-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-core-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-databind-2.2.3.jar", "$KB_TOP/lib/jars/kbase/auth/kbase-auth-1398468950-3552bb2.jar", "$KB_TOP/lib/jars/kbase/workspace/WorkspaceClient-0.2.0.jar"] mc = "us.kbase.kbasefeaturevalues.transform.ExpressionUploader" argslist = ["--workspace_service_url {0}".format(workspace_service_url), "--workspace_name {0}".format(workspace_name), "--object_name {0}".format(object_name), "--input_directory {0}".format(input_directory), "--working_directory {0}".format(working_directory)] if output_file_name: argslist.append("--output_file_name {0}".format(output_file_name)) if input_mapping: argslist.append("--input_mapping {0}".format(input_mapping)) if format_type: argslist.append("--format_type {0}".format(format_type)) if genome_object_name: argslist.append("--genome_object_name {0}".format(genome_object_name)) if fill_missing_values: argslist.append("--fill_missing_values") if data_type: argslist.append("--data_type {0}".format(data_type)) if data_scale: argslist.append("--data_scale {0}".format(data_scale)) arguments = ["java", "-classpath", ":".join(classpath), mc, " ".join(argslist)] logger.debug(arguments) # need shell in this case because the java code is depending on finding the KBase token in the environment tool_process = subprocess.Popen(" ".join(arguments), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.error("Transformation from TSV.Expression to KBaseFeatureValues.ExpressionMatrix failed on {0}".format(input_directory)) logger.error(stderr) sys.exit(1) logger.info("Conversion completed.")
def validate(input_directory, working_directory, level=logging.INFO, logger=None): """ Validates any file containing sequence data. Args: input_directory: A directory containing one or more SequenceRead files. working_directory: A directory where any output files produced by validation can be written. level: Logging level, defaults to logging.INFO. Returns: Currently writes to stderr with a Java Exception trace on error, otherwise no output. Authors: Srividya Ramikrishnan, Matt Henderson """ if logger is None: logger = script_utils.stderrlogger(__file__) fasta_extensions = [".fa",".fas",".fasta",".fna"] fastq_extensions = [".fq",".fastq",".fnq"] extensions = fasta_extensions + fastq_extensions checked = False validated = True for input_file_name in os.listdir(input_directory): logger.info("Checking for SequenceReads file : {0}".format(input_file_name)) filePath = os.path.join(os.path.abspath(input_directory), input_file_name) if not os.path.isfile(filePath): logger.warning("Skipping directory {0}".format(input_file_name)) continue elif os.path.splitext(input_file_name)[-1] not in extensions: logger.warning("Unrecognized file type, skipping.") continue logger.info("Starting SequenceReads validation of {0}".format(input_file_name)) if os.path.splitext(input_file_name)[-1] in fasta_extensions: # TODO This needs to be changed, this is really just a demo program for this library and not a serious tool java_classpath = os.path.join(os.environ.get("KB_TOP"), "lib/jars/FastaValidator/FastaValidator-1.0.jar") arguments = ["java", "-classpath", java_classpath, "FVTester", filePath] elif os.path.splitext(input_file_name)[-1] in fastq_extensions: line_count = int(subprocess.check_output(["wc", "-l", filePath]).split()[0]) if line_count % 4 > 0: #cleans out lines that are empty. SRA Tool box puts newline on the end. cmd_list = ["sed","-i", r"/^$/d",filePath] filtering = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = filtering.communicate() if filtering.returncode != 0: raise Exception("sed execution failed for the file {0}".format(filePath)) if (check_interleavedPE(filePath) == 1): arguments = ["fastQValidator", "--file", filePath, "--maxErrors", "10", "--disableSeqIDCheck"] else : arguments = ["fastQValidator", "--file", filePath, "--maxErrors", "10"] tool_process = subprocess.Popen(arguments, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if tool_process.returncode != 0: logger.error("Validation failed on {0}".format(input_file_name)) validated = False break else: logger.info("Validation passed on {0}".format(input_file_name)) checked = True if not validated: raise Exception("Validation failed!") elif not checked: raise Exception("No files were found that had a valid fasta or fastq extension.") else: logger.info("Validation passed.")
def upload_assembly(shock_service_url = None, handle_service_url = None, input_directory = None, # shock_id = None, # handle_id = None, input_mapping = None, workspace_name = None, workspace_service_url = None, taxon_reference = None, assembly_name = None, source = None, date_string = None, contig_information_dict = None, logger = None): """ Uploads CondensedGenomeAssembly Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle service. shock_id: If the shock id exists use same file (NEEDS TO BE UPDATED TO HANDLE ID) input_mapping: (not sure, I think for mapping multiple files, not needed here only 1 file expected) workspace_name: Name of ws to load into workspace_service_url: URL of WS server instance the WS is on. taxon_reference: The ws reference the assembly points to. (Optional) assembly_name: Name of the assembly object to be created. (Optional) (defaults to file_name) source: The source of the data (Ex: Refseq) date_string: Date (or date range) associated with data. (Optional) contig_information_dict: A mapping that has is_circular and description information (Optional) Returns: JSON file on disk that can be saved as a KBase workspace object. Authors: Jason Baumohl, Matt Henderson """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of FASTA to Assembly object") token = os.environ.get('KB_AUTH_TOKEN') if input_mapping is None: logger.info("Scanning for FASTA files.") valid_extensions = [".fa",".fasta",".fna",".fas"] # files = os.listdir(input_directory) files = os.listdir(os.path.abspath(input_directory)) fasta_files = [x for x in files if os.path.splitext(x)[-1] in valid_extensions] if (len(fasta_files) == 0): raise Exception("The input file does not have one of the following extensions .fa, .fasta, .fas or .fna") logger.info("Found {0}".format(str(fasta_files))) fasta_file_name = os.path.join(input_directory,fasta_files[0]) if len(fasta_files) > 1: logger.warning("Not sure how to handle multiple FASTA files in this context. Using {0}".format(fasta_file_name)) else: logger.info("Input Mapping not none : " + str(input_mapping)) fasta_file_name = os.path.join(os.path.join(input_directory, "FASTA.DNA.Assembly"), simplejson.loads(input_mapping)["FASTA.DNA.Assembly"]) logger.info("Building Object.") if not os.path.isfile(fasta_file_name): raise Exception("The fasta file name {0} is not a file!".format(fasta_file_name)) if not os.path.isdir(input_directory): raise Exception("The input directory {0} is not a valid directory!".format(input_directory)) ws_client = biokbase.workspace.client.Workspace(workspace_service_url) workspace_object = ws_client.get_workspace_info({'workspace':workspace_name}) workspace_id = workspace_object[0] workspace_name = workspace_object[1] print "FASTA FILE Name :"+ fasta_file_name + ":" if assembly_name is None: base = os.path.basename(fasta_file_name) assembly_name = "{0}_assembly".format(os.path.splitext(base)[0]) ########################################## #ASSEMBLY CREATION PORTION - consume Fasta File ########################################## logger.info("Starting conversion of FASTA to Assemblies") logger.info("Building Assembly Object.") input_file_handle = TextFileDecoder.open_textdecoder(fasta_file_name, 'ISO-8859-1') fasta_header = None fasta_description = None sequence_list = [] fasta_dict = dict() first_header_found = False contig_set_md5_list = [] # Pattern for replacing white space pattern = re.compile(r'\s+') sequence_exists = False total_length = 0 gc_length = 0 #Note added X and x due to kb|g.1886.fasta valid_chars = "-AaCcGgTtUuWwSsMmKkRrYyBbDdHhVvNnXx" amino_acid_specific_characters = "PpLlIiFfQqEe" #Base_counts - is dict of base characters and their counts. base_counts = dict() sequence_start = 0 sequence_stop = 0 current_line = input_file_handle.readline() while current_line != None and len(current_line) > 0: # print "CURRENT LINE: " + current_line if (current_line[0] == ">"): # found a header line # Wrap up previous fasta sequence if (not sequence_exists) and first_header_found: logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) if not first_header_found: first_header_found = True sequence_start = 0 else: sequence_stop = input_file_handle.tell() - len(current_line) # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence : logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) # for character in total_sequence: # if character not in valid_chars: # if character in amino_acid_specific_characters: # raise Exception("This fasta file may have amino acids in it instead of the required nucleotides.") # raise Exception("This FASTA file has non nucleic acid characters : {0}".format(character)) seq_count = collections.Counter(total_sequence.upper()) seq_dict = dict(seq_count) for character in seq_dict: if character in base_counts: base_counts[character] = base_counts[character] + seq_dict[character] else: base_counts[character] = seq_dict[character] if character not in valid_chars: if character in amino_acid_specific_characters: raise Exception("This fasta file may have amino acids in it instead of the required nucleotides.") raise Exception("This FASTA file has non nucleic acid characters : {0}".format(character)) contig_dict = dict() Ncount = 0 if "N" in seq_dict: Ncount = seq_dict["N"] contig_dict["Ncount"] = Ncount length = len(total_sequence) total_length = total_length + length contig_gc_length = len(re.findall('G|g|C|c',total_sequence)) contig_dict["gc_content"] = float(contig_gc_length)/float(length) gc_length = gc_length + contig_gc_length fasta_key = fasta_header.strip() contig_dict["contig_id"] = fasta_key contig_dict["length"] = length contig_dict["name"] = fasta_key contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"] = contig_md5 contig_set_md5_list.append(contig_md5) contig_dict["is_circular"] = "Unknown" if fasta_description is not None: contig_dict["description"] = fasta_description if contig_information_dict is not None: if contig_information_dict[fasta_key] is not None: if contig_information_dict[fasta_key]["definition"] is not None: contig_dict["description"] = contig_information_dict[fasta_key]["definition"] if contig_information_dict[fasta_key]["is_circular"] is not None: contig_dict["is_circular"] = contig_information_dict[fasta_key]["is_circular"] contig_dict["start_position"] = sequence_start contig_dict["num_bytes"] = sequence_stop - sequence_start # print "Sequence Start: " + str(sequence_start) + "Fasta: " + fasta_key # print "Sequence Stop: " + str(sequence_stop) + "Fasta: " + fasta_key if fasta_key in fasta_dict: raise Exception("The fasta header {0} appears more than once in the file ".format(fasta_key)) else: fasta_dict[fasta_key] = contig_dict # get set up for next fasta sequence sequence_list = [] sequence_exists = False # sequence_start = input_file_handle.tell() sequence_start = 0 fasta_header_line = current_line.strip().replace('>','') try: fasta_header , fasta_description = fasta_header_line.split(' ',1) except: fasta_header = fasta_header_line fasta_description = None else: if sequence_start == 0: sequence_start = input_file_handle.tell() - len(current_line) sequence_list.append(current_line) sequence_exists = True current_line = input_file_handle.readline() # print "ENDING CURRENT LINE: " + current_line # wrap up last fasta sequence if (not sequence_exists) and first_header_found: logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) elif not first_header_found : logger.error("There are no contigs in this file") raise Exception("There are no contigs in this file") else: sequence_stop = input_file_handle.tell() # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence : logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) # for character in total_sequence: seq_count = collections.Counter(total_sequence.upper()) seq_dict = dict(seq_count) for character in seq_dict: if character in base_counts: base_counts[character] = base_counts[character] + seq_dict[character] else: base_counts[character] = seq_dict[character] if character not in valid_chars: if character in amino_acid_specific_characters: raise Exception("This fasta file may have amino acids in it instead of the required nucleotides.") raise Exception("This FASTA file has non nucleic acid characters : {0}".format(character)) contig_dict = dict() Ncount = 0 if "N" in seq_dict: Ncount = seq_dict["N"] contig_dict["Ncount"] = Ncount length = len(total_sequence) total_length = total_length + length contig_gc_length = len(re.findall('G|g|C|c',total_sequence)) contig_dict["gc_content"] = float(contig_gc_length)/float(length) gc_length = gc_length + contig_gc_length fasta_key = fasta_header.strip() contig_dict["contig_id"] = fasta_key contig_dict["length"] = length contig_dict["name"] = fasta_key contig_dict["is_circular"] = "Unknown" if fasta_description is not None: contig_dict["description"] = fasta_description if contig_information_dict is not None: if contig_information_dict[fasta_key] is not None: if contig_information_dict[fasta_key]["definition"] is not None: contig_dict["description"] = contig_information_dict[fasta_key]["definition"] if contig_information_dict[fasta_key]["is_circular"] is not None: contig_dict["is_circular"] = contig_information_dict[fasta_key]["is_circular"] contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"]= contig_md5 contig_set_md5_list.append(contig_md5) contig_dict["start_position"] = sequence_start contig_dict["num_bytes"] = sequence_stop - sequence_start if fasta_key in fasta_dict: raise Exception("The fasta header {0} appears more than once in the file ".format(fasta_key)) else: fasta_dict[fasta_key] = contig_dict input_file_handle.close() contig_set_dict = dict() contig_set_dict["md5"] = hashlib.md5(",".join(sorted(contig_set_md5_list))).hexdigest() contig_set_dict["assembly_id"] = assembly_name contig_set_dict["name"] = assembly_name contig_set_dict["external_source"] = source contig_set_dict["external_source_id"] = os.path.basename(fasta_file_name) # contig_set_dict["external_source_origination_date"] = str(os.stat(fasta_file_name).st_ctime) if date_string is not None: contig_set_dict["external_source_origination_date"] = date_string contig_set_dict["contigs"] = fasta_dict contig_set_dict["dna_size"] = total_length contig_set_dict["gc_content"] = float(gc_length)/float(total_length) # print "Fasta dict Keys :"+",".join(fasta_dict.keys())+":" contig_set_dict["num_contigs"] = len(fasta_dict.keys()) contig_set_dict["type"] = "Unknown" contig_set_dict["notes"] = "Note MD5s are generated from uppercasing the sequences" contig_set_dict["base_counts"] = base_counts if taxon_reference is not None: contig_set_dict["taxon_ref"] = taxon_reference shock_id = None handle_id = None if shock_id is None: shock_info = script_utils.upload_file_to_shock(logger, shock_service_url, fasta_file_name, token=token) shock_id = shock_info["id"] handles = script_utils.getHandles(logger, shock_service_url, handle_service_url, [shock_id], [handle_id], token) handle_id = handles[0] contig_set_dict["fasta_handle_ref"] = handle_id # For future development if the type is updated to the handle_reference instead of a shock_reference assembly_not_saved = True assembly_provenance = [{"script": __file__, "script_ver": "0.1", "description": "Generated from fasta files generated from v5 of the CS."}] while assembly_not_saved: try: assembly_info = ws_client.save_objects({"workspace": workspace_name,"objects":[ {"type":"KBaseGenomeAnnotations.Assembly", "data":contig_set_dict, "name": assembly_name, "provenance":assembly_provenance}]}) assembly_not_saved = False except biokbase.workspace.client.ServerError as err: print "ASSEMBLY SAVE FAILED ON genome " + str(assembly_name) + " ERROR: " + str(err) raise except: print "ASSEMBLY SAVE FAILED ON genome " + str(assembly_name) + " GENERAL_EXCEPTION: " + str(sys.exc_info()[0]) raise logger.info("Conversion completed.")
def main(): """ KBase Convert task manager for converting between KBase objects. Step 1 - Run a converter to pull the source object and save the destination object. Args: workspace_service_url: URL for a KBase Workspace service where KBase objects are stored. ujs_service_url: URL for a User and Job State service to report task progress back to the user. shock_service_url: URL for a KBase SHOCK data store service for storing files and large reference data. handle_service_url: URL for a KBase Handle service that maps permissions from the Workspace to SHOCK for KBase types that specify a Handle reference instead of a SHOCK reference. source_workspace_name: The name of the source workspace. destination_workspace_name: The name of the destination workspace. source_object_name: The source object name. destination_object_name: The destination object name. source_kbase_type: The KBase Workspace type string that indicates the module and type of the object being created. destination_kbase_type: The KBase Workspace type string that indicates the module and type of the object being created. optional_arguments: This is a JSON string containing optional parameters that can be passed in for custom behavior per conversion. ujs_job_id: The job id from the User and Job State service that can be used to report status on task progress back to the user. job_details: This is a JSON string that passes in the script specific command line options for a given conversion type. The service pulls these config settings from a script config created by the developer of the conversion script and passes that into the AWE job that calls this script. working_directory: The working directory on disk where files can be created and will be cleaned when the job ends with success or failure. keep_working_directory: A flag to tell the script not to delete the working directory, which is mainly for debugging purposes. Returns: Literal return value is 0 for success and 1 for failure. Actual data output is one or more Workspace objects saved to a user's workspace. Authors: Matt Henderson, Gavin Price """ logger = script_utils.stderrlogger(__file__, level=logging.DEBUG) logger.info("Executing KBase Convert tasks") script_details = script_utils.parse_docs(main.__doc__) logger.debug(script_details["Args"]) parser = script_utils.ArgumentParser(description=script_details["Description"], epilog=script_details["Authors"]) # provided by service config parser.add_argument('--workspace_service_url', help=script_details["Args"]["workspace_service_url"], action='store', required=True) parser.add_argument('--ujs_service_url', help=script_details["Args"]["ujs_service_url"], action='store', required=True) # optional because not all KBase Workspace types contain a SHOCK or Handle reference parser.add_argument('--shock_service_url', help=script_details["Args"]["shock_service_url"], action='store', default=None) parser.add_argument('--handle_service_url', help=script_details["Args"]["handle_service_url"], action='store', default=None) # workspace info for pulling the data parser.add_argument('--source_workspace_name', help=script_details["Args"]["source_workspace_name"], action='store', required=True) parser.add_argument('--source_object_name', help=script_details["Args"]["source_object_name"], action='store', required=True) # workspace info for saving the data parser.add_argument('--destination_workspace_name', help=script_details["Args"]["destination_workspace_name"], action='store', required=True) parser.add_argument('--destination_object_name', help=script_details["Args"]["destination_object_name"], action='store', required=True) # the types that we are transforming between, currently assumed one to one parser.add_argument('--source_kbase_type', help=script_details["Args"]["source_kbase_type"], action='store', required=True) parser.add_argument('--destination_kbase_type', help=script_details["Args"]["destination_kbase_type"], action='store', required=True) # any user options provided, encoded as a jason string parser.add_argument('--optional_arguments', help=script_details["Args"]["optional_arguments"], action='store', default='{}') # Used if you are restarting a previously executed job? parser.add_argument('--ujs_job_id', help=script_details["Args"]["ujs_job_id"], action='store', default=None, required=False) # config information for running the validate and transform scripts parser.add_argument('--job_details', help=script_details["Args"]["job_details"], action='store', default=None) # the working directory is where all the files for this job will be written, # and normal operation cleans it after the job ends (success or fail) parser.add_argument('--working_directory', help=script_details["Args"]["working_directory"], action='store', default=None, required=True) parser.add_argument('--keep_working_directory', help=script_details["Args"]["keep_working_directory"], action='store_true') # ignore any extra arguments args, unknown = parser.parse_known_args() kb_token = os.environ.get('KB_AUTH_TOKEN') ujs = UserAndJobState(url=args.ujs_service_url, token=kb_token) est = datetime.datetime.utcnow() + datetime.timedelta(minutes=3) if args.ujs_job_id is not None: ujs.update_job_progress(args.ujs_job_id, kb_token, "KBase Data Convert started", 1, est.strftime('%Y-%m-%dT%H:%M:%S+0000')) # parse all the json strings from the argument list into dicts # TODO had issues with json.loads and unicode strings, workaround was using simplejson and base64 args.optional_arguments = simplejson.loads(base64.urlsafe_b64decode(args.optional_arguments)) args.job_details = simplejson.loads(base64.urlsafe_b64decode(args.job_details)) if not os.path.exists(args.working_directory): os.mkdir(args.working_directory) if args.ujs_job_id is not None: ujs.update_job_progress(args.ujs_job_id, kb_token, "Converting from {0} to {1}".format(args.source_kbase_type,args.destination_kbase_type), 1, est.strftime('%Y-%m-%dT%H:%M:%S+0000') ) # Step 1 : Convert the objects try: logger.info(args) convert_args = args.job_details["transform"] convert_args["optional_arguments"] = args.optional_arguments convert_args["working_directory"] = args.working_directory convert_args["workspace_service_url"] = args.workspace_service_url convert_args["source_workspace_name"] = args.source_workspace_name convert_args["source_object_name"] = args.source_object_name convert_args["destination_workspace_name"] = args.destination_workspace_name convert_args["destination_object_name"] = args.destination_object_name logger.info(convert_args) task_output = handler_utils.run_task(logger, convert_args) if task_output["stdout"] is not None: logger.debug("STDOUT : " + str(task_output["stdout"])) if task_output["stderr"] is not None: logger.debug("STDERR : " + str(task_output["stderr"])) except Exception, e: handler_utils.report_exception(logger, {"message": 'ERROR : Conversion from {0} to {1}'.format(args.source_kbase_type,args.destination_kbase_type), "exc": e, "ujs": ujs, "ujs_job_id": args.ujs_job_id, "token": kb_token, }, {"keep_working_directory": args.keep_working_directory, "working_directory": args.working_directory}) ujs.complete_job(args.ujs_job_id, kb_token, "Convert to {0} failed.".format( args.destination_workspace_name), str(e), None)
def transform(workspace_service_url=None, workspace_name=None, object_name=None, version=None, working_directory=None, output_file_name=None, level=logging.INFO, logger=None): """ Converts KBaseEnigmaMetals.ChromatographyMatrix to TSV-formatted file. Args: workspace_service_url: A url for the KBase Workspace service workspace_name: Name of the workspace object_name: Name of the object in the workspace version: Version number of workspace object, defaults to most recent version working_directory: The working directory where the output file should be stored. output_file_name: The desired file name of the result file. level: Logging level, defaults to logging.INFO. Returns: TSV-formatted file containing data from ChromatographyMatrix object. Authors: Roman Sutormin, Alexey Kazakov """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of KBaseEnigmaMetals.ChromatographyMatrix to TSV.Chromatography") token = os.environ.get("KB_AUTH_TOKEN") if not working_directory or not os.path.isdir(args.working_directory): raise Exception("The working directory {0} does not exist".format(working_directory)) logger.info("Grabbing Data.") classpath = ["$KB_TOP/lib/jars/kbase/transform/kbase_transform_deps.jar", "$KB_TOP/lib/jars/apache_commons/commons-cli-1.2.jar", "$KB_TOP/lib/jars/ini4j/ini4j-0.5.2.jar", "$KB_TOP/lib/jars/jackson/jackson-annotations-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-core-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-databind-2.2.3.jar", "$KB_TOP/lib/jars/jetty/jetty-all-7.0.0.jar", "$KB_TOP/lib/jars/jna/jna-3.4.0.jar", "$KB_TOP/lib/jars/kbase/auth/kbase-auth-0.3.1.jar", "$KB_TOP/lib/jars/kbase/common/kbase-common-0.0.10.jar", "$KB_TOP/lib/jars/servlet/servlet-api-2.5.jar", "$KB_TOP/lib/jars/syslog4j/syslog4j-0.9.46.jar", "$KB_TOP/lib/jars/kbase/workspace/WorkspaceClient-0.2.0.jar"] mc = "us.kbase.kbaseenigmametals.ChromatographyMatrixDownloader" argslist = ["--workspace_service_url {0}".format(workspace_service_url), "--workspace_name {0}".format(workspace_name), "--object_name {0}".format(object_name), "--working_directory {0}".format(working_directory)] if output_file_name: argslist.append("--output_file_name {0}".format(output_file_name)) if version: argslist.append("--version {0}".format(version)) arguments = ["java", "-classpath", ":".join(classpath), mc, " ".join(argslist)] logger.debug(arguments) # need shell in this case because the java code is depending on finding the KBase token in the environment tool_process = subprocess.Popen(" ".join(arguments), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.error("Transformation from KBaseEnigmaMetals.ChromatographyMatrix to TSV.Chromatography failed") logger.error(stderr) sys.exit(1) logger.info("Conversion completed.")
def convert(shock_service_url, handle_service_url, input_directory, object_name, level=logging.INFO, logger=None): """ Converts FASTQ file to KBaseAssembly.PairedEndLibrary json string. Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. input_directory: Where the FASTQ file can be found. object_name: A name to use when storing the JSON string. mean_insert: The average insert size. std_dev: standard deviation of the inserts interleaved: Are the reads interleaved? read_orientation: Do the reads have an outward orientation? level: Logging level, defaults to logging.INFO. """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info( "Starting conversion of FASTQ to KBaseAssembly.PairedEndLibrary.") token = os.environ.get('KB_AUTH_TOKEN') # scan the directory for files logger.info("Scanning for FASTQ files.") valid_extensions = [".fq", ".fastq", ".fnq"] files = os.listdir(working_directory) fastq_files = [ x for x in files if os.path.splitext(x)[-1] in valid_extensions ] assert len(fastq_files) != 0 # put the files in shock, get handles shock_ids = list() for x in fastq_files: shock_info = script_utils.upload_file_to_shock(logger, shock_service_url, input_file_name, token=token) shock_ids.append(shock_info["id"]) logger.info("Gathering information.") handles = script_utils.getHandles(logger, shock_service_url, handle_service_url, shock_ids, [handle_id], token) assert len(handles) != 0 # fill out the object details resultObject = dict() resultObject["handle_1"] = handles[0] if len(handles) == 2: resultObject["handle_2"] = handles[1] if mean_insert is not None: resultObject["insert_size_mean"] = mean_insert if std_dev is not None: resultObject["insert_size_std_dev"] = std_dev if interleaved: resultObject["interleaved"] = 1 if read_orientation: resultObject["read_orientation_outward"] = 1 objectString = json.dumps(resultObject, sort_keys=True, indent=4) logger.info("Writing out JSON.") with open(args.output_filename, "w") as outFile: outFile.write(objectString) logger.info("Conversion completed.")
def upload_assembly( shock_service_url=None, handle_service_url=None, input_directory=None, # shock_id = None, # handle_id = None, input_mapping=None, workspace_name=None, workspace_service_url=None, taxon_reference=None, assembly_name=None, source=None, date_string=None, contig_information_dict=None, logger=None): """ Uploads CondensedGenomeAssembly Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle service. shock_id: If the shock id exists use same file (NEEDS TO BE UPDATED TO HANDLE ID) input_mapping: (not sure, I think for mapping multiple files, not needed here only 1 file expected) workspace_name: Name of ws to load into workspace_service_url: URL of WS server instance the WS is on. taxon_reference: The ws reference the assembly points to. (Optional) assembly_name: Name of the assembly object to be created. (Optional) (defaults to file_name) source: The source of the data (Ex: Refseq) date_string: Date (or date range) associated with data. (Optional) contig_information_dict: A mapping that has is_circular and description information (Optional) Returns: JSON file on disk that can be saved as a KBase workspace object. Authors: Jason Baumohl, Matt Henderson """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of FASTA to Assembly object") token = os.environ.get('KB_AUTH_TOKEN') if input_mapping is None: logger.info("Scanning for FASTA files.") valid_extensions = [".fa", ".fasta", ".fna", ".fas"] # files = os.listdir(input_directory) files = os.listdir(os.path.abspath(input_directory)) fasta_files = [ x for x in files if os.path.splitext(x)[-1] in valid_extensions ] if (len(fasta_files) == 0): raise Exception( "The input file does not have one of the following extensions .fa, .fasta, .fas or .fna" ) logger.info("Found {0}".format(str(fasta_files))) fasta_file_name = os.path.join(input_directory, fasta_files[0]) if len(fasta_files) > 1: logger.warning( "Not sure how to handle multiple FASTA files in this context. Using {0}" .format(fasta_file_name)) else: logger.info("Input Mapping not none : " + str(input_mapping)) fasta_file_name = os.path.join( os.path.join(input_directory, "FASTA.DNA.Assembly"), simplejson.loads(input_mapping)["FASTA.DNA.Assembly"]) logger.info("Building Object.") if not os.path.isfile(fasta_file_name): raise Exception( "The fasta file name {0} is not a file!".format(fasta_file_name)) if not os.path.isdir(input_directory): raise Exception( "The input directory {0} is not a valid directory!".format( input_directory)) ws_client = biokbase.workspace.client.Workspace(workspace_service_url) workspace_object = ws_client.get_workspace_info( {'workspace': workspace_name}) workspace_id = workspace_object[0] workspace_name = workspace_object[1] print "FASTA FILE Name :" + fasta_file_name + ":" if assembly_name is None: base = os.path.basename(fasta_file_name) assembly_name = "{0}_assembly".format(os.path.splitext(base)[0]) ########################################## #ASSEMBLY CREATION PORTION - consume Fasta File ########################################## logger.info("Starting conversion of FASTA to Assemblies") logger.info("Building Assembly Object.") input_file_handle = TextFileDecoder.open_textdecoder( fasta_file_name, 'ISO-8859-1') fasta_header = None fasta_description = None sequence_list = [] fasta_dict = dict() first_header_found = False contig_set_md5_list = [] # Pattern for replacing white space pattern = re.compile(r'\s+') sequence_exists = False total_length = 0 gc_length = 0 #Note added X and x due to kb|g.1886.fasta valid_chars = "-AaCcGgTtUuWwSsMmKkRrYyBbDdHhVvNnXx" amino_acid_specific_characters = "PpLlIiFfQqEe" #Base_counts - is dict of base characters and their counts. base_counts = dict() sequence_start = 0 sequence_stop = 0 current_line = input_file_handle.readline() while current_line != None and len(current_line) > 0: # print "CURRENT LINE: " + current_line if (current_line[0] == ">"): # found a header line # Wrap up previous fasta sequence if (not sequence_exists) and first_header_found: logger.error( "There is no sequence related to FASTA record : {0}". format(fasta_header)) raise Exception( "There is no sequence related to FASTA record : {0}". format(fasta_header)) if not first_header_found: first_header_found = True sequence_start = 0 else: sequence_stop = input_file_handle.tell() - len(current_line) # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence: logger.error( "There is no sequence related to FASTA record : {0}". format(fasta_header)) raise Exception( "There is no sequence related to FASTA record : {0}". format(fasta_header)) # for character in total_sequence: # if character not in valid_chars: # if character in amino_acid_specific_characters: # raise Exception("This fasta file may have amino acids in it instead of the required nucleotides.") # raise Exception("This FASTA file has non nucleic acid characters : {0}".format(character)) seq_count = collections.Counter(total_sequence.upper()) seq_dict = dict(seq_count) for character in seq_dict: if character in base_counts: base_counts[character] = base_counts[ character] + seq_dict[character] else: base_counts[character] = seq_dict[character] if character not in valid_chars: if character in amino_acid_specific_characters: raise Exception( "This fasta file may have amino acids in it instead of the required nucleotides." ) raise Exception( "This FASTA file has non nucleic acid characters : {0}" .format(character)) contig_dict = dict() Ncount = 0 if "N" in seq_dict: Ncount = seq_dict["N"] contig_dict["Ncount"] = Ncount length = len(total_sequence) total_length = total_length + length contig_gc_length = len(re.findall('G|g|C|c', total_sequence)) contig_dict["gc_content"] = float(contig_gc_length) / float( length) gc_length = gc_length + contig_gc_length fasta_key = fasta_header.strip() contig_dict["contig_id"] = fasta_key contig_dict["length"] = length contig_dict["name"] = fasta_key contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"] = contig_md5 contig_set_md5_list.append(contig_md5) contig_dict["is_circular"] = "Unknown" if fasta_description is not None: contig_dict["description"] = fasta_description if contig_information_dict is not None: if contig_information_dict[fasta_key] is not None: if contig_information_dict[fasta_key][ "definition"] is not None: contig_dict[ "description"] = contig_information_dict[ fasta_key]["definition"] if contig_information_dict[fasta_key][ "is_circular"] is not None: contig_dict[ "is_circular"] = contig_information_dict[ fasta_key]["is_circular"] contig_dict["start_position"] = sequence_start contig_dict["num_bytes"] = sequence_stop - sequence_start # print "Sequence Start: " + str(sequence_start) + "Fasta: " + fasta_key # print "Sequence Stop: " + str(sequence_stop) + "Fasta: " + fasta_key if fasta_key in fasta_dict: raise Exception( "The fasta header {0} appears more than once in the file " .format(fasta_key)) else: fasta_dict[fasta_key] = contig_dict # get set up for next fasta sequence sequence_list = [] sequence_exists = False # sequence_start = input_file_handle.tell() sequence_start = 0 fasta_header_line = current_line.strip().replace('>', '') try: fasta_header, fasta_description = fasta_header_line.split( ' ', 1) except: fasta_header = fasta_header_line fasta_description = None else: if sequence_start == 0: sequence_start = input_file_handle.tell() - len(current_line) sequence_list.append(current_line) sequence_exists = True current_line = input_file_handle.readline() # print "ENDING CURRENT LINE: " + current_line # wrap up last fasta sequence if (not sequence_exists) and first_header_found: logger.error( "There is no sequence related to FASTA record : {0}".format( fasta_header)) raise Exception( "There is no sequence related to FASTA record : {0}".format( fasta_header)) elif not first_header_found: logger.error("There are no contigs in this file") raise Exception("There are no contigs in this file") else: sequence_stop = input_file_handle.tell() # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence: logger.error( "There is no sequence related to FASTA record : {0}".format( fasta_header)) raise Exception( "There is no sequence related to FASTA record : {0}".format( fasta_header)) # for character in total_sequence: seq_count = collections.Counter(total_sequence.upper()) seq_dict = dict(seq_count) for character in seq_dict: if character in base_counts: base_counts[ character] = base_counts[character] + seq_dict[character] else: base_counts[character] = seq_dict[character] if character not in valid_chars: if character in amino_acid_specific_characters: raise Exception( "This fasta file may have amino acids in it instead of the required nucleotides." ) raise Exception( "This FASTA file has non nucleic acid characters : {0}". format(character)) contig_dict = dict() Ncount = 0 if "N" in seq_dict: Ncount = seq_dict["N"] contig_dict["Ncount"] = Ncount length = len(total_sequence) total_length = total_length + length contig_gc_length = len(re.findall('G|g|C|c', total_sequence)) contig_dict["gc_content"] = float(contig_gc_length) / float(length) gc_length = gc_length + contig_gc_length fasta_key = fasta_header.strip() contig_dict["contig_id"] = fasta_key contig_dict["length"] = length contig_dict["name"] = fasta_key contig_dict["is_circular"] = "Unknown" if fasta_description is not None: contig_dict["description"] = fasta_description if contig_information_dict is not None: if contig_information_dict[fasta_key] is not None: if contig_information_dict[fasta_key][ "definition"] is not None: contig_dict["description"] = contig_information_dict[ fasta_key]["definition"] if contig_information_dict[fasta_key][ "is_circular"] is not None: contig_dict["is_circular"] = contig_information_dict[ fasta_key]["is_circular"] contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"] = contig_md5 contig_set_md5_list.append(contig_md5) contig_dict["start_position"] = sequence_start contig_dict["num_bytes"] = sequence_stop - sequence_start if fasta_key in fasta_dict: raise Exception( "The fasta header {0} appears more than once in the file ". format(fasta_key)) else: fasta_dict[fasta_key] = contig_dict input_file_handle.close() contig_set_dict = dict() contig_set_dict["md5"] = hashlib.md5(",".join( sorted(contig_set_md5_list))).hexdigest() contig_set_dict["assembly_id"] = assembly_name contig_set_dict["name"] = assembly_name contig_set_dict["external_source"] = source contig_set_dict["external_source_id"] = os.path.basename(fasta_file_name) # contig_set_dict["external_source_origination_date"] = str(os.stat(fasta_file_name).st_ctime) if date_string is not None: contig_set_dict["external_source_origination_date"] = date_string contig_set_dict["contigs"] = fasta_dict contig_set_dict["dna_size"] = total_length contig_set_dict["gc_content"] = float(gc_length) / float(total_length) # print "Fasta dict Keys :"+",".join(fasta_dict.keys())+":" contig_set_dict["num_contigs"] = len(fasta_dict.keys()) contig_set_dict["type"] = "Unknown" contig_set_dict[ "notes"] = "Note MD5s are generated from uppercasing the sequences" contig_set_dict["base_counts"] = base_counts if taxon_reference is not None: contig_set_dict["taxon_ref"] = taxon_reference shock_id = None handle_id = None if shock_id is None: shock_info = script_utils.upload_file_to_shock(logger, shock_service_url, fasta_file_name, token=token) shock_id = shock_info["id"] handles = script_utils.getHandles(logger, shock_service_url, handle_service_url, [shock_id], [handle_id], token) handle_id = handles[0] contig_set_dict["fasta_handle_ref"] = handle_id # For future development if the type is updated to the handle_reference instead of a shock_reference assembly_not_saved = True assembly_provenance = [{ "script": __file__, "script_ver": "0.1", "description": "Generated from fasta files generated from v5 of the CS." }] while assembly_not_saved: try: assembly_info = ws_client.save_objects({ "workspace": workspace_name, "objects": [{ "type": "KBaseGenomeAnnotations.Assembly", "data": contig_set_dict, "name": assembly_name, "provenance": assembly_provenance }] }) assembly_not_saved = False except biokbase.workspace.client.ServerError as err: print "ASSEMBLY SAVE FAILED ON genome " + str( assembly_name) + " ERROR: " + str(err) raise except: print "ASSEMBLY SAVE FAILED ON genome " + str( assembly_name) + " GENERAL_EXCEPTION: " + str( sys.exc_info()[0]) raise logger.info("Conversion completed.")
def PluginManager(directory=None, logger=script_utils.stderrlogger(__file__)): if directory is None: raise Exception("Must provide a directory to read plugin configs from!") manager = PlugIns(directory, logger) return manager
def diff_p_distribution(self, ctx, args): # ctx is the context object # return variables are: result #BEGIN diff_p_distribution try: os.makedirs(self.RAWEXPR_DIR) except: pass try: os.makedirs(self.FLTRD_DIR) except: pass try: os.makedirs(self.FINAL_DIR) except: pass if self.logger is None: self.logger = script_utils.stderrlogger(__file__) result = {} self.logger.info("Starting conversion of KBaseFeatureValues.ExpressionMatrix to TSV") token = ctx['token'] eenv = os.environ.copy() eenv['KB_AUTH_TOKEN'] = token param = args from biokbase.workspace.client import Workspace ws = Workspace(url=self.__WS_URL, token=token) expr = ws.get_objects([{'workspace': param['workspace_name'], 'name' : param['object_name']}])[0]['data'] cmd_dowload_cvt_tsv = [self.FVE_2_TSV, '--workspace_service_url', self.__WS_URL, '--workspace_name', param['workspace_name'], '--object_name', param['object_name'], '--working_directory', self.RAWEXPR_DIR, '--output_file_name', self.EXPRESS_FN ] # need shell in this case because the java code is depending on finding the KBase token in the environment # -- copied from FVE_2_TSV tool_process = subprocess.Popen(" ".join(cmd_dowload_cvt_tsv), stderr=subprocess.PIPE, shell=True, env=eenv) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: self.logger.info(stdout) if stderr is not None and len(stderr) > 0: self.logger.info(stderr) self.logger.info("Identifying differentially expressed genes") ## Prepare sample file # detect num of columns with open("{0}/{1}".format(self.RAWEXPR_DIR, self.EXPRESS_FN), 'r') as f: fl = f.readline() ncol = len(fl.split('\t')) # force to use ANOVA if the number of sample is two if(ncol == 3): param['method'] = 'anova' with open("{0}/{1}".format(self.RAWEXPR_DIR, self.SAMPLE_FN), 'wt') as s: s.write("0") for j in range(1,ncol-1): s.write("\t{0}".format(j)) s.write("\n") ## Run coex_filter cmd_coex_filter = [self.COEX_FILTER, '-i', "{0}/{1}".format(self.RAWEXPR_DIR, self.EXPRESS_FN), '-o', "{0}/{1}".format(self.FLTRD_DIR, self.FLTRD_FN), '-m', param['method'], '-n', '10', '-s', "{0}/{1}".format(self.RAWEXPR_DIR, self.SAMPLE_FN), '-x', "{0}/{1}".format(self.RAWEXPR_DIR, self.GENELST_FN), '-t', 'y', '-j', self.PVFDT_FN] if 'num_features' in param: cmd_coex_filter.append("-n") cmd_coex_filter.append(str(param['num_features'])) if 'p_value' in param: cmd_coex_filter.append("-p") cmd_coex_filter.append(str(param['p_value'])) tool_process = subprocess.Popen(cmd_coex_filter, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: self.logger.info(stdout) if stderr is not None and len(stderr) > 0: self.logger.info(stderr) ## loading pvalue distribution FDT pvfdt = {'row_labels' :[], 'column_labels' : [], "data" : [[]]}; pvfdt = OrderedDict(pvfdt) with open(self.PVFDT_FN, 'r') as myfile: pvfdt = json.load(myfile) data_obj_name = "{0}.fdt".format(param['out_figure_object_name']) pvfdt['id'] = data_obj_name fig_properties = {"xlabel" : "-log2(p-value)", "ylabel" : "Number of features", "xlog_mode" : "-log2", "ylog_mode" : "none", "title" : "Histogram of P-values", "plot_type" : "histogram"} sstatus = ws.save_objects({'workspace' : param['workspace_name'], 'objects' : [{'type' : 'MAK.FloatDataTable', 'data' : pvfdt, 'name' : data_obj_name}]}) data_ref = "{0}/{1}/{2}".format(sstatus[0][6], sstatus[0][0], sstatus[0][4]) fig_properties['data_ref'] = data_ref sstatus = ws.save_objects({'workspace' : param['workspace_name'], 'objects' : [{'type' : 'CoExpression.FigureProperties', 'data' : fig_properties, 'name' : (param['out_figure_object_name'])}]}) result = fig_properties #END diff_p_distribution # At some point might do deeper type checking... if not isinstance(result, dict): raise ValueError('Method diff_p_distribution return value ' + 'result is not type dict as required.') # return the results return [result]
def run_filter_genes(workspace_service_url=None, param_file=None, level=logging.INFO, logger=None): """ Narrative Job Wrapper script to execute coex_filter Args: workspace_service_url: A url for the KBase Workspace service param_file: parameter file object_name: Name of the object in the workspace level: Logging level, defaults to logging.INFO. Returns: Output is written back in WS Authors: Shinjae Yoo """ try: os.makedirs(RAWEXPR_DIR) except: pass try: os.makedirs(FLTRD_DIR) except: pass try: os.makedirs(FINAL_DIR) except: pass if logger is None: logger = script_utils.stderrlogger(__file__) logger.info( "Starting conversion of KBaseFeatureValues.ExpressionMatrix to TSV") token = os.environ.get("KB_AUTH_TOKEN") with open(param_file) as paramh: param = json.load(paramh) cmd_dowload_cvt_tsv = [ FVE_2_TSV, '--workspace_service_url', workspace_service_url, '--workspace_name', param['workspace_name'], '--object_name', param['object_name'], '--working_directory', RAWEXPR_DIR, '--output_file_name', EXPRESS_FN ] # need shell in this case because the java code is depending on finding the KBase token in the environment # -- copied from FVE_2_TSV tool_process = subprocess.Popen(" ".join(cmd_dowload_cvt_tsv), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.info(stderr) logger.info("Identifying differentially expressed genes") ## Prepare sample file # detect num of columns with open("{0}/{1}".format(RAWEXPR_DIR, EXPRESS_FN), 'r') as f: fl = f.readline() ncol = len(fl.split('\t')) with open("{0}/{1}".format(RAWEXPR_DIR, SAMPLE_FN), 'wt') as s: s.write("0") for j in range(1, ncol - 1): s.write("\t{0}".format(j)) s.write("\n") ## Run coex_filter cmd_coex_filter = [ COEX_FILTER, '-i', "{0}/{1}".format(RAWEXPR_DIR, EXPRESS_FN), '-o', "{0}/{1}".format(FLTRD_DIR, FLTRD_FN), '-m', param['method'], '-s', "{0}/{1}".format(RAWEXPR_DIR, SAMPLE_FN), '-x', "{0}/{1}".format(RAWEXPR_DIR, GENELST_FN), '-t', 'y' ] if 'num_features' in param: cmd_coex_filter.append("-n") cmd_coex_filter.append(param['num_features']) if 'num_features' not in param and 'p_value' in param: cmd_coex_filter.append("-p") cmd_coex_filter.append(param['p_value']) if 'p_value' not in param and 'num_features' not in param: logger.error("One of p_value or num_features must be defined") sys.exit(2) #if 'p_value' in param and 'num_features' in param: # logger.error("Both of p_value and num_features cannot be defined together"); # sys.exit(3) tool_process = subprocess.Popen(cmd_coex_filter, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.info(stderr) ## Header correction with open("{0}/{1}".format(FLTRD_DIR, FLTRD_FN), 'r') as ff: fe = ff.readlines() with open("{0}/{1}".format(FLTRD_DIR, FLTRD_FN), 'w') as ff: ff.write( fl) # use original first line that has correct header information fe.pop(0) ff.writelines(fe) ## Upload FVE from biokbase.workspace.client import Workspace ws = Workspace(url=workspace_service_url, token=os.environ['KB_AUTH_TOKEN']) expr = ws.get_objects([{ 'workspace': param['workspace_name'], 'name': param['object_name'] }])[0]['data'] # change workspace to be the referenced object's workspace_name because it may not be in the same working ws due to referencing cmd_upload_expr = [ TSV_2_FVE, '--workspace_service_url', workspace_service_url, '--object_name', param['out_expr_object_name'], '--working_directory', FINAL_DIR, '--input_directory', FLTRD_DIR, '--output_file_name', FINAL_FN ] tmp_ws = param['workspace_name'] if 'genome_ref' in expr: cmd_upload_expr.append('--genome_object_name') obj_infos = ws.get_object_info_new( {"objects": [{ 'ref': expr['genome_ref'] }]})[0] if len(obj_infos) < 1: logger.error("Couldn't find {0} from the workspace".format( expr['genome_ref'])) raise Exception("Couldn't find {0} from the workspace".format( expr['genome_ref'])) cmd_upload_expr.append(obj_infos[1]) tmp_ws = obj_infos[7] logger.info("{0} => {1} / {2}".format(expr['genome_ref'], tmp_ws, obj_infos[1])) # updated ws name cmd_upload_expr.append('--workspace_name') cmd_upload_expr.append(tmp_ws) tool_process = subprocess.Popen(" ".join(cmd_upload_expr), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.info(stderr) with open("{0}/{1}".format(FINAL_DIR, FINAL_FN), 'r') as et: eo = json.load(et) if 'description' in expr: expr['description'] = "{0}, coex_filter by {1}".format( expr['description'], " ".join(cmd_coex_filter)) if 'feature_mapping' in expr: expr['feature_mapping'] = eo['feature_mapping'] expr['data'] = eo['data'] ws.save_objects({ 'workspace': param['workspace_name'], 'objects': [{ 'type': 'KBaseFeatureValues.ExpressionMatrix', 'data': expr, 'name': (param['out_expr_object_name']) }] }) ## Upload FeatureSet fs = { 'description': 'Differentially expressed genes generated by {0}'.format( " ".join(cmd_coex_filter)), 'elements': {} } with open("{0}/{1}".format(RAWEXPR_DIR, GENELST_FN), 'r') as glh: gl = glh.readlines() gl = [x.strip('\n') for x in gl] for g in gl: if 'genome_ref' in expr: fs['elements'][g] = [expr['genome_ref']] else: fs['elements'][g] = [] ws.save_objects({ 'workspace': param['workspace_name'], 'objects': [{ 'type': 'KBaseCollections.FeatureSet', 'data': fs, 'name': (param['out_fs_object_name']) }] })
def const_coex_net_clust(self, ctx, args): # ctx is the context object # return variables are: result #BEGIN const_coex_net_clust try: os.makedirs(self.RAWEXPR_DIR) except: pass try: os.makedirs(self.CLSTR_DIR) except: pass try: os.makedirs(self.FINAL_DIR) except: pass if self.logger is None: self.logger = script_utils.stderrlogger(__file__) result = {} self.logger.info("Starting conversion of KBaseFeatureValues.ExpressionMatrix to TSV") token = ctx['token'] param = args from biokbase.workspace.client import Workspace ws = Workspace(url=self.__WS_URL, token=token) expr = ws.get_objects([{'workspace': param['workspace_name'], 'name' : param['object_name']}])[0]['data'] eenv = os.environ.copy() eenv['KB_AUTH_TOKEN'] = token cmd_dowload_cvt_tsv = [self.FVE_2_TSV, '--workspace_service_url', self.__WS_URL, '--workspace_name', param['workspace_name'], '--object_name', param['object_name'], '--working_directory', self.RAWEXPR_DIR, '--output_file_name', self.EXPRESS_FN ] # need shell in this case because the java code is depending on finding the KBase token in the environment # -- copied from FVE_2_TSV tool_process = subprocess.Popen(" ".join(cmd_dowload_cvt_tsv), stderr=subprocess.PIPE, shell=True, env=eenv) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: self.logger.info(stdout) if stderr is not None and len(stderr) > 0: self.logger.info(stderr) #raise Exception(stderr) self.logger.info("Coexpression clustering analysis") ## Prepare sample file # detect num of columns with open("{0}/{1}".format(self.RAWEXPR_DIR, self.EXPRESS_FN), 'r') as f: fl = f.readline() ncol = len(fl.split('\t')) with open("{0}/{1}".format(self.RAWEXPR_DIR, self.SAMPLE_FN), 'wt') as s: s.write("0") for j in range(1,ncol-1): s.write("\t{0}".format(j)) s.write("\n") ## Run coex_cluster cmd_coex_cluster = [self.COEX_CLUSTER, '-t', 'y', '-i', "{0}/{1}".format(self.RAWEXPR_DIR, self.EXPRESS_FN), '-o', "{0}/{1}".format(self.CLSTR_DIR, self.CLSTR_FN), '-m', "{0}/{1}".format(self.CLSTR_DIR, self.CSTAT_FN) ] for p in ['net_method', 'minRsq', 'maxmediank', 'maxpower', 'clust_method', 'minModuleSize', 'detectCutHeight']: if p in param: cmd_coex_cluster.append("--{0}".format(p)) cmd_coex_cluster.append(str(param[p])) #sys.exit(2) #TODO: No error handling in narrative so we do graceful termination #if 'p_value' in param and 'num_features' in param: # self.logger.error("Both of p_value and num_features cannot be defined together"); # sys.exit(3) tool_process = subprocess.Popen(cmd_coex_cluster, stderr=subprocess.PIPE) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: self.logger.info(stdout) if stderr is not None and len(stderr) > 0: if re.search(r'^There were \d+ warnings \(use warnings\(\) to see them\)', stderr): self.logger.info(stderr) else: self.logger.error(stderr) raise Exception(stderr) # build index for gene list pos_index ={expr['data']['row_ids'][i]: i for i in range(0, len(expr['data']['row_ids']))} # parse clustering results cid2genelist = {} cid2stat = {} with open("{0}/{1}".format(self.CLSTR_DIR, self.CSTAT_FN),'r') as glh: glh.readline() # skip header for line in glh: cluster, mcor, msec = line.rstrip().replace('"','').split("\t") cid2stat[cluster]= [mcor, msec] with open("{0}/{1}".format(self.CLSTR_DIR, self.CLSTR_FN),'r') as glh: glh.readline() # skip header for line in glh: gene, cluster = line.rstrip().replace('"','').split("\t") if cluster not in cid2genelist: cid2genelist[cluster] = [] cid2genelist[cluster].append(gene) if(len(cid2genelist) < 1) : self.logger.error("Clustering failed") return empty_results("Error: No cluster output", expr,self.__WS_URL, param, self.logger, ws) #sys.exit(4) self.logger.info("Uploading the results onto WS") feature_clusters = [] for cluster in cid2genelist: feature_clusters.append( {"meancor": float(cid2stat[cluster][0]), "msec": float(cid2stat[cluster][0]), "id_to_pos" : { gene : pos_index[gene] for gene in cid2genelist[cluster]}}) ## Upload Clusters feature_clusters ={"original_data": "{0}/{1}".format(param['workspace_name'],param['object_name']), "feature_clusters": feature_clusters} ws.save_objects({'workspace' : param['workspace_name'], 'objects' : [{'type' : 'KBaseFeatureValues.FeatureClusters', 'data' : feature_clusters, 'name' : (param['out_object_name'])}]}) result = {'workspace_name' : param['workspace_name'], 'out_object_name' : param['out_object_name']} #END const_coex_net_clust # At some point might do deeper type checking... if not isinstance(result, dict): raise ValueError('Method const_coex_net_clust return value ' + 'result is not type dict as required.') # return the results return [result]
def transform(shock_service_url=None, handle_service_url=None, output_file_name=None, input_directory=None, working_directory=None, shock_id=None, handle_id=None, input_mapping=None, mzml_file_name=None, polarity=None, atlases=None, group=None, inclusion_order=None, normalization_factor=None, retention_correction=None, level=logging.INFO, logger=None): """ Converts mzML file to MetaboliteAtlas2_MAFileInfo json string. Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. output_file_name: A file name where the output JSON string should be stored. If the output file name is not specified the name will default to the name of the input file appended with '_finfo'. input_directory: The directory where files will be read from. working_directory: The directory the resulting json file will be written to. shock_id: Shock id for the hdf file if it already exists in shock handle_id: Handle id for the hdf file if it already exists as a handle input_mapping: JSON string mapping of input files to expected types. If you don't get this you need to scan the input directory and look for your files. level: Logging level, defaults to logging.INFO. atlases: List of MetaboliteAtlas atlas IDs. mzml_file_name: Name of the file, optional. Defaults to the file name. polarity: Run polarity. group: Run group. inclusion_order: Run inclusion_order. retention_correction: Run retention_correction. normalization_factor: Run normalization factor. Returns: JSON files on disk that can be saved as a KBase workspace objects. Authors: Steven Silvester """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of mzML to MetaboliteAtlas2.MAFileInfo") token = os.environ.get('KB_AUTH_TOKEN') if not working_directory or not os.path.isdir(working_directory): raise Exception( "The working directory {0} is not a valid directory!".format( working_directory)) logger.info("Scanning for mzML files.") valid_extensions = [".mzML"] files = os.listdir(input_directory) mzml_files = [ x for x in files if os.path.splitext(x)[-1] in valid_extensions ] assert len(mzml_files) != 0 logger.info("Found {0} files".format(len(mzml_files))) for fname in mzml_files: path = os.path.join(input_directory, fname) if not os.path.isfile(path): raise Exception( "The input file name {0} is not a file!".format(path)) hdf_file = mzml_loader.mzml_to_hdf(path) if shock_service_url: shock_info = script_utils.upload_file_to_shock(logger, shock_service_url, hdf_file, token=token) run_info = dict() run_info['mzml_file_name'] = (mzml_file_name or fname.replace('.mzML', '')) run_info['atlases'] = atlases or [] if polarity is not None: run_info['polarity'] = polarity if group is not None: run_info['group'] = group if inclusion_order is not None: run_info['inclusion_order'] = inclusion_order if normalization_factor is not None: run_info['normalization_factor'] = normalization_factor if retention_correction is not None: run_info['retention_correction'] = retention_correction if shock_service_url: handle_id = script_utils.getHandles(logger, shock_service_url, handle_service_url, [shock_info["id"]], token=token)[0] run_info["run_file_id"] = handle_id else: run_info['run_file_id'] = hdf_file output_file_name = fname.replace('.mzML', '_finfo.json') # This generates the json for the object objectString = simplejson.dumps(run_info, sort_keys=True, indent=4) output_file_path = os.path.join(working_directory, output_file_name) with open(output_file_path, "w") as outFile: outFile.write(objectString) logger.info("Conversion completed.")
parser.add_argument('--handle_id', help=script_details["Args"]["handle_id"], action='store', type=str, nargs='?', default=None, required=False) parser.add_argument('--input_mapping', help=script_details["Args"]["input_mapping"], action='store', type=unicode, nargs='?', default=None, required=False) # Example of a custom argument specific to this uploader parser.add_argument('--fasta_reference_only', help=script_details["Args"]["fasta_reference_only"], action='store', type=str, default="False", required=False) args, unknown = parser.parse_known_args() logger = script_utils.stderrlogger(__file__) logger.debug(args) try: if args.fasta_reference_only.lower() == "true": ref_only = True elif args.fasta_reference_only.lower() == "false": ref_only = False else: raise Exception("Expected true or false for fasta_reference_only.") transform(shock_service_url = args.shock_service_url, handle_service_url = args.handle_service_url, output_file_name = args.output_file_name, input_directory = args.input_directory, working_directory = args.working_directory,
def transform(shock_service_url=None, workspace_service_url=None, workspace_name=None, object_name=None, object_id=None, object_version=None, working_directory=None, output_file_name=None, level=logging.INFO, logger=None): """ Transforms KBaseGenomes.Genome and KBaseGenomes.ContigSet objects to Genbank file. Args: shock_service_url: If you have shock references you need to make. workspace_service_url: KBase Workspace URL workspace_name: Name of the workspace to save the data to object_name: Name of the genome object to save object_id: Id of the genome object to save object_version: Version of the genome object to save working_directory: A directory where you can do work output_file_name: File name for Genbank output Returns: Genbank output file. Authors: Shinjae Yoo, Matt Henderson, Marcin Joachimiak """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting transformation of KBaseGenomes.Genome to Genbank") classpath = [ "$KB_TOP/lib/jars/kbase/transform/GenBankTransform.jar", "$KB_TOP/lib/jars/kbase/genomes/kbase-genomes-20140411.jar", "$KB_TOP/lib/jars/kbase/common/kbase-common-0.0.6.jar", "$KB_TOP/lib/jars/jackson/jackson-annotations-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-core-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-databind-2.2.3.jar", "$KB_TOP/lib/jars/kbase/auth/kbase-auth-1398468950-3552bb2.jar", "$KB_TOP/lib/jars/kbase/workspace/WorkspaceClient-0.2.0.jar" ] argslist = [ "--shock_service_url {0}".format(shock_service_url), "--workspace_service_url {0}".format(workspace_service_url), "--workspace_name {0}".format(workspace_name), "--working_directory {0}".format(working_directory) ] logger.debug(object_name) if object_id is not None and len(object_id) > 0: argslist.append("--object_id {0}".format(object_id)) elif object_name is not None and len(object_name) > 0: object_name_print = object_name.replace("|", "\|") argslist.append("--object_name {0}".format(object_name_print)) else: logger.error( "Transformation from KBaseGenomes.Genome to Genbank.Genome failed due to no object name or id" ) sys.exit(1) if object_version is not None: try: int(object_version) except: logger.error( "Version number not correct! Expected integer, but found {0}". format(type(object_version))) sys.exit(1) argslist.append("--object_version {0}".format(object_version)) if output_file_name is not None and len(output_file_name) > 0: argslist.append("--output_file {0}".format(output_file_name)) arguments = [ "java", "-classpath", ":".join(classpath), "us.kbase.genbank.GenometoGbk", " ".join(argslist) ] logger.debug(arguments) # need shell in this case because the java code is depending on finding the KBase token in the environment tool_process = subprocess.Popen(" ".join(arguments), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.error( "Transformation from KBaseGenomes.Genome to Genbank.Genome failed on {0}" .format(object_name)) logger.error(stderr) sys.exit(1) logger.info( "Transformation from KBaseGenomes.Genome to Genbank.Genome completed.") sys.exit(0)
def main(): script_details = script_utils.parse_docs(transform.__doc__) import argparse parser = argparse.ArgumentParser(prog=__file__, description=script_details["Description"], epilog=script_details["Authors"]) parser.add_argument('--shock_service_url', help=script_details["Args"]["shock_service_url"], action='store', type=str, nargs='?', required=True) parser.add_argument('--handle_service_url', help=script_details["Args"]["handle_service_url"], action='store', type=str, nargs='?', default=None, required=False) parser.add_argument('--input_directory', help=script_details["Args"]["input_directory"], action='store', type=str, nargs='?', required=True) parser.add_argument('--working_directory', help=script_details["Args"]["working_directory"], action='store', type=str, nargs='?', required=True) parser.add_argument('--output_file_name', help=script_details["Args"]["output_file_name"], action='store', type=str, nargs='?', default=None, required=False) parser.add_argument('--shock_id', help=script_details["Args"]["shock_id"], action='store', type=str, nargs='?', default=None, required=False) parser.add_argument('--handle_id', help=script_details["Args"]["handle_id"], action='store', type=str, nargs='?', default=None, required=False) parser.add_argument('--input_mapping', help=script_details["Args"]["input_mapping"], action='store', type=unicode, nargs='?', default=None, required=False) # custom arguments specific to this uploader parser.add_argument('--polarity', help=script_details["Args"]["polarity"], action='store', type=int, required=False) parser.add_argument('--group', help=script_details["Args"]["group"], action='store', type=str, required=False) parser.add_argument('--inclusion_order', help=script_details["Args"]["inclusion_order"], action='store', type=int, required=False) parser.add_argument('--retention_correction', help=script_details["Args"]["retention_correction"], action='store', type=float, required=False) parser.add_argument('--atlases', help=script_details["Args"]["atlases"], action='store', type=str, nargs='?', required=False) parser.add_argument('--mzml_file_name', help=script_details["Args"]["mzml_file_name"], action='store', type=str, required=False) parser.add_argument('--normalization_factor', help=script_details["Args"]["normalization_factor"], action='store', type=float, required=False) args, unknown = parser.parse_known_args() logger = script_utils.stderrlogger(__file__) logger.debug(args) try: transform(shock_service_url=args.shock_service_url, handle_service_url=args.handle_service_url, output_file_name=args.output_file_name, input_directory=args.input_directory, working_directory=args.working_directory, shock_id=args.shock_id, handle_id=args.handle_id, input_mapping=args.input_mapping, mzml_file_name=args.mzml_file_name, polarity=args.polarity, atlases=args.atlases, group=args.group, inclusion_order=args.inclusion_order, normalization_factor=args.normalization_factor, retention_correction=args.retention_correction, logger=logger) except Exception as e: logger.exception(e) sys.exit(1)
def main(): parser = script_utils.ArgumentParser( prog=SCRIPT_NAME, description='Converts KBaseFile.AssemblyFile to ' + 'KBaseGenomes.ContigSet.', epilog='Authors: Jason Baumohl, Matt Henderson, Gavin Price') # The following 7 arguments should be standard to all uploaders parser.add_argument('--working_directory', help='Directory for temporary files', action='store', type=str, required=True) # Example of a custom argument specific to this uploader parser.add_argument('--workspace_service_url', help='workspace service url', action='store', type=str, required=True) parser.add_argument('--source_workspace_name', help='name of the source workspace', action='store', type=str, required=True) parser.add_argument('--destination_workspace_name', help='name of the target workspace', action='store', type=str, required=True) parser.add_argument('--source_object_name', help='name of the workspace object to convert', action='store', type=str, required=True) parser.add_argument('--destination_object_name', help='name for the produced ContigSet.', action='store', type=str, required=True) parser.add_argument( '--fasta_reference_only', help='Creates a reference to the fasta file in Shock, but does not ' + 'store the sequences in the workspace object. Not recommended ' + 'unless the fasta file is larger than 1GB. This is the default ' + 'behavior for files that large.', action='store_true', required=False) # ignore unknown arguments for now args, _ = parser.parse_known_args() logger = script_utils.stderrlogger(__file__) try: # make there's at least something for a token if not TOKEN: raise Exception("Unable to retrieve KBase Authentication token!") shock_url, shock_id, ref, source = download_workspace_data( args.workspace_service_url, args.source_workspace_name, args.source_object_name, args.working_directory, logger) inputfile = os.path.join(args.working_directory, args.source_object_name) cs = convert_to_contigs(None, None, inputfile, args.destination_object_name, args.working_directory, shock_id, None, args.fasta_reference_only, source, logger=logger) upload_workspace_data(cs, args.workspace_service_url, ref, args.destination_workspace_name, args.destination_object_name) except Exception, e: logger.exception(e) sys.exit(1)
parser.add_argument("--output_file_name", help=script_details["Args"]["output_file_name"], action="store", type=str, nargs="?", required=False) parser.add_argument("--working_directory", help=script_details["Args"]["working_directory"], action="store", type=str, nargs='?', required=False) args, unknown = parser.parse_known_args() logger = script_utils.stderrlogger(__file__, level=logging.DEBUG) try: transform(shock_service_url=args.shock_service_url, workspace_service_url=args.workspace_service_url, workspace_name=args.workspace_name, object_name=args.object_name, object_version=args.object_version, working_directory=args.working_directory, output_file_name=args.output_file_name, logger=logger) except Exception, e: logger.exception(e) sys.exit(1) sys.exit(0)
def transform(workspace_service_url=None, workspace_name=None, object_name=None, output_file_name=None, input_directory=None, working_directory=None, has_replicates=None, input_mapping=None, format_type=None, level=logging.INFO, logger=None): """ Converts Growth TSV file to json string of KBaseEnigmaMetals.GrowthMatrix type. Args: workspace_service_url: URL for a KBase Workspace service where KBase objects. are stored. workspace_name: The name of the destination workspace. object_name: The destination object name. output_file_name: A file name where the output JSON string should be stored. If the output file name is not specified the name will default to the name of the input file appended with '_output.json'. input_directory: The directory where files will be read from. working_directory: The directory the resulting json file will be written to. has_replicates: 0 if the input file contains marked series of replicates, 1 if the input file contains non-marked series of replicates, 2 if the input file contains no replicates. input_mapping: JSON string mapping of input files to expected types. If you don't get this you need to scan the input directory and look for your files. format_type: Mannually defined type of TSV file format. Returns: JSON files on disk that can be saved as a KBase workspace objects. Authors: Roman Sutormin, Alexey Kazakov """ if logger is None: logger = script_utils.stderrlogger(__file__) # logger.info("Starting conversion of Growth TSV to KBaseEnigmaMetals.GrowthMatrix") # token = os.environ.get('KB_AUTH_TOKEN') if not working_directory or not os.path.isdir(working_directory): raise Exception( "The working directory {0} is not a valid directory!".format( working_directory)) classpath = [ "$KB_TOP/lib/jars/kbase/transform/kbase_transform_deps.jar", "$KB_TOP/lib/jars/apache_commons/commons-cli-1.2.jar", "$KB_TOP/lib/jars/apache_commons/commons-lang3-3.1.jar", "$KB_TOP/lib/jars/ini4j/ini4j-0.5.2.jar", "$KB_TOP/lib/jars/jackson/jackson-annotations-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-core-2.2.3.jar", "$KB_TOP/lib/jars/jackson/jackson-databind-2.2.3.jar", "$KB_TOP/lib/jars/jetty/jetty-all-7.0.0.jar", "$KB_TOP/lib/jars/jna/jna-3.4.0.jar", "$KB_TOP/lib/jars/kbase/auth/kbase-auth-0.3.1.jar", "$KB_TOP/lib/jars/kbase/common/kbase-common-0.0.10.jar", "$KB_TOP/lib/jars/servlet/servlet-api-2.5.jar", "$KB_TOP/lib/jars/syslog4j/syslog4j-0.9.46.jar", "$KB_TOP/lib/jars/kbase/workspace/WorkspaceClient-0.2.0.jar" ] mc = "us.kbase.kbaseenigmametals.GrowthMatrixUploader" argslist = [ "--workspace_service_url {0}".format(workspace_service_url), "--workspace_name {0}".format(workspace_name), "--object_name {0}".format(object_name), "--input_directory {0}".format(input_directory), "--has_replicates {0}".format(has_replicates), "--working_directory {0}".format(working_directory) ] if output_file_name: argslist.append("--output_file_name {0}".format(output_file_name)) if input_mapping: argslist.append("--input_mapping {0}".format(input_mapping)) argslist.append("--format_type {0}".format(format_type)) arguments = [ "java", "-classpath", ":".join(classpath), mc, " ".join(argslist) ] logger.info(arguments) # need shell in this case because the java code is depending on finding the KBase token in the environment tool_process = subprocess.Popen(" ".join(arguments), stderr=subprocess.PIPE, shell=True) stdout, stderr = tool_process.communicate() if stdout is not None and len(stdout) > 0: logger.info(stdout) if stderr is not None and len(stderr) > 0: logger.error(stderr) if tool_process.returncode: logger.error( "Transformation from TSV.Growth to KBaseEnigmaMetals.GrowthMatrix failed on {0}" .format(input_directory)) sys.exit(1) logger.info("Conversion completed.")
if __name__ == "__main__": script_details = script_utils.parse_docs(validate.__doc__) import argparse parser = argparse.ArgumentParser(prog=__file__, description=script_details["Description"], epilog=script_details["Authors"]) parser.add_argument("--input_directory", help=script_details["Args"]["input_directory"], type=str, nargs="?", required=True) parser.add_argument("--working_directory", help=script_details["Args"]["working_directory"], type=str, nargs="?", required=True) args, unknown = parser.parse_known_args() returncode = 0 try: validate(input_directory = args.input_directory, working_directory = args.working_directory) except Exception, e: logger = script_utils.stderrlogger(__file__, logging.INFO) logger.exception(e) returncode = 1 sys.stdout.flush() sys.stderr.flush() os.close(sys.stdout.fileno()) os.close(sys.stderr.fileno()) sys.exit(returncode)
def transform(shock_service_url=None, handle_service_url=None, output_file_name=None, input_directory=None, working_directory=None, shock_id=None, handle_id=None, input_mapping=None, fasta_reference_only=False, level=logging.INFO, logger=None): """ Converts FASTA file to KBaseGenomes.ContigSet json string. Note the MD5 for the contig is generated by uppercasing the sequence. The ContigSet MD5 is generated by taking the MD5 of joining the sorted list of individual contig's MD5s with a comma separator. Args: shock_service_url: A url for the KBase SHOCK service. handle_service_url: A url for the KBase Handle Service. output_file_name: A file name where the output JSON string should be stored. If the output file name is not specified the name will default to the name of the input file appended with '_contig_set' input_directory: The directory where files will be read from. working_directory: The directory the resulting json file will be written to. shock_id: Shock id for the fasta file if it already exists in shock handle_id: Handle id for the fasta file if it already exists as a handle input_mapping: JSON string mapping of input files to expected types. If you don't get this you need to scan the input directory and look for your files. fasta_reference_only: Creates a reference to the fasta file in Shock, but does not store the sequences in the workspace object. Not recommended unless the fasta file is larger than 1GB. This is the default behavior for files that large. level: Logging level, defaults to logging.INFO. Returns: JSON file on disk that can be saved as a KBase workspace object. Authors: Jason Baumohl, Matt Henderson """ if logger is None: logger = script_utils.stderrlogger(__file__) logger.info("Starting conversion of FASTA to KBaseGenomes.ContigSet") token = os.environ.get('KB_AUTH_TOKEN') if input_mapping is None: logger.info("Scanning for FASTA files.") valid_extensions = [".fa",".fasta",".fna",".fas"] files = os.listdir(input_directory) fasta_files = [x for x in files if os.path.splitext(x)[-1] in valid_extensions] if (len(fasta_files) == 0): raise Exception("The input file does not have one of the following extensions .fa, .fasta, .fas or .fna") logger.info("Found {0}".format(str(fasta_files))) input_file_name = os.path.join(input_directory,files[0]) if len(fasta_files) > 1: logger.warning("Not sure how to handle multiple FASTA files in this context. Using {0}".format(input_file_name)) else: input_file_name = os.path.join(os.path.join(input_directory, "FASTA.DNA.Assembly"), simplejson.loads(input_mapping)["FASTA.DNA.Assembly"]) logger.info("Building Object.") if not os.path.isfile(input_file_name): raise Exception("The input file name {0} is not a file!".format(input_file_name)) if not os.path.isdir(working_directory): raise Exception("The working directory {0} is not a valid directory!".format(working_directory)) logger.debug(fasta_reference_only) # default if not too large contig_set_has_sequences = True if fasta_reference_only: contig_set_has_sequences = False fasta_filesize = os.stat(input_file_name).st_size if fasta_filesize > 1000000000: # Fasta file too large to save sequences into the ContigSet object. contigset_warn = """The FASTA input file seems to be too large. A ContigSet object will be created without sequences, but will contain a reference to the file.""" logger.warning(contigset_warn) contig_set_has_sequences = False input_file_handle = open(input_file_name, 'r') fasta_header = None sequence_list = [] fasta_dict = dict() first_header_found = False contig_set_md5_list = [] # Pattern for replacing white space pattern = re.compile(r'\s+') sequence_exists = False valid_chars = "-AaCcGgTtUuWwSsMmKkRrYyBbDdHhVvNn" amino_acid_specific_characters = "PpLlIiFfQqEe" for current_line in input_file_handle: if (current_line[0] == ">"): # found a header line # Wrap up previous fasta sequence if (not sequence_exists) and first_header_found: logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) if not first_header_found: first_header_found = True else: # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence : logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) # for character in total_sequence: seq_count = collections.Counter(total_sequence) seq_dict = dict(seq_count) for character in seq_dict: if character not in valid_chars: if character in amino_acid_specific_characters: raise Exception("This fasta file may have amino acids in it instead of the required nucleotides.") raise Exception("This FASTA file has non nucleic acid characters : {0}".format(character)) # fasta_key = fasta_header.strip() try: fasta_key , fasta_description = fasta_header.strip().split(' ',1) except: fasta_key = fasta_header.strip() fasta_description = None if fasta_key == '': raise Exception("One fasta header lines '>' does not have an identifier associated with it") contig_dict = dict() contig_dict["id"] = fasta_key contig_dict["length"] = len(total_sequence) contig_dict["name"] = fasta_key if fasta_description is None: contig_dict["description"] = "Note MD5 is generated from uppercasing the sequence" else: contig_dict["description"] = "%s. Note MD5 is generated from uppercasing the sequence" % (fasta_description) contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"] = contig_md5 contig_set_md5_list.append(contig_md5) if contig_set_has_sequences: contig_dict["sequence"]= total_sequence else: contig_dict["sequence"]= "" if fasta_key in fasta_dict: raise Exception("The fasta header {0} appears more than once in the file ".format(fasta_key)) else: fasta_dict[fasta_key] = contig_dict # get set up for next fasta sequence sequence_list = [] sequence_exists = False fasta_header = current_line.replace('>','') else: sequence_list.append(current_line) sequence_exists = True input_file_handle.close() # wrap up last fasta sequence if (not sequence_exists) and first_header_found: logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) elif not first_header_found : logger.error("There are no contigs in this file") raise Exception("There are no contigs in this file") else: # build up sequence and remove all white space total_sequence = ''.join(sequence_list) total_sequence = re.sub(pattern, '', total_sequence) if not total_sequence : logger.error("There is no sequence related to FASTA record : {0}".format(fasta_header)) raise Exception("There is no sequence related to FASTA record : {0}".format(fasta_header)) # for character in total_sequence: seq_count = collections.Counter(total_sequence) seq_dict = dict(seq_count) for character in seq_dict: if character not in valid_chars: if character in amino_acid_specific_characters: raise Exception("This fasta file may have amino acids in it instead of the required nucleotides.") raise Exception("This FASTA file has non nucleic acid characters : {0}".format(character)) # fasta_key = fasta_header.strip() try: fasta_key , fasta_description = fasta_header.strip().split(' ',1) except: fasta_key = fasta_header.strip() fasta_description = None if fasta_key == '': raise Exception("One fasta header lines '>' does not have an identifier associated with it") contig_dict = dict() contig_dict["id"] = fasta_key contig_dict["length"] = len(total_sequence) contig_dict["name"] = fasta_key if fasta_description is None: contig_dict["description"] = "Note MD5 is generated from uppercasing the sequence" else: contig_dict["description"] = "%s. Note MD5 is generated from uppercasing the sequence" % (fasta_description) contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() contig_dict["md5"]= contig_md5 contig_set_md5_list.append(contig_md5) if contig_set_has_sequences: contig_dict["sequence"] = total_sequence else: contig_dict["sequence"]= "" if fasta_key in fasta_dict: raise Exception("The fasta header {0} appears more than once in the file ".format(fasta_key)) else: fasta_dict[fasta_key] = contig_dict if output_file_name is None: # default to input file name minus file extenstion adding "_contig_set" to the end base = os.path.basename(input_file_name) output_file_name = "{0}_contig_set.json".format(os.path.splitext(base)[0]) contig_set_dict = dict() contig_set_dict["md5"] = hashlib.md5(",".join(sorted(contig_set_md5_list))).hexdigest() contig_set_dict["id"] = output_file_name contig_set_dict["name"] = output_file_name contig_set_dict["source"] = "KBase" contig_set_dict["source_id"] = os.path.basename(input_file_name) contig_set_dict["contigs"] = [fasta_dict[x] for x in sorted(fasta_dict.keys())] if shock_id is None: shock_info = script_utils.upload_file_to_shock(logger, shock_service_url, input_file_name, token=token) shock_id = shock_info["id"] contig_set_dict["fasta_ref"] = shock_id # For future development if the type is updated to the handle_reference instead of a shock_reference # This generates the json for the object objectString = simplejson.dumps(contig_set_dict, sort_keys=True, indent=4) if len(contig_set_dict["contigs"]) == 0: raise Exception("There appears to be no FASTA DNA Sequences in the input file.") #The workspace has a 1GB limit if sys.getsizeof(objectString) > 1E9 : contig_set_dict["contigs"] = [] objectString = simplejson.dumps(contig_set_dict, sort_keys=True, indent=4) logger.warning("The fasta file has a very large number of contigs thus resulting in an object being too large if " "the contigs are to have metadata. The resulting contigset will not have individual metadata for the contigs.") logger.info("ContigSet data structure creation completed. Writing out JSON.") output_file_path = os.path.join(working_directory,output_file_name) with open(output_file_path, "w") as outFile: outFile.write(objectString) logger.info("Conversion completed.")