def test_fastqc_app(self): # create ws, and load test reads wsName = self.getWsName() ru = ReadsUtils(os.environ['SDK_CALLBACK_URL']) input_file_ref = ru.upload_reads({ 'fwd_file': self.small_fq_test_file2, 'sequencing_tech': 'tech1', 'wsname': wsName, 'name': 'reads1', 'interleaved': 1 })['obj_ref'] input_params = {'input_ws': wsName, 'input_file_ref': input_file_ref} output = self.getImpl().runFastQC(self.getContext(), input_params)[0] self.assertIn('report_name', output) self.assertIn('report_ref', output) # pprint(output) report = self.getWsClient().get_objects2( {'objects': [{ 'ref': output['report_ref'] }]})['data'][0]['data'] # pprint(report) self.assertIn('direct_html', report) self.assertIn('file_links', report) self.assertIn('html_links', report) self.assertIn('objects_created', report) self.assertIn('text_message', report)
def getPairedEndLibInfo(self, lib_name): if hasattr(self.__class__, 'pairedEndLibInfo'): if self.__class__.pairedEndLibInfo.get(lib_name): return self.__class__.pairedEndLibInfo[lib_name] # copy the local test file to the shared scratch space so that the ReadsUtils # container can see it. test_fastq_file_local = os.path.join('data', 'reads', lib_name) test_fastq_file_scratch = os.path.join( self.scratch, os.path.basename(test_fastq_file_local)) shutil.copy(test_fastq_file_local, test_fastq_file_scratch) # call the ReadsUtils libary to upload the test data to KBase ru = ReadsUtils(os.environ['SDK_CALLBACK_URL']) paired_end_ref = ru.upload_reads({ 'fwd_file': test_fastq_file_scratch, 'sequencing_tech': 'artificial reads', 'interleaved': 1, 'wsname': self.getWsName(), 'name': lib_name })['obj_ref'] # get the object metadata for the new test dataset new_obj_info = self.ws.get_object_info_new( {'objects': [{ 'ref': paired_end_ref }]}) if not hasattr(self.__class__, 'pairedEndLibInfo'): self.__class__.pairedEndLibInfo = dict() self.__class__.pairedEndLibInfo[lib_name] = new_obj_info[0] return new_obj_info[0]
def loadPairedEndReads(self, forward_data_file, reverse_data_file, reads_name): # if hasattr(self.__class__, 'pairedEndLibInfo'): # return self.__class__.pairedEndLibInfo # 1) upload files to shock # shared_dir = "/kb/module/work/tmp" # forward_data_file = '../test/testReads/small.forward.fq' forward_file = os.path.join(self.scratch, os.path.basename(forward_data_file)) shutil.copy(forward_data_file, forward_file) # reverse_data_file = '../test/testReads/small.reverse.fq' reverse_file = os.path.join(self.scratch, os.path.basename(reverse_data_file)) shutil.copy(reverse_data_file, reverse_file) ru = ReadsUtils(os.environ['SDK_CALLBACK_URL']) pe_reads_ref = ru.upload_reads({ 'fwd_file': forward_file, 'rev_file': reverse_file, 'sequencing_tech': 'artificial reads', 'interleaved': 0, 'wsname': self.getWsName(), 'name': reads_name })['obj_ref'] # self.__class__.pe_reads_ref = pe_reads_ref print('Loaded PairedEndReads: ' + pe_reads_ref) new_obj_info = self.wsClient.get_object_info_new( {'objects': [{ 'ref': pe_reads_ref }]}) self.__class__.pairedEndLibInfo = new_obj_info[0] pprint(pformat(new_obj_info)) # return new_obj_info[0] return pe_reads_ref
def get_reads_RU(self, refs, console): readcli = ReadsUtils(self.callbackURL, token=self.token) typeerr = ('Supported types: KBaseFile.SingleEndLibrary ' + 'KBaseFile.PairedEndLibrary ' + 'KBaseAssembly.SingleEndLibrary ' + 'KBaseAssembly.PairedEndLibrary') try: reads = readcli.download_reads({'read_libraries': refs, 'interleaved': 'true', 'gzipped': None })['files'] except ServerError as se: self.log(console, 'logging stacktrace from dynamic client error') self.log(console, se.data) if typeerr in se.message: prefix = se.message.split('.')[0] raise ValueError( prefix + '. Only the types ' + 'KBaseAssembly.PairedEndLibrary ' + 'and KBaseFile.PairedEndLibrary are supported') else: raise self.log(console, 'Got reads data from converter:\n' + pformat(reads)) return reads
class DownloadFastqUtils: def __init__(self): self.callbackURL = os.environ['SDK_CALLBACK_URL'] self.au = AssemblyUtil(self.callbackURL) self.ru = ReadsUtils(self.callbackURL) pass def _stage_input_file(self, ref, reads_type): if reads_type == 'KBaseFile.PairedEndLibrary' or 'KBaseAssembly.PairedEndLibrary': input_file_info = self.ru.download_reads({ 'read_libraries': [ref], 'interleaved': 'true' })['files'][ref] elif reads_type == 'KBaseFile.SingleEndLibrary' or 'KBaseAssembly.SingleEndLibrary': input_file_info = self.ru.download_reads({'read_libraries': [ref]})['files'][ref] else: raise ValueError("Can't download_reads() for object type: '" + str(reads_type) + "'") input_file_info['input_ref'] = ref file_location = input_file_info['files']['fwd'] interleaved = False if input_file_info['files']['type'] == 'interleaved': interleaved = True return input_file_info def download_genome(self, genomeref): file = self.au.get_assembly_as_fasta({'ref': genomeref}) return file
def fetch_reads_from_reference(ref, callback_url): """ Fetch a FASTQ file (or 2 for paired-end) from a reads reference. Returns the following structure: { "style": "paired", "single", or "interleaved", "file_fwd": path_to_file, "file_rev": path_to_file, only if paired end, "object_ref": reads reference for downstream convenience. } """ try: print("Fetching reads from object {}".format(ref)) reads_client = ReadsUtils(callback_url) reads_dl = reads_client.download_reads({ "read_libraries": [ref], "interleaved": "false" }) pprint(reads_dl) reads_files = reads_dl['files'][ref]['files'] ret_reads = { "object_ref": ref, "style": reads_files["type"], "file_fwd": reads_files["fwd"] } if reads_files.get("rev", None) is not None: ret_reads["file_rev"] = reads_files["rev"] return ret_reads except: print( "Unable to fetch a file from expected reads object {}".format(ref)) raise
def prepare_single_run(self, input_info, assembly_or_genome_ref, bwa_index_info, ws_for_cache): ''' Given a reads ref and an assembly, setup the bwa index ''' # first setup the bwa index of the assembly input_configuration = {'bwa_index_info': bwa_index_info} if not bwa_index_info: bwaIndexBuilder = BwaIndexBuilder(self.scratch_dir, self.workspace_url, self.callback_url, self.srv_wiz_url, self.provenance) index_result = bwaIndexBuilder.get_index({ 'ref': assembly_or_genome_ref, 'ws_for_cache': ws_for_cache }) input_configuration['bwa_index_info'] = index_result # next download the reads read_lib_ref = input_info['ref'] read_lib_info = input_info['info'] reads_params = { 'read_libraries': [read_lib_ref], 'interleaved': 'false', 'gzipped': None } ru = ReadsUtils(self.callback_url) reads = ru.download_reads(reads_params)['files'] input_configuration['reads_lib_type'] = self.get_type_from_obj_info( read_lib_info).split('.')[1] input_configuration['reads_files'] = reads[read_lib_ref] input_configuration['reads_lib_ref'] = read_lib_ref return input_configuration
def download_interleaved_reads(callback_url, reads_upa): ru = ReadsUtils(callback_url) reads_info = ru.download_reads({ 'read_libraries': [reads_upa], 'interleaved': 'true', 'gzipped': None })['files'][reads_upa] return reads_info
def __init__(self, callbaack_url, scratch, wdl='../../metaAssembly/'): self.callback_url = callbaack_url self.scratch = scratch self.special = special(self.callback_url) self.ru = ReadsUtils(self.callback_url) self.au = AssemblyUtil(self.callback_url) self.report = KBaseReport(self.callback_url) self.wdl_base = wdl
def __init__(self, config, callback_url, workspace_id, cpus): self.shared_folder = config['scratch'] self.callback_url = callback_url self.cpus = cpus self.ru = ReadsUtils(self.callback_url) logging.basicConfig(format='%(created)s %(levelname)s: %(message)s', level=logging.INFO)
def __init__(self, config): self.callback_url = config['SDK_CALLBACK_URL'] self.token = config['KB_AUTH_TOKEN'] self.scratch = os.path.join(config['scratch'], 'import_SRA_' + str(uuid.uuid4())) handler_utils._mkdir_p(self.scratch) self.dfu = DataFileUtil(self.callback_url) self.ru = ReadsUtils(self.callback_url) self.uploader_utils = UploaderUtil(config)
def __init__(self, config): self.callback_url = os.environ['SDK_CALLBACK_URL'] self.scratch = config['scratch'] self.shock_url = config['shock-url'] self.ws_url = config['workspace-url'] self.dfu = DataFileUtil(self.callback_url) self.ru = ReadsUtils(self.callback_url) self.au = AssemblyUtil(self.callback_url) self.mgu = MetagenomeUtils(self.callback_url)
def download_reads(self, token, reads_ref): try: readsUtils_Client = ReadsUtils (url=self.callback_url, token=token) # SDK local readsLibrary = readsUtils_Client.download_reads ({'read_libraries': [reads_ref], 'interleaved': 'true' }) reads_file_path = readsLibrary['files'][reads_ref]['files']['fwd'] except Exception as e: raise ValueError('Unable to get reads library object from workspace: (' + reads_ref +")\n" + str(e)) return reads_file_path
def run_mash_sketch(self, ctx, params): """ Generate a sketch file from a fasta/fastq file :param params: instance of type "MashSketchParams" (* * Pass in **one of** input_path, assembly_ref, or reads_ref * input_path - string - local file path to an input fasta/fastq * assembly_ref - string - workspace reference to an Assembly type * reads_ref - string - workspace reference to a Reads type * Optionally, pass in a boolean indicating whether you are using paired-end reads. * paired_ends - boolean - whether you are passing in paired ends) -> structure: parameter "input_path" of String, parameter "assembly_ref" of String, parameter "reads_ref" of String, parameter "paired_ends" of type "boolean" (params: input_upa: workspace reference to an assembly object workspace_name: name of current workspace search_db: database to search n_max_results: number of results to return, integer between 1 and 100) :returns: instance of type "MashSketchResults" (* * Returns the local scratch file path of the generated sketch file. * Will have the extension '.msh') -> structure: parameter "sketch_path" of String """ # ctx is the context object # return variables are: results #BEGIN run_mash_sketch if 'reads_ref' in params: reads_utils = ReadsUtils(self.callbackURL) result = reads_utils.download_reads({ 'read_libraries': [params['reads_ref']], 'interleaved': 'true' }) input_path = result['files'][params['reads_ref']]['files']['fwd'] elif 'assembly_ref' in params: assembly_util = AssemblyUtil(self.callbackURL) result = assembly_util.get_assembly_as_fasta({'ref': params['assembly_ref']}) input_path = result['path'] elif 'input_path' in params: input_path = params['input_path'] else: raise ValueError( 'Invalid params; must provide one of `reads_ref`, `assembly_ref`, or `input_path`.' ) mash_utils = MashUtils(self.config, self.auth_token) output_file_path = mash_utils.mash_sketch(input_path, paired_ends=params.get('paired_ends')) results = {'sketch_path': output_file_path} #END run_mash_sketch # At some point might do deeper type checking... if not isinstance(results, dict): raise ValueError('Method run_mash_sketch return value ' + 'results is not type dict as required.') # return the results return [results]
def __init__(self, prj_dir, config): self.workspace_url = config['workspace-url'] self.callback_url = config['SDK_CALLBACK_URL'] self.token = config['KB_AUTH_TOKEN'] if 'shock-url' in config: self.shock_url = config['shock-url'] if 'handle-service-url' in config: self.handle_url = config['handle-service-url'] self.ws_client = Workspace(self.workspace_url, token=self.token) self.ru = ReadsUtils(self.callback_url, token=self.token) self.au = AssemblyUtil(self.callback_url, token=self.token) self.kbr = KBaseReport(self.callback_url) self.kbq = kb_quast(self.callback_url) self.proj_dir = prj_dir self.prog_runner = Program_Runner(self.MaSuRCA_BIN, self.proj_dir)
def load_pe_reads(fwd_file, rev_file): """ Copies from given dir to scratch. Then calls ReadsUtils to upload from scratch. """ callback_url = os.environ['SDK_CALLBACK_URL'] fwd_file_path = file_to_scratch(fwd_file, overwrite=True) rev_file_path = file_to_scratch(rev_file, overwrite=True) ru = ReadsUtils(callback_url) pe_reads_params = { 'fwd_file': fwd_file_path, 'rev_file': rev_file_path, 'sequencing_tech': 'Illumina', 'wsname': get_ws_name(), 'name': 'MyPairedEndLibrary' } return ru.upload_reads(pe_reads_params)['obj_ref']
def setUpClass(cls): token = environ.get('KB_AUTH_TOKEN', None) config_file = environ.get('KB_DEPLOYMENT_CONFIG', None) cls.cfg = {} config = ConfigParser() config.read(config_file) for nameval in config.items('kb_staging_exporter'): cls.cfg[nameval[0]] = nameval[1] # Getting username from Auth profile for token authServiceUrl = cls.cfg['auth-service-url'] auth_client = _KBaseAuth(authServiceUrl) user_id = auth_client.get_user(token) # WARNING: don't call any logging methods on the context object, # it'll result in a NoneType error cls.ctx = MethodContext(None) cls.ctx.update({'token': token, 'user_id': user_id, 'provenance': [ {'service': 'kb_staging_exporter', 'method': 'please_never_use_it_in_production', 'method_params': [] }], 'authenticated': 1}) cls.wsURL = cls.cfg['workspace-url'] cls.wsClient = workspaceService(cls.wsURL) cls.serviceImpl = kb_staging_exporter(cls.cfg) cls.scratch = cls.cfg['scratch'] cls.callback_url = os.environ['SDK_CALLBACK_URL'] cls.ru = ReadsUtils(cls.callback_url) cls.au = AssemblyUtil(cls.callback_url) cls.gfu = GenomeFileUtil(cls.callback_url, service_ver='dev') cls.rau = ReadsAlignmentUtils(cls.callback_url)
def load_reads(callback_url, ws_name, tech, file_fwd, file_rev, target_name): """ Loads FASTQ files as either SingleEndLibrary or PairedEndLibrary. If file_rev is None, then we get a single end, otherwise, paired. """ reads_util = ReadsUtils(callback_url) upload_params = { "wsname": ws_name, "fwd_file": file_fwd, "name": target_name, "sequencing_tech": tech } if file_rev is not None: upload_params["rev_file"] = file_rev reads_ref = reads_util.upload_reads(upload_params) return reads_ref["obj_ref"]
def setUpClass(cls): token = os.environ.get('KB_AUTH_TOKEN', None) config_file = os.environ.get('KB_DEPLOYMENT_CONFIG', None) cls.cfg = {} config = ConfigParser() config.read(config_file) for nameval in config.items('gottcha2'): cls.cfg[nameval[0]] = nameval[1] # Getting username from Auth profile for token authServiceUrl = cls.cfg['auth-service-url'] auth_client = _KBaseAuth(authServiceUrl) user_id = auth_client.get_user(token) # WARNING: don't call any logging methods on the context object, # it'll result in a NoneType error cls.ctx = MethodContext(None) cls.ctx.update({'token': token, 'user_id': user_id, 'provenance': [ {'service': 'gottcha2', 'method': 'please_never_use_it_in_production', 'method_params': [] }], 'authenticated': 1}) cls.wsURL = cls.cfg['workspace-url'] cls.wsClient = Workspace(cls.wsURL) cls.serviceImpl = gottcha2(cls.cfg) cls.scratch = cls.cfg['scratch'] cls.callback_url = os.environ['SDK_CALLBACK_URL'] suffix = int(time.time() * 1000) cls.wsName = "test_ContigFilter_" + str(suffix) cls.ru = ReadsUtils(os.environ['SDK_CALLBACK_URL']) ret = cls.wsClient.create_workspace({'workspace': cls.wsName}) # noqa
def loadSEReads(self, reads_file_path): # if hasattr(self.__class__, 'reads_ref'): # return self.__class__.reads_ref se_reads_name = os.path.basename(reads_file_path) fq_path = os.path.join(self.scratch, se_reads_name) shutil.copy(reads_file_path, fq_path) ru = ReadsUtils(self.callback_url) reads_ref = ru.upload_reads({ 'fwd_file': fq_path, 'wsname': self.getWsName(), 'name': se_reads_name.split('.')[0], 'sequencing_tech': 'kb reads' })['obj_ref'] # self.__class__.reads_ref = reads_ref return reads_ref
def __init__(self, prj_dir, config): self.workspace_url = config['workspace-url'] self.callback_url = config['SDK_CALLBACK_URL'] self.token = config['KB_AUTH_TOKEN'] if 'shock-url' in config: self.shock_url = config['shock-url'] if 'handle-service-url' in config: self.handle_url = config['handle-service-url'] self.ws_client = Workspace(self.workspace_url, token=self.token) self.ru = ReadsUtils(self.callback_url, token=self.token, service_ver='release') self.au = AssemblyUtil(self.callback_url, token=self.token, service_ver='release') self.kbr = KBaseReport(self.callback_url) self.kbq = kb_quast(self.callback_url) self.proj_dir = prj_dir self.spades_version = 'SPAdes-' + os.environ['SPADES_VERSION']
def loadSingleEndReads(self, single_end_fastq, params): if hasattr(self.__class__, 'se_reads_ref'): return self.__class__.se_reads_ref fq_path = os.path.join(self.scratch, 'extracted_WT_rep1.fastq') #shutil.copy(os.path.join('data', 'bt_test_data', 'extracted_WT_rep1.fastq'), fq_path) shutil.copy(single_end_fastq, fq_path) ru = ReadsUtils(self.callback_url) se_reads_ref = ru.upload_reads({ 'fwd_file': fq_path, 'wsname': params["workspace_name"], 'name': 'test_readsSE', 'sequencing_tech': 'artificial reads' })['obj_ref'] self.__class__.se_reads_ref = se_reads_ref print('Loaded SingleEndReads: ' + se_reads_ref) return se_reads_ref
def setUpClass(cls): token = environ.get('KB_AUTH_TOKEN', None) config_file = environ.get('KB_DEPLOYMENT_CONFIG', None) cls.cfg = {} config = ConfigParser() config.read(config_file) for nameval in config.items('kb_deseq'): cls.cfg[nameval[0]] = nameval[1] # Getting username from Auth profile for token authServiceUrl = cls.cfg['auth-service-url'] auth_client = _KBaseAuth(authServiceUrl) user_id = auth_client.get_user(token) # WARNING: don't call any logging methods on the context object, # it'll result in a NoneType error cls.ctx = MethodContext(None) cls.ctx.update({ 'token': token, 'user_id': user_id, 'provenance': [{ 'service': 'kb_deseq', 'method': 'please_never_use_it_in_production', 'method_params': [] }], 'authenticated': 1 }) cls.wsURL = cls.cfg['workspace-url'] cls.wsClient = Workspace(cls.wsURL) cls.ws = Workspace(cls.wsURL, token=token) cls.serviceImpl = kb_deseq(cls.cfg) cls.serviceImpl.status(cls.ctx) cls.scratch = cls.cfg['scratch'] cls.callback_url = os.environ['SDK_CALLBACK_URL'] cls.gfu = GenomeFileUtil(cls.callback_url, service_ver='dev') cls.dfu = DataFileUtil(cls.callback_url) cls.ru = ReadsUtils(cls.callback_url) cls.rau = ReadsAlignmentUtils(cls.callback_url) cls.stringtie = kb_stringtie(cls.callback_url) cls.eu = ExpressionUtils(cls.callback_url) cls.deseq_runner = DESeqUtil(cls.cfg) suffix = int(time.time() * 1000) cls.wsName = "test_kb_stringtie_" + str(suffix) cls.wsClient.create_workspace({'workspace': cls.wsName}) cls.dfu.ws_name_to_id(cls.wsName) # public on CI cls.expressionset_ref = '30957/52/41' cls.condition_1 = 'Ecoli_WT' cls.condition_2 = 'Ecoli_ydcR' # public on Appdev cls.expressionset_ref = '60454/19' cls.condition_1 = 'WT' cls.condition_2 = 'Hy5'
def loadSingleEndReads(self): if hasattr(self.__class__, 'se_reads_ref'): return self.__class__.se_reads_ref # return '23735/2/1' fq_path = os.path.join(self.scratch, 'extracted_WT_rep1.fastq') shutil.copy( os.path.join('data', 'bt_test_data', 'extracted_WT_rep1.fastq'), fq_path) ru = ReadsUtils(self.callback_url) se_reads_ref = ru.upload_reads({ 'fwd_file': fq_path, 'wsname': self.getWsName(), 'name': 'test_readsSE', 'sequencing_tech': 'artificial reads' })['obj_ref'] self.__class__.se_reads_ref = se_reads_ref print('Loaded SingleEndReads: ' + se_reads_ref) return se_reads_ref
def fetch_reads_files(self, reads_upas): """ From a list of reads UPAs, uses ReadsUtils to fetch the reads as files. Returns them as a dictionary from reads_upa -> filename """ if reads_upas is None: raise ValueError("reads_upas must be a list of UPAs") if len(reads_upas) == 0: raise ValueError("reads_upas must contain at least one UPA") ru = ReadsUtils(self.callback_url) reads_info = ru.download_reads(({ 'read_libraries': reads_upas, 'interleaved': 'true', 'gzipped': None }))['files'] file_set = dict() for reads in reads_info: file_set[reads] = reads_info[reads]['files']['fwd'] return file_set
def setUpClass(cls): cls.token = environ.get('KB_AUTH_TOKEN') cls.callbackURL = environ.get('SDK_CALLBACK_URL') print('CB URL: ' + cls.callbackURL) # WARNING: don't call any logging methods on the context object, # it'll result in a NoneType error cls.ctx = MethodContext(None) cls.ctx.update({ 'token': cls.token, 'provenance': [{ 'service': 'kb_unicycler', 'method': 'please_never_use_it_in_production', 'method_params': [] }], 'authenticated': 1 }) config_file = environ.get('KB_DEPLOYMENT_CONFIG', None) cls.cfg = {} config = ConfigParser() config.read(config_file) for nameval in config.items('kb_unicycler'): cls.cfg[nameval[0]] = nameval[1] cls.cfg["SDK_CALLBACK_URL"] = cls.callbackURL cls.cfg["KB_AUTH_TOKEN"] = cls.token cls.wsURL = cls.cfg['workspace-url'] cls.shockURL = cls.cfg['shock-url'] cls.hs = HandleService(url=cls.cfg['handle-service-url'], token=cls.token) # cls.wsClient = workspaceService(cls.wsURL, token=cls.token) cls.wsClient = Workspace(cls.wsURL, token=cls.token) wssuffix = int(time.time() * 1000) wsName = "test_kb_unicycler_" + str(wssuffix) cls.wsinfo = cls.wsClient.create_workspace({'workspace': wsName}) print('created workspace ' + cls.getWsName()) cls.PROJECT_DIR = 'unicycler_outputs' cls.scratch = cls.cfg['scratch'] if not os.path.exists(cls.scratch): os.makedirs(cls.scratch) cls.prjdir = os.path.join(cls.scratch, cls.PROJECT_DIR) if not os.path.exists(cls.prjdir): os.makedirs(cls.prjdir) cls.serviceImpl = kb_unicycler(cls.cfg) cls.readUtilsImpl = ReadsUtils(cls.callbackURL, token=cls.token) cls.dfuClient = DataFileUtil(url=cls.callbackURL, token=cls.token) cls.staged = {} cls.nodes_to_delete = [] cls.handles_to_delete = [] cls.setupTestData() print( '\n\n=============== Starting Unicycler tests ==================')
def setUpClass(cls): cls.token = environ.get('KB_AUTH_TOKEN', None) cls.callbackURL = environ.get('SDK_CALLBACK_URL') config_file = environ.get('KB_DEPLOYMENT_CONFIG', None) cls.cfg = {} config = ConfigParser() config.read(config_file) for nameval in config.items('ReadsAlignmentUtils'): cls.cfg[nameval[0]] = nameval[1] # Getting username from Auth profile for token authServiceUrl = cls.cfg['auth-service-url'] auth_client = _KBaseAuth(authServiceUrl) user_id = auth_client.get_user(cls.token) # WARNING: don't call any logging methods on the context object, # it'll result in a NoneType error cls.ctx = MethodContext(None) cls.ctx.update({ 'token': cls.token, 'user_id': user_id, 'provenance': [{ 'service': 'ReadsAlignmentUtils', 'method': 'please_never_use_it_in_production', 'method_params': [] }], 'authenticated': 1 }) cls.shockURL = cls.cfg['shock-url'] cls.wsURL = cls.cfg['workspace-url'] cls.wsClient = Workspace(cls.wsURL) cls.ws = Workspace(cls.wsURL, token=cls.token) cls.hs = HandleService(url=cls.cfg['handle-service-url'], token=cls.token) # create workspace wssuffix = int(time.time() * 1000) wsname = "test_alignment_" + str(wssuffix) cls.wsinfo = cls.wsClient.create_workspace({'workspace': wsname}) print('created workspace ' + cls.getWsName()) cls.serviceImpl = ReadsAlignmentUtils(cls.cfg) cls.readUtilsImpl = ReadsUtils(cls.callbackURL) cls.dfu = DataFileUtil(cls.callbackURL) cls.assemblyUtil = AssemblyUtil(cls.callbackURL) cls.gfu = GenomeFileUtil(cls.callbackURL) cls.scratch = cls.cfg['scratch'] cls.callback_url = os.environ['SDK_CALLBACK_URL'] cls.staged = {} cls.nodes_to_delete = [] cls.handles_to_delete = [] cls.setupTestData()
def getPairedEndLibInfo(self): if hasattr(self.__class__, 'pairedEndLibInfo'): return self.__class__.pairedEndLibInfo # 1) upload files to shock shared_dir = "/kb/module/work/tmp" forward_data_file = 'data/small.forward.fq' forward_file = os.path.join(shared_dir, os.path.basename(forward_data_file)) shutil.copy(forward_data_file, forward_file) reverse_data_file = 'data/small.reverse.fq' reverse_file = os.path.join(shared_dir, os.path.basename(reverse_data_file)) shutil.copy(reverse_data_file, reverse_file) ru = ReadsUtils(os.environ['SDK_CALLBACK_URL']) paired_end_ref = ru.upload_reads({'fwd_file': forward_file, 'rev_file': reverse_file, 'sequencing_tech': 'artificial reads', 'interleaved': 0, 'wsname': self.getWsName(), 'name': 'test.pe.reads'})['obj_ref'] new_obj_info = self.ws.get_object_info_new({'objects': [{'ref': paired_end_ref}]}) self.__class__.pairedEndLibInfo = new_obj_info[0] return new_obj_info[0]
def test_mash_sketch_valid_reads_ref(self): dir_path = os.path.dirname(os.path.realpath(__file__)) reads_file_name = 'reads-example.fastq' reads_test_path = os.path.join(dir_path, 'data', reads_file_name) reads_scratch_path = os.path.join(self.scratch, reads_file_name) shutil.copy(reads_test_path, reads_scratch_path) reads_utils = ReadsUtils(self.callback_url) upload_result = reads_utils.upload_reads({ 'wsname': self.getWsName(), 'interleaved': 'true', 'fwd_file': reads_scratch_path, 'name': 'example-reads', 'sequencing_tech': 'illumina' }) reads_ref = upload_result['obj_ref'] params = {'reads_ref': reads_ref, 'paired_ends': True} result = self.getImpl().run_mash_sketch(self.getContext(), params) output_path = result[0]['sketch_path'] with open(output_path, 'rb') as output_file: num_lines = sum(1 for line in output_file) self.assertTrue(os.path.exists(output_path)) self.assertEqual(num_lines, 25)
def __init__(self, config): self.ws_url = config["workspace-url"] self.callback_url = config['SDK_CALLBACK_URL'] self.token = config['KB_AUTH_TOKEN'] self.scratch = config['scratch'] self.dfu = DataFileUtil(self.callback_url) self.ru = ReadsUtils(self.callback_url) self.au = AssemblyUtil(self.callback_url) self.gfu = GenomeFileUtil(self.callback_url) self.rau = ReadsAlignmentUtils(self.callback_url) self.sp_uploader = sample_uploader(self.callback_url, service_ver='beta')