def test_fastqc_app(self):
# create ws, and load test reads
wsName = self.getWsName()
ru = ReadsUtils(os.environ['SDK_CALLBACK_URL'])
input_file_ref = ru.upload_reads({
'fwd_file': self.small_fq_test_file2,
'sequencing_tech': 'tech1',
'wsname': wsName,
'name': 'reads1',
'interleaved': 1
})['obj_ref']
input_params = {'input_ws': wsName, 'input_file_ref': input_file_ref}
output = self.getImpl().runFastQC(self.getContext(), input_params)[0]
self.assertIn('report_name', output)
self.assertIn('report_ref', output)
# pprint(output)
report = self.getWsClient().get_objects2(
{'objects': [{
'ref': output['report_ref']
}]})['data'][0]['data']
# pprint(report)
self.assertIn('direct_html', report)
self.assertIn('file_links', report)
self.assertIn('html_links', report)
self.assertIn('objects_created', report)
self.assertIn('text_message', report)
def getPairedEndLibInfo(self, lib_name):
if hasattr(self.__class__, 'pairedEndLibInfo'):
if self.__class__.pairedEndLibInfo.get(lib_name):
return self.__class__.pairedEndLibInfo[lib_name]
# copy the local test file to the shared scratch space so that the ReadsUtils
# container can see it.
test_fastq_file_local = os.path.join('data', 'reads', lib_name)
test_fastq_file_scratch = os.path.join(
self.scratch, os.path.basename(test_fastq_file_local))
shutil.copy(test_fastq_file_local, test_fastq_file_scratch)
# call the ReadsUtils libary to upload the test data to KBase
ru = ReadsUtils(os.environ['SDK_CALLBACK_URL'])
paired_end_ref = ru.upload_reads({
'fwd_file': test_fastq_file_scratch,
'sequencing_tech': 'artificial reads',
'interleaved': 1,
'wsname': self.getWsName(),
'name': lib_name
})['obj_ref']
# get the object metadata for the new test dataset
new_obj_info = self.ws.get_object_info_new(
{'objects': [{
'ref': paired_end_ref
}]})
if not hasattr(self.__class__, 'pairedEndLibInfo'):
self.__class__.pairedEndLibInfo = dict()
self.__class__.pairedEndLibInfo[lib_name] = new_obj_info[0]
return new_obj_info[0]
def loadPairedEndReads(self, forward_data_file, reverse_data_file,
reads_name):
# if hasattr(self.__class__, 'pairedEndLibInfo'):
# return self.__class__.pairedEndLibInfo
# 1) upload files to shock
# shared_dir = "/kb/module/work/tmp"
# forward_data_file = '../test/testReads/small.forward.fq'
forward_file = os.path.join(self.scratch,
os.path.basename(forward_data_file))
shutil.copy(forward_data_file, forward_file)
# reverse_data_file = '../test/testReads/small.reverse.fq'
reverse_file = os.path.join(self.scratch,
os.path.basename(reverse_data_file))
shutil.copy(reverse_data_file, reverse_file)
ru = ReadsUtils(os.environ['SDK_CALLBACK_URL'])
pe_reads_ref = ru.upload_reads({
'fwd_file': forward_file,
'rev_file': reverse_file,
'sequencing_tech': 'artificial reads',
'interleaved': 0,
'wsname': self.getWsName(),
'name': reads_name
})['obj_ref']
# self.__class__.pe_reads_ref = pe_reads_ref
print('Loaded PairedEndReads: ' + pe_reads_ref)
new_obj_info = self.wsClient.get_object_info_new(
{'objects': [{
'ref': pe_reads_ref
}]})
self.__class__.pairedEndLibInfo = new_obj_info[0]
pprint(pformat(new_obj_info))
# return new_obj_info[0]
return pe_reads_ref
def get_reads_RU(self, refs, console):
readcli = ReadsUtils(self.callbackURL, token=self.token)
typeerr = ('Supported types: KBaseFile.SingleEndLibrary ' +
'KBaseFile.PairedEndLibrary ' +
'KBaseAssembly.SingleEndLibrary ' +
'KBaseAssembly.PairedEndLibrary')
try:
reads = readcli.download_reads({'read_libraries': refs,
'interleaved': 'true',
'gzipped': None
})['files']
except ServerError as se:
self.log(console, 'logging stacktrace from dynamic client error')
self.log(console, se.data)
if typeerr in se.message:
prefix = se.message.split('.')[0]
raise ValueError(
prefix + '. Only the types ' +
'KBaseAssembly.PairedEndLibrary ' +
'and KBaseFile.PairedEndLibrary are supported')
else:
raise
self.log(console, 'Got reads data from converter:\n' + pformat(reads))
return reads
class DownloadFastqUtils:
def __init__(self):
self.callbackURL = os.environ['SDK_CALLBACK_URL']
self.au = AssemblyUtil(self.callbackURL)
self.ru = ReadsUtils(self.callbackURL)
pass
def _stage_input_file(self, ref, reads_type):
if reads_type == 'KBaseFile.PairedEndLibrary' or 'KBaseAssembly.PairedEndLibrary':
input_file_info = self.ru.download_reads({
'read_libraries': [ref],
'interleaved': 'true'
})['files'][ref]
elif reads_type == 'KBaseFile.SingleEndLibrary' or 'KBaseAssembly.SingleEndLibrary':
input_file_info = self.ru.download_reads({'read_libraries':
[ref]})['files'][ref]
else:
raise ValueError("Can't download_reads() for object type: '" +
str(reads_type) + "'")
input_file_info['input_ref'] = ref
file_location = input_file_info['files']['fwd']
interleaved = False
if input_file_info['files']['type'] == 'interleaved':
interleaved = True
return input_file_info
def download_genome(self, genomeref):
file = self.au.get_assembly_as_fasta({'ref': genomeref})
return file
def fetch_reads_from_reference(ref, callback_url):
"""
Fetch a FASTQ file (or 2 for paired-end) from a reads reference.
Returns the following structure:
{
"style": "paired", "single", or "interleaved",
"file_fwd": path_to_file,
"file_rev": path_to_file, only if paired end,
"object_ref": reads reference for downstream convenience.
}
"""
try:
print("Fetching reads from object {}".format(ref))
reads_client = ReadsUtils(callback_url)
reads_dl = reads_client.download_reads({
"read_libraries": [ref],
"interleaved": "false"
})
pprint(reads_dl)
reads_files = reads_dl['files'][ref]['files']
ret_reads = {
"object_ref": ref,
"style": reads_files["type"],
"file_fwd": reads_files["fwd"]
}
if reads_files.get("rev", None) is not None:
ret_reads["file_rev"] = reads_files["rev"]
return ret_reads
except:
print(
"Unable to fetch a file from expected reads object {}".format(ref))
raise
def prepare_single_run(self, input_info, assembly_or_genome_ref,
bwa_index_info, ws_for_cache):
''' Given a reads ref and an assembly, setup the bwa index '''
# first setup the bwa index of the assembly
input_configuration = {'bwa_index_info': bwa_index_info}
if not bwa_index_info:
bwaIndexBuilder = BwaIndexBuilder(self.scratch_dir,
self.workspace_url,
self.callback_url,
self.srv_wiz_url,
self.provenance)
index_result = bwaIndexBuilder.get_index({
'ref':
assembly_or_genome_ref,
'ws_for_cache':
ws_for_cache
})
input_configuration['bwa_index_info'] = index_result
# next download the reads
read_lib_ref = input_info['ref']
read_lib_info = input_info['info']
reads_params = {
'read_libraries': [read_lib_ref],
'interleaved': 'false',
'gzipped': None
}
ru = ReadsUtils(self.callback_url)
reads = ru.download_reads(reads_params)['files']
input_configuration['reads_lib_type'] = self.get_type_from_obj_info(
read_lib_info).split('.')[1]
input_configuration['reads_files'] = reads[read_lib_ref]
input_configuration['reads_lib_ref'] = read_lib_ref
return input_configuration
def download_interleaved_reads(callback_url, reads_upa):
ru = ReadsUtils(callback_url)
reads_info = ru.download_reads({
'read_libraries': [reads_upa],
'interleaved': 'true',
'gzipped': None
})['files'][reads_upa]
return reads_info
def __init__(self, callbaack_url, scratch, wdl='../../metaAssembly/'):
self.callback_url = callbaack_url
self.scratch = scratch
self.special = special(self.callback_url)
self.ru = ReadsUtils(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.report = KBaseReport(self.callback_url)
self.wdl_base = wdl
def __init__(self, config, callback_url, workspace_id, cpus):
self.shared_folder = config['scratch']
self.callback_url = callback_url
self.cpus = cpus
self.ru = ReadsUtils(self.callback_url)
logging.basicConfig(format='%(created)s %(levelname)s: %(message)s',
level=logging.INFO)
def __init__(self, config):
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.scratch = os.path.join(config['scratch'], 'import_SRA_' + str(uuid.uuid4()))
handler_utils._mkdir_p(self.scratch)
self.dfu = DataFileUtil(self.callback_url)
self.ru = ReadsUtils(self.callback_url)
self.uploader_utils = UploaderUtil(config)
def __init__(self, config):
self.callback_url = os.environ['SDK_CALLBACK_URL']
self.scratch = config['scratch']
self.shock_url = config['shock-url']
self.ws_url = config['workspace-url']
self.dfu = DataFileUtil(self.callback_url)
self.ru = ReadsUtils(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.mgu = MetagenomeUtils(self.callback_url)
def download_reads(self, token, reads_ref):
try:
readsUtils_Client = ReadsUtils (url=self.callback_url, token=token) # SDK local
readsLibrary = readsUtils_Client.download_reads ({'read_libraries': [reads_ref],
'interleaved': 'true'
})
reads_file_path = readsLibrary['files'][reads_ref]['files']['fwd']
except Exception as e:
raise ValueError('Unable to get reads library object from workspace: (' + reads_ref +")\n" + str(e))
return reads_file_path
def run_mash_sketch(self, ctx, params):
"""
Generate a sketch file from a fasta/fastq file
:param params: instance of type "MashSketchParams" (* * Pass in **one
of** input_path, assembly_ref, or reads_ref * input_path -
string - local file path to an input fasta/fastq * assembly_ref
- string - workspace reference to an Assembly type * reads_ref -
string - workspace reference to a Reads type * Optionally, pass in
a boolean indicating whether you are using paired-end reads. *
paired_ends - boolean - whether you are passing in paired ends) ->
structure: parameter "input_path" of String, parameter
"assembly_ref" of String, parameter "reads_ref" of String,
parameter "paired_ends" of type "boolean" (params:
input_upa: workspace reference to an assembly object
workspace_name: name of current workspace search_db: database to
search n_max_results: number of results to return, integer between
1 and 100)
:returns: instance of type "MashSketchResults" (* * Returns the local
scratch file path of the generated sketch file. * Will have the
extension '.msh') -> structure: parameter "sketch_path" of String
"""
# ctx is the context object
# return variables are: results
#BEGIN run_mash_sketch
if 'reads_ref' in params:
reads_utils = ReadsUtils(self.callbackURL)
result = reads_utils.download_reads({
'read_libraries': [params['reads_ref']],
'interleaved': 'true'
})
input_path = result['files'][params['reads_ref']]['files']['fwd']
elif 'assembly_ref' in params:
assembly_util = AssemblyUtil(self.callbackURL)
result = assembly_util.get_assembly_as_fasta({'ref': params['assembly_ref']})
input_path = result['path']
elif 'input_path' in params:
input_path = params['input_path']
else:
raise ValueError(
'Invalid params; must provide one of `reads_ref`, `assembly_ref`, or `input_path`.'
)
mash_utils = MashUtils(self.config, self.auth_token)
output_file_path = mash_utils.mash_sketch(input_path, paired_ends=params.get('paired_ends'))
results = {'sketch_path': output_file_path}
#END run_mash_sketch
# At some point might do deeper type checking...
if not isinstance(results, dict):
raise ValueError('Method run_mash_sketch return value ' +
'results is not type dict as required.')
# return the results
return [results]
def __init__(self, prj_dir, config):
self.workspace_url = config['workspace-url']
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
if 'shock-url' in config:
self.shock_url = config['shock-url']
if 'handle-service-url' in config:
self.handle_url = config['handle-service-url']
self.ws_client = Workspace(self.workspace_url, token=self.token)
self.ru = ReadsUtils(self.callback_url, token=self.token)
self.au = AssemblyUtil(self.callback_url, token=self.token)
self.kbr = KBaseReport(self.callback_url)
self.kbq = kb_quast(self.callback_url)
self.proj_dir = prj_dir
self.prog_runner = Program_Runner(self.MaSuRCA_BIN, self.proj_dir)
def load_pe_reads(fwd_file, rev_file):
"""
Copies from given dir to scratch. Then calls ReadsUtils to upload from scratch.
"""
callback_url = os.environ['SDK_CALLBACK_URL']
fwd_file_path = file_to_scratch(fwd_file, overwrite=True)
rev_file_path = file_to_scratch(rev_file, overwrite=True)
ru = ReadsUtils(callback_url)
pe_reads_params = {
'fwd_file': fwd_file_path,
'rev_file': rev_file_path,
'sequencing_tech': 'Illumina',
'wsname': get_ws_name(),
'name': 'MyPairedEndLibrary'
}
return ru.upload_reads(pe_reads_params)['obj_ref']
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_staging_exporter'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({'token': token,
'user_id': user_id,
'provenance': [
{'service': 'kb_staging_exporter',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated': 1})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = workspaceService(cls.wsURL)
cls.serviceImpl = kb_staging_exporter(cls.cfg)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.ru = ReadsUtils(cls.callback_url)
cls.au = AssemblyUtil(cls.callback_url)
cls.gfu = GenomeFileUtil(cls.callback_url, service_ver='dev')
cls.rau = ReadsAlignmentUtils(cls.callback_url)
def load_reads(callback_url, ws_name, tech, file_fwd, file_rev, target_name):
"""
Loads FASTQ files as either SingleEndLibrary or PairedEndLibrary. If file_rev is None,
then we get a single end, otherwise, paired.
"""
reads_util = ReadsUtils(callback_url)
upload_params = {
"wsname": ws_name,
"fwd_file": file_fwd,
"name": target_name,
"sequencing_tech": tech
}
if file_rev is not None:
upload_params["rev_file"] = file_rev
reads_ref = reads_util.upload_reads(upload_params)
return reads_ref["obj_ref"]
def setUpClass(cls):
token = os.environ.get('KB_AUTH_TOKEN', None)
config_file = os.environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('gottcha2'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({'token': token,
'user_id': user_id,
'provenance': [
{'service': 'gottcha2',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated': 1})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = Workspace(cls.wsURL)
cls.serviceImpl = gottcha2(cls.cfg)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
suffix = int(time.time() * 1000)
cls.wsName = "test_ContigFilter_" + str(suffix)
cls.ru = ReadsUtils(os.environ['SDK_CALLBACK_URL'])
ret = cls.wsClient.create_workspace({'workspace': cls.wsName}) # noqa
def loadSEReads(self, reads_file_path):
# if hasattr(self.__class__, 'reads_ref'):
# return self.__class__.reads_ref
se_reads_name = os.path.basename(reads_file_path)
fq_path = os.path.join(self.scratch, se_reads_name)
shutil.copy(reads_file_path, fq_path)
ru = ReadsUtils(self.callback_url)
reads_ref = ru.upload_reads({
'fwd_file': fq_path,
'wsname': self.getWsName(),
'name': se_reads_name.split('.')[0],
'sequencing_tech': 'kb reads'
})['obj_ref']
# self.__class__.reads_ref = reads_ref
return reads_ref
def __init__(self, prj_dir, config):
self.workspace_url = config['workspace-url']
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
if 'shock-url' in config:
self.shock_url = config['shock-url']
if 'handle-service-url' in config:
self.handle_url = config['handle-service-url']
self.ws_client = Workspace(self.workspace_url, token=self.token)
self.ru = ReadsUtils(self.callback_url, token=self.token, service_ver='release')
self.au = AssemblyUtil(self.callback_url, token=self.token, service_ver='release')
self.kbr = KBaseReport(self.callback_url)
self.kbq = kb_quast(self.callback_url)
self.proj_dir = prj_dir
self.spades_version = 'SPAdes-' + os.environ['SPADES_VERSION']
def loadSingleEndReads(self, single_end_fastq, params):
if hasattr(self.__class__, 'se_reads_ref'):
return self.__class__.se_reads_ref
fq_path = os.path.join(self.scratch, 'extracted_WT_rep1.fastq')
#shutil.copy(os.path.join('data', 'bt_test_data', 'extracted_WT_rep1.fastq'), fq_path)
shutil.copy(single_end_fastq, fq_path)
ru = ReadsUtils(self.callback_url)
se_reads_ref = ru.upload_reads({
'fwd_file': fq_path,
'wsname': params["workspace_name"],
'name': 'test_readsSE',
'sequencing_tech': 'artificial reads'
})['obj_ref']
self.__class__.se_reads_ref = se_reads_ref
print('Loaded SingleEndReads: ' + se_reads_ref)
return se_reads_ref
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_deseq'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
token,
'user_id':
user_id,
'provenance': [{
'service': 'kb_deseq',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = Workspace(cls.wsURL)
cls.ws = Workspace(cls.wsURL, token=token)
cls.serviceImpl = kb_deseq(cls.cfg)
cls.serviceImpl.status(cls.ctx)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.gfu = GenomeFileUtil(cls.callback_url, service_ver='dev')
cls.dfu = DataFileUtil(cls.callback_url)
cls.ru = ReadsUtils(cls.callback_url)
cls.rau = ReadsAlignmentUtils(cls.callback_url)
cls.stringtie = kb_stringtie(cls.callback_url)
cls.eu = ExpressionUtils(cls.callback_url)
cls.deseq_runner = DESeqUtil(cls.cfg)
suffix = int(time.time() * 1000)
cls.wsName = "test_kb_stringtie_" + str(suffix)
cls.wsClient.create_workspace({'workspace': cls.wsName})
cls.dfu.ws_name_to_id(cls.wsName)
# public on CI
cls.expressionset_ref = '30957/52/41'
cls.condition_1 = 'Ecoli_WT'
cls.condition_2 = 'Ecoli_ydcR'
# public on Appdev
cls.expressionset_ref = '60454/19'
cls.condition_1 = 'WT'
cls.condition_2 = 'Hy5'
def loadSingleEndReads(self):
if hasattr(self.__class__, 'se_reads_ref'):
return self.__class__.se_reads_ref
# return '23735/2/1'
fq_path = os.path.join(self.scratch, 'extracted_WT_rep1.fastq')
shutil.copy(
os.path.join('data', 'bt_test_data', 'extracted_WT_rep1.fastq'),
fq_path)
ru = ReadsUtils(self.callback_url)
se_reads_ref = ru.upload_reads({
'fwd_file': fq_path,
'wsname': self.getWsName(),
'name': 'test_readsSE',
'sequencing_tech': 'artificial reads'
})['obj_ref']
self.__class__.se_reads_ref = se_reads_ref
print('Loaded SingleEndReads: ' + se_reads_ref)
return se_reads_ref
def fetch_reads_files(self, reads_upas):
"""
From a list of reads UPAs, uses ReadsUtils to fetch the reads as files.
Returns them as a dictionary from reads_upa -> filename
"""
if reads_upas is None:
raise ValueError("reads_upas must be a list of UPAs")
if len(reads_upas) == 0:
raise ValueError("reads_upas must contain at least one UPA")
ru = ReadsUtils(self.callback_url)
reads_info = ru.download_reads(({
'read_libraries': reads_upas,
'interleaved': 'true',
'gzipped': None
}))['files']
file_set = dict()
for reads in reads_info:
file_set[reads] = reads_info[reads]['files']['fwd']
return file_set
def setUpClass(cls):
cls.token = environ.get('KB_AUTH_TOKEN')
cls.callbackURL = environ.get('SDK_CALLBACK_URL')
print('CB URL: ' + cls.callbackURL)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
cls.token,
'provenance': [{
'service': 'kb_unicycler',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_unicycler'):
cls.cfg[nameval[0]] = nameval[1]
cls.cfg["SDK_CALLBACK_URL"] = cls.callbackURL
cls.cfg["KB_AUTH_TOKEN"] = cls.token
cls.wsURL = cls.cfg['workspace-url']
cls.shockURL = cls.cfg['shock-url']
cls.hs = HandleService(url=cls.cfg['handle-service-url'],
token=cls.token)
# cls.wsClient = workspaceService(cls.wsURL, token=cls.token)
cls.wsClient = Workspace(cls.wsURL, token=cls.token)
wssuffix = int(time.time() * 1000)
wsName = "test_kb_unicycler_" + str(wssuffix)
cls.wsinfo = cls.wsClient.create_workspace({'workspace': wsName})
print('created workspace ' + cls.getWsName())
cls.PROJECT_DIR = 'unicycler_outputs'
cls.scratch = cls.cfg['scratch']
if not os.path.exists(cls.scratch):
os.makedirs(cls.scratch)
cls.prjdir = os.path.join(cls.scratch, cls.PROJECT_DIR)
if not os.path.exists(cls.prjdir):
os.makedirs(cls.prjdir)
cls.serviceImpl = kb_unicycler(cls.cfg)
cls.readUtilsImpl = ReadsUtils(cls.callbackURL, token=cls.token)
cls.dfuClient = DataFileUtil(url=cls.callbackURL, token=cls.token)
cls.staged = {}
cls.nodes_to_delete = []
cls.handles_to_delete = []
cls.setupTestData()
print(
'\n\n=============== Starting Unicycler tests ==================')
def setUpClass(cls):
cls.token = environ.get('KB_AUTH_TOKEN', None)
cls.callbackURL = environ.get('SDK_CALLBACK_URL')
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('ReadsAlignmentUtils'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(cls.token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
cls.token,
'user_id':
user_id,
'provenance': [{
'service': 'ReadsAlignmentUtils',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
cls.shockURL = cls.cfg['shock-url']
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = Workspace(cls.wsURL)
cls.ws = Workspace(cls.wsURL, token=cls.token)
cls.hs = HandleService(url=cls.cfg['handle-service-url'],
token=cls.token)
# create workspace
wssuffix = int(time.time() * 1000)
wsname = "test_alignment_" + str(wssuffix)
cls.wsinfo = cls.wsClient.create_workspace({'workspace': wsname})
print('created workspace ' + cls.getWsName())
cls.serviceImpl = ReadsAlignmentUtils(cls.cfg)
cls.readUtilsImpl = ReadsUtils(cls.callbackURL)
cls.dfu = DataFileUtil(cls.callbackURL)
cls.assemblyUtil = AssemblyUtil(cls.callbackURL)
cls.gfu = GenomeFileUtil(cls.callbackURL)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.staged = {}
cls.nodes_to_delete = []
cls.handles_to_delete = []
cls.setupTestData()
def getPairedEndLibInfo(self):
if hasattr(self.__class__, 'pairedEndLibInfo'):
return self.__class__.pairedEndLibInfo
# 1) upload files to shock
shared_dir = "/kb/module/work/tmp"
forward_data_file = 'data/small.forward.fq'
forward_file = os.path.join(shared_dir, os.path.basename(forward_data_file))
shutil.copy(forward_data_file, forward_file)
reverse_data_file = 'data/small.reverse.fq'
reverse_file = os.path.join(shared_dir, os.path.basename(reverse_data_file))
shutil.copy(reverse_data_file, reverse_file)
ru = ReadsUtils(os.environ['SDK_CALLBACK_URL'])
paired_end_ref = ru.upload_reads({'fwd_file': forward_file, 'rev_file': reverse_file,
'sequencing_tech': 'artificial reads',
'interleaved': 0, 'wsname': self.getWsName(),
'name': 'test.pe.reads'})['obj_ref']
new_obj_info = self.ws.get_object_info_new({'objects': [{'ref': paired_end_ref}]})
self.__class__.pairedEndLibInfo = new_obj_info[0]
return new_obj_info[0]
def test_mash_sketch_valid_reads_ref(self):
dir_path = os.path.dirname(os.path.realpath(__file__))
reads_file_name = 'reads-example.fastq'
reads_test_path = os.path.join(dir_path, 'data', reads_file_name)
reads_scratch_path = os.path.join(self.scratch, reads_file_name)
shutil.copy(reads_test_path, reads_scratch_path)
reads_utils = ReadsUtils(self.callback_url)
upload_result = reads_utils.upload_reads({
'wsname': self.getWsName(),
'interleaved': 'true',
'fwd_file': reads_scratch_path,
'name': 'example-reads',
'sequencing_tech': 'illumina'
})
reads_ref = upload_result['obj_ref']
params = {'reads_ref': reads_ref, 'paired_ends': True}
result = self.getImpl().run_mash_sketch(self.getContext(), params)
output_path = result[0]['sketch_path']
with open(output_path, 'rb') as output_file:
num_lines = sum(1 for line in output_file)
self.assertTrue(os.path.exists(output_path))
self.assertEqual(num_lines, 25)
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.scratch = config['scratch']
self.dfu = DataFileUtil(self.callback_url)
self.ru = ReadsUtils(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.sp_uploader = sample_uploader(self.callback_url,
service_ver='beta')
