def upload_STARalignment(self, input_params, reads_ref, reads_info, output_bam_file):
"""
Uploads the alignment file + metadata.
Returns the STAR alignment reference.
"""
aligner_opts = dict()
for k in input_params:
aligner_opts[k] = str(input_params[k])
pprint(reads_info)
alignment_name = reads_ref['alignment_output_name']
align_upload_params = {
"destination_ref": "{}/{}".format(input_params[self.PARAM_IN_WS], alignment_name),
"file_path": output_bam_file,
"assembly_or_genome_ref": input_params[self.PARAM_IN_GENOME],
"read_library_ref": reads_info['object_ref'],
"library_type": reads_info['style'],
"condition": reads_info['condition'],
"aligned_using": 'STAR',
"aligner_version":self.STAR_VERSION,
"aligner_opts": aligner_opts
}
pprint(align_upload_params)
ra_util = ReadsAlignmentUtils(self.callback_url, service_ver='beta')
rau_upload_ret = ra_util.upload_alignment(align_upload_params)
alignment_ref = rau_upload_ret["obj_ref"]
print("STAR alignment uploaded as object {}".format(alignment_ref))
return rau_upload_ret
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('AlignmentSetEditor'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({'token': token,
'user_id': user_id,
'provenance': [
{'service': 'AlignmentSetEditor',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated': 1})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = workspaceService(cls.wsURL)
cls.serviceImpl = AlignmentSetEditor(cls.cfg)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.setAPI = SetAPI(cls.callback_url)
cls.gfu = GenomeFileUtil(cls.callback_url)
cls.ru = ReadsUtils(cls.callback_url)
cls.rau = ReadsAlignmentUtils(cls.callback_url)
suffix = int(time.time() * 1000)
cls.wsName = "test_AlignmentSetEditor_" + str(suffix)
cls.wsClient.create_workspace({'workspace': cls.wsName})
def _gen_ctab_files(self, params, alignment_ref):
source_dir = params.get(self.PARAM_IN_SRC_DIR)
if len(glob.glob(source_dir + '/*.ctab')) < 5:
self.__LOGGER.info(' ======= Generating ctab files ==========')
gtf_file = os.path.join(source_dir, 'transcripts.gtf')
if not os.path.isfile(gtf_file):
raise ValueError("{} file is required to generate ctab files, found missing".
format(gtf_file))
if self.PARAM_IN_BAM_FILE_PATH in params and \
params[self.PARAM_IN_BAM_FILE_PATH] is not None:
bam_file_path = params[self.PARAM_IN_BAM_FILE_PATH]
else:
self.__LOGGER.info('Downloading bam file from alignment object')
rau = ReadsAlignmentUtils(self.callback_url)
alignment_retVal = rau.download_alignment({'source_ref': alignment_ref})
alignment_dir = alignment_retVal.get('destination_dir')
tmp_file_path = os.path.join(alignment_dir, 'accepted_hits.bam')
if os.path.isfile(tmp_file_path):
bam_file_path = tmp_file_path
else:
tmp_file_path = os.path.join(alignment_dir, 'accepted_hits_sorted.bam')
if os.path.isfile(tmp_file_path):
bam_file_path = tmp_file_path
else:
raise ValueError('accepted_hits.bam or accepted_hits_sorted.bam not found in {}'.
format(alignment_dir))
result = self.table_maker.build_ctab_files(
ref_genome_path=gtf_file,
alignment_path=bam_file_path,
output_dir=source_dir)
if result != 0:
raise ValueError('Tablemaker failed')
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_cufflinks'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({'token': token,
'user_id': user_id,
'provenance': [
{'service': 'kb_cufflinks',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated': 1})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = Workspace(url=cls.wsURL, token=token)
cls.serviceImpl = kb_cufflinks(cls.cfg)
cls.scratch = cls.cfg['scratch']
cls.callback_url = environ.get('SDK_CALLBACK_URL')
cls.srv_wiz_url = cls.cfg['srv-wiz-url']
# cls.wsName = 'cufflinks_test_' + user_id # reuse existing workspace
suffix = int(time.time() * 1000)
cls.wsName = "test_kb_cufflinks_" + str(suffix)
print('workspace_name: ' + cls.wsName)
try:
# reuse existing (previously torn down) workspace
cls.wsClient.undelete_workspace({'workspace': cls.wsName})
print('reusing old workspace...')
except BaseException:
try:
# create if workspace does not exist
cls.wsClient.create_workspace({'workspace': cls.wsName})
except BaseException:
# get workspace if it exists and was not previously deleted (previously
# not torn down)
ws_info = cls.wsClient.get_workspace_info({'workspace': cls.wsName})
print("creating new workspace: " + str(ws_info))
cls.dfu = DataFileUtil(cls.callback_url)
cls.gfu = GenomeFileUtil(cls.callback_url)
cls.ru = ReadsUtils(cls.callback_url)
cls.rau = ReadsAlignmentUtils(cls.callback_url)
cls.set_api = SetAPI(cls.srv_wiz_url, service_ver='dev')
cls.cufflinks_runner = CufflinksUtils(cls.cfg)
cls.prepare_data()
def __init__(self, config):
"""
:param config:
:param logger:
:param directory: Working directory
:param urls: Service urls
"""
# BEGIN_CONSTRUCTOR
self.ws_url = config["workspace-url"]
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.srv_wiz_url = config['srv-wiz-url']
self.token = config['KB_AUTH_TOKEN']
self.shock_url = config['shock-url']
self.dfu = DataFileUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.set_api = SetAPI(self.srv_wiz_url, service_ver='dev')
self.eu = ExpressionUtils(self.callback_url)
self.ws = Workspace(self.ws_url, token=self.token)
self.scratch = os.path.join(config['scratch'], str(uuid.uuid4()))
self._mkdir_p(self.scratch)
self.tool_used = "Cufflinks"
self.tool_version = os.environ['VERSION']
# END_CONSTRUCTOR
pass
def __init__(self, scratch_dir, callback_url, workspace_url, srv_wiz_url):
self.scratch_dir = scratch_dir
self.rau = ReadsAlignmentUtils(callback_url)
self.kbr = KBaseReport(callback_url)
self.dfu = DataFileUtil(callback_url)
self.set_api = SetAPI(srv_wiz_url)
self.ws = Workspace(workspace_url)
self.valid_commands = ['bamqc', 'multi-bamqc']
def __init__(self, config):
log('--->\nInitializing RNASeqDownloaderUtils instance:\n config: %s' %
config)
self.scratch = config['scratch']
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.dfu = DataFileUtil(self.callback_url, token=self.token)
self.rau = ReadsAlignmentUtils(self.callback_url, token=self.token)
def __init__(self, ctx, config):
log('--->\nInitializing RNASeqDownloaderUtils instance:\n config: %s' % config)
self.scratch = config['scratch']
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.ctx = ctx
self.staging_base = os.path.join(self.STAGING_FILE_PREFIX, ctx['user_id'])
self.rau = ReadsAlignmentUtils(self.callback_url, token=self.token)
def setUpClass(cls):
cls.token = environ.get('KB_AUTH_TOKEN', None)
cls.callbackURL = environ.get('SDK_CALLBACK_URL')
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('ExpressionUtils'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(cls.token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
cls.token,
'user_id':
user_id,
'provenance': [{
'service': 'ExpressionUtils',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
cls.shockURL = cls.cfg['shock-url']
cls.wsURL = cls.cfg['workspace-url']
cls.service_wizard_url = cls.cfg['srv-wiz-url']
cls.wsClient = workspaceService(cls.wsURL)
cls.ws = Workspace(cls.wsURL, token=cls.token)
cls.hs = HandleService(url=cls.cfg['handle-service-url'],
token=cls.token)
# create workspace
wssuffix = int(time.time() * 1000)
wsname = "test_expression_" + str(wssuffix)
cls.wsinfo = cls.wsClient.create_workspace({'workspace': wsname})
print('created workspace ' + cls.getWsName())
cls.serviceImpl = ExpressionUtils(cls.cfg)
cls.readUtils = ReadsUtils(cls.callbackURL)
cls.dfu = DataFileUtil(cls.callbackURL, service_ver='dev')
cls.dfu.ws_name_to_id(wsname)
cls.assemblyUtil = AssemblyUtil(cls.callbackURL)
cls.gfu = GenomeFileUtil(cls.callbackURL)
cls.gaAPI = GenomeAnnotationAPI(cls.service_wizard_url)
cls.rau = ReadsAlignmentUtils(cls.callbackURL)
cls.scratch = cls.cfg['scratch']
cls.staged = {}
cls.nodes_to_delete = []
cls.handles_to_delete = []
cls.setupTestData()
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.scratch = config['scratch']
self.dfu = DataFileUtil(self.callback_url)
self.ru = ReadsUtils(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.shock_url = config['shock-url']
self.dfu = DataFileUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.fv = KBaseFeatureValues(self.callback_url)
self.deu = DifferentialExpressionUtils(self.callback_url, service_ver='dev')
self.ws = Workspace(self.ws_url, token=self.token)
self.scratch = config['scratch']
self.config = config
def setUpClass(cls):
cls.token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_ballgown'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(cls.token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
cls.token,
'user_id':
user_id,
'provenance': [{
'service': 'kb_ballgown',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
cls.hs = HandleService(url=cls.cfg['handle-service-url'],
token=cls.token)
cls.shockURL = cls.cfg['shock-url']
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = workspaceService(cls.wsURL, token=cls.token)
cls.serviceImpl = kb_ballgown(cls.cfg)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.gfu = GenomeFileUtil(cls.callback_url)
cls.dfu = DataFileUtil(cls.callback_url)
cls.ru = ReadsUtils(cls.callback_url)
cls.rau = ReadsAlignmentUtils(cls.callback_url, service_ver='dev')
cls.eu = ExpressionUtils(cls.callback_url, service_ver='dev')
cls.set_api = SetAPI(cls.callback_url)
suffix = int(time.time() * 1000)
cls.wsName = "test_kb_ballgown_" + str(suffix)
#cls.wsName = "test_kb_ballgown_1004"
cls.wsClient.create_workspace({'workspace': cls.wsName})
cls.nodes_to_delete = []
cls.handles_to_delete = []
cls.prepare_data()
def save_read_alignment_output(self, run_output_info, input_configuration, validated_params):
rau = ReadsAlignmentUtils(self.callback_url)
destination_ref = validated_params['output_workspace'] + '/' + validated_params['output_alignment_name']
condition = 'unknown'
if 'condition_label' in validated_params:
condition = validated_params['condition_label']
upload_params = {'file_path': run_output_info['output_sam_file'],
'destination_ref': destination_ref,
'read_library_ref': input_configuration['reads_lib_ref'],
'assembly_or_genome_ref': validated_params['assembly_or_genome_ref'],
'condition': condition}
upload_results = rau.upload_alignment(upload_params)
return upload_results
def load_bam_file(self, file_path, genome_ref, reads_ref, obj_name):
rau = ReadsAlignmentUtils(self.callback_url, service_ver='dev')
return rau.upload_alignment({
"destination_ref":
"{}/{}".format(self.ws_name, obj_name),
"file_path":
file_path,
"read_library_ref":
reads_ref,
"assembly_or_genome_ref":
genome_ref,
"condition":
"none"
})["obj_ref"]
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.shock_url = config['shock-url']
self.srv_wiz_url = config['srv-wiz-url']
self.scratch = config['scratch']
self.dfu = DataFileUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.eu = ExpressionUtils(self.callback_url, service_ver='dev')
self.ws = Workspace(self.ws_url, token=self.token)
self.set_client = SetAPI(self.srv_wiz_url)
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.shock_url = config['shock-url']
self.scratch = config['scratch']
self.srv_wiz_url = config['srv-wiz-url']
self.ws = Workspace(self.ws_url, token=self.token)
self.bt = kb_Bowtie2(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.qualimap = kb_QualiMap(self.callback_url)
self.ru = ReadsUtils(self.callback_url)
self.dfu = DataFileUtil(self.callback_url)
self.set_client = SetAPI(self.srv_wiz_url)
def __init__(self, config, services, logger=None):
self.config = config
self.logger = logger
self.callback_url = os.environ['SDK_CALLBACK_URL']
self.scratch = os.path.join(config['scratch'],
'cuffdiff_merge_' + str(uuid.uuid4()))
self.ws_url = config['workspace-url']
self.services = services
self.ws_client = Workspace(self.services['workspace_service_url'])
self.dfu = DataFileUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.eu = ExpressionUtils(self.callback_url)
self.deu = DifferentialExpressionUtils(self.callback_url)
self.cuffmerge_runner = CuffMerge(config, logger)
self.num_threads = mp.cpu_count()
handler_utils._mkdir_p(self.scratch)
def get_alignment_data_files(self, alignment_refs):
"""
Returns a dictionary of data files. Key = object name, value = path to the file.
"""
alignment_files = dict()
ru = ReadsAlignmentUtils(self.callback_url, service_ver='dev')
for ref in alignment_refs:
ref_name = get_object_name(ref, self.workspace_url)
align_file = ru.download_alignment({
"source_ref": ref,
"downloadBAI": 0
})
for f in os.listdir(align_file["destination_dir"]):
if f.endswith("bam"):
alignment_files[ref_name] = os.path.join(
align_file["destination_dir"], f)
return alignment_files
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.ctx = {'token': token}
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_cummerbund'):
cls.cfg[nameval[0]] = nameval[1]
cls.scratch = cls.cfg['scratch']
cls.wsURL = cls.cfg['ws_url']
cls.gfu = GenomeFileUtil(cls.callback_url)
cls.ru = ReadsUtils(cls.callback_url)
cls.rau = ReadsAlignmentUtils(cls.callback_url, service_ver='dev')
cls.eu = ExpressionUtils(cls.callback_url, service_ver='dev')
cls.wsClient = workspaceService(cls.wsURL, token=token)
cls.dfu = DataFileUtil(cls.callback_url, token=token)
cls.serviceImpl = kb_cummerbund(cls.cfg)
cls.prepare_data()
def upload_alignment(self, input_params, reads_info, alignment_name,
alignment_file):
"""
Uploads the alignment file + metadata.
This then returns the expected return dictionary from HISAT2.
"""
aligner_opts = dict()
for k in input_params:
aligner_opts[k] = str(input_params[k])
align_upload_params = {
"destination_ref":
"{}/{}".format(input_params["ws_name"], alignment_name),
"file_path":
alignment_file,
"library_type":
reads_info["style"], # single or paired end,
"condition":
reads_info["condition"],
"assembly_or_genome_ref":
input_params["genome_ref"],
"read_library_ref":
reads_info["object_ref"],
"aligned_using":
"hisat2",
"aligner_version":
HISAT_VERSION,
"aligner_opts":
aligner_opts
}
if "sampleset_ref" in reads_info:
align_upload_params["sampleset_ref"] = reads_info["sampleset_ref"]
print("Uploading completed alignment")
pprint(align_upload_params)
ra_util = ReadsAlignmentUtils(self.callback_url, service_ver="dev")
alignment_ref = ra_util.upload_alignment(
align_upload_params)["obj_ref"]
print(
"Done! New alignment uploaded as object {}".format(alignment_ref))
return alignment_ref
def upload_alignment(self, file_path, reads_upa, assembly_upa, workspace_name, alignment_name):
if not file_path:
raise ValueError("file_path must be defined")
if not os.path.exists(file_path):
raise ValueError("The given alignment file '{}' does not exist".format(file_path))
if not reads_upa:
raise ValueError("The reads UPA must be defined")
if not assembly_upa:
raise ValueError("The assembly UPA must be defined")
if not workspace_name:
raise ValueError("workspace_name must be defined")
if not alignment_name:
raise ValueError("alignment_name must be defined")
rau = ReadsAlignmentUtils(self.callback_url)
alignment_upa = rau.upload_alignment({
"file_path": file_path,
"read_library_ref": reads_upa,
"assembly_or_genome_ref": assembly_upa,
"destination_ref": "{}/{}".format(workspace_name, alignment_name),
"aligned_using": "BBMap",
"condition": "new_assembly"
})["obj_ref"]
return alignment_upa