def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_orthofinder'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({'token': token,
'user_id': user_id,
'provenance': [
{'service': 'kb_orthofinder',
'method': 'annotate_plant_transcripts',
'method_params': []
}],
'authenticated': 1})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = workspaceService(cls.wsURL)
cls.serviceImpl = kb_orthofinder(cls.cfg)
cls.scratch = cls.cfg['scratch']
cls.test_data = cls.cfg['test_data']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.gfu = GenomeFileUtil(cls.callback_url)
cls.dfu = DataFileUtil(cls.callback_url)
cls.genome = "Test_Genome"
cls.prepare_data()
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_staging_exporter'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({'token': token,
'user_id': user_id,
'provenance': [
{'service': 'kb_staging_exporter',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated': 1})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = workspaceService(cls.wsURL)
cls.serviceImpl = kb_staging_exporter(cls.cfg)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.ru = ReadsUtils(cls.callback_url)
cls.au = AssemblyUtil(cls.callback_url)
cls.gfu = GenomeFileUtil(cls.callback_url, service_ver='dev')
cls.rau = ReadsAlignmentUtils(cls.callback_url)
def test_reannotate_genome_official(self):
"""
This test takes about 25 minutes to run. It uploads the rhodobacter_gff, runs prokka genome reannotation
and then checks to see if a specific feature has been updated correctly
:return:
"""
gfu = GenomeFileUtil(os.environ["SDK_CALLBACK_URL"])
genome_test_file = os.path.join("/kb/module/test/data/", "rhodobacter_genomic.gbff")
genome_test_file_scratch = os.path.join("/kb/module/work/tmp", "rhodobacter_genomic.gbff")
copyfile(genome_test_file, genome_test_file_scratch)
genome_ref_original = gfu.genbank_to_genome({"file": {"path": genome_test_file_scratch},
"workspace_name": self.getWsName(),
"genome_name": "rhodobacter_genomic.gbff",
"generate_ids_if_needed": 1})["genome_ref"]
genome_name = "Rhodoannotated_by_prokka"
print("ABOUT TO ANNOTATE GENOME")
result = self.getImpl().annotate(self.getContext(),
{"object_ref": genome_ref_original,
"output_workspace": self.getWsName(),
"output_genome_name": genome_name,
"evalue": None,
"fast": 0,
"gcode": 0,
"genus": "genus",
"kingdom": "Bacteria",
"metagenome": 0,
"mincontiglen": 1,
"norrna": 0,
"notrna": 0,
"rawproduct": 0,
"rfam": 1,
"scientific_name": "RhodoBacter"
})[0]
genome_ref_new = self.getWsName() + "/" + genome_name
un_annotated_genome = self.getWsClient().get_objects([{"ref": genome_ref_original}])[0][
"data"]
re_annotated_genome = self.getWsClient().get_objects([{"ref": genome_ref_new}])[0]["data"]
scratch = "/kb/module/work/tmp/"
with open(scratch + "OUTPUT_GENOME_BEFORE.txt", "w+") as outfile:
json.dump(un_annotated_genome, outfile)
with open(scratch + "OUTPUT_GENOME_AFTER.txt", "w+") as outfile:
json.dump(un_annotated_genome, outfile)
for feature in un_annotated_genome["features"]:
if feature["id"] == "RSP_1441":
old_function = feature["functions"]
self.assertEqual(old_function, ["regulatory protein, GntR family"])
break
for feature in re_annotated_genome["features"]:
if feature["id"] == "RSP_1441":
new_function = feature["functions"]
self.assertEqual(new_function, ["N-acetylglucosamine repressor"])
break
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_deseq'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
token,
'user_id':
user_id,
'provenance': [{
'service': 'kb_deseq',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = Workspace(cls.wsURL)
cls.ws = Workspace(cls.wsURL, token=token)
cls.serviceImpl = kb_deseq(cls.cfg)
cls.serviceImpl.status(cls.ctx)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.gfu = GenomeFileUtil(cls.callback_url, service_ver='dev')
cls.dfu = DataFileUtil(cls.callback_url)
cls.ru = ReadsUtils(cls.callback_url)
cls.rau = ReadsAlignmentUtils(cls.callback_url)
cls.stringtie = kb_stringtie(cls.callback_url)
cls.eu = ExpressionUtils(cls.callback_url)
cls.deseq_runner = DESeqUtil(cls.cfg)
suffix = int(time.time() * 1000)
cls.wsName = "test_kb_stringtie_" + str(suffix)
cls.wsClient.create_workspace({'workspace': cls.wsName})
cls.dfu.ws_name_to_id(cls.wsName)
# public on CI
cls.expressionset_ref = '30957/52/41'
cls.condition_1 = 'Ecoli_WT'
cls.condition_2 = 'Ecoli_ydcR'
# public on Appdev
cls.expressionset_ref = '60454/19'
cls.condition_1 = 'WT'
cls.condition_2 = 'Hy5'
def __init__(self, config):
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.dfu = DataFileUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url, service_ver='dev')
self.uploader_utils = UploaderUtil(config)
self.scratch = os.path.join(config['scratch'],
'import_Metagenome_' + str(uuid.uuid4()))
handler_utils._mkdir_p(self.scratch)
def __init__(self, scratch_dir, callback_url, workspace_url, srv_wiz_url):
self.scratch_dir = scratch_dir
self.rau = ReadsAlignmentUtils(callback_url)
self.kbr = KBaseReport(callback_url)
self.dfu = DataFileUtil(callback_url)
self.gfu = GenomeFileUtil(callback_url)
self.set_api = SetAPI(srv_wiz_url)
self.ws = Workspace(workspace_url)
self.valid_commands = ['bamqc', 'multi-bamqc']
def __init__(self, config):
os.makedirs(self.workdir, exist_ok=True)
self.config = config
self.timestamp = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
self.callback_url = config['SDK_CALLBACK_URL']
self.scratch = config['scratch']
self.genome_api = GenomeAnnotationAPI(self.callback_url)
self.dfu = DataFileUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.kbr = KBaseReport(self.callback_url)
self.ws_client = Workspace(config["workspace-url"])
def setUpClass(cls):
cls.token = environ.get('KB_AUTH_TOKEN', None)
cls.callbackURL = environ.get('SDK_CALLBACK_URL')
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('ReadsAlignmentUtils'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(cls.token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
cls.token,
'user_id':
user_id,
'provenance': [{
'service': 'ReadsAlignmentUtils',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
cls.shockURL = cls.cfg['shock-url']
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = Workspace(cls.wsURL)
cls.ws = Workspace(cls.wsURL, token=cls.token)
cls.hs = HandleService(url=cls.cfg['handle-service-url'],
token=cls.token)
# create workspace
wssuffix = int(time.time() * 1000)
wsname = "test_alignment_" + str(wssuffix)
cls.wsinfo = cls.wsClient.create_workspace({'workspace': wsname})
print('created workspace ' + cls.getWsName())
cls.serviceImpl = ReadsAlignmentUtils(cls.cfg)
cls.readUtilsImpl = ReadsUtils(cls.callbackURL)
cls.dfu = DataFileUtil(cls.callbackURL)
cls.assemblyUtil = AssemblyUtil(cls.callbackURL)
cls.gfu = GenomeFileUtil(cls.callbackURL)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.staged = {}
cls.nodes_to_delete = []
cls.handles_to_delete = []
cls.setupTestData()
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_Msuite'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({'token': token,
'user_id': user_id,
'provenance': [
{'service': 'kb_Msuite',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated': 1})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = workspaceService(cls.wsURL)
cls.serviceImpl = kb_Msuite(cls.cfg)
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.scratch = cls.cfg['scratch']
cls.suffix = int(time.time() * 1000)
#cls.scratch = cls.cfg['scratch']+'_'+str(suffix)
#cls.cfg['scratch'] = cls.scratch
#if not os.path.exists(cls.scratch):
# os.mkdir(cls.scratch)
cls.checkm_runner = CheckMUtil(cls.cfg, cls.ctx)
cls.wsName = "test_kb_Msuite_" + str(cls.suffix)
cls.ws_info = cls.wsClient.create_workspace({'workspace': cls.wsName})
cls.au = AssemblyUtil(os.environ['SDK_CALLBACK_URL'])
cls.setAPI = SetAPI(url=cls.cfg['srv-wiz-url'], token=cls.ctx['token'])
cls.gfu = GenomeFileUtil(os.environ['SDK_CALLBACK_URL'], service_ver='dev')
cls.mu = MetagenomeUtils(os.environ['SDK_CALLBACK_URL'])
# stage an input and output directory
"""
cls.input_dir = os.path.join(cls.scratch, 'input_1')
cls.output_dir = os.path.join(cls.scratch, 'output_1')
cls.all_seq_fasta = os.path.join(cls.scratch, 'all_seq.fna')
shutil.copytree(os.path.join('data', 'example_out', 'input'), cls.input_dir)
shutil.copytree(os.path.join('data', 'example_out', 'output'), cls.output_dir)
shutil.copy(os.path.join('data', 'example_out', 'all_seq.fna'), cls.all_seq_fasta)
"""
# prepare WS data
cls.prepare_data()
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.scratch = config['scratch']
self.dfu = DataFileUtil(self.callback_url)
self.ru = ReadsUtils(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
def __init__(self, config):
#BEGIN_CONSTRUCTOR
self.callback_url = os.environ['SDK_CALLBACK_URL']
#self.dfu = DataFileUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.shared_folder = config['scratch']
logging.basicConfig(format='%(created)s %(levelname)s: %(message)s',
level=logging.INFO)
self.ws_url = config['workspace-url']
#END_CONSTRUCTOR
pass
def fetch_genome_files(self, params, gbk_dir):
gfu = GenomeFileUtil(self.callback_url)
gbk = gfu.genome_to_genbank({'genome_ref':params['input_file']})
gbk_file = gbk["genbank_file"]["file_path"]
base = ntpath.basename(gbk_file).rsplit(".", 1)[0]
name_gbff = base + ".gbff"
name_gbk = base + ".gbk"
shutil.copy(gbk_file, gbk_dir)
gbff_path = os.path.join(gbk_dir, name_gbff)
gbk_path = os.path.join(gbk_dir, name_gbk)
shutil.move(gbff_path, gbk_path)
return base, gbk_path
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
test_time_stamp = int(time.time() * 1000)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_Msuite'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
token,
'user_id':
user_id,
'provenance': [{
'service': 'kb_Msuite',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = Workspace(cls.wsURL)
cls.serviceImpl = kb_Msuite(cls.cfg)
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.scratch = cls.cfg['scratch']
cls.appdir = cls.cfg['appdir']
cls.test_data_dir = os.path.join(cls.scratch, 'test_data')
cls.suffix = test_time_stamp
cls.checkm_runner = CheckMUtil(cls.cfg, cls.ctx)
cls.wsName = "test_kb_Msuite_" + str(cls.suffix)
cls.ws_info = cls.wsClient.create_workspace({'workspace': cls.wsName})
cls.au = AssemblyUtil(os.environ['SDK_CALLBACK_URL'])
cls.gfu = GenomeFileUtil(os.environ['SDK_CALLBACK_URL'],
service_ver='dev')
cls.mu = MetagenomeUtils(os.environ['SDK_CALLBACK_URL'])
cls.setAPI = SetAPI(url=cls.cfg['srv-wiz-url'], token=cls.ctx['token'])
cls.kr = KBaseReport(os.environ['SDK_CALLBACK_URL'])
cls.data_loaded = False
def getGenomeInfo(self, genome_basename, item_i=0):
if hasattr(self.__class__, 'genomeInfo_list'):
try:
info = self.__class__.genomeInfo_list[item_i]
name = self.__class__.genomeName_list[item_i]
if info != None:
if name != genome_basename:
self.__class__.genomeInfo_list[item_i] = None
self.__class__.genomeName_list[item_i] = None
else:
return info
except:
pass
# 1) transform genbank to kbase genome object and upload to ws
shared_dir = "/kb/module/work/tmp"
genome_data_file = 'data/genomes/'+genome_basename+'.gbff.gz'
genome_file = os.path.join(shared_dir, os.path.basename(genome_data_file))
shutil.copy(genome_data_file, genome_file)
SERVICE_VER = 'release'
#SERVICE_VER = 'dev'
GFU = GenomeFileUtil(os.environ['SDK_CALLBACK_URL'],
token=self.getContext()['token'],
service_ver=SERVICE_VER
)
print ("UPLOADING genome: "+genome_basename+" to WORKSPACE "+self.getWsName()+" ...")
genome_upload_result = GFU.genbank_to_genome({'file': {'path': genome_file },
'workspace_name': self.getWsName(),
'genome_name': genome_basename
})
# })[0]
pprint(genome_upload_result)
genome_ref = genome_upload_result['genome_ref']
new_obj_info = self.getWsClient().get_object_info_new({'objects': [{'ref': genome_ref}]})[0]
# 2) store it
if not hasattr(self.__class__, 'genomeInfo_list'):
self.__class__.genomeInfo_list = []
self.__class__.genomeName_list = []
for i in range(item_i+1):
try:
assigned = self.__class__.genomeInfo_list[i]
except:
self.__class__.genomeInfo_list.append(None)
self.__class__.genomeName_list.append(None)
self.__class__.genomeInfo_list[item_i] = new_obj_info
self.__class__.genomeName_list[item_i] = genome_basename
return new_obj_info
def __init__(self, Config):
callback_url = os.environ['SDK_CALLBACK_URL']
ws_url = Config['ws_url']
self.wsc = Workspace(ws_url)
self.dfu = DataFileUtil(callback_url)
self.gfu = GenomeFileUtil(callback_url)
#service-wizard url
self.sw_url = Config['sw_url']
self.shock_url = Config['shock_url']
scratch = Config['scratch']
session = str(uuid.uuid4())
self.session_dir = (os.path.join(scratch, session))
os.mkdir(self.session_dir)
pass
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config["SDK_CALLBACK_URL"]
self.token = config["KB_AUTH_TOKEN"]
self.shock_url = config["shock-url"]
self.srv_wiz_url = config["srv-wiz-url"]
self.scratch = config["scratch"]
self.dfu = DataFileUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.eu = ExpressionUtils(self.callback_url)
self.ws = Workspace(self.ws_url, token=self.token)
self.set_client = SetAPI(self.srv_wiz_url, service_ver="dev")
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_stringtie'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
token,
'user_id':
user_id,
'provenance': [{
'service': 'kb_stringtie',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = workspaceService(cls.wsURL)
cls.ws = Workspace(cls.wsURL, token=token)
cls.serviceImpl = kb_stringtie(cls.cfg)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.gfu = GenomeFileUtil(cls.callback_url)
cls.dfu = DataFileUtil(cls.callback_url)
cls.ru = ReadsUtils(cls.callback_url)
cls.rau = ReadsAlignmentUtils(cls.callback_url)
cls.au = AssemblyUtil(cls.callback_url)
cls.stringtie_runner = StringTieUtil(cls.cfg)
suffix = int(time.time() * 1000)
cls.wsName = "test_kb_stringtie_" + str(suffix)
# cls.wsName = "jjeffryes:narrative_1516993063374"
cls.wsClient.create_workspace({'workspace': cls.wsName})
cls.prepare_data()
def __init__(self, config):
self.scratch = config["scratch"]
self.ctx = config['ctx']
self.callback_url = config["SDK_CALLBACK_URL"]
self.ws_client = workspaceService(config["workspace-url"])
self.gfu = GenomeFileUtil(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.kbr = KBaseReport(self.callback_url)
self.dfu = DataFileUtil(self.callback_url)
self.genome_api = GenomeAnnotationAPI(self.callback_url)
self.sso_ref = None
self.sso_event = None
self.ec_to_sso = {}
self.output_workspace = None
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.scratch = config['scratch']
self.dfu = DataFileUtil(self.callback_url)
self.ru = ReadsUtils(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.sp_uploader = sample_uploader(self.callback_url,
service_ver='beta')
self.dotfu = KBaseDataObjectToFileUtils(self.callback_url,
token=self.token,
service_ver='beta')
def load_genbank_file(callback_url, ws_name, local_file, target_name):
"""
Loads a Genbank (.gbk/.gbff/etc.) file into a workspace as a Genome object. This
has the side effect of building an Assembly to contain the genome sequence.
"""
gfu = GenomeFileUtil(callback_url)
genome_ref = gfu.genbank_to_genome({
"file": {
"path": local_file
},
"genome_name": target_name,
"workspace_name": ws_name,
"source": "Ensembl",
"type": "User upload",
"generate_ids_if_needed": 1
})
return genome_ref.get('genome_ref') # yeah, i know.
def setUpClass(cls):
token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('GenericsAPI'):
cls.cfg[nameval[0]] = nameval[1]
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
token,
'user_id':
user_id,
'provenance': [{
'service': 'GenericsAPI',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = workspaceService(cls.wsURL)
cls.serviceImpl = GenericsAPI(cls.cfg)
cls.scratch = cls.cfg['scratch']
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.gfu = GenomeFileUtil(cls.callback_url)
cls.dfu = DataFileUtil(cls.callback_url)
cls.sample_uploader = sample_uploader(cls.callback_url,
service_ver="dev")
cls.sample_ser = SampleService(cls.cfg['srv-wiz-url'])
suffix = int(time.time() * 1000)
cls.wsName = "test_GenericsAPI_" + str(suffix)
ret = cls.wsClient.create_workspace({'workspace': cls.wsName})
cls.wsId = ret[0]
cls.prepare_data()
def setUpClass(cls):
cls.token = environ.get('KB_AUTH_TOKEN', None)
config_file = environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('kb_uploadmethods'):
cls.cfg[nameval[0]] = nameval[1]
authServiceUrl = cls.cfg.get(
'auth-service-url',
"https://kbase.us/services/authorization/Sessions/Login")
auth_client = _KBaseAuth(authServiceUrl)
cls.user_id = auth_client.get_user(cls.token)
# WARNING: don't call any logging methods on the context object,
# it'll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
'token':
cls.token,
'user_id':
cls.user_id,
'provenance': [{
'service': 'kb_uploadmethods',
'method': 'please_never_use_it_in_production',
'method_params': []
}],
'authenticated':
1
})
cls.wsURL = cls.cfg['workspace-url']
cls.wsClient = workspaceService(cls.wsURL, token=cls.token)
cls.serviceImpl = kb_uploadmethods(cls.cfg)
cls.dfu = DataFileUtil(os.environ['SDK_CALLBACK_URL'], token=cls.token)
cls.gfu = GenomeFileUtil(os.environ['SDK_CALLBACK_URL'],
token=cls.token)
cls.fba_tools = fba_tools(os.environ['SDK_CALLBACK_URL'],
token=cls.token)
cls.scratch = cls.cfg['scratch']
cls.shockURL = cls.cfg['shock-url']
suffix = int(time.time() * 1000)
cls.wsName = "test_kb_uploadmethods_phenotype_set" + str(suffix)
cls.wsClient.create_workspace({'workspace': cls.wsName})
cls.prepare_data()
def loadGenome(self):
if hasattr(self.__class__, 'genome_ref'):
return self.__class__.genome_ref
genbank_file_path = os.path.join(self.scratch, 'minimal.gbff')
shutil.copy(os.path.join('data', 'minimal.gbff'), genbank_file_path)
gfu = GenomeFileUtil(self.callback_url)
genome_ref = gfu.genbank_to_genome({
'file': {
'path': genbank_file_path
},
'workspace_name': self.getWsName(),
'genome_name': 'test_genome',
'source': 'Ensembl',
'generate_ids_if_needed': 1,
'generate_missing_genes': 1
})['genome_ref']
self.__class__.genome_ref = genome_ref
return genome_ref
def __init__(self, config):
#BEGIN_CONSTRUCTOR
self.workspaceURL = config['workspace-url']
self.testing = False
if (config['testing'] == '1'):
self.testing = True
self.runOrthoFinder = True
if (config['run_orthofinder'] == '0'):
self.runOrthoFinder = False
self.token = os.environ['KB_AUTH_TOKEN']
self.scratch = os.path.abspath(config['scratch'])
self.callback_url = os.environ['SDK_CALLBACK_URL']
self.dfu = DataFileUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
#END_CONSTRUCTOR
pass
def setUpClass(cls):
config_file = environ.get("KB_DEPLOYMENT_CONFIG", None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items("ProkkaAnnotation"):
cls.cfg[nameval[0]] = nameval[1]
# Token validation
token = environ.get("KB_AUTH_TOKEN", None)
authServiceUrl = cls.cfg.get(
"auth-service-url",
"https://kbase.us/services/authorization/Sessions/Login")
auth_client = _KBaseAuth(authServiceUrl)
user_id = auth_client.get_user(token)
# WARNING: don"t call any logging methods on the context object,
# it"ll result in a NoneType error
cls.ctx = MethodContext(None)
cls.ctx.update({
"token":
token,
"user_id":
user_id,
"provenance": [{
"service": "ProkkaAnnotation",
"method": "please_never_use_it_in_production",
"method_params": []
}],
"authenticated":
1
})
cls.wsURL = cls.cfg["workspace-url"]
cls.wsClient = workspaceService(cls.wsURL, token=token)
cls.serviceImpl = ProkkaAnnotation(cls.cfg)
cls.callback_url = os.environ['SDK_CALLBACK_URL']
cls.gfu = GenomeFileUtil(cls.callback_url)
cls.au = AssemblyUtil(cls.callback_url)
cls.scratch = cls.cfg['scratch']
def getAMAInfo(self, ama_basename, item_i=0):
if hasattr(self.__class__, 'amaInfo_list'):
try:
info = self.__class__.amaInfo_list[item_i]
name = self.__class__.amaName_list[item_i]
if info != None:
if name != ama_basename:
self.__class__.amaInfo_list[item_i] = None
self.__class__.amaName_list[item_i] = None
else:
return info
except:
pass
# 1) transform GFF+FNA to kbase AMA object and upload to ws
shared_dir = "/kb/module/work/tmp"
ama_gff_srcfile = 'data/amas/'+ama_basename+'.gff'
ama_fna_srcfile = 'data/amas/'+ama_basename+'.fa'
ama_gff_dstfile = os.path.join(shared_dir, os.path.basename(ama_gff_srcfile))
ama_fna_dstfile = os.path.join(shared_dir, os.path.basename(ama_fna_srcfile))
shutil.copy(ama_gff_srcfile, ama_gff_dstfile)
shutil.copy(ama_fna_srcfile, ama_fna_dstfile)
try:
SERVICE_VER = 'release'
#SERVICE_VER = 'dev'
GFU = GenomeFileUtil(os.environ['SDK_CALLBACK_URL'],
token=self.getContext()['token'],
service_ver=SERVICE_VER
)
except:
raise ValueError ("unable to obtain GenomeFileUtil client")
print ("UPLOADING AMA: "+ama_basename+" to WORKSPACE "+self.getWsName()+" ...")
ama_upload_params = {
"workspace_name": self.getWsName(),
"genome_name": ama_basename,
"fasta_file": {"path": ama_fna_dstfile},
"gff_file": {"path": ama_gff_dstfile},
"source": "GFF",
"scientific_name": "TEST AMA",
"generate_missing_genes": "True"
}
try:
ama_upload_result = GFU.fasta_gff_to_metagenome(ama_upload_params)
except:
raise ValueError("unable to upload test AMA data object")
print ("AMA UPLOADED")
pprint(ama_upload_result)
ama_ref = ama_upload_result['metagenome_ref']
new_obj_info = self.getWsClient().get_object_info_new({'objects': [{'ref': ama_ref}]})[0]
# 2) store it
if not hasattr(self.__class__, 'amaInfo_list'):
self.__class__.amaInfo_list = []
self.__class__.amaName_list = []
for i in range(item_i+1):
try:
assigned = self.__class__.amaInfo_list[i]
except:
self.__class__.amaInfo_list.append(None)
self.__class__.amaName_list.append(None)
self.__class__.amaInfo_list[item_i] = new_obj_info
self.__class__.amaName_list[item_i] = ama_basename
return new_obj_info
def test_kb_blast_BLASTp_Search_04_AnnotatedMetagenomeAssembly(self):
obj_basename = 'BLASTp_AnnotatedMetagenomeAssembly'
obj_out_name = obj_basename + ".test_output.FS"
obj_out_type = "KBaseCollections.FeatureSet"
# upload test AMA
ama_name = "ama_test.AMA"
ama_feature_cnt = 888
ama_contigs_file_src = "data/AnnotatedMetagenomeAssembly/ama_contigs.fasta"
ama_genes_file_src = "data/AnnotatedMetagenomeAssembly/ama_genes.gff"
shared_dir = "/kb/module/work/tmp"
ama_contigs_file_upload = os.path.join(
shared_dir, os.path.basename(ama_contigs_file_src))
ama_genes_file_upload = os.path.join(
shared_dir, os.path.basename(ama_genes_file_src))
shutil.copy(ama_contigs_file_src, ama_contigs_file_upload)
shutil.copy(ama_genes_file_src, ama_genes_file_upload)
ama_upload_params = {
"workspace_name": self.getWsName(),
"genome_name": ama_name,
"fasta_file": {
"path": ama_contigs_file_upload
},
"gff_file": {
"path": ama_genes_file_upload
},
"source": "GFF",
"scientific_name": "TEST AMA",
"generate_missing_genes": "True"
}
try:
SERVICE_VER = 'dev'
GFU = GenomeFileUtil(os.environ['SDK_CALLBACK_URL'],
token=self.getContext()['token'],
service_ver=SERVICE_VER)
print("UPLOADING AMA: " + ama_name + " to WORKSPACE " +
self.getWsName() + " ...")
ama_upload_result = GFU.fasta_gff_to_metagenome(ama_upload_params)
except:
raise ValueError("unable to upload test AMA data object")
pprint(ama_upload_result)
ama_ref_1 = ama_upload_result['metagenome_ref']
# gene 5_267 from ama_test.AMA
query_seq_prot = 'MDRDALTKLVTDLVSIPSVNPLEGPVGNGRGEAELAAFIHSRLTEAGVVCELKEALPGRPNIIARLPGQSEEMIWFDAHMDTVSGEGMAFPPFEPLIEGDRLLGRGSSDNKGSIATMMAALMEVAKSGERPPLTVVFTATADEEYMMRGMLSLFEAGLTAKAGIVAEPTALEIVIAHKGVARFKISTTGKAAHSSRPEEGVNAIYRMGKVLGAIEAYAKRGVGRETHPLLGKGTLSVGIIRGGEYVNVVPDQCEVDVDRRLLPGEDPRRAVSDVRDYLSNALQEEVGLKVSGPTLTVPGLAVSAESPLVQAVAAAVREVTGKAPLTGMQGATHAGQMAAVDIPALVFGPGQMGQAHTATEELDLTQLERAAAVYERLMRTGL'
parameters = {
'workspace_name': self.getWsName(),
'input_one_sequence': query_seq_prot,
#'input_one_ref': "",
'output_one_name': obj_basename + '.' + "test_query.SS",
'input_many_ref': ama_ref_1,
'output_filtered_name': obj_out_name,
'e_value': ".001",
'bitscore': "50",
'ident_thresh': "40.0",
'overlap_fraction': "50.0",
'maxaccepts': "1000",
'output_extra_format': "none"
}
ret = self.getImpl().BLASTp_Search(self.getContext(), parameters)[0]
self.assertIsNotNone(ret['report_ref'])
# check created obj
#report_obj = self.getWsClient().get_objects2({'objects':[{'ref':ret['report_ref']}]})[0]['data']
report_obj = self.getWsClient().get_objects([{
'ref': ret['report_ref']
}])[0]['data']
self.assertIsNotNone(report_obj['objects_created'][0]['ref'])
created_obj_0_info = self.getWsClient().get_object_info_new(
{'objects': [{
'ref': report_obj['objects_created'][0]['ref']
}]})[0]
[
OBJID_I, NAME_I, TYPE_I, SAVE_DATE_I, VERSION_I, SAVED_BY_I,
WSID_I, WORKSPACE_I, CHSUM_I, SIZE_I, META_I
] = list(range(11)) # object_info tuple
self.assertEqual(created_obj_0_info[NAME_I], obj_out_name)
self.assertEqual(created_obj_0_info[TYPE_I].split('-')[0],
obj_out_type)
pass
def prepare_data(self):
updated = False
# create genome object
#
if not self.genome_ref:
# connect to client
try:
gfuClient = GenomeFileUtil(self.callback_url,
token=self.getContext()['token'])
except Exception as e:
raise ValueError(
'Unable to instantiate gfuClient with callbackURL: ' +
self.callback_url + ' ERROR: ' + str(e))
# upload data
sci_name = 'Thermodesulfobacterium thermophilum DSM 1276',
base_genome = 'GCF_000421605.1_ASM42160v1_genomic'
genome_gff_file = base_genome + '.gff.gz'
genome_fna_file = base_genome + '.fna.gz'
genome_gff_path = os.path.join(self.scratch, genome_gff_file)
genome_fna_path = os.path.join(self.scratch, genome_fna_file)
shutil.copy(os.path.join("data", genome_gff_file), genome_gff_path)
shutil.copy(os.path.join("data", genome_fna_file), genome_fna_path)
self.genome_ref = gfuClient.fasta_gff_to_genome({
'workspace_name':
self.getWsName(),
'fasta_file': {
'path': genome_fna_path
},
'gff_file': {
'path': genome_gff_path
},
'generate_missing_genes':
1,
'source':
'GFF',
#'scientific_name': sci_name, # this is causing an error for some reason
'genome_name':
base_genome + '.Genome'
}).get('genome_ref')
updated = True
# create reads objects
#
if not self.reads_refs:
# connect to client
try:
ruClient = ReadsUtils(self.callback_url,
token=self.getContext()['token'])
except Exception as e:
raise ValueError(
'Unable to instantiate ruClient with callbackURL: ' +
self.callback_url + ' ERROR: ' + str(e))
# upload data (ReadsUtils.upload_reads() won't take a gzipped file, so decompress)
base_reads_list = [
'Thermodesulfo_50K-0.inter', 'Thermodesulfo_50K-1.inter'
]
reads_refs = []
for base_reads in base_reads_list:
reads_file = src_file = base_reads + '.fq'
reads_path = os.path.join(self.scratch, reads_file)
#shutil.copy(os.path.join("data", reads_file+'.gz'), reads_path+'.gz')
src_path = os.path.join("data", src_file + '.gz')
dst_path = os.path.join(self.scratch, src_file)
with gzip.open(src_path, 'rb') as f_in:
with open(dst_path, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
reads_refs.append(
ruClient.upload_reads({
'wsname': self.getWsName(),
'fwd_file': reads_path,
'sequencing_tech': 'artificial reads',
'interleaved': 1,
'name': base_reads + '.Reads'
})['obj_ref'])
self.reads_refs = reads_refs
updated = True
return updated
def run_barseqR(self, ctx, params):
"""
Args:
:param params: instance of mapping from String to unspecified object
ctx:
client_ip: None or 'str',
user_id: str,
'authenticated': 1,
'token': str,
'module': None,
'method': None,
'call_id': None,
'rpc_context': None,
'provenance':list<prov_d>
prov_d: (d)
service: (str)
'method': 'please_never_use_it_in_production',
'method_params': []}]}
:returns: instance of type "ReportResults" -> structure: parameter
"report_name" of String, parameter "report_ref" of String
"""
# ctx is the context object
# return variables are: output
#BEGIN run_barseqR
# SETUP - Unrelated to inputs --------
logging.basicConfig(level=logging.DEBUG)
logging.info("Call back url: " + str(self.callback_url))
# We create important classes
dfu = DataFileUtil(self.callback_url)
logging.info("DFU VARS-- " * 8)
logging.info(vars(dfu))
gfu = GenomeFileUtil(self.callback_url)
smpl_s = SampleService(self.callback_url)
myToken = os.environ.get('KB_AUTH_TOKEN', None)
ws = Workspace(self.ws_url, token=myToken)
ws_id = ws.get_workspace_info({'workspace':
params['workspace_name']})[0]
logging.info(os.environ)
logging.info('ws-url')
logging.info(self.ws_url)
logging.info('ctx')
logging.info(ctx)
# We create indir, outdir, sets_dir (Input, Output, Sets)
indir = os.path.join(self.shared_folder, "indir")
os.mkdir(indir)
outdir = os.path.join(self.shared_folder, "outdir")
os.mkdir(outdir)
sets_dir = os.path.join(indir, "sets_dir")
os.mkdir(sets_dir)
metadir = '/kb/module/lib/RunDir/metadata'
if not (os.path.isdir(metadir)):
raise Exception(
"metadata directory not found at: {}".format(metadir))
# We prepare locations of input files
poolfile_path = os.path.join(indir, "pool.n10")
gene_table_fp = os.path.join(indir, "genes.GC")
exps_file = os.path.join(indir, "FEBA_Barseq.tsv")
# END SETUP
# VALIDATE PARAMS:
logging.info("PARAMS:")
logging.info(params)
# From Util.validate python file
val_par = validate_params(params)
'''
val_par contains keys:
genome_ref
poolfile_ref
exps_ref
sets_ref
output_name
workspace_name
'''
val_par['username'] = ctx['user_id']
# DOWNLOAD FILES
download_dict = {
"dfu": dfu,
"gfu": gfu,
"ws": ws,
"smpl_s": smpl_s,
"sets_dir": sets_dir,
"poolfile_path": poolfile_path,
"gene_table_fp": gene_table_fp,
"exps_file": exps_file,
"scratch_dir": self.shared_folder
}
# We copy input files to proper directories.
# vp must contain genome_ref, poolfile_ref, exps_ref, sets_refs (list)
# DownloadResults must contain keys 'org', 'set_names_list', 'set_fps_list'
# set_names_list value contains the names of the sets without extensions
DownloadResults = download_files(val_par, download_dict)
logging.debug(json.dumps(DownloadResults, indent=2))
# Get args in this format:
# [-org, org_name, -indir, Scratch_Dir_Input, -metadir, Fixed meta dir,
# -outdir, scratch_dir_output, -sets_dir, within scratch_dir_input,
# -sets, set1 (sets_dir), set2 (sets_dir), set3 (sets_dir), ... ]
# Note meta dir is called metadata and is in RunDir
# Running the entire program:
arg_list = [
"-org", DownloadResults['org'], '-indir', indir, '-metadir',
metadir, '-outdir', outdir, '-sets_dir', sets_dir, '-sets'
]
arg_list += DownloadResults['set_names_list']
RunBarSeq(arg_list)
# Returning files to user
report = KBaseReport(self.callback_url)
report_info = report.create({
'report': {
'objects_created': [],
'text_message': params['parameter_1']
},
'workspace_name': params['workspace_name']
})
output = {
'report_name': report_info['name'],
'report_ref': report_info['ref'],
}
#END run_barseqR
# At some point might do deeper type checking...
if not isinstance(output, dict):
raise ValueError('Method run_barseqR return value ' +
'output is not type dict as required.')
# return the results
return [output]
def __init__(self):
self.callback_url = os.environ['SDK_CALLBACK_URL']
self.au = AssemblyUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
pass