def getHandles(logger,
shock_url = "https://kbase.us/services/shock-api/",
handle_url = "https://kbase.us/services/handle_service/",
shock_ids = None,
handle_ids = None,
token = None):
hs = AbstractHandle(url=handle_url, token=token)
handles = list()
if shock_ids is not None:
header = dict()
header["Authorization"] = "Oauth {0}".format(token)
for sid in shock_ids:
info = None
try:
logger.info("Found shock id {0}, retrieving information about the data.".format(sid))
response = requests.get("{0}/node/{1}".format(shock_url, sid), headers=header, verify=True)
info = response.json()["data"]
except:
logger.error("There was an error retrieving information about the shock node id {0} from url {1}".format(sid, shock_url))
try:
logger.info("Retrieving a handle id for the data.")
handle_id = hs.persist_handle({"id" : sid,
"type" : "shock",
"url" : shock_url,
"file_name": info["file"]["name"],
"remote_md5": info["file"]["md5"]})
except:
try:
handle_id = hs.ids_to_handles([sid])[0]["hid"]
single_handle = hs.hids_to_handles([handle_id])
assert len(single_handle) != 0
if info is not None:
single_handle[0]["file_name"] = info["file"]["name"]
single_handle[0]["remote_md5"] = info["file"]["md5"]
print >> sys.stderr, single_handle
handles.append(single_handle[0])
except:
logger.error("The input shock node id {} is already registered or could not be registered".format(sid))
raise
elif handle_ids is not None:
for hid in handle_ids:
try:
single_handle = hs.hids_to_handles([hid])
assert len(single_handle) != 0
handles.append(single_handle[0])
except:
logger.error("Invalid handle id {0}".format(hid))
raise
return handles
def _own_handle(self, genome_data, handle_property):
"""
_own_handle: check that handle_property point to shock nodes owned by calling user
"""
log('start checking handle {} ownership'.format(handle_property))
if handle_property in genome_data:
handle_id = genome_data[handle_property]
hs = HandleService(self.handle_url, token=self.token)
handles = hs.hids_to_handles([handle_id])
shock_id = handles[0]['id']
# Copy from DataFileUtil.own_shock_node implementation:
header = {'Authorization': 'Oauth {}'.format(self.token)}
res = requests.get(self.shock_url + '/node/' + shock_id +
'/acl/?verbosity=full',
headers=header,
allow_redirects=True)
self._check_shock_response(
res, 'Error getting ACLs for Shock node {}: '.format(shock_id))
owner = res.json()['data']['owner']['username']
user_id = self.auth_client.get_user(self.token)
if owner != user_id:
log('start copying node to owner: {}'.format(user_id))
dfu_shock = self.dfu.copy_shock_node({
'shock_id': shock_id,
'make_handle': True
})
handle_id = dfu_shock['handle']['hid']
genome_data[handle_property] = handle_id
def own_handle(self, genome, handle_property, ctx):
if not handle_property in genome:
return
token = ctx['token']
handle_id = genome[handle_property]
hs = HandleService(self.handle_url, token=token)
handles = hs.hids_to_handles([handle_id])
shock_id = handles[0]['id']
## Copy from DataFileUtil.own_shock_node implementation:
header = {'Authorization': 'Oauth {}'.format(token)}
res = requests.get(self.shock_url + '/node/' + shock_id +
'/acl/?verbosity=full',
headers=header, allow_redirects=True)
self.check_shock_response(
res, 'Error getting ACLs for Shock node {}: '.format(shock_id))
owner = res.json()['data']['owner']['username']
if owner != ctx['user_id']:
shock_id = self.copy_shock_node(ctx, shock_id)
r = requests.get(self.shock_url + '/node/' + shock_id,
headers=header, allow_redirects=True)
errtxt = ('Error downloading attributes from shock ' +
'node {}: ').format(shock_id)
self.check_shock_response(r, errtxt)
shock_data = r.json()['data']
handle = {'id': shock_data['id'],
'type': 'shock',
'url': self.shock_url,
'file_name': shock_data['file']['name'],
'remote_md5': shock_data['file']['checksum']['md5']
}
handle_id = hs.persist_handle(handle)
genome[handle_property] = handle_id
def main(argv):
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, prog='trns_transform_KBaseAssembly.FQ-to-KBaseAssembly.SingleEndLibrary', epilog=desc3)
parser.add_argument('-s', '--shock_url', help='Shock url', action='store', dest='shock_url', default='https://kbase.us/services/shock-api')
parser.add_argument('-n', '--hndl_svc_url', help='Handle service url', action='store', dest='hndl_url', default='https://kbase.us/services/handle_service')
parser.add_argument('-i', '--in_id', help='Input Shock node id', action='store', dest='inobj_id', default=None, required=False)
parser.add_argument('-f','--file_name', help = 'File Name', action= 'store', dest='file_name',default=None,required=False)
parser.add_argument('-d','--hid', help = 'Handle id', action= 'store', dest='hid',default=None, required=False)
parser.add_argument('-o', '--out_file_name', help='Output file name', action='store', dest='out_fn', default=None, required=True)
usage = parser.format_usage()
parser.description = desc1 + ' ' + usage + desc2
parser.usage = argparse.SUPPRESS
args = parser.parse_args()
if args.inobj_id is None and args.hid is None:
print >> sys.stderr, parser.description
print >> sys.stderr, "Need to provide either shock node id or handle id"
exit(1)
kb_token = os.environ.get('KB_AUTH_TOKEN')
hs = AbstractHandle(url=args.hndl_url, token = kb_token)
if args.hid is None:
try:
args.hid = hs.persist_handle({ "id" : args.inobj_id , "type" : "shock" , "url" : args.shock_url})
except:
try:
args.hid=hs.ids_to_handles([args.inobj_id])[0]["hid"]
except:
traceback.print_exc(file=sys.stderr)
print >> sys.stderr, "Please provide handle id.\nThe input shock node id {} is already registered or could not be registered".format(args.inobj_id)
exit(3)
hds = hs.hids_to_handles([args.hid])
if len(hds) <= 0:
print >> sys.stderr, 'Could not register a new handle with shock node id {} or wrong input handle id'.format(args.inobj_id)
exit(2)
ret = { "handle" : hds[0] }
of = open(args.out_fn, "w")
of.write(to_JSON(ret))
of.close()
class HandleService(object):
def __init__(self, url=None, timeout=30 * 60, user_id=None,
password=None, token=None, ignore_authrc=False,
trust_all_ssl_certificates=False):
self.url = url
self.timeout = int(timeout)
self._headers = dict()
self.trust_all_ssl_certificates = trust_all_ssl_certificates
self.token = token
# token overrides user_id and password
if token is not None: pass
elif user_id is not None and password is not None:
self.token = _get_token(user_id, password)
elif 'KB_AUTH_TOKEN' in _os.environ:
self.token = _os.environ.get('KB_AUTH_TOKEN')
elif not ignore_authrc:
authdata = _read_inifile()
if authdata is None:
authdata = _read_rcfile()
if authdata is not None:
if authdata.get('token') is not None:
self.token = authdata['token']
elif(authdata.get('user_id') is not None
and authdata.get('password') is not None):
self.token = _get_token(
authdata['user_id'], authdata['password'])
if self.timeout < 1:
raise ValueError('Timeout value must be at least 1 second')
self.dsi = AbstractHandle(url=url, token=self.token, trust_all_ssl_certificates=trust_all_ssl_certificates)
def upload(self,infile) :
handle = self.dsi.new_handle()
url = "{}/node/{}".format(handle["url"], handle["id"]);
ref_data = {}
try:
ref_data = _upload_file_to_shock(url,filePath=infile,token=self.token)
except:
raise
remote_md5 = ref_data["file"]["checksum"]["md5"]
#remote_sha1 = ref_data["file"]["checksum"]["sha1"] #SHOCK PUT would not create remote_sha1
if remote_md5 is None: raise Exception("looks like upload failed and no md5 returned from remote server")
handle["remote_md5"] = remote_md5
#handle["remote_sha1"] = remote_sha1
handle["file_name"] = _os.path.basename(infile)
self.dsi.persist_handle(handle)
return handle
def download (self, handle, outfile):
if( isinstance(handle, dict)): raise Exception("hande is not a dictionary")
if "id" not in handle: raise Exception("no id in handle")
if "url" not in handle: raise Exception("no url in handle")
if outfile is None: raise Exception("outfile is not defined")
raise Exception("Not implemented yet")
def new_handle(self, *arg):
return self.dsi.new_handle(*arg)
def localize_handle(self, *arg):
return self.dsi.localize_handle(*arg)
def initialize_handle (self, *arg):
return self.dsi.initialize_handle (*arg)
def persist_handle(self, *arg):
return self.dsi.persist_handle (*arg)
def upload_metadata(self, handle, infile):
raise Exception("Not implemented yet")
def download_metadata (self, handle, outfile):
raise Exception("Not implemented yet")
def list_handles (self, *arg):
return self.dsi.list_handles (*arg)
def are_readable (self, *arg):
return self.dsi.are_readable(*arg)
def is_readable(self, *arg):
return self.dsi.is_readable(*arg)
def hids_to_handles(self, *arg):
return self.dsi.hids_to_handles(*arg)
def ids_to_handles(self, *arg):
return self.dsi.ids_to_handles(*arg)
def main(argv):
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, prog='trnf_Convert_fastq', epilog=desc3)
parser.add_argument('-s', '--shock_url', help='Shock url', action='store', dest='shock_url', default='https://kbase.us/services/shock-api')
parser.add_argument('-n', '--handle_service_url', help='Handle service url', action='store', dest='hndl_url', default='https://kbase.us/services/handle_service')
parser.add_argument('-i', '--in_ids', help='Two input Shock node ids (comma separated)', action='store', dest='inobj_id', default=None, required=False)
parser.add_argument('-f','--file_names', help = 'Two optional handle file names (comma separated)', action= 'store', dest='loc_filepath',default=None,nargs=1,required=False)
parser.add_argument('-d','--hids', help = 'Two handle ids (comma separated)', action= 'store', dest='hid',default=None,required=False)
parser.add_argument('-m','--ins_mean', help = 'Mean insert size', action= 'store', dest='ins_mean',type=float,default=None)
parser.add_argument('-k','--std_dev', help = 'Standard deviation', action= 'store', dest='std_dev',type=float,default=None)
parser.add_argument('-l','--inl', help = 'Interleaved -- true/false', action= 'store', dest='inl',default=None)
parser.add_argument('-r','--r_ori', help = 'Read Orientation -- true/false', action= 'store', dest='read_orient',default=None)
parser.add_argument('-o', '--out_file_name', help='Output file name', action='store', dest='out_fn', default=None, required=True)
usage = parser.format_usage()
parser.description = desc1 + ' ' + usage + desc2
parser.usage = argparse.SUPPRESS
args = parser.parse_args()
if args.inobj_id is None and args.hid is None:
print >> sys.stderr, parser.description
print >> sys.stderr, "Need to provide either shock node ids or handle ids"
exit(1)
kb_token = os.environ.get('KB_AUTH_TOKEN')
hs = AbstractHandle(url=args.hndl_url, token = kb_token)
hids = []
if args.hid is None:
snids = args.inobj_id.split(',')
if len(snids) != 2:
print >> sys.stderr, "Please provide two shock node ids for pairend library"
exit(4)
try:
hids.append(hs.persist_handle({ "id" : snids[0] , "type" : "shock" , "url" : args.shock_url}))
except:
try:
hids.append(hs.ids_to_handles([snids[0]])[0]["hid"])
except:
traceback.print_exc(file=sys.stderr)
e,v = sys.exc_info()[:2]
print >> sys.stderr, "Please provide handle id.\nThe input shock node id {} is already registered or could not be registered : {} -- {}".format(snids[0], str(e), str(v))
exit(3)
try:
hids.append(hs.persist_handle({ "id" : snids[1] , "type" : "shock" , "url" : args.shock_url}))
except:
try:
hids.append(hs.ids_to_handles([snids[1]])[0]["hid"])
except:
traceback.print_exc(file=sys.stderr)
e,v = sys.exc_info()[:2]
print >> sys.stderr, "Please provide handle id.\nThe input shock node id {} is already registered or could not be registered : {} -- {}".format(snids[1], str(e), str(v))
exit(3)
else:
hids = args.hid.split(',')
if len(hids) != 2:
print >> sys.stderr, "Please provide two handle ids for pairend library"
exit(5)
hds = hs.hids_to_handles(hids)
if len(hds) != 2:
print >> sys.stderr, 'Could not register a new handle with shock node id {} or wrong input handle id'.format(args.inobj_id)
exit(2)
ret = {"handle_1" : hds[0], "handle_2" : hds[1]}
if args.ins_mean is not None :
ret["insert_size_mean"] = args.ins_mean
if args.std_dev is not None:
ret["insert_size_std_dev"] = args.std_dev
if args.inl == 'true':
ret["interleaved"] = 0
if args.read_orient == 'true':
ret["read_orientation_outward"] = 0
of = open(args.out_fn, "w")
of.write(to_JSON(ret))
of.close()
def main(argv):
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
prog='trnf_Convert_fastq',
epilog=desc3)
parser.add_argument('-s',
'--shock_url',
help='Shock url',
action='store',
dest='shock_url',
default='https://kbase.us/services/shock-api')
parser.add_argument('-n',
'--handle_service_url',
help='Handle service url',
action='store',
dest='hndl_url',
default='https://kbase.us/services/handle_service')
parser.add_argument('-i',
'--in_ids',
help='Two input Shock node ids (comma separated)',
action='store',
dest='inobj_id',
default=None,
required=False)
parser.add_argument(
'-f',
'--file_names',
help='Two optional handle file names (comma separated)',
action='store',
dest='loc_filepath',
default=None,
nargs=1,
required=False)
parser.add_argument('-d',
'--hids',
help='Two handle ids (comma separated)',
action='store',
dest='hid',
default=None,
required=False)
parser.add_argument('-m',
'--ins_mean',
help='Mean insert size',
action='store',
dest='ins_mean',
type=float,
default=None)
parser.add_argument('-k',
'--std_dev',
help='Standard deviation',
action='store',
dest='std_dev',
type=float,
default=None)
parser.add_argument('-l',
'--inl',
help='Interleaved -- true/false',
action='store',
dest='inl',
default=None)
parser.add_argument('-r',
'--r_ori',
help='Read Orientation -- true/false',
action='store',
dest='read_orient',
default=None)
parser.add_argument('-o',
'--out_file_name',
help='Output file name',
action='store',
dest='out_fn',
default=None,
required=True)
usage = parser.format_usage()
parser.description = desc1 + ' ' + usage + desc2
parser.usage = argparse.SUPPRESS
args = parser.parse_args()
if args.inobj_id is None and args.hid is None:
print >> sys.stderr, parser.description
print >> sys.stderr, "Need to provide either shock node ids or handle ids"
exit(1)
kb_token = os.environ.get('KB_AUTH_TOKEN')
hs = AbstractHandle(url=args.hndl_url, token=kb_token)
hids = []
if args.hid is None:
snids = args.inobj_id.split(',')
if len(snids) != 2:
print >> sys.stderr, "Please provide two shock node ids for pairend library"
exit(4)
try:
hids.append(
hs.persist_handle({
"id": snids[0],
"type": "shock",
"url": args.shock_url
}))
except:
try:
hids.append(hs.ids_to_handles([snids[0]])[0]["hid"])
except:
traceback.print_exc(file=sys.stderr)
e, v = sys.exc_info()[:2]
print >> sys.stderr, "Please provide handle id.\nThe input shock node id {} is already registered or could not be registered : {} -- {}".format(
snids[0], str(e), str(v))
exit(3)
try:
hids.append(
hs.persist_handle({
"id": snids[1],
"type": "shock",
"url": args.shock_url
}))
except:
try:
hids.append(hs.ids_to_handles([snids[1]])[0]["hid"])
except:
traceback.print_exc(file=sys.stderr)
e, v = sys.exc_info()[:2]
print >> sys.stderr, "Please provide handle id.\nThe input shock node id {} is already registered or could not be registered : {} -- {}".format(
snids[1], str(e), str(v))
exit(3)
else:
hids = args.hid.split(',')
if len(hids) != 2:
print >> sys.stderr, "Please provide two handle ids for pairend library"
exit(5)
hds = hs.hids_to_handles(hids)
if len(hds) != 2:
print >> sys.stderr, 'Could not register a new handle with shock node id {} or wrong input handle id'.format(
args.inobj_id)
exit(2)
ret = {"handle_1": hds[0], "handle_2": hds[1]}
if args.ins_mean is not None:
ret["insert_size_mean"] = args.ins_mean
if args.std_dev is not None:
ret["insert_size_std_dev"] = args.std_dev
if args.inl == 'true':
ret["interleaved"] = 0
if args.read_orient == 'true':
ret["read_orientation_outward"] = 0
of = open(args.out_fn, "w")
of.write(to_JSON(ret))
of.close()
