def upload(files):
for f in files:
attr = json.dumps({
"type": "inbox",
"id": mgrast_auth['id'],
"user": mgrast_auth['login'],
"email": mgrast_auth['email']
})
# get format
if f.endswith(".gz"):
fformat = "gzip"
elif f.endswith(".bz2"):
fformat = "bzip2"
else:
fformat = "upload"
# POST to shock
data = {"attributes_str": attr}
result = post_file(SHOCK_URL + "/node",
fformat,
f,
data=data,
auth=mgrast_auth['token'],
debug=DEBUG)
# compute file info
info = obj_from_url(API_URL + "/inbox/info/" + result['data']['id'],
auth=mgrast_auth['token'],
debug=DEBUG)
print(info['status'])
# compute sequence stats
if info['stats_info']['file_type'] in ['fasta', 'fastq']:
stats = obj_from_url(API_URL + "/inbox/stats/" +
result['data']['id'],
auth=mgrast_auth['token'],
debug=DEBUG)
print(stats['status'].replace("stats computation", "validation"))
def upload_archive(afile):
attr = json.dumps({
"type": "inbox",
"id": mgrast_auth['id'],
"user": mgrast_auth['login'],
"email": mgrast_auth['email']
})
# get format
if afile.endswith(".tar.gz"):
aformat = "tar.gz"
elif afile.endswith(".tar.bz2"):
aformat = "tar.bz2"
elif afile.endswith(".tar"):
aformat = "tar"
elif afile.endswith(".zip"):
aformat = "zip"
else:
sys.stderr.write("ERROR: input file %s is incorrect archive format\n" %
afile)
sys.exit(1)
# POST to shock / unpack
data = {"file_name": os.path.basename(afile), "attributes_str": attr}
result = post_file(SHOCK_URL + "/node",
"upload",
afile,
data=data,
auth=mgrast_auth['token'],
debug=DEBUG)
data = {
"unpack_node": result['data']['id'],
"archive_format": aformat,
"attributes_str": attr
}
unpack = obj_from_url(SHOCK_URL + "/node",
data=data,
auth=mgrast_auth['token'],
debug=DEBUG)
# process new nodes
for node in unpack['data']:
# compute file info
info = obj_from_url(API_URL + "/inbox/info/" + node['id'],
auth=mgrast_auth['token'],
debug=DEBUG)
print(info['status'])
# compute sequence stats
if info['stats_info']['file_type'] in ['fasta', 'fastq']:
stats = obj_from_url(API_URL + "/inbox/stats/" + node['id'],
auth=mgrast_auth['token'],
debug=DEBUG)
print(stats['status'].replace("stats computation", "validation"))
def upload(files, verbose):
fids = []
attr = json.dumps({
"type": "inbox",
"id": mgrast_auth['id'],
"user": mgrast_auth['login'],
"email": mgrast_auth['email']
})
for i, f in enumerate(files):
# get format
if f.endswith(".gz"):
fformat = "gzip"
fname = os.path.basename(f[:-3])
elif f.endswith(".bz2"):
fformat = "bzip2"
fname = os.path.basename(f[:-4])
else:
fformat = "upload"
fname = os.path.basename(f)
# POST to shock
data = {
"file_name": fname,
"attributes_str": attr
}
if verbose:
if len(files) > 1:
print("Uploading file %d of %d (%s) to MG-RAST Shock"%(i+1, len(files), f))
else:
print("Uploading file %s to MG-RAST Shock"%(f))
result = post_file(SHOCK_URL+"/node", fformat, f, data=data, auth=mgrast_auth['token'], debug=verbose)
if verbose:
print(json.dumps(result['data']))
if len(files) > 1:
print("Setting info for file %d of %d (%s) in MG-RAST inbox"%(i+1, len(files), f))
else:
print("Setting info for file %s in MG-RAST inbox"%(f))
# compute file info
info = obj_from_url(API_URL+"/inbox/info/"+result['data']['id'], auth=mgrast_auth['token'], debug=verbose)
if verbose:
print(json.dumps(info))
else:
print(info['status'])
fids.append(result['data']['id'])
return fids
def archive_upload(afile, verbose):
attr = json.dumps({
"type": "inbox",
"id": mgrast_auth['id'],
"user": mgrast_auth['login'],
"email": mgrast_auth['email']
})
# get format
if afile.endswith(".tar.gz"):
aformat = "tar.gz"
elif afile.endswith(".tar.bz2"):
aformat = "tar.bz2"
elif afile.endswith(".tar"):
aformat = "tar"
elif afile.endswith(".zip"):
aformat = "zip"
else:
sys.stderr.write("ERROR: input file %s is incorrect archive format\n"%afile)
sys.exit(1)
# POST to shock / unpack
if verbose:
print("Uploading file %s to MG-RAST Shock"%(afile))
data = {
"file_name": os.path.basename(afile),
"attributes_str": attr
}
result = post_file(SHOCK_URL+"/node", "upload", afile, data=data, auth=mgrast_auth['token'], debug=verbose)
if verbose:
print(json.dumps(result['data']))
print("Unpacking archive file %s"%(afile))
data = {
"unpack_node": result['data']['id'],
"archive_format": aformat,
"attributes_str": attr
}
unpack = obj_from_url(SHOCK_URL+"/node", data=data, auth=mgrast_auth['token'], debug=verbose)
if verbose:
print(json.dumps(unpack['data']))
fids = map(lambda x: x['id'], unpack['data'])
return fids
def main(args):
global API_URL
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='',
description=prehelp % VERSION,
epilog=posthelp % AUTH_LIST)
# access options
parser.add_argument("-u",
"--url",
dest="url",
default=API_URL,
help="MG-RAST API url")
parser.add_argument("-t",
"--token",
dest="token",
default=None,
help="MG-RAST token")
# other options
parser.add_argument("-f",
"--file",
dest="mdfile",
default=None,
help="metadata .xlsx file")
parser.add_argument(
"--taxa",
dest="taxa",
default=None,
help=
"metagenome_taxonomy for project: http://www.ebi.ac.uk/ena/data/view/Taxon:408169"
)
parser.add_argument("--debug",
dest="debug",
action="store_true",
default=False,
help="Run in debug mode")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Verbose STDOUT")
parser.add_argument("args",
type=str,
nargs="+",
help="Action (" + ",".join(valid_actions) + ")")
# get inputs
opts = parser.parse_args()
args = opts.args
API_URL = opts.url
# validate inputs
if len(args) < 1:
sys.stderr.write("ERROR: missing action\n")
return 1
action = args[0]
if action not in valid_actions:
sys.stderr.write("ERROR: invalid action. use one of: %s\n" %
", ".join(valid_actions))
return 1
if len(args) < 2:
sys.stderr.write("ERROR: missing Project ID\n")
return 1
pid = args[1]
DEBUG = opts.verbose + opts.debug
# get token
token = get_auth_token(opts)
if not token:
token = input('Enter your MG-RAST auth token: ')
# actions
if action == "get-info":
data = obj_from_url(opts.url + '/project/' + pid +
'?verbosity=verbose&nocache=1',
auth=token)
print(json.dumps(data, sort_keys=True, indent=4))
elif action == "get-metadata":
data = obj_from_url(opts.url + '/metadata/export/' + pid, auth=token)
print(json.dumps(data, sort_keys=True, indent=4))
elif action == "update-metadata":
result = post_file(opts.url + '/metadata/update',
'upload',
opts.mdfile,
auth=token,
data=json.dumps({'project': pid},
separators=(',', ':')),
debug=DEBUG)
print(json.dumps(data, sort_keys=True, indent=4))
elif action == "make-public":
data = obj_from_url(opts.url + '/project/' + pid + '/makepublic',
auth=token)
print(json.dumps(data, sort_keys=True, indent=4))
elif action == "submit-ebi":
debug = 1 if opts.debug else 0
info = {'project_id': pid, 'debug': debug}
if opts.taxa:
info['project_taxonomy'] = opts.taxa
data = obj_from_url(opts.url + '/submission/ebi',
auth=token,
data=json.dumps(info, separators=(',', ':')))
print(json.dumps(data, sort_keys=True, indent=4))
elif action == "status-ebi":
data = obj_from_url(opts.url + '/submission/' + pid, auth=token)
print(json.dumps(data, sort_keys=True, indent=4))
return 0
def upload(files, verbose):
''' upload(files, verbose) -- call MG-RAST api to upload one or more files to inbox '''
fids = []
attr = json.dumps({
"type": "inbox",
"id": mgrast_auth['id'],
"user": mgrast_auth['login'],
"email": mgrast_auth['email']
})
results = {
'submitted': [],
'failed': [],
'files': files
}
# Settings for nr tries
maxtries = 3
current = 0
sleep = 60
while len(results['files']) and current < maxtries:
# increase counter
current += 1
for i, f in enumerate(results['files']):
# get format
print(i,f)
if f.endswith(".gz"):
fformat = "gzip"
fname = os.path.basename(f[:-3])
elif f.endswith(".bz2"):
fformat = "bzip2"
fname = os.path.basename(f[:-4])
else:
fformat = "upload"
fname = os.path.basename(f)
# POST to shock
data = {
"file_name": fname,
"attributes_str": attr
}
if verbose:
if len(files) > 1:
print("Uploading file %d of %d (%s) to MG-RAST Shock"%(i+1, len(files), f))
else:
print("Uploading file %s to MG-RAST Shock"%(f))
if True: # change to debug
print("Submitting %s to %s " % (f,SHOCK_URL))
result = post_file(SHOCK_URL+"/node", fformat, f, data=data, auth=mgrast_auth['token'], debug=verbose)
if result:
if verbose:
print(json.dumps(result['data']))
if len(files) > 1:
print("Setting info for file %d of %d (%s) in MG-RAST inbox"%(i+1, len(files), f))
else:
print("Setting info for file %s in MG-RAST inbox"%(f))
# compute file info
info = obj_from_url(API_URL+"/inbox/info/"+result['data']['id'], auth=mgrast_auth['token'], debug=verbose)
if verbose:
print(json.dumps(info))
else:
print(info['status'])
fids.append(result['data']['id'])
results['submitted'].append(f)
else:
print(f)
sys.stderr.write("ERROR: can not submit %s\n" % (f))
results['failed'].append(f)
if verbose:
print(results)
print("Processed %d\tFailed %d" % (len(results['files']), len(results['failed'])))
# switch list, process failed again
results['files'] = results['failed']
results['failed'] = []
# wait
time.sleep(current * sleep)
return fids
def main(args):
global API_URL
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='', description=prehelp%VERSION, epilog=posthelp%AUTH_LIST)
# access options
parser.add_argument("-u", "--url", dest="url", default=API_URL, help="MG-RAST API url")
parser.add_argument("-t", "--token", dest="token", default=None, help="MG-RAST token")
# other options
parser.add_argument("-f", "--file", dest="mdfile", default=None, help="metadata .xlsx file")
parser.add_argument("--taxa", dest="taxa", default=None, help="metagenome_taxonomy for project: http://www.ebi.ac.uk/ena/data/view/Taxon:408169")
parser.add_argument("--debug", dest="debug", action="store_true", default=False, help="Run in debug mode")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Verbose STDOUT")
parser.add_argument("args",type=str, nargs="+", help="Action (" + ",".join(valid_actions)+")" )
# get inputs
opts = parser.parse_args()
args = opts.args
API_URL = opts.url
# validate inputs
if len(args) < 1:
sys.stderr.write("ERROR: missing action\n")
return 1
action = args[0]
if action not in valid_actions:
sys.stderr.write("ERROR: invalid action. use one of: %s\n"%", ".join(valid_actions))
return 1
if len(args) < 2:
sys.stderr.write("ERROR: missing Project ID\n")
return 1
pid = args[1]
DEBUG = opts.verbose + opts.debug
# get token
token = get_auth_token(opts)
if not token:
token = input('Enter your MG-RAST auth token: ')
# actions
if action == "get-info":
data = obj_from_url(opts.url+'/project/'+pid+'?verbosity=verbose&nocache=1', auth=token)
print(json.dumps(data, sort_keys=True, indent=4))
elif action == "get-metadata":
data = obj_from_url(opts.url+'/metadata/export/'+pid, auth=token)
print(json.dumps(data, sort_keys=True, indent=4))
elif action == "update-metadata":
result = post_file(opts.url+'/metadata/update', 'upload', opts.mdfile, auth=token, data=json.dumps({'project': pid}, separators=(',',':')), debug=DEBUG)
print(json.dumps(data, sort_keys=True, indent=4))
elif action == "make-public":
data = obj_from_url(opts.url+'/project/'+pid+'/makepublic', auth=token)
print(json.dumps(data, sort_keys=True, indent=4))
elif action == "submit-ebi":
debug = 1 if opts.debug else 0
info = {
'project_id': pid,
'debug': debug
}
if opts.taxa:
info['project_taxonomy'] = opts.taxa
data = obj_from_url(opts.url+'/submission/ebi', auth=token, data=json.dumps(info, separators=(',',':')))
print(json.dumps(data, sort_keys=True, indent=4))
elif action == "status-ebi":
data = obj_from_url(opts.url+'/submission/'+pid, auth=token)
print(json.dumps(data, sort_keys=True, indent=4))
return 0