def main():
parser = argparse.ArgumentParser(description='Upload a payload using SONG')
parser.add_argument('-s', '--study-id', dest="study_id", help="Study ID", required=True)
parser.add_argument('-u', '--server-url', dest="server_url", help="Server URL", required=True)
parser.add_argument('-p', '--payload', dest="payload", help="JSON Payload", required=True)
parser.add_argument('-o', '--output', dest="output", help="Output manifest file", required=True)
parser.add_argument('-d', '--input-dir', dest="input_dir", help="Payload files directory", required=True)
parser.add_argument('-t', '--access-token', dest="access_token", default=os.environ.get('ACCESSTOKEN',None),help="Server URL")
parser.add_argument('-j','--json',dest="json_output")
results = parser.parse_args()
study_id = results.study_id
server_url = results.server_url
access_token = results.access_token
payload_file = results.payload
analysis_id = json.load(open(payload_file)).get('analysisId')
api_config = ApiConfig(server_url,study_id,access_token)
api = Api(api_config)
upload_status = upload_payload(api,payload_file)
try:
api.save(upload_status.uploadId, ignore_analysis_id_collisions=True)
except Exception as e:
song_error = json.loads(str(e))
if song_error.get('errorId') != "duplicate.analysis.attempt":
raise e
if api.get_analysis(analysis_id).__dict__['analysisState'] == "PUBLISHED":
validate_payload_against_analysis(api, analysis_id, payload_file)
manifest_filename = results.output
create_manifest(api,analysis_id,payload_file,manifest_filename,results.input_dir)
if not api.get_analysis(analysis_id).__dict__['analysisState'] == "PUBLISHED":
subprocess.check_output(['icgc-storage-client','upload','--manifest',os.path.join(results.input_dir,manifest_filename), '--force'])
response = requests.put(server_url+'/studies/'+study_id+'/analysis/publish/'+analysis_id, headers={'Authorization':'Bearer '+access_token})
if response.status_code > 300:
raise Exception(response.text)
if not api.get_analysis(analysis_id).__dict__['analysisState'] == "PUBLISHED":
raise Exception("The analysis %s has not been published correctly." % analysis_id)
if results.json_output:
with open(os.path.join(results.input_dir,manifest_filename),'r') as f:
manifest_json = {}
manifest_json['analysis_id'] = f.readline().split('\t')[0]
manifest_json['files'] = []
for line in f.readlines():
_file = {}
_file['object_id'] = line.split('\t')[0]
_file['file_name'] = line.split('\t')[1]
_file['md5'] = line.split('\t')[2].strip('\n')
manifest_json['files'].append(_file)
with open(os.path.join(results.input_dir,results.json_output),'w') as outfile:
json.dump(manifest_json, outfile)
def main():
parser = argparse.ArgumentParser(description='Upload a payload using SONG')
parser.add_argument('-s', '--study-id', dest="study_id", help="Study ID", required=True)
parser.add_argument('-u', '--server-url', dest="server_url", help="Server URL", required=True)
parser.add_argument('-p', '--payload', dest="payload", help="JSON Payload", required=True)
parser.add_argument('-o', '--output', dest="output", help="Output manifest file", required=True)
parser.add_argument('-d', '--input-dir', dest="input_dir", help="Payload files directory", required=True)
parser.add_argument('-t', '--access-token', dest="access_token", default=os.environ.get('ACCESSTOKEN',None),help="Server URL")
parser.add_argument('-j','--json',dest="json_output")
results = parser.parse_args()
study_id = results.study_id
server_url = results.server_url
access_token = results.access_token
payload_file = results.payload
analysis_id = json.load(open(payload_file)).get('analysisId')
api_config = ApiConfig(server_url,study_id,access_token)
api = Api(api_config)
manifest_filename = results.output
create_manifest(api,"EGAZ00001257788",manifest_filename,results.input_dir)
def main():
api_config = ApiConfig("http://localhost:8080",
'ABC123',
"ad83ebde-a55c-11e7-abc4-cec278b6b50a",
debug=True)
upload(api_config)
def main():
parser = argparse.ArgumentParser(
description='Generate a song payload using minimal arguments')
parser.add_argument('-s',
'--study-id',
dest="study_id",
help="Study ID",
required=True)
parser.add_argument('-u',
'--server-url',
dest="server_url",
help="Server URL",
required=True)
parser.add_argument('-p',
'--payload',
dest="payload",
help="JSON Payload",
required=True)
parser.add_argument('-o',
'--output',
dest="output",
help="Output manifest file",
required=True)
parser.add_argument('-t',
'--access-token',
dest="access_token",
default=os.environ.get('ACCESSTOKEN', None),
help="Server URL")
results = parser.parse_args()
study_id = results.study_id
server_url = results.server_url
access_token = results.access_token
payload = results.payload
config = ApiConfig(server_url, study_id, access_token, debug=True)
api = Api(config)
study_client = StudyClient(api)
client = FileUploadClient(api,
payload,
is_async_validation=True,
ignore_analysis_id_collisions=True)
client.upload()
client.update_status()
api.save(client.upload_id, ignore_analysis_id_collisions=True)
client.save()
manifest_filename = results.output
manifest_client = ManifestClient(api)
manifest = create_manifest(api, client.analysis_id)
with open(manifest_filename, 'w') as fh:
fh.write(str(manifest))
subprocess.check_output([
'icgc-storage-client', 'upload', '--manifest',
os.path.join(os.getcwd(), manifest_filename), '--force'
])
requests.put(results.server_url + '/studies/' + results.study_id +
'/analysis/publish/' + client.analysis_id,
headers={
"Accept": "application/json",
"Content-Type": "application/json",
"Authorization": "Bearer " + results.access_token
})
#!/usr/bin/env python3
from overture_song.model import ApiConfig
from overture_song.client import Api
from overture_song.client import StudyClient
from overture_song.client import Study
from overture_song.entities import *
if __name__ == "__main__":
api_config = ApiConfig("http://84.88.186.194/song_eucancan_bsc", "ABC123",
"f69b726d-d40f-4261-b105-1ec7e6bf04d5")
api = Api(api_config)
## Check that we have access to the server
print("Access to the server: ", end="")
print(api.is_alive())
## Create the study
study_client = StudyClient(api)
if not study_client.has(api_config.study_id):
print("The study does not exists, creating it")
study = Study.create(api_config.study_id, "myStudyName",
"mySudyDescription", "myStudyOrganization")
#study_client(study)
else:
print("The study already exists")
## Create donor
donor = Donor()
donor.studyId = api_config.study_id
donor.donorGender = "male"
donor.donorSubmitterId = "dsId1"
def main(config, access_token, profile):
app_ctx = yaml.load(config)
ega_xml_info = app_ctx['ega_xml_info']
ega_xml = {}
with open(ega_xml_info, 'r', newline='') as f:
for l in f:
analysisId = l.split('\t')[0]
ega_xml[analysisId] = l.rstrip()
if os.path.isfile(app_ctx['log']):
os.remove(app_ctx['log'])
if os.path.isfile(app_ctx['xml_mismatch']):
os.remove(app_ctx['xml_mismatch'])
if not os.path.isdir(app_ctx['xml_fix_dir']):
os.mkdir(app_ctx['xml_fix_dir'])
song_file = app_ctx[profile]
with open(song_file, 'r', newline='') as f:
for l in f:
projectCode, objectId, analysisId, fileName, songMd5 = l.rstrip(
).split('\t')
# double check for safety reason
if profile == 'aws' and not projectCode in app_ctx['aws_approved']:
with open(app_ctx['log'], 'a') as f:
f.write(
'{0}::{1} object {2}: is not allowed in AWS\n'.format(
projectCode, analysisId, objectId))
continue
# only download if there is no local copy
fpath = os.path.join(app_ctx['xml_dir'], fileName)
if not os.path.isfile(fpath):
# download the xml from collab
subprocess.check_output([
'score-client', '--profile', profile, 'download',
'--object-id', objectId, '--validate', 'false', '--force',
'--output-dir', app_ctx['xml_dir']
])
# get file md5sum for the one downloaded by score client
scoreMd5 = get_md5(fpath)
scoreSize = os.path.getsize(fpath)
# handle the ega xml
if analysisId.startswith('EGA') and fileName.startswith('bundle'):
# generate the xml from ega jobs
if not ega_xml.get(analysisId):
click.echo(
'{}::{}: the ega transfer job is missing'.format(
projectCode, analysisId))
with open(app_ctx['log'], 'a') as f:
f.write(
'{0}::{1}: the ega transfer job is missing in the completed folder\n'
.format(projectCode, analysisId))
continue
if not os.path.isdir(app_ctx['ega_xml_dir']):
os.makedirs(app_ctx['ega_xml_dir'])
fpath = os.path.join(app_ctx['ega_xml_dir'], fileName)
if not os.path.isfile(fpath):
generate_metadata_xml(ega_xml[analysisId], app_ctx)
# get file md5sum for the one to upload
fileMd5 = get_md5(fpath)
fileSize = os.path.getsize(fpath)
if not fileMd5 == songMd5 or not fileMd5 == scoreMd5:
with open(app_ctx['xml_mismatch'], 'a') as f:
f.write('\t'.join(
[projectCode, analysisId, scoreMd5, songMd5, fileMd5]))
f.write('\n')
# skip the fix ones
fixpath = os.path.join(app_ctx['xml_fix_dir'], fileName + '.fix')
if os.path.isfile(fixpath): continue
# upload to storage
subprocess.check_output([
'score-client', '--profile', profile, 'upload', '--md5',
fileMd5, '--file', fpath, '--object-id', objectId, '--force'
])
# copy xml to open meta bucket
if profile == 'collab':
subprocess.check_output([
'aws', '--endpoint-url',
app_ctx[profile + '_endpoint_url'], '--profile', profile,
's3', 'cp', fpath, app_ctx['meta_bucket_url'] + objectId
])
else:
subprocess.check_output([
'aws', '--profile', 'amazon_pay', 's3', 'cp', fpath,
app_ctx['meta_bucket_url'] + objectId
])
# update the song
server_url = app_ctx['song'][profile]
api_config = ApiConfig(server_url, projectCode, access_token)
api = Api(api_config)
# check whether the song payload need to be updated
if not fileMd5 == songMd5:
# update the file
fileUpdate = FileUpdateRequest()
fileUpdate.fileSize = fileSize
fileUpdate.fileMd5sum = fileMd5
api.update_file(objectId, fileUpdate)
# publish the analysis
if not api.get_analysis(
analysisId).__dict__['analysisState'] == "PUBLISHED":
try:
api.publish(analysisId)
except:
with open(app_ctx['log'], 'a') as f:
f.write('{0}::{1}: can not be published\n'.format(
projectCode, analysisId))
with open(fixpath, 'w') as w:
w.write('')
return
def __build_api_config(self, study_id):
return ApiConfig(self.server_url, study_id, self.access_token, debug=self.debug)
def main():
parser = argparse.ArgumentParser(
description='Generate a song payload using minimal arguments')
parser.add_argument('-s',
'--study-id',
dest="study_id",
help="Study ID",
required=True)
parser.add_argument('-u',
'--server-url',
dest="server_url",
help="Server URL",
required=True)
parser.add_argument('-p',
'--payload',
dest="payload",
help="JSON Payload",
required=True)
parser.add_argument('-o',
'--output',
dest="output",
help="Output manifest file",
required=True)
parser.add_argument('-d',
'--input-dir',
dest="input_dir",
help="Payload files directory",
required=True)
parser.add_argument('-t',
'--access-token',
dest="access_token",
default=os.environ.get('ACCESSTOKEN', None),
help="Server URL")
parser.add_argument('-j', '--json', dest="json_output")
results = parser.parse_args()
study_id = results.study_id
server_url = results.server_url
access_token = results.access_token
payload = results.payload
analysis_id = json.load(open(payload)).get('analysisId')
config = ApiConfig(server_url, study_id, access_token, debug=True)
api = Api(config)
client = FileUploadClient(api,
payload,
is_async_validation=True,
ignore_analysis_id_collisions=True)
analysis = None
try:
analysis = api.get_analysis(analysis_id)
if analysis.analysisId is '':
raise Exception('Analysis id ' + analysis_id + ' not found')
payload_files = json.load(open(payload)).get('file')
api_files = api.get_analysis_files(analysis_id)
for i in range(0, len(api_files)):
if not exists_in_file_array(file_name=api_files[i].fileName,
file_md5=api_files[i].fileMd5sum,
file_type=api_files[i].fileType,
file_size=api_files[i].fileSize,
file_access=api_files[i].fileAccess,
files_array=payload_files):
print(
"Files in payload do not match the files on song server.")
exit(1)
except Exception:
if analysis is None or analysis is '':
client.upload()
client.update_status()
client.save()
manifest_filename = results.output
create_manifest(api, analysis_id, manifest_filename, results.input_dir)
subprocess.check_output([
'icgc-storage-client', 'upload', '--manifest',
os.path.join(results.input_dir, manifest_filename), '--force'
])
api.publish(analysis_id)
#client.publish()
if results.json_output:
with open(os.path.join(results.input_dir, manifest_filename),
'r') as f:
manifest_json = {}
manifest_json['analysis_id'] = f.readline().split('\t')[0]
manifest_json['files'] = []
for line in f.readlines():
_file = {}
_file['object_id'] = line.split('\t')[0]
_file['file_name'] = line.split('\t')[1]
_file['md5'] = line.split('\t')[2].strip('\n')
manifest_json['files'].append(_file)
with open(os.path.join(results.input_dir, results.json_output),
'w') as outfile:
json.dump(manifest_json, outfile)