def main(): parser = argparse.ArgumentParser(description='Upload a payload using SONG') parser.add_argument('-s', '--study-id', dest="study_id", help="Study ID", required=True) parser.add_argument('-u', '--server-url', dest="server_url", help="Server URL", required=True) parser.add_argument('-p', '--payload', dest="payload", help="JSON Payload", required=True) parser.add_argument('-o', '--output', dest="output", help="Output manifest file", required=True) parser.add_argument('-d', '--input-dir', dest="input_dir", help="Payload files directory", required=True) parser.add_argument('-t', '--access-token', dest="access_token", default=os.environ.get('ACCESSTOKEN',None),help="Server URL") parser.add_argument('-j','--json',dest="json_output") results = parser.parse_args() study_id = results.study_id server_url = results.server_url access_token = results.access_token payload_file = results.payload analysis_id = json.load(open(payload_file)).get('analysisId') api_config = ApiConfig(server_url,study_id,access_token) api = Api(api_config) upload_status = upload_payload(api,payload_file) try: api.save(upload_status.uploadId, ignore_analysis_id_collisions=True) except Exception as e: song_error = json.loads(str(e)) if song_error.get('errorId') != "duplicate.analysis.attempt": raise e if api.get_analysis(analysis_id).__dict__['analysisState'] == "PUBLISHED": validate_payload_against_analysis(api, analysis_id, payload_file) manifest_filename = results.output create_manifest(api,analysis_id,payload_file,manifest_filename,results.input_dir) if not api.get_analysis(analysis_id).__dict__['analysisState'] == "PUBLISHED": subprocess.check_output(['icgc-storage-client','upload','--manifest',os.path.join(results.input_dir,manifest_filename), '--force']) response = requests.put(server_url+'/studies/'+study_id+'/analysis/publish/'+analysis_id, headers={'Authorization':'Bearer '+access_token}) if response.status_code > 300: raise Exception(response.text) if not api.get_analysis(analysis_id).__dict__['analysisState'] == "PUBLISHED": raise Exception("The analysis %s has not been published correctly." % analysis_id) if results.json_output: with open(os.path.join(results.input_dir,manifest_filename),'r') as f: manifest_json = {} manifest_json['analysis_id'] = f.readline().split('\t')[0] manifest_json['files'] = [] for line in f.readlines(): _file = {} _file['object_id'] = line.split('\t')[0] _file['file_name'] = line.split('\t')[1] _file['md5'] = line.split('\t')[2].strip('\n') manifest_json['files'].append(_file) with open(os.path.join(results.input_dir,results.json_output),'w') as outfile: json.dump(manifest_json, outfile)
def main(): parser = argparse.ArgumentParser(description='Upload a payload using SONG') parser.add_argument('-s', '--study-id', dest="study_id", help="Study ID", required=True) parser.add_argument('-u', '--server-url', dest="server_url", help="Server URL", required=True) parser.add_argument('-p', '--payload', dest="payload", help="JSON Payload", required=True) parser.add_argument('-o', '--output', dest="output", help="Output manifest file", required=True) parser.add_argument('-d', '--input-dir', dest="input_dir", help="Payload files directory", required=True) parser.add_argument('-t', '--access-token', dest="access_token", default=os.environ.get('ACCESSTOKEN',None),help="Server URL") parser.add_argument('-j','--json',dest="json_output") results = parser.parse_args() study_id = results.study_id server_url = results.server_url access_token = results.access_token payload_file = results.payload analysis_id = json.load(open(payload_file)).get('analysisId') api_config = ApiConfig(server_url,study_id,access_token) api = Api(api_config) manifest_filename = results.output create_manifest(api,"EGAZ00001257788",manifest_filename,results.input_dir)
def upload(config): api = Api(config) check_state(api.is_alive(), "The server '{}' is not running", config) study_client = StudyClient(api) if not study_client.has(config.study_id): study_client.create(Study.create(config.study_id)) donor = Donor() donor.studyId = config.study_id donor.donorGender = "male" donor.donorSubmitterId = "dsId1" donor.set_info("randomDonorField", "someDonorValue") specimen = Specimen() specimen.specimenClass = "Tumour" specimen.specimenSubmitterId = "sp_sub_1" specimen.specimenType = "Normal - EBV immortalized" specimen.set_info("randomSpecimenField", "someSpecimenValue") sample = Sample() sample.sampleSubmitterId = "ssId1" sample.sampleType = "RNA" sample.set_info("randomSample1Field", "someSample1Value") # File 1 file1 = File() file1.fileName = "example.bam" file1.studyId = config.study_id file1.fileAccess = "controlled" file1.fileMd5sum = "73a90acaae2b1ccc0e969709665bc62f" file1.fileSize = 1234561 file1.fileType = "BAM" file1.set_info("randomFile1Field", "someFile1Value") # File 2 file2 = File() file2.fileName = "example.bam.bai" file2.studyId = config.study_id file2.fileAccess = "controlled" file2.fileMd5sum = "67a97437c474cc1d95e0a1672b291b70" file2.fileSize = 1234562 file2.fileType = "BAI" file2.set_info("randomFile2Field", "someFile2Value") # SequencingRead sequencing_read_experiment = SequencingRead() sequencing_read_experiment.aligned = True sequencing_read_experiment.alignmentTool = "myAlignmentTool" sequencing_read_experiment.pairedEnd = True sequencing_read_experiment.insertSize = 0 sequencing_read_experiment.libraryStrategy = "WXS" sequencing_read_experiment.referenceGenome = "GR37" sequencing_read_experiment.set_info("randomSRField", "someSRValue") builder = SimplePayloadBuilder(donor, specimen, sample, [file1, file2], sequencing_read_experiment) print("-----------------------Uploading payload-------------------------") upload_response = api.upload(json_payload=builder.to_dict(), is_async_validation=False) print("upload_response: \n{}".format(upload_response)) sleep(4) print("\n\n\n-------------------Check status----------------------------") status_response = api.status(upload_response.uploadId) print("status_response: \n{}".format(status_response)) sleep(4) print("\n\n\n----------------------Saving-------------------------------") save_response = api.save(status_response.uploadId, ignore_analysis_id_collisions=True) print("save_response: \n{}".format(save_response)) sleep(4) source_dir = "./" manifest_client = ManifestClient(api) print("\n\n\n-----------------Creating manifest-------------------------") manifest = manifest_client.create_manifest(source_dir, save_response.analysisId) print("manifest_file_contents: \n{}".format(manifest))
def main(): parser = argparse.ArgumentParser( description='Generate a song payload using minimal arguments') parser.add_argument('-s', '--study-id', dest="study_id", help="Study ID", required=True) parser.add_argument('-u', '--server-url', dest="server_url", help="Server URL", required=True) parser.add_argument('-p', '--payload', dest="payload", help="JSON Payload", required=True) parser.add_argument('-o', '--output', dest="output", help="Output manifest file", required=True) parser.add_argument('-t', '--access-token', dest="access_token", default=os.environ.get('ACCESSTOKEN', None), help="Server URL") results = parser.parse_args() study_id = results.study_id server_url = results.server_url access_token = results.access_token payload = results.payload config = ApiConfig(server_url, study_id, access_token, debug=True) api = Api(config) study_client = StudyClient(api) client = FileUploadClient(api, payload, is_async_validation=True, ignore_analysis_id_collisions=True) client.upload() client.update_status() api.save(client.upload_id, ignore_analysis_id_collisions=True) client.save() manifest_filename = results.output manifest_client = ManifestClient(api) manifest = create_manifest(api, client.analysis_id) with open(manifest_filename, 'w') as fh: fh.write(str(manifest)) subprocess.check_output([ 'icgc-storage-client', 'upload', '--manifest', os.path.join(os.getcwd(), manifest_filename), '--force' ]) requests.put(results.server_url + '/studies/' + results.study_id + '/analysis/publish/' + client.analysis_id, headers={ "Accept": "application/json", "Content-Type": "application/json", "Authorization": "Bearer " + results.access_token })
#!/usr/bin/env python3 from overture_song.model import ApiConfig from overture_song.client import Api from overture_song.client import StudyClient from overture_song.client import Study from overture_song.entities import * if __name__ == "__main__": api_config = ApiConfig("http://84.88.186.194/song_eucancan_bsc", "ABC123", "f69b726d-d40f-4261-b105-1ec7e6bf04d5") api = Api(api_config) ## Check that we have access to the server print("Access to the server: ", end="") print(api.is_alive()) ## Create the study study_client = StudyClient(api) if not study_client.has(api_config.study_id): print("The study does not exists, creating it") study = Study.create(api_config.study_id, "myStudyName", "mySudyDescription", "myStudyOrganization") #study_client(study) else: print("The study already exists") ## Create donor donor = Donor() donor.studyId = api_config.study_id donor.donorGender = "male" donor.donorSubmitterId = "dsId1"
def main(config, access_token, profile): app_ctx = yaml.load(config) ega_xml_info = app_ctx['ega_xml_info'] ega_xml = {} with open(ega_xml_info, 'r', newline='') as f: for l in f: analysisId = l.split('\t')[0] ega_xml[analysisId] = l.rstrip() if os.path.isfile(app_ctx['log']): os.remove(app_ctx['log']) if os.path.isfile(app_ctx['xml_mismatch']): os.remove(app_ctx['xml_mismatch']) if not os.path.isdir(app_ctx['xml_fix_dir']): os.mkdir(app_ctx['xml_fix_dir']) song_file = app_ctx[profile] with open(song_file, 'r', newline='') as f: for l in f: projectCode, objectId, analysisId, fileName, songMd5 = l.rstrip( ).split('\t') # double check for safety reason if profile == 'aws' and not projectCode in app_ctx['aws_approved']: with open(app_ctx['log'], 'a') as f: f.write( '{0}::{1} object {2}: is not allowed in AWS\n'.format( projectCode, analysisId, objectId)) continue # only download if there is no local copy fpath = os.path.join(app_ctx['xml_dir'], fileName) if not os.path.isfile(fpath): # download the xml from collab subprocess.check_output([ 'score-client', '--profile', profile, 'download', '--object-id', objectId, '--validate', 'false', '--force', '--output-dir', app_ctx['xml_dir'] ]) # get file md5sum for the one downloaded by score client scoreMd5 = get_md5(fpath) scoreSize = os.path.getsize(fpath) # handle the ega xml if analysisId.startswith('EGA') and fileName.startswith('bundle'): # generate the xml from ega jobs if not ega_xml.get(analysisId): click.echo( '{}::{}: the ega transfer job is missing'.format( projectCode, analysisId)) with open(app_ctx['log'], 'a') as f: f.write( '{0}::{1}: the ega transfer job is missing in the completed folder\n' .format(projectCode, analysisId)) continue if not os.path.isdir(app_ctx['ega_xml_dir']): os.makedirs(app_ctx['ega_xml_dir']) fpath = os.path.join(app_ctx['ega_xml_dir'], fileName) if not os.path.isfile(fpath): generate_metadata_xml(ega_xml[analysisId], app_ctx) # get file md5sum for the one to upload fileMd5 = get_md5(fpath) fileSize = os.path.getsize(fpath) if not fileMd5 == songMd5 or not fileMd5 == scoreMd5: with open(app_ctx['xml_mismatch'], 'a') as f: f.write('\t'.join( [projectCode, analysisId, scoreMd5, songMd5, fileMd5])) f.write('\n') # skip the fix ones fixpath = os.path.join(app_ctx['xml_fix_dir'], fileName + '.fix') if os.path.isfile(fixpath): continue # upload to storage subprocess.check_output([ 'score-client', '--profile', profile, 'upload', '--md5', fileMd5, '--file', fpath, '--object-id', objectId, '--force' ]) # copy xml to open meta bucket if profile == 'collab': subprocess.check_output([ 'aws', '--endpoint-url', app_ctx[profile + '_endpoint_url'], '--profile', profile, 's3', 'cp', fpath, app_ctx['meta_bucket_url'] + objectId ]) else: subprocess.check_output([ 'aws', '--profile', 'amazon_pay', 's3', 'cp', fpath, app_ctx['meta_bucket_url'] + objectId ]) # update the song server_url = app_ctx['song'][profile] api_config = ApiConfig(server_url, projectCode, access_token) api = Api(api_config) # check whether the song payload need to be updated if not fileMd5 == songMd5: # update the file fileUpdate = FileUpdateRequest() fileUpdate.fileSize = fileSize fileUpdate.fileMd5sum = fileMd5 api.update_file(objectId, fileUpdate) # publish the analysis if not api.get_analysis( analysisId).__dict__['analysisState'] == "PUBLISHED": try: api.publish(analysisId) except: with open(app_ctx['log'], 'a') as f: f.write('{0}::{1}: can not be published\n'.format( projectCode, analysisId)) with open(fixpath, 'w') as w: w.write('') return
def __build_api(self, study_id): return Api(self.__build_api_config(study_id))
def main(): parser = argparse.ArgumentParser( description='Generate a song payload using minimal arguments') parser.add_argument('-s', '--study-id', dest="study_id", help="Study ID", required=True) parser.add_argument('-u', '--server-url', dest="server_url", help="Server URL", required=True) parser.add_argument('-p', '--payload', dest="payload", help="JSON Payload", required=True) parser.add_argument('-o', '--output', dest="output", help="Output manifest file", required=True) parser.add_argument('-d', '--input-dir', dest="input_dir", help="Payload files directory", required=True) parser.add_argument('-t', '--access-token', dest="access_token", default=os.environ.get('ACCESSTOKEN', None), help="Server URL") parser.add_argument('-j', '--json', dest="json_output") results = parser.parse_args() study_id = results.study_id server_url = results.server_url access_token = results.access_token payload = results.payload analysis_id = json.load(open(payload)).get('analysisId') config = ApiConfig(server_url, study_id, access_token, debug=True) api = Api(config) client = FileUploadClient(api, payload, is_async_validation=True, ignore_analysis_id_collisions=True) analysis = None try: analysis = api.get_analysis(analysis_id) if analysis.analysisId is '': raise Exception('Analysis id ' + analysis_id + ' not found') payload_files = json.load(open(payload)).get('file') api_files = api.get_analysis_files(analysis_id) for i in range(0, len(api_files)): if not exists_in_file_array(file_name=api_files[i].fileName, file_md5=api_files[i].fileMd5sum, file_type=api_files[i].fileType, file_size=api_files[i].fileSize, file_access=api_files[i].fileAccess, files_array=payload_files): print( "Files in payload do not match the files on song server.") exit(1) except Exception: if analysis is None or analysis is '': client.upload() client.update_status() client.save() manifest_filename = results.output create_manifest(api, analysis_id, manifest_filename, results.input_dir) subprocess.check_output([ 'icgc-storage-client', 'upload', '--manifest', os.path.join(results.input_dir, manifest_filename), '--force' ]) api.publish(analysis_id) #client.publish() if results.json_output: with open(os.path.join(results.input_dir, manifest_filename), 'r') as f: manifest_json = {} manifest_json['analysis_id'] = f.readline().split('\t')[0] manifest_json['files'] = [] for line in f.readlines(): _file = {} _file['object_id'] = line.split('\t')[0] _file['file_name'] = line.split('\t')[1] _file['md5'] = line.split('\t')[2].strip('\n') manifest_json['files'].append(_file) with open(os.path.join(results.input_dir, results.json_output), 'w') as outfile: json.dump(manifest_json, outfile)