def check_input(args): dxpy.set_security_context({ "auth_token_type": "Bearer", "auth_token": args.api_token}) # Check API token and project context try: dxpy.get_handler(args.project).describe() except dxpy.exceptions.DXAPIError as e: if e.name == "InvalidAuthentication": raise_error("API token (%s) is not valid. %s" % (args.api_token, e)) if e.name == "PermissionDenied": raise_error("Project (%s) is not valid. %s" % (args.project, e)) except dxpy.exceptions.DXError as e: raise_error("Error getting project handler for project (%s). %s" % (args.project, e)) # Check that chained downstream applet is valid if args.applet: try: dxpy.get_handler(args.applet).describe() except dxpy.exceptions.DXAPIError as e: raise_error("Unable to resolve applet %s. %s" %(args.applet, e)) except dxpy.exceptions.DXError as e: raise_error("Error getting handler for applet (%s). %s" %(args.applet, e)) # Check that chained downstream workflow is valid if args.workflow: try: dxpy.get_handler(args.workflow).describe() except dxpy.exceptions.DXAPIError as e: raise_error("Unable to resolve workflow %s. %s" %(args.workflow, e)) except dxpy.exceptions.DXError as e: raise_error("Error getting handler for workflow (%s). %s" %(args.workflow, e)) # Check that executable to launch locally is executable if args.script: if not (os.path.isfile(args.script) and os.access(args.script, os.X_OK)): raise_error("Executable/script passed by -s: (%s) is not executable" %(args.script)) if not args.dxpy_upload: print_stderr("Checking if ua is in $PATH") try: sub.check_call(['ua', '--version'], stdout=open(os.devnull, 'w'), close_fds=True) except sub.CalledProcessError: raise_error("Upload agent executable 'ua' was not found in the $PATH") try: # We assume that dx_sync_directory is located in the same folder as this script # This is resolved by absolute path of invocation sub.check_call(['python', '{curr_dir}/dx_sync_directory.py'.format(curr_dir=sys.path[0]), '-h'], stdout=open(os.devnull, 'w'), close_fds=True) except sub.CalledProcessError: raise_error("dx_sync_directory.py not found. Please run incremental " + "upload from the directory containing incremental_upload.py "+ "and dx_sync_directory.py")
def _wrap_dx_calls(): """Updates the dx_auth_token from settings for dxpy Bubbles all dxpy exceptions as `DNAnexusError` classes """ auth_token = settings.get()['dx']['auth_token'] if auth_token: # pragma: no cover dxpy.set_security_context({ 'auth_token_type': 'Bearer', 'auth_token': auth_token }) try: yield except DXError as e: six.raise_from(_dx_error_to_descriptive_exception(e), e)
def _sync_dxpy_state(self): dxpy.set_api_server_info(host=environ.get("DX_APISERVER_HOST", None), port=environ.get("DX_APISERVER_PORT", None), protocol=environ.get("DX_APISERVER_PROTOCOL", None)) if "DX_SECURITY_CONTEXT" in environ: dxpy.set_security_context(json.loads(environ["DX_SECURITY_CONTEXT"])) if "DX_JOB_ID" in environ: dxpy.set_job_id(environ["DX_JOB_ID"]) dxpy.set_workspace_id(environ.get("DX_WORKSPACE_ID")) else: dxpy.set_job_id(None) dxpy.set_workspace_id(environ.get("DX_PROJECT_CONTEXT_ID")) dxpy.set_project_context(environ.get("DX_PROJECT_CONTEXT_ID"))
def main(): # Parse CLI arguments. Some arguments will exit the program intentionally. See docstring for detail. args = cli_parser() # Setup logging for module. Submodules inherit log handlers and filters mokaguys_logger.log_setup(args.logfile) logger = logging.getLogger(__name__) logger.info(f'START') # Setup dxpy authentication token read from command line file. with open(args.auth) as f: auth_token = f.read().rstrip() dxpy.set_security_context({ 'auth_token_type': 'Bearer', 'auth_token': auth_token }) # Set root directory and search it for runfolders # If dry-run CLI flag is given, no directories are deleted by the runfolder manager. RFM = RunFolderManager(args.root, dry_run=args.dry_run) logger.info(f'Root directory {args.root}') logger.info(f'Identifying local runfolders to consider deleting') local_runfolders = RFM.find_runfolders(min_age=args.min_age) logger.debug( f'Found local runfolders to consider deleting: {[rf.name for rf in local_runfolders]}' ) for runfolder in local_runfolders: logger.info(f'Processing {runfolder.name}') # Delete runfolder if it meets the backup criteria # runfolder.dx_project is evaluated first as following criteria checks depend on it if runfolder.dx_project: fastqs_uploaded = RFM.check_fastqs(runfolder) logfiles_uploaded = RFM.check_logfiles(runfolder, args.logfile_count) if fastqs_uploaded and logfiles_uploaded: RFM.delete(runfolder) elif not fastqs_uploaded: logger.warning(f'{runfolder.name} - FASTQ MISMATCH') elif not logfiles_uploaded: logger.warning(f'{runfolder.name} - LOGFILE MISMATCH') else: logger.warning(f'{runfolder.name} - DX PROJECT MISMATCH') # Record runfolders removed by this iteration logger.info(f'Runfolders deleted in this instance: {RFM.deleted}') logger.info(f'END')
def main(): # set token to env for dx authentication DX_SECURITY_CONTEXT = { "auth_token_type": "Bearer", "auth_token": AUTH_TOKEN } dxpy.set_security_context(DX_SECURITY_CONTEXT) # get date to name as output dir for workflow date = get_date() # make output dir out_dir = make_dir(date) # run workflow run_check(out_dir)
def set_dx_authorization(authorization_header): """Set up authorization to DNAnexus. This appears to be done through a module-level global, so we can just do it once. Args: authorization_header (string): The bearer token passed in the header of a request to DNAnexus. Looks like "Bearer: abc123xzy" Returns: auth_header_json (string): Stringified JSON that can be used to set environment variables for authentication. """ dx_token = authorization_header.replace("Bearer ", "") auth_header = {"auth_token_type": "Bearer", "auth_token": dx_token} dxpy.set_security_context(auth_header) return json.dumps(auth_header)
def main(): dnanexus_token = os.environ['DNANEXUS_TOKEN'] # env variable for dx authentication dx_security_context = { "auth_token_type": "Bearer", "auth_token": dnanexus_token } # set token to env dx.set_security_context(dx_security_context) check_dx_login() projects = get_002_projects() project2jobs, project_no_run = get_jobs_per_project(projects) send_msg_using_hermes(project2jobs, project_no_run)
def main(token): # Configure dxpy authentication dxpy.set_security_context({'auth_token_type': 'Bearer', 'auth_token': token}) # Resolve FACTORY_PROJECT by ID proj = dxpy.DXProject(FACTORY_PROJECT) print 'Resolved project:', proj.describe()['name'], proj.get_id() # Set FACTORY_PROJECT as the workspace for subsequent operations # (sort of like the current working directory) dxpy.set_workspace_id(FACTORY_PROJECT) # Resolve the workflow by name. (Could also store ID like the project) wf = list(dxpy.search.find_data_objects(classname="workflow", name="RNA-seq pipeline", return_handler=True))[0] print 'Resolved workflow:', wf.describe()['name'], wf.get_id() # TODO: Stage the inputs. Here we find them in the IN folder left_reads = list(dxpy.search.find_data_objects(classname="file", name="ENCFF001JPX.1k.fastq.gz", folder="/IN", return_handler=True))[0] print 'Resolved left reads:', left_reads.describe()['name'], left_reads.get_id() right_reads = list(dxpy.search.find_data_objects(classname="file", name="ENCFF001JQB.1k.fastq.gz", folder="/IN", return_handler=True))[0] print 'Resolved right reads:', right_reads.describe()['name'], right_reads.get_id() # Launch the workflow analysis = wf.run({'0.fastqs': [dxpy.dxlink(left_reads.get_id())], '0.fastq_pairs': [dxpy.dxlink(right_reads.get_id())]}) print 'Launched analysis:', analysis.get_id() print 'Analysis state:', analysis.describe()['state'] # TODO: Poll for (or come back when) analysis state 'done' or 'failed'. # Handle any failures. # Cooking-show-style substitution with completed analysis analysis = dxpy.DXAnalysis(COMPLETED_ANALYSIS) print 'Analysis state:', analysis.describe()['state'] # Enumerate outputs print 'Analysis outputs:' for one_output_name, one_output_link in analysis.describe()['output'].iteritems(): one_output = dxpy.get_handler(one_output_link) # one_output : dxpy.DXFile one_file_name = one_output.describe()['name'] one_file_url, _ = one_output.get_download_url(preauthenticated=True, filename=one_file_name) print one_file_name, one_file_url
def __init__(self, run_name, lane_index, project_id, rta_version, lims_url, lims_token, dx_token, dashboard_project_id, release=False, test_mode=False, develop=False): self.run_name = run_name self.project_id = project_id self.lane_index = lane_index self.rta_version = rta_version self.lims_url = lims_url self.lims_token = lims_token self.dx_token = dx_token self.release = release self.test_mode = test_mode self.develop = develop # Workflow variables self.workflow_name = None self.workflow_id = None self.workflow_project_id = None self.workflow_json_file = None self.workflow_inputs = None self.workflow_object = None self.analysis_input = None self.record_id = None self.dashboard_project_id = dashboard_project_id self.record_properties = None self.metadata_tar_id = None self.interop_tar_id = None self.lane_tar_id = None self.viewer_emails = [] dxpy.set_security_context({"auth_token_type": "bearer", "auth_token": self.dx_token}) self.connection = Connection(lims_url=lims_url, lims_token=lims_token) self.run_info = RunInfo(conn=self.connection, run=run_name) print '\nRUN INFO\n' print self.run_info.data print '\n' self.lane_info = self.run_info.get_lane(self.lane_index) print '\nLANE INFO\n' print self.lane_info print '\n' dna_library_id = int(self.lane_info['dna_library_id']) self.dna_library_info = self.connection.getdnalibraryinfo(dna_library_id) print '\nLIBRARY INFO\n' print self.dna_library_info print '\n' # Bcl2fastq & demultiplexing variables self.barcode_mismatches = int(1) # Get sequencing queue & tag project tags = [] tags.append(str(self.lane_info['queue'])) if self.dna_library_info['project_id']: tags.append(str(self.dna_library_info['project_id'])) if self.develop: tags.append('dev') dxpy.api.project_add_tags(self.project_id, input_params={'tags':tags}) comments = str(self.dna_library_info['comments']) dxpy.DXProject(project_id).update(description=comments) # Mapping variables try: self.mapper = self.lane_info['mapping_requests'][0]['mapping_program'] if self.mapper == 'bwa': # Update March 1, 2016: Reverting back to bwa_aln since there is an issue reporting # unique vs non-unique reads with bwa-mem # Currently no option for users; only LIMS option we use is "bwa". # Defaulting to BWA_MEM # self.mapper = 'bwa_mem' self.mapper = 'bwa_aln' # Changed to bwa_aln since qc_sample only accepts 'bwa_mem' or 'bwa_aln' #self.barcode_mismatches = int(self.lane_info['mapping_requests'][0]['max_mismatches']) self.barcode_mismatches = int(1) self.reference_genome = self.lane_info['mapping_requests'][0]['reference_sequence_name'] # PBR, 160921: Hack to stop samples with "Other" listed as reference, from dying # Should probably remove as submission option; follow-up with SeqCore if self.reference_genome == 'Other': self.reference_genome = None else: self.mapper = None self.barcode_mismatches = int(1) self.reference_genome = None except: print 'Warning: No mapping information found for %s' % self.run_name self.mapper = None self.barcode_mismatches = int(1) self.reference_genome = None self.reference_genome_dxid = None self.reference_index_dxid = None if self.reference_genome: self.get_reference_ids() self.get_lane_input_files()
def check_input(args): dxpy.set_security_context({ "auth_token_type": "Bearer", "auth_token": args.api_token }) # Check API token and project context try: dxpy.get_handler(args.project).describe() except dxpy.exceptions.DXAPIError as e: if e.name == "InvalidAuthentication": raise_error("API token (%s) is not valid. %s" % (args.api_token, e)) if e.name == "PermissionDenied": raise_error("Project (%s) is not valid. %s" % (args.project, e)) except dxpy.exceptions.DXError as e: raise_error("Error getting project handler for project (%s). %s" % (args.project, e)) # Check that chained downstream applet is valid if args.applet: try: dxpy.get_handler(args.applet).describe() except dxpy.exceptions.DXAPIError as e: raise_error("Unable to resolve applet %s. %s" % (args.applet, e)) except dxpy.exceptions.DXError as e: raise_error("Error getting handler for applet (%s). %s" % (args.applet, e)) # Check that chained downstream workflow is valid if args.workflow: try: dxpy.get_handler(args.workflow).describe() except dxpy.exceptions.DXAPIError as e: raise_error("Unable to resolve workflow %s. %s" % (args.workflow, e)) except dxpy.exceptions.DXError as e: raise_error("Error getting handler for workflow (%s). %s" % (args.workflow, e)) # Check that executable to launch locally is executable if args.script: if not (os.path.isfile(args.script) and os.access(args.script, os.X_OK)): raise_error( "Executable/script passed by -s: (%s) is not executable" % (args.script)) if not args.dxpy_upload: print_stderr("Checking if ua is in $PATH") try: sub.check_call(['ua', '--version'], stdout=open(os.devnull, 'w'), close_fds=True) except sub.CalledProcessError: raise_error( "Upload agent executable 'ua' was not found in the $PATH") try: # We assume that dx_sync_directory is located in the same folder as this script # This is resolved by absolute path of invocation sub.check_call([ 'python3', '{curr_dir}/dx_sync_directory.py'.format( curr_dir=sys.path[0]), '-h' ], stdout=open(os.devnull, 'w'), close_fds=True) except sub.CalledProcessError: raise_error( "dx_sync_directory.py not found. Please run incremental " + "upload from the directory containing incremental_upload.py " + "and dx_sync_directory.py")
def main(token): # Configure dxpy authentication dxpy.set_security_context({ 'auth_token_type': 'Bearer', 'auth_token': token }) # Resolve FACTORY_PROJECT by ID proj = dxpy.DXProject(FACTORY_PROJECT) print 'Resolved project:', proj.describe()['name'], proj.get_id() # Set FACTORY_PROJECT as the workspace for subsequent operations # (sort of like the current working directory) dxpy.set_workspace_id(FACTORY_PROJECT) # Resolve the workflow by name. (Could also store ID like the project) wf = list( dxpy.search.find_data_objects(classname="workflow", name="RNA-seq pipeline", return_handler=True))[0] print 'Resolved workflow:', wf.describe()['name'], wf.get_id() # TODO: Stage the inputs. Here we find them in the IN folder left_reads = list( dxpy.search.find_data_objects(classname="file", name="ENCFF001JPX.1k.fastq.gz", folder="/IN", return_handler=True))[0] print 'Resolved left reads:', left_reads.describe( )['name'], left_reads.get_id() right_reads = list( dxpy.search.find_data_objects(classname="file", name="ENCFF001JQB.1k.fastq.gz", folder="/IN", return_handler=True))[0] print 'Resolved right reads:', right_reads.describe( )['name'], right_reads.get_id() # Launch the workflow analysis = wf.run({ '0.fastqs': [dxpy.dxlink(left_reads.get_id())], '0.fastq_pairs': [dxpy.dxlink(right_reads.get_id())] }) print 'Launched analysis:', analysis.get_id() print 'Analysis state:', analysis.describe()['state'] # TODO: Poll for (or come back when) analysis state 'done' or 'failed'. # Handle any failures. # Cooking-show-style substitution with completed analysis analysis = dxpy.DXAnalysis(COMPLETED_ANALYSIS) print 'Analysis state:', analysis.describe()['state'] # Enumerate outputs print 'Analysis outputs:' for one_output_name, one_output_link in analysis.describe( )['output'].iteritems(): one_output = dxpy.get_handler( one_output_link) # one_output : dxpy.DXFile one_file_name = one_output.describe()['name'] one_file_url, _ = one_output.get_download_url(preauthenticated=True, filename=one_file_name) print one_file_name, one_file_url
def fileno(self): return 0 if __name__ == "__main__": upload_path="." dx_path="" remote_path="/" curr_proj = "project-Bz06Zv80pjKvypFvX4kqp1Vk" API_token={"auth_token_type" : "Bearer", "auth_token" : "Y1WoDbANKOKDF0ZfPV0wt9cQaIotIOWC"} # try to find the remote path # by default use "/From_<hostname>" remote_path = "/From_" + socket.gethostname() # create a random named file if len(sys.argv) > 1: given_rpath=sys.argv[1] if given_rpath.startswith("/"): remote_path=given_rpath else: remote_path = remote_path + "/" + given_rpath # set the security context dxpy.set_security_context(API_token) # upload to infinity and beyond! f = randf() dxpy.bindings.dxfile_functions.upload_local_file(file=f, folder=remote_path, parents=True, project=curr_proj)