def check_input(args):
dxpy.set_security_context({
"auth_token_type": "Bearer",
"auth_token": args.api_token})
# Check API token and project context
try:
dxpy.get_handler(args.project).describe()
except dxpy.exceptions.DXAPIError as e:
if e.name == "InvalidAuthentication":
raise_error("API token (%s) is not valid. %s"
% (args.api_token, e))
if e.name == "PermissionDenied":
raise_error("Project (%s) is not valid. %s"
% (args.project, e))
except dxpy.exceptions.DXError as e:
raise_error("Error getting project handler for project (%s). %s" %
(args.project, e))
# Check that chained downstream applet is valid
if args.applet:
try:
dxpy.get_handler(args.applet).describe()
except dxpy.exceptions.DXAPIError as e:
raise_error("Unable to resolve applet %s. %s" %(args.applet, e))
except dxpy.exceptions.DXError as e:
raise_error("Error getting handler for applet (%s). %s" %(args.applet, e))
# Check that chained downstream workflow is valid
if args.workflow:
try:
dxpy.get_handler(args.workflow).describe()
except dxpy.exceptions.DXAPIError as e:
raise_error("Unable to resolve workflow %s. %s" %(args.workflow, e))
except dxpy.exceptions.DXError as e:
raise_error("Error getting handler for workflow (%s). %s" %(args.workflow, e))
# Check that executable to launch locally is executable
if args.script:
if not (os.path.isfile(args.script) and os.access(args.script, os.X_OK)):
raise_error("Executable/script passed by -s: (%s) is not executable" %(args.script))
if not args.dxpy_upload:
print_stderr("Checking if ua is in $PATH")
try:
sub.check_call(['ua', '--version'],
stdout=open(os.devnull, 'w'), close_fds=True)
except sub.CalledProcessError:
raise_error("Upload agent executable 'ua' was not found in the $PATH")
try:
# We assume that dx_sync_directory is located in the same folder as this script
# This is resolved by absolute path of invocation
sub.check_call(['python', '{curr_dir}/dx_sync_directory.py'.format(curr_dir=sys.path[0]), '-h'],
stdout=open(os.devnull, 'w'), close_fds=True)
except sub.CalledProcessError:
raise_error("dx_sync_directory.py not found. Please run incremental " +
"upload from the directory containing incremental_upload.py "+
"and dx_sync_directory.py")
def _wrap_dx_calls():
"""Updates the dx_auth_token from settings for dxpy
Bubbles all dxpy exceptions as `DNAnexusError` classes
"""
auth_token = settings.get()['dx']['auth_token']
if auth_token: # pragma: no cover
dxpy.set_security_context({
'auth_token_type': 'Bearer',
'auth_token': auth_token
})
try:
yield
except DXError as e:
six.raise_from(_dx_error_to_descriptive_exception(e), e)
def _sync_dxpy_state(self):
dxpy.set_api_server_info(host=environ.get("DX_APISERVER_HOST", None),
port=environ.get("DX_APISERVER_PORT", None),
protocol=environ.get("DX_APISERVER_PROTOCOL", None))
if "DX_SECURITY_CONTEXT" in environ:
dxpy.set_security_context(json.loads(environ["DX_SECURITY_CONTEXT"]))
if "DX_JOB_ID" in environ:
dxpy.set_job_id(environ["DX_JOB_ID"])
dxpy.set_workspace_id(environ.get("DX_WORKSPACE_ID"))
else:
dxpy.set_job_id(None)
dxpy.set_workspace_id(environ.get("DX_PROJECT_CONTEXT_ID"))
dxpy.set_project_context(environ.get("DX_PROJECT_CONTEXT_ID"))
def _sync_dxpy_state(self):
dxpy.set_api_server_info(host=environ.get("DX_APISERVER_HOST", None),
port=environ.get("DX_APISERVER_PORT", None),
protocol=environ.get("DX_APISERVER_PROTOCOL", None))
if "DX_SECURITY_CONTEXT" in environ:
dxpy.set_security_context(json.loads(environ["DX_SECURITY_CONTEXT"]))
if "DX_JOB_ID" in environ:
dxpy.set_job_id(environ["DX_JOB_ID"])
dxpy.set_workspace_id(environ.get("DX_WORKSPACE_ID"))
else:
dxpy.set_job_id(None)
dxpy.set_workspace_id(environ.get("DX_PROJECT_CONTEXT_ID"))
dxpy.set_project_context(environ.get("DX_PROJECT_CONTEXT_ID"))
def main():
# Parse CLI arguments. Some arguments will exit the program intentionally. See docstring for detail.
args = cli_parser()
# Setup logging for module. Submodules inherit log handlers and filters
mokaguys_logger.log_setup(args.logfile)
logger = logging.getLogger(__name__)
logger.info(f'START')
# Setup dxpy authentication token read from command line file.
with open(args.auth) as f:
auth_token = f.read().rstrip()
dxpy.set_security_context({
'auth_token_type': 'Bearer',
'auth_token': auth_token
})
# Set root directory and search it for runfolders
# If dry-run CLI flag is given, no directories are deleted by the runfolder manager.
RFM = RunFolderManager(args.root, dry_run=args.dry_run)
logger.info(f'Root directory {args.root}')
logger.info(f'Identifying local runfolders to consider deleting')
local_runfolders = RFM.find_runfolders(min_age=args.min_age)
logger.debug(
f'Found local runfolders to consider deleting: {[rf.name for rf in local_runfolders]}'
)
for runfolder in local_runfolders:
logger.info(f'Processing {runfolder.name}')
# Delete runfolder if it meets the backup criteria
# runfolder.dx_project is evaluated first as following criteria checks depend on it
if runfolder.dx_project:
fastqs_uploaded = RFM.check_fastqs(runfolder)
logfiles_uploaded = RFM.check_logfiles(runfolder,
args.logfile_count)
if fastqs_uploaded and logfiles_uploaded:
RFM.delete(runfolder)
elif not fastqs_uploaded:
logger.warning(f'{runfolder.name} - FASTQ MISMATCH')
elif not logfiles_uploaded:
logger.warning(f'{runfolder.name} - LOGFILE MISMATCH')
else:
logger.warning(f'{runfolder.name} - DX PROJECT MISMATCH')
# Record runfolders removed by this iteration
logger.info(f'Runfolders deleted in this instance: {RFM.deleted}')
logger.info(f'END')
def main():
# set token to env for dx authentication
DX_SECURITY_CONTEXT = {
"auth_token_type": "Bearer",
"auth_token": AUTH_TOKEN
}
dxpy.set_security_context(DX_SECURITY_CONTEXT)
# get date to name as output dir for workflow
date = get_date()
# make output dir
out_dir = make_dir(date)
# run workflow
run_check(out_dir)
def set_dx_authorization(authorization_header):
"""Set up authorization to DNAnexus. This appears to be done through a
module-level global, so we can just do it once.
Args:
authorization_header (string): The bearer token passed in the header of
a request to DNAnexus. Looks like "Bearer: abc123xzy"
Returns:
auth_header_json (string): Stringified JSON that can be used to set
environment variables for authentication.
"""
dx_token = authorization_header.replace("Bearer ", "")
auth_header = {"auth_token_type": "Bearer", "auth_token": dx_token}
dxpy.set_security_context(auth_header)
return json.dumps(auth_header)
def main():
dnanexus_token = os.environ['DNANEXUS_TOKEN']
# env variable for dx authentication
dx_security_context = {
"auth_token_type": "Bearer",
"auth_token": dnanexus_token
}
# set token to env
dx.set_security_context(dx_security_context)
check_dx_login()
projects = get_002_projects()
project2jobs, project_no_run = get_jobs_per_project(projects)
send_msg_using_hermes(project2jobs, project_no_run)
def main(token):
# Configure dxpy authentication
dxpy.set_security_context({'auth_token_type': 'Bearer', 'auth_token': token})
# Resolve FACTORY_PROJECT by ID
proj = dxpy.DXProject(FACTORY_PROJECT)
print 'Resolved project:', proj.describe()['name'], proj.get_id()
# Set FACTORY_PROJECT as the workspace for subsequent operations
# (sort of like the current working directory)
dxpy.set_workspace_id(FACTORY_PROJECT)
# Resolve the workflow by name. (Could also store ID like the project)
wf = list(dxpy.search.find_data_objects(classname="workflow", name="RNA-seq pipeline",
return_handler=True))[0]
print 'Resolved workflow:', wf.describe()['name'], wf.get_id()
# TODO: Stage the inputs. Here we find them in the IN folder
left_reads = list(dxpy.search.find_data_objects(classname="file", name="ENCFF001JPX.1k.fastq.gz",
folder="/IN", return_handler=True))[0]
print 'Resolved left reads:', left_reads.describe()['name'], left_reads.get_id()
right_reads = list(dxpy.search.find_data_objects(classname="file", name="ENCFF001JQB.1k.fastq.gz",
folder="/IN", return_handler=True))[0]
print 'Resolved right reads:', right_reads.describe()['name'], right_reads.get_id()
# Launch the workflow
analysis = wf.run({'0.fastqs': [dxpy.dxlink(left_reads.get_id())],
'0.fastq_pairs': [dxpy.dxlink(right_reads.get_id())]})
print 'Launched analysis:', analysis.get_id()
print 'Analysis state:', analysis.describe()['state']
# TODO: Poll for (or come back when) analysis state 'done' or 'failed'.
# Handle any failures.
# Cooking-show-style substitution with completed analysis
analysis = dxpy.DXAnalysis(COMPLETED_ANALYSIS)
print 'Analysis state:', analysis.describe()['state']
# Enumerate outputs
print 'Analysis outputs:'
for one_output_name, one_output_link in analysis.describe()['output'].iteritems():
one_output = dxpy.get_handler(one_output_link) # one_output : dxpy.DXFile
one_file_name = one_output.describe()['name']
one_file_url, _ = one_output.get_download_url(preauthenticated=True, filename=one_file_name)
print one_file_name, one_file_url
def __init__(self, run_name, lane_index, project_id, rta_version, lims_url, lims_token,
dx_token, dashboard_project_id, release=False, test_mode=False, develop=False):
self.run_name = run_name
self.project_id = project_id
self.lane_index = lane_index
self.rta_version = rta_version
self.lims_url = lims_url
self.lims_token = lims_token
self.dx_token = dx_token
self.release = release
self.test_mode = test_mode
self.develop = develop
# Workflow variables
self.workflow_name = None
self.workflow_id = None
self.workflow_project_id = None
self.workflow_json_file = None
self.workflow_inputs = None
self.workflow_object = None
self.analysis_input = None
self.record_id = None
self.dashboard_project_id = dashboard_project_id
self.record_properties = None
self.metadata_tar_id = None
self.interop_tar_id = None
self.lane_tar_id = None
self.viewer_emails = []
dxpy.set_security_context({"auth_token_type": "bearer", "auth_token": self.dx_token})
self.connection = Connection(lims_url=lims_url, lims_token=lims_token)
self.run_info = RunInfo(conn=self.connection, run=run_name)
print '\nRUN INFO\n'
print self.run_info.data
print '\n'
self.lane_info = self.run_info.get_lane(self.lane_index)
print '\nLANE INFO\n'
print self.lane_info
print '\n'
dna_library_id = int(self.lane_info['dna_library_id'])
self.dna_library_info = self.connection.getdnalibraryinfo(dna_library_id)
print '\nLIBRARY INFO\n'
print self.dna_library_info
print '\n'
# Bcl2fastq & demultiplexing variables
self.barcode_mismatches = int(1)
# Get sequencing queue & tag project
tags = []
tags.append(str(self.lane_info['queue']))
if self.dna_library_info['project_id']:
tags.append(str(self.dna_library_info['project_id']))
if self.develop:
tags.append('dev')
dxpy.api.project_add_tags(self.project_id, input_params={'tags':tags})
comments = str(self.dna_library_info['comments'])
dxpy.DXProject(project_id).update(description=comments)
# Mapping variables
try:
self.mapper = self.lane_info['mapping_requests'][0]['mapping_program']
if self.mapper == 'bwa':
# Update March 1, 2016: Reverting back to bwa_aln since there is an issue reporting
# unique vs non-unique reads with bwa-mem
# Currently no option for users; only LIMS option we use is "bwa".
# Defaulting to BWA_MEM
# self.mapper = 'bwa_mem'
self.mapper = 'bwa_aln' # Changed to bwa_aln since qc_sample only accepts 'bwa_mem' or 'bwa_aln'
#self.barcode_mismatches = int(self.lane_info['mapping_requests'][0]['max_mismatches'])
self.barcode_mismatches = int(1)
self.reference_genome = self.lane_info['mapping_requests'][0]['reference_sequence_name']
# PBR, 160921: Hack to stop samples with "Other" listed as reference, from dying
# Should probably remove as submission option; follow-up with SeqCore
if self.reference_genome == 'Other':
self.reference_genome = None
else:
self.mapper = None
self.barcode_mismatches = int(1)
self.reference_genome = None
except:
print 'Warning: No mapping information found for %s' % self.run_name
self.mapper = None
self.barcode_mismatches = int(1)
self.reference_genome = None
self.reference_genome_dxid = None
self.reference_index_dxid = None
if self.reference_genome:
self.get_reference_ids()
self.get_lane_input_files()
def check_input(args):
dxpy.set_security_context({
"auth_token_type": "Bearer",
"auth_token": args.api_token
})
# Check API token and project context
try:
dxpy.get_handler(args.project).describe()
except dxpy.exceptions.DXAPIError as e:
if e.name == "InvalidAuthentication":
raise_error("API token (%s) is not valid. %s" %
(args.api_token, e))
if e.name == "PermissionDenied":
raise_error("Project (%s) is not valid. %s" % (args.project, e))
except dxpy.exceptions.DXError as e:
raise_error("Error getting project handler for project (%s). %s" %
(args.project, e))
# Check that chained downstream applet is valid
if args.applet:
try:
dxpy.get_handler(args.applet).describe()
except dxpy.exceptions.DXAPIError as e:
raise_error("Unable to resolve applet %s. %s" % (args.applet, e))
except dxpy.exceptions.DXError as e:
raise_error("Error getting handler for applet (%s). %s" %
(args.applet, e))
# Check that chained downstream workflow is valid
if args.workflow:
try:
dxpy.get_handler(args.workflow).describe()
except dxpy.exceptions.DXAPIError as e:
raise_error("Unable to resolve workflow %s. %s" %
(args.workflow, e))
except dxpy.exceptions.DXError as e:
raise_error("Error getting handler for workflow (%s). %s" %
(args.workflow, e))
# Check that executable to launch locally is executable
if args.script:
if not (os.path.isfile(args.script)
and os.access(args.script, os.X_OK)):
raise_error(
"Executable/script passed by -s: (%s) is not executable" %
(args.script))
if not args.dxpy_upload:
print_stderr("Checking if ua is in $PATH")
try:
sub.check_call(['ua', '--version'],
stdout=open(os.devnull, 'w'),
close_fds=True)
except sub.CalledProcessError:
raise_error(
"Upload agent executable 'ua' was not found in the $PATH")
try:
# We assume that dx_sync_directory is located in the same folder as this script
# This is resolved by absolute path of invocation
sub.check_call([
'python3', '{curr_dir}/dx_sync_directory.py'.format(
curr_dir=sys.path[0]), '-h'
],
stdout=open(os.devnull, 'w'),
close_fds=True)
except sub.CalledProcessError:
raise_error(
"dx_sync_directory.py not found. Please run incremental " +
"upload from the directory containing incremental_upload.py " +
"and dx_sync_directory.py")
def main(token):
# Configure dxpy authentication
dxpy.set_security_context({
'auth_token_type': 'Bearer',
'auth_token': token
})
# Resolve FACTORY_PROJECT by ID
proj = dxpy.DXProject(FACTORY_PROJECT)
print 'Resolved project:', proj.describe()['name'], proj.get_id()
# Set FACTORY_PROJECT as the workspace for subsequent operations
# (sort of like the current working directory)
dxpy.set_workspace_id(FACTORY_PROJECT)
# Resolve the workflow by name. (Could also store ID like the project)
wf = list(
dxpy.search.find_data_objects(classname="workflow",
name="RNA-seq pipeline",
return_handler=True))[0]
print 'Resolved workflow:', wf.describe()['name'], wf.get_id()
# TODO: Stage the inputs. Here we find them in the IN folder
left_reads = list(
dxpy.search.find_data_objects(classname="file",
name="ENCFF001JPX.1k.fastq.gz",
folder="/IN",
return_handler=True))[0]
print 'Resolved left reads:', left_reads.describe(
)['name'], left_reads.get_id()
right_reads = list(
dxpy.search.find_data_objects(classname="file",
name="ENCFF001JQB.1k.fastq.gz",
folder="/IN",
return_handler=True))[0]
print 'Resolved right reads:', right_reads.describe(
)['name'], right_reads.get_id()
# Launch the workflow
analysis = wf.run({
'0.fastqs': [dxpy.dxlink(left_reads.get_id())],
'0.fastq_pairs': [dxpy.dxlink(right_reads.get_id())]
})
print 'Launched analysis:', analysis.get_id()
print 'Analysis state:', analysis.describe()['state']
# TODO: Poll for (or come back when) analysis state 'done' or 'failed'.
# Handle any failures.
# Cooking-show-style substitution with completed analysis
analysis = dxpy.DXAnalysis(COMPLETED_ANALYSIS)
print 'Analysis state:', analysis.describe()['state']
# Enumerate outputs
print 'Analysis outputs:'
for one_output_name, one_output_link in analysis.describe(
)['output'].iteritems():
one_output = dxpy.get_handler(
one_output_link) # one_output : dxpy.DXFile
one_file_name = one_output.describe()['name']
one_file_url, _ = one_output.get_download_url(preauthenticated=True,
filename=one_file_name)
print one_file_name, one_file_url
def fileno(self):
return 0
if __name__ == "__main__":
upload_path="."
dx_path=""
remote_path="/"
curr_proj = "project-Bz06Zv80pjKvypFvX4kqp1Vk"
API_token={"auth_token_type" : "Bearer", "auth_token" : "Y1WoDbANKOKDF0ZfPV0wt9cQaIotIOWC"}
# try to find the remote path
# by default use "/From_<hostname>"
remote_path = "/From_" + socket.gethostname()
# create a random named file
if len(sys.argv) > 1:
given_rpath=sys.argv[1]
if given_rpath.startswith("/"):
remote_path=given_rpath
else:
remote_path = remote_path + "/" + given_rpath
# set the security context
dxpy.set_security_context(API_token)
# upload to infinity and beyond!
f = randf()
dxpy.bindings.dxfile_functions.upload_local_file(file=f, folder=remote_path, parents=True, project=curr_proj)
