def mkdirs():
files = load_file_list()
dirs = set([os.path.dirname(f.path) for f in files])
proj = dxpy.DXProject(dxpy.PROJECT_CONTEXT_ID)
for dn in dirs:
proj.new_folder("/" + dn, parents=True)
def make_dir(date):
"""Make dir for weekly run output from current date"""
project = dxpy.DXProject(projectID)
out_dir = "/dx_weekly_check/{}".format(date)
project.new_folder(folder=out_dir)
return out_dir
def get_project(project_name):
'''Try to find the project with the given name or id.'''
# First, see if the project is a project-id.
try:
project = dxpy.DXProject(project_name)
return project
except dxpy.DXError:
pass
project = dxpy.find_projects(name=project_name,
name_mode='glob',
return_handler=True,
level="VIEW")
project = [p for p in project]
if len(project) < 1:
print('Did not find project {0}.'.format(project_name),
file=sys.stderr)
sys.exit(1)
elif len(project) > 1:
print('Found more than 1 project matching {0}.'.format(project_name),
file=sys.stderr)
sys.exit(1)
else:
project = project[0]
return project
def get_project(project_name):
"""Try to find the project with the given name or id on DNAnexus.
It is assumed that the user or job is logged in prior to running this
function so dx API queries are returned."""
# First, see if the project is a project-id.
try:
project = dxpy.DXProject(project_name)
return project
except dxpy.DXError:
pass
project = dxpy.find_projects(name=project_name,
name_mode='glob',
return_handler=True,
level="VIEW")
project = [p for p in project]
if len(project) < 1:
print('Did not find project {0}.'.format(project_name),
file=sys.stderr)
sys.exit(1)
elif len(project) > 1:
print('Found more than 1 project matching {0}.'.format(project_name),
file=sys.stderr)
sys.exit(1)
else:
project = project[0]
return project
def dnanexus_workflows_post(event, context):
"""Handle a WES workflows POST request and turn it into a DNAnexus
/{workflow-id}/run request.
"""
# Create an id for the workflow. We'll return this to the user and also
# tag the dnanexus executions with it.
wes_workflow_id = str(uuid.uuid4())
# Set up the token for making DNAnexus API requests
set_dx_authorization(event["headers"]["Authorization"])
# Get a reference to the "project" on DNAnexus in which we'll be building
# and executing this workflow. We'll put everything in a folder named after
# the WES workflow id so it doesn't get too cluttered, and we avoid name
# collisions.
project_id = event["body"]["key_values"]["dx_project_id"]
dxpy.set_project_context(project_id)
dxpy.DXProject(project_id).new_folder('/' + wes_workflow_id)
# Find URLs that need to be localized, and launch jobs to do that. Note
# that after this step, we'll have jobs pending on dnanexus.
inputs_dict = json.loads(event["body"]["workflow_params"])
dx_localized_input_dict, localization_job_ids = launch_localization_jobs(
inputs_dict, project_id, wes_workflow_id)
# Launch the dnanexus job that will run dxWDL.
run_dxwdl(event["body"]["workflow_descriptor"],
json.dumps(dx_localized_input_dict), wes_workflow_id, project_id,
(event["body"]["workflow_dependencies"]
if "workflow_dependencies" in event["body"] else None),
localization_job_ids)
return {"workflow_id": wes_workflow_id}
def main(reorg_conf___=None, reorg_status___=None):
job_describe = dxpy.describe(dxpy.JOB_ID)
analysis_id = job_describe["analysis"]
stages = dxpy.describe(analysis_id)["stages"]
# key is the name of the output and the value is the link of the file.
output_map = [
x['execution']['output'] for x in stages if x['id'] == 'stage-outputs'
][0]
print(output_map)
output_file = list(output_map.get('output_file').values())
output_config_file = list(output_map.get('output_config_file').values())
output_folder_1 = '/tests/test_reorg/out_1'
output_folder_2 = '/tests/test_reorg/out_2'
dx_container = dxpy.DXProject(dxpy.PROJECT_CONTEXT_ID)
dx_container.move(destination=output_folder_1, objects=output_file)
dx_container.move(
objects=output_config_file,
destination=output_folder_2,
)
return {
"outputs":
[output_map.get('output_file'),
output_map.get('output_config_file')]
}
def temporary_project(name='dx client tests temporary project',
cleanup=True,
reclaim_permissions=False,
select=False):
"""Creates a temporary project scoped to the context manager, and
yields a DXProject handler for the project.
:param cleanup: if False, do not clean up the project when done (useful for debugging so you can examine the state of the project)
:type cleanup: bool
:param reclaim_permissions: if True, attempts a project-xxxx/join before trying to destroy the project. May be needed if the test reduced its own permissions in the project.
:type reclaim_permissions: bool
:param select:
if True, sets the environment variable DX_PROJECT_CONTEXT_ID
(and restores the previous value afterwards) so that subprocess
calls made within the block use the new project by default.
:type select: bool
"""
temp_project = dxpy.DXProject(dxpy.api.project_new({'name': name})['id'])
try:
if select:
with select_project(temp_project):
yield temp_project
else:
yield temp_project
finally:
if reclaim_permissions:
dxpy.DXHTTPRequest('/' + temp_project.get_id() + '/join',
{'level': 'ADMINISTER'})
if cleanup:
dxpy.api.project_destroy(temp_project.get_id(),
{"terminateJobs": True})
def locate_or_create_dx_project(project_name, billTo=None, skip_share=False):
'''Try to find the project with the given name. If one doesn't exist,
we'll create it.'''
projects = dxpy.find_projects(name=project_name,
name_mode='glob',
return_handler=True,
level='CONTRIBUTE')
project = [p for p in projects]
if len(project) < 1:
project_params = {'name': project_name, 'summary': 'VGP Data Project'}
if billTo:
project_params['billTo'] = billTo
project = dxpy.DXProject(dxpy.api.project_new(project_params)['id'])
elif len(project) > 1:
print 'Found more than 1 project matching ' + project_name + '.'
print 'Please provide a unique project!'
sys.exit(1)
else:
project = project[0]
if skip_share is False:
project.invite(SHARE_WITH, "VIEW")
return project
def main():
parser = get_parser()
args = parser.parse_args()
days_ago = args.days_ago
days_ago = "-" + str(days_ago) + "d"
org = args.org
if not org.startswith("org-"):
parser.error("Argument --org must be passed a value prefixed with 'org-'.")
dx_projects = dxpy.find_projects(created_after=days_ago,billed_to=org)
#dx_projects is a generater of dicts of the form {u'permissionSources': [u'user-nathankw'], u'public': False, u'id': u'project-BzzP0j0070XJ8vkJpk0Vgkb7', u'level': u'ADMINISTER'}
for i in dx_projects:
proj_id = i["id"]
proj = dxpy.DXProject(proj_id)
# Use a quick filter to check if this project has been cleaned already:
try:
folder_list = proj.list_folder("/raw_data")
except dxpy.exceptions.ResourceNotFound:
continue
raw_files = folder_list["objects"]
if len(raw_files) < 3:
#Then this project should already have been cleaned, otherwise there'd be at least three files.
#When cleaned, the only files present should be the RunInfo.xml and runParameters.xml.
continue
APP.run(app_input={},project=proj_id,folder=RAW_DATA_FOLDER)
print(proj.name + " (" + proj_id + ")")
def __init__(self, record_link):
self.record_link = record_link.strip()
link_elements = self.record_link.split(':')
record_project = link_elements[0]
record_dxid = link_elements[1]
self.record = dxpy.DXRecord(dxid=record_dxid, project=record_project)
self.properties = self.record.get_properties()
self.details = self.record.get_details()
# Details (Used for Dashboard information)
self.lane_project_id = self.details['laneProject']
self.run_name = self.details['run']
self.run_date = self.run_name.split('_')[0]
self.lane_index = int(self.details['lane'])
self.library_id = self.details['library_id']
self.lane_id = self.details['lane_id']
# Parse library name ("DL_set2_rep1 rcvd 1/4/16")
library_label = self.details['library']
elements = library_label.split('rcvd')
library_name = elements[0].rstrip()
self.library_name = re.sub(r"[^a-zA-Z0-9]+", "-", library_name)
# Properties
self.lims_url = self.properties['lims_url']
self.lims_token = self.properties['lims_token']
self.rta_version = self.properties['rta_version']
self.seq_instrument = self.properties['seq_instrument']
self.flowcell_id = self.properties['flowcell_id']
self.lane_project = dxpy.DXProject(dxid=self.lane_project_id)
self.home = os.getcwd()
self.sample_sheet = None
self.output_dir = None
self.bcl2fastq_version = None
self.lane_barcode = None
self.flowcell_id = None
# Choose bcl2fastq version based on rta_version
## DEV: Update version to match official documentation: i.e. 1.18.54 or later
if StrictVersion(self.rta_version) < StrictVersion('1.18.54'):
self.bcl2fastq_version = 1
elif StrictVersion(self.rta_version) >= StrictVersion('1.18.54'):
self.bcl2fastq_version = 2
# Get barcode information (codepoint + name) from LIMS
# Used to add barcode name to FastQ files
self.connection = Connection(lims_url=self.lims_url,
lims_token=self.lims_token)
self.run_info = RunInfo(conn=self.connection, run=self.run_name)
self.lane_info = self.run_info.get_lane(self.lane_index)
self.barcode_dict = {}
barcode_list = self.lane_info['barcodes']
for barcode_info in barcode_list:
self.barcode_dict[barcode_info['codepoint']] = barcode_info['name']
def main(reorg_conf___=None, reorg_status___=None): # pylint: disable=unused-argument
# find the output stage of the current analysis
analysis_id = dxpy.describe(dxpy.JOB_ID)["analysis"]
stages = dxpy.describe(analysis_id)["stages"]
# retrieve the dictionary containing outputs
output_map = [
x["execution"]["output"] for x in stages if x["id"] == "stage-outputs"
][0]
folder_location = dxpy.describe(analysis_id)["folder"]
project_container = dxpy.DXProject(dxpy.PROJECT_CONTEXT_ID)
# move required outputfiles to their preferred permanent folders
for file_identifiers in output_map.values():
if isinstance(file_identifiers, (list, tuple)):
for indvfile in file_identifiers:
try:
default_location = dxpy.describe(
dxpy.describe(indvfile["$dnanexus_link"])["createdBy"]
["job"])["runInput"]["default_location"]
folder = folder_location + "/" + default_location
except:
folder = folder_location
project_container.new_folder(folder, parents=True)
file_container = dxpy.describe(
indvfile["$dnanexus_link"])["project"]
file_object = dxpy.bindings.DXFile(indvfile["$dnanexus_link"],
project=file_container)
if file_container == dxpy.PROJECT_CONTEXT_ID:
file_object.move(folder)
else:
cloned_file = file_object.clone( # pylint: disable=unused-variable
dxpy.PROJECT_CONTEXT_ID,
folder=folder)
elif isinstance(file_identifiers, dict):
if '$dnanexus_link' in file_identifiers:
try:
default_location = dxpy.describe(
dxpy.describe(file_identifiers["$dnanexus_link"])
["createdBy"]["job"])["runInput"]["default_location"]
folder = folder_location + "/" + default_location
except:
folder = folder_location
project_container.new_folder(folder, parents=True)
file_container = dxpy.describe(
file_identifiers["$dnanexus_link"])["project"]
file_object = dxpy.bindings.DXFile(
file_identifiers["$dnanexus_link"], project=file_container)
if file_container == dxpy.PROJECT_CONTEXT_ID:
file_object.move(folder)
else:
cloned_file = file_object.clone(dxpy.PROJECT_CONTEXT_ID,
folder=folder)
def accept_project_transfers(access_level, queue, org, share_with_org=None):
"""
Args:
access_level: `str`. Permissions level the new member should have on transferred projects. Should
be one of ["VIEW","UPLOAD","CONTRIBUTE","ADMINISTER"]. See
https://wiki.dnanexus.com/API-Specification-v1.0.0/Project-Permissions-and-Sharing for more details on access levels.
queue: `str`. The value of the queue property on a DNAnexus project. Only projects that are
pending transfer that have this value for the queue property will be transferred to the
specified org.
org: `str`. The name of the DNAnexus org under which to accept the project transfers for projects
that have their queue property set to the value of the 'queue' argument.
share_with_org: `str`. Set this argument if you'd like to share the transferred projects
with the org so that all users of the org will have access to the project. The value you
supply should be the access level that members of the org will have.
Returns:
`dict`: The projects that were transferred to the specified billing account. Keys are the
project IDs, and values are the project names.
"""
dx_username = gbsc_dnanexus.utils.get_dx_username()
#gbsc_dnanexus.utils.log_into_dnanexus(dx_username)
org = gbsc_dnanexus.utils.add_dx_orgprefix(org)
pending_transfers = gbsc_dnanexus.utils.pending_transfers(dx_username)
#pending_transfers is a list of project IDs
transferred = {}
for proj_id in pending_transfers:
dx_proj = dxpy.DXProject(proj_id)
props = dx_proj.describe(
input_params={"properties": True})["properties"]
try:
project_queue = props["queue"]
except KeyError:
raise DxProjectMissingQueueProperty(
"DNAnexus project {proj_name} ({proj_id}) is missing the queue property."
.format(proj_name=dx_proj.name, proj_id=proj_id))
if queue != project_queue:
continue
msg = "Accepting project transfer of {proj_name} ({proj_id}) for user {user}, to be billed under the org {org}.".format(
proj_name=dx_proj.name, proj_id=proj_id, user=dx_username, org=org)
debug_logger.debug(msg)
dxpy.DXHTTPRequest("/" + proj_id + "/acceptTransfer", {"billTo": org})
success_logger.info(msg)
transferred[proj_id] = dx_proj.name
if share_with_org:
msg = "Sharing project {proj_id} with {org} with access level {share_with_org}.".format(
proj_id=proj_id, org=org, share_with_org=share_with_org)
debug_logger.debug(msg)
share_with_org(project_ids=[proj_id],
org=org,
access_level=share_with_org)
dxpy.api.project_invite(object_id=proj_id,
input_params={
"invitee": org,
"level": share_with_org,
"suppressEmailNotification": True
})
success_logger.info(msg)
return transferred
def main():
args = get_args()
if args.debug:
logger.setLevel(logging.DEBUG)
logger.debug("Logging level set to DEBUG")
else:
logger.setLevel(logging.INFO)
if args.analysis_ids:
ids = args.analysis_ids
else:
ids = args.infile
first_row = True
for string in ids:
analysis_id = string.strip()
if not analysis_id:
continue
try:
new_analysis = rerun_with_frip(analysis_id, args.dryrun, args.assay_type)
except:
row = "%s\terror" % (analysis_id)
print("%s\t%s" % (analysis_id, "error"), file=sys.stderr)
traceback.print_exc(file=sys.stderr)
else:
if args.dryrun:
# since it's only a temporary workflow, the project is just a termporary
# container, which has only an ID and not a name
project_name = new_analysis.describe().get('project')
else:
project = dxpy.DXProject(new_analysis.describe().get('project'))
project_name = project.name
row = "%s\t%s\t%s" % (
analysis_id,
new_analysis.get_id(),
project_name
)
if args.accession:
try:
accessioning_job = accession_analysis(new_analysis)
except Exception as e:
logger.error("acccession_analysis failed with %s" % (e))
row += "\tfailed"
else:
row += "\t%s" % (None if not accessioning_job else accessioning_job.get_id())
else:
row += " manually"
if first_row:
print("old_analysis\tnew_analysis\tproject\taccession_job")
print(row)
first_row = False
def virtual_path(self):
"""The DXVirtualPath instance equivalent to the canonical path within the specified project
"""
proj = dxpy.DXProject(dxid=self.project)
virtual_p = DXVirtualPath(self.drive + proj.name + ':/')
if self.resource:
file_h = dxpy.DXFile(dxid=self.canonical_resource,
project=self.canonical_project)
virtual_p = virtual_p / file_h.folder[1:] / file_h.name
return virtual_p
def resolve_project(project_name, level=None):
try:
project = dxpy.find_one_project(name=project_name,
name_mode='exact',
level=level,
return_handler=False)
except:
print 'Could not find 1 and only 1 project named %s; ' % format(
project_name)
sys.exit(1)
return dxpy.DXProject(project['id'])
def stat(self):
"""Performs a stat on the path.
Raises:
MultipleObjectsSameNameError: If project or resource is not unique
NotFoundError: When the project or resource cannot be found
ValueError: If path is folder path
"""
if not self.resource:
return dxpy.DXProject(dxid=self.canonical_project).describe()
return dxpy.DXFile(dxid=self.canonical_resource,
project=self.canonical_project).describe()
def listdir(self, only='all', canonicalize=False):
"""List the path as a dir, returning top-level directories and files.
Args:
canonicalize (bool, default False): if True, return canonical paths
only (str): "objects" for only objects, "folders" for only folders,
"all" for both
Returns:
List[DXPath]: Iterates over listed files directly within the resource
Raises:
NotFoundError: When resource folder is not present on DX platform
"""
proj_id = self.canonical_project
proj_name = self.virtual_project
ans_list = []
kwargs = {
'only': only,
'describe': {
'fields': {
'name': True,
'folder': True
}
},
'folder': '/' + (self.resource or '')
}
with _wrap_dx_calls():
obj_dict = dxpy.DXProject(dxid=proj_id).list_folder(**kwargs)
for key, values in obj_dict.items():
for entry in values:
if canonicalize:
ans_list.append(
DXCanonicalPath('dx://{}:/{}'.format(
proj_id, (entry.lstrip('/')
if key == 'folders' else entry['id']))))
else:
if key == 'folders':
ans_list.append(
DXVirtualPath('{drive}{proj_name}:{folder}'.format(
drive=self.drive,
proj_name=proj_name,
folder=entry)))
else:
ans_list.append(
DXVirtualPath(
'{drive}{proj_name}:{folder}/{name}'.format(
drive=self.drive,
proj_name=proj_name,
folder=entry['describe']['folder'].rstrip(
'/'),
name=entry['describe']['name'])))
return ans_list
def get_project(projectName, level=None):
'''Returns the DXProject by name or errors out if not found.'''
try:
project = dxpy.find_one_project(name=projectName,
name_mode='exact',
level=level,
return_handler=False)
except:
print "Could not find 1 and only 1 project named '" + projectName + "'."
sys.exit(1)
return dxpy.DXProject(project['id'])
def __init__(self, project, ref_genome="1kg_v37", url_duration=ONE_YEAR):
"""
:param project:
:param ref_genome:
:param url_duration: number of seconds for which the generated URL will be valid
"""
if isinstance(project, dxpy.DXProject):
pass
elif project.startswith("project-"):
project = dxpy.DXProject(project)
else:
project = dxpy.DXProject(dxpy.find_one_project(name=project)["id"])
assert isinstance(project, dxpy.DXProject)
self.project = project
Global = Element('Global')
Global.set("name", project.name)
Global.set("version", "1")
self.Global = Global
self.url_duration = url_duration
self.genome = ref_genome
def main(worker_max, f_ids, bandwidth, species_name=None):
"""
Input variables removed:
"""
_run_cmd('aws --version', True)
print('file ids: ' + str(f_ids))
# Remove any files which are already symlinks
f_ids = filter(lambda x: not _is_symlink(x), f_ids)
if species_name is None:
species_name = _get_species_name()
# Set upload root to user specified directory or project
projdx = dxpy.DXProject(os.environ['DX_PROJECT_CONTEXT_ID'])
dir_file = os.path.join(S3_ROOT_FOLDER, species_name, projdx.name)
# Trim trailing / in upload dir
dir_file = dir_file.strip('/')
print('Upload directory: ' + dir_file)
# Programatically split files into equal list based on size and max workers
split_list_dxlinks = _split_partition(f_ids, worker_max)
# Select instance type based on user input
trans_worker_inst = instance_from_bandwidth(bandwidth)
# Run subjobs on list
uploadjobs = [dxpy.new_dxjob(
fn_input={'target_s3': TARGET_S3,
'assigned_files': f_group,
'up_dir': dir_file},
fn_name='s3_upload',
instance_type=trans_worker_inst)
for f_group in split_list_dxlinks]
# Merge S3 status upload reports from subjobs
report_fileDXLinks = [subjob.get_output_ref('report_file_link')
for subjob in uploadjobs]
print('Creating S3 upload report')
report_job = dxpy.new_dxjob(
fn_input={'filelinks': report_fileDXLinks}, fn_name='create_upload_report')
# Output merged report
print('Output final report')
finalreportDXLink = report_job.get_output_ref('reportDXLink')
output = {}
output['upload_report'] = finalreportDXLink
return output
def main():
argparser = argparse.ArgumentParser(description="Initialize a git repository for DNAnexus workflow development & continuous integration.")
argparser.add_argument("--name", help="workflow name (default: current directory name)")
argparser.add_argument("--project", help="DNAnexus project ID for continuous integration (default: create a new project)")
argparser.add_argument("--force", help="Overwrite existing local files", action="store_true", default=False)
args = argparser.parse_args()
# ensure we're in a git repository
if not os.path.isdir(".git"):
print("The current directory should be the root of a git repository. Aborting.", file=sys.stderr)
sys.exit(1)
# ensure we're not going to clobber any existing files
if not args.force:
for fn in files_to_create:
if os.path.exists(fn):
print("Local file {} already exists. Aborting.".format(fn), file=sys.stderr)
sys.exit(1)
# detect current directory name
if not args.name:
args.name = os.path.split(os.getcwd())[1]
print("Workflow name: " + args.name)
# create the DNAnexus project for continuous integration
if not args.project:
args.project = dxpy.api.project_new({"name": "dx-ci-{}".format(args.name)})["id"]
project = dxpy.DXProject(args.project)
initialize_project(project)
# generate local applet and workflow source
generate_applet()
generate_build_workflow_py(args.name, project)
generate_run_tests_py(args.name)
generate_travis_yml()
print("""
Initialized templates for DNAnexus workflow continuous integration; modify
them so that './build_workflow.py --run-tests' does everything you'd like.
If this is a GitHub repository, you can complete Travis CI setup with these
steps:
1. Enable this repository in Travis CI's Accounts settings.
2. Create a long-lived DNAnexus auth token with CONTRIBUTE access to the
continuous integration project (xxxx).
3. Run 'travis encrypt DX_AUTH_TOKEN=xxxx --add' here.
('gem install travis' first, if needed)
4. Commit the modified .travis.yml and push everything to GitHub.
Travis CI integration is optional; you can always just run
./build_workflow.py --run-tests
locally instead.
""")
def makedirs_p(self, mode=0o777):
"""Make directories, including parents on DX from DX folder paths.
Args:
mode: unused, present for compatibility
(access permissions are managed at project level)
"""
if not self.resource:
if not self.exists():
raise ValueError('Cannot create a project via makedirs_p()')
return
proj_handler = dxpy.DXProject(self.canonical_project)
with _wrap_dx_calls():
proj_handler.new_folder('/' + self.resource, parents=True)
def main():
args = get_args()
if args.debug:
logger.setLevel(logging.DEBUG)
logger.debug("Logging level set to DEBUG")
else:
logger.setLevel(logging.INFO)
if args.analysis_ids:
ids = args.analysis_ids
else:
ids = args.infile
first_row = True
for string in ids:
analysis_id = string.strip()
if not analysis_id:
continue
try:
new_analysis = rerun_with_applet(analysis_id, args.stage,
args.applet, args.folder)
except:
row = "%s\terror" % (analysis_id)
print("%s\t%s" % (analysis_id, "error"), file=sys.stderr)
traceback.print_exc(file=sys.stderr)
else:
project = dxpy.DXProject(new_analysis.describe()['project'])
row = "%s\t%s\t%s" % (analysis_id, new_analysis.get_id(),
project.name)
if args.accession:
try:
accessioning_job = accession_analysis(new_analysis)
except Exception as e:
logger.error("acccession_analysis failed with %s" % (e))
row += "\tfailed"
else:
row += "\t%s" % (None if not accessioning_job else
accessioning_job.get_id())
else:
row += "\tmanually"
if first_row:
print("old_analysis\tnew_analysis\tproject\taccession_job")
print(row)
first_row = False
def locate_or_create_dx_project(project_name):
'''Try to find the project with the given name. If one doesn't exist,
we'll create it.'''
projects = dxpy.find_projects(name=project_name, name_mode='glob', return_handler=True, level='CONTRIBUTE')
project = [p for p in projects]
if len(project) < 1:
project = dxpy.DXProject(dxpy.api.project_new({'name': project_name, 'summary': 'FALCON Unzip Assembly'})['id'])
elif len(project) > 1:
print 'Found more than 1 project matching ' + project_name + '.'
print 'Please provide a unique project!'
sys.exit(1)
else:
project = project[0]
return project
def get_002_projects():
"""
Return list of 002 projects
Returns:
list: List of project ids
"""
project_objects = []
projects = dx.find_projects(name="002_*", name_mode="glob")
for project in projects:
project_objects.append(dx.DXProject(project["id"]))
return project_objects
def logThisRun(runId, resultsFolder, projectId):
'''Adds a runId to the runsLaunched file in resultsFolder.'''
# NOTE: DX manual lies?! Append not possible?! Then write new/delete old
launchFilePath = resultsFolder + '/' + RUNS_LAUNCHED_FILE
oldFid = dxencode.find_file(launchFilePath, projectId)
newFh = dxpy.new_dxfile('a',
project=projectId,
folder=resultsFolder,
name=RUNS_LAUNCHED_FILE)
newFh.write(runId + ' started:' + str(datetime.now()) + '\n')
if oldFid is not None:
with dxpy.open_dxfile(oldFid) as oldFh:
for oldRunId in oldFh:
newFh.write(oldRunId + '\n')
proj = dxpy.DXProject(projectId)
proj.remove_objects([oldFid])
newFh.close()
def move_files(fids, folder, projectId):
'''Moves files to supplied folder. Expected to be in the same project.'''
for fid in fids:
try:
dxlink = FILES[fid]
except:
#print >> sys.stderr, "File %s not in cache, trying id" % fid
dxlink = fid
fileDict = dxpy.describe(dxlink) # FILES contain dxLinks
if fileDict['project'] != projectId:
print >> sys.stderr, "ERROR: Failed to move '" + fileDict['name'] + "' as it is not in '" + \
projectId + "'."
sys.exit(1)
proj = dxpy.DXProject(projectId)
if not project_has_folder(proj, folder):
proj.new_folder(folder, parents=True)
proj.move(folder, fids)
def _movetree(self, dest):
"""Moves the project or directory to a different folder within project.
Like copytree, if the destination exists as a folder already, the source dir is
moved inside that dest folder with its original name.
The source cannot be the root directory.
Refer to copytree or copytree for detailed information.
Args:
dest (Path): The destination directory path within same project
Raises:
TargetExistsError: When destination directory already exists
DNAnexusError: When attempting to move across projects
"""
if dest.canonical_project != self.canonical_project:
raise DNAnexusError('Cannot movetree across different projects')
if not self.resource:
raise DNAnexusError(
'Cannot move root folder within same project on DX')
if self == dest:
return
if dest == (self.drive +
dest.project): # need to convert dx://proj to dx://proj:
dest = dest + ':'
target_dest, should_rename, moved_folder_path = self._prep_for_copytree(
dest)
project_handler = dxpy.DXProject(self.canonical_project)
with _wrap_dx_calls():
project_handler.move_folder(folder='/' + self.resource,
destination='/' +
(target_dest.parent.resource or ''))
if should_rename:
dxpy.api.project_rename_folder(dest.canonical_project,
input_params={
'folder':
'/' +
moved_folder_path.resource,
'name':
target_dest.name
})
self.clear_cached_properties()
def select_newest_project(dx_project_ids):
"""
Given a list of DNAnexus project IDs, returns the one that is newest as determined by creation date.
Args:
dx_project_ids: `list` of DNAnexus project IDs.
Returns:
`str`.
"""
if len(dx_project_ids) == 1:
return dx_project_ids[0]
projects = [dxpy.DXProject(x) for x in dx_project_ids]
created_times = [x.describe()["created"] for x in projects]
paired = zip(created_times, projects)
paired.sort(reverse=True)
return paired[0][0]
def main(reorg_conf___=None, reorg_status___=None):
# download and parse `reorg_conf___`
#conf_file = dxpy.DXFile(reorg_conf___)
#dxpy.download_dxfile(conf_file.get_id(), "conf.json")
#with open('conf.json') as f:
# conf = json.load(f)
# find the output stage of the current analysis
analysis_id = dxpy.describe(dxpy.JOB_ID)["analysis"]
stages = dxpy.describe(analysis_id)["stages"]
# retrieve the dictionary containing outputs, where key is the name of output and value is the link to the file.
output_map = [x['execution']['output'] for x in stages if x['id'] == 'stage-outputs'][0]
out = output_map['out']
bam = [x for x in out if dxpy.describe(x)["name"].endswith('.bam')][0]
vcf = [x for x in out if dxpy.describe(x)["name"].endswith('.vcf')][0]
vcf_folder = "/results/out/vcf" #conf['vcf']
bam_folder = "/results/out/bam" #conf['bam']
# get the container instance
dx_container = dxpy.DXProject(dxpy.PROJECT_CONTEXT_ID)
dx_container.new_folder(vcf_folder,parents=True)
dx_container.new_folder(bam_folder,parents=True)
dx_container.move(
destination=vcf_folder,
objects=[ vcf['$dnanexus_link'] ]
)
dx_container.move(
destination=bam_folder,
objects=[ bam['$dnanexus_link'] ],
)
