def download_url_create_symlink(self, url, sym_name): print("url = {}".format(url)) tmp_file = "localfile" # download [url] cmd = ["wget", "--tries=5", "--quiet", "-O", tmp_file, url] try: print("Downloading original link with wget") subprocess.check_call(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: msg = "" if e and e.output: msg = e.output.strip() err_exit("Failed to download with wget: {cmd}\n{msg}\n".format( cmd=str(cmd), msg=msg)) # calculate its md5 checksum digest = md5_checksum(tmp_file) os.remove(tmp_file) # create a symlink on the platform, with the correct checksum input_params = { 'name': sym_name, 'project': self.proj_id, 'drive': "drive-PUBLISHED", 'md5sum': digest, 'symlinkPath': { 'object': url } } result = dxpy.api.file_new(input_params=input_params) return dxpy.DXFile(dxid=result["id"], project=self.proj_id)
def md5_checksum(filename): md5sum_exe = which("md5sum") if md5sum_exe is None: err_exit("md5sum is not installed on this system") cmd = [md5sum_exe, "-b", filename] try: print("Calculating checksum") cmd_out = subprocess.check_output(cmd) except subprocess.CalledProcessError: err_exit("Failed to run md5sum: " + str(cmd)) line = cmd_out.strip().split() if len(line) != 2: err_exit("md5sum returned weird results: " + str(line)) result = line[0] # convert to string assert (isinstance(result, bytes)) result = result.decode("ascii") return result
def main(**kwargs): """ Entry point for dx-build-app(let). Don't call this function as a subroutine in your program! It is liable to sys.exit your program when it detects certain error conditions, so you can't recover from those as you could if it raised exceptions. Instead, call dx_build_app.build_and_upload_locally which provides the real implementation for dx-build-app(let) but is easier to use in your program. """ if len(sys.argv) > 0: if sys.argv[0].endswith('dx-build-app'): logging.warn('Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.') elif sys.argv[0].endswith('dx-build-applet'): logging.warn('Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.') if len(kwargs) == 0: args = parser.parse_args() else: args = parser.parse_args(**kwargs) if dxpy.AUTH_HELPER is None and not args.dry_run: parser.error('Authentication required to build an executable on the platform; please run "dx login" first') if args.src_dir is None: args.src_dir = os.getcwd() if USING_PYTHON2: args.src_dir = args.src_dir.decode(sys.getfilesystemencoding()) if args.mode == "app" and args.destination != '.': parser.error("--destination cannot be used when creating an app (only an applet)") if args.dx_toolkit_autodep in ['beta', 'unstable']: logging.warn('The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.') if args.overwrite and args.archive: parser.error("Options -f/--overwrite and -a/--archive cannot be specified together") if args.run is not None and args.dry_run: parser.error("Options --dry-run and --run cannot be specified together") if args.run and args.remote and args.mode == 'app': parser.error("Options --remote, --app, and --run cannot all be specified together. Try removing --run and then separately invoking dx run.") executable_id = _build_app(args, json.loads(args.extra_args) if args.extra_args else {}) if args.run is not None: if executable_id is None: raise AssertionError('Expected executable_id to be set here') try: subprocess.check_call(['dx', 'run', executable_id, '--priority', 'high'] + args.run) except subprocess.CalledProcessError as e: sys.exit(e.returncode) except: err_exit() return
def main(**kwargs): """ Entry point for dx-build-app(let). Don't call this function as a subroutine in your program! It is liable to sys.exit your program when it detects certain error conditions, so you can't recover from those as you could if it raised exceptions. Instead, call dx_build_app.build_and_upload_locally which provides the real implementation for dx-build-app(let) but is easier to use in your program. """ if len(sys.argv) > 0: if sys.argv[0].endswith('dx-build-app'): logging.warn('Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.') elif sys.argv[0].endswith('dx-build-applet'): logging.warn('Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.') if len(kwargs) == 0: args = parser.parse_args() else: args = parser.parse_args(**kwargs) if dxpy.AUTH_HELPER is None and not args.dry_run: parser.error('Authentication required to build an executable on the platform; please run "dx login" first') if args.src_dir is None: args.src_dir = os.getcwd() if args.mode == "app" and args.destination != '.': parser.error("--destination cannot be used when creating an app (only an applet)") if args.dx_toolkit_autodep in ['beta', 'unstable']: logging.warn('The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.') if args.overwrite and args.archive: parser.error("Options -f/--overwrite and -a/--archive cannot be specified together") extra_args = json.loads(args.extra_args) if args.extra_args else {} if not args.remote: # LOCAL BUILD try: output = build_and_upload_locally( args.src_dir, args.mode, overwrite=args.overwrite, archive=args.archive, publish=args.publish, destination_override=args.destination, version_override=args.version_override, bill_to_override=args.bill_to, use_temp_build_project=args.use_temp_build_project, do_parallel_build=args.parallel_build, do_version_autonumbering=args.version_autonumbering, do_try_update=args.update, dx_toolkit_autodep=args.dx_toolkit_autodep, do_build_step=args.build_step, do_upload_step=args.upload_step, do_check_syntax=args.check_syntax, dry_run=args.dry_run, return_object_dump=args.json, **extra_args ) if output is not None: print json.dumps(output) except dxpy.app_builder.AppBuilderException as e: # AppBuilderException represents errors during app or applet building # that could reasonably have been anticipated by the user. print >> sys.stderr, "Error: %s" % (e.message,) sys.exit(3) except dxpy.exceptions.DXAPIError as e: print >> sys.stderr, "Error: %s" % (e,) sys.exit(3) if args.run is not None: if output is None: err_exit("The --run option was given, but no executable was created") cmd = os.path.join(os.environ['DNANEXUS_HOME'], 'bin', 'dx') os.execv(cmd, ['dx', 'run', output['id']] + args.run) return else: # REMOTE BUILD try: app_json = _parse_app_spec(args.src_dir) _verify_app_source_dir(args.src_dir) if args.mode == "app" and not args.dry_run: _verify_app_writable(app_json['name']) except dxpy.app_builder.AppBuilderException as e: print >> sys.stderr, "Error: %s" % (e.message,) sys.exit(3) # The following flags might be useful in conjunction with # --remote. To enable these, we need to learn how to pass these # options through to the interior call of dx_build_app(let). if args.dry_run: parser.error('--remote cannot be combined with --dry-run') if args.overwrite: parser.error('--remote cannot be combined with --overwrite/-f') if args.archive: parser.error('--remote cannot be combined with --archive/-a') # The following flags are probably not useful in conjunction # with --remote. if not args.build_step: parser.error('--remote cannot be combined with --no-build-step') if not args.upload_step: parser.error('--remote cannot be combined with --no-upload-step') if args.json: parser.error('--remote cannot be combined with --json') if not args.use_temp_build_project: parser.error('--remote cannot be combined with --no-temp-build-project') more_kwargs = {} if args.version_override: more_kwargs['version_override'] = args.version_override if args.bill_to: more_kwargs['bill_to_override'] = args.bill_to if not args.version_autonumbering: more_kwargs['do_version_autonumbering'] = False if not args.update: more_kwargs['do_try_update'] = False if not args.parallel_build: more_kwargs['do_parallel_build'] = False if not args.check_syntax: more_kwargs['do_check_syntax'] = False return _build_app_remote(args.mode, args.src_dir, destination_override=args.destination, publish=args.publish, dx_toolkit_autodep=args.dx_toolkit_autodep, **more_kwargs)
def mount_all_inputs(exclude=None, verbose=False): ''' :param exclude: List of input variables that should not be mounted. :type exclude: Array of strings :returns: dict of lists of strings where each key is the input variable and each list element is the full path to the file that has been mounted. :param verbose: Start dxfuse with '-verbose 2' logging :type verbose: boolean This function mounts all files that were supplied as inputs to the app. By convention, if an input parameter "FOO" has value {"$dnanexus_link": "file-xxxx"} and filename INPUT.TXT, then the linked file will be mounted into the path: $HOME/in/FOO/INPUT.TXT If an input is an array of files, then all files will be placed into numbered subdirectories under a parent directory named for the input. For example, if the input key is FOO, and the inputs are {A, B, C}.vcf then, the directory structure will be: $HOME/in/FOO/0/A.vcf 1/B.vcf 2/C.vcf Zero padding is used to ensure argument order. For example, if there are 12 input files {A, B, C, D, E, F, G, H, I, J, K, L}.txt, the directory structure will be: $HOME/in/FOO/00/A.vcf ... 11/L.vcf This allows using shell globbing (FOO/*/*.vcf) to get all the files in the input order and prevents issues with files which have the same filename.''' print("Mounting inputs...") home_dir = os.environ["HOME"] mount_dir = os.path.join(home_dir, "in") mount_manifest_file = os.path.join(home_dir, "mount-manifest.json") dxfuse_cmd = _which("dxfuse") if dxfuse_cmd is None: err_exit("dxfuse is not installed on this system") subprocess.check_output(["mkdir", mount_dir]) try: job_input_file = file_load_utils.get_input_json_file() dirs, inputs, rest = file_load_utils.get_job_input_filenames(job_input_file) except IOError: msg = 'Error: Could not find the input json file: {0}.\n'.format(job_input_file) msg += ' This function should only be called from within a running job.' print(msg) raise # Remove excluded inputs if exclude: inputs = file_load_utils.filter_dict(inputs, exclude) # Convert to a flat list of elements to mount to_mount = [] for ival_list in inputs.values(): to_mount.extend(ival_list) files_manifest = _build_mount_manifest(to_mount) with open(mount_manifest_file, 'w') as mfile: json.dump(files_manifest, mfile) dxfuse_version = subprocess.check_output([dxfuse_cmd, "-version"]) print("Using dxfuse version " + str(dxfuse_version)) uid = str(int(subprocess.check_output(["id", "-u"]))) gid = str(int(subprocess.check_output(["id", "-g"]))) cmd = [dxfuse_cmd, "-uid", uid, "-gid", gid, mount_dir, mount_manifest_file] if verbose: cmd[1:1] = ["-verbose", "2"] print(subprocess.check_output(cmd)) print("Done mounting inputs.") subprocess.call(["find", mount_dir, "-name", "*"]) helper_vars = _gen_helper_dict(inputs) return helper_vars
def main(**kwargs): """ Entry point for dx-build-app(let). Don't call this function as a subroutine in your program! It is liable to sys.exit your program when it detects certain error conditions, so you can't recover from those as you could if it raised exceptions. Instead, call dx_build_app.build_and_upload_locally which provides the real implementation for dx-build-app(let) but is easier to use in your program. """ if len(sys.argv) > 0: if sys.argv[0].endswith('dx-build-app'): logging.warn( 'Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.' ) elif sys.argv[0].endswith('dx-build-applet'): logging.warn( 'Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.' ) if len(kwargs) == 0: args = parser.parse_args() else: args = parser.parse_args(**kwargs) if dxpy.AUTH_HELPER is None and not args.dry_run: parser.error( 'Authentication required to build an executable on the platform; please run "dx login" first' ) if args.src_dir is None: args.src_dir = os.getcwd() if args.mode == "app" and args.destination != '.': parser.error( "--destination cannot be used when creating an app (only an applet)" ) if args.dx_toolkit_autodep in ['beta', 'unstable']: logging.warn( 'The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.' ) if args.overwrite and args.archive: parser.error( "Options -f/--overwrite and -a/--archive cannot be specified together" ) extra_args = json.loads(args.extra_args) if args.extra_args else {} if not args.remote: # LOCAL BUILD try: output = build_and_upload_locally( args.src_dir, args.mode, overwrite=args.overwrite, archive=args.archive, publish=args.publish, destination_override=args.destination, version_override=args.version_override, bill_to_override=args.bill_to, use_temp_build_project=args.use_temp_build_project, do_parallel_build=args.parallel_build, do_version_autonumbering=args.version_autonumbering, do_try_update=args.update, dx_toolkit_autodep=args.dx_toolkit_autodep, do_build_step=args.build_step, do_upload_step=args.upload_step, do_check_syntax=args.check_syntax, dry_run=args.dry_run, return_object_dump=args.json, **extra_args) if output is not None: print json.dumps(output) except dxpy.app_builder.AppBuilderException as e: # AppBuilderException represents errors during app or applet building # that could reasonably have been anticipated by the user. print >> sys.stderr, "Error: %s" % (e.message, ) sys.exit(3) except dxpy.exceptions.DXAPIError as e: print >> sys.stderr, "Error: %s" % (e, ) sys.exit(3) if args.run is not None: if output is None: err_exit( "The --run option was given, but no executable was created" ) cmd = os.path.join(os.environ['DNANEXUS_HOME'], 'bin', 'dx') os.execv(cmd, ['dx', 'run', output['id']] + args.run) return else: # REMOTE BUILD try: app_json = _parse_app_spec(args.src_dir) _verify_app_source_dir(args.src_dir) if args.mode == "app" and not args.dry_run: _verify_app_writable(app_json['name']) except dxpy.app_builder.AppBuilderException as e: print >> sys.stderr, "Error: %s" % (e.message, ) sys.exit(3) # The following flags might be useful in conjunction with # --remote. To enable these, we need to learn how to pass these # options through to the interior call of dx_build_app(let). if args.dry_run: parser.error('--remote cannot be combined with --dry-run') if args.overwrite: parser.error('--remote cannot be combined with --overwrite/-f') if args.archive: parser.error('--remote cannot be combined with --archive/-a') # The following flags are probably not useful in conjunction # with --remote. if not args.build_step: parser.error('--remote cannot be combined with --no-build-step') if not args.upload_step: parser.error('--remote cannot be combined with --no-upload-step') if args.json: parser.error('--remote cannot be combined with --json') if not args.use_temp_build_project: parser.error( '--remote cannot be combined with --no-temp-build-project') more_kwargs = {} if args.version_override: more_kwargs['version_override'] = args.version_override if args.bill_to: more_kwargs['bill_to_override'] = args.bill_to if not args.version_autonumbering: more_kwargs['do_version_autonumbering'] = False if not args.update: more_kwargs['do_try_update'] = False if not args.parallel_build: more_kwargs['do_parallel_build'] = False if not args.check_syntax: more_kwargs['do_check_syntax'] = False return _build_app_remote(args.mode, args.src_dir, destination_override=args.destination, publish=args.publish, dx_toolkit_autodep=args.dx_toolkit_autodep, **more_kwargs)