Beispiel #1
0
    def download_url_create_symlink(self, url, sym_name):
        print("url = {}".format(url))

        tmp_file = "localfile"
        # download [url]
        cmd = ["wget", "--tries=5", "--quiet", "-O", tmp_file, url]

        try:
            print("Downloading original link with wget")
            subprocess.check_call(cmd, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            msg = ""
            if e and e.output:
                msg = e.output.strip()
            err_exit("Failed to download with wget: {cmd}\n{msg}\n".format(
                cmd=str(cmd), msg=msg))

        # calculate its md5 checksum
        digest = md5_checksum(tmp_file)
        os.remove(tmp_file)

        # create a symlink on the platform, with the correct checksum
        input_params = {
            'name': sym_name,
            'project': self.proj_id,
            'drive': "drive-PUBLISHED",
            'md5sum': digest,
            'symlinkPath': {
                'object': url
            }
        }
        result = dxpy.api.file_new(input_params=input_params)
        return dxpy.DXFile(dxid=result["id"], project=self.proj_id)
Beispiel #2
0
def md5_checksum(filename):
    md5sum_exe = which("md5sum")
    if md5sum_exe is None:
        err_exit("md5sum is not installed on this system")
    cmd = [md5sum_exe, "-b", filename]
    try:
        print("Calculating checksum")
        cmd_out = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        err_exit("Failed to run md5sum: " + str(cmd))

    line = cmd_out.strip().split()
    if len(line) != 2:
        err_exit("md5sum returned weird results: " + str(line))
    result = line[0]

    # convert to string
    assert (isinstance(result, bytes))
    result = result.decode("ascii")
    return result
Beispiel #3
0
def main(**kwargs):
    """
    Entry point for dx-build-app(let).

    Don't call this function as a subroutine in your program! It is liable to
    sys.exit your program when it detects certain error conditions, so you
    can't recover from those as you could if it raised exceptions. Instead,
    call dx_build_app.build_and_upload_locally which provides the real
    implementation for dx-build-app(let) but is easier to use in your program.
    """

    if len(sys.argv) > 0:
        if sys.argv[0].endswith('dx-build-app'):
            logging.warn('Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.')
        elif sys.argv[0].endswith('dx-build-applet'):
            logging.warn('Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.')

    if len(kwargs) == 0:
        args = parser.parse_args()
    else:
        args = parser.parse_args(**kwargs)

    if dxpy.AUTH_HELPER is None and not args.dry_run:
        parser.error('Authentication required to build an executable on the platform; please run "dx login" first')

    if args.src_dir is None:
        args.src_dir = os.getcwd()
        if USING_PYTHON2:
            args.src_dir = args.src_dir.decode(sys.getfilesystemencoding())

    if args.mode == "app" and args.destination != '.':
        parser.error("--destination cannot be used when creating an app (only an applet)")

    if args.dx_toolkit_autodep in ['beta', 'unstable']:
        logging.warn('The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.')

    if args.overwrite and args.archive:
        parser.error("Options -f/--overwrite and -a/--archive cannot be specified together")

    if args.run is not None and args.dry_run:
        parser.error("Options --dry-run and --run cannot be specified together")

    if args.run and args.remote and args.mode == 'app':
        parser.error("Options --remote, --app, and --run cannot all be specified together. Try removing --run and then separately invoking dx run.")

    executable_id = _build_app(args,
                               json.loads(args.extra_args) if args.extra_args else {})

    if args.run is not None:

        if executable_id is None:
            raise AssertionError('Expected executable_id to be set here')

        try:
            subprocess.check_call(['dx', 'run', executable_id, '--priority', 'high'] + args.run)
        except subprocess.CalledProcessError as e:
            sys.exit(e.returncode)
        except:
            err_exit()

    return
def main(**kwargs):
    """
    Entry point for dx-build-app(let).

    Don't call this function as a subroutine in your program! It is liable to
    sys.exit your program when it detects certain error conditions, so you
    can't recover from those as you could if it raised exceptions. Instead,
    call dx_build_app.build_and_upload_locally which provides the real
    implementation for dx-build-app(let) but is easier to use in your program.
    """

    if len(sys.argv) > 0:
        if sys.argv[0].endswith('dx-build-app'):
            logging.warn('Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.')
        elif sys.argv[0].endswith('dx-build-applet'):
            logging.warn('Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.')

    if len(kwargs) == 0:
        args = parser.parse_args()
    else:
        args = parser.parse_args(**kwargs)

    if dxpy.AUTH_HELPER is None and not args.dry_run:
        parser.error('Authentication required to build an executable on the platform; please run "dx login" first')

    if args.src_dir is None:
        args.src_dir = os.getcwd()

    if args.mode == "app" and args.destination != '.':
        parser.error("--destination cannot be used when creating an app (only an applet)")

    if args.dx_toolkit_autodep in ['beta', 'unstable']:
        logging.warn('The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.')

    if args.overwrite and args.archive:
        parser.error("Options -f/--overwrite and -a/--archive cannot be specified together")

    extra_args = json.loads(args.extra_args) if args.extra_args else {}

    if not args.remote:
        # LOCAL BUILD

        try:
            output = build_and_upload_locally(
                args.src_dir,
                args.mode,
                overwrite=args.overwrite,
                archive=args.archive,
                publish=args.publish,
                destination_override=args.destination,
                version_override=args.version_override,
                bill_to_override=args.bill_to,
                use_temp_build_project=args.use_temp_build_project,
                do_parallel_build=args.parallel_build,
                do_version_autonumbering=args.version_autonumbering,
                do_try_update=args.update,
                dx_toolkit_autodep=args.dx_toolkit_autodep,
                do_build_step=args.build_step,
                do_upload_step=args.upload_step,
                do_check_syntax=args.check_syntax,
                dry_run=args.dry_run,
                return_object_dump=args.json,
                **extra_args
                )

            if output is not None:
                print json.dumps(output)
        except dxpy.app_builder.AppBuilderException as e:
            # AppBuilderException represents errors during app or applet building
            # that could reasonably have been anticipated by the user.
            print >> sys.stderr, "Error: %s" % (e.message,)
            sys.exit(3)
        except dxpy.exceptions.DXAPIError as e:
            print >> sys.stderr, "Error: %s" % (e,)
            sys.exit(3)

        if args.run is not None:
            if output is None:
                err_exit("The --run option was given, but no executable was created")
            cmd = os.path.join(os.environ['DNANEXUS_HOME'], 'bin', 'dx')
            os.execv(cmd, ['dx', 'run', output['id']] + args.run)

        return

    else:
        # REMOTE BUILD

        try:
            app_json = _parse_app_spec(args.src_dir)
            _verify_app_source_dir(args.src_dir)
            if args.mode == "app" and not args.dry_run:
                _verify_app_writable(app_json['name'])
        except dxpy.app_builder.AppBuilderException as e:
            print >> sys.stderr, "Error: %s" % (e.message,)
            sys.exit(3)

        # The following flags might be useful in conjunction with
        # --remote. To enable these, we need to learn how to pass these
        # options through to the interior call of dx_build_app(let).
        if args.dry_run:
            parser.error('--remote cannot be combined with --dry-run')
        if args.overwrite:
            parser.error('--remote cannot be combined with --overwrite/-f')
        if args.archive:
            parser.error('--remote cannot be combined with --archive/-a')

        # The following flags are probably not useful in conjunction
        # with --remote.
        if not args.build_step:
            parser.error('--remote cannot be combined with --no-build-step')
        if not args.upload_step:
            parser.error('--remote cannot be combined with --no-upload-step')
        if args.json:
            parser.error('--remote cannot be combined with --json')
        if not args.use_temp_build_project:
            parser.error('--remote cannot be combined with --no-temp-build-project')

        more_kwargs = {}
        if args.version_override:
            more_kwargs['version_override'] = args.version_override
        if args.bill_to:
            more_kwargs['bill_to_override'] = args.bill_to
        if not args.version_autonumbering:
            more_kwargs['do_version_autonumbering'] = False
        if not args.update:
            more_kwargs['do_try_update'] = False
        if not args.parallel_build:
            more_kwargs['do_parallel_build'] = False
        if not args.check_syntax:
            more_kwargs['do_check_syntax'] = False

        return _build_app_remote(args.mode, args.src_dir, destination_override=args.destination, publish=args.publish, dx_toolkit_autodep=args.dx_toolkit_autodep, **more_kwargs)
Beispiel #5
0
def mount_all_inputs(exclude=None, verbose=False):
    '''
    :param exclude: List of input variables that should not be mounted.
    :type exclude: Array of strings
    :returns: dict of lists of strings where each key is the input variable
                and each list element is the full path to the file that has
                been mounted.
    :param verbose: Start dxfuse with '-verbose 2' logging
    :type verbose: boolean


    This function mounts all files that were supplied as inputs to the app.
    By convention, if an input parameter "FOO" has value

        {"$dnanexus_link": "file-xxxx"}

    and filename INPUT.TXT, then the linked file will be mounted into the
    path:

        $HOME/in/FOO/INPUT.TXT

    If an input is an array of files, then all files will be placed into
    numbered subdirectories under a parent directory named for the
    input. For example, if the input key is FOO, and the inputs are {A, B,
    C}.vcf then, the directory structure will be:

        $HOME/in/FOO/0/A.vcf
                     1/B.vcf
                     2/C.vcf

    Zero padding is used to ensure argument order. For example, if there are
    12 input files {A, B, C, D, E, F, G, H, I, J, K, L}.txt, the directory
    structure will be:

        $HOME/in/FOO/00/A.vcf
                     ...
                     11/L.vcf

    This allows using shell globbing (FOO/*/*.vcf) to get all the files in the input
    order and prevents issues with files which have the same filename.'''

    print("Mounting inputs...")

    home_dir = os.environ["HOME"]
    mount_dir = os.path.join(home_dir, "in")
    mount_manifest_file = os.path.join(home_dir, "mount-manifest.json")
    dxfuse_cmd = _which("dxfuse")
    if dxfuse_cmd is None:
        err_exit("dxfuse is not installed on this system")

    subprocess.check_output(["mkdir", mount_dir])

    try:
        job_input_file = file_load_utils.get_input_json_file()
        dirs, inputs, rest = file_load_utils.get_job_input_filenames(job_input_file)
    except IOError:
        msg = 'Error: Could not find the input json file: {0}.\n'.format(job_input_file)
        msg += '       This function should only be called from within a running job.'
        print(msg)
        raise

    # Remove excluded inputs
    if exclude:
        inputs = file_load_utils.filter_dict(inputs, exclude)

    # Convert to a flat list of elements to mount
    to_mount = []
    for ival_list in inputs.values():
        to_mount.extend(ival_list)

    files_manifest = _build_mount_manifest(to_mount)
    with open(mount_manifest_file, 'w') as mfile:
        json.dump(files_manifest, mfile)

    dxfuse_version = subprocess.check_output([dxfuse_cmd, "-version"])
    print("Using dxfuse version " + str(dxfuse_version))

    uid = str(int(subprocess.check_output(["id", "-u"])))
    gid = str(int(subprocess.check_output(["id", "-g"])))
    cmd = [dxfuse_cmd, "-uid", uid, "-gid", gid, mount_dir, mount_manifest_file]
    if verbose:
        cmd[1:1] = ["-verbose", "2"]
    print(subprocess.check_output(cmd))

    print("Done mounting inputs.")

    subprocess.call(["find", mount_dir, "-name", "*"])

    helper_vars = _gen_helper_dict(inputs)
    return helper_vars
Beispiel #6
0
def main(**kwargs):
    """
    Entry point for dx-build-app(let).

    Don't call this function as a subroutine in your program! It is liable to
    sys.exit your program when it detects certain error conditions, so you
    can't recover from those as you could if it raised exceptions. Instead,
    call dx_build_app.build_and_upload_locally which provides the real
    implementation for dx-build-app(let) but is easier to use in your program.
    """

    if len(sys.argv) > 0:
        if sys.argv[0].endswith('dx-build-app'):
            logging.warn(
                'Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.'
            )
        elif sys.argv[0].endswith('dx-build-applet'):
            logging.warn(
                'Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.'
            )

    if len(kwargs) == 0:
        args = parser.parse_args()
    else:
        args = parser.parse_args(**kwargs)

    if dxpy.AUTH_HELPER is None and not args.dry_run:
        parser.error(
            'Authentication required to build an executable on the platform; please run "dx login" first'
        )

    if args.src_dir is None:
        args.src_dir = os.getcwd()

    if args.mode == "app" and args.destination != '.':
        parser.error(
            "--destination cannot be used when creating an app (only an applet)"
        )

    if args.dx_toolkit_autodep in ['beta', 'unstable']:
        logging.warn(
            'The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.'
        )

    if args.overwrite and args.archive:
        parser.error(
            "Options -f/--overwrite and -a/--archive cannot be specified together"
        )

    extra_args = json.loads(args.extra_args) if args.extra_args else {}

    if not args.remote:
        # LOCAL BUILD

        try:
            output = build_and_upload_locally(
                args.src_dir,
                args.mode,
                overwrite=args.overwrite,
                archive=args.archive,
                publish=args.publish,
                destination_override=args.destination,
                version_override=args.version_override,
                bill_to_override=args.bill_to,
                use_temp_build_project=args.use_temp_build_project,
                do_parallel_build=args.parallel_build,
                do_version_autonumbering=args.version_autonumbering,
                do_try_update=args.update,
                dx_toolkit_autodep=args.dx_toolkit_autodep,
                do_build_step=args.build_step,
                do_upload_step=args.upload_step,
                do_check_syntax=args.check_syntax,
                dry_run=args.dry_run,
                return_object_dump=args.json,
                **extra_args)

            if output is not None:
                print json.dumps(output)
        except dxpy.app_builder.AppBuilderException as e:
            # AppBuilderException represents errors during app or applet building
            # that could reasonably have been anticipated by the user.
            print >> sys.stderr, "Error: %s" % (e.message, )
            sys.exit(3)
        except dxpy.exceptions.DXAPIError as e:
            print >> sys.stderr, "Error: %s" % (e, )
            sys.exit(3)

        if args.run is not None:
            if output is None:
                err_exit(
                    "The --run option was given, but no executable was created"
                )
            cmd = os.path.join(os.environ['DNANEXUS_HOME'], 'bin', 'dx')
            os.execv(cmd, ['dx', 'run', output['id']] + args.run)

        return

    else:
        # REMOTE BUILD

        try:
            app_json = _parse_app_spec(args.src_dir)
            _verify_app_source_dir(args.src_dir)
            if args.mode == "app" and not args.dry_run:
                _verify_app_writable(app_json['name'])
        except dxpy.app_builder.AppBuilderException as e:
            print >> sys.stderr, "Error: %s" % (e.message, )
            sys.exit(3)

        # The following flags might be useful in conjunction with
        # --remote. To enable these, we need to learn how to pass these
        # options through to the interior call of dx_build_app(let).
        if args.dry_run:
            parser.error('--remote cannot be combined with --dry-run')
        if args.overwrite:
            parser.error('--remote cannot be combined with --overwrite/-f')
        if args.archive:
            parser.error('--remote cannot be combined with --archive/-a')

        # The following flags are probably not useful in conjunction
        # with --remote.
        if not args.build_step:
            parser.error('--remote cannot be combined with --no-build-step')
        if not args.upload_step:
            parser.error('--remote cannot be combined with --no-upload-step')
        if args.json:
            parser.error('--remote cannot be combined with --json')
        if not args.use_temp_build_project:
            parser.error(
                '--remote cannot be combined with --no-temp-build-project')

        more_kwargs = {}
        if args.version_override:
            more_kwargs['version_override'] = args.version_override
        if args.bill_to:
            more_kwargs['bill_to_override'] = args.bill_to
        if not args.version_autonumbering:
            more_kwargs['do_version_autonumbering'] = False
        if not args.update:
            more_kwargs['do_try_update'] = False
        if not args.parallel_build:
            more_kwargs['do_parallel_build'] = False
        if not args.check_syntax:
            more_kwargs['do_check_syntax'] = False

        return _build_app_remote(args.mode,
                                 args.src_dir,
                                 destination_override=args.destination,
                                 publish=args.publish,
                                 dx_toolkit_autodep=args.dx_toolkit_autodep,
                                 **more_kwargs)
Beispiel #7
0
def main(**kwargs):
    """
    Entry point for dx-build-app(let).

    Don't call this function as a subroutine in your program! It is liable to
    sys.exit your program when it detects certain error conditions, so you
    can't recover from those as you could if it raised exceptions. Instead,
    call dx_build_app.build_and_upload_locally which provides the real
    implementation for dx-build-app(let) but is easier to use in your program.
    """

    if len(sys.argv) > 0:
        if sys.argv[0].endswith('dx-build-app'):
            logging.warn('Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.')
        elif sys.argv[0].endswith('dx-build-applet'):
            logging.warn('Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.')

    if len(kwargs) == 0:
        args = parser.parse_args()
    else:
        args = parser.parse_args(**kwargs)

    if dxpy.AUTH_HELPER is None and not args.dry_run:
        parser.error('Authentication required to build an executable on the platform; please run "dx login" first')

    if args.src_dir is None:
        args.src_dir = os.getcwd()
        if USING_PYTHON2:
            args.src_dir = args.src_dir.decode(sys.getfilesystemencoding())

    if args.mode == "app" and args.destination != '.':
        parser.error("--destination cannot be used when creating an app (only an applet)")

    if args.dx_toolkit_autodep in ['beta', 'unstable']:
        logging.warn('The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.')

    if args.overwrite and args.archive:
        parser.error("Options -f/--overwrite and -a/--archive cannot be specified together")

    if args.run is not None and args.dry_run:
        parser.error("Options --dry-run and --run cannot be specified together")

    if args.run and args.remote and args.mode == 'app':
        parser.error("Options --remote, --app, and --run cannot all be specified together. Try removing --run and then separately invoking dx run.")

    executable_id = _build_app(args,
                               json.loads(args.extra_args) if args.extra_args else {})

    if args.run is not None:

        if executable_id is None:
            raise AssertionError('Expected executable_id to be set here')

        try:
            subprocess.check_call(['dx', 'run', executable_id, '--priority', 'high'] + args.run)
        except subprocess.CalledProcessError as e:
            sys.exit(e.returncode)
        except:
            err_exit()

    return