Exemple #1
0
    def get_resource(name):
        """
        :param name: optional filename or http(s) url
        for the application or group resource
        :type name: str | None
        :returns: resource
        :rtype: dict
        """
        if name is not None:
            if os.path.isfile(name):
                with util.open_file(name) as resource_file:
                    return util.load_json(resource_file)
            else:
                try:
                    http.silence_requests_warnings()
                    req = http.get(name)
                    if req.status_code == 200:
                        data = b''
                        for chunk in req.iter_content(1024):
                            data += chunk
                        return util.load_jsons(data.decode('utf-8'))
                    else:
                        raise Exception
                except Exception:
                    logger.exception('Cannot read from resource %s', name)
                    raise DCOSException(
                        "Can't read from resource: {0}.\n"
                        "Please check that it exists.".format(name))

        example = "E.g.: dcos marathon app add < app_resource.json"
        ResourceReader._assert_no_tty(example)

        return util.load_json(sys.stdin)
Exemple #2
0
    def get_resource(name):
        """
        :param name: optional filename or http(s) url
        for the application or group resource
        :type name: str | None
        :returns: resource
        :rtype: dict
        """
        if name is not None:
            if os.path.isfile(name):
                with util.open_file(name) as resource_file:
                    return util.load_json(resource_file)
            else:
                try:
                    http.silence_requests_warnings()
                    req = http.get(name)
                    if req.status_code == 200:
                        data = b''
                        for chunk in req.iter_content(1024):
                            data += chunk
                        return util.load_jsons(data.decode('utf-8'))
                    else:
                        raise Exception
                except Exception:
                    logger.exception('Cannot read from resource %s', name)
                    raise DCOSException(
                        "Can't read from resource: {0}.\n"
                        "Please check that it exists.".format(name))

        example = "E.g.: dcos marathon app add < app_resource.json"
        ResourceReader._assert_no_tty(example)

        return util.load_json(sys.stdin)
Exemple #3
0
def get_resource(resource):
    """
    :param resource: optional filename or http(s) url
    for the application or group resource
    :type resource: str
    :returns: resource
    :rtype: dict
    """
    if resource is not None:
        if os.path.isfile(resource):
            with util.open_file(resource) as resource_file:
                return util.load_json(resource_file)
        else:
            try:
                http.silence_requests_warnings()
                req = http.get(resource)
                if req.status_code == 200:
                    data = b''
                    for chunk in req.iter_content(1024):
                        data += chunk
                    return util.load_jsons(data.decode('utf-8'))
                else:
                    raise Exception
            except Exception:
                raise DCOSException(
                    "Can't read from resource: {0}.\n"
                    "Please check that it exists.".format(resource))
Exemple #4
0
def _replace_directly(build_definition, build_schema,
                      build_definition_directory, ref):
    """ Replaces the local reference ref with the contents of
     the file pointed to by ref

    :param build_definition: The DC/OS Package Build Definition that
    may contain local references
    :type build_definition: dict
    :param build_definition_directory: The directory of the Build Definition
    :type build_definition_directory: str
    :param build_schema: The schema for the Build Definition
    :type build_schema: dict
    :param ref: The key in build_definition that will be replaced
    :type ref: str
    """
    if ref in build_definition and _is_local_reference(build_definition[ref]):
        location = build_definition[ref][1:]
        if not os.path.isabs(location):
            location = os.path.join(build_definition_directory, location)

        with util.open_file(location) as f:
            contents = util.load_json(f, True)

        build_definition[ref] = contents

        errs = util.validate_json(build_definition, build_schema)
        if errs:
            logger.debug("Failed during resolution of {}: \n"
                         "\tbuild definition: {}"
                         "".format(ref, build_definition))
            raise DCOSException(_validation_error(location))
Exemple #5
0
def _validate_json_file(fullpath):
    """Validates the content of the file against its schema. Throws an
    exception if the file is not valid.

    :param fullpath: full path to the file.
    :type fullpath: str
    :return: json object if it is a special file
    :rtype: dict
    """

    filename = os.path.basename(fullpath)
    if filename in ['command.json', 'config.json', 'package.json']:
        schema_path = 'data/universe-schema/{}'.format(filename)
    else:
        raise DCOSException(('Error bundling package. Unknown file in package '
                             'directory [{}]').format(fullpath))

    special_schema = util.load_jsons(
        pkg_resources.resource_string('dcoscli', schema_path).decode('utf-8'))

    with util.open_file(fullpath) as special_file:
        special_json = util.load_json(special_file)

    errs = util.validate_json(special_json, special_schema)
    if errs:
        emitter.publish(
            errors.DefaultError(
                'Error validating JSON file [{}]'.format(fullpath)))
        raise DCOSException(util.list_to_err(errs))

    return special_json
Exemple #6
0
def _validate_json_file(fullpath):
    """Validates the content of the file against its schema. Throws an
    exception if the file is not valid.

    :param fullpath: full path to the file.
    :type fullpath: str
    :return: json object if it is a special file
    :rtype: dict
    """

    filename = os.path.basename(fullpath)
    if filename in ['command.json', 'config.json', 'package.json']:
        schema_path = 'data/universe-schema/{}'.format(filename)
    else:
        raise DCOSException(
            ('Error bundling package. Unknown file in package '
             'directory [{}]').format(fullpath))

    special_schema = util.load_jsons(
        pkg_resources.resource_string('dcoscli', schema_path).decode('utf-8'))

    with util.open_file(fullpath) as special_file:
        special_json = util.load_json(special_file)

    errs = util.validate_json(special_json, special_schema)
    if errs:
        emitter.publish(
            errors.DefaultError(
                'Error validating JSON file [{}]'.format(fullpath)))
        raise DCOSException(util.list_to_err(errs))

    return special_json
Exemple #7
0
def _replace_directly(build_definition,
                      build_schema,
                      build_definition_directory,
                      ref):
    """ Replaces the local reference ref with the contents of
     the file pointed to by ref

    :param build_definition: The DC/OS Package Build Definition that
    may contain local references
    :type build_definition: dict
    :param build_definition_directory: The directory of the Build Definition
    :type build_definition_directory: str
    :param build_schema: The schema for the Build Definition
    :type build_schema: dict
    :param ref: The key in build_definition that will be replaced
    :type ref: str
    """
    if ref in build_definition and _is_local_reference(build_definition[ref]):
        location = build_definition[ref][1:]
        if not os.path.isabs(location):
            location = os.path.join(build_definition_directory, location)

        with util.open_file(location) as f:
            contents = util.load_json(f, True)

        build_definition[ref] = contents

        errs = util.validate_json(build_definition, build_schema)
        if errs:
            logger.debug("Failed during resolution of {}: \n"
                         "\tbuild definition: {}"
                         "".format(ref, build_definition))
            raise DCOSException(_validation_error(location))
Exemple #8
0
    def package_json(self):
        """
        :returns: contents of this subcommand's package.json file.
        :rtype: dict
        """

        package_json_path = os.path.join(self._dir(), 'package.json')
        with util.open_file(package_json_path) as package_json_file:
            return util.load_json(package_json_file)
Exemple #9
0
    def package_json(self):
        """
        :returns: contents of this subcommand's package.json file.
        :rtype: dict
        """

        package_json_path = os.path.join(self._dir(), 'package.json')
        with util.open_file(package_json_path) as package_json_file:
            return util.load_json(package_json_file)
Exemple #10
0
def _get_resource(resource):
    """
    :param resource: optional filename for the application or group resource
    :type resource: str
    :returns: resource
    :rtype: dict
    """
    if resource is not None:
        with util.open_file(resource) as resource_file:
            return util.load_json(resource_file)

    # Check that stdin is not tty
    if sys.stdin.isatty():
        # We don't support TTY right now. In the future we will start an
        # editor
        raise DCOSException(
            "We currently don't support reading from the TTY. Please "
            "specify an application JSON.\n"
            "Usage: dcos app add < app_resource.json")

    return util.load_json(sys.stdin)
Exemple #11
0
def _get_resource(resource):
    """
    :param resource: optional filename or http(s) url
    for the application or group resource
    :type resource: str
    :returns: resource
    :rtype: dict
    """
    if resource is not None:
        if os.path.isfile(resource):
            with util.open_file(resource) as resource_file:
                return util.load_json(resource_file)
        else:
            try:
                http.silence_requests_warnings()
                req = http.get(resource)
                if req.status_code == 200:
                    data = b''
                    for chunk in req.iter_content(1024):
                        data += chunk
                    return util.load_jsons(data.decode('utf-8'))
                else:
                    raise DCOSHTTPException("HTTP error code: {}"
                                            .format(req.status_code))
            except Exception:
                logger.exception('Cannot read from resource %s', resource)
                raise DCOSException(
                    "Can't read from resource: {0}.\n"
                    "Please check that it exists.".format(resource))

    # Check that stdin is not tty
    if sys.stdin.isatty():
        # We don't support TTY right now. In the future we will start an
        # editor
        raise DCOSException(
            "We currently don't support reading from the TTY. Please "
            "specify an application JSON.\n"
            "E.g.: dcos job add < app_resource.json")

    return util.load_json(sys.stdin)
Exemple #12
0
def _get_resource(resource):
    """
    :param resource: optional filename or http(s) url
    for the application or group resource
    :type resource: str
    :returns: resource
    :rtype: dict
    """
    if resource is not None:
        if os.path.isfile(resource):
            with util.open_file(resource) as resource_file:
                return util.load_json(resource_file)
        else:
            try:
                http.silence_requests_warnings()
                req = http.get(resource)
                if req.status_code == 200:
                    data = b''
                    for chunk in req.iter_content(1024):
                        data += chunk
                    return util.load_jsons(data.decode('utf-8'))
                else:
                    raise DCOSHTTPException("HTTP error code: {}"
                                            .format(req.status_code))
            except Exception:
                logger.exception('Cannot read from resource %s', resource)
                raise DCOSException(
                    "Can't read from resource: {0}.\n"
                    "Please check that it exists.".format(resource))

    # Check that stdin is not tty
    if sys.stdin.isatty():
        # We don't support TTY right now. In the future we will start an
        # editor
        raise DCOSException(
            "We currently don't support reading from the TTY. Please "
            "specify an application JSON.\n"
            "E.g.: dcos job add < app_resource.json")

    return util.load_json(sys.stdin)
Exemple #13
0
def _user_options(path):
    """ Read the options at the given file path.

    :param path: file path
    :type path: str
    :returns: options
    :rtype: dict
    """
    if path is None:
        return {}
    else:
        with util.open_file(path) as options_file:
            return util.load_json(options_file)
Exemple #14
0
def _user_options(path):
    """ Read the options at the given file path.

    :param path: file path
    :type path: str
    :returns: options
    :rtype: dict
    """
    if path is None:
        return {}
    else:
        with util.open_file(path) as options_file:
            return util.load_json(options_file)
Exemple #15
0
def _build(output_json, build_definition, output_directory):
    """ Creates a DC/OS Package from a DC/OS Package Build Definition

    :param output_json: whether to output json
    :type output_json: None | bool
    :param build_definition: The path to a DC/OS Package Build Definition
    :type build_definition: str
    :param output_directory: The directory where the DC/OS Package
    will be stored
    :type output_directory: str
    :returns: The process status
    :rtype: int
    """
    # get the path of the build definition
    cwd = os.getcwd()
    build_definition_path = build_definition
    if not os.path.isabs(build_definition_path):
        build_definition_path = os.path.join(cwd, build_definition_path)

    build_definition_directory = os.path.dirname(build_definition_path)

    if not os.path.exists(build_definition_path):
        raise DCOSException(
            "The file [{}] does not exist".format(build_definition_path))

    # get the path to the output directory
    if output_directory is None:
        output_directory = cwd

    if not os.path.exists(output_directory):
        raise DCOSException("The output directory [{}]"
                            " does not exist".format(output_directory))

    logger.debug("Using [%s] as output directory", output_directory)

    # load raw build definition
    with util.open_file(build_definition_path) as bd:
        build_definition_raw = util.load_json(bd, keep_order=True)

    # validate DC/OS Package Build Definition with local references
    build_definition_schema_path = "data/schemas/build-definition-schema.json"
    build_definition_schema = util.load_jsons(
        pkg_resources.resource_string("dcoscli",
                                      build_definition_schema_path).decode())

    errs = util.validate_json(build_definition_raw, build_definition_schema)

    if errs:
        logger.debug("Failed before resolution: \n"
                     "\tbuild definition: {}"
                     "".format(build_definition_raw))
        raise DCOSException(_validation_error(build_definition_path))

    # resolve local references in build definition
    _resolve_local_references(build_definition_raw, build_definition_schema,
                              build_definition_directory)

    # at this point all the local references have been resolved
    build_definition_resolved = build_definition_raw

    # validate resolved build definition
    metadata_schema_path = "data/schemas/metadata-schema.json"
    metadata_schema = util.load_jsons(
        pkg_resources.resource_string("dcoscli",
                                      metadata_schema_path).decode())

    errs = util.validate_json(build_definition_resolved, metadata_schema)

    if errs:
        logger.debug("Failed after resolution: \n"
                     "\tbuild definition: {}"
                     "".format(build_definition_resolved))
        raise DCOSException('Error validating package: '
                            'there was a problem resolving '
                            'the local references in '
                            '[{}]'.format(build_definition_path))

    # create the manifest
    manifest_json = {'built-by': "dcoscli.version={}".format(dcoscli.version)}

    # create the metadata
    metadata_json = build_definition_resolved

    # create zip file
    with tempfile.NamedTemporaryFile() as temp_file:
        with zipfile.ZipFile(temp_file.file,
                             mode='w',
                             compression=zipfile.ZIP_DEFLATED,
                             allowZip64=True) as zip_file:
            metadata = json.dumps(metadata_json, indent=2).encode()
            zip_file.writestr("metadata.json", metadata)

            manifest = json.dumps(manifest_json, indent=2).encode()
            zip_file.writestr("manifest.json", manifest)

        # name the package appropriately
        temp_file.file.seek(0)
        dcos_package_name = '{}-{}-{}.dcos'.format(
            metadata_json['name'], metadata_json['version'],
            md5_hash_file(temp_file.file))

        # get the dcos package path
        dcos_package_path = os.path.join(output_directory, dcos_package_name)

        if os.path.exists(dcos_package_path):
            raise DCOSException(
                'Output file [{}] already exists'.format(dcos_package_path))

        # create a new file to contain the package
        temp_file.file.seek(0)
        with util.open_file(dcos_package_path, 'w+b') as dcos_package:
            shutil.copyfileobj(temp_file.file, dcos_package)

    if output_json:
        message = {'package_path': dcos_package_path}
    else:
        message = 'Created DC/OS Universe Package [{}]'.format(
            dcos_package_path)
    emitter.publish(message)

    return 0
Exemple #16
0
def _build(output_json,
           build_definition,
           output_directory):
    """ Creates a DC/OS Package from a DC/OS Package Build Definition

    :param output_json: whether to output json
    :type output_json: None | bool
    :param build_definition: The Path to a DC/OS package build definition
    :type build_definition: str
    :param output_directory: The directory where the DC/OS Package
    will be stored
    :type output_directory: str
    :returns: The process status
    :rtype: int
    """
    # get the path of the build definition
    cwd = os.getcwd()
    build_definition_path = build_definition
    if not os.path.isabs(build_definition_path):
        build_definition_path = os.path.join(cwd, build_definition_path)

    build_definition_directory = os.path.dirname(build_definition_path)

    if not os.path.exists(build_definition_path):
        raise DCOSException(
            "The file [{}] does not exist".format(build_definition_path))

    # get the path to the output directory
    if output_directory is None:
        output_directory = cwd

    if not os.path.exists(output_directory):
        raise DCOSException(
            "The output directory [{}]"
            " does not exist".format(output_directory))

    logger.debug("Using [%s] as output directory", output_directory)

    # load raw build definition
    with util.open_file(build_definition_path) as bd:
        build_definition_raw = util.load_json(bd, keep_order=True)

    # validate DC/OS Package Build Definition with local references
    build_definition_schema_path = "data/schemas/build-definition-schema.json"
    build_definition_schema = util.load_jsons(
        pkg_resources.resource_string(
            "dcoscli", build_definition_schema_path).decode())

    errs = util.validate_json(build_definition_raw, build_definition_schema)

    if errs:
        logger.debug("Failed before resolution: \n"
                     "\tbuild definition: {}"
                     "".format(build_definition_raw))
        raise DCOSException(_validation_error(build_definition_path))

    # resolve local references in build definition
    _resolve_local_references(
        build_definition_raw,
        build_definition_schema,
        build_definition_directory
    )

    # at this point all the local references have been resolved
    build_definition_resolved = build_definition_raw

    # validate resolved build definition
    metadata_schema_path = "data/schemas/metadata-schema.json"
    metadata_schema = util.load_jsons(
        pkg_resources.resource_string(
            "dcoscli", metadata_schema_path).decode())

    errs = util.validate_json(build_definition_resolved, metadata_schema)

    if errs:
        logger.debug("Failed after resolution: \n"
                     "\tbuild definition: {}"
                     "".format(build_definition_resolved))
        raise DCOSException('Error validating package: '
                            'there was a problem resolving '
                            'the local references in '
                            '[{}]'.format(build_definition_path))

    # create the manifest
    manifest_json = {
        'built-by': "dcoscli.version={}".format(dcoscli.version)
    }

    # create the metadata
    metadata_json = build_definition_resolved

    # create zip file
    with tempfile.NamedTemporaryFile() as temp_file:
        with zipfile.ZipFile(
                temp_file.file,
                mode='w',
                compression=zipfile.ZIP_DEFLATED,
                allowZip64=True) as zip_file:
            metadata = json.dumps(metadata_json, indent=2).encode()
            zip_file.writestr("metadata.json", metadata)

            manifest = json.dumps(manifest_json, indent=2).encode()
            zip_file.writestr("manifest.json", manifest)

        # name the package appropriately
        temp_file.file.seek(0)
        dcos_package_name = '{}-{}-{}.dcos'.format(
            metadata_json['name'],
            metadata_json['version'],
            md5_hash_file(temp_file.file))

        # get the dcos package path
        dcos_package_path = os.path.join(output_directory, dcos_package_name)

        if os.path.exists(dcos_package_path):
            raise DCOSException(
                'Output file [{}] already exists'.format(
                    dcos_package_path))

        # create a new file to contain the package
        temp_file.file.seek(0)
        with util.open_file(dcos_package_path, 'w+b') as dcos_package:
            shutil.copyfileobj(temp_file.file, dcos_package)

    if output_json:
        message = {'package_path': dcos_package_path}
    else:
        message = 'Created DC/OS Universe Package [{}]'.format(
            dcos_package_path)
    emitter.publish(message)

    return 0