def _validate_json_file(fullpath): """Validates the content of the file against its schema. Throws an exception if the file is not valid. :param fullpath: full path to the file. :type fullpath: str :return: json object if it is a special file :rtype: dict """ filename = os.path.basename(fullpath) if filename in ['command.json', 'config.json', 'package.json']: schema_path = 'data/universe-schema/{}'.format(filename) else: raise DCOSException(('Error bundling package. Unknown file in package ' 'directory [{}]').format(fullpath)) special_schema = util.load_jsons( pkg_resources.resource_string('dcoscli', schema_path).decode('utf-8')) with util.open_file(fullpath) as special_file: special_json = util.load_json(special_file) errs = util.validate_json(special_json, special_schema) if errs: emitter.publish( errors.DefaultError( 'Error validating JSON file [{}]'.format(fullpath))) raise DCOSException(util.list_to_err(errs)) return special_json
def _replace_marathon(build_definition, build_schema, build_definition_directory): """ Replaces the marathon v2AppMustacheTemplate ref with the base64 encoding of the file pointed to by the reference :param build_definition: The DC/OS Package Build Definition that may contain local references :type build_definition: dict :param build_definition_directory: The directory of the Build Definition :type build_definition_directory: str :param build_schema: The schema for the Build Definition :type build_schema: dict """ ref = "marathon" template = "v2AppMustacheTemplate" if ref in build_definition and \ _is_local_reference(build_definition[ref][template]): location = (build_definition[ref])[template][1:] if not os.path.isabs(location): location = os.path.join(build_definition_directory, location) # convert the contents of the marathon file into base64 with util.open_file(location) as f: contents = base64.b64encode(f.read().encode()).decode() build_definition[ref][template] = contents errs = util.validate_json(build_definition, build_schema) if errs: logger.debug("Failed during resolution of marathon: \n" "\tbuild definition: {}" "".format(build_definition)) raise DCOSException(_validation_error(location))
def _validate_json_file(fullpath): """Validates the content of the file against its schema. Throws an exception if the file is not valid. :param fullpath: full path to the file. :type fullpath: str :return: json object if it is a special file :rtype: dict """ filename = os.path.basename(fullpath) if filename in ['command.json', 'config.json', 'package.json']: schema_path = 'data/universe-schema/{}'.format(filename) else: raise DCOSException( ('Error bundling package. Unknown file in package ' 'directory [{}]').format(fullpath)) special_schema = util.load_jsons( pkg_resources.resource_string('dcoscli', schema_path).decode('utf-8')) with util.open_file(fullpath) as special_file: special_json = util.load_json(special_file) errs = util.validate_json(special_json, special_schema) if errs: emitter.publish( errors.DefaultError( 'Error validating JSON file [{}]'.format(fullpath))) raise DCOSException(util.list_to_err(errs)) return special_json
def test_open_file(): path = 'nonexistant_file_name.txt' with pytest.raises(DCOSException) as excinfo: with util.open_file(path): pass assert 'Error opening file [{}]: No such file or directory'.format(path) \ in str(excinfo.value)
def package_add_local(self, dcos_package): """ Adds a locally stored DC/OS package to DC/OS :param dcos_package: path to the DC/OS package :type dcos_package: None | str :return: Response to the package add request :rtype: requests.Response """ try: with util.open_file(dcos_package, 'rb') as pkg: extra_headers = { 'Content-Type': 'application/vnd.dcos.' 'universe.package+zip;version=v1', 'X-Dcos-Content-MD5': util.md5_hash_file(pkg) } return self._post('add', headers=extra_headers, data=pkg) except DCOSHTTPException as e: if e.status() == 404: message = 'Your version of DC/OS ' \ 'does not support this operation' raise DCOSException(message) else: raise e
def get_resource(name): """ :param name: optional filename or http(s) url for the application or group resource :type name: str | None :returns: resource :rtype: dict """ if name is not None: if os.path.isfile(name): with util.open_file(name) as resource_file: return util.load_json(resource_file) else: try: http.silence_requests_warnings() req = http.get(name) if req.status_code == 200: data = b'' for chunk in req.iter_content(1024): data += chunk return util.load_jsons(data.decode('utf-8')) else: raise Exception except Exception: logger.exception('Cannot read from resource %s', name) raise DCOSException( "Can't read from resource: {0}.\n" "Please check that it exists.".format(name)) example = "E.g.: dcos marathon app add < app_resource.json" ResourceReader._assert_no_tty(example) return util.load_json(sys.stdin)
def get_resource(resource): """ :param resource: optional filename or http(s) url for the application or group resource :type resource: str :returns: resource :rtype: dict """ if resource is not None: if os.path.isfile(resource): with util.open_file(resource) as resource_file: return util.load_json(resource_file) else: try: http.silence_requests_warnings() req = http.get(resource) if req.status_code == 200: data = b'' for chunk in req.iter_content(1024): data += chunk return util.load_jsons(data.decode('utf-8')) else: raise Exception except Exception: raise DCOSException( "Can't read from resource: {0}.\n" "Please check that it exists.".format(resource))
def _replace_directly(build_definition, build_schema, build_definition_directory, ref): """ Replaces the local reference ref with the contents of the file pointed to by ref :param build_definition: The DC/OS Package Build Definition that may contain local references :type build_definition: dict :param build_definition_directory: The directory of the Build Definition :type build_definition_directory: str :param build_schema: The schema for the Build Definition :type build_schema: dict :param ref: The key in build_definition that will be replaced :type ref: str """ if ref in build_definition and _is_local_reference(build_definition[ref]): location = build_definition[ref][1:] if not os.path.isabs(location): location = os.path.join(build_definition_directory, location) with util.open_file(location) as f: contents = util.load_json(f, True) build_definition[ref] = contents errs = util.validate_json(build_definition, build_schema) if errs: logger.debug("Failed during resolution of {}: \n" "\tbuild definition: {}" "".format(ref, build_definition)) raise DCOSException(_validation_error(location))
def _replace_marathon(build_definition, build_schema, build_definition_directory): """ Replaces the marathon v2AppMustacheTemplate ref with the base64 encoding of the file pointed to by the reference :param build_definition: The DC/OS Package Build Definition that may contain local references :type build_definition: dict :param build_definition_directory: The directory of the Build Definition :type build_definition_directory: str :param build_schema: The schema for the Build Definition :type build_schema: dict """ ref = "marathon" template = "v2AppMustacheTemplate" if ref in build_definition and \ _is_local_reference(build_definition[ref][template]): location = (build_definition[ref])[template][1:] if not os.path.isabs(location): location = os.path.join(build_definition_directory, location) # convert the contents of the marathon file into base64 with util.open_file(location) as f: contents = base64.b64encode( f.read().encode()).decode() build_definition[ref][template] = contents errs = util.validate_json(build_definition, build_schema) if errs: logger.debug("Failed during resolution of marathon: \n" "\tbuild definition: {}" "".format(build_definition)) raise DCOSException(_validation_error(location))
def _save_auth_keys(key_dict): """ :param key_dict: auth parameters dict :type key_dict: dict :rtype: None """ config_path = os.environ[constants.DCOS_CONFIG_ENV] toml_config = config.mutable_load_from_path(config_path) section = 'core' config_schema = json.loads( pkg_resources.resource_string( 'dcoscli', 'data/config-schema/core.json').decode('utf-8')) for k, v in iteritems(key_dict): python_value = jsonitem.parse_json_value(k, v, config_schema) name = '{}.{}'.format(section, k) toml_config[name] = python_value serial = toml.dumps(toml_config._dictionary) with util.open_file(config_path, 'w') as config_file: config_file.write(serial) return None
def package_json(self): """ :returns: contents of this subcommand's package.json file. :rtype: dict """ package_json_path = os.path.join(self._dir(), 'package.json') with util.open_file(package_json_path) as package_json_file: return util.load_json(package_json_file)
def save(toml_config): """ :param toml_config: TOML configuration object :type toml_config: MutableToml or Toml """ serial = toml.dumps(toml_config._dictionary) path = util.get_config_path() with util.open_file(path, 'w') as config_file: config_file.write(serial)
def mutable_load_from_path(path): """Loads a TOML file from the path :param path: Path to the TOML file :type path: str :returns: Mutable map for the configuration file :rtype: MutableToml """ with util.open_file(path) as config_file: return MutableToml(toml.loads(config_file.read()))
def _save_config_file(config_path, toml_config): """ :param config_path: path to configuration file. :type config_path: str :param toml_config: TOML configuration object :type toml_config: MutableToml or Toml """ serial = toml.dumps(toml_config._dictionary) with util.open_file(config_path, 'w') as config_file: config_file.write(serial)
def _user_options(path): """ Read the options at the given file path. :param path: file path :type path: str :returns: options :rtype: dict """ if path is None: return {} else: with util.open_file(path) as options_file: return util.load_json(options_file)
def save(toml_config): """ :param toml_config: TOML configuration object :type toml_config: MutableToml or Toml """ serial = toml.dumps(toml_config._dictionary) path = get_config_path() util.ensure_file_exists(path) util.enforce_file_permissions(path) with util.open_file(path, 'w') as config_file: config_file.write(serial)
def _write_package_source(pkg): """ Write package source locally. :param pkg: the package being installed :type pkg: Package :rtype: None """ pkg_dir = package_dir(pkg.name()) source_path = os.path.join(pkg_dir, 'source') with util.open_file(source_path, 'w') as source_file: source_file.write(pkg.registry.source.url)
def _write_package_json(pkg): """ Write package.json locally. :param pkg: the package being installed :type pkg: PackageVersion :rtype: None """ pkg_dir = _package_dir(pkg.name()) package_path = os.path.join(pkg_dir, 'package.json') package_json = pkg.package_json() with util.open_file(package_path, 'w') as package_file: json.dump(package_json, package_file)
def _write_package_revision(pkg, revision): """ Write package revision locally. :param pkg: the package being installed :type pkg: Package :param revision: the package revision to install :type revision: str :rtype: None """ pkg_dir = package_dir(pkg.name()) revision_path = os.path.join(pkg_dir, 'version') with util.open_file(revision_path, 'w') as revision_file: revision_file.write(revision)
def _write_package_version(pkg, version): """ Write package version locally. :param pkg: the package being installed :type pkg: Package :param version: the package version to install :type version: str :rtype: None """ pkg_dir = package_dir(pkg.name()) version_path = os.path.join(pkg_dir, 'version') with util.open_file(version_path, 'w') as version_file: version_file.write(version)
def _write_package_json(pkg, pkg_dir): """ Write package.json locally. :param pkg: the package being installed :type pkg: PackageVersion :param pkg_dir: directory to install package :type pkg_dir: str :rtype: None """ package_path = os.path.join(pkg_dir, 'package.json') package_json = pkg.package_json() with util.open_file(package_path, 'w') as package_file: json.dump(package_json, package_file)
def _write_package_revision(pkg, revision): """ Write package revision locally. :param pkg: the package being installed :type pkg: Package :param revision: the package revision to install :type revision: str :rtype: None """ pkg_dir = package_dir(pkg.name()) revision_path = os.path.join(pkg_dir, "version") with util.open_file(revision_path, "w") as revision_file: revision_file.write(revision)
def save(toml_config, config_path=None): """ :param toml_config: TOML configuration object :type toml_config: MutableToml or Toml :param config_path: path to config to use :type config_path: str """ serial = toml.dumps(toml_config._dictionary) if config_path is None: config_path = get_config_path() util.ensure_file_exists(config_path) util.enforce_file_permissions(config_path) with util.open_file(config_path, 'w') as config_file: config_file.write(serial)
def get_index(self): """Retuprns the index of packages in this registry. :rtype: dict """ # The package index is found in $BASE/repo/meta/index.json index_path = os.path.join(self._base_path, "repo", "meta", "index.json") if not os.path.isfile(index_path): raise DCOSException("Path [{}] is not a file".format(index_path)) try: with util.open_file(index_path) as fd: return json.load(fd) except ValueError: raise DCOSException("Unable to parse [{}]".format(index_path))
def get_version(self): """Returns the version of this registry. :rtype: str """ # The package version is found in $BASE/repo/meta/version.json index_path = os.path.join(self._base_path, "repo", "meta", "version.json") if not os.path.isfile(index_path): raise DCOSException("Path [{}] is not a file".format(index_path)) try: with util.open_file(index_path) as fd: version_json = json.load(fd) return version_json.get("version") except ValueError: raise DCOSException("Unable to parse [{}]".format(index_path))
def _write_package_json(pkg, revision): """ Write package.json locally. :param pkg: the package being installed :type pkg: Package :param revision: the package revision to install :type revision: str :rtype: None """ pkg_dir = package_dir(pkg.name()) package_path = os.path.join(pkg_dir, "package.json") package_json = pkg.package_json(revision) with util.open_file(package_path, "w") as package_file: json.dump(package_json, package_file)
def load_from_path(path, mutable=False): """Loads a TOML file from the path :param path: Path to the TOML file :type path: str :param mutable: True if the returned Toml object should be mutable :type mutable: boolean :returns: Map for the configuration file :rtype: Toml | MutableToml """ util.ensure_file_exists(path) with util.open_file(path, 'r') as config_file: try: toml_obj = toml.loads(config_file.read()) except Exception as e: raise DCOSException('Error parsing config file at [{}]: {}'.format( path, e)) return (MutableToml if mutable else Toml)(toml_obj)
def load_from_path(path, mutable=False): """Loads a TOML file from the path :param path: Path to the TOML file :type path: str :param mutable: True if the returned Toml object should be mutable :type mutable: boolean :returns: Map for the configuration file :rtype: Toml | MutableToml """ util.ensure_file_exists(path) with util.open_file(path, 'r') as config_file: try: toml_obj = toml.loads(config_file.read()) except Exception as e: raise DCOSException( 'Error parsing config file at [{}]: {}'.format(path, e)) return (MutableToml if mutable else Toml)(toml_obj)
def get_index(self): """Retuprns the index of packages in this registry. :rtype: dict """ # The package index is found in $BASE/repo/meta/index.json index_path = os.path.join(self._base_path, 'repo', 'meta', 'index.json') if not os.path.isfile(index_path): raise DCOSException('Path [{}] is not a file'.format(index_path)) try: with util.open_file(index_path) as fd: return json.load(fd) except ValueError: logger.exception('Unable to parse JSON: %s', index_path) raise DCOSException('Unable to parse [{}]'.format(index_path))
def _get_resource(resource): """ :param resource: optional filename for the application or group resource :type resource: str :returns: resource :rtype: dict """ if resource is not None: with util.open_file(resource) as resource_file: return util.load_json(resource_file) # Check that stdin is not tty if sys.stdin.isatty(): # We don't support TTY right now. In the future we will start an # editor raise DCOSException( "We currently don't support reading from the TTY. Please " "specify an application JSON.\n" "Usage: dcos app add < app_resource.json") return util.load_json(sys.stdin)
def get_version(self): """Returns the version of this registry. :rtype: str """ # The package version is found in $BASE/repo/meta/version.json index_path = os.path.join(self._base_path, 'repo', 'meta', 'version.json') if not os.path.isfile(index_path): raise DCOSException('Path [{}] is not a file'.format(index_path)) try: with util.open_file(index_path) as fd: version_json = json.load(fd) return version_json.get('version') except ValueError: logger.exception('Unable to parse JSON: %s', index_path) raise DCOSException('Unable to parse [{}]'.format(index_path))
def _get_resource(resource): """ :param resource: optional filename or http(s) url for the application or group resource :type resource: str :returns: resource :rtype: dict """ if resource is not None: if os.path.isfile(resource): with util.open_file(resource) as resource_file: return util.load_json(resource_file) else: try: http.silence_requests_warnings() req = http.get(resource) if req.status_code == 200: data = b'' for chunk in req.iter_content(1024): data += chunk return util.load_jsons(data.decode('utf-8')) else: raise DCOSHTTPException("HTTP error code: {}" .format(req.status_code)) except Exception: logger.exception('Cannot read from resource %s', resource) raise DCOSException( "Can't read from resource: {0}.\n" "Please check that it exists.".format(resource)) # Check that stdin is not tty if sys.stdin.isatty(): # We don't support TTY right now. In the future we will start an # editor raise DCOSException( "We currently don't support reading from the TTY. Please " "specify an application JSON.\n" "E.g.: dcos job add < app_resource.json") return util.load_json(sys.stdin)
def get_index(self): """Returns the index of packages in this registry. :rtype: dict """ # The package index is found in $BASE/repo/meta/index.json index_path = os.path.join( self._base_path, 'repo', 'meta', 'index.json') if not os.path.isfile(index_path): raise DCOSException('Path [{}] is not a file'.format(index_path)) try: with util.open_file(index_path) as fd: return json.load(fd) except ValueError: logger.exception('Unable to parse JSON: %s', index_path) raise DCOSException('Unable to parse [{}]'.format(index_path))
def get_version(self): """Returns the version of this registry. :rtype: str """ # The package version is found in $BASE/repo/meta/version.json index_path = os.path.join( self._base_path, 'repo', 'meta', 'version.json') if not os.path.isfile(index_path): raise DCOSException('Path [{}] is not a file'.format(index_path)) try: with util.open_file(index_path) as fd: version_json = json.load(fd) return version_json.get('version') except ValueError: logger.exception('Unable to parse JSON: %s', index_path) raise DCOSException('Unable to parse [{}]'.format(index_path))
def _build(output_json, build_definition, output_directory): """ Creates a DC/OS Package from a DC/OS Package Build Definition :param output_json: whether to output json :type output_json: None | bool :param build_definition: The path to a DC/OS Package Build Definition :type build_definition: str :param output_directory: The directory where the DC/OS Package will be stored :type output_directory: str :returns: The process status :rtype: int """ # get the path of the build definition cwd = os.getcwd() build_definition_path = build_definition if not os.path.isabs(build_definition_path): build_definition_path = os.path.join(cwd, build_definition_path) build_definition_directory = os.path.dirname(build_definition_path) if not os.path.exists(build_definition_path): raise DCOSException( "The file [{}] does not exist".format(build_definition_path)) # get the path to the output directory if output_directory is None: output_directory = cwd if not os.path.exists(output_directory): raise DCOSException("The output directory [{}]" " does not exist".format(output_directory)) logger.debug("Using [%s] as output directory", output_directory) # load raw build definition with util.open_file(build_definition_path) as bd: build_definition_raw = util.load_json(bd, keep_order=True) # validate DC/OS Package Build Definition with local references build_definition_schema_path = "data/schemas/build-definition-schema.json" build_definition_schema = util.load_jsons( pkg_resources.resource_string("dcoscli", build_definition_schema_path).decode()) errs = util.validate_json(build_definition_raw, build_definition_schema) if errs: logger.debug("Failed before resolution: \n" "\tbuild definition: {}" "".format(build_definition_raw)) raise DCOSException(_validation_error(build_definition_path)) # resolve local references in build definition _resolve_local_references(build_definition_raw, build_definition_schema, build_definition_directory) # at this point all the local references have been resolved build_definition_resolved = build_definition_raw # validate resolved build definition metadata_schema_path = "data/schemas/metadata-schema.json" metadata_schema = util.load_jsons( pkg_resources.resource_string("dcoscli", metadata_schema_path).decode()) errs = util.validate_json(build_definition_resolved, metadata_schema) if errs: logger.debug("Failed after resolution: \n" "\tbuild definition: {}" "".format(build_definition_resolved)) raise DCOSException('Error validating package: ' 'there was a problem resolving ' 'the local references in ' '[{}]'.format(build_definition_path)) # create the manifest manifest_json = {'built-by': "dcoscli.version={}".format(dcoscli.version)} # create the metadata metadata_json = build_definition_resolved # create zip file with tempfile.NamedTemporaryFile() as temp_file: with zipfile.ZipFile(temp_file.file, mode='w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_file: metadata = json.dumps(metadata_json, indent=2).encode() zip_file.writestr("metadata.json", metadata) manifest = json.dumps(manifest_json, indent=2).encode() zip_file.writestr("manifest.json", manifest) # name the package appropriately temp_file.file.seek(0) dcos_package_name = '{}-{}-{}.dcos'.format( metadata_json['name'], metadata_json['version'], md5_hash_file(temp_file.file)) # get the dcos package path dcos_package_path = os.path.join(output_directory, dcos_package_name) if os.path.exists(dcos_package_path): raise DCOSException( 'Output file [{}] already exists'.format(dcos_package_path)) # create a new file to contain the package temp_file.file.seek(0) with util.open_file(dcos_package_path, 'w+b') as dcos_package: shutil.copyfileobj(temp_file.file, dcos_package) if output_json: message = {'package_path': dcos_package_path} else: message = 'Created DC/OS Universe Package [{}]'.format( dcos_package_path) emitter.publish(message) return 0
def _build(output_json, build_definition, output_directory): """ Creates a DC/OS Package from a DC/OS Package Build Definition :param output_json: whether to output json :type output_json: None | bool :param build_definition: The Path to a DC/OS package build definition :type build_definition: str :param output_directory: The directory where the DC/OS Package will be stored :type output_directory: str :returns: The process status :rtype: int """ # get the path of the build definition cwd = os.getcwd() build_definition_path = build_definition if not os.path.isabs(build_definition_path): build_definition_path = os.path.join(cwd, build_definition_path) build_definition_directory = os.path.dirname(build_definition_path) if not os.path.exists(build_definition_path): raise DCOSException( "The file [{}] does not exist".format(build_definition_path)) # get the path to the output directory if output_directory is None: output_directory = cwd if not os.path.exists(output_directory): raise DCOSException( "The output directory [{}]" " does not exist".format(output_directory)) logger.debug("Using [%s] as output directory", output_directory) # load raw build definition with util.open_file(build_definition_path) as bd: build_definition_raw = util.load_json(bd, keep_order=True) # validate DC/OS Package Build Definition with local references build_definition_schema_path = "data/schemas/build-definition-schema.json" build_definition_schema = util.load_jsons( pkg_resources.resource_string( "dcoscli", build_definition_schema_path).decode()) errs = util.validate_json(build_definition_raw, build_definition_schema) if errs: logger.debug("Failed before resolution: \n" "\tbuild definition: {}" "".format(build_definition_raw)) raise DCOSException(_validation_error(build_definition_path)) # resolve local references in build definition _resolve_local_references( build_definition_raw, build_definition_schema, build_definition_directory ) # at this point all the local references have been resolved build_definition_resolved = build_definition_raw # validate resolved build definition metadata_schema_path = "data/schemas/metadata-schema.json" metadata_schema = util.load_jsons( pkg_resources.resource_string( "dcoscli", metadata_schema_path).decode()) errs = util.validate_json(build_definition_resolved, metadata_schema) if errs: logger.debug("Failed after resolution: \n" "\tbuild definition: {}" "".format(build_definition_resolved)) raise DCOSException('Error validating package: ' 'there was a problem resolving ' 'the local references in ' '[{}]'.format(build_definition_path)) # create the manifest manifest_json = { 'built-by': "dcoscli.version={}".format(dcoscli.version) } # create the metadata metadata_json = build_definition_resolved # create zip file with tempfile.NamedTemporaryFile() as temp_file: with zipfile.ZipFile( temp_file.file, mode='w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_file: metadata = json.dumps(metadata_json, indent=2).encode() zip_file.writestr("metadata.json", metadata) manifest = json.dumps(manifest_json, indent=2).encode() zip_file.writestr("manifest.json", manifest) # name the package appropriately temp_file.file.seek(0) dcos_package_name = '{}-{}-{}.dcos'.format( metadata_json['name'], metadata_json['version'], md5_hash_file(temp_file.file)) # get the dcos package path dcos_package_path = os.path.join(output_directory, dcos_package_name) if os.path.exists(dcos_package_path): raise DCOSException( 'Output file [{}] already exists'.format( dcos_package_path)) # create a new file to contain the package temp_file.file.seek(0) with util.open_file(dcos_package_path, 'w+b') as dcos_package: shutil.copyfileobj(temp_file.file, dcos_package) if output_json: message = {'package_path': dcos_package_path} else: message = 'Created DC/OS Universe Package [{}]'.format( dcos_package_path) emitter.publish(message) return 0