def _decode_and_add_context(app_id, labels): """ Create an enhanced package JSON from Marathon labels { 'appId': <appId>, 'packageSource': <source>, 'registryVersion': <app_version>, 'releaseVersion': <release_version>, ..<package.json properties>.. } :param app_id: Marathon application id :type app_id: str :param labels: Marathon label dictionary :type labels: dict :rtype: dict """ encoded = labels.get(PACKAGE_METADATA_KEY, {}) decoded = base64.b64decode(six.b(encoded)).decode() decoded_json = util.load_jsons(decoded) decoded_json['appId'] = app_id decoded_json['packageSource'] = labels.get(PACKAGE_SOURCE_KEY) decoded_json['releaseVersion'] = labels.get(PACKAGE_RELEASE_KEY) return decoded_json
def _parse_properties(properties): """ :param properties: JSON items in the form key=value :type properties: [str] :returns: resource JSON :rtype: dict """ if len(properties) == 0: if sys.stdin.isatty(): # We don't support TTY right now. In the future we will start an # editor raise DCOSException( "We currently don't support reading from the TTY. Please " "specify an application JSON.\n" "E.g. dcos marathon app update your-app-id < app_update.json") else: return util.load_jsons(sys.stdin.read()) resource_json = {} for prop in properties: key, value = jsonitem.parse_json_item(prop, None) key = jsonitem.clean_value(key) if key in resource_json: raise DCOSException( 'Key {!r} was specified more than once'.format(key)) resource_json[key] = value return resource_json
def get_resource(resource): """ :param resource: optional filename or http(s) url for the application or group resource :type resource: str :returns: resource :rtype: dict """ if resource is not None: if os.path.isfile(resource): with util.open_file(resource) as resource_file: return util.load_json(resource_file) else: try: http.silence_requests_warnings() req = http.get(resource) if req.status_code == 200: data = b'' for chunk in req.iter_content(1024): data += chunk return util.load_jsons(data.decode('utf-8')) else: raise Exception except Exception: raise DCOSException( "Can't read from resource: {0}.\n" "Please check that it exists.".format(resource))
def _validate_json_file(fullpath): """Validates the content of the file against its schema. Throws an exception if the file is not valid. :param fullpath: full path to the file. :type fullpath: str :return: json object if it is a special file :rtype: dict """ filename = os.path.basename(fullpath) if filename in ['command.json', 'config.json', 'package.json']: schema_path = 'data/universe-schema/{}'.format(filename) else: raise DCOSException( ('Error bundling package. Unknown file in package ' 'directory [{}]').format(fullpath)) special_schema = util.load_jsons( pkg_resources.resource_string('dcoscli', schema_path).decode('utf-8')) with util.open_file(fullpath) as special_file: special_json = util.load_json(special_file) errs = util.validate_json(special_json, special_schema) if errs: emitter.publish( errors.DefaultError( 'Error validating JSON file [{}]'.format(fullpath))) raise DCOSException(util.list_to_err(errs)) return special_json
def get_resource_from_properties(properties): """ :param properties: JSON items in the form key=value :type properties: [str] :returns: resource JSON :rtype: dict """ if len(properties) == 0: example =\ "E.g. dcos marathon app update your-app-id < app_update.json" ResourceReader._assert_no_tty(example) return util.load_jsons(sys.stdin.read()) resource_json = {} for prop in properties: key, value = jsonitem.parse_json_item(prop, None) key = jsonitem.clean_value(key) if key in resource_json: raise DCOSException( 'Key {!r} was specified more than once'.format(key)) resource_json[key] = value return resource_json
def get_resource(name): """ :param name: optional filename or http(s) url for the application or group resource :type name: str | None :returns: resource :rtype: dict """ if name is not None: if os.path.isfile(name): with util.open_file(name) as resource_file: return util.load_json(resource_file) else: try: http.silence_requests_warnings() req = http.get(name) if req.status_code == 200: data = b'' for chunk in req.iter_content(1024): data += chunk return util.load_jsons(data.decode('utf-8')) else: raise Exception except Exception: logger.exception('Cannot read from resource %s', name) raise DCOSException( "Can't read from resource: {0}.\n" "Please check that it exists.".format(name)) example = "E.g.: dcos marathon app add < app_resource.json" ResourceReader._assert_no_tty(example) return util.load_json(sys.stdin)
def decode_and_add_context(pair): app_id, labels = pair encoded = labels.get(PACKAGE_METADATA_KEY, {}) decoded = base64.b64decode(six.b(encoded)).decode() decoded_json = util.load_jsons(decoded) decoded_json["appId"] = app_id decoded_json["packageSource"] = labels.get(PACKAGE_SOURCE_KEY) decoded_json["releaseVersion"] = labels.get(PACKAGE_RELEASE_KEY) return decoded_json
def _json(self, path): """Returns the json content of the supplied file, relative to the base path. :param path: The relative path to the file to read :type path: str :rtype: dict """ data = self._data(path) return util.load_jsons(data)
def command_json(self, options): """Returns the JSON content of the command.json template, after rendering it with options. :param options: the template options to use in rendering :type options: dict :returns: Package data :rtype: dict """ rendered = pystache.render(json.dumps(self._command_json), options) return util.load_jsons(rendered)
def _json(self, revision, name): """Returns the json content of the file named `name` in the directory named `revision` :param revision: the package revision :type revision: str :param name: file name :type name: str :rtype: dict """ data = self._data(revision, name) return util.load_jsons(data)
def _read_http_response_body(http_response): """ Get an requests HTTP response, read it and deserialize to json. :param http_response: http response :type http_response: requests.Response onject :return: deserialized json :rtype: dict """ data = b'' try: for chunk in http_response.iter_content(1024): data += chunk bundle_response = util.load_jsons(data.decode('utf-8')) return bundle_response except DCOSException: raise
def _get_resource(resource): """ :param resource: optional filename or http(s) url for the application or group resource :type resource: str :returns: resource :rtype: dict """ if resource is not None: if os.path.isfile(resource): with util.open_file(resource) as resource_file: return util.load_json(resource_file) else: try: http.silence_requests_warnings() req = http.get(resource) if req.status_code == 200: data = b'' for chunk in req.iter_content(1024): data += chunk return util.load_jsons(data.decode('utf-8')) else: raise DCOSHTTPException("HTTP error code: {}" .format(req.status_code)) except Exception: logger.exception('Cannot read from resource %s', resource) raise DCOSException( "Can't read from resource: {0}.\n" "Please check that it exists.".format(resource)) # Check that stdin is not tty if sys.stdin.isatty(): # We don't support TTY right now. In the future we will start an # editor raise DCOSException( "We currently don't support reading from the TTY. Please " "specify an application JSON.\n" "E.g.: dcos job add < app_resource.json") return util.load_json(sys.stdin)
def _build(output_json, build_definition, output_directory): """ Creates a DC/OS Package from a DC/OS Package Build Definition :param output_json: whether to output json :type output_json: None | bool :param build_definition: The path to a DC/OS Package Build Definition :type build_definition: str :param output_directory: The directory where the DC/OS Package will be stored :type output_directory: str :returns: The process status :rtype: int """ # get the path of the build definition cwd = os.getcwd() build_definition_path = build_definition if not os.path.isabs(build_definition_path): build_definition_path = os.path.join(cwd, build_definition_path) build_definition_directory = os.path.dirname(build_definition_path) if not os.path.exists(build_definition_path): raise DCOSException( "The file [{}] does not exist".format(build_definition_path)) # get the path to the output directory if output_directory is None: output_directory = cwd if not os.path.exists(output_directory): raise DCOSException("The output directory [{}]" " does not exist".format(output_directory)) logger.debug("Using [%s] as output directory", output_directory) # load raw build definition with util.open_file(build_definition_path) as bd: build_definition_raw = util.load_json(bd, keep_order=True) # validate DC/OS Package Build Definition with local references build_definition_schema_path = "data/schemas/build-definition-schema.json" build_definition_schema = util.load_jsons( pkg_resources.resource_string("dcoscli", build_definition_schema_path).decode()) errs = util.validate_json(build_definition_raw, build_definition_schema) if errs: logger.debug("Failed before resolution: \n" "\tbuild definition: {}" "".format(build_definition_raw)) raise DCOSException(_validation_error(build_definition_path)) # resolve local references in build definition _resolve_local_references(build_definition_raw, build_definition_schema, build_definition_directory) # at this point all the local references have been resolved build_definition_resolved = build_definition_raw # validate resolved build definition metadata_schema_path = "data/schemas/metadata-schema.json" metadata_schema = util.load_jsons( pkg_resources.resource_string("dcoscli", metadata_schema_path).decode()) errs = util.validate_json(build_definition_resolved, metadata_schema) if errs: logger.debug("Failed after resolution: \n" "\tbuild definition: {}" "".format(build_definition_resolved)) raise DCOSException('Error validating package: ' 'there was a problem resolving ' 'the local references in ' '[{}]'.format(build_definition_path)) # create the manifest manifest_json = {'built-by': "dcoscli.version={}".format(dcoscli.version)} # create the metadata metadata_json = build_definition_resolved # create zip file with tempfile.NamedTemporaryFile() as temp_file: with zipfile.ZipFile(temp_file.file, mode='w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_file: metadata = json.dumps(metadata_json, indent=2).encode() zip_file.writestr("metadata.json", metadata) manifest = json.dumps(manifest_json, indent=2).encode() zip_file.writestr("manifest.json", manifest) # name the package appropriately temp_file.file.seek(0) dcos_package_name = '{}-{}-{}.dcos'.format( metadata_json['name'], metadata_json['version'], md5_hash_file(temp_file.file)) # get the dcos package path dcos_package_path = os.path.join(output_directory, dcos_package_name) if os.path.exists(dcos_package_path): raise DCOSException( 'Output file [{}] already exists'.format(dcos_package_path)) # create a new file to contain the package temp_file.file.seek(0) with util.open_file(dcos_package_path, 'w+b') as dcos_package: shutil.copyfileobj(temp_file.file, dcos_package) if output_json: message = {'package_path': dcos_package_path} else: message = 'Created DC/OS Universe Package [{}]'.format( dcos_package_path) emitter.publish(message) return 0
def _build(output_json, build_definition, output_directory): """ Creates a DC/OS Package from a DC/OS Package Build Definition :param output_json: whether to output json :type output_json: None | bool :param build_definition: The Path to a DC/OS package build definition :type build_definition: str :param output_directory: The directory where the DC/OS Package will be stored :type output_directory: str :returns: The process status :rtype: int """ # get the path of the build definition cwd = os.getcwd() build_definition_path = build_definition if not os.path.isabs(build_definition_path): build_definition_path = os.path.join(cwd, build_definition_path) build_definition_directory = os.path.dirname(build_definition_path) if not os.path.exists(build_definition_path): raise DCOSException( "The file [{}] does not exist".format(build_definition_path)) # get the path to the output directory if output_directory is None: output_directory = cwd if not os.path.exists(output_directory): raise DCOSException( "The output directory [{}]" " does not exist".format(output_directory)) logger.debug("Using [%s] as output directory", output_directory) # load raw build definition with util.open_file(build_definition_path) as bd: build_definition_raw = util.load_json(bd, keep_order=True) # validate DC/OS Package Build Definition with local references build_definition_schema_path = "data/schemas/build-definition-schema.json" build_definition_schema = util.load_jsons( pkg_resources.resource_string( "dcoscli", build_definition_schema_path).decode()) errs = util.validate_json(build_definition_raw, build_definition_schema) if errs: logger.debug("Failed before resolution: \n" "\tbuild definition: {}" "".format(build_definition_raw)) raise DCOSException(_validation_error(build_definition_path)) # resolve local references in build definition _resolve_local_references( build_definition_raw, build_definition_schema, build_definition_directory ) # at this point all the local references have been resolved build_definition_resolved = build_definition_raw # validate resolved build definition metadata_schema_path = "data/schemas/metadata-schema.json" metadata_schema = util.load_jsons( pkg_resources.resource_string( "dcoscli", metadata_schema_path).decode()) errs = util.validate_json(build_definition_resolved, metadata_schema) if errs: logger.debug("Failed after resolution: \n" "\tbuild definition: {}" "".format(build_definition_resolved)) raise DCOSException('Error validating package: ' 'there was a problem resolving ' 'the local references in ' '[{}]'.format(build_definition_path)) # create the manifest manifest_json = { 'built-by': "dcoscli.version={}".format(dcoscli.version) } # create the metadata metadata_json = build_definition_resolved # create zip file with tempfile.NamedTemporaryFile() as temp_file: with zipfile.ZipFile( temp_file.file, mode='w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_file: metadata = json.dumps(metadata_json, indent=2).encode() zip_file.writestr("metadata.json", metadata) manifest = json.dumps(manifest_json, indent=2).encode() zip_file.writestr("manifest.json", manifest) # name the package appropriately temp_file.file.seek(0) dcos_package_name = '{}-{}-{}.dcos'.format( metadata_json['name'], metadata_json['version'], md5_hash_file(temp_file.file)) # get the dcos package path dcos_package_path = os.path.join(output_directory, dcos_package_name) if os.path.exists(dcos_package_path): raise DCOSException( 'Output file [{}] already exists'.format( dcos_package_path)) # create a new file to contain the package temp_file.file.seek(0) with util.open_file(dcos_package_path, 'w+b') as dcos_package: shutil.copyfileobj(temp_file.file, dcos_package) if output_json: message = {'package_path': dcos_package_path} else: message = 'Created DC/OS Universe Package [{}]'.format( dcos_package_path) emitter.publish(message) return 0