def _decode_and_add_context(app_id, labels):
""" Create an enhanced package JSON from Marathon labels
{
'appId': <appId>,
'packageSource': <source>,
'registryVersion': <app_version>,
'releaseVersion': <release_version>,
..<package.json properties>..
}
:param app_id: Marathon application id
:type app_id: str
:param labels: Marathon label dictionary
:type labels: dict
:rtype: dict
"""
encoded = labels.get(PACKAGE_METADATA_KEY, {})
decoded = base64.b64decode(six.b(encoded)).decode()
decoded_json = util.load_jsons(decoded)
decoded_json['appId'] = app_id
decoded_json['packageSource'] = labels.get(PACKAGE_SOURCE_KEY)
decoded_json['releaseVersion'] = labels.get(PACKAGE_RELEASE_KEY)
return decoded_json
def _parse_properties(properties):
"""
:param properties: JSON items in the form key=value
:type properties: [str]
:returns: resource JSON
:rtype: dict
"""
if len(properties) == 0:
if sys.stdin.isatty():
# We don't support TTY right now. In the future we will start an
# editor
raise DCOSException(
"We currently don't support reading from the TTY. Please "
"specify an application JSON.\n"
"E.g. dcos marathon app update your-app-id < app_update.json")
else:
return util.load_jsons(sys.stdin.read())
resource_json = {}
for prop in properties:
key, value = jsonitem.parse_json_item(prop, None)
key = jsonitem.clean_value(key)
if key in resource_json:
raise DCOSException(
'Key {!r} was specified more than once'.format(key))
resource_json[key] = value
return resource_json
def get_resource(resource):
"""
:param resource: optional filename or http(s) url
for the application or group resource
:type resource: str
:returns: resource
:rtype: dict
"""
if resource is not None:
if os.path.isfile(resource):
with util.open_file(resource) as resource_file:
return util.load_json(resource_file)
else:
try:
http.silence_requests_warnings()
req = http.get(resource)
if req.status_code == 200:
data = b''
for chunk in req.iter_content(1024):
data += chunk
return util.load_jsons(data.decode('utf-8'))
else:
raise Exception
except Exception:
raise DCOSException(
"Can't read from resource: {0}.\n"
"Please check that it exists.".format(resource))
def _validate_json_file(fullpath):
"""Validates the content of the file against its schema. Throws an
exception if the file is not valid.
:param fullpath: full path to the file.
:type fullpath: str
:return: json object if it is a special file
:rtype: dict
"""
filename = os.path.basename(fullpath)
if filename in ['command.json', 'config.json', 'package.json']:
schema_path = 'data/universe-schema/{}'.format(filename)
else:
raise DCOSException(
('Error bundling package. Unknown file in package '
'directory [{}]').format(fullpath))
special_schema = util.load_jsons(
pkg_resources.resource_string('dcoscli', schema_path).decode('utf-8'))
with util.open_file(fullpath) as special_file:
special_json = util.load_json(special_file)
errs = util.validate_json(special_json, special_schema)
if errs:
emitter.publish(
errors.DefaultError(
'Error validating JSON file [{}]'.format(fullpath)))
raise DCOSException(util.list_to_err(errs))
return special_json
def get_resource_from_properties(properties):
"""
:param properties: JSON items in the form key=value
:type properties: [str]
:returns: resource JSON
:rtype: dict
"""
if len(properties) == 0:
example =\
"E.g. dcos marathon app update your-app-id < app_update.json"
ResourceReader._assert_no_tty(example)
return util.load_jsons(sys.stdin.read())
resource_json = {}
for prop in properties:
key, value = jsonitem.parse_json_item(prop, None)
key = jsonitem.clean_value(key)
if key in resource_json:
raise DCOSException(
'Key {!r} was specified more than once'.format(key))
resource_json[key] = value
return resource_json
def get_resource(name):
"""
:param name: optional filename or http(s) url
for the application or group resource
:type name: str | None
:returns: resource
:rtype: dict
"""
if name is not None:
if os.path.isfile(name):
with util.open_file(name) as resource_file:
return util.load_json(resource_file)
else:
try:
http.silence_requests_warnings()
req = http.get(name)
if req.status_code == 200:
data = b''
for chunk in req.iter_content(1024):
data += chunk
return util.load_jsons(data.decode('utf-8'))
else:
raise Exception
except Exception:
logger.exception('Cannot read from resource %s', name)
raise DCOSException(
"Can't read from resource: {0}.\n"
"Please check that it exists.".format(name))
example = "E.g.: dcos marathon app add < app_resource.json"
ResourceReader._assert_no_tty(example)
return util.load_json(sys.stdin)
def decode_and_add_context(pair):
app_id, labels = pair
encoded = labels.get(PACKAGE_METADATA_KEY, {})
decoded = base64.b64decode(six.b(encoded)).decode()
decoded_json = util.load_jsons(decoded)
decoded_json["appId"] = app_id
decoded_json["packageSource"] = labels.get(PACKAGE_SOURCE_KEY)
decoded_json["releaseVersion"] = labels.get(PACKAGE_RELEASE_KEY)
return decoded_json
def _json(self, path):
"""Returns the json content of the supplied file, relative to the
base path.
:param path: The relative path to the file to read
:type path: str
:rtype: dict
"""
data = self._data(path)
return util.load_jsons(data)
def command_json(self, options):
"""Returns the JSON content of the command.json template, after
rendering it with options.
:param options: the template options to use in rendering
:type options: dict
:returns: Package data
:rtype: dict
"""
rendered = pystache.render(json.dumps(self._command_json), options)
return util.load_jsons(rendered)
def _json(self, revision, name):
"""Returns the json content of the file named `name` in the directory
named `revision`
:param revision: the package revision
:type revision: str
:param name: file name
:type name: str
:rtype: dict
"""
data = self._data(revision, name)
return util.load_jsons(data)
def _read_http_response_body(http_response):
"""
Get an requests HTTP response, read it and deserialize to json.
:param http_response: http response
:type http_response: requests.Response onject
:return: deserialized json
:rtype: dict
"""
data = b''
try:
for chunk in http_response.iter_content(1024):
data += chunk
bundle_response = util.load_jsons(data.decode('utf-8'))
return bundle_response
except DCOSException:
raise
def _get_resource(resource):
"""
:param resource: optional filename or http(s) url
for the application or group resource
:type resource: str
:returns: resource
:rtype: dict
"""
if resource is not None:
if os.path.isfile(resource):
with util.open_file(resource) as resource_file:
return util.load_json(resource_file)
else:
try:
http.silence_requests_warnings()
req = http.get(resource)
if req.status_code == 200:
data = b''
for chunk in req.iter_content(1024):
data += chunk
return util.load_jsons(data.decode('utf-8'))
else:
raise DCOSHTTPException("HTTP error code: {}"
.format(req.status_code))
except Exception:
logger.exception('Cannot read from resource %s', resource)
raise DCOSException(
"Can't read from resource: {0}.\n"
"Please check that it exists.".format(resource))
# Check that stdin is not tty
if sys.stdin.isatty():
# We don't support TTY right now. In the future we will start an
# editor
raise DCOSException(
"We currently don't support reading from the TTY. Please "
"specify an application JSON.\n"
"E.g.: dcos job add < app_resource.json")
return util.load_json(sys.stdin)
def _get_resource(resource):
"""
:param resource: optional filename or http(s) url
for the application or group resource
:type resource: str
:returns: resource
:rtype: dict
"""
if resource is not None:
if os.path.isfile(resource):
with util.open_file(resource) as resource_file:
return util.load_json(resource_file)
else:
try:
http.silence_requests_warnings()
req = http.get(resource)
if req.status_code == 200:
data = b''
for chunk in req.iter_content(1024):
data += chunk
return util.load_jsons(data.decode('utf-8'))
else:
raise DCOSHTTPException("HTTP error code: {}"
.format(req.status_code))
except Exception:
logger.exception('Cannot read from resource %s', resource)
raise DCOSException(
"Can't read from resource: {0}.\n"
"Please check that it exists.".format(resource))
# Check that stdin is not tty
if sys.stdin.isatty():
# We don't support TTY right now. In the future we will start an
# editor
raise DCOSException(
"We currently don't support reading from the TTY. Please "
"specify an application JSON.\n"
"E.g.: dcos job add < app_resource.json")
return util.load_json(sys.stdin)
def _build(output_json, build_definition, output_directory):
""" Creates a DC/OS Package from a DC/OS Package Build Definition
:param output_json: whether to output json
:type output_json: None | bool
:param build_definition: The path to a DC/OS Package Build Definition
:type build_definition: str
:param output_directory: The directory where the DC/OS Package
will be stored
:type output_directory: str
:returns: The process status
:rtype: int
"""
# get the path of the build definition
cwd = os.getcwd()
build_definition_path = build_definition
if not os.path.isabs(build_definition_path):
build_definition_path = os.path.join(cwd, build_definition_path)
build_definition_directory = os.path.dirname(build_definition_path)
if not os.path.exists(build_definition_path):
raise DCOSException(
"The file [{}] does not exist".format(build_definition_path))
# get the path to the output directory
if output_directory is None:
output_directory = cwd
if not os.path.exists(output_directory):
raise DCOSException("The output directory [{}]"
" does not exist".format(output_directory))
logger.debug("Using [%s] as output directory", output_directory)
# load raw build definition
with util.open_file(build_definition_path) as bd:
build_definition_raw = util.load_json(bd, keep_order=True)
# validate DC/OS Package Build Definition with local references
build_definition_schema_path = "data/schemas/build-definition-schema.json"
build_definition_schema = util.load_jsons(
pkg_resources.resource_string("dcoscli",
build_definition_schema_path).decode())
errs = util.validate_json(build_definition_raw, build_definition_schema)
if errs:
logger.debug("Failed before resolution: \n"
"\tbuild definition: {}"
"".format(build_definition_raw))
raise DCOSException(_validation_error(build_definition_path))
# resolve local references in build definition
_resolve_local_references(build_definition_raw, build_definition_schema,
build_definition_directory)
# at this point all the local references have been resolved
build_definition_resolved = build_definition_raw
# validate resolved build definition
metadata_schema_path = "data/schemas/metadata-schema.json"
metadata_schema = util.load_jsons(
pkg_resources.resource_string("dcoscli",
metadata_schema_path).decode())
errs = util.validate_json(build_definition_resolved, metadata_schema)
if errs:
logger.debug("Failed after resolution: \n"
"\tbuild definition: {}"
"".format(build_definition_resolved))
raise DCOSException('Error validating package: '
'there was a problem resolving '
'the local references in '
'[{}]'.format(build_definition_path))
# create the manifest
manifest_json = {'built-by': "dcoscli.version={}".format(dcoscli.version)}
# create the metadata
metadata_json = build_definition_resolved
# create zip file
with tempfile.NamedTemporaryFile() as temp_file:
with zipfile.ZipFile(temp_file.file,
mode='w',
compression=zipfile.ZIP_DEFLATED,
allowZip64=True) as zip_file:
metadata = json.dumps(metadata_json, indent=2).encode()
zip_file.writestr("metadata.json", metadata)
manifest = json.dumps(manifest_json, indent=2).encode()
zip_file.writestr("manifest.json", manifest)
# name the package appropriately
temp_file.file.seek(0)
dcos_package_name = '{}-{}-{}.dcos'.format(
metadata_json['name'], metadata_json['version'],
md5_hash_file(temp_file.file))
# get the dcos package path
dcos_package_path = os.path.join(output_directory, dcos_package_name)
if os.path.exists(dcos_package_path):
raise DCOSException(
'Output file [{}] already exists'.format(dcos_package_path))
# create a new file to contain the package
temp_file.file.seek(0)
with util.open_file(dcos_package_path, 'w+b') as dcos_package:
shutil.copyfileobj(temp_file.file, dcos_package)
if output_json:
message = {'package_path': dcos_package_path}
else:
message = 'Created DC/OS Universe Package [{}]'.format(
dcos_package_path)
emitter.publish(message)
return 0
def _build(output_json,
build_definition,
output_directory):
""" Creates a DC/OS Package from a DC/OS Package Build Definition
:param output_json: whether to output json
:type output_json: None | bool
:param build_definition: The Path to a DC/OS package build definition
:type build_definition: str
:param output_directory: The directory where the DC/OS Package
will be stored
:type output_directory: str
:returns: The process status
:rtype: int
"""
# get the path of the build definition
cwd = os.getcwd()
build_definition_path = build_definition
if not os.path.isabs(build_definition_path):
build_definition_path = os.path.join(cwd, build_definition_path)
build_definition_directory = os.path.dirname(build_definition_path)
if not os.path.exists(build_definition_path):
raise DCOSException(
"The file [{}] does not exist".format(build_definition_path))
# get the path to the output directory
if output_directory is None:
output_directory = cwd
if not os.path.exists(output_directory):
raise DCOSException(
"The output directory [{}]"
" does not exist".format(output_directory))
logger.debug("Using [%s] as output directory", output_directory)
# load raw build definition
with util.open_file(build_definition_path) as bd:
build_definition_raw = util.load_json(bd, keep_order=True)
# validate DC/OS Package Build Definition with local references
build_definition_schema_path = "data/schemas/build-definition-schema.json"
build_definition_schema = util.load_jsons(
pkg_resources.resource_string(
"dcoscli", build_definition_schema_path).decode())
errs = util.validate_json(build_definition_raw, build_definition_schema)
if errs:
logger.debug("Failed before resolution: \n"
"\tbuild definition: {}"
"".format(build_definition_raw))
raise DCOSException(_validation_error(build_definition_path))
# resolve local references in build definition
_resolve_local_references(
build_definition_raw,
build_definition_schema,
build_definition_directory
)
# at this point all the local references have been resolved
build_definition_resolved = build_definition_raw
# validate resolved build definition
metadata_schema_path = "data/schemas/metadata-schema.json"
metadata_schema = util.load_jsons(
pkg_resources.resource_string(
"dcoscli", metadata_schema_path).decode())
errs = util.validate_json(build_definition_resolved, metadata_schema)
if errs:
logger.debug("Failed after resolution: \n"
"\tbuild definition: {}"
"".format(build_definition_resolved))
raise DCOSException('Error validating package: '
'there was a problem resolving '
'the local references in '
'[{}]'.format(build_definition_path))
# create the manifest
manifest_json = {
'built-by': "dcoscli.version={}".format(dcoscli.version)
}
# create the metadata
metadata_json = build_definition_resolved
# create zip file
with tempfile.NamedTemporaryFile() as temp_file:
with zipfile.ZipFile(
temp_file.file,
mode='w',
compression=zipfile.ZIP_DEFLATED,
allowZip64=True) as zip_file:
metadata = json.dumps(metadata_json, indent=2).encode()
zip_file.writestr("metadata.json", metadata)
manifest = json.dumps(manifest_json, indent=2).encode()
zip_file.writestr("manifest.json", manifest)
# name the package appropriately
temp_file.file.seek(0)
dcos_package_name = '{}-{}-{}.dcos'.format(
metadata_json['name'],
metadata_json['version'],
md5_hash_file(temp_file.file))
# get the dcos package path
dcos_package_path = os.path.join(output_directory, dcos_package_name)
if os.path.exists(dcos_package_path):
raise DCOSException(
'Output file [{}] already exists'.format(
dcos_package_path))
# create a new file to contain the package
temp_file.file.seek(0)
with util.open_file(dcos_package_path, 'w+b') as dcos_package:
shutil.copyfileobj(temp_file.file, dcos_package)
if output_json:
message = {'package_path': dcos_package_path}
else:
message = 'Created DC/OS Universe Package [{}]'.format(
dcos_package_path)
emitter.publish(message)
return 0
