def to_str(self, x):
if self.sub_value_type:
x = traverse(x, self.sub_value_type.to_str)
# we sort the set before we serialize!
x = sorted(x, key=lambda x: json_utils.dumps_canonical(x))
return json_utils.dumps_canonical(x)
def build_signature(name, params, extra=None):
"""
Returns a canonical string used to identify a particular task
:param name:
:param params: a list mapping parameter names to their serialized values
:return: A unique, shortened identifier corresponding to the family and params
"""
# task_id is a concatenation of task family,
# sorted by parameter name and a md5hash of the family/parameters as a cananocalised json.
params = {key: value for key, value in params}
signature_dict = {"name": name, "params": params}
if extra:
signature_dict["extra"] = extra
# we can't handle sets
signature_dict = traverse_frozen_set(signature_dict)
param_str = json_utils.dumps_canonical(signature_dict)
signature = user_friendly_signature(param_str)
return Signature(name=name,
signature=signature,
signature_source=param_str)
def run_cmd_java(self, jar, main_class):
options = self._get_options()
if "labels" in options:
options["labels"] = dumps_canonical(options["labels"])
if main_class:
cmd = ["java", "-cp", jar, main_class]
else:
cmd = ["java", "-jar", jar]
for attr, value in six.iteritems(options):
if value is None:
continue
cmd.append("--" + attr + "=" + str(value))
# run job
self._run_cmd(cmd)
def to_str(self, x):
# value should be the "deferred value" -> Target of any kind
return json_utils.dumps_canonical(x)
def _frozen_set(struct):
if isinstance(struct, set):
from dbnd._core.utils import json_utils
return sorted(struct, key=lambda x: json_utils.dumps_canonical(x))
return struct