def id_for_memo_list(denormalized_list, output_ref=False): if type(denormalized_list) != list: raise ValueError("id_for_memo_list cannot work on subclasses of list") normalized_list = [] for e in denormalized_list: normalized_list.append(id_for_memo(e, output_ref=output_ref)) return serialize_object(normalized_list)[0]
def make_hash(self, task): ''' Create a hash of the task inputs. This uses a serialization library borrowed from ipyparallel. If this fails here, then all ipp calls are also likely to fail due to failure at serialization. Args: - task (dict) : Task dictionary from dfk.tasks Returns: - hash (str) : A unique hash string ''' # Function name TODO: Add fn body later t = [serialize_object(task['func_name'])[0], serialize_object(task['fn_hash'])[0], serialize_object(task['args'])[0], serialize_object(task['kwargs'])[0], serialize_object(task['env'])[0]] x = b''.join(t) hashedsum = hashlib.md5(x).hexdigest() return hashedsum
def id_for_memo_dict(denormalized_dict, output_ref=False): """This normalises the keys and values of the supplied dictionary. When output_ref=True, the values are normalised as output refs, but the keys are not. """ if type(denormalized_dict) != dict: raise ValueError("id_for_memo_dict cannot work on subclasses of dict") keys = sorted(denormalized_dict) normalized_list = [] for k in keys: normalized_list.append(id_for_memo(k)) normalized_list.append( id_for_memo(denormalized_dict[k], output_ref=output_ref)) return serialize_object(normalized_list)[0]
def id_for_memo_serialize(obj, output_ref=False): return serialize_object(obj)[0]