def reverse_opt_map(opt_map):
"""Reverse the key/value pairs of the option map in the interface classes.
Parameters
----------
opt_map : dict
Dictionary mapping the attribute name to a command line flag.
Each interface class defines these for the command it wraps.
Returns
-------
rev_opt_map : dict
Dictionary mapping the flags to the attribute name.
"""
# For docs, we only care about the mapping from our attribute
# names to the command-line flags. The 'v.split()[0]' below
# strips off the string format characters.
# if (k != 'flags' and v) , key must not be flags as it is generic,
# v must not be None or it cannot be parsed by this line
revdict = {}
for key, value in opt_map.items():
if is_container(value):
# The value is a tuple where the first element is the
# format string and the second element is a docstring.
value = value[0]
if (key != 'flags' and value is not None):
revdict[value.split()[0]] = key
return revdict
def _hash_infile(self, adict, key):
""" Inject file hashes into adict[key]"""
stuff = adict[key]
if not is_container(stuff):
stuff = [stuff]
file_list = []
for afile in stuff:
if is_container(afile):
hashlist = self._hash_infile({"infiles": afile}, "infiles")
hash = [val[1] for val in hashlist]
else:
if config.get("execution", "hash_method").lower() == "timestamp":
hash = hash_timestamp(afile)
elif config.get("execution", "hash_method").lower() == "content":
hash = hash_infile(afile)
else:
raise Exception("Unknown hash method: %s" % config.get("execution", "hash_method"))
file_list.append((afile, hash))
return file_list
def filename_to_list(filename):
"""Returns a list given either a string or a list
"""
if isinstance(filename,(str, unicode)):
return [filename]
elif isinstance(filename,list):
return filename
elif is_container(filename):
return [x for x in filename]
else:
return None
def filename_to_list(filename):
"""Returns a list given either a string or a list
"""
if isinstance(filename, (str, unicode)):
return [filename]
elif isinstance(filename, list):
return filename
elif is_container(filename):
return [x for x in filename]
else:
return None
def _get_bunch_hash(self):
"""Return a dictionary of our items with hashes for each file.
Searches through dictionary items and if an item is a file, it
calculates the md5 hash of the file contents and stores the
file name and hash value as the new key value.
However, the overall bunch hash is calculated only on the hash
value of a file. The path and name of the file are not used in
the overall hash calculation.
Returns
-------
dict_withhash : dict
Copy of our dictionary with the new file hashes included
with each file.
hashvalue : str
The md5 hash value of the `dict_withhash`
"""
infile_list = []
for key, val in self.items():
if is_container(val):
# XXX - SG this probably doesn't catch numpy arrays
# containing embedded file names either.
if isinstance(val, dict):
# XXX - SG should traverse dicts, but ignoring for now
item = None
else:
if len(val) == 0:
raise AttributeError("%s attribute is empty" % key)
item = val[0]
else:
item = val
try:
if os.path.isfile(item):
infile_list.append(key)
except TypeError:
# `item` is not a file or string.
continue
dict_withhash = self.dictcopy()
dict_nofilename = self.dictcopy()
for item in infile_list:
dict_withhash[item] = self._hash_infile(dict_withhash, item)
dict_nofilename[item] = [val[1] for val in dict_withhash[item]]
# Sort the items of the dictionary, before hashing the string
# representation so we get a predictable order of the
# dictionary.
sorted_dict = str(sorted(dict_nofilename.items()))
return (dict_withhash, md5(sorted_dict).hexdigest())
def _hash_infile(self, adict, key):
# Inject file hashes into adict[key]
stuff = adict[key]
if not is_container(stuff):
stuff = [stuff]
file_list = []
for afile in stuff:
if os.path.isfile(afile):
md5obj = md5()
fp = file(afile, "rb")
while True:
data = fp.read(8192)
if not data:
break
md5obj.update(data)
fp.close()
md5hex = md5obj.hexdigest()
else:
md5hex = None
file_list.append((afile, md5hex))
return file_list