def __init__(self, config, stores):
"""
:param config: configuration for the parser
:param stores: instances of the available stores
"""
self.config = config
self.stores = stores
self.filters = config.getarray('filters')
self.filters = dict([
(cname,
cls(stores=listvalues(stores),
config=config.get_section(section='filter.' +
cls.CONFIG_SECTION, )))
for cname, cls in iteritems(filters.FILTERS)
if cname in self.filters or 'all' in self.filters
])
self.sources = config.getarray('sources')
self.sources = dict([
(cname,
cls(stores=listvalues(stores),
config=config.get_section(section='source.' +
cls.CONFIG_SECTION, )))
for cname, cls in iteritems(sources.SOURCES)
if cname in self.sources or 'all' in self.sources
])
def updatePileupLocation(self):
dataItems = self.backend.getActivePileupData()
# fullResync incorrect with multiple dbs's - fix!!!
dataLocations = DataLocationMapper.__call__(self, dataItems)
self.logger.info("Found %d unique pileup data to update location",
len(dataItems))
# Given that there might be multiple data items to be updated
# handle it like a dict such that element lookup becomes easier
modified = {}
for dataMapping in listvalues(dataLocations):
for data, locations in viewitems(dataMapping):
elements = self.backend.getElementsForPileupData(data)
for element in elements:
if element.get('NoPileupUpdate', False):
continue
if element.id in modified:
element = modified[element.id]
for pData in element['PileupData']:
if pData == data:
if sorted(locations) != sorted(
element['PileupData'][pData]):
self.logger.info("%s, setting location to: %s",
data, locations)
element['PileupData'][pData] = locations
modified[element.id] = element
break
self.logger.info("Updating %d elements for Pileup location update",
len(modified))
self.backend.saveElements(*listvalues(modified))
return len(modified)
def applySiiQcValidation(siiContainer, specfile):
"""Iterates over all Sii entries of a specfile in siiContainer and validates
if they surpass a user defined quality threshold. The parameters for
validation are defined in ``siiContainer.info[specfile]``:
- ``qcAttr``, ``qcCutoff`` and ``qcLargerBetter``
In addition to passing this validation a ``Sii`` has also to be at the first
list position in the ``siiContainer.container``. If both criteria are met
the attribute ``Sii.isValid`` is set to ``True``.
:param siiContainer: instance of :class:`maspy.core.SiiContainer`
:param specfile: unambiguous identifier of a ms-run file. Is also used as
a reference to other MasPy file containers.
"""
attr = siiContainer.info[specfile]['qcAttr']
cutOff = siiContainer.info[specfile]['qcCutoff']
if siiContainer.info[specfile]['qcLargerBetter']:
evaluator = lambda sii: getattr(sii, attr) >= cutOff and sii.rank == 1
else:
evaluator = lambda sii: getattr(sii, attr) <= cutOff and sii.rank == 1
for itemList in listvalues(siiContainer.container[specfile]):
#Set the .isValid attribute of all Sii to False
for sii in itemList:
sii.isValid = False
#Validate the first Sii
sii = itemList[0]
if evaluator(sii):
sii.isValid = True
def __init__(self, skim_dict):
self.offset_mapper = skim_dict.offset_mapper
self.skim_dict = skim_dict
# - key1_blocks dict maps key1 to block number
# DISTWALK: 0,
# DRV_COM_WLK_BOARDS: 0, ...
key1_block_offsets = skim_dict.skim_info['key1_block_offsets']
self.key1_blocks = {k: v[0] for k, v in iteritems(key1_block_offsets)}
# - skim_dim3 dict maps key1 to dict of key2 absolute offsets into block
# DRV_COM_WLK_BOARDS: {'MD': 4, 'AM': 3, 'PM': 5}, ...
block_offsets = skim_dict.skim_info['block_offsets']
skim_dim3 = OrderedDict()
for skim_key in block_offsets:
if not isinstance(skim_key, tuple):
continue
key1, key2 = skim_key
block, offset = block_offsets[skim_key]
assert block == self.key1_blocks[key1]
skim_dim3.setdefault(key1, OrderedDict())[key2] = offset
self.skim_dim3 = skim_dim3
logger.info("SkimStack.__init__ loaded %s keys with %s total skims" %
(len(self.skim_dim3),
sum([len(d) for d in listvalues(self.skim_dim3)])))
self.usage = set()
def make_unique(self, caller):
"""
Remove duplicate command-keys (unsafe)
Args:
caller (object): Commands on this object will
get preference in the duplicate removal.
Notes:
This is an unsafe command meant to clean out a cmdset of
doublet commands after it has been created. It is useful
for commands inheriting cmdsets from the cmdhandler where
obj-based cmdsets always are added double. Doublets will
be weeded out with preference to commands defined on
caller, otherwise just by first-come-first-served.
"""
unique = {}
for cmd in self.commands:
if cmd.key in unique:
ocmd = unique[cmd.key]
if (hasattr(cmd, 'obj') and cmd.obj == caller) and not \
(hasattr(ocmd, 'obj') and ocmd.obj == caller):
unique[cmd.key] = cmd
else:
unique[cmd.key] = cmd
self.commands = listvalues(unique)
def load_graph_toro(filename):
from collections import OrderedDict
vertices = OrderedDict()
edges = []
with open(filename) as f:
for linenum, line in enumerate(f):
d = line.split()
i = lambda idx: int(d[idx])
f = lambda idx: float(d[idx])
if d[0] == 'VERTEX2':
id_ = i(1)
xyt = np.array([f(2), f(3), f(4)])
vertices[id_] = VertexXYT(xyt)
elif d[0] == 'EDGE2':
ndx_out, ndx_in = i(1), i(2)
v_out, v_in = vertices[ndx_out], vertices[ndx_in]
xyt = np.array([f(3), f(4), f(5)])
P = np.array([[f(6), f(7), f(10)], [f(7), f(8),
f(11)],
[f(10), f(11), f(9)]])
g = MultiVariateGaussian(xyt, P)
edges.append(XYTConstraint(v_out, v_in, g))
else:
msg = "Unknown edge or vertex type %s in line %d" % (d[0],
linenum)
raise Exception(msg)
return Graph(listvalues(vertices), edges)
def _pre_run(self):
assert (self.LPU_obj)
assert (all(
[var in self.memory_manager.variables for var in self.variables]))
self.get_inds_func = {}
for var, d in self.variables.items():
v_dict = self.memory_manager.variables[var]
if not d['uids']:
uids = list(v_dict['uids'])
inds = listvalues(v_dict['uids'])
o = np.argsort(inds)
d['uids'] = [uids[i] for i in o]
self.src_inds[var] = garray.to_gpu(np.arange(len(d['uids'])))
else:
uids = []
inds = []
for uid in d['uids']:
try:
inds.append(v_dict['uids'][uid])
uids.append(uid)
except:
pass
inds = np.array(inds, np.int32)
o = np.argsort(inds)
self.src_inds[var] = garray.to_gpu(inds[o])
d['uids'] = [uids[i] for i in o]
self._d_output[var] = garray.empty(len(d['uids']),
v_dict['buffer'].dtype)
d['output'] = np.zeros(len(d['uids']), v_dict['buffer'].dtype)
self.get_inds_func[var] = get_inds_kernel(self.src_inds[var].dtype,
v_dict['buffer'].dtype)
self.pre_run()
def find_node(self, attribute, value):
"""
Find and return the node with the specified attribute / value pair.
@type attribute: str
@param attribute: Attribute name we are looking for
@type value: mixed
@param value: Value of attribute we are looking for
@rtype: Mixed
@return: Node, if attribute / value pair is matched. None otherwise.
"""
# if the attribute to search for is the id, simply return the node from the internal hash.
if attribute == "id" and value in self.nodes:
return self.nodes[value]
# step through all the nodes looking for the given attribute/value pair.
else:
for node in listvalues(self.nodes):
if hasattr(node, attribute):
if getattr(node, attribute) == value:
return node
return None
def pandaify_results(gort, ssrt, tb=.68, clmap=None, bootstrap=False, bootinfo={'nsubjects':25, 'ntrials':1000, 'groups':['ssd']}, ssd=np.array([[.2, .25, .3, .35, .4]])):
nlevels = gort.shape[0]
if nlevels==1:
clmap = {'flat': ['flat']}
conds = list(clmap)
cond_levels = list(product(*[list(levels) for levels in listvalues(clmap)]))
dfColumns=['ttype', 'ssd', 'response', 'acc', 'rt', 'ssrt', 'trial']
dfColumns = conds + dfColumns
# bootinfo['groups'] = bootinfo['groups'] + conds
dfList = []
for i in range(nlevels):
df = create_sim_dataframe(gort[i], ssrt[i], tb=tb, ssd=ssd)
for ii, c in enumerate(conds):
df[c] = cond_levels[i][ii]
dfList.append(df[dfColumns])
resultsdf = pd.concat(dfList)
resultsdf.reset_index(drop=True, inplace=True)
if bootstrap:
resultsdf = bootstrap_data(resultsdf, nsubjects=bootinfo['nsubjects'], n=bootinfo['ntrials'], groups=bootinfo['groups'])
return resultsdf
def _gen_update_info(result_mapping, Gt, Gd):
node_info = {}
for task, device in iteritems(result_mapping):
if Gt.node[task][GtInfo.DEVICE]:
assert device == Gt.node[task][GtInfo.DEVICE]
device_type = Gd.node[device][GdInfo.DEVICE_TYPE]
latency_info = Gt.node[task][GtInfo.LATENCY_INFO]
exec_cmd = deepcopy(Gt.node[task][GtInfo.EXEC_CMD])
compute_latency = 0 if not latency_info else (listvalues(
latency_info[device_type])[0])
node_info[task] = {
GtInfo.CUR_DEVICE: device,
GtInfo.CUR_LATENCY: compute_latency,
GtInfo.EXEC_CMD: exec_cmd
}
edge_info = {}
for edge in Gt.edges():
t1, t2 = edge
d1 = node_info[t1][GtInfo.CUR_DEVICE]
d2 = node_info[t2][GtInfo.CUR_DEVICE]
transmission_latency = Gd[d1][d2][GdInfo.LATENCY]
edge_info[LinkHelper.get_edge(t1, t2)] = {
GtInfo.CUR_LATENCY: transmission_latency,
}
return node_info, edge_info
def _parseDataBySchema(self, dataRoot, fileSchema, outGlbData):
""" first sort the schema Root by location """
locFieldDict = sortSchemaByLocation(fileSchema)
# for each data entry, parse data by location
floatKey = getKeys(dataRoot, float)
for ien in floatKey:
if float(ien) <=0:
continue
dataEntry = dataRoot[ien]
outDataEntry = FileManDataEntry(fileSchema.getFileNo(), ien)
dataKeys = [x for x in dataEntry]
sortedKey = sorted(dataKeys, key=functools.cmp_to_key(sortDataEntryFloatFirst))
for locKey in sortedKey:
if locKey == '0' and fileSchema.getFileNo() == '1':
self._parseFileDetail(dataEntry[locKey], ien)
if locKey in locFieldDict:
fieldDict = locFieldDict[locKey] # a dict of {pos: field}
curDataRoot = dataEntry[locKey]
if len(fieldDict) == 1:
fieldAttr = listvalues(fieldDict)[0]
if fieldAttr.isSubFilePointerType(): # Multiple
self._parseSubFileField(curDataRoot, fieldAttr, outDataEntry)
else:
self._parseSingleDataValueField(curDataRoot, fieldAttr,
outDataEntry)
else:
self._parseDataValueField(curDataRoot, fieldDict, outDataEntry)
outGlbData.addFileManDataEntry(ien, outDataEntry)
if fileSchema.getFileNo() == self._curFileNo:
self._addFileKeyIndex(self._curFileNo, ien, outDataEntry.name)
def __init__(self, skim_dict):
self.offset_mapper = skim_dict.offset_mapper
self.skim_dict = skim_dict
# - key1_blocks dict maps key1 to block number
# DISTWALK: 0,
# DRV_COM_WLK_BOARDS: 0, ...
key1_block_offsets = skim_dict.skim_info['key1_block_offsets']
self.key1_blocks = {k: v[0] for k, v in iteritems(key1_block_offsets)}
# - skim_dim3 dict maps key1 to dict of key2 absolute offsets into block
# DRV_COM_WLK_BOARDS: {'MD': 4, 'AM': 3, 'PM': 5}, ...
block_offsets = skim_dict.skim_info['block_offsets']
skim_dim3 = OrderedDict()
for skim_key in block_offsets:
if not isinstance(skim_key, tuple):
continue
key1, key2 = skim_key
block, offset = block_offsets[skim_key]
assert block == self.key1_blocks[key1]
skim_dim3.setdefault(key1, OrderedDict())[key2] = offset
self.skim_dim3 = skim_dim3
logger.info("SkimStack.__init__ loaded %s keys with %s total skims"
% (len(self.skim_dim3),
sum([len(d) for d in listvalues(self.skim_dim3)])))
self.usage = set()
def cap_cache(self):
if self._total > self.cap:
for ent in listvalues(self._entries):
if self._total < self.cap or len(
self._entries) < self.min_entries:
break
self._remove(ent, True)
def applySiiQcValidation(siiContainer, specfile):
"""Iterates over all Sii entries of a specfile in siiContainer and validates
if they surpass a user defined quality threshold. The parameters for
validation are defined in ``siiContainer.info[specfile]``:
- ``qcAttr``, ``qcCutoff`` and ``qcLargerBetter``
In addition to passing this validation a ``Sii`` has also to be at the first
list position in the ``siiContainer.container``. If both criteria are met
the attribute ``Sii.isValid`` is set to ``True``.
:param siiContainer: instance of :class:`maspy.core.SiiContainer`
:param specfile: unambiguous identifier of a ms-run file. Is also used as
a reference to other MasPy file containers.
"""
attr = siiContainer.info[specfile]['qcAttr']
cutOff = siiContainer.info[specfile]['qcCutoff']
if siiContainer.info[specfile]['qcLargerBetter']:
evaluator = lambda sii: getattr(sii, attr) >= cutOff and sii.rank == 1
else:
evaluator = lambda sii: getattr(sii, attr) <= cutOff and sii.rank == 1
for itemList in listvalues(siiContainer.container[specfile]):
#Set the .isValid attribute of all Sii to False
for sii in itemList:
sii.isValid = False
#Validate the first Sii
sii = itemList[0]
if evaluator(sii):
sii.isValid = True
def update_node_id(self, current_id, new_id):
"""
Simply updating the id attribute of a node will sever the edges to / from the given node. This routine will
correctly update the edges as well.
@type current_id: long
@param current_id: Current ID of node whose ID we want to update
@type new_id: long
@param new_id: New ID to update to.
"""
if current_id not in self.nodes:
return
# update the node.
node = self.nodes[current_id]
del self.nodes[current_id]
node.id = new_id
self.nodes[node.id] = node
# update the edges.
for edge in [
edge for edge in listvalues(self.edges)
if current_id in (edge.src, edge.dst)
]:
del self.edges[edge.id]
if edge.src == current_id:
edge.src = new_id
if edge.dst == current_id:
edge.dst = new_id
edge.id = (edge.src << 32) + edge.dst
self.edges[edge.id] = edge
def calcDistMatchArr(matchArr, tKey, mKey):
"""Calculate the euclidean distance of all array positions in "matchArr".
:param matchArr: a dictionary of ``numpy.arrays`` containing at least two
entries that are treated as cartesian coordinates.
:param tKey: #TODO: docstring
:param mKey: #TODO: docstring
:returns: #TODO: docstring
{'eucDist': numpy.array([eucDistance, eucDistance, ...]),
'posPairs': numpy.array([[pos1, pos2], [pos1, pos2], ...])
}
"""
#Calculate all sorted list of all eucledian feature distances
matchArrSize = listvalues(matchArr)[0].size
distInfo = {'posPairs': list(), 'eucDist': list()}
_matrix = numpy.swapaxes(numpy.array([matchArr[tKey], matchArr[mKey]]), 0,
1)
for pos1 in range(matchArrSize - 1):
for pos2 in range(pos1 + 1, matchArrSize):
distInfo['posPairs'].append((pos1, pos2))
distInfo['posPairs'] = numpy.array(distInfo['posPairs'])
distInfo['eucDist'] = scipy.spatial.distance.pdist(_matrix)
distSort = numpy.argsort(distInfo['eucDist'])
for key in list(viewkeys(distInfo)):
distInfo[key] = distInfo[key][distSort]
return distInfo
def _get_path_length(path, Gt, Gd, result_mapping):
log.info('path: {}'.format(path))
path_vars = []
# first node: src
ti = path[0]
di = Gt.node[ti][GtInfo.DEVICE]
# device_mapping start from path[1] to path[N-1]
for j in range(1, len(path) - 1):
tj = path[j]
dj = result_mapping[tj]
# get transmission latency from di -> dj
Ld_di_dj = Gd[di][dj][GdInfo.LATENCY]
path_vars.append(Ld_di_dj)
log.debug('Ld_di_dj (move {} -> {}) = {}'.format(di, dj, Ld_di_dj))
# get computation latency for task tj at dj
dj_type = Gd.node[dj][GdInfo.DEVICE_TYPE]
# get latency of the default build flavor
Lt_tj_dj = listvalues(Gt.node[tj][GtInfo.LATENCY_INFO][dj_type])[0]
path_vars.append(Lt_tj_dj)
log.debug('Lt_tj_dj (do {} at {}) = {}'.format(tj, dj, Lt_tj_dj))
ti = tj
di = dj
# last node: dst
tj = path[len(path) - 1]
dj = Gt.node[tj][GtInfo.DEVICE]
Ld_di_dj = Gd[di][dj][GdInfo.LATENCY]
path_vars.append(Ld_di_dj)
log.debug('Ld_di_dj (move {} -> {}) = {}'.format(di, dj, Ld_di_dj))
path_length = sum(path_vars)
log.info('\tlength: {}, {}'.format(path_length, path_vars))
return path_length
def __init__(self, path_response, context_name, context_id,
req_sort_order_type):
# common.debug('VideoListSorted data: {}', path_response)
self.perpetual_range_selector = path_response.get(
'_perpetual_range_selector')
self.data = path_response
self.context_name = context_name
has_data = bool(
(context_id and path_response.get(context_name)
and path_response[context_name].get(context_id))
or (not context_id and path_response.get(context_name)))
self.data_lists = {}
self.videos = OrderedDict()
self.artitem = None
self.contained_titles = None
self.videoids = None
if has_data:
self.data_lists = path_response[context_name][context_id][req_sort_order_type] \
if context_id else path_response[context_name][req_sort_order_type]
self.videos = OrderedDict(resolve_refs(self.data_lists, self.data))
if self.videos:
# self.artitem = next(itervalues(self.videos))
self.artitem = listvalues(self.videos)[0]
self.contained_titles = _get_titles(self.videos)
try:
self.videoids = _get_videoids(self.videos)
except KeyError:
self.videoids = None
def find_edge(self, attribute, value):
"""
Find and return the edge with the specified attribute / value pair.
@type attribute: str
@param attribute: Attribute name we are looking for
@type value: Mixed
@param value: Value of attribute we are looking for
@rtype: Mixed
@return: Edge, if attribute / value pair is matched. None otherwise.
"""
# if the attribute to search for is the id, simply return the edge from the internal hash.
if attribute == "id" and value in self.edges:
return self.edges[value]
# step through all the edges looking for the given attribute/value pair.
else:
# TODO: Verify that this actually works? Was broken when I got here ;-P
for node_edge in listvalues(self.edges):
if hasattr(node_edge, attribute):
if getattr(node_edge, attribute) == value:
return node_edge
return None
def loadConfigurationFile(filename):
"""
_loadConfigurationFile_
Load a Configuration File
"""
cfgBaseName = os.path.basename(filename).replace(".py", "")
cfgDirName = os.path.dirname(filename)
if not cfgDirName:
modPath = imp.find_module(cfgBaseName)
else:
modPath = imp.find_module(cfgBaseName, [cfgDirName])
try:
modRef = imp.load_module(cfgBaseName, modPath[0], modPath[1],
modPath[2])
except Exception as ex:
msg = "Unable to load Configuration File:\n"
msg += "%s\n" % filename
msg += "Due to error:\n"
msg += str(ex)
msg += str(traceback.format_exc())
raise RuntimeError(msg)
for attr in listvalues(modRef.__dict__):
if isinstance(attr, Configuration):
return attr
# //
# // couldnt find a Configuration instance
# //
msg = "Unable to find a Configuration object instance in file:\n"
msg += "%s\n" % filename
raise RuntimeError(msg)
def set_conditions(self, depends_on=None, bwfactors=None):
data = self.data.copy()
self.depends_on = depends_on
self.conds = np.unique(np.hstack(listvalues(self.depends_on))).tolist()
self.nconds = len(self.conds)
if 'flat' in self.conds:
self.is_flat = True
data['flat'] = 'flat'
self.data = data.copy()
else:
self.is_flat = False
clevels = [np.sort(data[c].unique()) for c in self.conds]
clevels = [np.array([str(lvl) for lvl in levels]) for levels in clevels]
self.clmap = {c: lvls for c, lvls in zip(self.conds, clevels)}
self.cond_matrix = np.array([lvls.size for lvls in clevels])
self.nlevels = np.cumprod(self.cond_matrix)[-1]
self.groups = np.hstack([['idx'], self.conds]).tolist()
self.__format_pcmap__()
if hasattr(self, 'ssdDF') and not self.is_nested:
self.__set_ssd_info__()
if hasattr(self, 'fitparams') and not self.is_nested:
self.generate_model_id()
self.set_fitparams(nlevels=self.nlevels, clmap=self.clmap, model_id=self.model_id)
def set_skim_wrapper_targets(df, skims):
"""
Add the dataframe to the SkimDictWrapper object so that it can be dereferenced
using the parameters of the skims object.
Parameters
----------
df : pandas.DataFrame
Table to which to add skim data as new columns.
`df` is modified in-place.
skims : SkimDictWrapper or SkimStackWrapper object, or a list or dict of skims
The skims object is used to contain multiple matrices of
origin-destination impedances. Make sure to also add it to the
locals_d below in order to access it in expressions. The *only* job
of this method in regards to skims is to call set_df with the
dataframe that comes back from interacting choosers with
alternatives. See the skims module for more documentation on how
the skims object is intended to be used.
"""
if isinstance(skims, list):
for skim in skims:
assert isinstance(skim, SkimDictWrapper) or isinstance(skim, SkimStackWrapper)
skim.set_df(df)
elif isinstance(skims, dict):
# it it is a dict, then check for known types, ignore anything we don't recognize as a skim
# (this allows putting skim column names in same dict as skims for use in locals_dicts)
for skim in listvalues(skims):
if isinstance(skim, SkimDictWrapper) or isinstance(skim, SkimStackWrapper):
skim.set_df(df)
else:
assert isinstance(skims, SkimDictWrapper) or isinstance(skims, SkimStackWrapper)
skims.set_df(df)
def calcDistMatchArr(matchArr, tKey, mKey):
"""Calculate the euclidean distance of all array positions in "matchArr".
:param matchArr: a dictionary of ``numpy.arrays`` containing at least two
entries that are treated as cartesian coordinates.
:param tKey: #TODO: docstring
:param mKey: #TODO: docstring
:returns: #TODO: docstring
{'eucDist': numpy.array([eucDistance, eucDistance, ...]),
'posPairs': numpy.array([[pos1, pos2], [pos1, pos2], ...])
}
"""
#Calculate all sorted list of all eucledian feature distances
matchArrSize = listvalues(matchArr)[0].size
distInfo = {'posPairs': list(), 'eucDist': list()}
_matrix = numpy.swapaxes(numpy.array([matchArr[tKey], matchArr[mKey]]), 0, 1)
for pos1 in range(matchArrSize-1):
for pos2 in range(pos1+1, matchArrSize):
distInfo['posPairs'].append((pos1, pos2))
distInfo['posPairs'] = numpy.array(distInfo['posPairs'])
distInfo['eucDist'] = scipy.spatial.distance.pdist(_matrix)
distSort = numpy.argsort(distInfo['eucDist'])
for key in list(viewkeys(distInfo)):
distInfo[key] = distInfo[key][distSort]
return distInfo
def __init__(self, path_response, list_id=None):
# common.debug('VideoList data: {}', path_response)
self.perpetual_range_selector = path_response.get(
'_perpetual_range_selector')
self.data = path_response
has_data = bool(path_response.get('lists'))
self.videos = OrderedDict()
self.artitem = None
self.contained_titles = None
self.videoids = None
if has_data:
# Generate one videoid, or from the first id of the list or with specified one
self.videoid = common.VideoId(videoid=(
list_id if list_id else next(iter(self.data['lists']))))
self.videos = OrderedDict(
resolve_refs(self.data['lists'][self.videoid.value],
self.data))
if self.videos:
# self.artitem = next(itervalues(self.videos))
self.artitem = listvalues(self.videos)[0]
self.contained_titles = _get_titles(self.videos)
try:
self.videoids = _get_videoids(self.videos)
except KeyError:
self.videoids = None
def __init__(self, path_response, list_id):
# common.debug('VideoListLoCo data: {}', path_response)
self.perpetual_range_selector = path_response.get(
'_perpetual_range_selector')
self.data = path_response
self.list_id = list_id
self.videoids = None
# Set a 'UNSPECIFIED' type videoid (special handling for menus see parse_info in infolabels.py)
self.videoid = common.VideoId(videoid=list_id)
self.contained_titles = None
self.artitem = None
if 'lists' not in path_response:
# No data in path response
return
# Set videos data for the specified list id
self.videos = OrderedDict(
resolve_refs(self.data['lists'][list_id], self.data))
if not self.videos:
return
# Set first videos titles (special handling for menus see parse_info in infolabels.py)
self.contained_titles = _get_titles(self.videos)
# Set art data of first video (special handling for menus see parse_info in infolabels.py)
self.artitem = listvalues(self.videos)[0]
try:
self.videoids = _get_videoids(self.videos)
except KeyError:
self.videoids = None
def create(self):
self._validate_creation()
self._create_cfn_template()
tags = [{'Key': k, 'Value': v} for (k, v) in viewitems(self.tags)]
cf = boto3.session.Session().resource('cloudformation')
reporter = ResourceReporter()
try:
stack = cf.create_stack(StackName=self.stack_name, TemplateBody=self.tpl.to_yaml(),
Capabilities=['CAPABILITY_NAMED_IAM'], Tags=tags)
stack = reporter.report_stack_creation(self.stack_name, listvalues(self.resources), stack.stack_id)
role = {o.get('OutputKey'): o.get('OutputValue') for o in stack.outputs}.get(self.RESOURCE_NG_ROLE.name)
except Exception as e:
ec2 = boto3.session.Session().resource('ec2')
if self.keypair_imported:
r = copy(self.RESOURCE_NG_KEYPAIR)
r.resource_id = self.keypair
ec2.KeyPair(self.keypair).delete()
r.status = Status.deleted
reporter.succeed(resource=r)
raise e
self._update_configmap(os.path.expanduser(self.kubeconf), role)
return NodeGroupInfo(self.name, self.instance, self.min, self.max, role)
def make_unique(self, caller):
"""
Remove duplicate command-keys (unsafe)
Args:
caller (object): Commands on this object will
get preference in the duplicate removal.
Notes:
This is an unsafe command meant to clean out a cmdset of
doublet commands after it has been created. It is useful
for commands inheriting cmdsets from the cmdhandler where
obj-based cmdsets always are added double. Doublets will
be weeded out with preference to commands defined on
caller, otherwise just by first-come-first-served.
"""
unique = {}
for cmd in self.commands:
if cmd.key in unique:
ocmd = unique[cmd.key]
if (hasattr(cmd, 'obj') and cmd.obj == caller) and not \
(hasattr(ocmd, 'obj') and ocmd.obj == caller):
unique[cmd.key] = cmd
else:
unique[cmd.key] = cmd
self.commands = listvalues(unique)
def fixDBSmissingFileAssoc():
os.environ[
'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
connectToDB()
myThread = threading.currentThread()
formatter = DBFormatter(logging, myThread.dbi)
problemFilesSql = """
select dbsbuffer_file.id as fileid, dbsbuffer_location.id as seid from wmbs_file_location fl
inner join wmbs_file_details fd on fd.id = fl.fileid
inner join wmbs_location_pnns wls on wls.location = fl.location
inner join wmbs_location wl on wl.id = fl.location
inner join dbsbuffer_location on dbsbuffer_location.pnn = wls.pnn
inner join dbsbuffer_file on dbsbuffer_file.lfn = fd.lfn
where fd.lfn in (select df.lfn from dbsbuffer_file df
left outer join dbsbuffer_file_location dfl on df.id = dfl.filename
where dfl.location is null)
"""
unfinishedTasks = formatter.formatDict(
formatter.dbi.processData(problemFilesSql))
print("%s lenth" % len(unfinishedTasks))
result = {}
for row in unfinishedTasks:
result.setdefault(row["fileid"], row)
print(row)
print("trimed %s lenth" % len(result))
insertSQL = """INSERT INTO dbsbuffer_file_location (filename, location)
VALUES (:fileid, :seid)"""
done = formatter.dbi.processData(insertSQL, listvalues(result))
print("inserted %s" % done)
def formatWorkflow(self, results):
workflow = {}
tran = Transitions()
for result in results:
if result["task"] not in workflow:
workflow[result["task"]] = {}
for state in tran.states():
workflow[result["task"]][state] = 0
workflow[result["task"]][result["state"]] = result["num_job"]
workflow[result["task"]]['total_jobs'] = result["num_job"]
workflow[result["task"]]["real_success"] = int(result["success"])
workflow[result["task"]]["id"] = result["id"]
workflow[result["task"]]["wmspec"] = result["wmspec"]
workflow[result["task"]]["task"] = result["task"]
workflow[result["task"]]["real_fail"] = self.failCount(result)
workflow[result["task"]]['processing'] = self.processingCount(result)
else:
workflow[result["task"]][result["state"]] = result["num_job"]
workflow[result["task"]]['total_jobs'] += result["num_job"]
workflow[result["task"]]["real_success"] += int(result["success"])
workflow[result["task"]]["real_fail"] += self.failCount(result)
workflow[result["task"]]['processing'] += self.processingCount(result)
# need to order by id (client side)
return listvalues(workflow)
def render_graph_graphviz(self):
"""
Render the graphviz graph structure.
@rtype: pydot.Dot
@return: Pydot object representing entire graph
"""
dot_graph = pydot.Dot()
for node in listvalues(self.nodes):
dot_graph.add_node(node.render_node_graphviz(self))
for edge in listvalues(self.edges):
dot_graph.add_edge(edge.render_edge_graphviz(self))
return dot_graph
def repack_small_blocks(self, force=False, sync=False, closed_file_size=0):
"""Packs small blocks together before uploading"""
self._pending_write_size += closed_file_size
# Check if there are enough small blocks for filling up one in full
if not (force or (self._pending_write_size >= config.KEEP_BLOCK_SIZE)):
return
# Search blocks ready for getting packed together before being
# committed to Keep.
# A WRITABLE block always has an owner.
# A WRITABLE block with its owner.closed() implies that its
# size is <= KEEP_BLOCK_SIZE/2.
try:
small_blocks = [
b for b in listvalues(self._bufferblocks)
if b.state() == _BufferBlock.WRITABLE and b.owner.closed()
]
except AttributeError:
# Writable blocks without owner shouldn't exist.
raise UnownedBlockError()
if len(small_blocks) <= 1:
# Not enough small blocks for repacking
return
for bb in small_blocks:
bb.repack_writes()
# Update the pending write size count with its true value, just in case
# some small file was opened, written and closed several times.
self._pending_write_size = sum([b.size() for b in small_blocks])
if self._pending_write_size < config.KEEP_BLOCK_SIZE and not force:
return
new_bb = self._alloc_bufferblock()
new_bb.owner = []
files = []
while len(small_blocks) > 0 and (
new_bb.write_pointer +
small_blocks[0].size()) <= config.KEEP_BLOCK_SIZE:
bb = small_blocks.pop(0)
new_bb.owner.append(bb.owner)
self._pending_write_size -= bb.size()
new_bb.append(bb.buffer_view[0:bb.write_pointer].tobytes())
files.append((bb, new_bb.write_pointer - bb.size()))
self.commit_bufferblock(new_bb, sync=sync)
for bb, new_bb_segment_offset in files:
newsegs = bb.owner.segments()
for s in newsegs:
if s.locator == bb.blockid:
s.locator = new_bb.blockid
s.segment_offset = new_bb_segment_offset + s.segment_offset
bb.owner.set_segments(newsegs)
self._delete_bufferblock(bb.blockid)
def graph_cat(self, other_graph):
"""
Concatenate the other graph into the current one.
@todo: Add support for clusters
@type other_graph: pgraph.Graph
@param other_graph: Graph to concatenate into this one.
"""
for other_node in listvalues(other_graph.nodes):
self.add_node(other_node)
for other_edge in listvalues(other_graph.edges):
self.add_edge(other_edge)
return self
def execute(self, siteName, conn=None, transaction=False):
binds = {"location": siteName}
result = self.dbi.processData(self.sql,
binds,
conn=conn,
transaction=transaction)
return listvalues(result[0].fetchall()[0])[0]
def __init__(self, path_response):
self.perpetual_range_selector = path_response.get('_perpetual_range_selector')
self.data = path_response
self.videos = OrderedDict(self.data.get('videos', {}))
self.videoids = _get_videoids(self.videos)
# self.artitem = next(itervalues(self.videos))
self.artitem = listvalues(self.videos)[0] if self.videos else None
self.contained_titles = _get_titles(self.videos)
def render_graph_gml(self):
"""
Render the GML graph description.
@rtype: String
@return: GML graph description.
"""
gml = 'Creator "pGRAPH - Pedram Amini <*****@*****.**>"\n'
gml += "directed 1\n"
# open the graph tag.
gml += "graph [\n"
# add the nodes to the GML definition.
for node in listvalues(self.nodes):
gml += node.render_node_gml(self)
# add the edges to the GML definition.
for edge in listvalues(self.edges):
gml += edge.render_edge_gml(self)
# close the graph tag.
gml += "]\n"
"""
TODO: Complete cluster rendering
# if clusters exist.
if len(self.clusters):
# open the rootcluster tag.
gml += 'rootcluster [\n'
# add the clusters to the GML definition.
for cluster in self.clusters:
gml += cluster.render()
# add the clusterless nodes to the GML definition.
for node in self.nodes:
if not self.find_cluster_by_node("id", node.id):
gml += ' vertex "%d"\n' % node.id
# close the rootcluster tag.
gml += ']\n'
"""
return gml
def format(self, results):
if len(results) == 0:
return False
else:
format = []
for i in results[0].fetchall():
format.append(listvalues(i)[0])
return format
def saygo(depends_on={}, cond_map=None, kind='xdpm', fit_on='subjects'):
""" generate confirmation message that model is prepared and ready to fit.
repeats structure and important fit details for user to confirm """
depkeys = describe_model(depends_on)
if 'x' in kind:
bias = '(w/ dynamic bias)'
else:
bias = ""
dep = listvalues(depends_on)
# flatten list of all cond levels
lbls = ', '.join(sum(listvalues(depends_on), []))
msg = get_nonsense()
strings = (kind, bias, fit_on, pdep, dep, lbls, msg)
print("""
Model is prepared to fit %s model %s to %s data,
allowing %s to vary across levels of %s (%s)
%s \n""" % strings)
return True
def graph_sub(self, other_graph):
"""
Remove the elements shared between the current graph and other graph from the current
graph.
@todo: Add support for clusters
@type other_graph: pgraph.Graph
@param other_graph: Graph to diff/remove against
"""
for other_node in listvalues(other_graph.nodes):
self.del_node(other_node.id)
for other_edge in listvalues(other_graph.edges):
self.del_edge(None, other_edge.src, other_edge.dst)
return self
def make_f_cols(self):
""" make header names for various fit statistics in fitDF
(model parameters, goodness-of-fit measures, etc)
"""
params = np.sort(list(self.inits))
if not self.model.is_flat:
cond_param_names = listvalues(self.model.clmap)
params = np.hstack([params, np.squeeze(cond_param_names)]).tolist()
fit_cols = ['nfev', 'nvary', 'df', 'chi', 'rchi', 'logp', 'AIC', 'BIC', 'cnvrg']
self.f_cols = np.hstack([['idx'], params, fit_cols]).tolist()
def repack_small_blocks(self, force=False, sync=False, closed_file_size=0):
"""Packs small blocks together before uploading"""
self._pending_write_size += closed_file_size
# Check if there are enough small blocks for filling up one in full
if not (force or (self._pending_write_size >= config.KEEP_BLOCK_SIZE)):
return
# Search blocks ready for getting packed together before being
# committed to Keep.
# A WRITABLE block always has an owner.
# A WRITABLE block with its owner.closed() implies that its
# size is <= KEEP_BLOCK_SIZE/2.
try:
small_blocks = [b for b in listvalues(self._bufferblocks)
if b.state() == _BufferBlock.WRITABLE and b.owner.closed()]
except AttributeError:
# Writable blocks without owner shouldn't exist.
raise UnownedBlockError()
if len(small_blocks) <= 1:
# Not enough small blocks for repacking
return
for bb in small_blocks:
bb.repack_writes()
# Update the pending write size count with its true value, just in case
# some small file was opened, written and closed several times.
self._pending_write_size = sum([b.size() for b in small_blocks])
if self._pending_write_size < config.KEEP_BLOCK_SIZE and not force:
return
new_bb = self._alloc_bufferblock()
new_bb.owner = []
files = []
while len(small_blocks) > 0 and (new_bb.write_pointer + small_blocks[0].size()) <= config.KEEP_BLOCK_SIZE:
bb = small_blocks.pop(0)
new_bb.owner.append(bb.owner)
self._pending_write_size -= bb.size()
new_bb.append(bb.buffer_view[0:bb.write_pointer].tobytes())
files.append((bb, new_bb.write_pointer - bb.size()))
self.commit_bufferblock(new_bb, sync=sync)
for bb, new_bb_segment_offset in files:
newsegs = bb.owner.segments()
for s in newsegs:
if s.locator == bb.blockid:
s.locator = new_bb.blockid
s.segment_offset = new_bb_segment_offset+s.segment_offset
bb.owner.set_segments(newsegs)
self._delete_bufferblock(bb.blockid)
def set_conditions(self, depends_on=None):
data = self.data.copy()
self.depends_on = depends_on
self.conds = np.unique(listvalues(self.depends_on)).tolist()
if 'flat' in self.conds:
self.is_flat = True
data['flat'] = 'flat'
self.data = data.copy()
else:
self.is_flat = False
self.nconds = len(self.conds)
self.clmap = {c: np.sort(data[c].unique()) for c in self.conds}
self.nlevels = np.sum([len(lvls) for lvls in listvalues(self.clmap)])
self.groups = np.hstack([['idx'], self.conds]).tolist()
self.__format_pcmap__()
if hasattr(self, 'ssd'):
self.__set_ssd_info__()
if hasattr(self, 'fitparams'):
self.generate_model_id()
self.set_fitparams(nlevels=self.nlevels, clmap=self.clmap)
def _collection_size(self, collection):
"""
Recursively get the total size of the collection
"""
size = 0
for item in listvalues(collection):
if isinstance(item, arvados.collection.Collection) or isinstance(item, arvados.collection.Subcollection):
size += self._collection_size(item)
else:
size += item.size()
return size
def manifest_text(self, strip=False):
manifest_text = [self.name().replace(' ', '\\040')]
if strip:
for d in self._data_locators:
m = re.match(r'^[0-9a-f]{32}\+\d+', d.locator)
manifest_text.append(m.group(0))
else:
manifest_text.extend([d.locator for d in self._data_locators])
manifest_text.extend([' '.join(["{}:{}:{}".format(seg.locator, seg.range_size, f.name.replace(' ', '\\040'))
for seg in f.segments])
for f in listvalues(self._files)])
return ' '.join(manifest_text) + '\n'
def photos(self):
if not self._photo_assets:
child_assets = self.data.get('childAssetsBinaryFeed')
if not child_assets:
raise PyiCloudBinaryFeedParseError(
"Missing childAssetsBinaryFeed in photo album")
self._photo_assets = listvalues(_parse_binary_feed(child_assets))
for asset in self._photo_assets:
asset.album = self
return self._photo_assets
def get_cpals(name='all', aslist=False, random=False):
rpal = lambda nc: sns.blend_palette(['#e88379', '#de143d'], n_colors=nc)
bpal = lambda nc: sns.blend_palette(['#81aedb', '#3572C6'], n_colors=nc)
gpal = lambda nc: sns.blend_palette(['#65b88f', '#27ae60'], n_colors=nc)
ppal = lambda nc: sns.blend_palette(['#9B59B6', "#663399"], n_colors=nc)
heat = lambda nc: sns.blend_palette(['#f39c12', '#e5344a'], n_colors=nc)
cool = lambda nc: sns.blend_palette(["#4168B7", "#27ae60"], n_colors=nc)
slate = lambda nc: sns.blend_palette(['#95A5A6', "#6C7A89"], n_colors=nc)
wet = lambda nc: sns.blend_palette(['#34495e', "#99A4AE"], n_colors=nc)
fire = lambda nc: sns.blend_palette(['#e5344a', "#f39c12"], n_colors=nc)
bupu = lambda nc: sns.blend_palette(['#8E44AD', "#3498db"], n_colors=nc)
color_dict = {'bpal': bpal, 'gpal': gpal, 'rpal': rpal, 'ppal': ppal, 'heat': heat, 'cool': cool, 'slate': slate, 'wet': wet, 'fire':fire, 'bupu': bupu}
if random:
pals = listvalues(color_dict)
i = randint(0, len(pals), 1)
return pals[i]
if name=='all':
if aslist:
return listvalues(color_dict)
return color_dict
else:
return color_dict[name]
def Shutdown( self ):
"""Properly shutdown the server."""
self.shutdown_requested = True
# Shutdown waitress threads.
self.task_dispatcher.shutdown()
# Close asyncore channels.
# We don't use itervalues here because _map is modified while looping
# through it.
# NOTE: _map is an attribute from the asyncore.dispatcher class, which is a
# base class of TcpWSGIServer. This may change in future versions of
# waitress so extra care should be taken when updating waitress.
for channel in listvalues( self._map ):
channel.close()
def get_choice_label(self, field, choice_value, lang='English'):
choices = [
choice for choice in field.children if choice.name == choice_value
]
if len(choices):
choice = choices[0]
label = choice.label
if isinstance(label, dict):
label = label.get(lang, listvalues(choice.label)[0])
return label
return choice_value
def all(self, interval=None):
"""
Get all subscriptions.
Args:
interval (int): Limit match to tickers with this interval.
Returns:
tickers (list): If `interval` was given, this is a list of
tickers using that interval.
tickerpool_layout (dict): If `interval` was *not* given,
this is a dict {interval1: [ticker1, ticker2, ...], ...}
"""
if interval is None:
# return dict of all, ordered by interval
return dict((interval, listvalues(ticker.subscriptions))
for interval, ticker in self.ticker_pool.tickers.items())
else:
# get individual interval
ticker = self.ticker_pool.tickers.get(interval, None)
if ticker:
return listvalues(ticker.subscriptions)
def get_cpals(name='all', aslist=False):
rpal = lambda nc: sns.blend_palette(['#e88379', '#c0392b'], n_colors=nc)
bpal = lambda nc: sns.blend_palette(['#81aedb', '#3A539B'], n_colors=nc)
gpal = lambda nc: sns.blend_palette(['#65b88f', '#27ae60'], n_colors=nc)
ppal = lambda nc: sns.blend_palette(['#848bb6', "#9B59B6"], n_colors=nc)
heat = lambda nc: sns.blend_palette(['#f39c12', '#c0392b'], n_colors=nc)
cool = lambda nc: sns.blend_palette(["#4168B7", "#27ae60"], n_colors=nc)
slate = lambda nc: sns.blend_palette(['#95A5A6', "#6C7A89"], n_colors=nc)
color_dict = {'bpal': bpal, 'gpal': gpal, 'rpal': rpal, 'ppal': ppal, 'heat': heat, 'cool': cool, 'slate': slate}
if name=='all':
if aslist:
return listvalues(color_dict)
return color_dict
else:
return color_dict[name]
def logger(param_report, finfo={}, popt={}, fitparams={}, kind='xdpm', fit_on='average', array_names = ['y', 'wts', 'yhat']):
""" logs information by opening and appending to an existing log file
(named according to parameter dependencies) or creating a new log.
"""
# functions for writing numpy arrays to strings (ex. "y = np.array([1,2,3])"")
name_equals = lambda name, strvector: '{0} = array([{1}])'.format(name, strvector)
stringify = lambda x: name_equals(x[0], ', '.join('{:f}'.format(n) for n in x[1]))
# brief-ify fitparams reference
fp = dict(deepcopy(fitparams))
# list flattened y, wts, and yhat arrays
arrays = [fp[k].flatten() for k in array_names]
# write arays to strings for easy logging
names_arrays = zip(array_names, arrays)
y_str, wts_str, yhat_str = map(stringify, names_arrays)
if fp['nlevels']==1:
fit_on = ' '.join([fit_on, 'FLAT'])
dep_id = "flat model (no conditional parameters)"
fname = './' + kind + '_flat.txt'
else:
depends_on = fp['depends_on']
fit_on = ' '.join([fit_on, 'FULL'])
pkeys = '_'.join(list(depends_on))
pconds = '_'.join(listvalues(depends_on))
dep_id = "{0} depends on {1}".format(pconds, pkeys)
fname = '_'.join(['./' + kind, pconds + '.txt'])
with open(fname, 'a') as f:
f.write('\n\n')
f.write('==' * 30 + '\n\n')
f.write(str(fit_on) + '\n')
f.write(str(kind) + '\n')
f.write(str(dep_id) + '\n\n')
f.write(wts_str + '\n\n')
f.write(yhat_str + '\n\n')
f.write(y_str + '\n\n')
f.write('--' * 30 + '\n')
f.write("FIT REPORT")
f.write('\n' + '--' * 30 + '\n')
f.write(param_report)
f.write('\n' + '--' * 30 + '\n')
f.write('='.join(['popt', repr(popt)]) + '\n')
f.write('AIC: %.8f' % finfo['AIC'] + '\n')
f.write('BIC: %.8f' % finfo['BIC'] + '\n')
f.write('chi: %.8f' % finfo['chi'] + '\n')
f.write('rchi: %.8f' % finfo['rchi'] + '\n')
f.write('converged: %s' % finfo['cnvrg'] + '\n\n')
f.write('==' * 30 + '\n\n\n')
def updateIntensities(self, fiContainer, iKey='intensity'):
""" #TODO: docstring
:param fiContainer:
:param iKey: Attribute name of :class:`Fi` that contains the feature
intensity or an abundance measure. Default "intensity"
"""
for fgi in listvalues(self.container):
intensities = list()
specfileIds = {i: j for i, j in zip(fgi.specfiles, fgi.featureIds)}
for specfile in self._matrixTemplate:
if specfile in specfileIds:
fi = fiContainer.getItem(specfile, specfileIds[specfile])
intensities.append(getattr(fi, iKey))
else:
intensities.append(None)
fgi.intensities = intensities
def get_sessions(self, include_unloggedin=False):
"""
Returns the connected session objects.
Args:
include_unloggedin (bool, optional): Also list Sessions
that have not yet authenticated.
Returns:
sessions (list): A list of `Session` objects.
"""
if include_unloggedin:
return listvalues(self)
else:
return [session for session in self.values() if session.logged_in]
def scalarize_params(params, pc_map=None, is_flat=True):
""" scalarize all parameters in params dict """
exclude = []
if isinstance(params, pd.Series):
params = params.to_dict()
if pc_map is not None and not is_flat:
exclude = np.sort(list(pc_map))
p_conds = np.sort(listvalues(pc_map)).squeeze()
for pkc in exclude:
params[pkc] = array([params[pc] for pc in p_conds])
for pk in list(params):
if pk in exclude:
continue
if hasattr(params[pk], '__iter__'):
try:
params[pk] = np.asscalar(params[pk])
except ValueError:
params[pk] = np.mean(params[pk])
return params
def get_users(project, context, all_perms=True):
"""
Return a list of users and organizations that have access to the project.
"""
if all_perms:
users = cache.get('{}{}'.format(PROJ_PERM_CACHE, project.pk))
if users:
return users
data = {}
for perm in project.projectuserobjectpermission_set.all():
if perm.user_id not in data:
user = perm.user
if all_perms or user in [
context['request'].user, project.organization
]:
data[perm.user_id] = {
'permissions': [],
'is_org': is_organization(user.profile),
'metadata': user.profile.metadata,
'first_name': user.first_name,
'last_name': user.last_name,
'user': user.username
}
if perm.user_id in data:
data[perm.user_id]['permissions'].append(perm.permission.codename)
for k in list(data):
data[k]['permissions'].sort()
data[k]['role'] = get_role(data[k]['permissions'], project)
del data[k]['permissions']
results = listvalues(data)
if all_perms:
cache.set('{}{}'.format(PROJ_PERM_CACHE, project.pk), results)
return results
def updateFgiAnnotationFromFi(fgiContainer, fiContainer, largerBetter):
""" #TODO: docstring
:param fgiContainer:
:param fiContainer:
:param largerBetter:
"""
for fgi in listvalues(fgiContainer.container):
annotations = list()
for specfile, fiId in zip(fgi.specfiles, fgi.featureIds):
fi = fiContainer.getItem(specfile, fiId)
if not fi.isAnnotated:
continue
annotations.append([fi.score, fi.peptide, fi.sequence])
annotations.sort(reverse=largerBetter)
if len(annotations) > 0:
fgi.isAnnotated = True
fgi.score = annotations[0][0]
fgi.peptide = annotations[0][1]
fgi.sequence = annotations[0][2]
else:
fgi.isAnnotated = False
def massTimeContinuityGroups(arrays, mKey, tKey, mLimit, tLimit):
""" #TODO docstring
:param arrays: a dictionary containing ``numpy.arrays``, must be sorted
according to the "mKey" (mass key) value.
:param mKey: "arrays" key that contains the mass ``numpy.array``
:param tKey: "arrays" key that contains the time ``numpy.array``
:param mLimit: maximal mass difference for separating continuity groups
:param tLimit: maximal time difference for separating continuity groups
:returns: a list containing array positions of continuous groups."""
arrayPositions = numpy.array(range(listvalues(arrays)[0].size))
finalGroupPositions = list()
for start, end in continuityGrouping(arrays[mKey], mLimit):
if start == end:
finalGroupPositions.append(arrayPositions[start:end+1])
continue
#Perform time continuity grouping on the mass continuity groups
preSelectionT = arrays[tKey][start:end+1]
preSelectionM = arrays[mKey][start:end+1]
preSelectionPositions = arrayPositions[start:end+1]
_sort = numpy.argsort(preSelectionT)
preGroups = continuityGrouping(preSelectionT[_sort], tLimit)
#Perform a second round of mass continuity grouping
finalGroupPrePos = list()
for _start, _end in preGroups:
preGroupPos = sorted(_sort[_start:_end+1])
secGroups = continuityGrouping(preSelectionM[preGroupPos], mLimit)
for fStart, fEnd in secGroups:
finalGroupPrePos.append(preGroupPos[fStart:fEnd+1])
#Add the final group positions
for _pos in finalGroupPrePos:
finalGroupPositions.append(preSelectionPositions[_pos])
return finalGroupPositions
def proximityGrouping(matchArr, distInfo, distLimit, categoryKey):
""" #TODO: docstring. Group according to the distance value provided by
``distInfo['eucDist']`` with the limitation that each ... category value
can occur only once per group.
:param matchArr: #TODO: docstring
:param distInfo: #TODO: docstring, must be sorted, provide keys "posPairs"
and "eucDist". As generated by :func:`calcDistMatchArr()`
:param distLimit: #TODO: docstring
:param categoryKey: #TODO: docstring
:returns: #TODO: docstring
"""
#Group fi according to their proximity
matchArrSize = listvalues(matchArr)[0].size
linkageGroups = {p: [p] for p in range(matchArrSize)}
posToGroup = {p: p for p in range(matchArrSize)}
groupCategories = {p: set([s]) for p, s in zip(range(matchArrSize),
matchArr[categoryKey]
)
}
for (pos1, pos2), dist in zip(distInfo['posPairs'], distInfo['eucDist']):
if dist > distLimit:
break
id1 = posToGroup[pos1]
id2 = posToGroup[pos2]
if groupCategories[id1].intersection(groupCategories[id2]):
continue
linkageGroups[id1].extend(linkageGroups[id2])
groupCategories[id1].update(groupCategories[id2])
for _pos in linkageGroups[id2]:
posToGroup[_pos] = id1
del linkageGroups[id2]
del groupCategories[id2]
return linkageGroups
def _datablocks_on_item(self, item):
"""
Return a list of datablock locators, recursively navigating
through subcollections
"""
if isinstance(item, arvados.arvfile.ArvadosFile):
if item.size() == 0:
# Empty file locator
return ["d41d8cd98f00b204e9800998ecf8427e+0"]
else:
locators = []
for segment in item.segments():
loc = segment.locator
locators.append(loc)
return locators
elif isinstance(item, arvados.collection.Collection):
l = [self._datablocks_on_item(x) for x in listvalues(item)]
# Fast list flattener method taken from:
# http://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python
return [loc for sublist in l for loc in sublist]
else:
return None
def get_users(self, obj):
xform_perms = []
if obj:
xform_perms = cache.get(
'{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk))
if xform_perms:
return xform_perms
cache.set('{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk),
xform_perms)
data = {}
for perm in obj.xformuserobjectpermission_set.all():
if perm.user_id not in data:
user = perm.user
data[perm.user_id] = {
'permissions': [],
'is_org': is_organization(user.profile),
'metadata': user.profile.metadata,
'first_name': user.first_name,
'last_name': user.last_name,
'user': user.username
}
if perm.user_id in data:
data[perm.user_id]['permissions'].append(
perm.permission.codename)
for k in list(data):
data[k]['permissions'].sort()
data[k]['role'] = get_role(data[k]['permissions'], XForm)
del (data[k]['permissions'])
xform_perms = listvalues(data)
cache.set('{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk), xform_perms)
return xform_perms
def applySiiRanking(siiContainer, specfile):
"""Iterates over all Sii entries of a specfile in siiContainer and sorts Sii
elements of the same spectrum according to the score attribute specified in
``siiContainer.info[specfile]['rankAttr']``. Sorted Sii elements are then
ranked according to their sorted position, if multiple Sii have the same
score, all get the same rank and the next entries rank is its list position.
:param siiContainer: instance of :class:`maspy.core.SiiContainer`
:param specfile: unambiguous identifier of a ms-run file. Is also used as
a reference to other MasPy file containers.
"""
attr = siiContainer.info[specfile]['rankAttr']
reverse = siiContainer.info[specfile]['rankLargerBetter']
for itemList in listvalues(siiContainer.container[specfile]):
sortList = [(getattr(sii, attr), sii) for sii in itemList]
itemList = [sii for score, sii in sorted(sortList, reverse=reverse)]
#Rank Sii according to their position
lastValue = None
for itemPosition, item in enumerate(itemList, 1):
if getattr(item, attr) != lastValue:
rank = itemPosition
item.rank = rank
lastValue = getattr(item, attr)
