def parseFile(filename, patIdx, medIdx, diagIdx, labelIdx, delim="|"): """ Parse a csv file using the delimiter and the appropriate columns of interest. The resultant sparse tensor has patient on the 0th mode, diagnosis on the 1st mode, and medications on the 2nd mode. Tensor info contains the axis information for each mode. """ print "Creating the tensor for " + filename patList = OrderedDict(sorted({}.items(), key=lambda t:t[1])) medList = OrderedDict(sorted({}.items(), key=lambda t:t[1])) diagList = OrderedDict(sorted({}.items(), key=lambda t:t[1])) patClass = OrderedDict(sorted({}.items(), key=lambda t:t[1])) ## storing tensor class as empty array tensorIdx = np.array([[0, 0, 0]]) datfile = open(filename) for i, line in enumerate(datfile): line = line.rstrip('\r\n') parse = line.split(delim) # insert them into the list if necessary if not patList.has_key(parse[patIdx]): patList[parse[patIdx]] = len(patList) patClass[parse[patIdx]] = parse[labelIdx] if not diagList.has_key(parse[diagIdx]): diagList[parse[diagIdx]] = len(diagList) if not medList.has_key(parse[medIdx]): medList[parse[medIdx]] = len(medList) patId = patList.get(parse[patIdx]) diagId = diagList.get(parse[diagIdx]) medId = medList.get(parse[medIdx]) # we know the first one is already mapped if i > 1: tensorIdx = np.append(tensorIdx, [[patId, diagId, medId]], axis=0) tensorVal = np.ones((tensorIdx.shape[0], 1)) # initialize size siz = np.array([len(patList), len(diagList), len(medList)]) X = sptensor.sptensor(tensorIdx, tensorVal, siz) tensorInfo = {} tensorInfo['axis'] = [patList.keys(), diagList.keys(), medList.keys()] tensorInfo['pat'] = patList.keys() tensorInfo['med'] = medList.keys() tensorInfo['diag'] = diagList.keys() tensorInfo['class'] = patClass.values() return X, tensorInfo
class LRUCache(OrderedDict): """ #实现一个LRU算法的dict,用于记录order_id #避免storm平台重发log记录,造成redis数据偏大! """ def __init__(self, capacity): """ @param capacity 字典长度 """ self.capacity = capacity self.cache = OrderedDict() def get(self,key): if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value else: value = None return value def set(self,key,value): if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value else: if len(self.cache) == self.capacity: #pop出第一个item self.cache.popitem(last = False) self.cache[key] = value else: self.cache[key] = value
def getParentTransforms(self, variableName, topLayerName=None): layerNames = self.layers.keys() if topLayerName: layerIndex = layerNames.index(topLayerName) else: layerIndex = len(self.layers) + 1 transforms = OrderedDict() transforms[variableName] = CaffeTransform([1., 1.], [1., 1.], [1., 1.]) for layerName in reversed(layerNames[0:layerIndex]): layer = self.layers[layerName] layerTfs = layer.getTransforms(self) for i, inputName in enumerate(layer.inputs): tfs = [] if transforms.has_key(inputName): tfs.append(transforms[inputName]) for j, outputName in enumerate(layer.outputs): if layerTfs[i][j] is None: continue if transforms.has_key(outputName): composed = composeTransforms(layerTfs[i][j], transforms[outputName]) tfs.append(composed) if len(tfs) > 0: # should resolve conflicts, not simply pick the first tf transforms[inputName] = tfs[0] return transforms
class LRUCache(object): def __init__(self, capacity): """ :type capacity: int """ self.cache = OrderedDict() self.capacity = capacity def get(self, key): """ :rtype: int """ if self.cache.has_key(key): val = self.cache[key] del self.cache[key] self.cache[key] = val return val return -1 def set(self, key, value): """ :type key: int :type value: int :rtype: nothing """ if self.cache.has_key(key): del self.cache[key] self.cache[key] = value else: if len(self.cache) >= self.capacity: self.cache.popitem(last=False) self.cache[key] = value else: self.cache[key] = value
class LRUCache(OrderedDict): '''不能存储可变类型对象,不能并发访问set()''' def __init__(self,capacity): self.capacity = capacity self.cache = OrderedDict() def get(self,key): if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value else: value = None return value def set(self,key,value): if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value else: if len(self.cache) == self.capacity: self.cache.popitem(last = False) #pop出第一个item self.cache[key] = value else: self.cache[key] = value
class LRUCache(object): def __init__(self, capacity): self.cap = capacity self.cache = OrderedDict() def get(self, key): if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value return value else: return -1 def set(self, key, value): if self.cache.has_key(key): self.cache.pop(key) self.cache[key] = value else: if len(self.cache) == self.cap: self.cache.popitem(False) self.cache[key] = value else: self.cache[key] = value
class LRUCache(OrderedDict): def __init__(self, capacity): self.capacity = capacity self.cache = OrderedDict() def get(self, key): if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value else: value = None return value def set(self, key, value): global count_s global count_f global count_m if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value count_s += 1 else: count_f += 1 if len(self.cache) == self.capacity: self.cache.popitem(last=False) self.cache[key] = value count_m += 1 else: self.cache[key] = value
class StatusConsole: """ A static console for displaying parameters that change over time. """ def __init__(self,x,y,width): self.parameters = OrderedDict() self.paramCount = 0 self.x = x self.y = y self.title = '' self.width = width self.label = pyglet.text.Label(self.title, font_name='Consolas', font_size=10, color = (255, 255, 255, 255), x=self.x, y=self.y, width=self.width, multiline=True) def addParameter(self, parameter): if(not self.parameters.has_key(parameter)): self.parameters[parameter] = None self.modified = True def setParameter(self, parameter, value): if(self.parameters.has_key(parameter)): self.parameters[parameter] = value self.modified = True def setTitle(self, string=''): self.title = string def updateLabel(self): s = ''.join([str(a) + " : " + str(b) + '\n' for a,b in self.parameters.items()]) s = self.title + '\n' + s self.label.text = s def draw(self): if(self.modified): self.updateLabel() self.label.draw()
class FIFOCache: def __init__(self, max_cache_size=10): self.cache = OrderedDict() self.max_size = max_cache_size def set(self, key, value): ''' 添加元素, 1.判断是否已满,如果已满则先删除最先插入的元素 2.判断key是否存在,如果存在则直接返回其值 3.不存在则直接添加 :param key: :param value: :return: ''' if self.max_size <= len(self.cache): #缓存已满,删除最先插入的元素 self.cache.popitem(last=False) self.cache[key] = value elif self.cache.has_key(key): self.cache[key] = value else: self.cache[key] = value def get(self, key): ''' 获取元素,存在则先删除,再添加并返回对应的value;否则返回None :param key: :return: ''' if self.cache.has_key(key): value = self.cache[key] return value else: return None
def nodeLabelDict(nodes, trace): # Inverse look up dict for node -> symbol from trace.globalEnv inv_env_dict = OrderedDict() for (sym, env_node) in trace.globalEnv.frame.iteritems(): assert isinstance(env_node, Node) assert not inv_env_dict.has_key(env_node) inv_env_dict[env_node] = sym label_dict = OrderedDict() for node in nodes: if inv_env_dict.has_key(node): label = inv_env_dict[node] elif isOutputNode(node): label = 'O' # 'Output' #: ' + str(node.value) elif isRequestNode(node): label = 'R' # 'Request' #: ' + str(node.value) elif isLookupNode(node): label = 'L' # 'Lookup' elif isConstantNode(node): label = 'C' # 'Constant' else: label = '' # str(node.value) label_dict[node] = label return label_dict
def get_samples(self, manager, cache, resources): meminfo = OrderedDict() if os.path.isfile("/proc/meminfo"): with open("/proc/meminfo") as f: for line in f: temp = line.split(":") meminfo[temp[0]] = temp[1].strip() if (meminfo.has_key('MemTotal') and meminfo.has_key('MemFree')): memtotal = int(meminfo['MemTotal'].strip().split(' ')[0]) memfree = int(meminfo['MemFree'].strip().split(' ')[0]) memutil = (memtotal - memfree)/(1.0*memtotal) memutil = float("%.2f"%memutil)*100 hostinfo = os.popen("uname -a").read() host = hostinfo.split(' ')[1] yield util.make_sample_from_host( host, name='host_mem_util', type=sample.TYPE_DELTA, unit='%', volume=memutil, additional_metadata=None, ) else: LOG.error(" the /proc/meminfo is not exist")
def climbingLeaderboard(scores, alice): ranks = OrderedDict() rank = 1 for i in range(len(scores)): if not ranks.has_key(scores[i]): ranks[scores[i]] = rank rank += 1 serach_from_index = len(scores) - 1 # Loop over Alice's scores highest_rank_acheived = False for i in range(len(alice)): if highest_rank_acheived: print 1 continue if ranks.has_key(alice[i]): print ranks[alice[i]] continue next_largest_score, serach_from_index = find_rank_index( scores, alice[i], serach_from_index) #print "i =", i, " alice_score =", alice[i], " next_higher_score =", next_largest_score if next_largest_score is None: # If there is no greater score than this, then alice has reached RANK 1 highest_rank_acheived = True print 1 else: print ranks[next_largest_score] + 1
class LRUCache(OrderedDict): '''不能存储可变类型对象,不能并发访问set()''' def __init__(self, capacity): self.capacity = capacity self.cache = OrderedDict() def get(self, key): if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value else: value = None return value def set(self, key, value): if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value else: if len(self.cache) == self.capacity: self.cache.popitem(last=False) #pop出第一个item self.cache[key] = value else: self.cache[key] = value
class LRUCache(object): def __init__(self, capacity): """ :type capacity: int """ self.capacity = capacity self.items = OrderedDict() def get(self, key): """ :type key: int :rtype: int """ if not self.items.has_key(key): return -1 value = self.items.pop(key) self.items[key] = value return value def put(self, key, value): """ :type key: int :type value: int :rtype: void """ if self.items.has_key(key): self.items.pop(key) self.items[key] = value return if len(self.items) >= self.capacity: self.items.popitem(False) self.items[key] = value
class LRUCache(object): def __init__(self, capacity): """ :type capacity: int """ self._len = capacity self._dict = OrderedDict([]) def get(self, key): """ :type key: int :rtype: int """ if not self._dict.has_key(key): return -1 self._dict[key] = self._dict.pop(key) return self._dict[key] def put(self, key, value): """ :type key: int :type value: int :rtype: void """ if self._dict.has_key(key): self._dict.pop(key) if len(self._dict) < self._len: self._dict.popitem(last=False) self._dict[key] = value
def score_with_cocoeval(samples_valid, samples_test, engine): scorer = COCOScorer() if samples_valid: gts_valid = OrderedDict() for ID in engine.val_data_ids: vidID, capID = ID.split('|') words = engine.get_cap_tokens(vidID, int(capID), mode='val') caption = ' '.join(words) if gts_valid.has_key(vidID): gts_valid[vidID].append({'image_id': vidID, 'caption': caption, 'cap_id': capID}) else: gts_valid[vidID] = [{'image_id': vidID, 'caption': caption, 'cap_id': capID}] valid_score = scorer.score(gts_valid, samples_valid, gts_valid.keys()) else: valid_score = None if samples_test: gts_test = OrderedDict() for ID in engine.test_data_ids: vidID, capID = ID.split('|') words = engine.get_cap_tokens(vidID, int(capID), mode='test') caption = ' '.join(words) if gts_test.has_key(vidID): gts_test[vidID].append({'image_id': vidID, 'caption': caption, 'cap_id': capID}) else: gts_test[vidID] = [{'image_id': vidID, 'caption': caption, 'cap_id': capID}] test_score = scorer.score(gts_test, samples_test, gts_test.keys()) else: test_score = None return valid_score, test_score
def compare_betweenPair(num): dbconn = MySQLdb.connect('10.141.221.73', 'root', 'root', 'fdroid') dbcursor = dbconn.cursor() # sql = 'select block_id,block_code from fdroid.cc_block where detection_id=1 and detection_tp = "20150101--20150131"' sql = 'SELECT distinct cc_clonepair.block1_id \ from fdroid.cc_clonepair where \ clonepair_id<%s and cc_clonepair.detection_tp="20150101--20150131"' % num blocksql = "SELECT block_id, block_code FROM fdroid.cc_block \ where detection_id=1 and detection_tp = '20150101--20150131' \ and block_id = %s" dbcursor.execute(sql) SourceCodedistanceDict = OrderedDict() CodeTokendistanceDict = OrderedDict() tem = dbcursor.fetchall() for i in range(len(tem)-1): print i blockid1 = tem[i][0] blockid2 = tem[i+1][0] sql1 = blocksql % blockid1 sql2 = blocksql % blockid2 dbcursor.execute(sql1) code1 = dbcursor.fetchone()[1].replace(r"\n","").replace(r"\t","").replace(r"b'","").replace(r"',","") hash1 = getHash.get_nilsimsa(code1) with open('lexer/tem.code', 'w') as f: f.write(code1) os.popen('java -jar codeLexer.jar lexer/tem.code') token1 = lexerparser.parse() dbcursor.execute(sql2) code2 = dbcursor.fetchone()[1].replace(r"\n","").replace(r"\t","").replace(r"b'","").replace(r"',","") hash2 = getHash.get_nilsimsa(code2) with open('lexer/tem.code', 'w') as f: f.write(code2) os.popen('java -jar codeLexer.jar lexer/tem.code') token2 = lexerparser.parse() dist = getHash.compare_hash(hash1,hash2) if SourceCodedistanceDict.has_key(dist): SourceCodedistanceDict[dist] += 1 else: SourceCodedistanceDict[dist] = 1 dist = getHash.compare_hash(getHash.get_nilsimsa(token1),getHash.get_nilsimsa(token2)) # print code1 # print code2 # print token1,token2 if CodeTokendistanceDict.has_key(dist): CodeTokendistanceDict[dist] += 1 else: CodeTokendistanceDict[dist] = 1 dbcursor.close() dbconn.close() return SourceCodedistanceDict, CodeTokendistanceDict
class Recording(object): """Recording Class is used for managing the recording video process.""" #----------------------------------------------------------------------# # Devices Class Constructor # #----------------------------------------------------------------------# def __init__(self): """Devices Class Constructor.""" # Creates a dictionary for managering multiple cameras. self.__videos = OrderedDict() self.__fourcc = cv2.VideoWriter_fourcc(*"WMV2") #----------------------------------------------------------------------# # Public Class Methods # #----------------------------------------------------------------------# def AddVideo(self, index, fps=30.0, size=(640, 480)): """Add a new video in the Python dictionary.""" print index key = str(index) if self.__videos.has_key(key): return False self.__videos[key] = cv2.VideoWriter(index, self.__fourcc, fps, size) return True def DelVideo(self, index): """Remove a video from the Python dictionary.""" key = str(index) if not self.__videos.has_key(key): return False self.__videos[key].release() del self.__videos[key] return True def Write(self, images): """Writes the next video frame.""" if type(images).__module__ == np.__name__: images = [images] for key, image in zip(self.__videos, images): self.__videos[key].write(image) def Release(self): """Closes all videos.""" for key in self.__videos: self.__videos[key].release() self.__videos.clear() #----------------------------------------------------------------------# # Class Methods # #----------------------------------------------------------------------# def __repr__(self): """Get a object representation in a string format.""" return "Framework.RecordingVideos.Recording object."
class ATimeCache(object): """Cache class (dictionary) with a limited size, where only the 'max_entries' most recently added or accessed entries are stored.""" def __init__(self, max_entries): self._cache = OrderedDict() self._max_entries = max_entries def _shrink(self): while len(self._cache) > self._max_entries: self._cache.popitem(last=False) def get_max_entries(self): return self._max_entries def set_max_entries(self, value): self._max_entries = value self._shrink() max_entries = property( get_max_entries, set_max_entries, None, "Set or get the cache size") def has_key(self, key): return self._cache.has_key(key) def __eq__(self, other): try: return self._cache.__eq__(other._cache) except: return False def __len__(self): return self._cache.__len__() def __getitem__(self, key): value = self._cache.pop(key) self._cache[key] = value return value def __setitem__(self, key, value): if self._cache.has_key(key): self._cache.pop(key) self._cache.__setitem__(key, value) self._shrink() def __contains__(self, key): return self.has_key(key) def __str__(self): return self.cache.__str__() def __iter__(self): # Iterate directly on the underlying dict, rather than on this # class, in order to change the order of cached items (as # opposed to []/__getitem__, which will reinsert an item on top # of the stack whenever it is looked up. return iter(self._cache)
class ATimeCache(object): """Cache class (dictionary) with a limited size, where only the 'max_entries' most recently added or accessed entries are stored.""" def __init__(self, max_entries): self._cache = OrderedDict() self._max_entries = max_entries def _shrink(self): while len(self._cache) > self._max_entries: self._cache.popitem(last=False) def get_max_entries(self): return self._max_entries def set_max_entries(self, value): self._max_entries = value self._shrink() max_entries = property(get_max_entries, set_max_entries, None, "Set or get the cache size") def has_key(self, key): return self._cache.has_key(key) def __eq__(self, other): try: return self._cache.__eq__(other._cache) except: return False def __len__(self): return self._cache.__len__() def __getitem__(self, key): value = self._cache.pop(key) self._cache[key] = value return value def __setitem__(self, key, value): if self._cache.has_key(key): self._cache.pop(key) self._cache.__setitem__(key, value) self._shrink() def __contains__(self, key): return self.has_key(key) def __str__(self): return self.cache.__str__() def __iter__(self): # Iterate directly on the underlying dict, rather than on this # class, in order to change the order of cached items (as # opposed to []/__getitem__, which will reinsert an item on top # of the stack whenever it is looked up. return iter(self._cache)
class Recording(object): """Recording Class is used for managing the recording video process.""" #----------------------------------------------------------------------# # Devices Class Constructor # #----------------------------------------------------------------------# def __init__(self): """Devices Class Constructor.""" # Creates a dictionary for managering multiple cameras. self.__videos = OrderedDict() self.__fourcc = cv2.VideoWriter_fourcc(*"WMV2") #----------------------------------------------------------------------# # Public Class Methods # #----------------------------------------------------------------------# def AddVideo(self, index, fps=30.0, size=(640, 480)): """Add a new video in the Python dictionary.""" key = str(index) if self.__videos.has_key(key): return False self.__videos[key] = cv2.VideoWriter(index, self.__fourcc, fps, size) return True def DelVideo(self, index): """Remove a video from the Python dictionary.""" key = str(index) if not self.__videos.has_key(key): return False self.__videos[key].release() del self.__videos[key] return True def Write(self, images): """Writes the next video frame.""" if type(images).__module__ == np.__name__: images = [images] for key, image in zip(self.__videos, images): self.__videos[key].write(image) def Release(self): """Closes all videos.""" for key in self.__videos: self.__videos[key].release() self.__videos.clear() #----------------------------------------------------------------------# # Class Methods # #----------------------------------------------------------------------# def __repr__(self): """Get a object representation in a string format.""" return "Framework.RecordingVideos.Recording object."
def parseFile(filename, patIdx, medIdx, diagIdx, labelIdx, delim="|"): """ Parse a csv file using the delimiter and the appropriate columns of interest. The resultant sparse tensor has patient on the 0th mode, diagnosis on the 1st mode, and medications on the 2nd mode. Tensor info contains the axis information for each mode. """ print "Creating the tensor for " + filename patList = OrderedDict(sorted({}.items(), key=lambda t: t[1])) medList = OrderedDict(sorted({}.items(), key=lambda t: t[1])) diagList = OrderedDict(sorted({}.items(), key=lambda t: t[1])) patClass = OrderedDict(sorted({}.items(), key=lambda t: t[1])) ## storing tensor class as empty array tensorIdx = np.array([[0, 0, 0]]) datfile = open(filename) for i, line in enumerate(datfile): line = line.rstrip('\r\n') parse = line.split(delim) # insert them into the list if necessary if not patList.has_key(parse[patIdx]): patList[parse[patIdx]] = len(patList) patClass[parse[patIdx]] = parse[labelIdx] if not diagList.has_key(parse[diagIdx]): diagList[parse[diagIdx]] = len(diagList) if not medList.has_key(parse[medIdx]): medList[parse[medIdx]] = len(medList) patId = patList.get(parse[patIdx]) diagId = diagList.get(parse[diagIdx]) medId = medList.get(parse[medIdx]) # we know the first one is already mapped if i > 1: tensorIdx = np.append(tensorIdx, [[patId, diagId, medId]], axis=0) tensorVal = np.ones((tensorIdx.shape[0], 1)) # initialize size siz = np.array([len(patList), len(diagList), len(medList)]) X = sptensor.sptensor(tensorIdx, tensorVal, siz) tensorInfo = {} tensorInfo['axis'] = [patList.keys(), diagList.keys(), medList.keys()] tensorInfo['pat'] = patList.keys() tensorInfo['med'] = medList.keys() tensorInfo['diag'] = diagList.keys() tensorInfo['class'] = patClass.values() return X, tensorInfo
def get_net_param_info(self): param_set = OrderedDict() total_count = 0 for layer in self.layers: if (not isinstance(layer,la.InputLayer)): if (not param_set.has_key(layer.params.weights)): param_set[layer.params.weights] = True total_count += layer.params.weights.count if (not param_set.has_key(layer.params.biases)): param_set[layer.params.biases] = True total_count += layer.params.biases.count return param_set,total_count
def main(): parser = argparse.ArgumentParser(description='this is hello example.') parser.add_argument('name', metavar='NAME',type=str, nargs='?',default=None, help='display name.') parser.add_argument('msg', metavar='MESSAGE',type=str, nargs='*', help='display a line of text') parser.add_argument('-v','--verbose',action='store_true',default=None, help='enable verbose.') parser.add_argument('-d','--debug',action='store_true',default=None, help='enable debug.') args = parser.parse_args() conf=config.defaults() if config.has_section("fajoy"): conf=OrderedDict(config.items("fajoy")) log_fmt="%(asctime)s - [%(threadName)s:%(name)s.%(funcName)s((%(lineno)d))] - %(levelname)s - %(message)s" if conf.has_key("log_format"): log_fmt=conf["log_format"] if args.debug: logging.basicConfig(stream=sys.stderr , level=logging.DEBUG, format=log_fmt) if args.verbose: logging.basicConfig(stream=sys.stderr , level=logging.INFO, format=log_fmt) logging.basicConfig(stream=sys.stderr , level=logging.WARNING, format=log_fmt) if conf.has_key("log"): enable_log(conf["log"]) if log.isEnabledFor(logging.INFO): log.info(conf) if log.isEnabledFor(logging.DEBUG): log.debug(conf) name = args.name or "%s%s" % (conf.get("name"),"(%s)" % conf.get("mail") if conf.has_key("mail") else "") msg = " ".join(args.msg) or "hello" e = Exception("test") log.exception(e) print "%s : %s" %(name,msg)
class ParameterSet(object): def __init__(self, definition, parameter_tuple): if len(definition) != len(parameter_tuple): raise ExperimentError('ParameterSet: Parameter set definition and ' 'provided parameters do not agree') self.definition = definition self.ps = parameter_tuple self.names = OrderedDict() for i, n in enumerate(definition.values()): self.names[n] = i self.shortnames = OrderedDict() for i, n in enumerate(definition.keys()): self.shortnames[n] = i def has(self, n): return self.shortnames.has_key(n) or self.names.has_key(n) def get(self, n): if self.shortnames.has_key(n): return self.ps[self.shortnames[n]] elif self.names.has_key(n): return self.ps[self.names[n]] else: raise ExperimentError( 'Parameter "{}" does not exist in this parameter set'.format( n)) def __getattr__(self, n): shortnames = object.__getattribute__(self, 'shortnames') ps = object.__getattribute__(self, 'ps') if not shortnames.has_key(n): raise AttributeError() return ps[shortnames[n]] def __getitem__(self, n): if isinstance(n, int): return self.ps[n] if not self.names.has_key(n): raise IndexError() return self.ps[self.names[n]] def __repr__(self): s = '(' s += ','.join([ '{}={}'.format(n, v) for n, v in zip(self.shortnames.keys(), self.ps) ]) s += ')' return s
def get_notes(sno): db = connect.connect() cursor = db.cursor() cursor.execute('''SELECT line from notes where serialno = %s order by lineno''', sno) results = cursor.fetchall() cursor.close() notes_dict = OrderedDict() ndate, op = "", "" #a line is like ('\x01REC\x0c\x08m\x0c\x08m\n\x08',) for line, in results: ntype, note, operator, date2 = notes.decipher_noteline(line) if date2 != "": ndate = date2 if operator != "": op = operator key = (ndate, op) if notes_dict.has_key(key): notes_dict[key].append((ntype, note)) else: notes_dict[key] = [(ntype, note)] return notes_dict
def longest_amicable_chain_min_member(n): # ), queue_size=600): '''Return the smallest member of the longest amicable chain with no element exceeding n-1. Assumed to be unique, with chain length <= queue_size.''' d, x, c = DCalculator(n).divisor_sum(), 1, np.zeros((n + 1,), dtype=np.int) c.fill(-1) # Initially, set all nodes to unvisited. 0=visited. >0: cycle length at smallest element of the cycle while x < n: # x points to next unvisited element #print 'x', x y, p = x, 0 q = OrderedDict([(y, p)]) #print '\t', y, p while y < n: # Advance along cycle y, p = d[y], p + 1 appeared_before = q.has_key(y) #print '\t', y, p if y > n or c[y] > 0 or appeared_before: break q[y] = p #print '\tlast', y, p, q r = q.keys() c[r] = 0 if appeared_before: i = q[y] z = r[i + np.argmin(r[i:])] c[z] = p - i print x, '\t', 'c[%d] = %d, i=%d' % (z, c[z], i) while c[x] >= 0 and x < n: x += 1 # Advance to next unvisited element # np.set_printoptions(threshold=np.nan) # print c return np.argmax(c)
def from_files(filenames): """Return an iterator that provides a sequence of Histograms for the histograms defined in filenames. """ all_histograms = OrderedDict() for filename in filenames: parser = FILENAME_PARSERS[os.path.basename(filename)] histograms = parser(filename) # OrderedDicts are important, because then the iteration order over # the parsed histograms is stable, which makes the insertion into # all_histograms stable, which makes ordering in generated files # stable, which makes builds more deterministic. if not isinstance(histograms, OrderedDict): raise BaseException, "histogram parser didn't provide an OrderedDict" for (name, definition) in histograms.iteritems(): if all_histograms.has_key(name): raise DefinitionException, "duplicate histogram name %s" % name all_histograms[name] = definition # We require that all USE_COUNTER2_* histograms be defined in a contiguous # block. use_counter_indices = filter(lambda x: x[1].startswith("USE_COUNTER2_"), enumerate(all_histograms.iterkeys())); if use_counter_indices: lower_bound = use_counter_indices[0][0] upper_bound = use_counter_indices[-1][0] n_counters = upper_bound - lower_bound + 1 if n_counters != len(use_counter_indices): raise DefinitionException, "use counter histograms must be defined in a contiguous block" for (name, definition) in all_histograms.iteritems(): yield Histogram(name, definition)
def _check_md5(self, projectDir): # type: (str) -> list, list """ :param projectDir: :return: """ readMe = "" title = "" files = OrderedDict() not_match_files = [] with open(os.path.join(projectDir, "readme.txt"), "rb") as f: i = 0 for line in f: line = line.strip() if i == 0: title = line if i > 2: l = line.split("\t") if len(l) != 2: print line continue files[l[0]] = l[1] i += 1 for key in files.keys(): p = os.path.join(projectDir, key) if files.has_key(key) == False: print "key is not in files:", files, key continue md5Str = files[key] if md5Str != self.md5_checksum(p): not_match_files.append(key) return not_match_files, files
def tags_sorted (self): tags = reversed(sorted(self.tags.iteritems(), key=operator.itemgetter(1))) grouped = OrderedDict() totals = {} for tag in tags: temp = tag[0].split(':') if len(temp) > 1: g = temp[0] t = temp[1] else: g = 'Ungrouped' t = temp[0] if grouped.has_key(g): grouped[g].append([t, None, tag[1]]) totals[g] += tag[1] else: grouped[g] = [[t, None, tag[1]]] totals[g] = tag[1] for key in grouped.keys(): if key != 'Ungrouped': total = 0 for t in grouped[key]: total += t[2] for t in grouped[key]: t[1] = round((float(t[2]) / total) * 100, 1) return grouped.items()
def treasures(): results = sqlrelay.execute_results(""" SELECT GROUP_ID , ITEM_ID , FREQUENCY FROM ARPG_BT_TREASURE """) treasure_list = OrderedDict() for r in results: group_id = int(r[0]) item_id = int(r[1]) frequency = int(r[2]) if treasure_list.has_key(group_id): treasure = treasure_list[group_id] else: treasure = OrderedDict() treasure_list[group_id] = treasure treasure[item_id] = { 'frequency': frequency, } # 토탈 확률계산 for treasure in treasure_list.values(): total = 0 for item in treasure.values(): total += item['frequency'] treasure['total'] = total return treasure_list
def flat(self, pattern='ptc/*_ptc.*.fits', time_stamp=None, verbose=True): if time_stamp is None: time_stamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') exptime = lambda x : pyfits.open(x)[0].header['EXPTIME'] infiles = self._infiles(pattern) # Group files by exposure time and therefore into pairs, presumably. groups = OrderedDict() for infile in infiles: my_exptime = exptime(infile) if not groups.has_key(my_exptime): groups[my_exptime] = [] groups[my_exptime].append(infile) # Translate first two files in each exptime group as flat1 and flat2. for key, infiles in groups.items(): if key == 0 or len(infiles) < 2: # Skip zero exposure frames and groups with only one frame. continue seqno = '%09.4f_flat1' % key self.translate(infiles[0], 'flat', 'flat', seqno, time_stamp=time_stamp, verbose=verbose) seqno = '%09.4f_flat2' % key self.translate(infiles[1], 'flat', 'flat', seqno, time_stamp=time_stamp, verbose=verbose) return time_stamp
def parse(dat): f = open(dat, 'r') first = True labels = [] lines = f.readlines() data = OrderedDict() min = OrderedDict() max = OrderedDict() scale_by = None cur_name = None it = iter(xrange(len(lines))) for i in it: if first: cur_name = lines[i] labels.append(lines[i]) first = False elif lines[i] == '\n': for d in data.values(): if not d.has_key(cur_name): d[cur_name] = 0 for d in min.values(): if not d.has_key(cur_name): d[cur_name] = 0 for d in max.values(): if not d.has_key(cur_name): d[cur_name] = 0 first = True scale_by = None else: print lines[i], lines[i + 1] if not scale_by: scale_by = int(cur_name[:2]) if SCALE_CORE else 1 #float(lines[i+1]) if not data.has_key(lines[i]): data[lines[i]] = OrderedDict() min[lines[i]] = OrderedDict() max[lines[i]] = OrderedDict() # initialize previously missed values to 0 for l in labels: data[lines[i]][l] = 0 min[lines[i]][l] = 0 max[lines[i]][l] = 0 raw_dat = eval(lines[i + 1]) # very safe lol throughputs, neworders = raw_dat print "XXXXX" print throughputs print median(throughputs) print numpy.amin(throughputs) print numpy.amax(throughputs) data[lines[i]][cur_name] = median(throughputs) / scale_by min[lines[i]][cur_name] = numpy.amin(throughputs) / scale_by max[lines[i]][cur_name] = numpy.amax(throughputs) / scale_by next(it) print data return data, min, max, labels
def loadSettings(self): returnData = OD() cur_dict = None try: inifile = open(self.currentFile, 'r') except: raise Exception("Error opening file at %s" % self.currentFile) #Here's the business end of the function for line in inifile: #Ignore any commented out lines if line[0] == "#": continue #New group if line[0] == "[": grpname = re.match("\[(.*?)\]", line).group(1) cur_dict = grpname if returnData.has_key("grpname") is False: returnData[grpname] = OD() #Options elif re.match("(.*?)(?:\s+)?=(?:\s+)?(.*)", line) is not None: if cur_dict is not None: option_line = re.match("(.*?)(?:\s+)?=(?:\s+)?(.*)", line) returnData[cur_dict][str(option_line.group(1)).lower()] = str(option_line.group(2)) else: pass #Gotta close it before we quit inifile.close() return returnData
def delete_duplicates(input_file, my_file): content = "" last_first = re.compile("\w+,\w+") last_first_middle = re.compile("\w+,\w+ \w+") fid = open(input_file) fid_wrt = open(my_file, "w+") lines = fid.readlines() dict = OrderedDict() for line in lines: if len(re.split(":", line)) != 1: name, SSN = re.split(":", line) SSN = SSN.strip("\n") if dict.has_key(SSN): dict[SSN].append(name) else: dict.update({SSN: []}) dict[SSN].append(name) for SSN in dict.iterkeys(): name_array = dict[SSN] split_name = [] name_array.sort(key=lambda item: (-len(item), item)) if re.match(last_first_middle, name_array[0]) or re.match(last_first, name_array[0]): split_name = re.findall(r"[\w']+", name_array[0]) if re.match(last_first_middle, name_array[0]): normal_form = split_name[1] + " " + split_name[2] + " " + split_name[0] elif re.match(last_first, name_array[0]): normal_form = split_name[1] + " " + split_name[0] else: normal_form = name_array[0] content += normal_form + ":" + SSN + "\n" fid_wrt.write(content.rstrip("\n")) fid_wrt.close()
class Cost(): def __init__(self, cost, params, constants=None): self.cost = cost self.grads = OrderedDict() self.computed_cost = False self.params = OrderedDict() for p in params: self.params[p] = True self.constants = OrderedDict() constants = [] if constants is None else constants for c in constants: self.constants[c] = True def compute_gradients(self, lr, multipliers=None): multipliers = OrderedDict() if multipliers is None else multipliers grads = T.grad(self.cost, self.params.keys(), consider_constant=self.constants.keys(), disconnected_inputs='ignore') for param, gparam in zip(self.params.keys(), grads): param_lr = multipliers.get(param.name, 1.0) * lr self.grads[param] = param_lr * gparam self.computed_cost = True def update_gradient(self, param, new_grad): assert self.computed_cost assert self.grads.has_key(param) self.grads[param] = new_grad
def UpdateNLHTIV(nlhtiv): if (nlhtiv.has_key("Trigger_Condition")): tc_list = [] tc = nlhtiv["Trigger_Condition"] #del nlhtiv["Trigger_Condition"] if (tc == "TriggerList"): if (nlhtiv.has_key("Trigger_Condition_List")): tc_list = nlhtiv["Trigger_Condition_List"] else: raise Exception("Expected Trigger_Condition_List") elif (tc == "TriggerString"): if (nlhtiv.has_key("Trigger_Condition_String")): tc_list.append(nlhtiv["Trigger_Condition_String"]) del nlhtiv["Trigger_Condition_String"] else: raise Exception("Expected Trigger_Condition_String") else: tc_list.append(tc) nlhtiv = OrderedDict([("Trigger_Condition_List", tc_list) if k == "Trigger_Condition" else (k, v) for k, v in nlhtiv.items()]) elif (not nlhtiv.has_key("Trigger_Condition_List")): raise Exception("Expected Trigger_Condition") return nlhtiv
def parse_solver(solverfile): solver = OrderedDict() lines = open(solverfile).readlines() for line in lines: line = line.strip() if line[0] == '#': continue if line.find('#') >= 0: line = line.split('#')[0] items = line.split(':') key = items[0].strip() value = items[1].strip().strip('"') try: _has_key = solver.has_key(key) except: _has_key = key in solver if not _has_key: solver[key] = value elif not type(solver[key]) == list: solver[key] = [solver[key], value] else: solver[key].append(value) return solver
def train(self): config = self.config print config.train_batch_size % len(self.unlabeled_loader) self.param_init() self.iter_cnt = 0 iter, min_dev_incorrect = 0, 1e6 monitor = OrderedDict() batch_per_epoch = int((len(self.unlabeled_loader) + config.train_batch_size - 1) / config.train_batch_size) while True: if iter % batch_per_epoch == 0: epoch = iter / batch_per_epoch if epoch >= config.max_epochs: break epoch_ratio = float(epoch) / float(config.max_epochs) # use another outer max to prevent any float computation precision problem self.dis_optimizer.param_groups[0]['lr'] = config.dis_lr * max(0., min(3. * (1. - epoch_ratio), 1.)) self.gen_optimizer.param_groups[0]['lr'] = config.gen_lr * max(0., min(3. * (1. - epoch_ratio), 1.)) iter_vals = self._train(iter=iter) for k, v in iter_vals.items(): if not monitor.has_key(k): monitor[k] = 0. monitor[k] += v if iter % config.vis_period == 0: self.visualize() if iter % config.eval_period == 0: train_loss, train_incorrect = self.eval(self.labeled_loader) dev_loss, dev_incorrect = self.eval(self.dev_loader) min_dev_incorrect = min(min_dev_incorrect, dev_incorrect) disp_str = '#{}\ttrain: {:.4f}, {} | dev: {:.4f}, {} | best: {}'.format( iter, train_loss, train_incorrect, dev_loss, dev_incorrect, min_dev_incorrect) for k, v in monitor.items(): disp_str += ' | {}: {:.4f}'.format(k, v / config.eval_period) disp_str += ' | lr: dis {:.5f}, gen {:.5f}'.format( self.dis_optimizer.param_groups[0]['lr'], self.gen_optimizer.param_groups[0]['lr']) monitor = OrderedDict() print disp_str noise = Variable(torch.Tensor(400, self.config.noise_size).uniform_().cuda(), volatile=True) images = self.gen(noise) images = (images - 0.5) / 0.5 images = images.view(-1, 1, 28, 28) logits = self.pixelcnn(images) log_probs = - pixelcnn_loss.discretized_mix_logistic_loss_c1(images.permute(0, 2, 3, 1), logits.permute(0, 2, 3, 1), sum_all=False).data.cpu() gen_ploss_stats = log_probs.min(), log_probs.max(), log_probs.mean(), log_probs.var() print 'gen stats', gen_ploss_stats print 'unl stats', self.unl_ploss_stats iter += 1 self.iter_cnt += 1
def parse_block(fp): block = OrderedDict() line = fp.readline().strip() while line != '}': ltype = line_type(line) if ltype == 0: # key: value #print line line = line.split('#')[0] key, value = line.split(':') key = key.strip() value = value.strip().strip('"') if block.has_key(key): if type(block[key]) == list: block[key].append(value) else: block[key] = [block[key], value] else: block[key] = value elif ltype == 1: # blockname { key = line.split('{')[0].strip() sub_block = parse_block(fp) block[key] = sub_block line = fp.readline().strip() line = line.split('#')[0] return block
def rougeS(refSent , sents): skip2 = list() newDict = OrderedDict() refLen = len(refSent) noSents = len(sents) noComb = (refLen*(refLen - 1)/2) for i in range(refLen): for j in range(i+1 , refLen): newDict.update({refSent[i] + '_' + refSent[j] : True}) for sent in sents: sentLen = len(sent) count = 0 for p in range(sentLen): for q in range(p+1 , sentLen): if newDict.has_key(sent[p] + '_' + sent[q]): count = count + 1 skip2.append((float(count)/noComb,sent)) return skip2
class Device_Pool(object): '''Holds a number of telemedicine devices to be assigned to patients. Maintains a list of Device objects as well as a dictionary of which device is assigned to which ''' def __init__(self, number_of_devices): self.devices = [ Device(self) for x in xrange(number_of_devices) ] #Instantiate device objects, with a default UUID4 id from the Device's constructor. self.device_assignments = OrderedDict( ) #Maintain an ordered dictionary of which patients are assigned which device. def get_device_count(self): return len( self.devices) #Return the length of the device pool as an integer. def assign_device(self, patient): '''Assign the next available device in the devices attribute, or raise an exception.''' if self.get_device_count( ) > 0: #Still have some devices left to assign. self.device_assignments[ patient.get_patient_id()] = self.devices.pop( ) #Assign a device from the list of device objects. else: raise Exception("No devices left to assign!") #This is where code to add a queue for patients who are recruited but don't have a device yet would go. def return_device(self, patient, device): self.devices.append(device) if self.device_assignments.has_key(patient.get_patient_id()): del self.device_assignments[patient.get_patient_id( )] #Will raise an KeyError if the patient ID isn't in the device assignment. else: pass #Or catch the KeyError.
def format(self, data, keys=None, group_by=None, domain=None): rows_dict = OrderedDict() tmp_data = OrderedDict() sorted_data = [] value_chains = get_domain_configuration(domain).by_type_hierarchy for key, row in data.items(): to_list = list(key) def find_name(list, deep): for element in list: if deep == len(key)-3 and key[deep+1] == element.val: return element.text elif key[deep+1] == element.val: return find_name(element.next, deep+1) name = find_name(value_chains, 0) to_list[2] = name tmp_data.update({tuple(to_list): row}) if tmp_data: sorted_data = sorted(tmp_data.items(), key=lambda x: (x[0][0], x[0][2])) for row in sorted_data: formatted_row = self._format.format_row(row[1]) if not rows_dict.has_key(formatted_row[0]): rows_dict[formatted_row[0]] = [] rows_dict[formatted_row[0]].append(formatted_row[1]) min_length = min([len(item[1]) for item in rows_dict.items()]) for key, row in rows_dict.items(): total_column = self.calculate_total_column(row) res = [key, total_column] res.extend(row[0:min_length]) yield res
def get(self): user = users.get_current_user() if user: #current user name current_username = user.nickname() #report query reportQry = DailyReportModel.query().order(-DailyReportModel.reportDay, -DailyReportModel.writeTime) reportFetch = reportQry.fetch() reportDict = OrderedDict() for item in reportFetch: reportDay = item.getReportDay() if reportDict.has_key(reportDay): reportDict[reportDay].append(item) else: reportDict[reportDay] = [item] # sort OrderedDict(sorted(reportDict.items(), key=lambda t: t[0], reverse=True)) # jinja2 param template_values = { 'current_username' : current_username, 'report_dict' : reportDict, 'date_today' : datetime.datetime.now().strftime("%Y-%m-%d") } template = jinja_env.get_template('MainPage.html') self.response.out.write(template.render(template_values)) else: self.redirect(users.create_login_url(self.request.uri))
def get_idmap(idfile, key_column, value_column, header=False): """ get_dict_from_file ------------------ Use when you want to get the mappings between id's, e.g.,from MGI to Entrez Gene ID Args: idfile: Gene id file (tab-separated values) key_column: column index of keys value_column: column index of values header: True if datafile has a header row Returns: gene_id_map (OrderedDict) """ with open(idfile) as fh: if header is True: fh.next() mydict = OrderedDict() for curline in fh: curline = curline.rstrip() items = curline.split("\t") key = items[key_column] value = items[value_column] if mydict.has_key(key): if not mydict[key] == value: LOG.debug( "Conflicting items: key=%s, registered value=%s <=> new value=%s" % (key, mydict[key], value)) else: mydict[key] = value return mydict
def extract_student_options(options): options.update({ 'poll': Polls.query.order_by(Polls.id.desc()).limit(1).first() }) options.update({ 'history': History.query.order_by(History.timestamp.desc()).limit(10).all() }) options.update({ 'categories': ResourceCategory.query.all() }) if 'testing' in session: testing = session.pop('testing') if session['testing'].has_key('finished') else session['testing'] options.update({ 'testing': { 'tests': Test.query.filter(Test.id.in_(testing['tests'])).all(), 'index': testing['index'], 'answers': testing['answers'], 'correct': testing['correct'], 'finished': testing['finished'] if testing.has_key('finished') else False } }) uploaded_files = File.query.join(Account).join(Student).filter(Student.id == current_user.student.id).order_by( File.timestamp.asc()).all() # if url in ['index', 'files']: options.update({ 'uploadedFiles': uploaded_files }) if current_user.student.stats.all(): options.update({ 'stats': { 'last': current_user.student.stats.order_by(Stat.timestamp.desc()).first(), 'best': current_user.student.stats.order_by(Stat.answers.desc()).first() } }) # STATISTIC group_files_stat = OrderedDict() if current_user.student.group: for student in current_user.student.group.students.order_by(Student.id.desc()): student_files = File.query.join(Account).join(Student).filter( Student.id == student.id).order_by(File.timestamp.desc()).all() files_stat = OrderedDict() for studentFile in student_files: ts = studentFile.timestamp.strftime('%Y.%m.%d') files_stat.update({ ts: files_stat[ts] + 1 if files_stat.has_key(ts) else 1 }) group_files_stat.update({student: files_stat}) options.update({'group_files_stat': group_files_stat})
def client_devices_new(client_id): detail = get_client_detail(client_id) data = detail['data'] device_types_keys = sorted(data["device_types"]) device_types_list = OrderedDict() for key in device_types_keys: device_types_list[key] = data["device_types"][key] products = {} products_per_type = OrderedDict() if "products" in data: products_list = data["products"] products_list = sorted(products_list, key=itemgetter("name")) for prod in products_list: product_label = data['device_types'][prod["type"]]['name'] products[prod["name"]] = prod["type"] #if not products_per_type.has_key(prod["type"]): if not products_per_type.has_key(product_label): products_per_type[product_label] = OrderedDict() products_per_type[product_label][prod['name']] = prod["type"] # TODO : include products icons return render_template('client_device_new.html', device_types = device_types_list, products = products, products_per_type = products_per_type, clientid = client_id, mactive="clients", active = 'devices', client_detail = detail, )
def from_files(filenames): """Return an iterator that provides a sequence of Histograms for the histograms defined in filenames. """ all_histograms = OrderedDict() for filename in filenames: parser = FILENAME_PARSERS[os.path.basename(filename)] histograms = parser(filename) # OrderedDicts are important, because then the iteration order over # the parsed histograms is stable, which makes the insertion into # all_histograms stable, which makes ordering in generated files # stable, which makes builds more deterministic. if not isinstance(histograms, OrderedDict): raise BaseException, "histogram parser didn't provide an OrderedDict" for (name, definition) in histograms.iteritems(): if all_histograms.has_key(name): raise DefinitionException, "duplicate histogram name %s" % name all_histograms[name] = definition # We require that all USE_COUNTER_* histograms be defined in a contiguous # block. use_counter_indices = filter(lambda x: x[1].startswith("USE_COUNTER_"), enumerate(all_histograms.iterkeys())) if use_counter_indices: lower_bound = use_counter_indices[0][0] upper_bound = use_counter_indices[-1][0] n_counters = upper_bound - lower_bound + 1 if n_counters != len(use_counter_indices): raise DefinitionException, "use counter histograms must be defined in a contiguous block" for (name, definition) in all_histograms.iteritems(): yield Histogram(name, definition)
def clients(): cli = MQSyncReq(app.zmq_context) msg = MQMessage() msg.set_action('client.list.get') res = cli.request('manager', msg.get(), timeout=10) if res is not None: client_list = res.get_data() else: client_list = {} client_list_per_host_per_type = OrderedDict() for client in client_list: cli_type = client_list[client]['type'] cli_host = client_list[client]['host'] if not client_list_per_host_per_type.has_key(cli_host): client_list_per_host_per_type[cli_host] = {} if not client_list_per_host_per_type[cli_host].has_key(cli_type): client_list_per_host_per_type[cli_host][cli_type] = {} client_list_per_host_per_type[cli_host][cli_type][client] = client_list[client] return render_template('clients.html', mactive="clients", overview_state="collapse", clients=client_list, client_list_per_host_per_type=client_list_per_host_per_type )
class DebugBox(Window): def __init__(self,*args): super(DebugBox,self).__init__(*args) self.attr_list = OrderedDict() self.next_free = 1 def register(self,text,width,win): ''' store details of watch in an ordered dict of tuples dict1{"attr"+win} = attr, win_ref, width, index index = order on the screen (y coord) ''' if not self.attr_list.has_key(text + str(win)): self.attr_list.__setitem__(text + str(win),(text,win,width,self.next_free)) self.next_free += 1 else: pass #raise Exception # nothing should try to register twice def run(self): while 1: for key in self.attr_list.keys(): text,win,width,index = self.attr_list.__getitem__(key) self.addstr_no_cursor(text,index,1,width,win) time.sleep(1) self.refresh()
def get_representation(self, infile): histograms = self.get_all_histograms(infile) #create dictionary with [category][sample][systematic] -> bin data nd = OrderedDict() cats = set([]) samps = set([]) systs = set([]) for k in sorted(histograms.keys()): samp, cat, syst = self.get_key(k) if not self.filter_key((samp, cat, syst)): continue if not nd.has_key(cat): nd[cat] = OrderedDict() if not nd[cat].has_key(samp): nd[cat][samp] = OrderedDict() if not nd[cat][samp].has_key(syst): nd[cat][samp][syst] = OrderedDict() nd[cat][samp][syst] = get_bins(histograms[k]) samps.add(samp) systs.add(syst) cats.add(cat) self.all_categories = list(cats) self.all_samples = list(samps) self.all_systematics = list(systs) return nd
def getTeacherField(self, item): fields = ['localLang' , 'certification', 'study' , 'qualified_teacher', \ 'ethnic_teacher', 'education' , 'experience', 'teaching_years' , 'remarks'] fieldsName = { 'localLang': _(u'Local Language'), 'certification': _(u'Ethnic language certification'), 'study': _(u'Revitalization study'), 'qualified_teacher': _(u'Teaching class (Qualified teacher)'), 'ethnic_teacher': _(u'Teaching class (Ethnic teacher)'), 'education': _(u'Education'), 'experience': _(u'work experience'), 'teaching_years': _(u'Teaching years'), 'remarks': _(u'Remarks') } fieldsDict = OrderedDict() for field in fields: field_value = getattr(item, field, '') if field_value: fieldsDict.update({fieldsName[field]: field_value}) if fieldsDict.has_key(fieldsName['localLang']): localLangValue = '\r\n'.join([ lang.split(',')[1] for lang in fieldsDict[fieldsName['localLang']].split('/') ]) fieldsDict[fieldsName['localLang']] = localLangValue return fieldsDict
def format(self, data, keys=None, group_by=None, domain=None): rows_dict = OrderedDict() tmp_data = OrderedDict() sorted_data = [] value_chains = get_domain_configuration(domain).by_type_hierarchy for key, row in data.items(): to_list = list(key) def find_name(list, deep): for element in list: if deep == len(key) - 3 and key[deep + 1] == element.val: return element.text elif key[deep + 1] == element.val: return find_name(element.next, deep + 1) name = find_name(value_chains, 0) to_list[2] = name tmp_data.update({tuple(to_list): row}) if tmp_data: sorted_data = sorted(tmp_data.items(), key=lambda x: (x[0][0], x[0][2])) for row in sorted_data: formatted_row = self._format.format_row(row[1]) if not rows_dict.has_key(formatted_row[0]): rows_dict[formatted_row[0]] = [] rows_dict[formatted_row[0]].append(formatted_row[1]) for key, row in rows_dict.items(): total_column = self.calculate_total_column(row) res = [key, total_column] res.extend(row) yield res
def get_readgroup_and_seq_dict_from_bam(bam_files, allow_collision=False): print "Gather seq dict and read groups from %s bam files" % len(bam_files) all_read_groups = {} all_seq_dict = OrderedDict() for bam_file in bam_files: command = "samtools view -H %s | egrep '@RG|@SQ' " % bam_file stdout, process = utils_commands.get_output_stream_from_command(command) for line in stdout: if line.startswith('@RG'): read_group_dict = {} for element in line.strip().split('\t'): if element != '@RG': key, value = element.split(':') read_group_dict[key] = value if read_group_dict.has_key('ID') and read_group_dict.get('ID') not in all_read_groups: all_read_groups[read_group_dict.get('ID')] = read_group_dict if line.startswith('@SQ'): seq_dict = {} for element in line.strip().split('\t'): if element != '@SQ': key, value = element.split(':') if key == 'LN': value = int(value) seq_dict[key] = value if seq_dict.has_key('SN'): name = seq_dict.get('SN') if all_seq_dict.has_key(name) and not allow_collision: raise StandardError( "Identical sequence dictionary name %s in %s and previous bam entry and collision not allowed" % ( name, bam_file)) all_seq_dict[name] = seq_dict return all_read_groups.values(), all_seq_dict.values()