def make_tags(article, index): def score(word, num): word = word.lower() md = hashlib.md5(word.encode('utf-8')).hexdigest()[:2] df = redis_tag.hget(md, word) df = max(int(df), 0) if df is not None else 0 if df < 10: return 0 return num * max(math.log(df + 1, 50), 1.5) index = [(x, score(x, y)) for x, y in sorted(index.iteritems(), key=lambda x: -x[1])] index = filter(lambda x: x[1] > 0, index) tags = OrderedDict(sorted(index, key=lambda x: -x[1])).keys() for tag in list(tags): sub = False for other in tags: if tag == other: sub = True elif tag in other: if len(tag) < 4 and len(other) >= 4 or sub == False: if tag in tags: tags.remove(tag) else: tags.remove(other) return tags[:6]
class Adducts: def __init__(self, ion_mode=None, e=0.0005486): self.e = e if ion_mode == "pos": self.lib = OrderedDict() elif ion_mode == "neg": self.lib = OrderedDict() elif ion_mode is None: self.lib = OrderedDict() def add(self, name, mass, charge): self.lib[name] = OrderedDict([("mass", float(mass)), ("charge", int(charge))]) self.lib = OrderedDict( sorted(self.lib.items(), key=lambda x: x[1]['mass'])) def remove(self, name): if name == "*": self.lib = OrderedDict() else: if name in self.lib: self.lib.remove(name) else: raise IOError("Entry not in library: {}".format(name)) def __str__(self): out = "Adducts in library\n" out += "-----------------\n" out += "name\texact_mass\n" for key in self.lib: out += "%s\t%s\n" % (key, self.lib[key]) return out
def number_of_strongly_connected_components(adj): indegree_adj = [[] for _ in range(len(adj))] # print(indegree_adj) for i in range(len(adj)): for val in adj[i]: indegree_adj[val].append(i) # print(indegree_adj) # postorder = [[0,0] for _ in range(len(adj))] ## Moving to Ordered Dict to handle large datasets postordered_od = OrderedDict({i: [0,0] for i in range(len(adj))}) # find_postorders_reverse_graph(postorder,indegree_adj) find_postorders_reverse_graph(postordered_od,indegree_adj) # print(postordered_od) # Heap solution not viable as after DFS we need to remove traversed components as well # postorder_heap = list((-val[1],node) for node, val in enumerate(postorder)) # heapq.heapify(postorder_heap) # adding minus as heapq is min heap # print(postorder_heap) postordered_od = sorted(postordered_od, key=lambda x: postordered_od[x][1], reverse=True) # print(postordered_od) visited = [False] * len(adj) scc = [] def do_dfs(node, entry): visited[node] = True entry.add(node) for neighbor in adj[node]: if not visited[neighbor]: do_dfs(neighbor, entry) return while not all(visited): # max_postorder_node = find_max_postorder_node(postorder) # Replacing with heap to handle large datasets # max_postorder_val, max_postorder_node = heapq.heappop(postorder_heap) max_postorder_node = postordered_od[0] scc_entry = set() do_dfs(max_postorder_node, scc_entry) # print(scc_entry) scc.append(scc_entry) # scratch_postorder(scc_entry, postorder) # Replacing with heap to handle large datasets for e in scc_entry: postordered_od.remove(e) # print(scc) return len(scc)
class CombInfoDictionary(object): def __init__(self, jsonString=None): self.__dict__ = OrderedDict() if jsonString is not None: decoder = JSONDecoder(object_hook=OrderedDict) self.__dict__ = decoder.decode(jsonString) def __getitem__(self, key): for k, v in self.__dict__.items(): if k == key: return v if type(v) is dict: ret = self._getitem(key, v) if ret is not None: return ret elif type(v) is list: for x in v: ret = self._getitem(key, x) if ret is not None: return ret return None def __setitem__(self, key, val): if key not in self.__dict__.keys(): self.__dict__[key] = val def __delitem__(self, key): if key in self.__dict__.keys(): self.__dict__.remove(key) def __iter__(self): for key in self.__dict__.keys(): node = self.__dict__[key] for x in node.keys(): yield node[x] def __str__(self): return json.dumps(self, default=lambda x: x.__dict__, sort_keys=False, indent=4) def Save(self): if not os.path.isdir(Consts.COMB_OUTPUT_FOLDER): os.mkdir(Consts.COMB_OUTPUT_FOLDER) jsonpath = os.path.realpath(r'{}/CombInfo-{}.json'.format( Consts.COMB_OUTPUT_FOLDER, IdGenerator())) with open(jsonpath, 'w') as outputfile: outputfile.write(str(self))
class Frame: def __init__(self, id): """ Initialize video frame. :return: int sequence number of frame """ self.id = id self.timecode = frames2timecode(self.id) self._droplets = OrderedDict() @property def droplets(self): return self._droplets @property def droplet_count(self): return len(self._droplets) def add_droplet(self): """ Add new droplet to the video frame. :param droplet_id: """ d = Droplet() self._droplets[d.id] = d return d def remove_droplet(self, droplet_id): """ Remove a droplet from the video frame. :param droplet_id: """ self._droplets.remove(droplet_id)
class Metrix(QtGui.QTreeWidget): def __init__(self, parent=None, updateInterval=300): QtGui.QTreeWidget.__init__(self, parent) self.variables = OrderedDict() self.treeitems = {} headerItem = QtGui.QTreeWidgetItem() headerItem.setData(0,0, "Name") headerItem.setData(1,0, "Value") self.setHeaderItem(headerItem) self.setRootIsDecorated(False) self.setWordWrap(True) self.timer = QtCore.QTimer(self) self.timer.setInterval(updateInterval) #self.timer.timerEvent = self.update self.timer.timeout.connect(self.update) self.timer.start() def add(self, name, callback): self.variables[name] = callback self.treeitems[name] = QtGui.QTreeWidgetItem() self.treeitems[name].setData(0, 0, QtCore.QString(name)) self.treeitems[name].setData(1, 0, QtCore.QString(repr(callback()))) self.addTopLevelItem(self.treeitems[name]) self.resizeColumnToContents(0) self.resizeColumnToContents(1) def remove(self, name): self.variables.remove(name) self.removeItemWidget(self.treeitems[name]) del self.treeitems[name] @QtCore.pyqtSlot() def update(self, dummy=None): for name, callback in self.variables.iteritems(): self.treeitems[name].setData(1, 0, repr(callback()))
def main(): pstat = { 'status': {}, 'text': {}, 'short': {}, } now = time.strftime("%Y-%m-%d %H:%M:%S") jid = os.getpid() ckanlistrequests = ['package_list', 'group_list', 'tag_list'] ## Get options and arguments args = get_args(ckanlistrequests) # Output instance OUT = Output(pstat, now, jid, args) logger = OUT.setup_custom_logger('root', args.verbose) ## Settings for CKAN client and API ckanapi3 = 'http://' + args.ckan + '/api/3' if PY2: ckan = ckanclient.CkanClient(ckanapi3) else: auth = '12345' ckan = CKAN_CLIENT(args.ckan, auth) ckan_limit = 500000 start = time.time() if args.request.endswith('list'): try: if args.request == 'community_list': action = 'group_list' else: action = args.request if PY2: answer = ckan.action(action, rows=ckan_limit) else: answer = ckan.action(action) except ckanclient.CkanApiError as e: print('\t\tError %s Supported list requests are %s.' % (e, ckanlistrequests)) sys.exit(1) ## print '|- The list of %ss :\n\t%s' % (args.request.split('_')[0],'\n\t'.join(answer).encode('utf8')) print('\n\t%s' % '\n\t'.join(answer).encode('utf8')) sys.exit(0) # create CKAN search pattern : ckan_pattern = '' sand = '' pattern = ' '.join(args.pattern) if (args.community): ckan_pattern += "groups:%s" % args.community sand = " AND " if (args.pattern): ckan_pattern += sand + pattern print(' | - Search\n\t|- in\t%s\n\t|- for\t%s\n' % (args.ckan, ckan_pattern)) if args.request == 'package_search': if PY2: answer = ckan.action('package_search', q=ckan_pattern, rows=ckan_limit) else: answer = ckan.action('package_search', {"q": ckan_pattern}) for key, value in answer.items(): logger.warning('answer has key %s' % key) if PY2: tcount = answer['count'] else: tcount = answer['result']['count'] print(' | - Results:\n\t|- %d records found in %d sec' % (tcount, time.time() - start)) # Read in B2FIND metadata schema and fields schemafile = '%s/mapfiles/b2find_schema.json' % (os.getcwd()) with open(schemafile, 'r') as f: b2findfields = json.loads(f.read(), object_pairs_hook=OrderedDict) if tcount > 0 and args.keys is not None: if len(args.keys) == 0: akeys = [] else: if args.keys[0] == 'B2FIND.*': akeys = OrderedDict(sorted(b2findfields.keys())) else: akeys = args.keys suppid = b2findfields.keys() fh = io.open(args.output, "w", encoding='utf8') record = {} totlist = [] count = {} count['id'] = 0 statc = {} for outt in akeys: if outt not in suppid: print(' [WARNING] Not supported key %s is removed' % outt) akeys.remove(outt) else: count[outt] = 0 statc[outt] = Counter() printfacets = '' if (len(akeys) > 0): printfacets = "and related facets %s " % ", ".join(akeys) print('\t|- IDs %sare written to %s ...' % (printfacets, args.output)) counter = 0 cstart = 0 oldperc = 0 start2 = time.time() while (cstart < tcount): if (cstart > 0): if PY2: answer = ckan.action('package_search', q=ckan_pattern, rows=ckan_limit, start=cstart) else: answer = ckan.action('package_search', { "q": ckan_pattern, "rows": ckan_limit, "start": cstart }) if PY2: if len(answer['results']) == 0: break #HEW-D else: ##HEW-D if len(answer['result']['results']) == 0 : ##HEW-D break # loop over found records if PY2: results = answer['results'] else: results = answer['result']['results'] for ds in results: #### answer['results']: counter += 1 logger.debug(' | %-4d | %-40s |' % (counter, ds['name'])) perc = int(counter * 100 / tcount) bartags = perc / 5 if perc % 10 == 0 and perc != oldperc: oldperc = perc print('\r\t[%-20s] %5d (%3d%%) in %d sec' % ('=' * int(bartags), counter, perc, time.time() - start2)) sys.stdout.flush() record['id'] = '%s' % (ds['name']) outline = record['id'] # loop over facets for facet in akeys: ##HEW-T print 'facet : %s' % facet ckanFacet = b2findfields[facet]["ckanName"] if ckanFacet in ds: ## CKAN default field if facet == 'Group': record[facet] = ds[ckanFacet][0]['display_name'] else: record[facet] = ds[ckanFacet] else: ## CKAN extra field ##HEW-T print 'ds extras %s' % ds['extras'] efacet = [e for e in ds['extras'] if e['key'] == facet] if efacet: ##HEW-T print 'rrrr %s effff %s' % (record[facet],efacet[0]['value']) record[facet] = efacet[0]['value'] else: record[facet] = 'N/A' if record[facet] is None: record[facet] = 'None' statc[facet][record[facet]] += 1 else: if not isinstance(record[facet], list): words = record[facet].split(';') else: words = record[facet] for word in words: if isinstance(word, dict): word = word['name'] statc[facet][word] += 1 if not (record[facet] == 'N/A' or record[facet] == 'Not Stated') and len(record[facet]) > 0: count[facet] += 1 outline += '\t | %-30s' % record[facet][:30] fh.write(outline + '\n') cstart += len(results) logger.warning('%d records done, %d in total' % (cstart, tcount)) fh.close() if len(akeys) > 0: statfh = io.open('stat_' + args.output, "w", encoding='utf8') ##print "\n|- Statistics :\n\t| %-16s | %-10s | %6s |\n\t%s " % ('Facet','Occurence','%',"-" * 50) print('|- Statistics written to file %s' % 'stat_' + args.output) statline = unicode("") for outt in akeys: statline += "| %-16s\n\t| %-15s | %-6d | %3d |\n" % ( outt, '-Total-', count[outt], int(count[outt] * 100 / tcount)) for word in statc[outt].most_common(10): statline += '\t| %-15s | %-6d | %3d |\n' % ( word[0][:100], word[1], int(word[1] * 100 / tcount)) statfh.write(statline) statfh.close()
def main(): ckanlistrequests=['package_list','group_list','tag_list'] ## Get options and arguments args = get_args(ckanlistrequests) ## Settings for CKAN client and API ckanapi3='http://'+args.ckan+'/api/3' ckan = ckanclient.CkanClient(ckanapi3) ckan_limit=500000 start=time.time() if args.request.endswith('list'): try: if args.request == 'community_list' : action='group_list' else: action=args.request answer = ckan.action(action, rows=ckan_limit) except ckanclient.CkanApiError as e : print '\t\tError %s Supported list requests are %s.' % (e,ckanlistrequests) sys.exit(1) ## print '|- The list of %ss :\n\t%s' % (args.request.split('_')[0],'\n\t'.join(answer).encode('utf8')) print '\n\t%s' % '\n\t'.join(answer).encode('utf8') sys.exit(0) # create CKAN search pattern : ckan_pattern = '' sand='' pattern=' '.join(args.pattern) if (args.community): ckan_pattern += "groups:%s" % args.community sand=" AND " if (args.pattern): ckan_pattern += sand + pattern print ' | - Search\n\t|- in\t%s\n\t|- for\t%s\n' % (args.ckan,ckan_pattern) if args.request == 'package_search' : answer = ckan.action('package_search', q=ckan_pattern, rows=ckan_limit) tcount=answer['count'] print " | - Results:\n\t|- %d records found in %d sec" % (tcount,time.time()-start) # Read in B2FIND metadata schema and fields schemafile = '%s/mapfiles/b2find_schema.json' % (os.getcwd()) with open(schemafile, 'r') as f: b2findfields=json.loads(f.read(), object_pairs_hook=OrderedDict) if tcount>0 and args.keys is not None : if len(args.keys) == 0 : akeys=[] else: if args.keys[0] == 'B2FIND.*' : akeys=OrderedDict(sorted(b2findfields.keys())) else: akeys=args.keys suppid=b2findfields.keys() fh = io.open(args.output, "w", encoding='utf8') record={} totlist=[] count={} count['id']=0 statc={} for outt in akeys: if outt not in suppid : print ' [WARNING] Not supported key %s is removed' % outt akeys.remove(outt) else: count[outt]=0 statc[outt] = Counter() printfacets='' if (len(akeys) > 0): printfacets="and related facets %s " % ", ".join(akeys) print "\t|- ID's %sare written to %s ..." % (printfacets,args.output) counter=0 cstart=0 oldperc=0 start2=time.time() while (cstart < tcount) : if (cstart > 0): answer = ckan.action('package_search', q=ckan_pattern, rows=ckan_limit, start=cstart) if len(answer['results']) == 0 : break # loop over found records for ds in answer['results']: counter +=1 ##HEW-T print' | %-4d | %-40s |' % (counter,ds['name']) perc=int(counter*100/tcount) bartags=perc/5 if perc%10 == 0 and perc != oldperc : oldperc=perc print "\r\t[%-20s] %5d (%3d%%) in %d sec" % ('='*bartags, counter, perc, time.time()-start2 ) sys.stdout.flush() record['id'] = '%s' % (ds['name']) outline=record['id'] # loop over facets for facet in akeys: ##HEW-T print 'facet : %s' % facet ckanFacet=b2findfields[facet]["ckanName"] if ckanFacet in ds: ## CKAN default field if facet == 'Group': record[facet] = ds[ckanFacet][0]['display_name'] else: record[facet] = ds[ckanFacet] else: ## CKAN extra field ##HEW-T print 'ds extras %s' % ds['extras'] efacet=[e for e in ds['extras'] if e['key'] == facet] if efacet: ##HEW-T print 'rrrr %s effff %s' % (record[facet],efacet[0]['value']) record[facet] = efacet[0]['value'] else: record[facet] = 'N/A' if record[facet] is None : record[facet]='None' statc[facet][record[facet]]+=1 else: if not isinstance(record[facet],list): words=record[facet].split(';') else: words=record[facet] for word in words: if isinstance(word,dict): word=word['name'] statc[facet][word]+=1 if not ( record[facet] == 'N/A' or record[facet] == 'Not Stated') and len(record[facet])>0 : count[facet]+=1 outline+='\t | %-30s' % record[facet][:30] ##outline+='\n \t| %-20s' % (statc[aid].keys(),statc[aid].values()) fh.write(outline+'\n') cstart+=len(answer['results']) fh.close() if len(akeys) > 0 : statfh = io.open('stat_'+args.output, "w", encoding='utf8') ##print "\n|- Statistics :\n\t| %-16s | %-10s | %6s |\n\t%s " % ('Facet','Occurence','%',"-" * 50) print '|- Statistics written to file %s' % 'stat_'+args.output statline=unicode("") for outt in akeys: statline+= "| %-16s\n\t| %-15s | %-6d | %3d |\n" % (outt,'-Total-',count[outt],int(count[outt]*100/tcount)) for word in statc[outt].most_common(10): statline+= '\t| %-15s | %-6d | %3d |\n' % (word[0][:100], word[1], int(word[1]*100/tcount)) statfh.write(statline) statfh.close()
while (catstartnum < num): catstartnum += category_offset # move all VOD categories to VOD placeholder position if ("VOD" in category_order): vodindex = category_order.index("VOD") vodcategories = list( (cat for cat in category_order if cat.startswith('VOD -'))) if len(vodcategories): #remove the multi vod categories from their current location category_order = [ x for x in category_order if x not in vodcategories ] #insert the multi vod categories at the placeholder pos category_order[vodindex:vodindex] = vodcategories category_order.remove("VOD") # Check for and parse override map self.parse_map_channels_xml(dictchannels) # Have a look at what we have if DEBUG and TESTRUN: datafile = open(os.path.join(EPGIMPORTPATH, 'channels.debug'), "w+") for cat in category_order: if cat in dictchannels: for line in dictchannels[cat]: linevals = "" for key, value in line.items(): if type(value) is bool: linevals += str(value) + ":"
def main(): args = get_args() print "\n%s" %('-'*100) # create CKAN search pattern : ckan_pattern = '' sand='' pattern=' '.join(args.pattern) if (args.community): ckan_pattern += "groups:%s" % args.community sand=" AND " if (args.pattern): ckan_pattern += sand + pattern print ' | - Search\n\t|- in\t%s\n\t|- for\t%s\n' % (args.ckan,ckan_pattern) ckanapi3='http://'+args.ckan+'/api/3' ckan = ckanclient.CkanClient(ckanapi3) ckan_limit=100000 start=time.time() answer = ckan.action('package_search', q=ckan_pattern, rows=ckan_limit) tcount=answer['count'] print " | - Results:\n\t|- %d records found in %d sec" % (tcount,time.time()-start) b2findorder=['Title','Description','Tags','Source','DOI','PID','Checksum','Rights','Discipline','Creator','Publisher','PublicationYear','Language','TemporalCoverage','SpatialCoverage','Format','Contact','MetadataAccess'] b2findfacets={ 'Title':'title', 'Description':'notes', 'Tags':'tags', 'Source':'url', 'DOI':'DOI', 'PID':'PID', 'Checksum':'version', 'Rights':'Rights', 'Discipline':'Discipline', 'Creator':'author', 'Publisher':'Publisher', 'PublicationYear':'PublicationYear', 'Language':'Language', 'TemporalCoverage':'TemporalCoverage', 'SpatialCoverage':'SpatialCoverage', 'Format':'Format', 'Contact':'Contact', 'MetadataAccess':'MetadataAccess' } admin_fields={ 'ManagerVersion':'ManagerVersion', 'modified':'metadata_modified', 'created':'metadata_created', 'oai_set':'oai_set' } if tcount>0 and args.keys is not None : if len(args.keys) == 0 : akeys=[] else: if args.keys[0] == 'B2FIND.*' : akeys=OrderedDict(sorted(b2findfacets.items(), key=lambda i:b2findorder.index(i[0]))) else: akeys=args.keys suppid=b2findfacets suppid.update(admin_fields) fh = io.open(args.output, "w", encoding='utf8') record={} totlist=[] count={} count['id']=0 statc={} for outt in akeys: if outt not in suppid : print ' [WARNING] Not supported key %s is removed' % outt akeys.remove(outt) else: count[outt]=0 statc[outt] = Counter() printfacets='' if (len(akeys) > 0): printfacets="and related facets %s " % ", ".join(akeys) print "\t|- ID's %sare written to %s ..." % (printfacets,args.output) counter=0 cstart=0 oldperc=0 start2=time.time() while (cstart < tcount) : if (cstart > 0): answer = ckan.action('package_search', q=ckan_pattern, rows=ckan_limit, start=cstart) if len(answer['results']) == 0 : break # loop over records for ds in answer['results']: counter +=1 ##HEW-T print' | %-4d | %-40s |' % (counter,ds['name']) perc=int(counter*100/tcount) bartags=perc/5 if perc%10 == 0 and perc != oldperc : oldperc=perc print "\r\t[%-20s] %5d (%3d%%) in %d sec" % ('='*bartags, counter, perc, time.time()-start2 ) sys.stdout.flush() record['id'] = '%s' % (ds['name']) # loop over facets for facet in akeys: if suppid[facet] in ds: ## CKAN default field if facet == 'Group': record[facet] = ds[suppid[facet]][0]['display_name'] else: record[facet] = ds[suppid[facet]] else: ## CKAN extra field efacet=[e for e in ds['extras'] if e['key'] == facet] if efacet: record[facet] = efacet[0]['value'] else: record[facet] = 'N/A' if record[facet] is None : record[facet]='None' statc[facet][record[facet]]+=1 else: if not isinstance(record[facet],list): words=record[facet].split(';') else: words=record[facet] for word in words: if isinstance(word,dict): word=word['name'] statc[facet][word]+=1 if not ( record[facet] == 'N/A' or record[facet] == 'Not Stated') and len(record[facet])>0 : count[facet]+=1 outline=record['id'] for aid in akeys: outline+='\t | %-30s' % record[aid][:30] ##outline+='\n \t| %-20s' % (statc[aid].keys(),statc[aid].values()) fh.write(outline+'\n') cstart+=len(answer['results']) fh.close() if len(akeys) > 0 : statfh = io.open('stat_'+args.output, "w", encoding='utf8') ##print "\n|- Statistics :\n\t| %-16s | %-10s | %6s |\n\t%s " % ('Facet','Occurence','%',"-" * 50) print '|- Statistics written to file %s' % 'stat_'+args.output statline=unicode("") for outt in akeys: statline+= "| %-16s | %-10d | %6d |\n" % (outt,count[outt],int(count[outt]*100/tcount)) for word in statc[outt].most_common(10): statline+= '\t| %-10d | %-100s\n' % (word[1], word[0][:100]) statfh.write(statline) statfh.close()
class asmlAscii(object): """ Class for ASML jobfile Each object corresponds to an entire (possibly not fully specified) ASML jobfile, organized as a dictionary of lists of section objects, specified by section type. Class methods provide functionality to get, remove, and append sections, to merge jobfiles, to check for section interferences and jobfile completion, and to read and write ASCII strings and files. """ def __init__(self): """ Initializes dictionary of empty section lists """ self.sections = OrderedDict() for sectionname in ASMLJOBSECTIONS: self.sections[sectionname] = [] def append(self,newsection,check_interference=False): """ Adds a new section object to the jobfile Can check for interference between the new section and existing sections and only add the new section if it does not interfere. Returns boolean for convenience of calling processes. """ if check_interference: for section in self.sections[newsection.sectionname]: if asmlSection.interfere(section,newsection): return False # If the interference test passes, append the new section self.sections[newsection.sectionname].append(newsection) return True def get(self,sectionname,**id_element_values): """ Get sections from jobfile Identifies sections by name and optional specification of ID elements. If the specification matches several sections in the jobfile, returns a list of matching sections. """ matchsections = [] for section in self.sections[sectionname]: id_match = True for id_element_name in id_element_values: if id_element_name not in section.id_elements: raise ValueError('{} not a valid id element for {} section'.format(id_element_name,sectionname)) if section.get(id_element_name) != id_element_values[id_element_name]: id_match = False # Change match to False if any ID element disagrees if id_match: matchsections.append(section) return matchsections def remove(self,sectionname,**id_element_values): """ Removes sections from jobfile Identifies sections by name and optional specification of ID elements. If the specification matches several sections in the jobfile, removes all of them. """ for section in self.sections[sectionname]: id_match = True for id_element_name in id_element_values: if id_element_name not in section.id_elements: raise ValueError('{} not a valid id element for {} section'.format(id_element_name,sectionname)) if section.get(id_element_name) != id_element_values[id_element_name]: id_match = False if id_match: self.sections.remove(section) @staticmethod def merge(job1,job2): """ Merges two jobfiles If there is any interference between sections of the jobfiles ignore conflicting sections from the second jobfile. """ mergedjob = copy.deepcopy(job1) for sectionname in ASMLJOBSECTIONS: for section in job2.sections[sectionname]: mergedjob.append(section,check_interference=True) return mergedjob def isSpecified(self): """ Checks whether jobfile is adequately specified An adequately specified jobfile contains all non-optional sections, only one of each non-multiple-allowed sections, and has sections that are themselves adequately specified. """ # There may be other things to check. # # 1. Are layers in correct order? # 2. Does every layer have a PROCESS_DATA section? # for sectionname in ASMLJOBSECTIONS: if (not ASMLJOBSECTIONS[sectionname]['is_optional'] and len(self.sections[sectionname]) == 0): return False if (not ASMLJOBSECTIONS[sectionname]['multiple_allowed'] and len(self.sections[sectionname]) > 1): # raising exception because this shouldn't be possible using class methods raise ValueError('Job can only have one {} section'.format(sectionname)) for section in self.sections[sectionname]: if not section.isSpecified(): return False return True def makeAscii(self,fix_delim_bug=True): """ Generates ASCII string representation of jobfile """ asciistring = '' for sectionname in self.sections: for section in self.sections[sectionname]: if fix_delim_bug: section.fixDelimBug() asciistring += section.makeAscii() + '\n\n' return asciistring[:-2] # strip off final newlines def readAscii(self,asciistring): """ Reads an ASCII string representation of a jobfile section """ # Matches all section blocks in the string mm = re.findall(r'START_SECTION (\S*)\n(.*?)\nEND_SECTION',asciistring,re.DOTALL) for m in mm: sectionname = m[0] sectionbody = m[1] newsection = asmlSection(sectionname) newsection.readAscii(sectionbody) self.append(newsection) def readAsciiJobfile(self,filename): """ Loads and parses ASCII jobfile """ with open(filename,'r') as f: asciistring = f.read() self.readAscii(asciistring) def writeAsciiJobfile(self,filename): """ Generates and saves ASCII jobfile """ asciistring = self.makeAscii() with open(filename,'w') as f: f.write(asciistring) def printSummary(self): """ Print summary information on the jobfile Summary values could include: - number of images - number of instances - number of layers - number of image placements per layer - max. number of placements of each image - etc. """ print('Not yet implemented')
class BaseListSelector(AbstractSelector): sequence = List(context=True) _hidden_keys = [] _sort = Button('Sort') _remove = Button('-') def __init__(self): self._parameters = OrderedDict() self._selected_settings = None def add_parameter(self, parameter, label=None, default_value=None): if parameter in self._parameters: raise ValueError('Parameter already exists') if label is None: label = parameter self._parameters[parameter] = label for setting in self.sequence: setting[parameter] = default_value def remove_parameter(self, parameter): if parameter not in self._parameters: raise ValueError('Parameter not in list') self._parameters.remove(parameter) for setting in self.sequence: del(setting[parameter]) def _new_setting(self): if self._selected_settings: return self._selected_settings[-1].copy() elif self.sequence: return self.sequence[-1].copy() else: return dict((p, 0) for p in self._parameters) def add_setting(self, setting=None): if setting is None: setting = self._new_setting() valid_keys = set(self._parameters.keys() + self._hidden_keys) if set(setting.keys()) != valid_keys: raise ValueError('Setting contains extra parameters') self.sequence.append(setting) def remove_setting(self, setting): self.sequence.remove(setting) def remove_selected_settings(self): for setting in self._selected_settings: self.remove_setting(setting) def __remove_fired(self): self.remove_selected_settings() def __sort_fired(self): self.sequence = sorted(self.sequence) def _buttons_view(self): raise NotImplementedError def _settings_view(self): raise NotImplementedError def traits_view(self): _column_map = list((v, k) for k, v in self._parameters.items()) _editor = TabularEditor( adapter=DictTabularAdapter(columns=_column_map), auto_update=True, selected='_selected_settings', operations=['edit'], editable=True, editable_labels=True, multi_select=True, ) return View( VGroup( self._settings_view(), self._buttons_view(), Item('sequence', editor=_editor, show_label=False), ), resizable=True, )
def check_epkgs(epkg_list, lpps, efixes): """ For each epkg get the label, packaging date, filset and check prerequisites based on fileset current level and build a list ordered by packaging date that should not be locked at its installation. Note: in case of parsing error, keep the epkg (best effort) args: epkg_list (list): The list of efixes to check lpps (dict): The current lpps levels efixes (dict): The current efixes info returns: The list of epkg to install (ordered by packaging date) The list of epkg to rejected with the reason (ordered by label) """ epkgs_info = {} epkgs_reject = [] # Installed efix could lock some files we will try to modify, # let's build a dictionary indexed upon file location locked_files = {} for efix in efixes: for file in efixes[efix]['files']: if file not in locked_files: locked_files[file] = efix module.debug('locked_files: {0}'.format(locked_files)) # Get information on efix we want to install and check it can be installed for epkg_path in epkg_list: epkg = {'path': epkg_path, 'label': '', 'pkg_date': None, 'sec_from_epoch': -1, 'filesets': [], 'files': [], 'prereq': {}, 'reject': False} # get efix information cmd = '/usr/sbin/emgr -dXv3 -e {0} | /bin/grep -p -e PREREQ -e PACKAG'.format(epkg['path']) rc, stdout, stderr = module.run_command(cmd, use_unsafe_shell=True) if rc != 0: msg = 'Cannot get efix information {0}'.format(epkg['path']) module.log(msg) module.log('cmd:{0} failed rc={1} stdout:{2} stderr:{3}' .format(cmd, rc, stdout, stderr)) results['meta']['messages'].append(msg) # do not break or continue, we keep this efix, will try to install it anyway # ordered parsing: expecting the following line order: # LABEL, PACKAGING DATE, then PACKAGE, then prerequisites levels for line in stdout.splitlines(): # skip comments and empty lines line = line.rstrip() if not line or line.startswith('+'): continue # skip blank and comment line if not epkg['label']: # match: "LABEL: IJ02726s8a" match = re.match(r'^LABEL:\s+(\S+)$', line) if match: epkg['label'] = match.group(1) continue if not epkg['pkg_date']: # match: "PACKAGING DATE: Mon Oct 9 09:35:09 CDT 2017" match = re.match(r'^PACKAGING\s+DATE:\s+' r'(\S+\s+\S+\s+\d+\s+\d+:\d+:\d+\s+\S*\s*\S+).*$', line) if match: epkg['pkg_date'] = match.group(1) continue # match: " PACKAGE: devices.vdevice.IBM.vfc-client.rte" match = re.match(r'^\s+PACKAGE:\s+(\S+)\s*?$', line) if match: if match.group(1) not in epkg['filesets']: epkg['filesets'].append(match.group(1)) continue # match: " LOCATION: /usr/lib/boot/unix_64" match = re.match(r'^\s+LOCATION:\s+(\S+)\s*?$', line) if match: if match.group(1) not in epkg['files']: epkg['files'].append(match.group(1)) continue # match and convert prerequisite levels # line like: "bos.net.tcp.server 7.1.3.0 7.1.3.49" match = re.match(r'^(\S+)\s+([\d+\.]+)\s+([\d+\.]+)\s*?$', line) if match is None: continue (prereq, minlvl, maxlvl) = match.groups() epkg['prereq'][prereq] = {} epkg['prereq'][prereq]['minlvl'] = minlvl epkg['prereq'][prereq]['maxlvl'] = maxlvl # parsing done # check filseset prerequisite is present if prereq not in lpps: epkg['reject'] = '{0}: prerequisite missing: {1}'.format(os.path.basename(epkg['path']), prereq) module.log('reject {0}'.format(epkg['reject'])) break # stop parsing # check filseset prerequisite is present minlvl_i = list(map(int, epkg['prereq'][prereq]['minlvl'].split('.'))) maxlvl_i = list(map(int, epkg['prereq'][prereq]['maxlvl'].split('.'))) if lpps[prereq]['int'] < minlvl_i or lpps[prereq]['int'] > maxlvl_i: epkg['reject'] = '{0}: prerequisite {1} levels do not satisfy condition string: {2} =< {3} =< {4}'\ .format(os.path.basename(epkg['path']), prereq, epkg['prereq'][prereq]['minlvl'], lpps[prereq]['str'], epkg['prereq'][prereq]['maxlvl']) module.log('reject {0}'.format(epkg['reject'])) break if epkg['reject']: epkgs_reject.append(epkg['reject']) continue # check file locked by efix already installed for file in epkg['files']: if file in locked_files: results['meta']['messages'].append('installed efix {0} is locking {1} preventing the ' 'installation of {2}, remove it manually or set the ' '"force" option.' .format(locked_files[file], file, os.path.basename(epkg['path']))) epkg['reject'] = '{0}: installed efix {1} is locking {2}'\ .format(os.path.basename(epkg['path']), locked_files[file], file) module.log('reject {0}'.format(epkg['reject'])) epkgs_reject.append(epkg['reject']) continue if epkg['reject']: continue # convert packaging date into time in sec from epoch if epkg['pkg_date']: (sec_from_epoch, msg) = to_utc_epoch(epkg['pkg_date']) if sec_from_epoch == -1: module.log('{0}: "{1}" for epkg:{2}'.format(msg, epkg['pkg_date'], epkg)) epkg['sec_from_epoch'] = sec_from_epoch epkgs_info[epkg['path']] = epkg.copy() # sort the epkg by packing date (sec from epoch) sorted_epkgs = OrderedDict(sorted(epkgs_info.items(), key=lambda t: t[1]['sec_from_epoch'], reverse=True)).keys() # exclude epkg that will be interlocked global_file_locks = [] removed_epkg = [] for epkg in sorted_epkgs: if set(epkgs_info[epkg]['files']).isdisjoint(set(global_file_locks)): global_file_locks.extend(epkgs_info[epkg]['files']) module.log('keep {0}, files: {1}' .format(os.path.basename(epkgs_info[epkg]['path']), epkgs_info[epkg]['files'])) else: results['meta']['messages'].append('a previous efix to install will lock a file of {0} ' 'preventing its installation, install it manually or ' 'run the task again.' .format(os.path.basename(epkgs_info[epkg]['path']))) epkgs_info[epkg]['reject'] = '{0}: locked by previous efix to install'\ .format(os.path.basename(epkgs_info[epkg]['path'])) module.log('reject {0}'.format(epkgs_info[epkg]['reject'])) epkgs_reject.append(epkgs_info[epkg]['reject']) removed_epkg.append(epkg) for epkg in removed_epkg: sorted_epkgs.remove(epkg) epkgs_reject = sorted(epkgs_reject) # order the reject list by label return (sorted_epkgs, epkgs_reject)
class AutomationConfig(object): def __init__(self, source): self.__dict__ = OrderedDict() if os.path.isfile(source): with open(source, 'r') as f: jsonString = f.read() else: jsonString = source decoder = JSONDecoder(object_hook=OrderedDict) self.__dict__ = decoder.decode(jsonString) # json.loads(jsonString) self._jsonFile = source def __getitem__(self, key): for k, v in self.__dict__.items(): if k == key: return v if type(v) is dict: ret = self._getitem(key, v) if ret is not None: return ret elif type(v) is list: for x in v: ret = self._getitem(key, x) if ret is not None: return ret return None def _getitem(self, key, item): if key is None: return item elif key == item.keys()[0]: return item[key] if type(item) is dict: for k in item.keys(): if k == key: self._getitem(None, item[k]) elif type(item) is list: for x in item: self._getitem(key, x) def __setitem__(self, key, val): if key not in self.__dict__.keys(): self.__dict__[key] = val def __delitem__(self, key): if key in self.__dict__.keys(): self.__dict__.remove(key) def __iter__(self): for key in self.__dict__.keys(): node = self.__dict__[key] for x in node.keys(): yield node[x] def __str__(self): #return json.dumps(self, default=lambda x: x.__dict__, sort_keys=False, indent=4) dic = OrderedDict() for k, v in self.__dict__.items(): if not k.startswith('_'): dic[k] = v return json.dumps(dic, default=lambda x: x.__dict__, sort_keys=False, indent=4) def Save(self): if self._jsonFile is not None: with open(self._jsonFile, 'w') as f: f.write(str(self))
# alias new license types to existing licenses by recording them in the aliases dict below # "new type": "existing type", e.g. "cc-by-nc": "cc-nc" (the open definition only includes cc-nc, not cc-by-nc) aliases = {'cc-by-nc': 'cc-nc', 'cc-by-nc-sa': 'cc-nc-sa', 'cc-by-nc-nd': 'cc-nc-nd'} for alias, target in aliases.items(): LICENSES[alias] = LICENSES[target] licenses_dropdown = [('','')] # default: empty __process_licenses = OrderedDict(sorted(LICENSES.items(), key=lambda x: x[1]['title'])).items() # in alphabetical order of the titles (shown to the user) # Creative Commons first for lic_type, info in __process_licenses[:]: if lic_type.startswith('cc-'): choice = (lic_type, info.get('title', lic_type)) licenses_dropdown.append(choice) __process_licenses.remove((lic_type, info)) # Free to Read next for lic_type, info in __process_licenses[:]: if lic_type == 'free-to-read': choice = (lic_type, info.get('title', lic_type)) licenses_dropdown.append(choice) __process_licenses.remove((lic_type, info)) dropdown_separator = ('', '--------------------') licenses_dropdown.append(dropdown_separator) # Take out all "Other"-type licenses to append to end of options later other_license_choices = [] for lic_type, info in __process_licenses[:]: if lic_type.startswith('other-'):
def check_epkgs(epkg_list, lpps, efixes, machine, output): """ For each epkg get the label, packaging date, filset and check prerequisites based on fileset current level and build a list ordered by packaging date that should not be locked at its installation. Note: in case of parsing error, keep the epkg (best effort) args: epkg_list (list): The list of efixes to check lpps_lvl (dict): The current lpps levels efixes (dict): The current efixes info machine (str) : The target machine output (dict): The result of the command (use only before exit) returns: The list of epkg to install (ordered by packaging date) The list of epkg to rejected with the reason (ordered by label) """ epkgs_info = {} epkgs_reject = [] # Installed efix could lock some files we will try to modify, # build a dictionary indexed upon file location locked_files = {} for efix in efixes: for file in efixes[efix]['files']: if file not in locked_files: locked_files[file] = efix logging.debug('{}: locked_files: {}'.format(machine, locked_files)) # Get information on efix we want to install # and check it could be installed for epkg_path in epkg_list: epkg = {'path': epkg_path, 'label': '', 'pkg_date': None, 'sec_from_epoch': -1, 'filesets': [], 'files': [], 'prereq': {}, 'reject': False} # get efix information stdout = '' try: cmd = '/usr/sbin/emgr -dXv3 -e {} | /bin/grep -p -e PREREQ -e PACKAG'\ .format(epkg['path']) stdout = subprocess.check_output(args=cmd, shell=True, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: logging.warning('EXCEPTION cmd={} rc={} output={}' .format(exc.cmd, exc.returncode, exc.output)) # do not break or continue, we keep this efix, will try to install it anyway # ordered parsing: expecting the following line order: # LABEL, PACKAGING DATE, then PACKAGE, then prerequisites levels for line in stdout.splitlines(): # skip comments and empty lines line = line.rstrip() if not line or line.startswith('+'): continue # skip blank and comment line if not epkg['label']: # match: "LABEL: IJ02726s8a" match = re.match(r'^LABEL:\s+(\S+)$', line) if match: epkg['label'] = match.group(1) continue if not epkg['pkg_date']: # match: "PACKAGING DATE: Mon Oct 9 09:35:09 CDT 2017" match = re.match(r'^PACKAGING\s+DATE:\s+' '(\S+\s+\S+\s+\d+\s+\d+:\d+:\d+\s+\S*\s*\S+).*$', line) if match: epkg['pkg_date'] = match.group(1) continue # match: " PACKAGE: devices.vdevice.IBM.vfc-client.rte" match = re.match(r'^\s+PACKAGE:\s+(\S+)\s*?$', line) if match: if match.group(1) not in epkg['filesets']: epkg['filesets'].append(match.group(1)) continue # match: " LOCATION: /usr/lib/boot/unix_64" match = re.match(r'^\s+LOCATION:\s+(\S+)\s*?$', line) if match: if match.group(1) not in epkg['files']: epkg['files'].append(match.group(1)) continue # match and convert prerequisite levels # line like: "bos.net.tcp.server 7.1.3.0 7.1.3.49" match = re.match(r'^(\S+)\s+([\d+\.]+)\s+([\d+\.]+)\s*?$', line) if match is None: continue (prereq, minlvl, maxlvl) = match.groups() epkg['prereq'][prereq] = {} epkg['prereq'][prereq]['minlvl'] = minlvl epkg['prereq'][prereq]['maxlvl'] = maxlvl # parsing done # check filseset prerequisite is present if prereq not in lpps: epkg['reject'] = '{}: prerequisite missing: {}'.format(epkg['label'], prereq) logging.info('{}: reject {}'.format(machine, epkg['reject'])) break # stop parsing # check filseset prerequisite is present minlvl_i = list(map(int, epkg['prereq'][prereq]['minlvl'].split('.'))) maxlvl_i = list(map(int, epkg['prereq'][prereq]['maxlvl'].split('.'))) if lpps[prereq]['int'] < minlvl_i\ or lpps[prereq]['int'] > maxlvl_i: epkg['reject'] = '{}: prerequisite {} levels do not match: {} < {} < {}'\ .format(epkg['label'], prereq, epkg['prereq'][prereq]['minlvl'], lpps[prereq]['str'], epkg['prereq'][prereq]['maxlvl']) logging.info('{}: reject {}'.format(machine, epkg['reject'])) break if epkg['reject']: epkgs_reject.append(epkg['reject']) continue # check file locked by efix already installed on the machine for file in epkg['files']: if file in locked_files: output['messages'].append('installed efix {} is locking {} preventing the ' 'installation of {}, remove it manually or set the ' '"force" option.' .format(locked_files[file], file, epkg['label'])) epkg['reject'] = '{}: installed efix {} is locking {}'\ .format(epkg['label'], locked_files[file], file) logging.info('{}: reject {}'.format(machine, epkg['reject'])) epkgs_reject.append(epkg['reject']) continue if epkg['reject']: continue # convert packaging date into time in sec from epoch if epkg['pkg_date']: (sec_from_epoch, msg) = to_utc_epoch(epkg['pkg_date']) if sec_from_epoch == -1: logging.warning('{}: {}: "{}" for epkg:{} ' .format(machine, msg, epkg['pkg_date'], epkg)) epkg['sec_from_epoch'] = sec_from_epoch epkgs_info[epkg['path']] = epkg.copy() # sort the epkg by packing date (sec from epoch) sorted_epkgs = OrderedDict(sorted(epkgs_info.items(), key=lambda t: t[1]['sec_from_epoch'], reverse=True)).keys() # exclude epkg that will be interlocked global_file_locks = [] removed_epkg = [] for epkg in sorted_epkgs: lock_found = False if set(epkgs_info[epkg]['files']).isdisjoint(set(global_file_locks)): global_file_locks.extend(epkgs_info[epkg]['files']) logging.info('{}: keep {}, files: {}' .format(machine, epkgs_info[epkg]['label'], epkgs_info[epkg]['files'])) else: epkgs_info[epkg]['reject'] = '{}: locked by previous efix to install'\ .format(epkgs_info[epkg]['label']) logging.info('{}: reject {}'.format(machine, epkgs_info[epkg]['reject'])) epkgs_reject.append(epkgs_info[epkg]['reject']) removed_epkg.append(epkg) for epkg in removed_epkg: sorted_epkgs.remove(epkg) epkgs_reject = sorted(epkgs_reject) # order the reject list by label return (sorted_epkgs, epkgs_reject)
def main(): pstat = { 'status' : {}, 'text' : {}, 'short' : {}, } now = time.strftime("%Y-%m-%d %H:%M:%S") jid = os.getpid() ckanlistrequests=['package_list','group_list','tag_list'] ## Get options and arguments args = get_args(ckanlistrequests) # Output instance OUT = Output(pstat,now,jid,args) logger = OUT.setup_custom_logger('root',args.verbose) ## Settings for CKAN client and API ckanapi3='http://'+args.iphost+'/api/3' auth='12345' CKAN = CKAN_CLIENT(args.iphost,auth) ckan_limit=500000 start=time.time() if args.request.endswith('list'): try: if args.request == 'community_list' : action='group_list' else: action=args.request if PY2 : answer = CKAN.action(action, rows=ckan_limit) else: answer = CKAN.action(action) except ckanclient.CkanApiError as e : print('\t\tError %s Supported list requests are %s.' % (e,ckanlistrequests)) sys.exit(1) print('\n\t%s' % '\n\t'.join(answer).encode('utf8')) sys.exit(0) # create CKAN search pattern : ckan_pattern = '' sand='' pattern=' '.join(args.pattern) if (args.community): ckan_pattern += "groups:%s" % args.community sand=" AND " if (args.pattern): ckan_pattern += sand + pattern print(' | - Search\n\t|- in\t%s\n\t|- for\t%s\n' % (args.iphost,ckan_pattern)) if args.request == 'package_search' : if PY2: answer = CKAN.action('package_search', {"q":ckan_pattern}) ##HEW-D? , rows=ckan_limit) else: answer = CKAN.action('package_search',{"q":ckan_pattern}) for key, value in answer.items() : logger.warning('answer has key %s' % key) if PY2 : tcount=answer['result']['count'] ### ['count'] else: tcount=answer['result']['count'] print(' | - Results:\n\t|- %d records found in %d sec' % (tcount,time.time()-start)) # Read in B2FIND metadata schema and fields schemafile = '%s/mapfiles/b2find_schema.json' % (os.getcwd()) with open(schemafile, 'r') as f: b2findfields=json.loads(f.read(), object_pairs_hook=OrderedDict) if tcount>0 and args.keys is not None : if len(args.keys) == 0 : akeys=[] else: if args.keys[0] == 'B2FIND.*' : akeys=OrderedDict(sorted(b2findfields.keys())) else: akeys=args.keys suppid=b2findfields.keys() fh = io.open(args.output, "w", encoding='utf8') record={} totlist=[] count={} count['id']=0 statc={} for outt in akeys: if outt not in suppid : print(' [WARNING] Not supported key %s is removed' % outt) akeys.remove(outt) else: count[outt]=0 statc[outt] = Counter() printfacets='' if (len(akeys) > 0): printfacets="and related facets %s " % ", ".join(akeys) print('\t|- IDs %sare written to %s ...' % (printfacets,args.output)) counter=0 cstart=0 oldperc=0 start2=time.time() while (cstart < tcount) : if (cstart > 0): if PY2 : answer = CKAN.action('package_search', {"q":ckan_pattern,"rows":ckan_limit,"start":cstart}) ##HEW-D q=ckan_pattern, rows=ckan_limit, start=cstart) else: answer = CKAN.action('package_search',{"q":ckan_pattern,"rows":ckan_limit,"start":cstart}) if PY2 : if len(answer['result']) == 0 : break # loop over found records if PY2: results= answer['result']['results'] ### ['results'] else: results= answer['result']['results'] for ds in results : #### answer['results']: counter +=1 logger.debug(' | %-4d | %-40s |' % (counter,ds['name'])) perc=int(counter*100/tcount) bartags=perc/5 if perc%10 == 0 and perc != oldperc : oldperc=perc print('\r\t[%-20s] %5d (%3d%%) in %d sec' % ('='*int(bartags), counter, perc, time.time()-start2 )) sys.stdout.flush() record['id'] = '%s' % (ds['name']) outline='%s\n' % record['id'] # loop over facets for facet in akeys: ckanFacet=b2findfields[facet]["ckanName"] if ckanFacet in ds: ## CKAN default field if facet == 'Group': record[facet] = ds[ckanFacet][0]['display_name'] else: record[facet] = ds[ckanFacet] else: ## CKAN extra field efacet=[e for e in ds['extras'] if e['key'] == facet] if efacet: record[facet] = efacet[0]['value'] else: record[facet] = 'N/A' if record[facet] is None : record[facet]='None' statc[facet][record[facet]]+=1 else: if not isinstance(record[facet],list): words=record[facet].split(';') else: words=record[facet] for word in words: if isinstance(word,dict): word=word['name'] statc[facet][word]+=1 if not ( record[facet] == 'N/A' or record[facet] == 'Not Stated') and len(record[facet])>0 : count[facet]+=1 if PY2 : fh.writelines((record[facet]+'\n').decode('utf-8')) else : fh.writelines((record[facet]+'\n')) cstart+=len(results) logger.warning('%d records done, %d in total' % (cstart,tcount)) fh.close() if len(akeys) > 0 : statfh = io.open('stat_'+args.output, "w", encoding='utf8') print('|- Statistics written to file %s' % 'stat_'+args.output) statline="" for outt in akeys: statline+= "| %-16s\n\t| %-15s | %-6d | %3d |\n" % (outt,'-Total-',count[outt],int(count[outt]*100/tcount)) for word in statc[outt].most_common(10): statline+= '\t| %-15s | %-6d | %3d |\n' % (word[0][:100], word[1], int(word[1]*100/tcount)) if PY2 : statfh.write(statline.decode('utf-8')) else : statfh.write(statline) statfh.close()
class Run(object): __slots__ = [ "_modality", # modality associeted with this run "attribute", # dictionary of attr:regexp "entity", # dictionary for run entities "json", # dictionary for json fields "_suffix", # suffix associated with run "_bk_modality", # copy of self old values "_bk_attribute", "_bk_entity", "_bk_suffix", "_bk_json", "provenance", # file from which run is modelled "example", # bids name from provenance file "writable", "checked", # switch if run was confirmed by user "template" # switch if run was extracted from template ] def __init__(self, *, modality: str = "", attribute: dict = {}, entity: OrderedDict = {}, json: OrderedDict = {}, suffix: str = "", provenance: str = None, example: str = None): """ Run class contains all information needed to identify and generate the bidsified name. All init parameters must be named. Parameters ---------- modality: str the modality corresponding to given run attribute: dict dict of attributes (keys) and values (value) to match entity: OrderedDict dict of bids fields (keys) and values (value) to generate bidsified name json: OrderedDict dict of json fields (keys) and values (value) to fill sidecar json file suffix: str suffix associated with given run provenance: str path to the file from which this run was generated example: str example of bidsified name generated from provenance file """ self._bk_modality = None self._bk_attribute = dict() self._bk_entity = dict() self._bk_suffix = None self._bk_json = dict() self.writable = True self.template = False self.checked = False self.example = example self.provenance = provenance self._modality = check_type("modality", str, modality) self._suffix = check_type("suffix", str, suffix) self.attribute = dict(check_type("attribute", dict, attribute)) self.entity = OrderedDict(check_type("entity", dict, entity)) self.json = OrderedDict(check_type("json", dict, json)) def __bool__(self) -> bool: """ Return True if suffix is defined """ if self._suffix == "": return False return True @property def modality(self) -> str: """ Return Modality """ return self._modality @modality.setter def modality(self, value: str) -> None: """ Sets Modality. The old (unmodified) value is backuped """ if self._bk_modality is None: self._bk_modality = self._modality self.modality = value def restore_modality(self) -> None: """ Restore old (unmodified) modality """ if self._bk_modality is not None: self._modality = self._bk_modality self._bk_modality = None @property def suffix(self) -> str: """ Return Suffix """ return self._suffix @suffix.setter def suffix(self, value: str) -> None: """ Sets new suffix. The old (unmodified) value is backuped """ if self._bk_suffix is None: self._bk_suffix = self._suffix self.suffix = value def restore_suffix(self) -> None: """ Restore old (unmodified) suffix """ if self._bk_suffix is not None: self._suffix = self._bk_suffix self._bk_suffix = None def set_attribute(self, attr: str, val: object) -> None: """ Sets attribute value. Old (unmodified) value is backuped Parameters ---------- attr: str name of attribute to set value: object value to match given attribute """ if isinstance(val, str): val = val.encode('unicode_escape').decode() attr = check_type("attr", str, attr) if attr in self.attribute: if self.attribute[attr] == val: return if attr not in self._bk_attribute: self._bk_attribute[attr] = self.attribute[attr] self.attribute[attr] = val return else: if val == "": return self.attribute[attr] = val if attr not in self._bk_attribute: self._bk_attribute[attr] = None def set_entity(self, ent: str, val: str) -> None: """ Sets entity value. Old (unmodified) value is backuped Parameters ---------- ent: str name of entity to set val: str value of entity """ val = check_type("val", str, val) attr = check_type("ent", str, ent) if attr in self.entity: if self.entity[attr] == val: return if attr not in self._bk_entity: self._bk_entity[attr] = self.entity[attr] if val: self.entity[attr] = val else: self.entity.remove(attr) return else: if val == "": return self.entity[attr] = val if attr not in self._bk_entity: self._bk_entity[attr] = None def set_json_field(self, field: str, val: object) -> None: """ Sets the json field value. The old (unmodified) value is backuped Parameters: ----------- field: str name of field to set val: object value to set """ if isinstance(val, str): val = val.encode('unicode_escape').decode() attr = check_type("field", str, field) if attr in self.json: if self.json[attr] == val: return if attr not in self._bk_json: self._bk_json[attr] = self.json[attr] if val: self.json[attr] = val else: self.json.remove(attr) return else: if val == "": return self.json[attr] = val if attr not in self._bk_json: self._bk_json[attr] = None def restore_attribute(self, attr: str) -> None: """ Restore old value of given attribute. New value is lost Parameters: ----------- attr: str name of attribute to restore """ self.__restore_val(attr, self.attribute, self._bk_attribute) def restore_attributes(self): """ Restore all attributes to old values. New values and new attributes are lost """ for attr in self.attribute: self.__restore_val(attr, self.attribute, self._bk_attribute) def restore_entity(self, attr: str) -> None: """ Restore old value for given entity. New value is lost. """ self.__restore_val(attr, self.entity, self._bk_entity) def restore_entities(self) -> None: """ Restore old values for all entities. New values and new entities are lost. """ for attr in self.entity: self.__restore_val(attr, self.entity, self._bk_entity) def restore_json_field(self, attr: str) -> None: """ Restore old value for json field. New value is lost """ self.__restore_val(attr, self.json, self._bk_json) def restore_json(self) -> None: """ Restore old values for all json fields. New values and new fields are lost. """ for attr in self.json: self.__restore_val(attr, self.json, self._bk_json) def restore(self) -> None: """ Restores old values for modality, suffix, attributesm entities and json fields """ self.restore_modality() self.restore_suffix() self.restore_attributes() self.restore_entities() self.restore_json() def save(self) -> None: """ Saves all current values, old values are lost """ self._bk_modality = None self._bk_attribute = dict() self._bk_entity = dict() self._bk_json = dict() self._bk_suffix = None def __restore_val(self, attr: str, dest: dict, source: dict) -> None: """ Restore old value from given dictionary. Parameters: ----------- attr: str name of field to restore dest: dict dictionary where restore value source: dict dictionary from which retrieve values """ if attr not in source: return if source[attr] is None: if attr in dest: dest.remove(attr) return dest[attr] = source.pop(attr) def dump(self, empty_attributes: bool = True) -> dict: """ Dumps run into a dictionary Parameters: ----------- empty_attributes: bool if True, the void and empty attributes are also dumped, if False, viod values are ignored """ d = dict() d["provenance"] = self.provenance if self.example: d["example"] = self.example if self.template: d["template"] = self.template d["checked"] = self.checked d["suffix"] = self.suffix d["attributes"] = { k: v for k, v in self.attribute.items() if empty_attributes or v is not None } d["bids"] = self.entity d["json"] = self.json return d def genEntities(self, entities: list): """ Completes the existing entities by entities from list All added values will be set to None Parameters ---------- entities: list list of strings with names of entities to add """ for ent in entities: if ent not in self.entity: self.entity[ent] = None
# print len(corepoints) # print directly_reachable corepoints = OrderedDict((x, True) for x in corepoints).keys() noise = set(points) clusters = [] while len(corepoints) != 0: cluster = [] cluster.append(corepoints[0]) i = 0 while i < len(cluster): point = cluster[i] if point in corepoints: cluster += directly_reachable[point] corepoints.remove(point) cluster = OrderedDict((x, True) for x in cluster).keys() i += 1 clusters.append(cluster) noise = noise - set(cluster) # print len(clusters) for cluster in clusters: # coloring the clusters x, y = zip(*cluster) plt.scatter(x, y) if len(noise) > 0: # coloring the noise points x, y = zip(*noise) plt.scatter(x, y, color='gray') plt.show() # display the plotted graph