Esempio n. 1
0
def make_tags(article, index):
	def score(word, num):
		word = word.lower()
		md = hashlib.md5(word.encode('utf-8')).hexdigest()[:2]
		df = redis_tag.hget(md, word)
		df = max(int(df), 0) if df is not None else 0
		if df < 10:
			return 0
		return num * max(math.log(df + 1, 50), 1.5)

	index = [(x, score(x, y)) for x, y in sorted(index.iteritems(), key=lambda x: -x[1])]
	index = filter(lambda x: x[1] > 0, index)

	tags = OrderedDict(sorted(index, key=lambda x: -x[1])).keys()

	for tag in list(tags):
		sub = False
		for other in tags:
			if tag == other:
				sub = True
			elif tag in other:
				if len(tag) < 4 and len(other) >= 4 or sub == False:
					if tag in tags:
						tags.remove(tag)
				else:
					tags.remove(other)
	return tags[:6]
Esempio n. 2
0
def make_tags(article, index):
    def score(word, num):
        word = word.lower()
        md = hashlib.md5(word.encode('utf-8')).hexdigest()[:2]
        df = redis_tag.hget(md, word)
        df = max(int(df), 0) if df is not None else 0
        if df < 10:
            return 0
        return num * max(math.log(df + 1, 50), 1.5)

    index = [(x, score(x, y))
             for x, y in sorted(index.iteritems(), key=lambda x: -x[1])]
    index = filter(lambda x: x[1] > 0, index)

    tags = OrderedDict(sorted(index, key=lambda x: -x[1])).keys()

    for tag in list(tags):
        sub = False
        for other in tags:
            if tag == other:
                sub = True
            elif tag in other:
                if len(tag) < 4 and len(other) >= 4 or sub == False:
                    if tag in tags:
                        tags.remove(tag)
                else:
                    tags.remove(other)
    return tags[:6]
class Adducts:
    def __init__(self, ion_mode=None, e=0.0005486):

        self.e = e
        if ion_mode == "pos":
            self.lib = OrderedDict()
        elif ion_mode == "neg":
            self.lib = OrderedDict()
        elif ion_mode is None:
            self.lib = OrderedDict()

    def add(self, name, mass, charge):
        self.lib[name] = OrderedDict([("mass", float(mass)),
                                      ("charge", int(charge))])
        self.lib = OrderedDict(
            sorted(self.lib.items(), key=lambda x: x[1]['mass']))

    def remove(self, name):
        if name == "*":
            self.lib = OrderedDict()
        else:
            if name in self.lib:
                self.lib.remove(name)
            else:
                raise IOError("Entry not in library: {}".format(name))

    def __str__(self):
        out = "Adducts in library\n"
        out += "-----------------\n"
        out += "name\texact_mass\n"
        for key in self.lib:
            out += "%s\t%s\n" % (key, self.lib[key])
        return out
def number_of_strongly_connected_components(adj):
    indegree_adj = [[] for _ in range(len(adj))]
    # print(indegree_adj)
    for i in range(len(adj)):
        for val in adj[i]:
            indegree_adj[val].append(i)

    # print(indegree_adj)

    # postorder = [[0,0] for _ in range(len(adj))] ## Moving to Ordered Dict to handle large datasets

    postordered_od = OrderedDict({i: [0,0] for i in range(len(adj))})


    # find_postorders_reverse_graph(postorder,indegree_adj)
    find_postorders_reverse_graph(postordered_od,indegree_adj)

    # print(postordered_od)

    # Heap solution not viable as after DFS we need to remove traversed components as well
    # postorder_heap = list((-val[1],node) for node, val in enumerate(postorder))
    # heapq.heapify(postorder_heap)  # adding minus as heapq is min heap
    # print(postorder_heap)

    postordered_od = sorted(postordered_od, key=lambda x: postordered_od[x][1], reverse=True)

    # print(postordered_od)

    visited = [False] * len(adj)
    scc = []

    def do_dfs(node, entry):
        visited[node] = True
        entry.add(node)

        for neighbor in adj[node]:
            if not visited[neighbor]:
                do_dfs(neighbor, entry)

        return

    while not all(visited):
        # max_postorder_node = find_max_postorder_node(postorder) # Replacing with heap to handle large datasets
        # max_postorder_val, max_postorder_node = heapq.heappop(postorder_heap)
        max_postorder_node = postordered_od[0]
        scc_entry = set()
        do_dfs(max_postorder_node, scc_entry)
        # print(scc_entry)
        scc.append(scc_entry)
        # scratch_postorder(scc_entry, postorder) # Replacing with heap to handle large datasets
        for e in scc_entry:
            postordered_od.remove(e)

    # print(scc)
    return len(scc)
Esempio n. 5
0
class CombInfoDictionary(object):
    def __init__(self, jsonString=None):
        self.__dict__ = OrderedDict()
        if jsonString is not None:
            decoder = JSONDecoder(object_hook=OrderedDict)
            self.__dict__ = decoder.decode(jsonString)

    def __getitem__(self, key):
        for k, v in self.__dict__.items():
            if k == key:
                return v
            if type(v) is dict:
                ret = self._getitem(key, v)
                if ret is not None:
                    return ret
            elif type(v) is list:
                for x in v:
                    ret = self._getitem(key, x)
                    if ret is not None:
                        return ret
        return None

    def __setitem__(self, key, val):
        if key not in self.__dict__.keys():
            self.__dict__[key] = val

    def __delitem__(self, key):
        if key in self.__dict__.keys():
            self.__dict__.remove(key)

    def __iter__(self):
        for key in self.__dict__.keys():
            node = self.__dict__[key]
            for x in node.keys():
                yield node[x]

    def __str__(self):
        return json.dumps(self,
                          default=lambda x: x.__dict__,
                          sort_keys=False,
                          indent=4)

    def Save(self):
        if not os.path.isdir(Consts.COMB_OUTPUT_FOLDER):
            os.mkdir(Consts.COMB_OUTPUT_FOLDER)
        jsonpath = os.path.realpath(r'{}/CombInfo-{}.json'.format(
            Consts.COMB_OUTPUT_FOLDER, IdGenerator()))
        with open(jsonpath, 'w') as outputfile:
            outputfile.write(str(self))
Esempio n. 6
0
class Frame:
    def __init__(self, id):
        """
        Initialize video frame.

        :return: int sequence number of frame
        """
        self.id = id
        self.timecode = frames2timecode(self.id)

        self._droplets = OrderedDict()

    @property
    def droplets(self):
        return self._droplets

    @property
    def droplet_count(self):
        return len(self._droplets)

    def add_droplet(self):
        """
        Add new droplet to the video frame.

        :param droplet_id:
        """
        d = Droplet()
        self._droplets[d.id] = d

        return d

    def remove_droplet(self, droplet_id):
        """
        Remove a droplet from the video frame.

        :param droplet_id:
        """

        self._droplets.remove(droplet_id)
Esempio n. 7
0
class Metrix(QtGui.QTreeWidget):
	def __init__(self, parent=None, updateInterval=300):
		QtGui.QTreeWidget.__init__(self, parent)
		self.variables = OrderedDict()
		self.treeitems = {}

		headerItem = QtGui.QTreeWidgetItem()
		headerItem.setData(0,0, "Name")
		headerItem.setData(1,0, "Value")
		self.setHeaderItem(headerItem)
		self.setRootIsDecorated(False)
		self.setWordWrap(True)
		self.timer = QtCore.QTimer(self)
		self.timer.setInterval(updateInterval)
		#self.timer.timerEvent = self.update
		self.timer.timeout.connect(self.update)
		self.timer.start()

	def add(self, name, callback):
		self.variables[name] = callback

		self.treeitems[name] = QtGui.QTreeWidgetItem()
		self.treeitems[name].setData(0, 0, QtCore.QString(name))
		self.treeitems[name].setData(1, 0, QtCore.QString(repr(callback())))
		
		self.addTopLevelItem(self.treeitems[name])
		self.resizeColumnToContents(0)
		self.resizeColumnToContents(1)

	def remove(self, name):
		self.variables.remove(name)
		self.removeItemWidget(self.treeitems[name])
		del self.treeitems[name]

	@QtCore.pyqtSlot()
	def update(self, dummy=None):
		for name, callback in self.variables.iteritems():
			self.treeitems[name].setData(1, 0, repr(callback()))
Esempio n. 8
0
def main():

    pstat = {
        'status': {},
        'text': {},
        'short': {},
    }
    now = time.strftime("%Y-%m-%d %H:%M:%S")
    jid = os.getpid()
    ckanlistrequests = ['package_list', 'group_list', 'tag_list']

    ## Get options and arguments
    args = get_args(ckanlistrequests)

    # Output instance
    OUT = Output(pstat, now, jid, args)
    logger = OUT.setup_custom_logger('root', args.verbose)

    ## Settings for CKAN client and API
    ckanapi3 = 'http://' + args.ckan + '/api/3'
    if PY2:
        ckan = ckanclient.CkanClient(ckanapi3)
    else:
        auth = '12345'
        ckan = CKAN_CLIENT(args.ckan, auth)

    ckan_limit = 500000

    start = time.time()

    if args.request.endswith('list'):
        try:
            if args.request == 'community_list':
                action = 'group_list'
            else:
                action = args.request
            if PY2:
                answer = ckan.action(action, rows=ckan_limit)
            else:
                answer = ckan.action(action)
        except ckanclient.CkanApiError as e:
            print('\t\tError %s Supported list requests are %s.' %
                  (e, ckanlistrequests))
            sys.exit(1)
        ## print '|- The list of %ss :\n\t%s' % (args.request.split('_')[0],'\n\t'.join(answer).encode('utf8'))
        print('\n\t%s' % '\n\t'.join(answer).encode('utf8'))
        sys.exit(0)

    # create CKAN search pattern :
    ckan_pattern = ''
    sand = ''
    pattern = ' '.join(args.pattern)

    if (args.community):
        ckan_pattern += "groups:%s" % args.community
        sand = " AND "
    if (args.pattern):
        ckan_pattern += sand + pattern

    print(' | - Search\n\t|- in\t%s\n\t|- for\t%s\n' %
          (args.ckan, ckan_pattern))

    if args.request == 'package_search':
        if PY2:
            answer = ckan.action('package_search',
                                 q=ckan_pattern,
                                 rows=ckan_limit)
        else:
            answer = ckan.action('package_search', {"q": ckan_pattern})
    for key, value in answer.items():
        logger.warning('answer has key %s' % key)
    if PY2:
        tcount = answer['count']
    else:
        tcount = answer['result']['count']
    print(' | - Results:\n\t|- %d records found in %d sec' %
          (tcount, time.time() - start))

    # Read in B2FIND metadata schema and fields
    schemafile = '%s/mapfiles/b2find_schema.json' % (os.getcwd())
    with open(schemafile, 'r') as f:
        b2findfields = json.loads(f.read(), object_pairs_hook=OrderedDict)

    if tcount > 0 and args.keys is not None:
        if len(args.keys) == 0:
            akeys = []
        else:
            if args.keys[0] == 'B2FIND.*':
                akeys = OrderedDict(sorted(b2findfields.keys()))
            else:
                akeys = args.keys

        suppid = b2findfields.keys()

        fh = io.open(args.output, "w", encoding='utf8')
        record = {}

        totlist = []
        count = {}
        count['id'] = 0
        statc = {}
        for outt in akeys:
            if outt not in suppid:
                print(' [WARNING] Not supported key %s is removed' % outt)
                akeys.remove(outt)
            else:
                count[outt] = 0
                statc[outt] = Counter()

        printfacets = ''
        if (len(akeys) > 0):
            printfacets = "and related facets %s " % ", ".join(akeys)

            print('\t|- IDs %sare written to %s ...' %
                  (printfacets, args.output))

        counter = 0
        cstart = 0
        oldperc = 0
        start2 = time.time()

        while (cstart < tcount):
            if (cstart > 0):
                if PY2:
                    answer = ckan.action('package_search',
                                         q=ckan_pattern,
                                         rows=ckan_limit,
                                         start=cstart)
                else:
                    answer = ckan.action('package_search', {
                        "q": ckan_pattern,
                        "rows": ckan_limit,
                        "start": cstart
                    })
            if PY2:
                if len(answer['results']) == 0:
                    break
            #HEW-D else:
            ##HEW-D    if len(answer['result']['results']) == 0 :
            ##HEW-D        break

            # loop over found records
            if PY2:
                results = answer['results']
            else:
                results = answer['result']['results']
            for ds in results:  #### answer['results']:
                counter += 1
                logger.debug('    | %-4d | %-40s |' % (counter, ds['name']))
                perc = int(counter * 100 / tcount)
                bartags = perc / 5
                if perc % 10 == 0 and perc != oldperc:
                    oldperc = perc
                    print('\r\t[%-20s] %5d (%3d%%) in %d sec' %
                          ('=' * int(bartags), counter, perc,
                           time.time() - start2))
                    sys.stdout.flush()

                record['id'] = '%s' % (ds['name'])
                outline = record['id']

                # loop over facets
                for facet in akeys:
                    ##HEW-T print 'facet : %s' % facet
                    ckanFacet = b2findfields[facet]["ckanName"]
                    if ckanFacet in ds:  ## CKAN default field
                        if facet == 'Group':
                            record[facet] = ds[ckanFacet][0]['display_name']
                        else:
                            record[facet] = ds[ckanFacet]
                    else:  ## CKAN extra field
                        ##HEW-T print 'ds extras %s' % ds['extras']
                        efacet = [e for e in ds['extras'] if e['key'] == facet]
                        if efacet:
                            ##HEW-T print 'rrrr %s effff %s' % (record[facet],efacet[0]['value'])
                            record[facet] = efacet[0]['value']
                        else:
                            record[facet] = 'N/A'
                    if record[facet] is None:
                        record[facet] = 'None'
                        statc[facet][record[facet]] += 1
                    else:
                        if not isinstance(record[facet], list):
                            words = record[facet].split(';')
                        else:
                            words = record[facet]
                        for word in words:
                            if isinstance(word, dict): word = word['name']
                            statc[facet][word] += 1
                    if not (record[facet] == 'N/A' or record[facet]
                            == 'Not Stated') and len(record[facet]) > 0:
                        count[facet] += 1
                    outline += '\t | %-30s' % record[facet][:30]
                fh.write(outline + '\n')
            cstart += len(results)
            logger.warning('%d records done, %d in total' % (cstart, tcount))
        fh.close()

        if len(akeys) > 0:
            statfh = io.open('stat_' + args.output, "w", encoding='utf8')
            ##print "\n|- Statistics :\n\t| %-16s | %-10s | %6s |\n\t%s " % ('Facet','Occurence','%',"-" * 50)
            print('|- Statistics written to file %s' % 'stat_' + args.output)

            statline = unicode("")
            for outt in akeys:
                statline += "| %-16s\n\t| %-15s | %-6d | %3d |\n" % (
                    outt, '-Total-', count[outt],
                    int(count[outt] * 100 / tcount))
                for word in statc[outt].most_common(10):
                    statline += '\t| %-15s | %-6d | %3d |\n' % (
                        word[0][:100], word[1], int(word[1] * 100 / tcount))

            statfh.write(statline)

            statfh.close()
Esempio n. 9
0
def main():

    ckanlistrequests=['package_list','group_list','tag_list']

    ## Get options and arguments
    args = get_args(ckanlistrequests)
    
    ## Settings for CKAN client and API
    ckanapi3='http://'+args.ckan+'/api/3'
    ckan = ckanclient.CkanClient(ckanapi3)
    ckan_limit=500000

    start=time.time()

    if args.request.endswith('list'):
        try:
            if args.request == 'community_list' :
                action='group_list'
            else:
                action=args.request
            answer = ckan.action(action, rows=ckan_limit)
        except ckanclient.CkanApiError as e :
            print '\t\tError %s Supported list requests are %s.' % (e,ckanlistrequests)
            sys.exit(1)
        ## print '|- The list of %ss :\n\t%s' % (args.request.split('_')[0],'\n\t'.join(answer).encode('utf8'))
        print '\n\t%s' % '\n\t'.join(answer).encode('utf8')
        sys.exit(0)

    # create CKAN search pattern :
    ckan_pattern = ''
    sand=''
    pattern=' '.join(args.pattern)

    if (args.community):
        ckan_pattern += "groups:%s" % args.community
        sand=" AND "
    if (args.pattern):
        ckan_pattern += sand + pattern   

    print ' | - Search\n\t|- in\t%s\n\t|- for\t%s\n' % (args.ckan,ckan_pattern)

    if args.request == 'package_search' :
        answer = ckan.action('package_search', q=ckan_pattern, rows=ckan_limit)
    tcount=answer['count']
    print " | - Results:\n\t|- %d records found in %d sec" % (tcount,time.time()-start)

    # Read in B2FIND metadata schema and fields
    schemafile =  '%s/mapfiles/b2find_schema.json' % (os.getcwd())
    with open(schemafile, 'r') as f:
        b2findfields=json.loads(f.read(), object_pairs_hook=OrderedDict)   


    if tcount>0 and args.keys is not None :
        if len(args.keys) == 0 :
            akeys=[]
        else:
            if args.keys[0] == 'B2FIND.*' :
                akeys=OrderedDict(sorted(b2findfields.keys()))
            else:
                akeys=args.keys

        suppid=b2findfields.keys()

        fh = io.open(args.output, "w", encoding='utf8')
        record={} 
  
        totlist=[]
        count={}
        count['id']=0
        statc={}
        for outt in akeys:
                if outt not in suppid :
                    print ' [WARNING] Not supported key %s is removed' % outt
                    akeys.remove(outt)
                else:
                    count[outt]=0
                    statc[outt] = Counter()

        printfacets=''
        if (len(akeys) > 0):
            printfacets="and related facets %s " % ", ".join(akeys)

            print "\t|- ID's %sare written to %s ..." % (printfacets,args.output)

        counter=0
        cstart=0
        oldperc=0
        start2=time.time()

        while (cstart < tcount) :
	       if (cstart > 0):
	           answer = ckan.action('package_search', q=ckan_pattern, rows=ckan_limit, start=cstart)
	       if len(answer['results']) == 0 :
	           break
	
	       # loop over found records
	       for ds in answer['results']:
	            counter +=1
	            ##HEW-T print'    | %-4d | %-40s |' % (counter,ds['name'])
	            perc=int(counter*100/tcount)
	            bartags=perc/5
	            if perc%10 == 0 and perc != oldperc :
	                oldperc=perc
	                print "\r\t[%-20s] %5d (%3d%%) in %d sec" % ('='*bartags, counter, perc, time.time()-start2 )
	                sys.stdout.flush()
	
	            
	            record['id']  = '%s' % (ds['name'])
	            outline=record['id']
	
	            # loop over facets
	            for facet in akeys:
                        ##HEW-T print 'facet : %s' % facet
                        ckanFacet=b2findfields[facet]["ckanName"]
	                if ckanFacet in ds: ## CKAN default field
	                    if facet == 'Group':
	                        record[facet]  = ds[ckanFacet][0]['display_name']
	                    else:
	                        record[facet]  = ds[ckanFacet]
	
	                else: ## CKAN extra field
                            ##HEW-T print 'ds extras %s' % ds['extras']
	                    efacet=[e for e in ds['extras'] if e['key'] == facet]
	                    if efacet:
                                ##HEW-T print 'rrrr %s effff %s' % (record[facet],efacet[0]['value'])
	                        record[facet]  = efacet[0]['value']
	                    else:
	                        record[facet]  = 'N/A'
	                if record[facet] is None :
	                    record[facet]='None'
	                    statc[facet][record[facet]]+=1
	                else:
	                    if not isinstance(record[facet],list):
	                        words=record[facet].split(';')
	                    else:
	                        words=record[facet]
	                    for word in words:
	                        if isinstance(word,dict): word=word['name']
	                        statc[facet][word]+=1
	                if not ( record[facet] == 'N/A' or record[facet] == 'Not Stated') and len(record[facet])>0 : 
	                    count[facet]+=1
	
		        outline+='\t | %-30s' % record[facet][:30]
	                ##outline+='\n   \t| %-20s' % (statc[aid].keys(),statc[aid].values())
	            fh.write(outline+'\n')
	       cstart+=len(answer['results']) 
        fh.close()
	
        if len(akeys) > 0 :
	        statfh = io.open('stat_'+args.output, "w", encoding='utf8')
	        ##print "\n|- Statistics :\n\t| %-16s | %-10s | %6s |\n\t%s " % ('Facet','Occurence','%',"-" * 50)
	        print '|- Statistics written to file %s' % 'stat_'+args.output
	
	        statline=unicode("")
	        for outt in akeys:
	            statline+= "| %-16s\n\t| %-15s | %-6d | %3d |\n" % (outt,'-Total-',count[outt],int(count[outt]*100/tcount))
	            for word in statc[outt].most_common(10):
	                statline+= '\t| %-15s | %-6d | %3d |\n' % (word[0][:100], word[1], int(word[1]*100/tcount))
	
	        statfh.write(statline)
	
	        statfh.close()
Esempio n. 10
0
            while (catstartnum < num):
                catstartnum += category_offset

        # move all VOD categories to VOD placeholder position
        if ("VOD" in category_order):
            vodindex = category_order.index("VOD")
            vodcategories = list(
                (cat for cat in category_order if cat.startswith('VOD -')))
            if len(vodcategories):
                #remove the multi vod categories from their current location
                category_order = [
                    x for x in category_order if x not in vodcategories
                ]
                #insert the multi vod categories at the placeholder pos
                category_order[vodindex:vodindex] = vodcategories
                category_order.remove("VOD")

        # Check for and parse override map
        self.parse_map_channels_xml(dictchannels)

        # Have a look at what we have
        if DEBUG and TESTRUN:
            datafile = open(os.path.join(EPGIMPORTPATH, 'channels.debug'),
                            "w+")
            for cat in category_order:
                if cat in dictchannels:
                    for line in dictchannels[cat]:
                        linevals = ""
                        for key, value in line.items():
                            if type(value) is bool:
                                linevals += str(value) + ":"
Esempio n. 11
0
def main():
    args = get_args()
    
    print "\n%s" %('-'*100)
    # create CKAN search pattern :
    ckan_pattern = ''
    sand=''
    pattern=' '.join(args.pattern)

    if (args.community):
        ckan_pattern += "groups:%s" % args.community
        sand=" AND "
    if (args.pattern):
        ckan_pattern += sand + pattern   

    print ' | - Search\n\t|- in\t%s\n\t|- for\t%s\n' % (args.ckan,ckan_pattern)

    ckanapi3='http://'+args.ckan+'/api/3'
    ckan = ckanclient.CkanClient(ckanapi3)
    ckan_limit=100000
    start=time.time()
    answer = ckan.action('package_search', q=ckan_pattern, rows=ckan_limit)
    tcount=answer['count']
    print " | - Results:\n\t|- %d records found in %d sec" % (tcount,time.time()-start)

    b2findorder=['Title','Description','Tags','Source','DOI','PID','Checksum','Rights','Discipline','Creator','Publisher','PublicationYear','Language','TemporalCoverage','SpatialCoverage','Format','Contact','MetadataAccess']

    b2findfacets={
        'Title':'title',
        'Description':'notes',
        'Tags':'tags',
        'Source':'url',
        'DOI':'DOI',
        'PID':'PID',
        'Checksum':'version',
        'Rights':'Rights',
        'Discipline':'Discipline',
        'Creator':'author',
        'Publisher':'Publisher',
        'PublicationYear':'PublicationYear',
        'Language':'Language',
        'TemporalCoverage':'TemporalCoverage',
        'SpatialCoverage':'SpatialCoverage',
        'Format':'Format',
        'Contact':'Contact',
        'MetadataAccess':'MetadataAccess'
}

    admin_fields={
        'ManagerVersion':'ManagerVersion',
        'modified':'metadata_modified',
        'created':'metadata_created',
        'oai_set':'oai_set'
}

    if tcount>0 and args.keys is not None :
        if len(args.keys) == 0 :
            akeys=[]
        else:
            if args.keys[0] == 'B2FIND.*' :
                akeys=OrderedDict(sorted(b2findfacets.items(), key=lambda i:b2findorder.index(i[0])))
            else:
                akeys=args.keys

        suppid=b2findfacets
        suppid.update(admin_fields)

        fh = io.open(args.output, "w", encoding='utf8')
        record={} 
  
        totlist=[]
        count={}
        count['id']=0
        statc={}
        for outt in akeys:
                if outt not in suppid :
                    print ' [WARNING] Not supported key %s is removed' % outt
                    akeys.remove(outt)
                else:
                    count[outt]=0
                    statc[outt] = Counter()

        printfacets=''
        if (len(akeys) > 0):
            printfacets="and related facets %s " % ", ".join(akeys)

            print "\t|- ID's %sare written to %s ..." % (printfacets,args.output)

        counter=0
        cstart=0
        oldperc=0
        start2=time.time()
        while (cstart < tcount) :
	       if (cstart > 0):
	           answer = ckan.action('package_search', q=ckan_pattern, rows=ckan_limit, start=cstart)
	       if len(answer['results']) == 0 :
	           break
	
	       # loop over records
	       for ds in answer['results']:
	            counter +=1
	            ##HEW-T print'    | %-4d | %-40s |' % (counter,ds['name'])
	            perc=int(counter*100/tcount)
	            bartags=perc/5
	            if perc%10 == 0 and perc != oldperc :
	                oldperc=perc
	                print "\r\t[%-20s] %5d (%3d%%) in %d sec" % ('='*bartags, counter, perc, time.time()-start2 )
	                sys.stdout.flush()
	
	            
	            record['id']  = '%s' % (ds['name'])
	
	            # loop over facets
	            for facet in akeys:
	                if suppid[facet] in ds: ## CKAN default field
	                    if facet == 'Group':
	                        record[facet]  = ds[suppid[facet]][0]['display_name']
	                    else:
	                        record[facet]  = ds[suppid[facet]]
	
	                else: ## CKAN extra field
	                    efacet=[e for e in ds['extras'] if e['key'] == facet]
	                    if efacet:
	                        record[facet]  = efacet[0]['value']
	                    else:
	                        record[facet]  = 'N/A'
	                if record[facet] is None :
	                    record[facet]='None'
	                    statc[facet][record[facet]]+=1
	                else:
	                    if not isinstance(record[facet],list):
	                        words=record[facet].split(';')
	                    else:
	                        words=record[facet]
	                    for word in words:
	                        if isinstance(word,dict): word=word['name']
	                        statc[facet][word]+=1
	                if not ( record[facet] == 'N/A' or record[facet] == 'Not Stated') and len(record[facet])>0 : 
	                    count[facet]+=1
	
	
	            outline=record['id']
	            for aid in akeys:
	                outline+='\t | %-30s' % record[aid][:30]
	                ##outline+='\n   \t| %-20s' % (statc[aid].keys(),statc[aid].values())
	            fh.write(outline+'\n')
	       cstart+=len(answer['results']) 
        fh.close()
	
        if len(akeys) > 0 :
	        statfh = io.open('stat_'+args.output, "w", encoding='utf8')
	        ##print "\n|- Statistics :\n\t| %-16s | %-10s | %6s |\n\t%s " % ('Facet','Occurence','%',"-" * 50)
	        print '|- Statistics written to file %s' % 'stat_'+args.output
	
	        statline=unicode("")
	        for outt in akeys:
	            statline+= "| %-16s | %-10d | %6d |\n" % (outt,count[outt],int(count[outt]*100/tcount))
	            for word in statc[outt].most_common(10):
	                statline+= '\t| %-10d | %-100s\n' % (word[1], word[0][:100])
	
	        statfh.write(statline)
	
	        statfh.close()
Esempio n. 12
0
class asmlAscii(object):
    """ Class for ASML jobfile

    Each object corresponds to an entire (possibly not fully specified) ASML
    jobfile, organized as a dictionary of lists of section objects, specified
    by section type.

    Class methods provide functionality to get, remove, and append sections,
    to merge jobfiles, to check for section interferences and jobfile completion,
    and to read and write ASCII strings and files.
    """
    def __init__(self):
        """ Initializes dictionary of empty section lists """
        self.sections = OrderedDict()
        for sectionname in ASMLJOBSECTIONS:
            self.sections[sectionname] = []
    
    def append(self,newsection,check_interference=False):
        """ Adds a new section object to the jobfile

        Can check for interference between the new section and
        existing sections and only add the new section if it does
        not interfere.

        Returns boolean for convenience of calling processes.
        """
        if check_interference:
            for section in self.sections[newsection.sectionname]:
                if asmlSection.interfere(section,newsection):
                    return False

        # If the interference test passes, append the new section            
        self.sections[newsection.sectionname].append(newsection)
        return True

    def get(self,sectionname,**id_element_values):
        """ Get sections from jobfile

        Identifies sections by name and optional specification of
        ID elements. If the specification matches several sections
        in the jobfile, returns a list of matching sections.
        """
        matchsections = []
        for section in self.sections[sectionname]:
            id_match = True
            for id_element_name in id_element_values:
                if id_element_name not in section.id_elements:
                    raise ValueError('{} not a valid id element for {} section'.format(id_element_name,sectionname))

                if section.get(id_element_name) != id_element_values[id_element_name]:
                    id_match = False    # Change match to False if any ID element disagrees
                    
            if id_match:
                matchsections.append(section)
        
        return matchsections

    def remove(self,sectionname,**id_element_values):
        """ Removes sections from jobfile

        Identifies sections by name and optional specification of
        ID elements. If the specification matches several sections
        in the jobfile, removes all of them.
        """
        for section in self.sections[sectionname]:
            id_match = True
            for id_element_name in id_element_values:
                if id_element_name not in section.id_elements:
                    raise ValueError('{} not a valid id element for {} section'.format(id_element_name,sectionname))

                if section.get(id_element_name) != id_element_values[id_element_name]:
                    id_match = False
            if id_match:
                self.sections.remove(section)
    
    @staticmethod
    def merge(job1,job2):
        """ Merges two jobfiles

        If there is any interference between sections of the jobfiles
        ignore conflicting sections from the second jobfile.
        """
        mergedjob = copy.deepcopy(job1)
        for sectionname in ASMLJOBSECTIONS:
            for section in job2.sections[sectionname]:
                mergedjob.append(section,check_interference=True)
        
        return mergedjob
    
    def isSpecified(self):
        """ Checks whether jobfile is adequately specified

        An adequately specified jobfile contains all non-optional sections,
        only one of each non-multiple-allowed sections, and has sections
        that are themselves adequately specified.
        """

        # There may be other things to check.
        # 
        # 1. Are layers in correct order?
        # 2. Does every layer have a PROCESS_DATA section?
        # 

        for sectionname in ASMLJOBSECTIONS:
            if (not ASMLJOBSECTIONS[sectionname]['is_optional']
                and len(self.sections[sectionname]) == 0):
                return False
            
            if (not ASMLJOBSECTIONS[sectionname]['multiple_allowed']
                and len(self.sections[sectionname]) > 1):
                # raising exception because this shouldn't be possible using class methods
                raise ValueError('Job can only have one {} section'.format(sectionname))
            
            for section in self.sections[sectionname]:
                if not section.isSpecified():
                    return False
        
        return True
        
    def makeAscii(self,fix_delim_bug=True):
        """ Generates ASCII string representation of jobfile """
        asciistring = ''
        for sectionname in self.sections:
            for section in self.sections[sectionname]:
                if fix_delim_bug:
                    section.fixDelimBug()
                asciistring += section.makeAscii() + '\n\n'
        
        return asciistring[:-2]     # strip off final newlines
    
    def readAscii(self,asciistring):
        """ Reads an ASCII string representation of a jobfile section """

        # Matches all section blocks in the string
        mm = re.findall(r'START_SECTION (\S*)\n(.*?)\nEND_SECTION',asciistring,re.DOTALL)
        for m in mm:
            sectionname = m[0]
            sectionbody = m[1]
            newsection = asmlSection(sectionname)
            newsection.readAscii(sectionbody)

            self.append(newsection)

    def readAsciiJobfile(self,filename):
        """ Loads and parses ASCII jobfile """
        with open(filename,'r') as f:
            asciistring = f.read()
        
        self.readAscii(asciistring)
    
    def writeAsciiJobfile(self,filename):
        """ Generates and saves ASCII jobfile """
        asciistring = self.makeAscii()

        with open(filename,'w') as f:
            f.write(asciistring)

    def printSummary(self):
        """ Print summary information on the jobfile

        Summary values could include:
            - number of images
            - number of instances
            - number of layers
            - number of image placements per layer
            - max. number of placements of each image
            - etc. 
        """
        print('Not yet implemented')
Esempio n. 13
0
class BaseListSelector(AbstractSelector):

    sequence = List(context=True)
    _hidden_keys = []
    _sort = Button('Sort')
    _remove = Button('-')

    def __init__(self):
        self._parameters = OrderedDict()
        self._selected_settings = None

    def add_parameter(self, parameter, label=None, default_value=None):
        if parameter in self._parameters:
            raise ValueError('Parameter already exists')
        if label is None:
            label = parameter
        self._parameters[parameter] = label
        for setting in self.sequence:
            setting[parameter] = default_value

    def remove_parameter(self, parameter):
        if parameter not in self._parameters:
            raise ValueError('Parameter not in list')
        self._parameters.remove(parameter)
        for setting in self.sequence:
            del(setting[parameter])

    def _new_setting(self):
        if self._selected_settings:
            return self._selected_settings[-1].copy()
        elif self.sequence:
            return self.sequence[-1].copy()
        else:
            return dict((p, 0) for p in self._parameters)

    def add_setting(self, setting=None):
        if setting is None:
            setting = self._new_setting()
        valid_keys = set(self._parameters.keys() + self._hidden_keys)
        if set(setting.keys()) != valid_keys:
            raise ValueError('Setting contains extra parameters')
        self.sequence.append(setting)

    def remove_setting(self, setting):
        self.sequence.remove(setting)

    def remove_selected_settings(self):
        for setting in self._selected_settings:
            self.remove_setting(setting)

    def __remove_fired(self):
        self.remove_selected_settings()

    def __sort_fired(self):
        self.sequence = sorted(self.sequence)

    def _buttons_view(self):
        raise NotImplementedError

    def _settings_view(self):
        raise NotImplementedError

    def traits_view(self):
        _column_map = list((v, k) for k, v in self._parameters.items())
        _editor = TabularEditor(
            adapter=DictTabularAdapter(columns=_column_map),
            auto_update=True,
            selected='_selected_settings',
            operations=['edit'],
            editable=True,
            editable_labels=True,
            multi_select=True,
        )
        return View(
            VGroup(
                self._settings_view(),
                self._buttons_view(),
                Item('sequence', editor=_editor, show_label=False),
            ),
            resizable=True,
        )
Esempio n. 14
0
def check_epkgs(epkg_list, lpps, efixes):
    """
    For each epkg get the label, packaging date, filset and check prerequisites
    based on fileset current level and build a list ordered by packaging date
    that should not be locked at its installation.

    Note: in case of parsing error, keep the epkg (best effort)

    args:
        epkg_list (list): The list of efixes to check
        lpps      (dict): The current lpps levels
        efixes    (dict): The current efixes info
    returns:
        The list of epkg to install (ordered by packaging date)
        The list of epkg to rejected with the reason (ordered by label)
    """

    epkgs_info = {}
    epkgs_reject = []

    # Installed efix could lock some files we will try to modify,
    # let's build a dictionary indexed upon file location
    locked_files = {}
    for efix in efixes:
        for file in efixes[efix]['files']:
            if file not in locked_files:
                locked_files[file] = efix
    module.debug('locked_files: {0}'.format(locked_files))

    # Get information on efix we want to install and check it can be installed
    for epkg_path in epkg_list:
        epkg = {'path': epkg_path,
                'label': '',
                'pkg_date': None,
                'sec_from_epoch': -1,
                'filesets': [],
                'files': [],
                'prereq': {},
                'reject': False}

        # get efix information
        cmd = '/usr/sbin/emgr -dXv3 -e {0} | /bin/grep -p -e PREREQ -e PACKAG'.format(epkg['path'])
        rc, stdout, stderr = module.run_command(cmd, use_unsafe_shell=True)
        if rc != 0:
            msg = 'Cannot get efix information {0}'.format(epkg['path'])
            module.log(msg)
            module.log('cmd:{0} failed rc={1} stdout:{2} stderr:{3}'
                       .format(cmd, rc, stdout, stderr))
            results['meta']['messages'].append(msg)
            # do not break or continue, we keep this efix, will try to install it anyway

        # ordered parsing: expecting the following line order:
        # LABEL, PACKAGING DATE, then PACKAGE, then prerequisites levels
        for line in stdout.splitlines():
            # skip comments and empty lines
            line = line.rstrip()
            if not line or line.startswith('+'):
                continue  # skip blank and comment line

            if not epkg['label']:
                # match: "LABEL:            IJ02726s8a"
                match = re.match(r'^LABEL:\s+(\S+)$', line)
                if match:
                    epkg['label'] = match.group(1)
                    continue

            if not epkg['pkg_date']:
                # match: "PACKAGING DATE:   Mon Oct  9 09:35:09 CDT 2017"
                match = re.match(r'^PACKAGING\s+DATE:\s+'
                                 r'(\S+\s+\S+\s+\d+\s+\d+:\d+:\d+\s+\S*\s*\S+).*$',
                                 line)
                if match:
                    epkg['pkg_date'] = match.group(1)
                    continue

            # match: "   PACKAGE:       devices.vdevice.IBM.vfc-client.rte"
            match = re.match(r'^\s+PACKAGE:\s+(\S+)\s*?$', line)
            if match:
                if match.group(1) not in epkg['filesets']:
                    epkg['filesets'].append(match.group(1))
                continue

            # match: "   LOCATION:      /usr/lib/boot/unix_64"
            match = re.match(r'^\s+LOCATION:\s+(\S+)\s*?$', line)
            if match:
                if match.group(1) not in epkg['files']:
                    epkg['files'].append(match.group(1))
                continue

            # match and convert prerequisite levels
            # line like: "bos.net.tcp.server 7.1.3.0 7.1.3.49"
            match = re.match(r'^(\S+)\s+([\d+\.]+)\s+([\d+\.]+)\s*?$', line)
            if match is None:
                continue
            (prereq, minlvl, maxlvl) = match.groups()
            epkg['prereq'][prereq] = {}
            epkg['prereq'][prereq]['minlvl'] = minlvl
            epkg['prereq'][prereq]['maxlvl'] = maxlvl

            # parsing done
            # check filseset prerequisite is present
            if prereq not in lpps:
                epkg['reject'] = '{0}: prerequisite missing: {1}'.format(os.path.basename(epkg['path']), prereq)
                module.log('reject {0}'.format(epkg['reject']))
                break  # stop parsing

            # check filseset prerequisite is present
            minlvl_i = list(map(int, epkg['prereq'][prereq]['minlvl'].split('.')))
            maxlvl_i = list(map(int, epkg['prereq'][prereq]['maxlvl'].split('.')))
            if lpps[prereq]['int'] < minlvl_i or lpps[prereq]['int'] > maxlvl_i:
                epkg['reject'] = '{0}: prerequisite {1} levels do not satisfy condition string: {2} =< {3} =< {4}'\
                                 .format(os.path.basename(epkg['path']),
                                         prereq,
                                         epkg['prereq'][prereq]['minlvl'],
                                         lpps[prereq]['str'],
                                         epkg['prereq'][prereq]['maxlvl'])
                module.log('reject {0}'.format(epkg['reject']))
                break
        if epkg['reject']:
            epkgs_reject.append(epkg['reject'])
            continue

        # check file locked by efix already installed
        for file in epkg['files']:
            if file in locked_files:
                results['meta']['messages'].append('installed efix {0} is locking {1} preventing the '
                                                   'installation of {2}, remove it manually or set the '
                                                   '"force" option.'
                                                   .format(locked_files[file], file, os.path.basename(epkg['path'])))
                epkg['reject'] = '{0}: installed efix {1} is locking {2}'\
                                 .format(os.path.basename(epkg['path']), locked_files[file], file)
                module.log('reject {0}'.format(epkg['reject']))
                epkgs_reject.append(epkg['reject'])
                continue
        if epkg['reject']:
            continue

        # convert packaging date into time in sec from epoch

        if epkg['pkg_date']:
            (sec_from_epoch, msg) = to_utc_epoch(epkg['pkg_date'])
            if sec_from_epoch == -1:
                module.log('{0}: "{1}" for epkg:{2}'.format(msg, epkg['pkg_date'], epkg))
            epkg['sec_from_epoch'] = sec_from_epoch

        epkgs_info[epkg['path']] = epkg.copy()

    # sort the epkg by packing date (sec from epoch)
    sorted_epkgs = OrderedDict(sorted(epkgs_info.items(),
                                      key=lambda t: t[1]['sec_from_epoch'],
                                      reverse=True)).keys()

    # exclude epkg that will be interlocked
    global_file_locks = []
    removed_epkg = []
    for epkg in sorted_epkgs:
        if set(epkgs_info[epkg]['files']).isdisjoint(set(global_file_locks)):
            global_file_locks.extend(epkgs_info[epkg]['files'])
            module.log('keep {0}, files: {1}'
                       .format(os.path.basename(epkgs_info[epkg]['path']), epkgs_info[epkg]['files']))
        else:
            results['meta']['messages'].append('a previous efix to install will lock a file of {0} '
                                               'preventing its installation, install it manually or '
                                               'run the task again.'
                                               .format(os.path.basename(epkgs_info[epkg]['path'])))
            epkgs_info[epkg]['reject'] = '{0}: locked by previous efix to install'\
                                         .format(os.path.basename(epkgs_info[epkg]['path']))
            module.log('reject {0}'.format(epkgs_info[epkg]['reject']))
            epkgs_reject.append(epkgs_info[epkg]['reject'])
            removed_epkg.append(epkg)
    for epkg in removed_epkg:
        sorted_epkgs.remove(epkg)

    epkgs_reject = sorted(epkgs_reject)  # order the reject list by label

    return (sorted_epkgs, epkgs_reject)
Esempio n. 15
0
class AutomationConfig(object):
    def __init__(self, source):
        self.__dict__ = OrderedDict()
        if os.path.isfile(source):
            with open(source, 'r') as f:
                jsonString = f.read()
        else:
            jsonString = source
        decoder = JSONDecoder(object_hook=OrderedDict)
        self.__dict__ = decoder.decode(jsonString)  # json.loads(jsonString)
        self._jsonFile = source

    def __getitem__(self, key):
        for k, v in self.__dict__.items():
            if k == key:
                return v
            if type(v) is dict:
                ret = self._getitem(key, v)
                if ret is not None:
                    return ret
            elif type(v) is list:
                for x in v:
                    ret = self._getitem(key, x)
                    if ret is not None:
                        return ret
        return None

    def _getitem(self, key, item):
        if key is None:
            return item
        elif key == item.keys()[0]:
            return item[key]

        if type(item) is dict:
            for k in item.keys():
                if k == key:
                    self._getitem(None, item[k])
        elif type(item) is list:
            for x in item:
                self._getitem(key, x)

    def __setitem__(self, key, val):
        if key not in self.__dict__.keys():
            self.__dict__[key] = val

    def __delitem__(self, key):
        if key in self.__dict__.keys():
            self.__dict__.remove(key)

    def __iter__(self):
        for key in self.__dict__.keys():
            node = self.__dict__[key]
            for x in node.keys():
                yield node[x]

    def __str__(self):
        #return json.dumps(self, default=lambda x: x.__dict__, sort_keys=False, indent=4)
        dic = OrderedDict()
        for k, v in self.__dict__.items():
            if not k.startswith('_'):
                dic[k] = v
        return json.dumps(dic,
                          default=lambda x: x.__dict__,
                          sort_keys=False,
                          indent=4)

    def Save(self):
        if self._jsonFile is not None:
            with open(self._jsonFile, 'w') as f:
                f.write(str(self))
Esempio n. 16
0
# alias new license types to existing licenses by recording them in the aliases dict below
# "new type": "existing type", e.g. "cc-by-nc": "cc-nc" (the open definition only includes cc-nc, not cc-by-nc)
aliases = {'cc-by-nc': 'cc-nc', 'cc-by-nc-sa': 'cc-nc-sa', 'cc-by-nc-nd': 'cc-nc-nd'}
for alias, target in aliases.items():
    LICENSES[alias] = LICENSES[target]


licenses_dropdown = [('','')] # default: empty
__process_licenses = OrderedDict(sorted(LICENSES.items(), key=lambda x: x[1]['title'])).items() # in alphabetical order of the titles (shown to the user)

# Creative Commons first
for lic_type, info in __process_licenses[:]:
    if lic_type.startswith('cc-'):
        choice = (lic_type, info.get('title', lic_type))
        licenses_dropdown.append(choice)
        __process_licenses.remove((lic_type, info))

# Free to Read next
for lic_type, info in __process_licenses[:]:
    if lic_type == 'free-to-read':
        choice = (lic_type, info.get('title', lic_type))
        licenses_dropdown.append(choice)
        __process_licenses.remove((lic_type, info))

dropdown_separator = ('', '--------------------')
licenses_dropdown.append(dropdown_separator)

# Take out all "Other"-type licenses to append to end of options later
other_license_choices = []
for lic_type, info in __process_licenses[:]:
    if lic_type.startswith('other-'):
Esempio n. 17
0
def check_epkgs(epkg_list, lpps, efixes, machine, output):
    """
    For each epkg get the label, packaging date, filset and check prerequisites
    based on fileset current level and build a list ordered by packaging date
    that should not be locked at its installation.

    Note: in case of parsing error, keep the epkg (best effort)

    args:
        epkg_list (list): The list of efixes to check
        lpps_lvl  (dict): The current lpps levels
        efixes    (dict): The current efixes info
        machine   (str) : The target machine
        output    (dict): The result of the command (use only before exit)
    returns:
        The list of epkg to install (ordered by packaging date)
        The list of epkg to rejected with the reason (ordered by label)
    """

    epkgs_info = {}
    epkgs_reject = []

    # Installed efix could lock some files we will try to modify,
    # build a dictionary indexed upon file location
    locked_files = {}
    for efix in efixes:
        for file in efixes[efix]['files']:
            if file not in locked_files:
                locked_files[file] = efix

    logging.debug('{}: locked_files: {}'.format(machine, locked_files))

    # Get information on efix we want to install
    # and check it could be installed
    for epkg_path in epkg_list:
        epkg = {'path': epkg_path,
                'label': '',
                'pkg_date': None,
                'sec_from_epoch': -1,
                'filesets': [],
                'files': [],
                'prereq': {},
                'reject': False}

        # get efix information
        stdout = ''
        try:
            cmd = '/usr/sbin/emgr -dXv3 -e {} | /bin/grep -p -e PREREQ -e PACKAG'\
                  .format(epkg['path'])
            stdout = subprocess.check_output(args=cmd, shell=True, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as exc:
            logging.warning('EXCEPTION cmd={} rc={} output={}'
                            .format(exc.cmd, exc.returncode, exc.output))
            # do not break or continue, we keep this efix, will try to install it anyway

        # ordered parsing: expecting the following line order:
        # LABEL, PACKAGING DATE, then PACKAGE, then prerequisites levels
        for line in stdout.splitlines():
            # skip comments and empty lines
            line = line.rstrip()
            if not line or line.startswith('+'):
                continue  # skip blank and comment line

            if not epkg['label']:
                # match: "LABEL:            IJ02726s8a"
                match = re.match(r'^LABEL:\s+(\S+)$', line)
                if match:
                    epkg['label'] = match.group(1)
                    continue

            if not epkg['pkg_date']:
                # match: "PACKAGING DATE:   Mon Oct  9 09:35:09 CDT 2017"
                match = re.match(r'^PACKAGING\s+DATE:\s+'
                                 '(\S+\s+\S+\s+\d+\s+\d+:\d+:\d+\s+\S*\s*\S+).*$',
                                 line)
                if match:
                    epkg['pkg_date'] = match.group(1)
                    continue

            # match: "   PACKAGE:       devices.vdevice.IBM.vfc-client.rte"
            match = re.match(r'^\s+PACKAGE:\s+(\S+)\s*?$', line)
            if match:
                if match.group(1) not in epkg['filesets']:
                    epkg['filesets'].append(match.group(1))
                continue

            # match: "   LOCATION:      /usr/lib/boot/unix_64"
            match = re.match(r'^\s+LOCATION:\s+(\S+)\s*?$', line)
            if match:
                if match.group(1) not in epkg['files']:
                    epkg['files'].append(match.group(1))
                continue

            # match and convert prerequisite levels
            # line like: "bos.net.tcp.server 7.1.3.0 7.1.3.49"
            match = re.match(r'^(\S+)\s+([\d+\.]+)\s+([\d+\.]+)\s*?$', line)
            if match is None:
                continue
            (prereq, minlvl, maxlvl) = match.groups()
            epkg['prereq'][prereq] = {}
            epkg['prereq'][prereq]['minlvl'] = minlvl
            epkg['prereq'][prereq]['maxlvl'] = maxlvl

            # parsing done
            # check filseset prerequisite is present
            if prereq not in lpps:
                epkg['reject'] = '{}: prerequisite missing: {}'.format(epkg['label'], prereq)
                logging.info('{}: reject {}'.format(machine, epkg['reject']))
                break  # stop parsing

            # check filseset prerequisite is present
            minlvl_i = list(map(int, epkg['prereq'][prereq]['minlvl'].split('.')))
            maxlvl_i = list(map(int, epkg['prereq'][prereq]['maxlvl'].split('.')))
            if lpps[prereq]['int'] < minlvl_i\
               or lpps[prereq]['int'] > maxlvl_i:
                epkg['reject'] = '{}: prerequisite {} levels do not match: {} < {} < {}'\
                                 .format(epkg['label'],
                                         prereq,
                                         epkg['prereq'][prereq]['minlvl'],
                                         lpps[prereq]['str'],
                                         epkg['prereq'][prereq]['maxlvl'])
                logging.info('{}: reject {}'.format(machine, epkg['reject']))
                break
        if epkg['reject']:
            epkgs_reject.append(epkg['reject'])
            continue

        # check file locked by efix already installed on the machine
        for file in epkg['files']:
            if file in locked_files:
                output['messages'].append('installed efix {} is locking {} preventing the '
                                          'installation of {}, remove it manually or set the '
                                          '"force" option.'
                                          .format(locked_files[file], file, epkg['label']))
                epkg['reject'] = '{}: installed efix {} is locking {}'\
                                 .format(epkg['label'], locked_files[file], file)
                logging.info('{}: reject {}'.format(machine, epkg['reject']))
                epkgs_reject.append(epkg['reject'])
                continue
        if epkg['reject']:
            continue

        # convert packaging date into time in sec from epoch
        if epkg['pkg_date']:
                (sec_from_epoch, msg) = to_utc_epoch(epkg['pkg_date'])
                if sec_from_epoch == -1:
                    logging.warning('{}: {}: "{}" for epkg:{} '
                                    .format(machine, msg, epkg['pkg_date'], epkg))
                epkg['sec_from_epoch'] = sec_from_epoch

        epkgs_info[epkg['path']] = epkg.copy()

    # sort the epkg by packing date (sec from epoch)
    sorted_epkgs = OrderedDict(sorted(epkgs_info.items(),
                                      key=lambda t: t[1]['sec_from_epoch'],
                                      reverse=True)).keys()

    # exclude epkg that will be interlocked
    global_file_locks = []
    removed_epkg = []
    for epkg in sorted_epkgs:
        lock_found = False
        if set(epkgs_info[epkg]['files']).isdisjoint(set(global_file_locks)):
            global_file_locks.extend(epkgs_info[epkg]['files'])
            logging.info('{}: keep {}, files: {}'
                         .format(machine, epkgs_info[epkg]['label'], epkgs_info[epkg]['files']))
        else:
            epkgs_info[epkg]['reject'] = '{}: locked by previous efix to install'\
                                         .format(epkgs_info[epkg]['label'])
            logging.info('{}: reject {}'.format(machine, epkgs_info[epkg]['reject']))
            epkgs_reject.append(epkgs_info[epkg]['reject'])
            removed_epkg.append(epkg)
    for epkg in removed_epkg:
        sorted_epkgs.remove(epkg)

    epkgs_reject = sorted(epkgs_reject)  # order the reject list by label

    return (sorted_epkgs, epkgs_reject)
Esempio n. 18
0
def main():

    pstat = {
        'status' : {},
        'text' : {},
        'short' : {},
     }
    now = time.strftime("%Y-%m-%d %H:%M:%S")
    jid = os.getpid()
    ckanlistrequests=['package_list','group_list','tag_list']

    ## Get options and arguments
    args = get_args(ckanlistrequests)

    # Output instance
    OUT = Output(pstat,now,jid,args)
    logger = OUT.setup_custom_logger('root',args.verbose)
    
    ## Settings for CKAN client and API
    ckanapi3='http://'+args.iphost+'/api/3'
    auth='12345'
    CKAN = CKAN_CLIENT(args.iphost,auth)

    ckan_limit=500000

    start=time.time()

    if args.request.endswith('list'):
        try:
            if args.request == 'community_list' :
                action='group_list'
            else:
                action=args.request
            if PY2 :
                answer = CKAN.action(action, rows=ckan_limit)
            else:
                answer = CKAN.action(action)
        except ckanclient.CkanApiError as e :
            print('\t\tError %s Supported list requests are %s.' % (e,ckanlistrequests))
            sys.exit(1)

        print('\n\t%s' % '\n\t'.join(answer).encode('utf8'))
        sys.exit(0)

    # create CKAN search pattern :
    ckan_pattern = ''
    sand=''
    pattern=' '.join(args.pattern)

    if (args.community):
        ckan_pattern += "groups:%s" % args.community
        sand=" AND "
    if (args.pattern):
        ckan_pattern += sand + pattern   

    print(' | - Search\n\t|- in\t%s\n\t|- for\t%s\n' % (args.iphost,ckan_pattern))

    if args.request == 'package_search' :
        if PY2:
            answer = CKAN.action('package_search', {"q":ckan_pattern}) ##HEW-D? , rows=ckan_limit)
        else:
            answer = CKAN.action('package_search',{"q":ckan_pattern})
    for key, value in answer.items() :
        logger.warning('answer has key %s' % key)
    if PY2 :
        tcount=answer['result']['count'] ### ['count']
    else:
        tcount=answer['result']['count']
    print(' | - Results:\n\t|- %d records found in %d sec' % (tcount,time.time()-start))

    # Read in B2FIND metadata schema and fields
    schemafile =  '%s/mapfiles/b2find_schema.json' % (os.getcwd())
    with open(schemafile, 'r') as f:
        b2findfields=json.loads(f.read(), object_pairs_hook=OrderedDict)   


    if tcount>0 and args.keys is not None :
        if len(args.keys) == 0 :
            akeys=[]
        else:
            if args.keys[0] == 'B2FIND.*' :
                akeys=OrderedDict(sorted(b2findfields.keys()))
            else:
                akeys=args.keys

        suppid=b2findfields.keys()

        fh = io.open(args.output, "w", encoding='utf8')
        record={} 
  
        totlist=[]
        count={}
        count['id']=0
        statc={}
        for outt in akeys:
                if outt not in suppid :
                    print(' [WARNING] Not supported key %s is removed' % outt)
                    akeys.remove(outt)
                else:
                    count[outt]=0
                    statc[outt] = Counter()

        printfacets=''
        if (len(akeys) > 0):
            printfacets="and related facets %s " % ", ".join(akeys)

            print('\t|- IDs %sare written to %s ...' % (printfacets,args.output))

        counter=0
        cstart=0
        oldperc=0
        start2=time.time()

        while (cstart < tcount) :
            if (cstart > 0):
                if PY2 :
                    answer = CKAN.action('package_search', {"q":ckan_pattern,"rows":ckan_limit,"start":cstart}) ##HEW-D q=ckan_pattern, rows=ckan_limit, start=cstart)
                else:
                    answer = CKAN.action('package_search',{"q":ckan_pattern,"rows":ckan_limit,"start":cstart})
            if PY2 :
                if len(answer['result']) == 0 :
                    break
        
            # loop over found records
            if PY2:
                results= answer['result']['results'] ### ['results']
            else:
                results= answer['result']['results']
            for ds in results : #### answer['results']:
                    counter +=1
                    logger.debug('    | %-4d | %-40s |' % (counter,ds['name']))
                    perc=int(counter*100/tcount)
                    bartags=perc/5
                    if perc%10 == 0 and perc != oldperc :
                        oldperc=perc
                        print('\r\t[%-20s] %5d (%3d%%) in %d sec' % ('='*int(bartags), counter, perc, time.time()-start2 ))
                        sys.stdout.flush()
        
                    
                    record['id']  = '%s' % (ds['name'])
                    outline='%s\n' % record['id']
        
                    # loop over facets
                    for facet in akeys:
                        ckanFacet=b2findfields[facet]["ckanName"]
                        if ckanFacet in ds: ## CKAN default field
                            if facet == 'Group':
                                record[facet]  = ds[ckanFacet][0]['display_name']
                            else:
                                record[facet]  = ds[ckanFacet]
                        else: ## CKAN extra field
                            efacet=[e for e in ds['extras'] if e['key'] == facet]
                            if efacet:
                                record[facet]  = efacet[0]['value']
                            else:
                                record[facet]  = 'N/A'
                        if record[facet] is None :
                            record[facet]='None'
                            statc[facet][record[facet]]+=1
                        else:
                            if not isinstance(record[facet],list):
                                words=record[facet].split(';')
                            else:
                                words=record[facet]
                            for word in words:
                                if isinstance(word,dict): word=word['name']
                                statc[facet][word]+=1
                        if not ( record[facet] == 'N/A' or record[facet] == 'Not Stated') and len(record[facet])>0 : 
                            count[facet]+=1
                        if PY2 :
                            fh.writelines((record[facet]+'\n').decode('utf-8'))
                        else :
                            fh.writelines((record[facet]+'\n'))

            cstart+=len(results)
            logger.warning('%d records done, %d in total' % (cstart,tcount))
        fh.close()
        
        if len(akeys) > 0 :
                statfh = io.open('stat_'+args.output, "w", encoding='utf8')
                print('|- Statistics written to file %s' % 'stat_'+args.output)
        
                statline=""
                for outt in akeys:
                    statline+= "| %-16s\n\t| %-15s | %-6d | %3d |\n" % (outt,'-Total-',count[outt],int(count[outt]*100/tcount))
                    for word in statc[outt].most_common(10):
                        statline+= '\t| %-15s | %-6d | %3d |\n' % (word[0][:100], word[1], int(word[1]*100/tcount))
        
                if PY2 :
                    statfh.write(statline.decode('utf-8'))
                else :
                    statfh.write(statline)
        
                statfh.close()
Esempio n. 19
0
class Run(object):
    __slots__ = [
        "_modality",  # modality associeted with this run
        "attribute",  # dictionary of attr:regexp
        "entity",  # dictionary for run entities
        "json",  # dictionary for json fields
        "_suffix",  # suffix associated with run
        "_bk_modality",  # copy of self old values
        "_bk_attribute",
        "_bk_entity",
        "_bk_suffix",
        "_bk_json",
        "provenance",  # file from which run is modelled
        "example",  # bids name from provenance file
        "writable",
        "checked",  # switch if run was confirmed by user
        "template"  # switch if run was extracted from template
    ]

    def __init__(self,
                 *,
                 modality: str = "",
                 attribute: dict = {},
                 entity: OrderedDict = {},
                 json: OrderedDict = {},
                 suffix: str = "",
                 provenance: str = None,
                 example: str = None):
        """
        Run class contains all information needed to identify and generate
        the bidsified name. All init parameters must be named.

        Parameters
        ----------
        modality: str
            the modality corresponding to given run
        attribute: dict
            dict of attributes (keys) and values (value) to match
        entity: OrderedDict
            dict of bids fields (keys) and values (value) to
            generate bidsified name
        json: OrderedDict
            dict of json fields (keys) and values (value) to
            fill sidecar json file
        suffix: str
            suffix associated with given run
        provenance: str
            path to the file from which this run was generated
        example: str
            example of bidsified name generated from provenance
            file
        """
        self._bk_modality = None
        self._bk_attribute = dict()
        self._bk_entity = dict()
        self._bk_suffix = None
        self._bk_json = dict()
        self.writable = True
        self.template = False
        self.checked = False
        self.example = example

        self.provenance = provenance
        self._modality = check_type("modality", str, modality)
        self._suffix = check_type("suffix", str, suffix)
        self.attribute = dict(check_type("attribute", dict, attribute))
        self.entity = OrderedDict(check_type("entity", dict, entity))
        self.json = OrderedDict(check_type("json", dict, json))

    def __bool__(self) -> bool:
        """
        Return True if suffix is defined
        """
        if self._suffix == "":
            return False
        return True

    @property
    def modality(self) -> str:
        """
        Return Modality
        """
        return self._modality

    @modality.setter
    def modality(self, value: str) -> None:
        """
        Sets Modality. The old (unmodified) value is backuped
        """
        if self._bk_modality is None:
            self._bk_modality = self._modality
        self.modality = value

    def restore_modality(self) -> None:
        """
        Restore old (unmodified) modality
        """
        if self._bk_modality is not None:
            self._modality = self._bk_modality
            self._bk_modality = None

    @property
    def suffix(self) -> str:
        """
        Return Suffix
        """
        return self._suffix

    @suffix.setter
    def suffix(self, value: str) -> None:
        """
        Sets new suffix. The old (unmodified) value is backuped
        """
        if self._bk_suffix is None:
            self._bk_suffix = self._suffix
        self.suffix = value

    def restore_suffix(self) -> None:
        """
        Restore old (unmodified) suffix
        """
        if self._bk_suffix is not None:
            self._suffix = self._bk_suffix
            self._bk_suffix = None

    def set_attribute(self, attr: str, val: object) -> None:
        """
        Sets attribute value. Old (unmodified) value is backuped

        Parameters
        ----------
        attr: str
            name of attribute to set
        value: object
            value to match given attribute
        """
        if isinstance(val, str):
            val = val.encode('unicode_escape').decode()
        attr = check_type("attr", str, attr)
        if attr in self.attribute:
            if self.attribute[attr] == val:
                return

            if attr not in self._bk_attribute:
                self._bk_attribute[attr] = self.attribute[attr]

                self.attribute[attr] = val
            return

        else:
            if val == "":
                return
            self.attribute[attr] = val
            if attr not in self._bk_attribute:
                self._bk_attribute[attr] = None

    def set_entity(self, ent: str, val: str) -> None:
        """
        Sets entity value. Old (unmodified) value is backuped

        Parameters
        ----------
        ent: str
            name of entity to set
        val: str
            value of entity
        """
        val = check_type("val", str, val)
        attr = check_type("ent", str, ent)

        if attr in self.entity:
            if self.entity[attr] == val:
                return

            if attr not in self._bk_entity:
                self._bk_entity[attr] = self.entity[attr]

            if val:
                self.entity[attr] = val
            else:
                self.entity.remove(attr)
            return

        else:
            if val == "":
                return
            self.entity[attr] = val
            if attr not in self._bk_entity:
                self._bk_entity[attr] = None

    def set_json_field(self, field: str, val: object) -> None:
        """
        Sets the json field value. The old (unmodified) value is backuped

        Parameters:
        -----------
        field: str
            name of field to set
        val: object
            value to set
        """
        if isinstance(val, str):
            val = val.encode('unicode_escape').decode()
        attr = check_type("field", str, field)

        if attr in self.json:
            if self.json[attr] == val:
                return

            if attr not in self._bk_json:
                self._bk_json[attr] = self.json[attr]

            if val:
                self.json[attr] = val
            else:
                self.json.remove(attr)
            return

        else:
            if val == "":
                return
            self.json[attr] = val
            if attr not in self._bk_json:
                self._bk_json[attr] = None

    def restore_attribute(self, attr: str) -> None:
        """
        Restore old value of given attribute.
        New value is lost

        Parameters:
        -----------
        attr: str
            name of attribute to restore
        """
        self.__restore_val(attr, self.attribute, self._bk_attribute)

    def restore_attributes(self):
        """
        Restore all attributes to old values.
        New values and new attributes are lost
        """
        for attr in self.attribute:
            self.__restore_val(attr, self.attribute, self._bk_attribute)

    def restore_entity(self, attr: str) -> None:
        """
        Restore old value for given entity.
        New value is lost.
        """
        self.__restore_val(attr, self.entity, self._bk_entity)

    def restore_entities(self) -> None:
        """
        Restore old values for all entities.
        New values and new entities are lost.
        """
        for attr in self.entity:
            self.__restore_val(attr, self.entity, self._bk_entity)

    def restore_json_field(self, attr: str) -> None:
        """
        Restore old value for json field.
        New value is lost
        """
        self.__restore_val(attr, self.json, self._bk_json)

    def restore_json(self) -> None:
        """
        Restore old values for all json fields.
        New values and new fields are lost.
        """
        for attr in self.json:
            self.__restore_val(attr, self.json, self._bk_json)

    def restore(self) -> None:
        """
        Restores old values for modality, suffix,
        attributesm entities and json fields
        """
        self.restore_modality()
        self.restore_suffix()
        self.restore_attributes()
        self.restore_entities()
        self.restore_json()

    def save(self) -> None:
        """
        Saves all current values, old values
        are lost
        """
        self._bk_modality = None
        self._bk_attribute = dict()
        self._bk_entity = dict()
        self._bk_json = dict()
        self._bk_suffix = None

    def __restore_val(self, attr: str, dest: dict, source: dict) -> None:
        """
        Restore old value from given dictionary.

        Parameters:
        -----------
        attr: str
            name of field to restore
        dest: dict
            dictionary where restore value
        source: dict
            dictionary from which retrieve values
        """
        if attr not in source:
            return
        if source[attr] is None:
            if attr in dest:
                dest.remove(attr)
            return
        dest[attr] = source.pop(attr)

    def dump(self, empty_attributes: bool = True) -> dict:
        """
        Dumps run into a dictionary

        Parameters:
        -----------
        empty_attributes: bool
            if True, the void and empty attributes are also dumped,
            if False, viod values are ignored
        """
        d = dict()
        d["provenance"] = self.provenance
        if self.example:
            d["example"] = self.example
        if self.template:
            d["template"] = self.template
        d["checked"] = self.checked
        d["suffix"] = self.suffix
        d["attributes"] = {
            k: v
            for k, v in self.attribute.items()
            if empty_attributes or v is not None
        }
        d["bids"] = self.entity
        d["json"] = self.json

        return d

    def genEntities(self, entities: list):
        """
        Completes the existing entities by entities from list
        All added values will be set to None

        Parameters
        ----------
        entities: list
            list of strings with names of entities to add
        """
        for ent in entities:
            if ent not in self.entity:
                self.entity[ent] = None
Esempio n. 20
0
# print len(corepoints)
# print directly_reachable

corepoints = OrderedDict((x, True) for x in corepoints).keys()
noise = set(points)
clusters = []
while len(corepoints) != 0:
    cluster = []
    cluster.append(corepoints[0])
    i = 0
    while i < len(cluster):
        point = cluster[i]
        if point in corepoints:
            cluster += directly_reachable[point]
            corepoints.remove(point)
            cluster = OrderedDict((x, True) for x in cluster).keys()
        i += 1
    clusters.append(cluster)
    noise = noise - set(cluster)
# print len(clusters)

for cluster in clusters:  # coloring the clusters
    x, y = zip(*cluster)
    plt.scatter(x, y)

if len(noise) > 0:  # coloring the noise points
    x, y = zip(*noise)
    plt.scatter(x, y, color='gray')

plt.show()  # display the plotted graph