class TwitterListener(StreamListener): def __init__(self): self.cache = LFUCache(maxsize=50) self.cache2 = LFUCache(maxsize=50) Timer(interval=60, function=self.print_keys) Timer(interval=30, function=self.check_cached_words) def on_data(self, data): data_lst = json.loads(data) data_lst = data_lst.get('text', '').split() if self.cache.currsize == self.cache.maxsize: for key in list(self.cache.keys()): if self.cache[key] == 0: del self.cache[key] for word in data_lst: if word in self.cache.keys(): self.cache[word] += 1 else: self.cache[word] = 1 if self.cache[word] < 0: del self.cache[word] return True def print_keys(self): """ print recent words and update the second cache every 60 seconds """ print(list(self.cache.items())) self.cache2.update(self.cache) return True def check_cached_words(self): """ Decrease score of word by 1 if the score does not change within 60 seconds """ for word in list(self.cache.keys()): if self.cache.get(word) == self.cache2.get(word): self.cache[word] -= 1 return True
class VendorBase(object): _vendorlogo = None _partclass = VendorPartBase _invoiceclass = VendorInvoice _type = 'BASE' _url_base = None _ident_blacklist = [] def __init__(self, name, dname, pclass, mappath=None, currency_code=BASE_CURRENCY, currency_symbol=BASE_CURRENCY_SYMBOL, vendorlogo=None, sname=None, is_manufacturer=None, vtype=None): self._name = name self._dname = dname self._sname = sname self._instance_vendorlogo = vendorlogo self._is_manufacturer = is_manufacturer self._currency = currency.CurrencyDefinition(currency_code, currency_symbol) self._vtype = vtype self._pclass = pclass self._order = None self._orderbasecosts = [] self._orderadditionalcosts = [] self._partcache = LFUCache(1000) if mappath is not None: self._mappath = mappath else: self._mappath = self._name + '-' + self._pclass + '.csv' self._map = VendorMapFileDB(self) @property def name(self): if self._dname is not None: return self._dname else: return self._name @property def sname(self): if self._sname is not None: return self._sname else: return self._name @property def cname(self): return self._name @property def pclass(self): return self._pclass @property def type(self): return self._type @property def is_manufacturer(self): if self._is_manufacturer is None: return False return self._is_manufacturer @property def mappath(self): return self._mappath @property def map(self): return self._map @property def currency(self): return self._currency @currency.setter def currency(self, currency_def): """ :type currency_def: utils.currency.CurrencyDefinition """ self._currency = currency_def @property def additional_costs(self): return self._orderadditionalcosts @property def logo(self): if self._instance_vendorlogo is not None: return self._instance_vendorlogo return self._vendorlogo @property def url_base(self): try: return self._url_base except AttributeError: return None def get_idents(self): for ident in self._map.get_idents(): yield ident def get_all_vpnos(self): for ident in self.get_idents(): for vpno in self._map.get_all_partnos(ident): yield ident, vpno def get_all_vparts(self, max_age=VENDOR_DEFAULT_MAXAGE): for ident, vpno in self.get_all_vpnos(): vpart = self.get_vpart(vpartno=vpno, ident=ident, max_age=max_age) if vpart: yield vpart def get_vpnos(self, ident, max_age=VENDOR_DEFAULT_MAXAGE): if ident in self._ident_blacklist: return [] acquire = False mtime = self._map.get_map_time(canonical=ident) if max_age > 0: now = time.time() if not mtime: acquire = True elif now - mtime > max_age: maintenance.update_vpmap(self.cname, ident) elif max_age == 0 or mtime is None: acquire = True if acquire is True: try: vpnos, strategy = self.search_vpnos(ident) if not vpnos: vpnos = [] with get_session() as session: controller.set_strategy(vendor=self._name, ident=ident, strategy=strategy, session=session) controller.set_amap_vpnos(vendor=self._name, ident=ident, vpnos=vpnos, session=session) except (NotImplementedError, URLError, HTTPError): pass return self._map.get_partnos(ident) def search_vpnos(self, ident): raise NotImplementedError def get_vpart(self, vpartno, ident=None, max_age=VENDOR_DEFAULT_MAXAGE): idx = (vpartno, ident) if idx not in self._partcache.keys(): part = self._partclass(vpartno, ident=ident, vendor=self, max_age=max_age) self._partcache[idx] = part return self._partcache[idx] @staticmethod def _get_candidate_tcost(candidate, oqty): ubprice, nbprice = candidate.get_price(oqty) effprice = candidate.get_effective_price(ubprice) return effprice.extended_price(oqty).native_value def _get_candidate_isinfo(self, candidate, oqty): tcost = self._get_candidate_tcost(candidate, oqty) ubprice, nbprice = candidate.get_price(oqty) effprice = candidate.get_effective_price(ubprice) urationale = None olduprice = None if nbprice is not None: nubprice, nnbprice = candidate.get_price(nbprice.moq) neffprice = candidate.get_effective_price(nubprice) ntcost = neffprice.extended_price(nbprice.moq).native_value # bump_excess_qty = nubprice.moq - rqty if ntcost < tcost * 1.4: urationale = "TC Increase < 40%" oqty = nbprice.moq olduprice = ubprice ubprice = nubprice nbprice = nnbprice effprice = neffprice elif nubprice.unit_price.native_value < \ ubprice.unit_price.native_value * 0.2: # TODO This used to be 40%. Evelta threw this out the window. # Figure out a better way to handle sparse price breaks. urationale = "UP Decrease > 80%" olduprice = ubprice oqty = nbprice.moq ubprice = nubprice nbprice = nnbprice effprice = neffprice return SourcingInfo(self, candidate, oqty, nbprice, ubprice, effprice, urationale, olduprice) def get_optimal_pricing(self, ident, rqty, get_all=False, relax_moq=False): candidate_names = self.get_vpnos(ident) candidates = [] for name in candidate_names: try: candidates.append(self.get_vpart(name, ident=ident)) except VendorPartRetrievalError: continue if not relax_moq: candidates = [x for x in candidates if x.abs_moq <= rqty] candidates = [ x for x in candidates if x.vqtyavail is None or x.vqtyavail > rqty or x.vqtyavail == -2 ] oqty = rqty if len(candidates) == 0: if not get_all: return SourcingInfo(self, None, None, None, None, None, None, None) else: return [] if get_all: return [self._get_candidate_isinfo(x, oqty) for x in candidates] tcost = None idx = 0 while not tcost: selcandidate = candidates[idx] try: tcost = selcandidate.get_effective_price( selcandidate.get_price(rqty)[0]).extended_price( rqty).native_value except VendorPartPricingError: logger.error("Unable to price part {0}" "".format(selcandidate.vpno)) idx += 1 except IndexError: if not get_all: return SourcingInfo(self, None, None, None, None, None, None, None) else: return [] selcandidate = None for candidate in candidates: try: ntcost = self._get_candidate_tcost(candidate, oqty) except VendorPartPricingError: continue if ntcost <= tcost: tcost = ntcost selcandidate = candidate if selcandidate.vqtyavail == -2: logger.warning("Vendor available quantity could not be confirmed. " "Verify manually : " + self.name + " " + selcandidate.vpno + os.linesep + os.linesep + os.linesep) return self._get_candidate_isinfo(selcandidate, oqty) def add_order_additional_cost_component(self, desc, percent): self._orderadditionalcosts.append((desc, percent)) def get_effective_price(self, price): warnings.warn( "Deprecated access of VendorBase.get_effective_price. " "Use VendorPartBase.get_effective_price instead.", DeprecationWarning) effective_unitp = price.unit_price.source_value for additional_cost in self._orderadditionalcosts: effective_unitp += price.unit_price.source_value * \ float(additional_cost[1]) / 100 return VendorPrice(price.moq, effective_unitp, self.currency, price.oqmultiple) def get_additional_costs(self, price): rval = [] for desc, percent in self._orderadditionalcosts: rval.append((desc, price.source_value * percent / 100)) return rval @property def order_baseprice(self): t = 0 for price in self._orderbasecosts: t += price[1].native_value return currency.CurrencyValue(t, currency.native_currency_defn) def add_order_baseprice_component(self, desc, value): if isinstance(value, currency.CurrencyValue): self._orderbasecosts.append((desc, value)) else: self._orderbasecosts.append( (desc, currency.CurrencyValue(value, self.currency))) def add_to_order(self, line, orderref=None): if self._order is None: self._order = VendorOrder(self, orderref) logger.info("Adding to " + self._name + " order : " + line[0] + " : " + str(line[3])) self._order.add(line) def _dump_open_order(self, path): orderfile = os.path.join(path, self._name + '-order.csv') with open(orderfile, 'w') as orderf: w = csv.writer(orderf) w.writerow([ self._dname + " Order", None, None, None, None, time.strftime("%c") ]) w.writerow([ "Ident", "Vendor Part No", "Quantity", "Unit Price (" + self._currency.symbol + ")", "Extended Price (" + self._currency.symbol + ")", "Effective Price (" + currency.native_currency_defn.symbol + ")" ]) for line in self._order.lines: w.writerow([ line[0], line[2], line[3], line[5].unit_price.source_value, line[5].extended_price(line[3]).source_value, line[6].extended_price(line[3]).native_value ]) for basecost in self._orderbasecosts: w.writerow([ None, basecost[0], None, None, basecost[1].source_value, basecost[1].native_value ]) def _generate_purchase_order(self, path): stagebase = {} return stagebase def finalize_order(self, path): if self._order is None or len(self._order) == 0: logger.debug("Nothing in the order, " "not generating order file : " + self._name) return logger.info("Writing " + self._dname + " order to Folder : " + path) self._dump_open_order(path) self._generate_purchase_order(path) self._order = None # Filter Functions from Digikey and TI Vendor implementations @staticmethod def _filter_results_unfiltered(parts): """ Given a list of :class:`.vendors.SearchPart` instances, returns a :class:`.vendors.SearchResult` instance, whose ``parts`` attribute includes a list of part numbers. If any of the part numbers are not listed as Non-Stocked, only the Stocked results are returned along with the strategy ``UNFILTERED``. If all of the part numbers are listed as Non-Stocked, then all the part numbers are returned with the strategy ``UNFILTERED_ALLOW_NS``. :type parts: list of :class:`.vendors.SearchPart` :rtype: :class:`.vendors.SearchResult` """ pnos = [] strategy = 'UNFILTERED' for part in parts: if not part.ns: pnos.append(part.pno) if len(pnos) == 0: strategy += '_ALLOW_NS' for part in parts: pnos.append(part.pno) return SearchResult(True, pnos, strategy) @staticmethod def _find_exact_match_package(parts, value): """ Given a list of :class:`.vendors.SearchPart` instances and a known value, returns a :class:`.vendors.SearchResult` instance, whose ``parts`` attribute includes only the package of the part whose manufacturer part number (``mfgpno``) exactly matches the given value, if such an exact match can be found. The :class:`.vendors.SearchResult` returned on success has it's strategy attribute set to ``EXACT_MATCH_FFP``. :type parts: list of :class:`.vendors.SearchPart` :type value: str :rtype: :class:`.vendors.SearchResult` """ for part in parts: if part.mfgpno == value: return SearchResult(True, part.package, 'EXACT_MATCH_FFP') return SearchResult(False, None, None) @staticmethod def _find_consensus_package(parts): """ Given a list of :class:`.vendors.SearchPart` instances, returns a :class:`.vendors.SearchResult` instance, whose 'parts' attribute includes only the consensus package of all the parts in the provided list, if such a consensus can be reached. The :class:`.vendors.SearchResult` returned on success has it's strategy attribute set to ``CONSENSUS_FP_MATCH``. :type parts: list of :class:`.vendors.SearchPart` :rtype: :class:`.vendors.SearchResult` """ cpackage = parts[0].package for part in parts: if part.package != cpackage: cpackage = None if cpackage is not None: return SearchResult(True, cpackage, 'CONSENSUS_FP_MATCH') return SearchResult(False, None, None) @staticmethod def _filter_results_bycpackage(parts, cpackage, strategy): """ Given a list of :class:`.vendors.SearchPart` instances, and a consensus package string, returns a :class:`.vendors.SearchResult` instance, whose ``parts`` attribute includes the part numbers of all the parts in the provided list whose package attribute matches the consensus package. When used in the correct context, this function uses cpackage instead of the original footprint. cpackage is itself extracted from the result table, and therefore greatly decreases (though not eliminates) the odds of false negatives. A strategy is accepted as the third argument to this function, and is returned within the :class:`.vendors.SearchResult`, with modification to append ``_ALLOW_NS`` if necessary. :type parts: list of :class:`.vendors.SearchPart` :param cpackage: A consensus or exact match package. :type cpackage: str :type strategy: str :rtype: :class:`.vendors.SearchResult` """ pnos = [] for part in parts: if part.package == cpackage: if not part.ns: pnos.append(part.pno) if len(pnos) == 0: strategy += '_ALLOW_NS' for part in parts: if part.package == cpackage: pnos.append(part.pno) return SearchResult(True, pnos, strategy) @staticmethod def _filter_results_byfootprint(parts, footprint): """ Given a list of :class:`.vendors.SearchPart` instances and the target footprint, returns a :class:`.vendors.SearchResult` instance, whose ``parts`` attribute includes part numbers for all parts in the provided list whose package attribute contains the provided footprint. This is a last ditch effort. Due to the diversity in package nomenclature, this has a very low likelihood of success without an exceptionally well curated symbol library. The :class:`.vendors.SearchResult` returned on success has it's strategy attribute set to ``NAIVE_FP_MATCH`` or ``NAIVE_FP_MATCH_ALLOW_NS``. :type parts: list of :class:`.vendors.SearchPart` :type footprint: str :rtype: :class:`.vendors.SearchResult` """ pnos = [] strategy = 'NAIVE_FP_MATCH' if isinstance(footprint, list): for part in parts: if part.package in footprint: if not part.ns: pnos.append(part.pno) if len(pnos) == 0: strategy += '_ALLOW_NS' for part in parts: if part.package in footprint: pnos.append(part.pno) else: for part in parts: if footprint in part.package: if not part.ns: pnos.append(part.pno) if len(pnos) == 0: strategy += ' ALLOW NS' for part in parts: if footprint in part.package: pnos.append(part.pno) return SearchResult(True, pnos, strategy) def _filter_results(self, parts, value, footprint): """ Given a list of :class:`.vendors.SearchPart`, and the target value and footprint, returns a :class:`.vendors.SearchResult` instance, whose ``parts`` attribute includes part numbers for all parts in the provided list matching the required value and footprint. The filtering is done in the following sequence : - If the first part in ``parts`` contains no package information or the provided target ``footprint`` is ``None``, then filtering is done solely using :func:`_filter_results_unfiltered`. - If an exact match package (:func:`_find_exact_match_package`) or consensus package (:func:`_find_consensus_package`) is found, it is used as ``cpackage`` and filtered using :func:`_filter_results_bycpackage`. - If none of the previous conditions were met, :func:`_filter_results_byfootprint` is used. The returned :class:`.vendors.SearchResult` instance always has its ``status`` attribute set to ``True``, and the ``strategy`` attribute is passed along unmodified from the inner filter function. """ if parts[0].package is None or footprint is None: # No package, so no basis to filter sr = self._filter_results_unfiltered(parts) return SearchResult(True, sr.parts, sr.strategy) # Find Exact Match Package sr = self._find_exact_match_package(parts, value) cpackage = sr.parts strategy = sr.strategy if sr.success is False: # Did not find an exact match package. # Check for consensus package instead. sr = self._find_consensus_package(parts) cpackage = sr.parts strategy = sr.strategy if sr.success is False: # No exact match, no consensus on package sr = self._filter_results_byfootprint(parts, footprint) return SearchResult(True, sr.parts, sr.strategy) # cpackage exists sr = self._filter_results_bycpackage(parts, cpackage, strategy) if len(sr.parts) == 0: pnos = None else: pnos = sr.parts return SearchResult(True, pnos, sr.strategy) @staticmethod def _remove_duplicates(parts): """ Given a list of :class:`.vendors.SearchPart` instances, this function removes any duplicates that may have crept in. In this case, the necessary and sufficient condition for two :class:`.vendors.SearchPart` instances to be duplicates of each other is that they have the same vendor part number (``pno``). """ vpnos = set() rparts = [] for part in parts: if part.pno not in vpnos: vpnos.add(part.pno) rparts.append(part) return rparts def _process_results(self, parts, value, footprint): """ Processes a list of :class:`.vendors.SearchPart` instances, using :func:`_remove_duplicates` and :func:`_filter_results`, and returns the :class:`.vendors.SearchResult` instance returned by :func:`_filter_results`. """ parts = self._remove_duplicates(parts) return self._filter_results(parts, value, footprint)
n = 0 while True: n += 1 try: # remove expired items cache = expire_items(cache) # cache new item cache[n] = get_pep(), datetime.utcnow() # randomly access to one element random_key = random.choice(range(1, n + 1)) # if it doens't exist move on if random_key not in cache: continue random_element = cache[random_key] print("Randomly accessed to: {}".format(random_key)) # print("Value for selected element {}".format(cache[random_key])) print("All elements cached: {}".format(sorted(cache.keys()))) print("Total elements cached: {}".format(len(cache))) # wait to appreciate the moment print() sleep(1) except urllib.error.HTTPError: print('Not Found')