class TwitterListener(StreamListener):
def __init__(self):
self.cache = LFUCache(maxsize=50)
self.cache2 = LFUCache(maxsize=50)
Timer(interval=60, function=self.print_keys)
Timer(interval=30, function=self.check_cached_words)
def on_data(self, data):
data_lst = json.loads(data)
data_lst = data_lst.get('text', '').split()
if self.cache.currsize == self.cache.maxsize:
for key in list(self.cache.keys()):
if self.cache[key] == 0:
del self.cache[key]
for word in data_lst:
if word in self.cache.keys():
self.cache[word] += 1
else:
self.cache[word] = 1
if self.cache[word] < 0:
del self.cache[word]
return True
def print_keys(self):
"""
print recent words and update the second cache every 60 seconds
"""
print(list(self.cache.items()))
self.cache2.update(self.cache)
return True
def check_cached_words(self):
"""
Decrease score of word by 1 if the score does not change within
60 seconds
"""
for word in list(self.cache.keys()):
if self.cache.get(word) == self.cache2.get(word):
self.cache[word] -= 1
return True
class VendorBase(object):
_vendorlogo = None
_partclass = VendorPartBase
_invoiceclass = VendorInvoice
_type = 'BASE'
_url_base = None
_ident_blacklist = []
def __init__(self,
name,
dname,
pclass,
mappath=None,
currency_code=BASE_CURRENCY,
currency_symbol=BASE_CURRENCY_SYMBOL,
vendorlogo=None,
sname=None,
is_manufacturer=None,
vtype=None):
self._name = name
self._dname = dname
self._sname = sname
self._instance_vendorlogo = vendorlogo
self._is_manufacturer = is_manufacturer
self._currency = currency.CurrencyDefinition(currency_code,
currency_symbol)
self._vtype = vtype
self._pclass = pclass
self._order = None
self._orderbasecosts = []
self._orderadditionalcosts = []
self._partcache = LFUCache(1000)
if mappath is not None:
self._mappath = mappath
else:
self._mappath = self._name + '-' + self._pclass + '.csv'
self._map = VendorMapFileDB(self)
@property
def name(self):
if self._dname is not None:
return self._dname
else:
return self._name
@property
def sname(self):
if self._sname is not None:
return self._sname
else:
return self._name
@property
def cname(self):
return self._name
@property
def pclass(self):
return self._pclass
@property
def type(self):
return self._type
@property
def is_manufacturer(self):
if self._is_manufacturer is None:
return False
return self._is_manufacturer
@property
def mappath(self):
return self._mappath
@property
def map(self):
return self._map
@property
def currency(self):
return self._currency
@currency.setter
def currency(self, currency_def):
"""
:type currency_def: utils.currency.CurrencyDefinition
"""
self._currency = currency_def
@property
def additional_costs(self):
return self._orderadditionalcosts
@property
def logo(self):
if self._instance_vendorlogo is not None:
return self._instance_vendorlogo
return self._vendorlogo
@property
def url_base(self):
try:
return self._url_base
except AttributeError:
return None
def get_idents(self):
for ident in self._map.get_idents():
yield ident
def get_all_vpnos(self):
for ident in self.get_idents():
for vpno in self._map.get_all_partnos(ident):
yield ident, vpno
def get_all_vparts(self, max_age=VENDOR_DEFAULT_MAXAGE):
for ident, vpno in self.get_all_vpnos():
vpart = self.get_vpart(vpartno=vpno, ident=ident, max_age=max_age)
if vpart:
yield vpart
def get_vpnos(self, ident, max_age=VENDOR_DEFAULT_MAXAGE):
if ident in self._ident_blacklist:
return []
acquire = False
mtime = self._map.get_map_time(canonical=ident)
if max_age > 0:
now = time.time()
if not mtime:
acquire = True
elif now - mtime > max_age:
maintenance.update_vpmap(self.cname, ident)
elif max_age == 0 or mtime is None:
acquire = True
if acquire is True:
try:
vpnos, strategy = self.search_vpnos(ident)
if not vpnos:
vpnos = []
with get_session() as session:
controller.set_strategy(vendor=self._name,
ident=ident,
strategy=strategy,
session=session)
controller.set_amap_vpnos(vendor=self._name,
ident=ident,
vpnos=vpnos,
session=session)
except (NotImplementedError, URLError, HTTPError):
pass
return self._map.get_partnos(ident)
def search_vpnos(self, ident):
raise NotImplementedError
def get_vpart(self, vpartno, ident=None, max_age=VENDOR_DEFAULT_MAXAGE):
idx = (vpartno, ident)
if idx not in self._partcache.keys():
part = self._partclass(vpartno,
ident=ident,
vendor=self,
max_age=max_age)
self._partcache[idx] = part
return self._partcache[idx]
@staticmethod
def _get_candidate_tcost(candidate, oqty):
ubprice, nbprice = candidate.get_price(oqty)
effprice = candidate.get_effective_price(ubprice)
return effprice.extended_price(oqty).native_value
def _get_candidate_isinfo(self, candidate, oqty):
tcost = self._get_candidate_tcost(candidate, oqty)
ubprice, nbprice = candidate.get_price(oqty)
effprice = candidate.get_effective_price(ubprice)
urationale = None
olduprice = None
if nbprice is not None:
nubprice, nnbprice = candidate.get_price(nbprice.moq)
neffprice = candidate.get_effective_price(nubprice)
ntcost = neffprice.extended_price(nbprice.moq).native_value
# bump_excess_qty = nubprice.moq - rqty
if ntcost < tcost * 1.4:
urationale = "TC Increase < 40%"
oqty = nbprice.moq
olduprice = ubprice
ubprice = nubprice
nbprice = nnbprice
effprice = neffprice
elif nubprice.unit_price.native_value < \
ubprice.unit_price.native_value * 0.2:
# TODO This used to be 40%. Evelta threw this out the window.
# Figure out a better way to handle sparse price breaks.
urationale = "UP Decrease > 80%"
olduprice = ubprice
oqty = nbprice.moq
ubprice = nubprice
nbprice = nnbprice
effprice = neffprice
return SourcingInfo(self, candidate, oqty, nbprice, ubprice, effprice,
urationale, olduprice)
def get_optimal_pricing(self, ident, rqty, get_all=False, relax_moq=False):
candidate_names = self.get_vpnos(ident)
candidates = []
for name in candidate_names:
try:
candidates.append(self.get_vpart(name, ident=ident))
except VendorPartRetrievalError:
continue
if not relax_moq:
candidates = [x for x in candidates if x.abs_moq <= rqty]
candidates = [
x for x in candidates
if x.vqtyavail is None or x.vqtyavail > rqty or x.vqtyavail == -2
]
oqty = rqty
if len(candidates) == 0:
if not get_all:
return SourcingInfo(self, None, None, None, None, None, None,
None)
else:
return []
if get_all:
return [self._get_candidate_isinfo(x, oqty) for x in candidates]
tcost = None
idx = 0
while not tcost:
selcandidate = candidates[idx]
try:
tcost = selcandidate.get_effective_price(
selcandidate.get_price(rqty)[0]).extended_price(
rqty).native_value
except VendorPartPricingError:
logger.error("Unable to price part {0}"
"".format(selcandidate.vpno))
idx += 1
except IndexError:
if not get_all:
return SourcingInfo(self, None, None, None, None, None,
None, None)
else:
return []
selcandidate = None
for candidate in candidates:
try:
ntcost = self._get_candidate_tcost(candidate, oqty)
except VendorPartPricingError:
continue
if ntcost <= tcost:
tcost = ntcost
selcandidate = candidate
if selcandidate.vqtyavail == -2:
logger.warning("Vendor available quantity could not be confirmed. "
"Verify manually : " + self.name + " " +
selcandidate.vpno + os.linesep + os.linesep +
os.linesep)
return self._get_candidate_isinfo(selcandidate, oqty)
def add_order_additional_cost_component(self, desc, percent):
self._orderadditionalcosts.append((desc, percent))
def get_effective_price(self, price):
warnings.warn(
"Deprecated access of VendorBase.get_effective_price. "
"Use VendorPartBase.get_effective_price instead.",
DeprecationWarning)
effective_unitp = price.unit_price.source_value
for additional_cost in self._orderadditionalcosts:
effective_unitp += price.unit_price.source_value * \
float(additional_cost[1]) / 100
return VendorPrice(price.moq, effective_unitp, self.currency,
price.oqmultiple)
def get_additional_costs(self, price):
rval = []
for desc, percent in self._orderadditionalcosts:
rval.append((desc, price.source_value * percent / 100))
return rval
@property
def order_baseprice(self):
t = 0
for price in self._orderbasecosts:
t += price[1].native_value
return currency.CurrencyValue(t, currency.native_currency_defn)
def add_order_baseprice_component(self, desc, value):
if isinstance(value, currency.CurrencyValue):
self._orderbasecosts.append((desc, value))
else:
self._orderbasecosts.append(
(desc, currency.CurrencyValue(value, self.currency)))
def add_to_order(self, line, orderref=None):
if self._order is None:
self._order = VendorOrder(self, orderref)
logger.info("Adding to " + self._name + " order : " + line[0] + " : " +
str(line[3]))
self._order.add(line)
def _dump_open_order(self, path):
orderfile = os.path.join(path, self._name + '-order.csv')
with open(orderfile, 'w') as orderf:
w = csv.writer(orderf)
w.writerow([
self._dname + " Order", None, None, None, None,
time.strftime("%c")
])
w.writerow([
"Ident", "Vendor Part No", "Quantity",
"Unit Price (" + self._currency.symbol + ")",
"Extended Price (" + self._currency.symbol + ")",
"Effective Price (" + currency.native_currency_defn.symbol +
")"
])
for line in self._order.lines:
w.writerow([
line[0], line[2], line[3], line[5].unit_price.source_value,
line[5].extended_price(line[3]).source_value,
line[6].extended_price(line[3]).native_value
])
for basecost in self._orderbasecosts:
w.writerow([
None, basecost[0], None, None, basecost[1].source_value,
basecost[1].native_value
])
def _generate_purchase_order(self, path):
stagebase = {}
return stagebase
def finalize_order(self, path):
if self._order is None or len(self._order) == 0:
logger.debug("Nothing in the order, "
"not generating order file : " + self._name)
return
logger.info("Writing " + self._dname + " order to Folder : " + path)
self._dump_open_order(path)
self._generate_purchase_order(path)
self._order = None
# Filter Functions from Digikey and TI Vendor implementations
@staticmethod
def _filter_results_unfiltered(parts):
"""
Given a list of :class:`.vendors.SearchPart` instances, returns a
:class:`.vendors.SearchResult` instance, whose ``parts``
attribute includes a list of part numbers.
If any of the part numbers are not listed as Non-Stocked, only the
Stocked results are returned along with the strategy ``UNFILTERED``.
If all of the part numbers are listed as Non-Stocked, then all the
part numbers are returned with the strategy ``UNFILTERED_ALLOW_NS``.
:type parts: list of :class:`.vendors.SearchPart`
:rtype: :class:`.vendors.SearchResult`
"""
pnos = []
strategy = 'UNFILTERED'
for part in parts:
if not part.ns:
pnos.append(part.pno)
if len(pnos) == 0:
strategy += '_ALLOW_NS'
for part in parts:
pnos.append(part.pno)
return SearchResult(True, pnos, strategy)
@staticmethod
def _find_exact_match_package(parts, value):
"""
Given a list of :class:`.vendors.SearchPart` instances and a known
value, returns a :class:`.vendors.SearchResult` instance, whose
``parts`` attribute includes only the package of the part whose
manufacturer part number (``mfgpno``) exactly matches the given value,
if such an exact match can be found.
The :class:`.vendors.SearchResult` returned on success has it's
strategy attribute set to ``EXACT_MATCH_FFP``.
:type parts: list of :class:`.vendors.SearchPart`
:type value: str
:rtype: :class:`.vendors.SearchResult`
"""
for part in parts:
if part.mfgpno == value:
return SearchResult(True, part.package, 'EXACT_MATCH_FFP')
return SearchResult(False, None, None)
@staticmethod
def _find_consensus_package(parts):
"""
Given a list of :class:`.vendors.SearchPart` instances, returns a
:class:`.vendors.SearchResult` instance, whose 'parts' attribute
includes only the consensus package of all the parts in the provided
list, if such a consensus can be reached.
The :class:`.vendors.SearchResult` returned on success has it's
strategy attribute set to ``CONSENSUS_FP_MATCH``.
:type parts: list of :class:`.vendors.SearchPart`
:rtype: :class:`.vendors.SearchResult`
"""
cpackage = parts[0].package
for part in parts:
if part.package != cpackage:
cpackage = None
if cpackage is not None:
return SearchResult(True, cpackage, 'CONSENSUS_FP_MATCH')
return SearchResult(False, None, None)
@staticmethod
def _filter_results_bycpackage(parts, cpackage, strategy):
"""
Given a list of :class:`.vendors.SearchPart` instances, and a consensus
package string, returns a :class:`.vendors.SearchResult` instance, whose
``parts`` attribute includes the part numbers of all the parts in the
provided list whose package attribute matches the consensus package.
When used in the correct context, this function uses cpackage instead
of the original footprint. cpackage is itself extracted from the
result table, and therefore greatly decreases (though not eliminates)
the odds of false negatives.
A strategy is accepted as the third argument to this function, and is
returned within the :class:`.vendors.SearchResult`, with modification
to append ``_ALLOW_NS`` if necessary.
:type parts: list of :class:`.vendors.SearchPart`
:param cpackage: A consensus or exact match package.
:type cpackage: str
:type strategy: str
:rtype: :class:`.vendors.SearchResult`
"""
pnos = []
for part in parts:
if part.package == cpackage:
if not part.ns:
pnos.append(part.pno)
if len(pnos) == 0:
strategy += '_ALLOW_NS'
for part in parts:
if part.package == cpackage:
pnos.append(part.pno)
return SearchResult(True, pnos, strategy)
@staticmethod
def _filter_results_byfootprint(parts, footprint):
"""
Given a list of :class:`.vendors.SearchPart` instances and the target
footprint, returns a :class:`.vendors.SearchResult` instance, whose
``parts`` attribute includes part numbers for all parts in the
provided list whose package attribute contains the provided footprint.
This is a last ditch effort. Due to the diversity in package
nomenclature, this has a very low likelihood of success
without an exceptionally well curated symbol library. The
:class:`.vendors.SearchResult` returned on success has it's
strategy attribute set to ``NAIVE_FP_MATCH`` or
``NAIVE_FP_MATCH_ALLOW_NS``.
:type parts: list of :class:`.vendors.SearchPart`
:type footprint: str
:rtype: :class:`.vendors.SearchResult`
"""
pnos = []
strategy = 'NAIVE_FP_MATCH'
if isinstance(footprint, list):
for part in parts:
if part.package in footprint:
if not part.ns:
pnos.append(part.pno)
if len(pnos) == 0:
strategy += '_ALLOW_NS'
for part in parts:
if part.package in footprint:
pnos.append(part.pno)
else:
for part in parts:
if footprint in part.package:
if not part.ns:
pnos.append(part.pno)
if len(pnos) == 0:
strategy += ' ALLOW NS'
for part in parts:
if footprint in part.package:
pnos.append(part.pno)
return SearchResult(True, pnos, strategy)
def _filter_results(self, parts, value, footprint):
"""
Given a list of :class:`.vendors.SearchPart`, and the target value and
footprint, returns a :class:`.vendors.SearchResult` instance, whose
``parts`` attribute includes part numbers for all parts in the
provided list matching the required value and footprint.
The filtering is done in the following sequence :
- If the first part in ``parts`` contains no package information or
the provided target ``footprint`` is ``None``, then filtering is
done solely using :func:`_filter_results_unfiltered`.
- If an exact match package (:func:`_find_exact_match_package`) or
consensus package (:func:`_find_consensus_package`) is found, it
is used as ``cpackage`` and filtered using
:func:`_filter_results_bycpackage`.
- If none of the previous conditions were met,
:func:`_filter_results_byfootprint` is used.
The returned :class:`.vendors.SearchResult` instance always has its
``status`` attribute set to ``True``, and the ``strategy`` attribute
is passed along unmodified from the inner filter function.
"""
if parts[0].package is None or footprint is None:
# No package, so no basis to filter
sr = self._filter_results_unfiltered(parts)
return SearchResult(True, sr.parts, sr.strategy)
# Find Exact Match Package
sr = self._find_exact_match_package(parts, value)
cpackage = sr.parts
strategy = sr.strategy
if sr.success is False:
# Did not find an exact match package.
# Check for consensus package instead.
sr = self._find_consensus_package(parts)
cpackage = sr.parts
strategy = sr.strategy
if sr.success is False:
# No exact match, no consensus on package
sr = self._filter_results_byfootprint(parts, footprint)
return SearchResult(True, sr.parts, sr.strategy)
# cpackage exists
sr = self._filter_results_bycpackage(parts, cpackage, strategy)
if len(sr.parts) == 0:
pnos = None
else:
pnos = sr.parts
return SearchResult(True, pnos, sr.strategy)
@staticmethod
def _remove_duplicates(parts):
"""
Given a list of :class:`.vendors.SearchPart` instances, this function
removes any duplicates that may have crept in. In this case, the
necessary and sufficient condition for two :class:`.vendors.SearchPart`
instances to be duplicates of each other is that they have the same
vendor part number (``pno``).
"""
vpnos = set()
rparts = []
for part in parts:
if part.pno not in vpnos:
vpnos.add(part.pno)
rparts.append(part)
return rparts
def _process_results(self, parts, value, footprint):
"""
Processes a list of :class:`.vendors.SearchPart` instances, using
:func:`_remove_duplicates` and :func:`_filter_results`, and
returns the :class:`.vendors.SearchResult` instance returned
by :func:`_filter_results`.
"""
parts = self._remove_duplicates(parts)
return self._filter_results(parts, value, footprint)
n = 0
while True:
n += 1
try:
# remove expired items
cache = expire_items(cache)
# cache new item
cache[n] = get_pep(), datetime.utcnow()
# randomly access to one element
random_key = random.choice(range(1, n + 1))
# if it doens't exist move on
if random_key not in cache:
continue
random_element = cache[random_key]
print("Randomly accessed to: {}".format(random_key))
# print("Value for selected element {}".format(cache[random_key]))
print("All elements cached: {}".format(sorted(cache.keys())))
print("Total elements cached: {}".format(len(cache)))
# wait to appreciate the moment
print()
sleep(1)
except urllib.error.HTTPError:
print('Not Found')
