def paginated(self):
self.logger.info('[PAGINATED] Beginning paginated products traversal...')
res = self.pub.call({'method': 'get', 'resource': 'category', 'where': {}})
try:
categories = unserialize(res['result'])
except Exception as e:
self.logger.critical(str(e))
self.logger.info('[PAGINATED] Received {} categories'.format(len(categories)))
categories = categories[45:]
for category in categories:
time.sleep(self.delay * self.threads)
print(self.delay * self.threads)
paginated = Walcart.paginated(category.id)
for i in range(category.pages_parsed):
if 'nextPage' in paginated:
time.sleep(self.delay * self.threads)
paginated = Walcart.get_json(paginated['nextPage'])
else:
category.pages_parsed = -1
if 'items' in paginated:
products = []
products_data = paginated['items']
for product_data in products_data:
product = self.make_product(product_data)
products.append(product)
self.logger.info('\t[PAGINATED] Saving {} products...'.format(len(products)))
res = self.pub.call({'method': 'save', 'resource': serialize(products)})
if 'success' not in res:
self.logger.critical('[PAGINATED] '.format(res['message']))
category.pages_parsed += 1
self.pub.call({'method': 'save', 'resource': serialize(category)})
def train(dataset, estimator, feature_keys, label_key, **kwargs):
'''Implements the "train" command.'''
data = deserialize(dataset)
features, labels = reformat(data, feature_keys, label_key)
params = {
key: kwargs.get(key, default)
for (key, default) in CLF_DEFAULTS.items()
}
LOGGER.info('estimator params => %s', json.dumps(params))
clf = SGDClassifier(**params)
clf.fit(features, labels)
serialize(estimator, clf)
def track_product(self, vald):
username = vald['username']
product_id = vald['product_id']
try:
res = self.pub.call({
'method': 'get',
'resource': 'user',
'where': {
'username': username
}
})
user = unserialize(res['result'])
res = self.pub.call({
'method': 'get',
'resource': 'product',
'where': {
'id': product_id
}
})
product = unserialize(res['result'])
except Exception as e:
return {'message': str(e)}
tracked = Tracked(wishlist=vald['wishlist'])
tracked.product = product
user.products.append(tracked)
res = self.pub.call({'method': 'save', 'resource': serialize(user)})
if not res.get('success'):
self.logger.info(res['message'])
return res
def search_query(self, query):
result = Walcart.search(query.replace(' ', '%20'))
if 'message' in result:
return {'success': False, 'message': result['message']}
elif 'items' in result:
products = [
self.burro.make_product(product_data)
for product_data in result['items']
]
res = self.pub.call({
'method': 'save',
'resource': serialize(products)
})
if 'success' in res:
res = self.pub.call({
'method': 'get',
'resource': 'product',
'where': {
'id': [p.id for p in products]
}
})
data = unserialize(res['result'])
product = [p.to_dict() for p in data
] if type(data) == list else data.to_dict()
return {'success': True, 'product': product}
else:
return {'success': False}
def search(self, product_id):
res = self.pub.call({
'method': 'get',
'resource': 'product',
'where': {
'id': product_id
}
})
product = unserialize(res['result'])
if not product:
product_data = Walcart.product(product_id)
if product_data.get('message'):
return product_data
product = self.burro.make_product(product_data)
res = self.pub.call({
'method': 'save',
'resource': serialize(product)
})
recommended_products = self.get_recommended_products(product.id)
res = self.pub.call({
'method': 'save',
'resource': serialize(recommended_products)
})
res_recommended_products = []
res = self.pub.call({
'method': 'get',
'resource': 'product',
'where': {
'id': [p.id for p in recommended_products]
}
})
if unserialize(res['result']):
res_recommended_products = [
p.to_dict() for p in unserialize(res['result'])
]
res = self.pub.call({
'method': 'get',
'resource': 'product',
'where': {
'id': product.id
}
})
product = unserialize(res['result'])
return {
'product': product.to_dict(),
'recommended': res_recommended_products
}
def register(self, user):
newUser = User(**user)
res = self.pub.call({'method': 'save', 'resource': serialize(newUser)})
if 'success' in res:
return {
'success': True,
'message': 'User has been registered successfully!'
}
else:
self.logger.info(res['message'])
return {'success': False, 'message': 'This user already exists.'}
def update_price(self, product):
self.logger.info('[UPDATE_PRICES] Updating price for {}'.format(product.name))
product_data = Walcart.product(product.id)
current_price = product_data.get('salePrice')
stock = product_data.get('stock')
old_price = None if not product.prices else product.prices[-1].price
self.logger.info('\t[UPDATE_PRICES] Old Price: {}\tCurrent Price: {}'.format(old_price, current_price))
if old_price != current_price:
self.logger.info('\t\t[UPDATE_PRICES] Price has been updated successfully.')
product.prices.append(Price(price=current_price, stock=stock))
res = self.pub.call({'method': 'save', 'resource': serialize(product)})
if not res.get('success'):
self.logger.critical('[UPDATE_PRICES] {}'.format(res['message']))
else:
self.logger.info('\t\t[UPDATE_PRICES] Price has not changed. No update necessary.')
return (product, False)
def get(self, resource, where_clause):
response = []
if where_clause and type(list(where_clause.values())[0]) is list:
col = list(where_clause.keys())[0]
values = list(where_clause.values())[0]
result = self.session.query(resource).filter(
getattr(resource, col).in_(values)).all()
else:
result = self.session.query(resource).filter_by(**where_clause)
#self.session.expunge_all()
for item in result:
response.append(item)
self.logger.info(response)
if len(response) == 1:
response = response[0]
self.logger.info(response)
return {'result': serialize(response)}
def serialize(self, n = None, exclude=[]):
if n is None:
n = self._root
return serialize(n, exclude=exclude).strip()
for record in map(json.loads, records):
for (key, encoder) in encoders:
indices[key].append(encoder(record[key]))
return {
key: encoder.transform(indices[key])
for (key, encoder) in encoders
}
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('config', help='JSON processing config')
parser.add_argument('vocabulary',
help='JSON file of token-frequency mappings')
parser.add_argument('infile', help='raw JSON records, one per line')
parser.add_argument('outfile', help='pickled dict of numpy arrays')
args = parser.parse_args()
settings.configure_logging()
config = load_json(args.config)
vocabulary = load_json(args.vocabulary)
encoders = list(
filter(None, [get_encoder(item, vocabulary) for item in config]))
with open(args.infile) as fh:
LOGGER.info('reading %s', args.infile)
data = encode(encoders, fh)
serialize(args.outfile, data)
def __init__(self, data):
self.__data = data
self.value = serialize(self.__data)
def verify_legitimacy(self):
if self.value == serialize(self.__data):
return True
return False
def add_blockchain_unit(self, record):
unit = BlockChainUnit(record)
unit.add_previous_hash(serialize(self.__blockChainUnits[-1]))
self.__blockChainUnits.append(unit)
self.__populate()
import common
import hashimoto
'''sz = cache.get_size(0)
seed = bytes(64)
c = cache.make(sz, seed)
common.serialize(c, 'cache')'''
'''block_number = 0
cache_ = common.deserialize('cache')
header = b'represents the SHA3-256 hash of the RLP representation of a truncated block header, that is, of a header excluding the fields mixHash and nonce'
nonce = b'is the eight bytes of a 64 bit unsigned integer in big-endian order'
res = hashimoto.hashimoto_light(block_number, cache_, header, nonce)
print(common.bytes_to_list(res))'''
