class BotWorker(object):
def __init__(self, config, client):
self.config = config
self.sleep = self.config['TIME_TO_SLEEP']
self.lots_client = client(key=self.config['LOTS_API_TOKEN'],
host_url=self.config["API_URL"],
api_version=self.config["API_VERSION"])
self.assets_client = client(resource="assets",
key=self.config['ASSETS_API_TOKEN'],
host_url=self.config["API_URL"],
api_version=self.config["API_VERSION"])
self.db = Database(
"http://{login}:{password}@{host}:{port}/{db}".format(
**self.config['LOTS_DB']),
session=Session(retry_delays=range(10)))
def get_lots(self, view):
logger.info("Getting lots")
try:
return ({
"data": {
'id': lot.id,
'assets': lot.value['assets'],
'status': lot.value['status']
}
} for lot in self.db.view(view)
if lot.value['status'] in ['waiting', 'dissolved'])
except Exception, e:
ecode = e.args[0]
if ecode in RETRYABLE_ERRORS:
logger.error("Error while getting lots: {}".format(e))
class CouchdbOutput(OutputModule):
def __init__(self,
actor_config,
couchdb_url,
payload=None,
selection="data",
parallel_streams=1,
native_events=False,
**kw):
OutputModule.__init__(self, actor_config)
self.pool.createQueue("inbox")
self.registerConsumer(self.consume, "inbox")
self.couchdb = Database(couchdb_url)
def consume(self, event):
if event.isBulk():
bulk_docs = {}
for e in extractBulkItems(event):
doc = e.get(self.kwargs.selection)
doc_id = doc.pop('id', doc.pop('_id', ''))
if doc_id:
doc['_id'] = doc['id'] = doc_id
bulk_docs[doc['id']] = doc
for row in self.couchdb.view('_all_docs',
keys=list(bulk_docs.keys())).rows:
if row.id in bulk_docs:
bulk_docs[row.id]['_rev'] = row['value']['rev']
try:
responce = self.couchdb.update(list(bulk_docs.values()))
for ok, doc_id, rest in responce:
if ok:
self.logging.info("Saved {}".format(doc_id))
else:
self.logging.error(
"Error on save bulk. Type {}, message {}, doc {}".
format(rest, getattr(rest, 'message', ''), doc_id))
except Exception as e:
self.logging.error("Uncaught error {} on save bulk".format(
e, ))
else:
data = event.get(self.kwargs.selection)
doc_id = data.get('id', data.get('_id'))
if doc_id:
data['_id'] = data['id'] = doc_id
if doc_id in self.couchdb:
rev = self.couchdb.get(id).rev
data['_rev'] = rev
self.logging.debug(
"Update revision in data {} to {}".format(id, rev))
self.couchdb.save(data)
class ViewFilter(FlowModule, ExpressionMixin):
def __init__(
self,
actor_config,
couchdb_url,
view,
view_expression,
conditions=[],
selection="data"
):
FlowModule.__init__(self, actor_config)
self.couchdb = Database(couchdb_url)
self.pool.createQueue('inbox')
self.registerConsumer(self.consume, 'inbox')
self.prepare_expressions()
self.view_expression = jq.jq(view_expression)
def consume(self, event):
self.logging.debug("Event from inbox {}".format(event))
data = event.get(self.kwargs.selection)
try:
resp = self.couchdb.view(
self.kwargs.view,
key=self.view_expression.transform(data)
)
view_value = next(iter(resp.rows), False)
if view_value:
for expression in self.conditions:
result = expression['compiled'].transform([view_value, data])
if result:
self.logging.debug("Expression {} matches data {}".format(
expression['expression'], [view_value, data]
))
queue = expression.get('queue', 'outbox')
self.submit(event, queue)
else:
continue
else:
self.submit(event, 'outbox')
except Exception as e:
self.logging.error("Error on view filter {}".format(e))
class CouchdbFilter(FlowModule):
def __init__(
self,
actor_config,
couchdb_url,
view_path,
filter_key,
filter_value,
op,
):
FlowModule.__init__(self, actor_config)
self.pool.createQueue("outbox")
self.view_path = view_path
self.filter_key = jq.jq(filter_key)
self.filter_value = jq.jq(filter_value)
self.op = getattr(operator, op)
self.pool.createQueue('outbox')
self.pool.createQueue('inbox')
self.couchdb = Database(couchdb_url)
self.registerConsumer(self.consume, 'inbox')
def consume(self, event):
self.logging.debug("Event from inbox {}".format(event))
data = event.dump().get('data', {})
resp = self.couchdb.view(self.view_path,
key=self.filter_key.transform(data))
if resp.rows:
old = resp.rows[0].value
new = self.filter_value.transform(data)
if not self.op(old, new):
self.logging.debug(
'Skipped {} by filter. Op: {} old: {} new {}'.format(
data['id'], self.op.__name__, old, new))
del data
return
self.submit(event, 'outbox')
