class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version, key=''
)
self.couch_url = urljoin(
self.config_get('couch_url'),
self.config_get('public_db')
)
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
def config_get(self, name):
return self.config.get('main').get(name)
def get_teders_list(self):
for item in get_tenders(host=self.api_host, version=self.api_version,
key='', extra_params={'mode': '_all_'},
retrievers_params=self.retrievers_params):
yield (item["id"], item["dateModified"])
def save_tender_in_db(self, tender_id, date_modified):
tender_doc = self.db.get(tender_id)
if tender_doc:
if tender_doc['dateModified'] == date_modified:
return
tender = self.client.get_tender(tender_id).get('data')
if tender:
tender['_id'] = tender_id
tender['doc_type'] = 'Tender'
if tender_doc:
tender['_rev'] = tender_doc['_rev']
logger.info('Update tender {} '.format(tender_id))
else:
logger.info('Save tender {} '.format(tender_id))
try:
self.db.save(tender)
except Exception as e:
logger.info('Saving tender {} fail with error {}'.format(tender_id, e.message),
extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
else:
logger.info('Tender {} not found'.format(tender_id))
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
for tender_id, date_modified in self.get_teders_list():
self.save_tender_in_db(tender_id, date_modified)
class CouchdbPuller(OutputModule):
def __init__(self,
actor_config,
couchdb_url,
payload=None,
selection="data",
bulk=100,
parallel_streams=1,
native_events=False,
**kw):
OutputModule.__init__(self, actor_config)
self.pool.createQueue("inbox")
self.registerConsumer(self.consume, "inbox")
self.couchdb = Database(couchdb_url)
self._bulk_size = bulk
self._bulk = {}
def __save(self):
self.logging.debug("Saving: {} docs".format(len(self._bulk)))
try:
responce = self.couchdb.update(
[doc for doc in self._bulk.values()])
for ok, doc_id, rest in responce:
if ok:
self.logging.info("Saved {}".format(doc_id))
else:
self.logging.error(
"Error on save bulk. Type {}, message {}, doc {}".
format(rest, getattr(rest, 'message', ''), doc_id))
except Exception as e:
self.logging.error("Uncaught error {} on save bulk".format(e, ))
finally:
self._bulk = {}
self.logging.debug("Cleaned bulk")
return False
def consume(self, event):
data = self.encode(self.getDataToSubmit(event))
if not isinstance(data, dict):
try:
data = loads(data)
except ValueError:
self.logging.error(
"Unable to parse data from raw string. Skipping")
id = data.get('id', data.get('_id'))
if id:
data['_id'] = data['id'] = id
if id and (id in self.couchdb):
rev = self.couchdb.get(id).rev
data['_rev'] = rev
self.logging.debug("Update revision in data {} to {}".format(
id, rev))
self._bulk[data.get('_id', uuid4().hex)] = data
self.logging.debug("Added {} to bulk queue. Size {}".format(
id, len(self._bulk)))
if len(self._bulk) >= self._bulk_size:
g = spawn(self.__save)
g.join()
class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
self.couch_url = urljoin(self.config_get('couch_url'),
self.config_get('public_db'))
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
def config_get(self, name):
return self.config.get('main').get(name)
def get_teders_list(self):
for item in get_tenders(host=self.api_host,
version=self.api_version,
key='',
extra_params={'mode': '_all_'}):
yield (item["id"], item["dateModified"])
def save_tender_in_db(self, tender_id, date_modified):
tender_doc = self.db.get(tender_id)
if tender_doc:
if tender_doc['dateModified'] == date_modified:
return
tender = self.client.get_tender(tender_id).get('data')
if tender:
tender['_id'] = tender_id
tender['doc_type'] = 'Tender'
if tender_doc:
tender['_rev'] = tender_doc['_rev']
logger.info('Update tender {} '.format(tender_id))
else:
logger.info('Save tender {} '.format(tender_id))
try:
self.db.save(tender)
except Exception as e:
logger.info(
'Saving tender {} fail with error {}'.format(
tender_id, e.message),
extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
else:
logger.info('Tender {} not found'.format(tender_id))
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
for tender_id, date_modified in self.get_teders_list():
self.save_tender_in_db(tender_id, date_modified)
def authenticate(request,id):
result = ""
docs = Database("https://*****:*****@wazza.cloudant.com/api_keys")
sep = "_"
list = string.split(id, sep)
authenticate = docs.get(str(list[1]))
if authenticate == None:
result = result + '<?xml version="1.0" Name="WS Project 2C API"?> \n <error value = "Invalid API Key"></error>'
return HttpResponse(result)
else:
return HttpResponse(lookup(str(list[0])))
class Auction(object):
"""docstring for Auction"""
def __init__(self, auction_doc_id,
worker_defaults={},
auction_data={}):
super(Auction, self).__init__()
self.auction_doc_id = auction_doc_id
self.tender_url = urljoin(
worker_defaults["TENDERS_API_URL"],
'/api/{0}/tenders/{1}'.format(
worker_defaults["TENDERS_API_VERSION"], auction_doc_id
)
)
if auction_data:
self.debug = True
logger.setLevel(logging.DEBUG)
self._auction_data = auction_data
else:
self.debug = False
self._end_auction_event = Event()
self.bids_actions = BoundedSemaphore()
self.worker_defaults = worker_defaults
self._bids_data = {}
self.db = Database(str(self.worker_defaults["COUCH_DATABASE"]),
session=Session(retry_delays=range(10)))
self.retries = 10
def generate_request_id(self):
self.request_id = generate_request_id()
def get_auction_document(self):
retries = self.retries
while retries:
try:
self.auction_document = self.db.get(self.auction_doc_id)
if self.auction_document:
logger.info("Get auction document {0[_id]} with rev {0[_rev]}".format(self.auction_document),
extra={"JOURNAL_REQUEST_ID": self.request_id,
"MESSAGE_ID": AUCTION_WORKER_DB})
return
except HTTPError, e:
logger.error("Error while get document: {}".format(e),
extra={'MESSAGE_ID': AUCTION_WORKER_DB})
except Exception, e:
ecode = e.args[0]
if ecode in RETRYABLE_ERRORS:
logger.error("Error while save document: {}".format(e),
extra={'MESSAGE_ID': AUCTION_WORKER_DB})
else:
logger.critical("Unhandled error: {}".format(e),
extra={'MESSAGE_ID': AUCTION_WORKER_DB})
retries -= 1
def authenticate(request, id):
result = ""
docs = Database(
"https://*****:*****@wazza.cloudant.com/api_keys"
)
sep = "_"
list = string.split(id, sep)
authenticate = docs.get(str(list[1]))
if authenticate == None:
result = result + '<?xml version="1.0" Name="WS Project 2C API"?> \n <error value = "Invalid API Key"></error>'
return HttpResponse(result)
else:
return HttpResponse(lookup(str(list[0])))
class CouchdbOutput(OutputModule):
def __init__(self,
actor_config,
couchdb_url,
payload=None,
selection="data",
parallel_streams=1,
native_events=False,
**kw):
OutputModule.__init__(self, actor_config)
self.pool.createQueue("inbox")
self.registerConsumer(self.consume, "inbox")
self.couchdb = Database(couchdb_url)
def consume(self, event):
if event.isBulk():
bulk_docs = {}
for e in extractBulkItems(event):
doc = e.get(self.kwargs.selection)
doc_id = doc.pop('id', doc.pop('_id', ''))
if doc_id:
doc['_id'] = doc['id'] = doc_id
bulk_docs[doc['id']] = doc
for row in self.couchdb.view('_all_docs',
keys=list(bulk_docs.keys())).rows:
if row.id in bulk_docs:
bulk_docs[row.id]['_rev'] = row['value']['rev']
try:
responce = self.couchdb.update(list(bulk_docs.values()))
for ok, doc_id, rest in responce:
if ok:
self.logging.info("Saved {}".format(doc_id))
else:
self.logging.error(
"Error on save bulk. Type {}, message {}, doc {}".
format(rest, getattr(rest, 'message', ''), doc_id))
except Exception as e:
self.logging.error("Uncaught error {} on save bulk".format(
e, ))
else:
data = event.get(self.kwargs.selection)
doc_id = data.get('id', data.get('_id'))
if doc_id:
data['_id'] = data['id'] = doc_id
if doc_id in self.couchdb:
rev = self.couchdb.get(id).rev
data['_rev'] = rev
self.logging.debug(
"Update revision in data {} to {}".format(id, rev))
self.couchdb.save(data)
class CouchDB(object):
"""
This class is responsible for work with CouchDB
Attributes:
_db: Representation of a database to work with on a CouchDB server
:type _db: couchdb.Database
db_request_retries: Number of retries of database requesting in case
error occurred during getting or saving document
:type db_request_retries: int
"""
_db = None
db_request_retries = 10
def __init__(self, config):
self._db = Database(str(config["COUCH_DATABASE"]),
session=Session(retry_delays=range(10)))
def _update_revision(self, auction_document, auction_doc_id):
"""
Check if document in couchdb database has same '_rev' field value
with provided document object. If it differs, change '_rev' field value
for provided auction document with one from couchdb database
:param auction_document: auction document object
:param auction_doc_id: identifier of document in database
:return:
"""
public_document = self.get_auction_document(auction_doc_id)
if public_document and public_document.get(
'_rev') != auction_document['_rev']:
auction_document["_rev"] = public_document["_rev"]
def get_auction_document(self, auction_doc_id):
"""
Retrieve auction document from couchdb database using provided identifier
:param auction_doc_id: identifier of document in couchdb database
:return: auction document object from couchdb database
"""
request_id = generate_request_id()
retries = self.db_request_retries
while retries:
try:
public_document = self._db.get(auction_doc_id)
if public_document:
LOGGER.info(
"Get auction document {0[_id]} with rev {0[_rev]}".
format(public_document),
extra={
"JOURNAL_REQUEST_ID": request_id,
"MESSAGE_ID": AUCTION_WORKER_DB_GET_DOC
})
return public_document
except HTTPError, e:
LOGGER.error(
"Error while get document: {}".format(e),
extra={'MESSAGE_ID': AUCTION_WORKER_DB_GET_DOC_ERROR})
except Exception, e:
errcode = e.args[0]
if errcode in RETRYABLE_ERRORS:
LOGGER.error(
"Error while get document: {}".format(e),
extra={'MESSAGE_ID': AUCTION_WORKER_DB_GET_DOC_ERROR})
else:
LOGGER.critical(
"Unhandled error: {}".format(e),
extra={
'MESSAGE_ID':
AUCTION_WORKER_DB_GET_DOC_UNHANDLED_ERROR
})
retries -= 1
class Auction(object):
"""Auction Worker Class"""
def __init__(self, tender_id,
worker_defaults={},
auction_data={},
lot_id=None,
activate=False):
super(Auction, self).__init__()
self.generate_request_id()
self.tender_id = tender_id
self.lot_id = lot_id
if lot_id:
self.auction_doc_id = tender_id + "_" + lot_id
else:
self.auction_doc_id = tender_id
self.tender_url = urljoin(
worker_defaults["TENDERS_API_URL"],
'/api/{0}/tenders/{1}'.format(
worker_defaults["TENDERS_API_VERSION"], tender_id
)
)
self.activate = activate
if auction_data:
self.debug = True
logger.setLevel(logging.DEBUG)
self._auction_data = auction_data
else:
self.debug = False
self._end_auction_event = Event()
self.bids_actions = BoundedSemaphore()
self.session = RequestsSession()
self.worker_defaults = worker_defaults
if self.worker_defaults.get('with_document_service', False):
self.session_ds = RequestsSession()
self._bids_data = {}
self.db = Database(str(self.worker_defaults["COUCH_DATABASE"]),
session=Session(retry_delays=range(10)))
self.audit = {}
self.retries = 10
self.bidders_count = 0
self.bidders_data = []
self.bidders_features = {}
self.bidders_coeficient = {}
self.features = None
self.mapping = {}
self.rounds_stages = []
def generate_request_id(self):
self.request_id = generate_request_id()
def prepare_public_document(self):
public_document = deepcopy(dict(self.auction_document))
not_last_stage = self.auction_document["current_stage"] not in (len(self.auction_document["stages"]) - 1,
len(self.auction_document["stages"]) - 2,)
if self.features and not_last_stage:
for stage_name in ['initial_bids', 'stages', 'results']:
public_document[stage_name] = map(
filter_amount,
public_document[stage_name]
)
return public_document
def get_auction_document(self, force=False):
retries = self.retries
while retries:
try:
public_document = self.db.get(self.auction_doc_id)
if public_document:
logger.info("Get auction document {0[_id]} with rev {0[_rev]}".format(public_document),
extra={"JOURNAL_REQUEST_ID": self.request_id,
"MESSAGE_ID": AUCTION_WORKER_DB_GET_DOC})
if not hasattr(self, 'auction_document'):
self.auction_document = public_document
if force:
return public_document
elif public_document['_rev'] != self.auction_document['_rev']:
logger.warning("Rev error")
self.auction_document["_rev"] = public_document["_rev"]
logger.debug(json.dumps(self.auction_document, indent=4))
return public_document
except HTTPError, e:
logger.error("Error while get document: {}".format(e),
extra={'MESSAGE_ID': AUCTION_WORKER_DB_GET_DOC_ERROR})
except Exception, e:
ecode = e.args[0]
if ecode in RETRYABLE_ERRORS:
logger.error("Error while get document: {}".format(e),
extra={'MESSAGE_ID': AUCTION_WORKER_DB_GET_DOC_ERROR})
else:
logger.critical("Unhandled error: {}".format(e),
extra={'MESSAGE_ID': AUCTION_WORKER_DB_GET_DOC_UNHANDLED_ERROR})
retries -= 1
class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
try:
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
except MissingSchema:
raise DataBridgeConfigError(
'In config dictionary empty or missing \'tenders_api_server\'')
except ConnectionError as e:
raise e
self.couch_url = urljoin(self.config_get('couch_url'),
self.config_get('public_db'))
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
try:
self.db.info()
except ResourceNotFound:
error_message = "Database with name '" + self.config_get(
'public_db') + "' doesn\'t exist"
raise DataBridgeConfigError(error_message)
except error as e:
if e.errno == errno.ECONNREFUSED:
raise DataBridgeConfigError(
"Connection refused: 'couch_url' is invalid in config dictionary"
)
except AttributeError as e:
raise DataBridgeConfigError(
'\'couch_url\' is missed or empty in config dictionary.')
except KeyError as e:
if e.message == 'db_name':
raise DataBridgeConfigError(
'\'public_db\' name is missed or empty in config dictionary'
)
def config_get(self, name):
try:
return self.config.get('main').get(name)
except AttributeError as e:
raise DataBridgeConfigError(
'In config dictionary missed section \'main\'')
def get_teders_list(self):
for item in get_tenders(host=self.api_host,
version=self.api_version,
key='',
extra_params={'mode': '_all_'},
retrievers_params=self.retrievers_params):
yield (item["id"], item["dateModified"])
def save_tender_in_db(self, tender_id, date_modified):
tender_doc = self.db.get(tender_id)
if tender_doc:
if tender_doc['dateModified'] == date_modified:
return
tender = self.client.get_tender(tender_id).get('data')
if tender:
tender['_id'] = tender_id
tender['doc_type'] = 'Tender'
if tender_doc:
tender['_rev'] = tender_doc['_rev']
logger.info('Update tender {} '.format(tender_id))
else:
logger.info('Save tender {} '.format(tender_id))
try:
self.db.save(tender)
except Exception as e:
logger.info(
'Saving tender {} fail with error {}'.format(
tender_id, e.message),
extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
else:
logger.info('Tender {} not found'.format(tender_id))
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
for tender_id, date_modified in self.get_teders_list():
self.save_tender_in_db(tender_id, date_modified)
