예제 #1
0
class EdgeDataBridge(object):

    """Edge Bridge"""

    def __init__(self, config):
        super(EdgeDataBridge, self).__init__()
        self.config = config
        self.api_host = self.config_get('tenders_api_server')
        self.api_version = self.config_get('tenders_api_version')
        self.retrievers_params = self.config_get('retrievers_params')

        self.client = TendersClient(host_url=self.api_host,
            api_version=self.api_version, key=''
        )

        self.couch_url = urljoin(
            self.config_get('couch_url'),
            self.config_get('public_db')
        )
        self.db = Database(self.couch_url,
                           session=Session(retry_delays=range(10)))

    def config_get(self, name):
        return self.config.get('main').get(name)

    def get_teders_list(self):
        for item in get_tenders(host=self.api_host, version=self.api_version,
                                key='', extra_params={'mode': '_all_'},
                                retrievers_params=self.retrievers_params):
            yield (item["id"], item["dateModified"])

    def save_tender_in_db(self, tender_id, date_modified):
        tender_doc = self.db.get(tender_id)
        if tender_doc:
            if tender_doc['dateModified'] == date_modified:
                return
        tender = self.client.get_tender(tender_id).get('data')
        if tender:
            tender['_id'] = tender_id
            tender['doc_type'] = 'Tender'
            if tender_doc:
                tender['_rev'] = tender_doc['_rev']
                logger.info('Update tender {} '.format(tender_id))
            else:
                logger.info('Save tender {} '.format(tender_id))
            try:
                self.db.save(tender)
            except Exception as e:
                logger.info('Saving tender {} fail with error {}'.format(tender_id, e.message),
                    extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
        else:
            logger.info('Tender {} not found'.format(tender_id))

    def run(self):
        logger.info('Start Edge Bridge',
                    extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
        logger.info('Start data sync...',
                    extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
        for tender_id, date_modified in self.get_teders_list():
            self.save_tender_in_db(tender_id, date_modified)
예제 #2
0
class CouchdbPuller(OutputModule):
    def __init__(self,
                 actor_config,
                 couchdb_url,
                 payload=None,
                 selection="data",
                 bulk=100,
                 parallel_streams=1,
                 native_events=False,
                 **kw):
        OutputModule.__init__(self, actor_config)
        self.pool.createQueue("inbox")
        self.registerConsumer(self.consume, "inbox")
        self.couchdb = Database(couchdb_url)
        self._bulk_size = bulk
        self._bulk = {}

    def __save(self):
        self.logging.debug("Saving: {} docs".format(len(self._bulk)))
        try:
            responce = self.couchdb.update(
                [doc for doc in self._bulk.values()])
            for ok, doc_id, rest in responce:
                if ok:
                    self.logging.info("Saved {}".format(doc_id))
                else:
                    self.logging.error(
                        "Error on save bulk. Type {}, message {}, doc {}".
                        format(rest, getattr(rest, 'message', ''), doc_id))
        except Exception as e:
            self.logging.error("Uncaught error {} on save bulk".format(e, ))
        finally:
            self._bulk = {}
            self.logging.debug("Cleaned bulk")

        return False

    def consume(self, event):
        data = self.encode(self.getDataToSubmit(event))
        if not isinstance(data, dict):
            try:
                data = loads(data)
            except ValueError:
                self.logging.error(
                    "Unable to parse data from raw string. Skipping")
        id = data.get('id', data.get('_id'))
        if id:
            data['_id'] = data['id'] = id
        if id and (id in self.couchdb):
            rev = self.couchdb.get(id).rev
            data['_rev'] = rev
            self.logging.debug("Update revision in data {} to {}".format(
                id, rev))
        self._bulk[data.get('_id', uuid4().hex)] = data
        self.logging.debug("Added {} to bulk queue. Size {}".format(
            id, len(self._bulk)))
        if len(self._bulk) >= self._bulk_size:
            g = spawn(self.__save)
            g.join()
예제 #3
0
class EdgeDataBridge(object):
    """Edge Bridge"""
    def __init__(self, config):
        super(EdgeDataBridge, self).__init__()
        self.config = config
        self.api_host = self.config_get('tenders_api_server')
        self.api_version = self.config_get('tenders_api_version')

        self.client = TendersClient(host_url=self.api_host,
                                    api_version=self.api_version,
                                    key='')

        self.couch_url = urljoin(self.config_get('couch_url'),
                                 self.config_get('public_db'))
        self.db = Database(self.couch_url,
                           session=Session(retry_delays=range(10)))

    def config_get(self, name):
        return self.config.get('main').get(name)

    def get_teders_list(self):
        for item in get_tenders(host=self.api_host,
                                version=self.api_version,
                                key='',
                                extra_params={'mode': '_all_'}):
            yield (item["id"], item["dateModified"])

    def save_tender_in_db(self, tender_id, date_modified):
        tender_doc = self.db.get(tender_id)
        if tender_doc:
            if tender_doc['dateModified'] == date_modified:
                return
        tender = self.client.get_tender(tender_id).get('data')
        if tender:
            tender['_id'] = tender_id
            tender['doc_type'] = 'Tender'
            if tender_doc:
                tender['_rev'] = tender_doc['_rev']
                logger.info('Update tender {} '.format(tender_id))
            else:
                logger.info('Save tender {} '.format(tender_id))
            try:
                self.db.save(tender)
            except Exception as e:
                logger.info(
                    'Saving tender {} fail with error {}'.format(
                        tender_id, e.message),
                    extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
        else:
            logger.info('Tender {} not found'.format(tender_id))

    def run(self):
        logger.info('Start Edge Bridge',
                    extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
        logger.info('Start data sync...',
                    extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
        for tender_id, date_modified in self.get_teders_list():
            self.save_tender_in_db(tender_id, date_modified)
예제 #4
0
파일: views.py 프로젝트: utsavized/urcid
def authenticate(request,id):
    result = ""
    docs = Database("https://*****:*****@wazza.cloudant.com/api_keys")
    sep = "_"
    list = string.split(id, sep)
    authenticate = docs.get(str(list[1]))
    if authenticate == None:
        result = result + '<?xml version="1.0" Name="WS Project 2C API"?> \n <error value = "Invalid API Key"></error>'
        return HttpResponse(result)
    else:
        return HttpResponse(lookup(str(list[0])))
class Auction(object):
    """docstring for Auction"""
    def __init__(self, auction_doc_id,
                 worker_defaults={},
                 auction_data={}):
        super(Auction, self).__init__()
        self.auction_doc_id = auction_doc_id
        self.tender_url = urljoin(
            worker_defaults["TENDERS_API_URL"],
            '/api/{0}/tenders/{1}'.format(
                worker_defaults["TENDERS_API_VERSION"], auction_doc_id
            )
        )
        if auction_data:
            self.debug = True
            logger.setLevel(logging.DEBUG)
            self._auction_data = auction_data
        else:
            self.debug = False
        self._end_auction_event = Event()
        self.bids_actions = BoundedSemaphore()
        self.worker_defaults = worker_defaults
        self._bids_data = {}
        self.db = Database(str(self.worker_defaults["COUCH_DATABASE"]),
                           session=Session(retry_delays=range(10)))
        self.retries = 10

    def generate_request_id(self):
        self.request_id = generate_request_id()

    def get_auction_document(self):
        retries = self.retries
        while retries:
            try:
                self.auction_document = self.db.get(self.auction_doc_id)
                if self.auction_document:
                    logger.info("Get auction document {0[_id]} with rev {0[_rev]}".format(self.auction_document),
                                extra={"JOURNAL_REQUEST_ID": self.request_id,
                                       "MESSAGE_ID": AUCTION_WORKER_DB})
                return
            except HTTPError, e:
                logger.error("Error while get document: {}".format(e),
                             extra={'MESSAGE_ID': AUCTION_WORKER_DB})
            except Exception, e:
                ecode = e.args[0]
                if ecode in RETRYABLE_ERRORS:
                    logger.error("Error while save document: {}".format(e),
                                 extra={'MESSAGE_ID': AUCTION_WORKER_DB})
                else:
                    logger.critical("Unhandled error: {}".format(e),
                                    extra={'MESSAGE_ID': AUCTION_WORKER_DB})
            retries -= 1
예제 #6
0
def authenticate(request, id):
    result = ""
    docs = Database(
        "https://*****:*****@wazza.cloudant.com/api_keys"
    )
    sep = "_"
    list = string.split(id, sep)
    authenticate = docs.get(str(list[1]))
    if authenticate == None:
        result = result + '<?xml version="1.0" Name="WS Project 2C API"?> \n <error value = "Invalid API Key"></error>'
        return HttpResponse(result)
    else:
        return HttpResponse(lookup(str(list[0])))
예제 #7
0
class CouchdbOutput(OutputModule):
    def __init__(self,
                 actor_config,
                 couchdb_url,
                 payload=None,
                 selection="data",
                 parallel_streams=1,
                 native_events=False,
                 **kw):
        OutputModule.__init__(self, actor_config)
        self.pool.createQueue("inbox")
        self.registerConsumer(self.consume, "inbox")
        self.couchdb = Database(couchdb_url)

    def consume(self, event):
        if event.isBulk():
            bulk_docs = {}
            for e in extractBulkItems(event):
                doc = e.get(self.kwargs.selection)
                doc_id = doc.pop('id', doc.pop('_id', ''))
                if doc_id:
                    doc['_id'] = doc['id'] = doc_id
                bulk_docs[doc['id']] = doc

            for row in self.couchdb.view('_all_docs',
                                         keys=list(bulk_docs.keys())).rows:
                if row.id in bulk_docs:
                    bulk_docs[row.id]['_rev'] = row['value']['rev']
            try:
                responce = self.couchdb.update(list(bulk_docs.values()))
                for ok, doc_id, rest in responce:
                    if ok:
                        self.logging.info("Saved {}".format(doc_id))
                    else:
                        self.logging.error(
                            "Error on save bulk. Type {}, message {}, doc {}".
                            format(rest, getattr(rest, 'message', ''), doc_id))
            except Exception as e:
                self.logging.error("Uncaught error {} on save bulk".format(
                    e, ))
        else:
            data = event.get(self.kwargs.selection)
            doc_id = data.get('id', data.get('_id'))
            if doc_id:
                data['_id'] = data['id'] = doc_id
                if doc_id in self.couchdb:
                    rev = self.couchdb.get(id).rev
                    data['_rev'] = rev
                    self.logging.debug(
                        "Update revision in data {} to {}".format(id, rev))
            self.couchdb.save(data)
예제 #8
0
class CouchDB(object):
    """
    This class is responsible for work with CouchDB

    Attributes:
        _db: Representation of a database to work with on a CouchDB server
        :type _db: couchdb.Database
        db_request_retries: Number of retries of database requesting in case
                            error occurred during getting or saving document
        :type db_request_retries: int
    """
    _db = None
    db_request_retries = 10

    def __init__(self, config):
        self._db = Database(str(config["COUCH_DATABASE"]),
                            session=Session(retry_delays=range(10)))

    def _update_revision(self, auction_document, auction_doc_id):
        """
        Check if document in couchdb database has same '_rev' field value
        with provided document object. If it differs, change '_rev' field value
        for provided auction document with one from couchdb database

        :param auction_document: auction document object
        :param auction_doc_id: identifier of document in database
        :return:
        """
        public_document = self.get_auction_document(auction_doc_id)
        if public_document and public_document.get(
                '_rev') != auction_document['_rev']:
            auction_document["_rev"] = public_document["_rev"]

    def get_auction_document(self, auction_doc_id):
        """
        Retrieve auction document from couchdb database using provided identifier

        :param auction_doc_id: identifier of document in couchdb database
        :return: auction document object from couchdb database
        """
        request_id = generate_request_id()
        retries = self.db_request_retries
        while retries:
            try:
                public_document = self._db.get(auction_doc_id)
                if public_document:
                    LOGGER.info(
                        "Get auction document {0[_id]} with rev {0[_rev]}".
                        format(public_document),
                        extra={
                            "JOURNAL_REQUEST_ID": request_id,
                            "MESSAGE_ID": AUCTION_WORKER_DB_GET_DOC
                        })
                    return public_document

            except HTTPError, e:
                LOGGER.error(
                    "Error while get document: {}".format(e),
                    extra={'MESSAGE_ID': AUCTION_WORKER_DB_GET_DOC_ERROR})
            except Exception, e:
                errcode = e.args[0]
                if errcode in RETRYABLE_ERRORS:
                    LOGGER.error(
                        "Error while get document: {}".format(e),
                        extra={'MESSAGE_ID': AUCTION_WORKER_DB_GET_DOC_ERROR})
                else:
                    LOGGER.critical(
                        "Unhandled error: {}".format(e),
                        extra={
                            'MESSAGE_ID':
                            AUCTION_WORKER_DB_GET_DOC_UNHANDLED_ERROR
                        })
            retries -= 1
예제 #9
0
class Auction(object):
    """Auction Worker Class"""
    def __init__(self, tender_id,
                 worker_defaults={},
                 auction_data={},
                 lot_id=None,
                 activate=False):
        super(Auction, self).__init__()
        self.generate_request_id()
        self.tender_id = tender_id
        self.lot_id = lot_id
        if lot_id:
            self.auction_doc_id = tender_id + "_" + lot_id
        else:
            self.auction_doc_id = tender_id
        self.tender_url = urljoin(
            worker_defaults["TENDERS_API_URL"],
            '/api/{0}/tenders/{1}'.format(
                worker_defaults["TENDERS_API_VERSION"], tender_id
            )
        )
        self.activate = activate
        if auction_data:
            self.debug = True
            logger.setLevel(logging.DEBUG)
            self._auction_data = auction_data
        else:
            self.debug = False
        self._end_auction_event = Event()
        self.bids_actions = BoundedSemaphore()
        self.session = RequestsSession()
        self.worker_defaults = worker_defaults
        if self.worker_defaults.get('with_document_service', False):
            self.session_ds = RequestsSession()
        self._bids_data = {}
        self.db = Database(str(self.worker_defaults["COUCH_DATABASE"]),
                           session=Session(retry_delays=range(10)))
        self.audit = {}
        self.retries = 10
        self.bidders_count = 0
        self.bidders_data = []
        self.bidders_features = {}
        self.bidders_coeficient = {}
        self.features = None
        self.mapping = {}
        self.rounds_stages = []


    def generate_request_id(self):
        self.request_id = generate_request_id()

    def prepare_public_document(self):
        public_document = deepcopy(dict(self.auction_document))
        not_last_stage = self.auction_document["current_stage"] not in (len(self.auction_document["stages"]) - 1,
                                                                        len(self.auction_document["stages"]) - 2,)
        if self.features and not_last_stage:
            for stage_name in ['initial_bids', 'stages', 'results']:
                public_document[stage_name] = map(
                    filter_amount,
                    public_document[stage_name]
                )
        return public_document

    def get_auction_document(self, force=False):
        retries = self.retries
        while retries:
            try:
                public_document = self.db.get(self.auction_doc_id)
                if public_document:
                    logger.info("Get auction document {0[_id]} with rev {0[_rev]}".format(public_document),
                                extra={"JOURNAL_REQUEST_ID": self.request_id,
                                       "MESSAGE_ID": AUCTION_WORKER_DB_GET_DOC})
                    if not hasattr(self, 'auction_document'):
                        self.auction_document = public_document
                    if force:
                        return public_document
                    elif public_document['_rev'] != self.auction_document['_rev']:
                        logger.warning("Rev error")
                        self.auction_document["_rev"] = public_document["_rev"]
                    logger.debug(json.dumps(self.auction_document, indent=4))
                return public_document

            except HTTPError, e:
                logger.error("Error while get document: {}".format(e),
                             extra={'MESSAGE_ID': AUCTION_WORKER_DB_GET_DOC_ERROR})
            except Exception, e:
                ecode = e.args[0]
                if ecode in RETRYABLE_ERRORS:
                    logger.error("Error while get document: {}".format(e),
                                 extra={'MESSAGE_ID': AUCTION_WORKER_DB_GET_DOC_ERROR})
                else:
                    logger.critical("Unhandled error: {}".format(e),
                                    extra={'MESSAGE_ID': AUCTION_WORKER_DB_GET_DOC_UNHANDLED_ERROR})
            retries -= 1
예제 #10
0
class EdgeDataBridge(object):
    """Edge Bridge"""
    def __init__(self, config):
        super(EdgeDataBridge, self).__init__()
        self.config = config
        self.api_host = self.config_get('tenders_api_server')
        self.api_version = self.config_get('tenders_api_version')
        self.retrievers_params = self.config_get('retrievers_params')

        try:
            self.client = TendersClient(host_url=self.api_host,
                                        api_version=self.api_version,
                                        key='')
        except MissingSchema:
            raise DataBridgeConfigError(
                'In config dictionary empty or missing \'tenders_api_server\'')
        except ConnectionError as e:
            raise e

        self.couch_url = urljoin(self.config_get('couch_url'),
                                 self.config_get('public_db'))
        self.db = Database(self.couch_url,
                           session=Session(retry_delays=range(10)))
        try:
            self.db.info()
        except ResourceNotFound:
            error_message = "Database with name '" + self.config_get(
                'public_db') + "' doesn\'t exist"
            raise DataBridgeConfigError(error_message)
        except error as e:
            if e.errno == errno.ECONNREFUSED:
                raise DataBridgeConfigError(
                    "Connection refused: 'couch_url' is invalid in config dictionary"
                )
        except AttributeError as e:
            raise DataBridgeConfigError(
                '\'couch_url\' is missed or empty in config dictionary.')
        except KeyError as e:
            if e.message == 'db_name':
                raise DataBridgeConfigError(
                    '\'public_db\' name is missed or empty in config dictionary'
                )

    def config_get(self, name):
        try:
            return self.config.get('main').get(name)
        except AttributeError as e:
            raise DataBridgeConfigError(
                'In config dictionary missed section \'main\'')

    def get_teders_list(self):
        for item in get_tenders(host=self.api_host,
                                version=self.api_version,
                                key='',
                                extra_params={'mode': '_all_'},
                                retrievers_params=self.retrievers_params):
            yield (item["id"], item["dateModified"])

    def save_tender_in_db(self, tender_id, date_modified):
        tender_doc = self.db.get(tender_id)
        if tender_doc:
            if tender_doc['dateModified'] == date_modified:
                return
        tender = self.client.get_tender(tender_id).get('data')
        if tender:
            tender['_id'] = tender_id
            tender['doc_type'] = 'Tender'
            if tender_doc:
                tender['_rev'] = tender_doc['_rev']
                logger.info('Update tender {} '.format(tender_id))
            else:
                logger.info('Save tender {} '.format(tender_id))
            try:
                self.db.save(tender)
            except Exception as e:
                logger.info(
                    'Saving tender {} fail with error {}'.format(
                        tender_id, e.message),
                    extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
        else:
            logger.info('Tender {} not found'.format(tender_id))

    def run(self):
        logger.info('Start Edge Bridge',
                    extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
        logger.info('Start data sync...',
                    extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
        for tender_id, date_modified in self.get_teders_list():
            self.save_tender_in_db(tender_id, date_modified)