def setUp(self):
        #self._testMethodName
        self.app = Bottle()

        setup_routing(self.app)
        self.server = WSGIServer(('localhost', 20602), self.app, log=None)
        self.server.start()
        self.client = ContractingClient(API_KEY,  host_url=HOST_URL,
                                        api_version=API_VERSION)

        with open(ROOT + 'contract_' + TEST_CONTRACT_KEYS.contract_id + '.json') as contract:
            self.contract = munchify(load(contract))
            self.contract.update({'access':{"token": API_KEY}})
Example #2
0
    def contracting_client_init(self):
        logger.info('Initialization contracting clients.',  extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, {}))
        self.contracting_client = ContractingClient(
            self.config_get('api_token'),
            host_url=self.contracting_api_server, api_version=self.contracting_api_version
        )

        self.contracting_client_ro = self.contracting_client
        if self.config_get('public_tenders_api_server'):
            if self.api_server == self.contracting_api_server and self.api_version == self.contracting_api_version:
                self.contracting_client_ro = ContractingClient(
                    '',
                    host_url=self.ro_api_server, api_version=self.api_version
                )
    def __init__(self, config):
        super(ContractingDataBridge, self).__init__()
        self.config = config

        self.tenders_sync_client = TendersClientSync('',
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'),
        )

        self.client = TendersClient(
            self.config_get('api_token'),
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'),
        )

        self.contracting_client = ContractingClient(
            self.config_get('api_token'),
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version')
        )

        self.initial_sync_point = {}
        self.tenders_queue = Queue(maxsize=500)
        self.handicap_contracts_queue = Queue(maxsize=500)
        self.contracts_put_queue = Queue(maxsize=500)
        self.contracts_retry_put_queue = Queue(maxsize=500)
Example #4
0
class ContractingUserTestCase(unittest.TestCase):
    """"""
    def setUp(self):
        #self._testMethodName
        self.app = Bottle()

        setup_routing(self.app)
        self.server = WSGIServer(('localhost', 20602), self.app, log=None)
        self.server.start()
        self.client = ContractingClient(API_KEY,
                                        host_url=HOST_URL,
                                        api_version=API_VERSION)

        with open(ROOT + 'contract_' + TEST_CONTRACT_KEYS.contract_id +
                  '.json') as contract:
            self.contract = munchify(load(contract))
            self.contract.update({'access': {"token": API_KEY}})

    def tearDown(self):
        self.server.stop()

    ###########################################################################
    #             CREATE ITEM TEST
    ###########################################################################

    def test_create_contract(self):
        setup_routing(self.app, routs=["contract_create"])
        contract = munchify({'data': 'contract'})
        self.client.create_contract(contract)

    ###########################################################################
    #             DOCUMENTS FILE TEST
    ###########################################################################

    def test_upload_contract_document(self):
        setup_routing(self.app, routs=["contract_document_create"])
        file_ = StringIO()
        file_.name = 'test_document.txt'
        file_.write("test upload contract document text data")
        file_.seek(0)
        doc = self.client.upload_document(file_, self.contract)
        self.assertEqual(doc.data.title, file_.name)
        self.assertEqual(doc.data.id, TEST_CONTRACT_KEYS.new_document_id)
        file_.close()
class ContractingUserTestCase(unittest.TestCase):
    """"""
    def setUp(self):
        #self._testMethodName
        self.app = Bottle()

        setup_routing(self.app)
        self.server = WSGIServer(('localhost', 20602), self.app, log=None)
        self.server.start()
        self.client = ContractingClient(API_KEY,  host_url=HOST_URL,
                                        api_version=API_VERSION)

        with open(ROOT + 'contract_' + TEST_CONTRACT_KEYS.contract_id + '.json') as contract:
            self.contract = munchify(load(contract))
            self.contract.update({'access':{"token": API_KEY}})

    def tearDown(self):
        self.server.stop()

    ###########################################################################
    #             CREATE ITEM TEST
    ###########################################################################

    def test_create_contract(self):
        setup_routing(self.app, routs=["contract_create"])
        contract = munchify({'data': 'contract'})
        self.client.create_contract(contract)

    ###########################################################################
    #             DOCUMENTS FILE TEST
    ###########################################################################

    def test_upload_contract_document(self):
        setup_routing(self.app, routs=["contract_document_create"])
        file_ = StringIO()
        file_.name = 'test_document.txt'
        file_.write("test upload contract document text data")
        file_.seek(0)
        doc = self.client.upload_document(file_, self.contract)
        self.assertEqual(doc.data.title, file_.name)
        self.assertEqual(doc.data.id, TEST_CONTRACT_KEYS.new_document_id)
        file_.close()
    def setUp(self):
        # self._testMethodName
        self.app = Bottle()

        setup_routing(self.app)
        self.server = WSGIServer(("localhost", 20602), self.app, log=None)
        self.server.start()
        self.client = ContractingClient(API_KEY, host_url=HOST_URL, api_version=API_VERSION)

        with open(ROOT + "contract_" + TEST_CONTRACT_KEYS.contract_id + ".json") as contract:
            self.contract = munchify(load(contract))
            self.contract.update({"access": {"token": API_KEY}})
    def __init__(self, config):
        super(ContractingDataBridge, self).__init__()
        self.config = config
        self.contracting_client = ContractingClient(
            self.config_get('api_token'),
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version')
        )
        params = {'opt_fields': 'status,lots',
                  'mode': '_all_'}
        self.tenders_client_forward = TendersClient(
            '',
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'),
            params=params
        )

        params_desc = deepcopy(params)
        params_desc['descending'] = 1
        self.tenders_client_backward = TendersClient(
            '',
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'),
            params=params_desc
        )

        self.client = TendersClient(
            self.config_get('api_token'),
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'),
        )

        self.tenders_queue = Queue()
        self.handicap_contracts_queue = Queue()
        self.contracts_put_queue = Queue()
        self.contracts_retry_put_queue = Queue()
class ContractingDataBridge(object):
    """ Contracting Data Bridge """

    def __init__(self, config):
        super(ContractingDataBridge, self).__init__()
        self.config = config

        self.tenders_sync_client = TendersClientSync('',
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'),
        )

        self.client = TendersClient(
            self.config_get('api_token'),
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'),
        )

        self.contracting_client = ContractingClient(
            self.config_get('api_token'),
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version')
        )

        self.initial_sync_point = {}
        self.tenders_queue = Queue(maxsize=500)
        self.handicap_contracts_queue = Queue(maxsize=500)
        self.contracts_put_queue = Queue(maxsize=500)
        self.contracts_retry_put_queue = Queue(maxsize=500)

    def config_get(self, name):
        return self.config.get('main').get(name)

    @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
    def get_tender_credentials(self, tender_id):
        self.client.headers.update({'X-Client-Request-ID': generate_req_id()})
        logger.info("Getting credentials for tender {}".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_CREDENTIALS},
                                                                                                 {"TENDER_ID": tender_id}))
        data = self.client.extract_credentials(tender_id)
        logger.info("Got tender {} credentials".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GOT_CREDENTIALS},
                                                                                         {"TENDER_ID": tender_id}))
        return data

    @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
    def initialize_sync(self, params=None, direction=None):
        # TODO use gevent.Event to wake up forward sync instead of checking
        # initial sync point
        if direction == "backward":
            assert params['descending']
            response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
            # set values in reverse order due to 'descending' option
            self.initial_sync_point = {'forward_offset': response.prev_page.offset,
                                       'backward_offset': response.next_page.offset}
            logger.info("Initial sync point {}".format(self.initial_sync_point))
            return response
        elif not self.initial_sync_point:
            raise ValueError
        else:
            assert 'descending' not in params
            params['offset'] = self.initial_sync_point['forward_offset']
            logger.info("Starting forward sync from offset {}".format(params['offset']))
            return self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})

    def get_tenders(self, params={}, direction=""):
        response = self.initialize_sync(params=params, direction=direction)

        while not (params.get('descending') and not len(response.data) and params.get('offset') == response.next_page.offset):
            tenders_list = response.data
            params['offset'] = response.next_page.offset

            delay = 101
            if tenders_list:
                delay = 15
                logger.info("Client {} params: {}".format(direction, params))
            for tender in tenders_list:
                if tender['status'] in ("active.qualification",
                                        "active.awarded", "complete"):
                    if hasattr(tender, "lots"):
                        if any([1 for lot in tender['lots'] if lot['status'] == "complete"]):
                            logger.info('{} sync: Found multilot tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']),
                                        extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_MULTILOT_COMPLETE}, {"TENDER_ID": tender['id']}))
                            yield tender
                    elif tender['status'] == "complete":
                        logger.info('{} sync: Found tender in complete status {}'.format(direction.capitalize(), tender['id']),
                                    extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_NOLOT_COMPLETE}, {"TENDER_ID": tender['id']}))
                        yield tender
                else:
                    logger.debug('{} sync: Skipping tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']),
                                 extra=journal_context(params={"TENDER_ID": tender['id']}))

            logger.info('Sleep {} sync...'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP}))
            gevent.sleep(delay)
            logger.info('Restore {} sync'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_RESUME}))
            logger.debug('{} {}'.format(direction, params))
            response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})

    def get_tender_contracts(self):
        while True:
            try:
                tender_to_sync = self.tenders_queue.get()
                tender = self.tenders_sync_client.get_tender(tender_to_sync['id'],
                                                             extra_headers={'X-Client-Request-ID': generate_req_id()})['data']
                db.put(tender_to_sync['id'], {'dateModified': tender_to_sync['dateModified']})
            except Exception, e:
                logger.exception(e)
                logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']}))
                self.tenders_queue.put(tender_to_sync)
            else:
                if 'contracts' not in tender:
                    logger.warn('!!!No contracts found in tender {}'.format(tender['id']), extra=journal_context(params={"TENDER_ID": tender['id']}))
                    continue
                for contract in tender['contracts']:
                    if contract["status"] == "active":

                        try:
                            if not db.has(contract['id']):
                                self.contracting_client.get_contract(contract['id'])
                            else:
                                logger.info('Contract {} exists in local db'.format(contract['id']), extra=journal_context(params={"CONTRACT_ID": contract['id']}))
                                continue
                        except ResourceNotFound:
                            logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id']), extra=journal_context(
                                {"MESSAGE_ID": DATABRIDGE_CONTRACT_TO_SYNC}, {"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
                        except Exception, e:
                            logger.exception(e)
                            logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']}))
                            self.tenders_queue.put(tender_to_sync)
                            break
                        else:
                            db.put(contract['id'], True)
                            logger.info('Contract exists {}'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_CONTRACT_EXISTS},
                                                                                                           {"CONTRACT_ID": contract['id']}))
                            continue

                        contract['tender_id'] = tender['id']
                        contract['procuringEntity'] = tender['procuringEntity']

                        if not contract.get('items'):
                            logger.info('Copying contract {} items'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_COPY_CONTRACT_ITEMS},
                                                                                                                  {"CONTRACT_ID": contract['id']}))
                            if tender.get('lots'):
                                related_awards = [aw for aw in tender['awards'] if aw['id'] == contract['awardID']]
                                if related_awards:
                                    award = related_awards[0]
                                    if award.get("items"):
                                        logger.debug('Copying items from related award {}'.format(award['id']))
                                        contract['items'] = award['items']
                                    else:
                                        logger.debug('Copying items matching related lot {}'.format(award['relatedLot']))
                                        contract['items'] = [item for item in tender['items'] if item['relatedLot'] == award['lotID']]
                                else:
                                    logger.warn('Not found related award for contact {} of tender {}'.format(contract['id'], tender['id']),
                                                extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
                            else:
                                logger.debug('Copying all tender {} items into contract {}'.format(tender['id'], contract['id']),
                                             extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
                                contract['items'] = tender['items']

                        if not contract.get('items'):
                            logger.warn('Contact {} of tender {} does not contain items info'.format(contract['id'], tender['id']),
                                        extra=journal_context({"MESSAGE_ID": DATABRIDGE_MISSING_CONTRACT_ITEMS},
                                                              {"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))

                        self.handicap_contracts_queue.put(contract)
class ContractingDataBridge(object):
    """ Contracting Data Bridge """

    def __init__(self, config):
        super(ContractingDataBridge, self).__init__()
        self.config = config
        self.contracting_client = ContractingClient(
            self.config_get('api_token'),
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version')
        )
        params = {'opt_fields': 'status,lots',
                  'mode': '_all_'}
        self.tenders_client_forward = TendersClient(
            '',
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'),
            params=params
        )

        params_desc = deepcopy(params)
        params_desc['descending'] = 1
        self.tenders_client_backward = TendersClient(
            '',
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'),
            params=params_desc
        )

        self.client = TendersClient(
            self.config_get('api_token'),
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'),
        )

        self.tenders_queue = Queue()
        self.handicap_contracts_queue = Queue()
        self.contracts_put_queue = Queue()
        self.contracts_retry_put_queue = Queue()

    def config_get(self, name):
        return self.config.get('main').get(name)

    @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
    def get_tender_credentials(self, tender_id):
        return self.client.extract_credentials(tender_id)

    def get_tenders(self, client):
        while True:
            request_id = generate_req_id()
            client.headers.update({'X-Client-Request-ID': request_id})
            tenders_list = list(client.get_tenders())
            delay = 101
            if tenders_list:
                delay = 15
                logger.info("Client params: {}".format(client.params))
            for tender in tenders_list:
                if tender['status'] in ("active.qualification",
                                        "active.awarded", "complete"):
                    if hasattr(tender, "lots"):
                        if any([1 for lot in tender['lots'] if lot['status'] == "complete"]):
                            logger.info('found multilot tender {} in status {}'.format(tender['id'], tender['status']))
                            yield tender
                    elif tender['status'] == "complete":
                        logger.info('Found tender in complete status {}'.format(tender['id']))
                        yield tender
                else:
                    logger.debug('Skipping tender {} in status {}'.format(tender['id'], tender['status']))

            logger.info('Sleep...')
            time.sleep(delay)

    def get_tender_contracts(self):
        while True:
            request_id = generate_req_id()
            self.tenders_client_backward.headers.update({'X-Client-Request-ID': request_id})
            tender = self.tenders_client_backward.get_tender(self.tenders_queue.get()['id'])['data']
            if 'contracts' not in tender:
                logger.warn('!!!No contracts found in tender {}'.format(tender['id']))
                continue
            for contract in tender['contracts']:
                if contract["status"] == "active":
                    try:
                        self.contracting_client.get_contract(contract['id'])
                    except ResourceNotFound:
                        logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id']))
                    else:
                        logger.info('Contract exists {}'.format(contract['id']))
                        continue

                    contract['tender_id'] = tender['id']
                    contract['procuringEntity'] = tender['procuringEntity']
                    self.handicap_contracts_queue.put(contract)

    def prepare_contract_data(self):
        while True:
            contract = self.handicap_contracts_queue.get()
            try:
                logger.info("Getting extra info for tender {}".format(contract['tender_id']))
                tender_data = self.get_tender_credentials(contract['tender_id'])
            except Exception, e:
                logger.info("Can't get tender credentials {}".format(contract['tender_id']))
                self.handicap_contracts_queue.put(contract)
            else:
                logger.debug("Got extra info for tender {}".format(
                    contract['tender_id']))
                data = tender_data.data
                if data.get('mode'):
                    contract['mode'] = data['mode']
                contract['owner'] = data['owner']
                contract['tender_token'] = data['tender_token']
                del contract['status']  # create contract in 'draft' status
                self.contracts_put_queue.put(contract)
            gevent.sleep(0)
    def __init__(self, config):
        super(ContractingDataBridge, self).__init__()
        self.config = config

        self.cache_db = Db(self.config.get('main'))

        self._backend = "redis"
        self._host = self.config.get('cache_host')
        self._port = self.config.get('cache_port') or 6379
        self._db_name = self.config.get('cache_db_name') or 0

        logger.info(
            "Caching backend: '{}', db name: '{}', host: '{}', port: '{}'".
            format(self.cache_db._backend, self.cache_db._db_name,
                   self.cache_db._host, self.cache_db._port),
            extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, {}))

        self.on_error_delay = self.config_get('on_error_sleep_delay') or 5
        self.jobs_watcher_delay = self.config_get('jobs_watcher_delay') or 15
        queue_size = self.config_get('buffers_size') or 500
        self.full_stack_sync_delay = self.config_get(
            'full_stack_sync_delay') or 15
        self.empty_stack_sync_delay = self.config_get(
            'empty_stack_sync_delay') or 101

        api_server = self.config_get('tenders_api_server')
        api_version = self.config_get('tenders_api_version')
        ro_api_server = self.config_get(
            'public_tenders_api_server') or api_server

        contracting_api_server = self.config_get('contracting_api_server')
        contracting_api_version = self.config_get('contracting_api_version')

        self.tenders_sync_client = TendersClientSync(
            '',
            host_url=ro_api_server,
            api_version=api_version,
        )

        self.client = TendersClient(
            self.config_get('api_token'),
            host_url=api_server,
            api_version=api_version,
        )

        self.contracting_client = ContractingClient(
            self.config_get('api_token'),
            host_url=contracting_api_server,
            api_version=contracting_api_version)

        self.contracting_client_ro = self.contracting_client
        if self.config_get('public_tenders_api_server'):
            if api_server == contracting_api_server and api_version == contracting_api_version:
                self.contracting_client_ro = ContractingClient(
                    self.config_get('api_token'),
                    host_url=ro_api_server,
                    api_version=api_version)

        self.initial_sync_point = {}
        self.initialization_event = gevent.event.Event()
        self.tenders_queue = Queue(maxsize=queue_size)
        self.handicap_contracts_queue = Queue(maxsize=queue_size)
        self.contracts_put_queue = Queue(maxsize=queue_size)
        self.contracts_retry_put_queue = Queue(maxsize=queue_size)
        self.basket = {}