def __init__(self, convoy_conf):
LOGGER.info('Init Convoy...')
self.convoy_conf = convoy_conf
self.stop_transmitting = False
self.transmitter_timeout = self.convoy_conf.get('transmitter_timeout',
10)
self.documents_transfer_queue = Queue()
self.timeout = self.convoy_conf.get('timeout', 10)
self.api_client = APIClient(**self.convoy_conf['cdb'])
self.lots_client = LotsClient(**self.convoy_conf['lots_db'])
self.assets_client = AssetsClient(**self.convoy_conf['assets_db'])
self.keys = ['classification', 'additionalClassifications', 'address',
'unit', 'quantity', 'location', 'id']
self.document_keys = ['hash', 'description', 'title', 'url', 'format',
'documentType']
user = self.convoy_conf['couchdb'].get('user', '')
password = self.convoy_conf['couchdb'].get('password', '')
if user and password:
server = Server(
"http://{user}:{password}@{host}:{port}".format(
**self.convoy_conf['couchdb']),
session=Session(retry_delays=range(10)))
self.db = server[self.convoy_conf['couchdb']['db']] if \
self.convoy_conf['couchdb']['db'] in server else \
server.create(self.convoy_conf['couchdb']['db'])
else:
server = Server(
"http://{host}:{port}".format(
**self.convoy_conf['couchdb']),
session=Session(retry_delays=range(10)))
self.db = server[self.convoy_conf['couchdb']['db']] if \
self.convoy_conf['couchdb']['db'] in server else \
server.create(self.convoy_conf['couchdb']['db'])
push_filter_doc(self.db)
LOGGER.info('Added filters doc to db.')
def __init__(self, convoy_conf):
LOGGER.info('Init Convoy...')
self.convoy_conf = convoy_conf
self.stop_transmitting = False
self.transmitter_timeout = self.convoy_conf.get(
'transmitter_timeout', 10)
self.documents_transfer_queue = Queue()
self.timeout = self.convoy_conf.get('timeout', 10)
self.api_client = APIClient(**self.convoy_conf['cdb'])
self.lots_client = LotsClient(**self.convoy_conf['lots_db'])
self.assets_client = AssetsClient(**self.convoy_conf['assets_db'])
user = self.convoy_conf['couchdb'].get('user', '')
password = self.convoy_conf['couchdb'].get('password', '')
if user and password:
self.db = Server(
"http://{user}:{password}@{host}:{port}".format(
**self.convoy_conf['couchdb']),
session=Session(
retry_delays=range(10)))[self.convoy_conf['couchdb']['db']]
else:
self.db = Server(
"http://{host}:{port}".format(**self.convoy_conf['couchdb']),
session=Session(
retry_delays=range(10)))[self.convoy_conf['couchdb']['db']]
push_filter_doc(self.db)
LOGGER.info('Added filters doc to db.')
def setUp(self):
# Init client for 2 resources
self.lots_client = LotsClient(key=config['token'],
host_url=config['url'],
api_version=config['version'])
self.assets_client = AssetsClient(key=config['token'],
host_url=config['url'],
api_version=config['version'])
def main():
args = parse_args()
config = app_config(args.config)
LOGGER.info("Connecting to DB...")
db = connect_to_db(
config.contracting.db.protocol,
config.contracting.db.host,
config.contracting.db.port,
config.contracting.db.login,
config.contracting.db.password,
config.contracting.db.name,
)
if db is None:
LOGGER.info("Gracefully exiting.")
return
LOGGER.info("Connected to DB")
LOGGER.info("Init clients")
ceasefire_client = ContractingClient(
host_url=config.contracting.api.host,
api_version=config.contracting.api.version,
key=config.contracting.api.token,
)
loki_client = LotsClient(
host_url=config.lots.api.host,
api_version=config.lots.api.version,
key=config.lots.api.token,
)
LOGGER.info("Clients are ready")
sleep_time_range = (config.runner.sleep_seconds.min,
config.runner.sleep_seconds.max)
runner = CeasefireLokiRunner(db, ceasefire_client, loki_client,
sleep_time_range)
runner.start()
class Convoy(object):
"""
Convoy worker object.
Worker that get assets and transform them to item's, than
he patch lot and auction to specified statuses
"""
def __init__(self, convoy_conf):
LOGGER.info('Init Convoy...')
self.convoy_conf = convoy_conf
self.stop_transmitting = False
self.transmitter_timeout = self.convoy_conf.get('transmitter_timeout',
10)
self.documents_transfer_queue = Queue()
self.timeout = self.convoy_conf.get('timeout', 10)
self.api_client = APIClient(**self.convoy_conf['cdb'])
self.lots_client = LotsClient(**self.convoy_conf['lots_db'])
self.assets_client = AssetsClient(**self.convoy_conf['assets_db'])
self.keys = ['classification', 'additionalClassifications', 'address',
'unit', 'quantity', 'location', 'id']
self.document_keys = ['hash', 'description', 'title', 'url', 'format',
'documentType']
user = self.convoy_conf['couchdb'].get('user', '')
password = self.convoy_conf['couchdb'].get('password', '')
if user and password:
server = Server(
"http://{user}:{password}@{host}:{port}".format(
**self.convoy_conf['couchdb']),
session=Session(retry_delays=range(10)))
self.db = server[self.convoy_conf['couchdb']['db']] if \
self.convoy_conf['couchdb']['db'] in server else \
server.create(self.convoy_conf['couchdb']['db'])
else:
server = Server(
"http://{host}:{port}".format(
**self.convoy_conf['couchdb']),
session=Session(retry_delays=range(10)))
self.db = server[self.convoy_conf['couchdb']['db']] if \
self.convoy_conf['couchdb']['db'] in server else \
server.create(self.convoy_conf['couchdb']['db'])
push_filter_doc(self.db)
LOGGER.info('Added filters doc to db.')
def _get_documents(self, item):
documents = []
for doc in item.get('documents', []):
item_document = {
k: doc[k] for k in self.document_keys if k in doc
}
registered_doc = self.api_client.ds_client.register_document_upload(doc['hash'])
LOGGER.info('Registered document upload for item {} with hash'
' {}'.format(item.id, doc['hash']))
transfer_item = {
'get_url': doc.url,
'upload_url': registered_doc['upload_url']
}
self.documents_transfer_queue.put(transfer_item)
item_document['url'] = registered_doc['data']['url']
item_document['documentOf'] = 'item'
item_document['relatedItem'] = item.id
documents.append(item_document)
return documents
def _create_items_from_assets(self, assets_ids):
items = []
documents = []
for index, asset_id in enumerate(assets_ids):
asset = self.assets_client.get_asset(asset_id).data
LOGGER.info('Received asset {} with status {}'.format(
asset.id, asset.status))
# Convert asset to item
item = {k: asset[k] for k in self.keys if k in asset}
item['description'] = asset.title
items.append(item)
# Get documents from asset
for doc in self._get_documents(asset):
documents.append(doc)
# Get items and items documents from complex asset
for item in asset.get('items', []):
items.append(item)
for doc in self._get_documents(item):
documents.append(doc)
return items, documents
def invalidate_auction(self, auction_id):
self.api_client.patch_resource_item(
auction_id, {"data": {"status": "invalid"}}
)
LOGGER.info('Switch auction {} status to invalid'.format(auction_id))
def prepare_auction(self, auction_doc):
LOGGER.info('Prepare auction {}'.format(auction_doc.id))
lot_id = auction_doc.merchandisingObject
# Get lot
try:
lot = self.lots_client.get_lot(lot_id).data
except ResourceNotFound:
self.invalidate_auction(auction_doc.id)
return
LOGGER.info('Received lot {} from CDB'.format(lot_id))
if lot.status != u'active.salable':
# lot['status'] = 'active.salable'
# self.lots_client.patch_resource_item(lot)
LOGGER.warning(
'Lot status \'{}\' not equal \'active.salable\''.format(
lot.status),
extra={'MESSAGE_ID': 'invalid_lot_status'})
self.invalidate_auction(auction_doc.id)
return
# Lock lot
auctions_list = lot.get('auctions', [])
auctions_list.append(auction_doc.id)
lot_patch_data = {'data': {'status': 'active.awaiting', 'auctions': auctions_list}}
self.lots_client.patch_resource_item(lot.id, lot_patch_data)
LOGGER.info('Lock lot {}'.format(lot.id),
extra={'MESSAGE_ID': 'lock_lot'})
# Convert assets to items
items, documents = self._create_items_from_assets(lot.assets)
if not items:
self.lots_client.patch_resource_item(lot.id, {'data': {'status': 'active.salable'}})
LOGGER.info('Switch lot {} status to active.salable'.format(lot.id))
self.invalidate_auction(auction_doc.id)
return
api_auction_doc = self.api_client.get_resource_item(auction_doc['id']).data
LOGGER.info('Received auction {} from CDB'.format(auction_doc['id']))
# Add items to CDB
auction_patch_data = {'data': {'items': items}}
self.api_client.patch_resource_item(
api_auction_doc.id, auction_patch_data
)
LOGGER.info('Added {} items to auction {}'.format(len(items),
auction_doc['id']))
# Add documents to CDB
for document in documents:
self.api_client.create_resource_item_subitem(
api_auction_doc.id, {'data': document}, DOCUMENTS
)
LOGGER.info(
'Added document with hash {} to auction id: {} item id:'
' {} in CDB'.format(document['hash'],
auction_doc['id'],
document['relatedItem'])
)
# Switch lot
self.lots_client.patch_resource_item(lot['id'], {'data': {'status': 'active.auction'}})
LOGGER.info('Switch lot {} to (active.auction) status'.format(lot['id']))
# Switch auction
self.api_client.patch_resource_item(auction_doc['id'], {'data': {'status': 'active.tendering'}})
LOGGER.info('Switch auction {} to (active.tendering) status'.format(auction_doc['id']))
def report_results(self, auction_doc):
LOGGER.info('Report auction results {}'.format(auction_doc.id))
lot_id = auction_doc.merchandisingObject
# Get lot
try:
lot = self.lots_client.get_lot(lot_id).data
except ResourceNotFound:
LOGGER.warning('Lot {} not found when report auction {} results'.format(lot_id, auction_doc.id))
return
if lot.status != 'active.auction' and lot.auctions[-1] == auction_doc.id:
LOGGER.info('Auction {} results already reported to lot {}'.format(auction_doc.id, lot_id))
return
LOGGER.info('Received lot {} from CDB'.format(lot_id))
if auction_doc.status == 'complete':
next_lot_status = 'sold'
else:
next_lot_status = 'active.salable'
# Report results
self.lots_client.patch_resource_item(lot['id'], {'data': {'status': next_lot_status}})
LOGGER.info('Switch lot {} to ({}) status'.format(lot['id'], next_lot_status))
def file_bridge(self):
while not self.stop_transmitting:
try:
transfer_item = self.documents_transfer_queue.get(timeout=2)
try:
file_, _ = self.api_client.get_file(
transfer_item['get_url'])
LOGGER.debug('Received document file from asset DS')
# TODO: Fill headers valid data if needed
headers = {}
self.api_client.ds_client.document_upload_not_register(
file_, headers
)
LOGGER.debug('Uploaded document file to auction DS')
except:
LOGGER.error('While receiving or uploading document '
'something went wrong :(')
self.documents_transfer_queue.put(transfer_item)
sleep(1)
continue
except Empty:
sleep(self.transmitter_timeout)
def run(self):
self.transmitter = spawn(self.file_bridge)
sleep(1)
for auction_info in continuous_changes_feed(self.db):
LOGGER.info('Received auction {}'.format(repr(auction_info)))
if auction_info['status'] == 'pending.verification':
self.prepare_auction(auction_info)
else:
self.report_results(auction_info)
class InternalTest(unittest.TestCase):
'''
Internal TestCase for openregistry correctness.
openprocurement.client.python for request
Test workflow, concierge(bot), convoy(bot) and
check switching statuses
'''
def setUp(self):
# Init client for 2 resources
self.lots_client = LotsClient(key=config['token'],
host_url=config['url'],
api_version=config['version'])
self.assets_client = AssetsClient(key=config['token'],
host_url=config['url'],
api_version=config['version'])
# self.auctions_client = APIResourceClient(
# resource="auctions",
# key=config['auction_token'],
# host_url=config['auction_url'],
# api_version=config['auction_version']
# )
def ensure_resource_status(self, get_resource, id, status, *args,
**kwargs):
'''
Wait for switching resource's status
'''
times = kwargs.get("times", 20)
waiting_message = kwargs.get(
"waiting_message",
"Waiting for resource's ({}) '{}' status".format(id, status))
for i in reversed(range(times)):
time.sleep(i)
resource = get_resource(id).data
if resource.status == status:
break
else:
print waiting_message
resource = get_resource(id).data
self.assertEqual(resource.status, status)
return resource
def test_01_general_workflow(self):
'''
Create two assets and move them to pending status
Create lot with this assets and move to verification status
Create procedure from lot
Check auction is unsuccessful and lot is active.salable
Move lot to dissolved status and check assets pending status
'''
# Create assets =======================================================
assets = []
assets.append(
self.assets_client.create_resource_item(
{"data": test_asset_basic_data}))
assets.append(
self.assets_client.create_resource_item(
{"data": test_asset_basic_data}))
self.assertNotEqual(assets[0].data.id, assets[1].data.id)
self.assertEqual(assets[0].data.status, 'draft')
self.assertEqual(assets[1].data.status, 'draft')
print "Successfully created assets [{}, {}]".format(
assets[0].data.id, assets[1].data.id)
# Move assets to pending ==============================================
for asset in assets:
asset_id = asset.data.id
self.assets_client.patch_asset(asset.data.id,
{"data": {
"status": "pending"
}}, asset.access.token)
self.assertEqual(
self.assets_client.get_asset(asset_id).data.status, "pending")
print "Moved assets to 'pending' status"
# Create lot ==========================================================
test_lot_data['assets'] = [assets[0].data.id, assets[1].data.id]
lot = self.lots_client.create_resource_item({"data": test_lot_data})
self.assertEqual(lot.data.status, 'draft')
print "Successfully created lot [{}]".format(lot.data.id)
# Move lot to Pending =================================================
self.lots_client.patch_lot(lot.data.id,
{"data": {
"status": "pending"
}}, lot.access.token)
self.assertEqual(
self.lots_client.get_lot(lot.data.id).data.status, "pending")
print "Moved lot to 'pending' status"
# Move lot to Verification ============================================
self.lots_client.patch_lot(lot.data.id,
{"data": {
"status": "verification"
}}, lot.access.token)
self.assertEqual(
self.lots_client.get_lot(lot.data.id).data.status, "verification")
print "Moved lot to 'verification' status"
# Check lot and assets statuses =======================================
upd_lot = self.ensure_resource_status(
self.lots_client.get_lot,
lot.data.id,
"active.salable",
waiting_message="Waiting for Concierge ...")
for asset in upd_lot.assets:
upd_asset = self.assets_client.get_asset(asset).data
self.assertEqual(upd_asset.status, "active")
self.assertEqual(upd_asset.relatedLot, upd_lot.id)
print "Concierge has moved lot to 'active.salable' and assets to 'active' statuses"
# Create auction ======================================================
# test_auction_data['merchandisingObject'] = lot.data.id
#
# auction = self.auctions_client.create_resource_item({
# "data": test_auction_data
# })
# self.assertEqual(auction.data.status, 'pending.verification')
#
# print "Successfully created auction [{}]".format(auction.data.id)
# Check auction and lot statuses ======================================
# self.ensure_resource_status(
# self.auctions_client.get_resource_item,
# auction.data.id, "active.tendering",
# waiting_message="Waiting for Convoy ..."
# )
#
# self.assertEqual(self.lots_client.get_lot(lot.data.id).data.status,
# "active.auction")
#
# print "Convoy has moved lot to 'active.auction' status"
# Check auction finished ==============================================
# self.ensure_resource_status(
# self.auctions_client.get_resource_item,
# auction.data.id, "unsuccessful",
# times=35,
# waiting_message="Waiting for UNS ..."
# )
#
# print "Switched auction to 'unsuccessful' status"
# Check lot status ====================================================
# self.ensure_resource_status(
# self.lots_client.get_lot,
# lot.data.id, "active.salable",
# waiting_message="Waiting for Convoy ..."
# )
#
# print "Convoy has moved lot to 'active.salable' status and done his work!"
# Move lot to dissolved status ========================================
self.lots_client.patch_lot(lot.data.id,
{"data": {
"status": "dissolved"
}}, lot.access.token)
lot_status = self.lots_client.get_lot(lot.data.id).data.status
self.assertEqual(lot_status, "dissolved")
print "Moved lot to 'dissolved' status"
# Check assets status =================================================
self.ensure_resource_status(
self.assets_client.get_asset,
assets[0].data.id,
"pending",
waiting_message="Waiting for Concierge ...")
for asset in assets:
upd_asset = self.assets_client.get_asset(asset.data.id).data
self.assertEqual(upd_asset.status, "pending")
print "Concierge has moved assets to 'pending' status and done his work!"