def test_sync_from_rhic_serve_blocking(self):
self.assertEqual(len(identity.JOBS), 0)
rhics = ConsumerIdentity.objects()
self.assertEquals(len(rhics), 0)
sync_from_rhic_serve_blocking()
rhics = ConsumerIdentity.objects()
self.assertEquals(len(rhics), 3)
expected_rhics = ["fb647f68-aa01-4171-b62b-35c2984a5328",
"ef8548a9-c874-42a8-b5dc-bc5ab0b34cd7",
"a17013d8-e896-4749-9b37-8606d62bf643"]
for r in rhics:
self.assertIn(str(r.uuid), expected_rhics)
def test_update_consumer_that_has_been_marked_as_deleted(self):
item = {}
item["uuid"] = "734ed55f-c3fb-4249-ac4c-52e440cd9304"
item["engineering_ids"] = ["1", "2"]
create_or_update_consumer_identity(item)
rhics = ConsumerIdentity.objects()
self.assertEquals(len(rhics), 1)
self.assertEquals(str(rhics[0].uuid), item["uuid"])
self.assertEquals(rhics[0].engineering_ids, item["engineering_ids"])
item["deleted"] = True
create_or_update_consumer_identity(item)
rhics = ConsumerIdentity.objects()
self.assertEquals(len(rhics), 1)
self.assertEquals(str(rhics[0].uuid), item["uuid"])
self.assertTrue(rhics[0].deleted)
def test_sync_from_rhic_serve_threaded(self):
self.assertEqual(len(identity.JOBS), 0)
rhics = ConsumerIdentity.objects()
self.assertEquals(len(rhics), 0)
sync_thread = sync_from_rhic_serve()
for index in range(0,120):
if not sync_thread.finished:
time.sleep(.05)
self.assertTrue(sync_thread.finished)
rhics = ConsumerIdentity.objects()
self.assertEquals(len(rhics), 3)
expected_rhics = ["fb647f68-aa01-4171-b62b-35c2984a5328",
"ef8548a9-c874-42a8-b5dc-bc5ab0b34cd7",
"a17013d8-e896-4749-9b37-8606d62bf643"]
for r in rhics:
self.assertIn(str(r.uuid), expected_rhics)
def process_data(data):
"""
Imports data into mongo, will update documents that have changed, and remove those which have been deleted
@param data from rhic_serve
"""
start = time.time()
_LOG.info("Fetched %s rhics from rhic_serve" % (len(data)))
consumer_ids = [x["uuid"] for x in data if x.has_key("uuid")]
# Determine which of these IDs already exist in DB and which are new
existing_objects = ConsumerIdentity.objects(uuid__in=consumer_ids).only("uuid")
existing_ids = [str(x.uuid) for x in existing_objects]
new_consumer_ids = set(consumer_ids).difference(set(existing_ids))
existing_consumers = [x for x in data if x.has_key("uuid") and x["uuid"] in existing_ids]
new_consumers = [x for x in data if x.has_key("uuid") and x["uuid"] in new_consumer_ids]
end_determine_new_and_existing = time.time()
_LOG.info("%s RHICs from parent consist of %s new and %s updates, %s seconds to determine this" % \
(len(data), len(new_consumers), len(existing_consumers), end_determine_new_and_existing-start))
# Perform a bulk insert for all new_consumers
if new_consumers:
objectids = bulk_insert(new_consumers)
if len(objectids) != len(new_consumer_ids):
_LOG.warning("Some RHICs were not created in database: %s new RHICs were intended only %s were created" % \
(len(new_consumers), len(objectids)))
end_bulk_insert_new_items = time.time()
# Update existing consumer IDs serially
if existing_consumers:
for item in existing_consumers:
create_or_update_consumer_identity(item)
end_update_existing_items = time.time()
_LOG.info("%s seconds to process %s RHICs, %s new (in %s seconds) and %s updated (in %s seconds)" % \
(end_update_existing_items-start, len(data), len(new_consumers),
end_bulk_insert_new_items-end_determine_new_and_existing,
len(existing_consumers), end_update_existing_items-end_bulk_insert_new_items))
return consumer_ids
def test_create_new_consumer_identity(self):
item = {}
item["uuid"] = "734ed55f-c3fb-4249-ac4c-52e440cd9304"
item["engineering_ids"] = ["1", "2"]
create_or_update_consumer_identity(item)
rhics = ConsumerIdentity.objects()
self.assertEquals(len(rhics), 1)
self.assertEquals(str(rhics[0].uuid), item["uuid"])
self.assertEquals(rhics[0].engineering_ids, item["engineering_ids"])
def create_or_update_consumer_identity(item):
"""
Creates a new consumer identity or updates existing to match passed in item data
@param item: dict containing needed info to construct a ConsumerIdentity object
required keys: 'uuid', 'engineering_ids'
@type item: dict
@return: True on success, False on failure
@rtype: bool
"""
if not item.has_key("uuid"):
raise Exception("Missing required parameter: 'uuid'")
if not item.has_key("engineering_ids"):
raise Exception("Missing required parameter: 'engineering_ids'")
consumer_id = item["uuid"]
engineering_ids = item["engineering_ids"]
created_date = datetime.now(tzutc())
modified_date = datetime.now(tzutc())
deleted = False
deleted_date = None
if item.has_key("created_date"):
created_date = convert_to_datetime(item["created_date"])
if item.has_key("modified_date"):
modified_date = convert_to_datetime(item["modified_date"])
if item.has_key("deleted"):
deleted = item["deleted"]
if item.has_key("deleted_date"):
deleted_date = convert_to_datetime(item["deleted_date"])
if deleted and not deleted_date:
deleted_date = datetime.now(tzutc())
identity = ConsumerIdentity.objects(uuid=UUID(consumer_id)).first()
if not identity:
_LOG.info("Creating new ConsumerIdentity for: %s" % (consumer_id))
identity = ConsumerIdentity(uuid=UUID(consumer_id))
identity.engineering_ids = engineering_ids
identity.created_date = created_date
identity.modified_date = modified_date
identity.deleted = deleted
identity.deleted_date = deleted_date
try:
_LOG.debug("Updating ConsumerIdentity: %s" % (identity))
identity.save(safe=True)
return True
except Exception, e:
_LOG.exception(e)
return False
def bulk_insert(data):
# Form model objects out of dictionary data items
objects = []
for d in data:
objects.append(convert_dict_to_consumer_identity(d))
if objects:
# Perform bulk insert
q = ConsumerIdentity.objects()
return q.insert(objects, load_bulk=False, safe=False)
return []
def test_check_access_unallowed(self):
identity = ConsumerIdentity.objects(uuid=self.valid_identity_uuid).first()
allowed_products, unallowed_products = self.checkin.check_access(identity, ["100", "101"])
self.assertTrue("100" in unallowed_products)
self.assertTrue("101" in unallowed_products)
self.assertEquals(len(allowed_products), 0)
allowed_products, unallowed_products = self.checkin.check_access(identity, [self.valid_products[0], "101"])
self.assertTrue(self.valid_products[0] in allowed_products)
self.assertTrue("101" in unallowed_products)
self.assertEquals(len(allowed_products), 1)
self.assertEquals(len(unallowed_products), 1)
def test_sync_where_existing_rhics_product_mapping_changes(self):
self.assertEqual(len(identity.JOBS), 0)
# Create a RHIC with products that will change after sync
item = {}
item["uuid"] = "fb647f68-aa01-4171-b62b-35c2984a5328"
item["engineering_ids"] = ["1", "2"]
create_or_update_consumer_identity(item)
rhics = ConsumerIdentity.objects()
self.assertEquals(len(rhics), 1)
sync_from_rhic_serve_blocking()
rhics = ConsumerIdentity.objects()
self.assertEquals(len(rhics), 3)
expected_rhics = ["fb647f68-aa01-4171-b62b-35c2984a5328",
"ef8548a9-c874-42a8-b5dc-bc5ab0b34cd7",
"a17013d8-e896-4749-9b37-8606d62bf643"]
for r in rhics:
self.assertIn(str(r.uuid), expected_rhics)
# Ensure that the products have been updated
rhic_under_test = ConsumerIdentity.objects(uuid=item["uuid"]).first()
self.assertTrue(rhic_under_test)
expected_products = ["183", "83", "69"]
for ep in expected_products:
self.assertTrue(ep in rhic_under_test.engineering_ids)
def get_identity_object(self, consumer_uuid):
_LOG.info("Found ID from identity certificate is '%s' " % (consumer_uuid))
identity = ConsumerIdentity.objects(uuid=UUID(consumer_uuid)).first()
if not identity:
# Lookup if we have a cached response for this uuid
cached_status_code = identity_lookup.get_cached_status_code(consumer_uuid)
if cached_status_code:
_LOG.info("Found cached lookup for '%s' with status_code '%s'" % (consumer_uuid, cached_status_code))
if cached_status_code == 404:
raise NotFoundConsumerIdentity(consumer_uuid)
else:
raise UnexpectedStatusCodeException(consumer_uuid, cached_status_code)
# If not, create a new lookup and query parent
_LOG.info("Couldn't find RHIC with ID '%s', will query parent" % (consumer_uuid))
identity_lookup.create_rhic_lookup_task(consumer_uuid)
raise UnknownConsumerIdentity(consumer_uuid)
return identity
def convert_dict_to_consumer_identity(item):
"""
Converts a dictionary to a ConsumerIdentity
@param item: dict containing needed info to construct a ConsumerIdentity object
required keys: 'uuid', 'engineering_ids'
@type item: dict
@return: instance of a consumer identity, note this instance has not yet been saved
@rtype: splice.common.models.ConsumerIdentity
"""
if not item.has_key("uuid"):
raise Exception("Missing required parameter: 'uuid'")
if not item.has_key("engineering_ids"):
raise Exception("Missing required parameter: 'engineering_ids'")
consumer_id = item["uuid"]
engineering_ids = item["engineering_ids"]
created_date = datetime.now(tzutc())
modified_date = datetime.now(tzutc())
deleted = False
deleted_date = None
if item.has_key("created_date"):
created_date = convert_to_datetime(item["created_date"])
if item.has_key("modified_date"):
modified_date = convert_to_datetime(item["modified_date"])
if item.has_key("deleted"):
deleted = item["deleted"]
if item.has_key("deleted_date"):
deleted_date = convert_to_datetime(item["deleted_date"])
if deleted and not deleted_date:
deleted_date = datetime.now(tzutc())
identity = ConsumerIdentity(uuid=UUID(consumer_id))
identity.engineering_ids = engineering_ids
identity.created_date = created_date
identity.modified_date = modified_date
identity.deleted = deleted
identity.deleted_date = deleted_date
return identity
def test_check_access_allowed(self):
identity = ConsumerIdentity.objects(uuid=self.valid_identity_uuid).first()
allowed_products, unallowed_products = self.checkin.check_access(identity, self.valid_products)
for p in self.valid_products:
self.assertTrue(p in allowed_products)
self.assertEquals(len(unallowed_products), 0)
