def test_sync_from_rhic_serve_blocking(self): self.assertEqual(len(identity.JOBS), 0) rhics = ConsumerIdentity.objects() self.assertEquals(len(rhics), 0) sync_from_rhic_serve_blocking() rhics = ConsumerIdentity.objects() self.assertEquals(len(rhics), 3) expected_rhics = ["fb647f68-aa01-4171-b62b-35c2984a5328", "ef8548a9-c874-42a8-b5dc-bc5ab0b34cd7", "a17013d8-e896-4749-9b37-8606d62bf643"] for r in rhics: self.assertIn(str(r.uuid), expected_rhics)
def test_update_consumer_that_has_been_marked_as_deleted(self): item = {} item["uuid"] = "734ed55f-c3fb-4249-ac4c-52e440cd9304" item["engineering_ids"] = ["1", "2"] create_or_update_consumer_identity(item) rhics = ConsumerIdentity.objects() self.assertEquals(len(rhics), 1) self.assertEquals(str(rhics[0].uuid), item["uuid"]) self.assertEquals(rhics[0].engineering_ids, item["engineering_ids"]) item["deleted"] = True create_or_update_consumer_identity(item) rhics = ConsumerIdentity.objects() self.assertEquals(len(rhics), 1) self.assertEquals(str(rhics[0].uuid), item["uuid"]) self.assertTrue(rhics[0].deleted)
def test_sync_from_rhic_serve_threaded(self): self.assertEqual(len(identity.JOBS), 0) rhics = ConsumerIdentity.objects() self.assertEquals(len(rhics), 0) sync_thread = sync_from_rhic_serve() for index in range(0,120): if not sync_thread.finished: time.sleep(.05) self.assertTrue(sync_thread.finished) rhics = ConsumerIdentity.objects() self.assertEquals(len(rhics), 3) expected_rhics = ["fb647f68-aa01-4171-b62b-35c2984a5328", "ef8548a9-c874-42a8-b5dc-bc5ab0b34cd7", "a17013d8-e896-4749-9b37-8606d62bf643"] for r in rhics: self.assertIn(str(r.uuid), expected_rhics)
def process_data(data): """ Imports data into mongo, will update documents that have changed, and remove those which have been deleted @param data from rhic_serve """ start = time.time() _LOG.info("Fetched %s rhics from rhic_serve" % (len(data))) consumer_ids = [x["uuid"] for x in data if x.has_key("uuid")] # Determine which of these IDs already exist in DB and which are new existing_objects = ConsumerIdentity.objects(uuid__in=consumer_ids).only("uuid") existing_ids = [str(x.uuid) for x in existing_objects] new_consumer_ids = set(consumer_ids).difference(set(existing_ids)) existing_consumers = [x for x in data if x.has_key("uuid") and x["uuid"] in existing_ids] new_consumers = [x for x in data if x.has_key("uuid") and x["uuid"] in new_consumer_ids] end_determine_new_and_existing = time.time() _LOG.info("%s RHICs from parent consist of %s new and %s updates, %s seconds to determine this" % \ (len(data), len(new_consumers), len(existing_consumers), end_determine_new_and_existing-start)) # Perform a bulk insert for all new_consumers if new_consumers: objectids = bulk_insert(new_consumers) if len(objectids) != len(new_consumer_ids): _LOG.warning("Some RHICs were not created in database: %s new RHICs were intended only %s were created" % \ (len(new_consumers), len(objectids))) end_bulk_insert_new_items = time.time() # Update existing consumer IDs serially if existing_consumers: for item in existing_consumers: create_or_update_consumer_identity(item) end_update_existing_items = time.time() _LOG.info("%s seconds to process %s RHICs, %s new (in %s seconds) and %s updated (in %s seconds)" % \ (end_update_existing_items-start, len(data), len(new_consumers), end_bulk_insert_new_items-end_determine_new_and_existing, len(existing_consumers), end_update_existing_items-end_bulk_insert_new_items)) return consumer_ids
def test_create_new_consumer_identity(self): item = {} item["uuid"] = "734ed55f-c3fb-4249-ac4c-52e440cd9304" item["engineering_ids"] = ["1", "2"] create_or_update_consumer_identity(item) rhics = ConsumerIdentity.objects() self.assertEquals(len(rhics), 1) self.assertEquals(str(rhics[0].uuid), item["uuid"]) self.assertEquals(rhics[0].engineering_ids, item["engineering_ids"])
def create_or_update_consumer_identity(item): """ Creates a new consumer identity or updates existing to match passed in item data @param item: dict containing needed info to construct a ConsumerIdentity object required keys: 'uuid', 'engineering_ids' @type item: dict @return: True on success, False on failure @rtype: bool """ if not item.has_key("uuid"): raise Exception("Missing required parameter: 'uuid'") if not item.has_key("engineering_ids"): raise Exception("Missing required parameter: 'engineering_ids'") consumer_id = item["uuid"] engineering_ids = item["engineering_ids"] created_date = datetime.now(tzutc()) modified_date = datetime.now(tzutc()) deleted = False deleted_date = None if item.has_key("created_date"): created_date = convert_to_datetime(item["created_date"]) if item.has_key("modified_date"): modified_date = convert_to_datetime(item["modified_date"]) if item.has_key("deleted"): deleted = item["deleted"] if item.has_key("deleted_date"): deleted_date = convert_to_datetime(item["deleted_date"]) if deleted and not deleted_date: deleted_date = datetime.now(tzutc()) identity = ConsumerIdentity.objects(uuid=UUID(consumer_id)).first() if not identity: _LOG.info("Creating new ConsumerIdentity for: %s" % (consumer_id)) identity = ConsumerIdentity(uuid=UUID(consumer_id)) identity.engineering_ids = engineering_ids identity.created_date = created_date identity.modified_date = modified_date identity.deleted = deleted identity.deleted_date = deleted_date try: _LOG.debug("Updating ConsumerIdentity: %s" % (identity)) identity.save(safe=True) return True except Exception, e: _LOG.exception(e) return False
def bulk_insert(data): # Form model objects out of dictionary data items objects = [] for d in data: objects.append(convert_dict_to_consumer_identity(d)) if objects: # Perform bulk insert q = ConsumerIdentity.objects() return q.insert(objects, load_bulk=False, safe=False) return []
def test_check_access_unallowed(self): identity = ConsumerIdentity.objects(uuid=self.valid_identity_uuid).first() allowed_products, unallowed_products = self.checkin.check_access(identity, ["100", "101"]) self.assertTrue("100" in unallowed_products) self.assertTrue("101" in unallowed_products) self.assertEquals(len(allowed_products), 0) allowed_products, unallowed_products = self.checkin.check_access(identity, [self.valid_products[0], "101"]) self.assertTrue(self.valid_products[0] in allowed_products) self.assertTrue("101" in unallowed_products) self.assertEquals(len(allowed_products), 1) self.assertEquals(len(unallowed_products), 1)
def test_sync_where_existing_rhics_product_mapping_changes(self): self.assertEqual(len(identity.JOBS), 0) # Create a RHIC with products that will change after sync item = {} item["uuid"] = "fb647f68-aa01-4171-b62b-35c2984a5328" item["engineering_ids"] = ["1", "2"] create_or_update_consumer_identity(item) rhics = ConsumerIdentity.objects() self.assertEquals(len(rhics), 1) sync_from_rhic_serve_blocking() rhics = ConsumerIdentity.objects() self.assertEquals(len(rhics), 3) expected_rhics = ["fb647f68-aa01-4171-b62b-35c2984a5328", "ef8548a9-c874-42a8-b5dc-bc5ab0b34cd7", "a17013d8-e896-4749-9b37-8606d62bf643"] for r in rhics: self.assertIn(str(r.uuid), expected_rhics) # Ensure that the products have been updated rhic_under_test = ConsumerIdentity.objects(uuid=item["uuid"]).first() self.assertTrue(rhic_under_test) expected_products = ["183", "83", "69"] for ep in expected_products: self.assertTrue(ep in rhic_under_test.engineering_ids)
def get_identity_object(self, consumer_uuid): _LOG.info("Found ID from identity certificate is '%s' " % (consumer_uuid)) identity = ConsumerIdentity.objects(uuid=UUID(consumer_uuid)).first() if not identity: # Lookup if we have a cached response for this uuid cached_status_code = identity_lookup.get_cached_status_code(consumer_uuid) if cached_status_code: _LOG.info("Found cached lookup for '%s' with status_code '%s'" % (consumer_uuid, cached_status_code)) if cached_status_code == 404: raise NotFoundConsumerIdentity(consumer_uuid) else: raise UnexpectedStatusCodeException(consumer_uuid, cached_status_code) # If not, create a new lookup and query parent _LOG.info("Couldn't find RHIC with ID '%s', will query parent" % (consumer_uuid)) identity_lookup.create_rhic_lookup_task(consumer_uuid) raise UnknownConsumerIdentity(consumer_uuid) return identity
def convert_dict_to_consumer_identity(item): """ Converts a dictionary to a ConsumerIdentity @param item: dict containing needed info to construct a ConsumerIdentity object required keys: 'uuid', 'engineering_ids' @type item: dict @return: instance of a consumer identity, note this instance has not yet been saved @rtype: splice.common.models.ConsumerIdentity """ if not item.has_key("uuid"): raise Exception("Missing required parameter: 'uuid'") if not item.has_key("engineering_ids"): raise Exception("Missing required parameter: 'engineering_ids'") consumer_id = item["uuid"] engineering_ids = item["engineering_ids"] created_date = datetime.now(tzutc()) modified_date = datetime.now(tzutc()) deleted = False deleted_date = None if item.has_key("created_date"): created_date = convert_to_datetime(item["created_date"]) if item.has_key("modified_date"): modified_date = convert_to_datetime(item["modified_date"]) if item.has_key("deleted"): deleted = item["deleted"] if item.has_key("deleted_date"): deleted_date = convert_to_datetime(item["deleted_date"]) if deleted and not deleted_date: deleted_date = datetime.now(tzutc()) identity = ConsumerIdentity(uuid=UUID(consumer_id)) identity.engineering_ids = engineering_ids identity.created_date = created_date identity.modified_date = modified_date identity.deleted = deleted identity.deleted_date = deleted_date return identity
def test_check_access_allowed(self): identity = ConsumerIdentity.objects(uuid=self.valid_identity_uuid).first() allowed_products, unallowed_products = self.checkin.check_access(identity, self.valid_products) for p in self.valid_products: self.assertTrue(p in allowed_products) self.assertEquals(len(unallowed_products), 0)