def test_uploading_simple_product_usage(self): consumer = "consumer_uuid" splice_server = "splice_server_uuid" inst_id = "mac_addr" date = datetime.now(tzutc()) allowed_product_info = ["1", "2", "3"] unallowed_product_info = ["0"] facts = {"tbd": "values"} pu_a = self.create_product_usage(consumer, splice_server, inst_id, date, allowed_product_info=allowed_product_info, unallowed_product_info=unallowed_product_info, facts=facts) self.assertEquals(len(ProductUsage.objects()), 0) post_data = [pu_a._data] LOG.info("Calling api for productusage import with post data: '%s'" % (post_data)) resp = self.api_client.post('/api/v1/productusage/', format='json', data=post_data, SSL_CLIENT_CERT=self.expected_valid_splice_server_identity_pem) LOG.info("Response for productusage import: Status Code: %s, Response: %s" % (resp.status_code, resp)) self.assertEquals(resp.status_code, 202) # Now check that the server api saved the object as expected found = ProductUsage.objects() self.assertEquals(len(found), 1) self.assertEquals(found[0].consumer, consumer) self.assertEquals(found[0].splice_server, splice_server) self.assertEquals(found[0].instance_identifier, inst_id) self.assertEquals(found[0].date.year, date.year) self.assertEquals(found[0].date.month, date.month) self.assertEquals(found[0].date.day, date.day) # TODO: Fix timezone issue, we are doing something wrong and not handling timezones correctly # self.assertEquals(found[0].date.hour, date.hour) self.assertEquals(found[0].date.minute, date.minute) self.assertEquals(found[0].date.second, date.second) self.assertEquals(found[0].allowed_product_info, allowed_product_info) self.assertEquals(found[0].unallowed_product_info, unallowed_product_info) self.assertEquals(found[0].facts, facts)
def test_create_simple_product_usage(self): self.assertEquals(len(ProductUsage.objects()), 0) consumer = "consumer_uuid" splice_server = "splice_server_uuid" inst_id = "mac_addr" date = datetime.now(tzutc()) pu = self.create_product_usage(consumer, splice_server, inst_id, date) pu.save() self.assertEquals(len(ProductUsage.objects()), 1)
def record_rhic_usage(self, rhic, inst_index, usage_datetime, splice_server): """ Record one record of a RHIC usage. """ pu = ProductUsage( consumer=rhic.uuid, splice_server=splice_server, instance_identifier=rhic.instance_identifiers[inst_index], allowed_product_info=rhic.engineering_ids, facts=rhic.instance_facts[inst_index], date=usage_datetime) pu.save()
def create_product_usage_data(self, addr, num, save=True): retval = [] for index in range(0, num): pu = ProductUsage() pu.splice_server = addr pu.consumer = "consumer_uuid" pu.date = datetime.now(tzutc()) - timedelta(hours=num-index) pu.instance_identifier = "instance_identifier" pu.allowed_product_info = ["1"] pu.unallowed_product_info = ["0"] pu.facts = {"tbd":"values"} if save: pu.save() retval.append(pu) return retval
def test_duplicate_product_usage_not_allowed(self): self.assertEquals(len(ProductUsage.objects()), 0) consumer = "consumer_uuid" splice_server = "splice_server_uuid" inst_id = "mac_addr" date = datetime.now(tzutc()) pu_a = self.create_product_usage(consumer, splice_server, inst_id, date) pu_a.save() self.assertEquals(len(ProductUsage.objects()), 1) caught = False pu_b = self.create_product_usage(consumer, splice_server, inst_id, date) try: pu_b.save() except OperationError, e: caught = True
def record_usage(self, identity, consumer_identifier, facts, allowed_products, unallowed_products): """ @param identity consumer's identity @type identity: str @param consumer_identifier means of uniquely identifying different instances with same consumer identity an example could be a mac address @type consumer_identifier: str @param facts system facts @type facts: {} @param allowed_products: list of product ids that are installed and entitled for usage by consumer @type allowed_products: [str] @param unallowed_products: list of product ids that are installed but _not_ entitled for usage by consumer @type unallowed_products: [str] """ try: sanitized_facts = utils.sanitize_dict_for_mongo(facts) _LOG.info("Record usage for '%s' with " "allowed_products '%s', " "unallowed_products '%s' " "on instance with identifier '%s' and facts <%s>" %\ (identity, allowed_products, unallowed_products, consumer_identifier, sanitized_facts)) consumer_uuid_str = str(identity.uuid) prod_usage = ProductUsage( consumer=consumer_uuid_str, splice_server=self.get_this_server().uuid, instance_identifier=consumer_identifier, allowed_product_info=allowed_products, unallowed_product_info=unallowed_products, facts=sanitized_facts, date=datetime.now()) prod_usage.save() except Exception, e: _LOG.exception(e)
def test_uploading_multiple_product_usages(self): consumer = "consumer_uuid" splice_server = "splice_server_uuid" date = datetime.now(tzutc()) allowed_product_info = ["1", "2", "3"] unallowed_product_info = ["0"] facts = {"tbd": "values"} pu_a = self.create_product_usage(consumer, splice_server, "instance_a", date, allowed_product_info=allowed_product_info, unallowed_product_info=unallowed_product_info, facts=facts) pu_b = self.create_product_usage(consumer, splice_server, "instance_b", date, allowed_product_info=allowed_product_info, unallowed_product_info=unallowed_product_info, facts=facts) self.assertEquals(len(ProductUsage.objects()), 0) post_data = [pu_a._data, pu_b._data] LOG.info("Calling api for productusage import with post data: '%s'" % (post_data)) resp = self.api_client.post('/api/v1/productusage/', format='json', data=post_data, SSL_CLIENT_CERT=self.expected_valid_splice_server_identity_pem) LOG.info("Response for productusage import: Status Code: %s, Response: %s" % (resp.status_code, resp)) self.assertEquals(resp.status_code, 202) # Now check that the server api saved the object as expected found = ProductUsage.objects() self.assertEquals(len(found), 2)
def create_product_usage(self, consumer, splice_server, instance_identifier, date=None, allowed_product_info=None, unallowed_product_info=None, facts=None): if not date: date = datetime.now(tzutc()) if not allowed_product_info: allowed_product_info = ["1", "2", "3"] if not unallowed_product_info: unallowed_product_info = [] if not facts: facts = {"tbd": "values"} pu = ProductUsage() pu.consumer = consumer pu.splice_server = splice_server pu.date = date pu.instance_identifier = instance_identifier pu.allowed_product_info = allowed_product_info pu.unallowed_product_info = unallowed_product_info pu.facts = facts return pu
def _get_product_usage_data(addr, limit): """ Returns product usage data which has not yet been uploaded to 'addr' @param addr: remote server to upload data to @param limit: max amount of objects to process per request @return: list of product usage objects ordered by date """ #TODO: # - Modify query to not fetch the "tracker" field this way it is always blank prod_usage_data = ProductUsage.objects(tracker__nin=[addr]) prod_usage_data = prod_usage_data.order_by("date") if limit: prod_usage_data = prod_usage_data.limit(limit) # Keep 'tracker' information private to this server for pu in prod_usage_data: pu.tracker = [] # _LOG.info("Retrieved %s items to send to %s" % (len(prod_usage_data), addr)) return prod_usage_data
def post_list(self, request, **kwargs): if not request.raw_post_data: _LOG.info("Empty body in request") return http.HttpBadRequest("Empty body in request") try: raw_post_data = request.raw_post_data _LOG.info("ProductUsageResource::post_list() processing %s KB." % (len(request.raw_post_data)/1024.0)) if request.META.has_key("HTTP_CONTENT_ENCODING") and request.META["HTTP_CONTENT_ENCODING"] == "gzip": start_unzip = time.time() data = StringIO.StringIO(raw_post_data) gzipper = gzip.GzipFile(fileobj=data) raw_post_data = gzipper.read() end_unzip = time.time() _LOG.info("ProductUsageResource::post_list() uncompressed %s KB to %s KB in %s seconds" % \ (len(request.raw_post_data)/float(1024), len(raw_post_data)/float(1024), end_unzip - start_unzip)) a = time.time() product_usage = json.loads(raw_post_data, object_hook=json_util.object_hook) if isinstance(product_usage, dict): product_usage = [product_usage] pu_models = [ProductUsage._from_son(p) for p in product_usage] for pu in pu_models: if isinstance(pu.date, basestring): # We must convert from str to datetime for ReportServer to be able to process this data pu.date = utils.convert_to_datetime(pu.date) b = time.time() items_not_imported = self.import_hook(pu_models) c = time.time() _LOG.info("ProductUsageResource::post_list() Total Time: %s, %s seconds to convert %s KB to JSON. " "%s seconds to import %s objects into mongo with %s errors." % (c-a, b-a, len(raw_post_data)/1024.0, c-b, len(pu_models), items_not_imported)) if not items_not_imported: return http.HttpAccepted() else: return http.HttpConflict(items_not_imported) except Exception, e: _LOG.exception("Unable to process request with %s bytes in body" % (len(raw_post_data))) _LOG.info("Snippet of failed request body: \n%s\n" % (raw_post_data[:8*1024])) return http.HttpBadRequest(e)
def _unmark_sent(object_ids, addr): for oid in object_ids: ProductUsage.objects(id=oid).update(pull__tracker=addr)
def _mark_sent(object_ids, addr): for oid in object_ids: ProductUsage.objects(id=oid).update(add_to_set__tracker=addr)
def test_tracker_for_product_usage_prevents_duplicate_server_entries(self): self.assertEqual(len(ProductUsage.objects()), 0) pu = ProductUsage() pu.consumer = "consumer_uuid" pu.splice_server = "splice server uuid" pu.date = datetime.now(tzutc()) pu.instance_identifier = "mac addr" pu.allowed_product_info = ["1"] pu.unallowed_product_info = [] pu.facts = {"key":"value"} self.assertEqual(pu.tracker, []) pu.tracker.append("a.example.com") pu.tracker.append("a.example.com") pu.tracker.append("b.example.com") self.assertEqual(len(pu.tracker), 3) pu.save() self.assertEqual(len(pu.tracker), 2) found = ProductUsage.objects() self.assertEqual(len(found), 1) self.assertEqual(len(found[0].tracker), 2) self.assertIn("a.example.com", found[0].tracker) self.assertIn("b.example.com", found[0].tracker)
def upload_product_usage_data(host, port, url, pu_data, accept_gzip=False, gzip_body=False): status, data = send_data(host, port, url, pu_data, accept_gzip=accept_gzip, gzip_body=gzip_body) if status not in [200, 202, 204]: raise RequestException(status, data) return status, data if __name__ == "__main__": from datetime import datetime from dateutil.tz import tzutc from splice.common.models import ProductUsage # Create a dummy product usage object pu = ProductUsage() pu.consumer = "test_consumer" pu.splice_server = "test_splice_server" pu.instance_identifier = "test_instance_identifier" pu.allowed_product_info = ["1", "2", "3", "4"] pu.unallowed_product_info = ["100"] pu.facts = {"tbd": "values"} pu.date = datetime.now(tzutc()) config.init(settings.SPLICE_CONFIG_FILE) cfg = config.get_reporting_config_info() if cfg["servers"]: remote_server = cfg["servers"][0] else: remote_server = ("127.0.0.1", "443", "/splice/api/v1/productusage/") host = remote_server[0]