def test_uploading_simple_product_usage(self):
consumer = "consumer_uuid"
splice_server = "splice_server_uuid"
inst_id = "mac_addr"
date = datetime.now(tzutc())
allowed_product_info = ["1", "2", "3"]
unallowed_product_info = ["0"]
facts = {"tbd": "values"}
pu_a = self.create_product_usage(consumer, splice_server, inst_id, date,
allowed_product_info=allowed_product_info, unallowed_product_info=unallowed_product_info, facts=facts)
self.assertEquals(len(ProductUsage.objects()), 0)
post_data = [pu_a._data]
LOG.info("Calling api for productusage import with post data: '%s'" % (post_data))
resp = self.api_client.post('/api/v1/productusage/', format='json', data=post_data,
SSL_CLIENT_CERT=self.expected_valid_splice_server_identity_pem)
LOG.info("Response for productusage import: Status Code: %s, Response: %s" % (resp.status_code, resp))
self.assertEquals(resp.status_code, 202)
# Now check that the server api saved the object as expected
found = ProductUsage.objects()
self.assertEquals(len(found), 1)
self.assertEquals(found[0].consumer, consumer)
self.assertEquals(found[0].splice_server, splice_server)
self.assertEquals(found[0].instance_identifier, inst_id)
self.assertEquals(found[0].date.year, date.year)
self.assertEquals(found[0].date.month, date.month)
self.assertEquals(found[0].date.day, date.day)
# TODO: Fix timezone issue, we are doing something wrong and not handling timezones correctly
# self.assertEquals(found[0].date.hour, date.hour)
self.assertEquals(found[0].date.minute, date.minute)
self.assertEquals(found[0].date.second, date.second)
self.assertEquals(found[0].allowed_product_info, allowed_product_info)
self.assertEquals(found[0].unallowed_product_info, unallowed_product_info)
self.assertEquals(found[0].facts, facts)
def test_create_simple_product_usage(self):
self.assertEquals(len(ProductUsage.objects()), 0)
consumer = "consumer_uuid"
splice_server = "splice_server_uuid"
inst_id = "mac_addr"
date = datetime.now(tzutc())
pu = self.create_product_usage(consumer, splice_server, inst_id, date)
pu.save()
self.assertEquals(len(ProductUsage.objects()), 1)
def record_rhic_usage(self, rhic, inst_index, usage_datetime,
splice_server):
"""
Record one record of a RHIC usage.
"""
pu = ProductUsage(
consumer=rhic.uuid, splice_server=splice_server,
instance_identifier=rhic.instance_identifiers[inst_index],
allowed_product_info=rhic.engineering_ids,
facts=rhic.instance_facts[inst_index], date=usage_datetime)
pu.save()
def create_product_usage_data(self, addr, num, save=True):
retval = []
for index in range(0, num):
pu = ProductUsage()
pu.splice_server = addr
pu.consumer = "consumer_uuid"
pu.date = datetime.now(tzutc()) - timedelta(hours=num-index)
pu.instance_identifier = "instance_identifier"
pu.allowed_product_info = ["1"]
pu.unallowed_product_info = ["0"]
pu.facts = {"tbd":"values"}
if save:
pu.save()
retval.append(pu)
return retval
def test_duplicate_product_usage_not_allowed(self):
self.assertEquals(len(ProductUsage.objects()), 0)
consumer = "consumer_uuid"
splice_server = "splice_server_uuid"
inst_id = "mac_addr"
date = datetime.now(tzutc())
pu_a = self.create_product_usage(consumer, splice_server, inst_id, date)
pu_a.save()
self.assertEquals(len(ProductUsage.objects()), 1)
caught = False
pu_b = self.create_product_usage(consumer, splice_server, inst_id, date)
try:
pu_b.save()
except OperationError, e:
caught = True
def record_usage(self, identity, consumer_identifier, facts, allowed_products, unallowed_products):
"""
@param identity consumer's identity
@type identity: str
@param consumer_identifier means of uniquely identifying different instances with same consumer identity
an example could be a mac address
@type consumer_identifier: str
@param facts system facts
@type facts: {}
@param allowed_products: list of product ids that are
installed and entitled for usage by consumer
@type allowed_products: [str]
@param unallowed_products: list of product ids that are
installed but _not_ entitled for usage by consumer
@type unallowed_products: [str]
"""
try:
sanitized_facts = utils.sanitize_dict_for_mongo(facts)
_LOG.info("Record usage for '%s' with "
"allowed_products '%s', "
"unallowed_products '%s' "
"on instance with identifier '%s' and facts <%s>" %\
(identity, allowed_products, unallowed_products,
consumer_identifier, sanitized_facts))
consumer_uuid_str = str(identity.uuid)
prod_usage = ProductUsage(
consumer=consumer_uuid_str,
splice_server=self.get_this_server().uuid,
instance_identifier=consumer_identifier,
allowed_product_info=allowed_products,
unallowed_product_info=unallowed_products,
facts=sanitized_facts,
date=datetime.now())
prod_usage.save()
except Exception, e:
_LOG.exception(e)
def test_uploading_multiple_product_usages(self):
consumer = "consumer_uuid"
splice_server = "splice_server_uuid"
date = datetime.now(tzutc())
allowed_product_info = ["1", "2", "3"]
unallowed_product_info = ["0"]
facts = {"tbd": "values"}
pu_a = self.create_product_usage(consumer, splice_server, "instance_a", date,
allowed_product_info=allowed_product_info, unallowed_product_info=unallowed_product_info, facts=facts)
pu_b = self.create_product_usage(consumer, splice_server, "instance_b", date,
allowed_product_info=allowed_product_info, unallowed_product_info=unallowed_product_info, facts=facts)
self.assertEquals(len(ProductUsage.objects()), 0)
post_data = [pu_a._data, pu_b._data]
LOG.info("Calling api for productusage import with post data: '%s'" % (post_data))
resp = self.api_client.post('/api/v1/productusage/', format='json', data=post_data,
SSL_CLIENT_CERT=self.expected_valid_splice_server_identity_pem)
LOG.info("Response for productusage import: Status Code: %s, Response: %s" % (resp.status_code, resp))
self.assertEquals(resp.status_code, 202)
# Now check that the server api saved the object as expected
found = ProductUsage.objects()
self.assertEquals(len(found), 2)
def create_product_usage(self, consumer, splice_server, instance_identifier, date=None,
allowed_product_info=None, unallowed_product_info=None, facts=None):
if not date:
date = datetime.now(tzutc())
if not allowed_product_info:
allowed_product_info = ["1", "2", "3"]
if not unallowed_product_info:
unallowed_product_info = []
if not facts:
facts = {"tbd": "values"}
pu = ProductUsage()
pu.consumer = consumer
pu.splice_server = splice_server
pu.date = date
pu.instance_identifier = instance_identifier
pu.allowed_product_info = allowed_product_info
pu.unallowed_product_info = unallowed_product_info
pu.facts = facts
return pu
def _get_product_usage_data(addr, limit):
"""
Returns product usage data which has not yet been uploaded to 'addr'
@param addr: remote server to upload data to
@param limit: max amount of objects to process per request
@return: list of product usage objects ordered by date
"""
#TODO:
# - Modify query to not fetch the "tracker" field this way it is always blank
prod_usage_data = ProductUsage.objects(tracker__nin=[addr])
prod_usage_data = prod_usage_data.order_by("date")
if limit:
prod_usage_data = prod_usage_data.limit(limit)
# Keep 'tracker' information private to this server
for pu in prod_usage_data:
pu.tracker = [] #
_LOG.info("Retrieved %s items to send to %s" % (len(prod_usage_data), addr))
return prod_usage_data
def post_list(self, request, **kwargs):
if not request.raw_post_data:
_LOG.info("Empty body in request")
return http.HttpBadRequest("Empty body in request")
try:
raw_post_data = request.raw_post_data
_LOG.info("ProductUsageResource::post_list() processing %s KB." % (len(request.raw_post_data)/1024.0))
if request.META.has_key("HTTP_CONTENT_ENCODING") and request.META["HTTP_CONTENT_ENCODING"] == "gzip":
start_unzip = time.time()
data = StringIO.StringIO(raw_post_data)
gzipper = gzip.GzipFile(fileobj=data)
raw_post_data = gzipper.read()
end_unzip = time.time()
_LOG.info("ProductUsageResource::post_list() uncompressed %s KB to %s KB in %s seconds" % \
(len(request.raw_post_data)/float(1024),
len(raw_post_data)/float(1024), end_unzip - start_unzip))
a = time.time()
product_usage = json.loads(raw_post_data, object_hook=json_util.object_hook)
if isinstance(product_usage, dict):
product_usage = [product_usage]
pu_models = [ProductUsage._from_son(p) for p in product_usage]
for pu in pu_models:
if isinstance(pu.date, basestring):
# We must convert from str to datetime for ReportServer to be able to process this data
pu.date = utils.convert_to_datetime(pu.date)
b = time.time()
items_not_imported = self.import_hook(pu_models)
c = time.time()
_LOG.info("ProductUsageResource::post_list() Total Time: %s, %s seconds to convert %s KB to JSON. "
"%s seconds to import %s objects into mongo with %s errors." % (c-a, b-a,
len(raw_post_data)/1024.0, c-b, len(pu_models), items_not_imported))
if not items_not_imported:
return http.HttpAccepted()
else:
return http.HttpConflict(items_not_imported)
except Exception, e:
_LOG.exception("Unable to process request with %s bytes in body" % (len(raw_post_data)))
_LOG.info("Snippet of failed request body: \n%s\n" % (raw_post_data[:8*1024]))
return http.HttpBadRequest(e)
def _unmark_sent(object_ids, addr):
for oid in object_ids:
ProductUsage.objects(id=oid).update(pull__tracker=addr)
def _mark_sent(object_ids, addr):
for oid in object_ids:
ProductUsage.objects(id=oid).update(add_to_set__tracker=addr)
def test_tracker_for_product_usage_prevents_duplicate_server_entries(self):
self.assertEqual(len(ProductUsage.objects()), 0)
pu = ProductUsage()
pu.consumer = "consumer_uuid"
pu.splice_server = "splice server uuid"
pu.date = datetime.now(tzutc())
pu.instance_identifier = "mac addr"
pu.allowed_product_info = ["1"]
pu.unallowed_product_info = []
pu.facts = {"key":"value"}
self.assertEqual(pu.tracker, [])
pu.tracker.append("a.example.com")
pu.tracker.append("a.example.com")
pu.tracker.append("b.example.com")
self.assertEqual(len(pu.tracker), 3)
pu.save()
self.assertEqual(len(pu.tracker), 2)
found = ProductUsage.objects()
self.assertEqual(len(found), 1)
self.assertEqual(len(found[0].tracker), 2)
self.assertIn("a.example.com", found[0].tracker)
self.assertIn("b.example.com", found[0].tracker)
def upload_product_usage_data(host, port, url, pu_data, accept_gzip=False, gzip_body=False):
status, data = send_data(host, port, url, pu_data, accept_gzip=accept_gzip, gzip_body=gzip_body)
if status not in [200, 202, 204]:
raise RequestException(status, data)
return status, data
if __name__ == "__main__":
from datetime import datetime
from dateutil.tz import tzutc
from splice.common.models import ProductUsage
# Create a dummy product usage object
pu = ProductUsage()
pu.consumer = "test_consumer"
pu.splice_server = "test_splice_server"
pu.instance_identifier = "test_instance_identifier"
pu.allowed_product_info = ["1", "2", "3", "4"]
pu.unallowed_product_info = ["100"]
pu.facts = {"tbd": "values"}
pu.date = datetime.now(tzutc())
config.init(settings.SPLICE_CONFIG_FILE)
cfg = config.get_reporting_config_info()
if cfg["servers"]:
remote_server = cfg["servers"][0]
else:
remote_server = ("127.0.0.1", "443", "/splice/api/v1/productusage/")
host = remote_server[0]
