예제 #1
0
 def test_hours_per_consumer(self):
     ReportData.drop_collection()
     count = 0
     for key, value in products_dict.items():
         count += 1
         entry = TestData.create_entry(key, mem_high=True)
         entry.save(safe=True)
         lookup = len(ReportData.objects.all())
         self.assertEqual(lookup, count)
     
     end = datetime.now()
     delta=timedelta(days=1)
     start = datetime.now() - delta
     
     list_of_rhics = RHIC.objects.all()
     results = hours_per_consumer(start, end, list_of_rhics )
     
     # right now products_dict RHEL and JBoss are sharing a RHIC so.. -1 on length
     self.assertEqual(len(results), (len(products_dict) -1 ), "correct number of results returned")
     results_product_list = []
     for r in results:
         self.assertEqual(r[0]['nau'], '1', "number of checkins is accurate")
         results_product_list.append(r[0]['product_name'])
     
     intersect = set(results_product_list).intersection(products_dict.keys())
     self.assertEqual(len(intersect), (len(products_dict) -1), "number of products returned in results is accurate")
예제 #2
0
    def test_basic_mdu(self):
        """
        Basic test of mdu
        """
        ReportData.drop_collection()
        rhel_entry = TestData.create_entry(RHEL, mem_high=True)
        rhel_entry.save()            
        lookup = ReportData.objects.all()
        self.assertEqual(len(lookup), 1)          
        
        delta=timedelta(days=1)
        start = datetime.now() - delta
        end = datetime.now() + delta        
        
        

        filter_args = {
               "memtotal__gt": rhel_entry.memtotal - 1,
               "product": rhel_entry.product,
               "contract_id": rhel_entry.contract_id,
               "support": rhel_entry.support,
               "memtotal__lt": rhel_entry.memtotal + 1,
               "consumer_uuid": rhel_entry.consumer_uuid,
               "sla": rhel_entry.sla
        }
        
        args = {
            "start": start.strftime(constants.epoch),
            "end": end.strftime(constants.epoch),
            "filter_args_dict": filter_args,
            "description": {"Product": RHEL}
        }        
        test_dict = MaxUsage.get_MDU_MCU(**args)
        result = test_dict['mdu'][1]
        self.assertEqual(result[1], 1, "correct mdu found")
예제 #3
0
    def test_generic_config_RHEL_JBOSS_same_rhic(self):
        ReportData.drop_collection()
        # create 1 RHEL, 2 JBoss
        entry_high = TestData.create_entry(RHEL, mem_high=True)
        entry_high.save(safe=True)
        
        entry_high = TestData.create_entry(JBoss, socket=5)
        entry_high.save(safe=True)
        entry_low = TestData.create_entry(JBoss, socket=4 )
        entry_low.save(safe=True)

        delta=timedelta(days=1)
        start = datetime.now() - delta
        end = datetime.now() + delta
        
        environment = "us-east-1"
        
        lookup = ReportData.objects.all()
        self.assertEqual(len(lookup), 3)
        #test for RHEL Match
        rhic = RHIC.objects.filter(uuid=products_dict[RHEL][1])[0]
        p = Product.objects.filter(name=RHEL, sla=rhic.sla,
                                   support_level=rhic.support_level)[0]
        results_dicts = Product_Def.get_count(p, rhic, start, end, rhic.contract, environment, report_biz_rules)
        self.assertEqual(len(results_dicts), 1)
        
        #test for JBoss match
        rhic = RHIC.objects.filter(uuid=products_dict[JBoss][1])[0]
        p = Product.objects.filter(name=JBoss, sla=rhic.sla,
                                   support_level=rhic.support_level)[0]
        results_dicts = Product_Def.get_count(p, rhic, start, end, rhic.contract, environment, report_biz_rules)
        results_dicts = Product_Def.get_count(p, rhic, start, end, rhic.contract, environment, report_biz_rules)
        self.assertEqual(len(results_dicts), 2)
예제 #4
0
 def test_RHEL_Host_negative(self):
     ReportData.drop_collection()
     entry_high = TestData.create_entry(UNLIMITED, socket=3)
     entry_high.save(safe=True)
     entry_low = TestData.create_entry(UNLIMITED, socket=3, mem_high=True )
     entry_low.save(safe=True)
     
     self.check_product_result(1, 1)
예제 #5
0
 def test_OpenShift_Gear_negative(self):
     ReportData.drop_collection()
     entry_high = TestData.create_entry(GEAR, cpu=2, mem_high=True)
     entry_high.save(safe=True)
     entry_low = TestData.create_entry(GEAR, cpu=3, mem_high=True)
     entry_low.save(safe=True)
     
     self.check_product_result(1, 1)
예제 #6
0
 def test_JBoss_vcpu_negative(self):
     ReportData.drop_collection()
     entry_high = TestData.create_entry(JBoss, socket=5)
     entry_high.save(safe=True)
     entry_low = TestData.create_entry(JBoss, socket=5, mem_high=True )
     entry_low.save(safe=True)
     
     self.check_product_result(1, 1)
예제 #7
0
 def test_RHEL_memory_negative(self):
     ReportData.drop_collection()
     
     entry_high = TestData.create_entry(RHEL, mem_high=True)
     entry_high.save(safe=True)
     entry_low = TestData.create_entry(RHEL, mem_high=True, socket=12)
     entry_low.save(safe=True)
     
     self.check_product_result(1, 1)
예제 #8
0
    def test_advanced_mcu_mdu(self):
                """
                three report data entries, each w/ unique instance_ident
                2 in the same $hour
                1 in $hour + 1
                mcu = 2 , mdu = 3
                """
                delta_day=timedelta(days=1)
                delta_hour=timedelta(hours=1)
                start = datetime.now() - delta_day
                end = datetime.now() + delta_day  
                hour_plus_1 = datetime.now() + delta_hour
                
                ReportData.drop_collection()
                rhel_entry = TestData.create_entry(RHEL,
                                                   instance_identifier="00:10")
                rhel_entry.save()
                rhel_entry = TestData.create_entry(RHEL,
                                                   instance_identifier="00:11")
                rhel_entry.save() 
                rhel_entry = TestData.create_entry(RHEL,
                                                   date=hour_plus_1,
                                                   instance_identifier="00:12")
                rhel_entry.save()                 
                lookup = ReportData.objects.all()
                self.assertEqual(len(lookup), 3)          

                filter_args = {
                       "memtotal__gt": rhel_entry.memtotal - 1,
                       "product": rhel_entry.product,
                       "contract_id": rhel_entry.contract_id,
                       "support": rhel_entry.support,
                       "memtotal__lt": rhel_entry.memtotal + 1,
                       "consumer_uuid": rhel_entry.consumer_uuid,
                       "sla": rhel_entry.sla
                }
                
                args = {
                    "start": start.strftime(constants.epoch),
                    "end": end.strftime(constants.epoch),
                    "filter_args_dict": filter_args,
                    "description": {"Product": RHEL}
                }
                
                
                test_dict = MaxUsage.get_MDU_MCU(**args)
                mdu = test_dict['mdu'][1]
                mcu = test_dict['mcu'][1]
                self.assertEqual(mdu[1], 3, "correct mdu found") 
                self.assertEqual(mcu[1], 2, "correct mcu found") 
예제 #9
0
def main():
    SpliceServer.drop_collection()
    ProductUsage.drop_collection()
    ReportData.drop_collection()
    ss1 = TestData.create_splice_server("test01", "east")
    
    uuid_rhel_jboss = products_dict[RHEL][1]
    prod_rhel = products_dict[RHEL][0]
    prod_jboss = products_dict[JBoss][0]
    
    uuid_ha = products_dict[HA][1]
    prod_ha = products_dict[HA][0]
    
    uuid_edu = products_dict[EDU][1]
    prod_edu = products_dict[EDU][0]
    
    now = datetime.now()
    delta_day = timedelta(days=4)
    delta_hour = timedelta(days=3)
    
    
    for i in range(1, 10):
        this_time = now - (delta_hour * i)
        create_set_of_usage(
            prod_rhel, uuid_rhel_jboss, get_times(this_time), ss1, 7)
    
    for i in range(1, 10):
        this_time = now - (delta_hour * i)
        create_set_of_usage(
            prod_jboss, uuid_rhel_jboss, get_times(this_time), ss1, 4)
    
    for i in range(1, 4):
        this_time = now - (delta_hour * i)
        create_set_of_usage(prod_ha, uuid_ha, get_times(this_time), ss1, 7)
    
    for i in range(5, 10):
        this_time = now - (delta_hour * i)
        create_set_of_usage(
            prod_edu, uuid_edu, get_times(this_time), ss1, 5)
    # run import
    results = import_data(force_import=True)
    
    # verify 1 items in db
    lookup = ReportData.objects.all()
예제 #10
0
    def test_generic_config_RHEL(self):
        ReportData.drop_collection()        
        entry_high = TestData.create_entry(RHEL, mem_high=True)
        entry_high.save(safe=True)

        delta=timedelta(days=1)
        start = datetime.now() - delta
        end = datetime.now() + delta
        contract_num = "3116649"
        environment = "us-east-1"
        
        lookup = ReportData.objects.all()
        self.assertEqual(len(lookup), 1)
        #test perfect match
        p = Product.objects.filter(name=RHEL)[0]
        #print(products_dict[RHEL][1])
        rhic = RHIC.objects.filter(uuid=products_dict[RHEL][1])[0]
        results_dicts = Product_Def.get_count(p, rhic, start, end, contract_num, environment, report_biz_rules)
        self.assertEqual(len(results_dicts), 1)
예제 #11
0
 def test_find_each_product(self):
     ReportData.drop_collection()
     count = 0
     for key, value in products_dict.items():
         count += 1
         entry = TestData.create_entry(key, mem_high=True)
         entry.save(safe=True)
         lookup = len(ReportData.objects.all())
         self.assertEqual(lookup, count)
         
         
     end = datetime.now()
     delta=timedelta(days=1)
     start = datetime.now() - delta
     
     for key, value in products_dict.items():
         rhic = RHIC.objects.filter(uuid=value[1])[0]
         p = Product.objects.filter(name=key, sla=rhic.sla,
                                    support_level=rhic.support_level)[0]
         results_dicts = Product_Def.get_count(p, rhic, start, end, rhic.contract, "us-east-1", report_biz_rules)
         self.assertEqual(len(results_dicts), 1)
예제 #12
0
def import_data(product_usage=[],
                checkin_interval=1,
                from_splice_server="NA",
                force_import=False):
    """
    @param product_usage
    @type mongoengine cursor

    @param checkin_interval
    @type int
    @description the interval between client checkins range 1-23, 24=daily

    @return dict w/ keys start, end to measure how long the import took
    @rtype dict
    """
    # config fail/pass on missing rhic
    if not product_usage:
        product_usage = ProductUsage.objects.all()
    start_stop_time = []
    quarantined = []
    #total_import_count = len(product_usage)
    #remaining_import_count = total_import_count

    # debug
    start = datetime.utcnow()
    
    #provide a way to throttle how often import can be run
    time = {}
    time['start'] = start.strftime(constants.full_format)

    time_now = datetime.utcnow()
    threshold = 0
    if config.CONFIG.has_option('import', 'quiet_period'):
        threshold = int(config.CONFIG.get('import', 'quiet_period'))
    last_import_threshhold = time_now - timedelta(minutes=threshold)
    last_import = ImportHistory.objects.filter(
        date__gt=last_import_threshhold).count()
    
    if last_import > 0 and force_import == False:
        time['end'] = -1 # -1 send import skipped message
        start_stop_time.append(time)
        _LOG.info("import skipped")
        return [], start_stop_time
    else:
        record = ImportHistory(date=start, splice_server=from_splice_server)
        record.save()

    # committing every 100 records instead of every 1 record saves about 5
    # seconds.
    cached_rhics = {}
    cached_contracts = {}

    for pu in product_usage:
        uuid = pu.consumer
        
        #BEGIN SANITIZE THE PU DATA 
        if not pu.allowed_product_info:
            _LOG.critical('product usuage object does not have any allowed products (engineering ids)')
            continue
                
        try:
            splice_server = SpliceServer.objects.get(uuid=pu.splice_server)
            _LOG.info('splice server = ' + splice_server.hostname)
        except Exception:
            _LOG.critical('splice server named: ' + str(pu.splice_server) + ' not found')
            continue        
        
        
        if uuid in cached_rhics:
            rhic = cached_rhics[uuid]
        else:
            try:
                _LOG.info('using RHIC: ' + uuid)
                rhic = RHIC.objects.filter(uuid=uuid)[0]
                cached_rhics[uuid] = rhic
            except IndexError:
                _LOG.critical('rhic not found @ import: ' + uuid)
                _LOG.critical('product usuage object will NOT be imported')
                continue
        
        #END SANITIZE THE PU DATA 

        account = Account.objects(account_id=rhic.account_id).only('contracts').first()

        contract = None
        if rhic.contract in cached_contracts:
            contract = cached_contracts['rhic.contract']
        else:
            for c in account.contracts:
                if c.contract_id == rhic.contract:
                    cached_contracts['rhic.contract'] = c
                    contract = c
                    break

        # Set of used engineering ids for this checkin
        product_set = Set(pu.allowed_product_info)
        

        # Iterate over each product in the contract, see if it matches sla and
        # support level, and consumed engineering ids.  If so, save an instance
        # of ReportData
        for product in contract.products:
            # Match on sla and support level
            if not (product.sla == rhic.sla and
                    product.support_level == rhic.support_level):
                continue

            # Set of engineering ids for this product.
            product_eng_id_set = set(product.engineering_ids)

            # If the set of engineering ids for the product is a subset of the
            # used engineering ids for this checkin, create an instance of
            # ReportData, check for dupes, and save the instance.
            if product_eng_id_set.issubset(product_set):
                # This line isn't technically necessary, but it improves
                # performance by making the set we need to search smaller each
                # time.
                product_set.difference_update(product_eng_id_set)
                splice_server = SpliceServer.objects.get(uuid=pu.splice_server)

                for interval in range(checkin_interval):
                    td = timedelta(hours=interval)
                    this_time = pu.date + td
                    rd = ReportData(
                        instance_identifier=str(pu.instance_identifier),
                        consumer=rhic.name,
                        consumer_uuid=uuid,
                        product=product.engineering_ids,
                        product_name=product.name,
                        date=this_time,
                        hour=this_time.strftime(
                            constants.hr_fmt),
                        day=pu.date.strftime(
                            constants.day_fmt),
                        sla=product.sla,
                        support=product.support_level,
                        contract_id=rhic.contract,
                        contract_use=str(product.quantity),
                        memtotal=int(
                            pu.facts['memory_dot_memtotal']),
                        cpu_sockets=int(
                            pu.facts['lscpu_dot_cpu_socket(s)']),
                        cpu=int(
                            pu.facts['lscpu_dot_cpu(s)']),
                        environment=str(
                            splice_server.environment),
                        splice_server=str(
                            splice_server.hostname),
                        duplicate=interval,
                        record_identifier=rhic.name + 
                            str(pu.instance_identifier) +
                            pu.date.strftime(constants.day_fmt) +
                            product.name
                    )

                    try:
                        rd.save(safe=True)
                        _LOG.info(
                            'recording: ' + str(product.engineering_ids))
                    except NotUniqueError:
                        _LOG.info("Ignorning NotUniqueError for: %s" % (rd))
                    except OperationError as oe:
                        _LOG.info("could not import:" +
                                  str(pu) + "Exception: " + str(oe))
                        quarantined.append(rd)

    end = datetime.utcnow()
    time['end'] = end.strftime(constants.full_format)
    start_stop_time.append(time)
    _LOG.info('import complete')

    return quarantined, start_stop_time
예제 #13
0
 def drop_collections(self):
     ReportData.drop_collection()
예제 #14
0
 def drop_report_data(self):
     ReportData.drop_collection()
예제 #15
0
 def drop_collections(self):
     ReportData.drop_collection()
     QuarantinedReportData.drop_collection()