def __init__(self, platform, startdate, enddate): self.startdate = startdate self.enddate = enddate self.platform = platform self.reports = MongoConnector('reports').collection self.stream_logger = StreamLogger.getLogger(__name__) self.reporter_client = ReporterClient(self.platform)
def __init__(self, platform, date, mode='prod'): super().__init__() self.reports = MongoConnector('reports').collection self.date = date self.platform = platform self.publisher_fees = publisher_fees() self.inventory = inventory() self.platforms = platforms() self.reports_table = 'reports_e2e_tests' if mode == 'test' else 'reports_e2e_tests' self.insert_values = [] self.messages = []
def load(self, platform=None): reports = MongoConnector('reports').collection filter_dict = {'platform':platform} if platform else {} reports.delete_many(filter_dict) pl_table = [platform] if platform else self.map_table for pl in pl_table: mdata = self._get_mysql_data(pl) #print(pl) try: reports.insert_many(mdata) except BulkWriteError as bws: exc_type, exc_value, exc_traceback = sys.exc_info()
class ReporterPusher(SSH_Client): def __init__(self, platform, date, mode='prod'): super().__init__() self.reports = MongoConnector('reports').collection self.date = date self.platform = platform self.publisher_fees = publisher_fees() self.inventory = inventory() self.platforms = platforms() self.reports_table = 'reports_e2e_tests' if mode == 'test' else 'reports_e2e_tests' self.insert_values = [] self.messages = [] def is_float(self, str): if str is None: return False try: float(str) return True except ValueError: return False def build_inserts(self): ID_RX = re.compile(r'^(\d+)_.*$') SIZE_RX = re.compile(r'^.*_(\d+[xX]\d+)$') inventory_dict = self.inventory.get() publisher_fees = self.publisher_fees.get() platforms_ids = self.platforms.get() filters = {"date":self.date} projection = {"date":1, "platform":1, "placement_name":1, "clicks":1, "resold_impressions":1, "revenue_dict.EUR":1, "revenue_dict.USD":1} if self.platform != 'all': filters.update({ "platform": self.platform }) rows = self.reports.find(filters, projection) self.row_len = 0 self.id_reject_count = 0 self.size_reject_count = 0 self.inventory_reject_count = 0 self.publisher_fee_reject_count = 0 self.revenue_reject_count = 0 self.insert_values_eurohoops = 0 for row in rows: #print(row) placement_name = row['placement_name'] platform_name = row['platform'] # ignore appnexus placements if platform_name == 'appnexus': continue self.row_len += 1 if platform_name == 'taboola': #m_id = re.match(r'^.*#(\d+)$', placement_name) m_id = re.match(r'^.*-.(\d+)$', placement_name) # old naming convention for taboola if m_id is None: m_id = re.match(r'^.*#(\d+)$', placement_name) elif platform_name == 'teads': m_id = re.match(r'^.*_(\d+)$', placement_name) else: m_id = ID_RX.match(placement_name) if m_id is None: self.id_reject_count += 1 self.stream_logger.info('Platform %s. Rejecting "%s". Could_not_parse_placement_id' % (platform_name, placement_name)) self.messages.append('Platform {0}. Rejecting "{1}". Could_not_parse_placement_id'.format(platform_name, placement_name)) continue else: placement_id = m_id.group(1) m_size = SIZE_RX.match(placement_name) if m_size is None and platform_name != 'taboola' and platform_name != 'teads': self.size_reject_count += 1 self.stream_logger.info('Platform %s. Rejecting "%s". Could_not_parse_placement_size' % (platform_name, placement_name)) self.messages.append('Platform {0}. Rejecting "{1}". Could_not_parse_placement_size'.format(platform_name, placement_name)) continue else: if platform_name == 'taboola': m_id = re.match(r'^.*-(.+)$', placement_name) if m_id == None: # old naming convention for taboola if placement_name.startswith('Remnant'): placement_type = 'r' else: placement_type = 'p' else: placement_type = m_id.group(1)[0] if placement_type == 'r': #if placement_name.startswith('Remnant'): m_size = re.match(r'^.*#(\d+[xX]\d+)#.+$', placement_name) if m_size is None: self.size_reject_count += 1 self.stream_logger.info('Platform %s. Rejecting "%s". Could_not_parse_placement_size' % (platform_name, placement_name)) self.messages.append('Platform {0}. Rejecting "{1}". Could_not_parse_placement_size'.format(platform_name, placement_name)) continue placement_size = m_size.group(1) else: placement_size = 'Content Unit' elif platform_name == 'teads': placement_size = 'Magic Ads' else: placement_size = m_size.group(1) # Get insert row data from dailies view insert_row = { 'date': row['date'], 'size': placement_size if placement_size else None, 'imps': row['resold_impressions'] if 'resold_impressions' in row else 0, 'clicks': row['clicks'] if 'clicks' in row else 0, 'placement_id': placement_id, 'revenue_usd': row['revenue_dict']['USD'], } # Update insert row from inventory table plc_id_str = str(insert_row['placement_id']) if plc_id_str not in inventory_dict: self.inventory_reject_count += 1 self.stream_logger.info('Platform %s. Rejecting "%s". Could_not_find_it_in_inventory_table' % (platform_name, placement_name)) self.messages.append('Platform {0}. Rejecting "{1}". Could_not_find_it_in_inventory_table'.format(platform_name, placement_name)) continue else: insert_row.update(inventory_dict[plc_id_str]) # Check if we have a PA_MENA publisher and insert usd for revenue if inventory_dict[plc_id_str]['publisher_name'].startswith('PA_MENA'): insert_row['revenue'] = row['revenue_dict']['USD'] else: insert_row['revenue'] = row['revenue_dict']['EUR'] if not self.is_float(insert_row['revenue']): self.revenue_reject_count += 1 self.stream_logger.info('Platform %s. Rejecting "%s". revenue_is_not_float' % (platform_name, placement_name)) self.messages.append('Platform {0}. Rejecting "{1}". revenue_is_not_float'.format(platform_name, placement_name)) continue if insert_row['publisher_id'] not in publisher_fees: self.publisher_fee_reject_count += 1 self.stream_logger.info('Platform {0}. Rejecting "{1}". Could_not_find_fee_for_publisher_id_{2}'. format(platform_name, placement_name, insert_row['publisher_id'])) self.messages.append('Platform {0}. Rejecting "{1}". Could_not_find_fee_for_publisher_id_{2}'. format(platform_name, placement_name, insert_row['publisher_id'])) continue else: insert_row['revenue_est_net'] = publisher_fees[insert_row['publisher_id']][0] * float(insert_row['revenue']) # Update insert row with standard fields, buyer_member_id='2725', insert_row.update( { 'buyer_member_id': platforms_ids[platform_name] if platform_name != 'appnexus' else None, 'brand_id': '17', } ) # Append the insert value statement self.insert_values.append("\n('{date}', '{size}', '{imps}', '{clicks}', '{revenue}', '{revenue_usd}', '{revenue_est_net}', '{publisher_id}', '{site_id}', '{placement_id}', '{buyer_member_id}', '{brand_id}')".format(**insert_row)) # patch for Eurohoops if insert_row['publisher_id'] == '475297' and insert_row['site_id'] == '2551048': self.insert_values_eurohoops += 1 insert_row['publisher_id'] = '111111' insert_row['revenue_est_net'] = publisher_fees[insert_row['publisher_id']][0] * insert_row['revenue_est_net'] insert_row['revenue'] = insert_row['revenue_est_net'] / 0.70 insert_row['revenue_usd'] = '' # Append the insert value statement self.insert_values.append("\n('{date}', '{size}', '{imps}', '{clicks}', '{revenue}', '{revenue_usd}', '{revenue_est_net}', '{publisher_id}', '{site_id}', '{placement_id}', '{buyer_member_id}', '{brand_id}')".format(**insert_row)) @twpamonitorclient.context('Push Data To UI', 'push_data', {}) def push_dailies_to_ui(self, monitor): self.stream_logger.info('Begin push_dailies_to_ui') monitor.log("Start push for platform {0}".format(self.platform)) if self.platform is None: self.platform = 'all' # Parse inventory file to get inventory data inventory_dict = self.inventory.get() #print(inventory_dict) # Parse publisher fees file publisher_fees = self.publisher_fees.get() #print(publisher_fees) #exit(0) # Parse platforms file platforms_ids = self.platforms.get() self.stream_logger.debug('Publisher fees {0}'.format(publisher_fees)) # Scrap dates from publisher fees dictionary publisher_fees = { k: publisher_fees[k][0] for k in publisher_fees} self.stream_logger.debug('Publisher fees {0}'.format(publisher_fees)) self.stream_logger.debug('Platform ids {0}'.format(platforms_ids)) self.build_inserts() # Create file to upload if len(self.insert_values) > 0: with open('insert.sql', 'w') as fh: fh.write('INSERT INTO {0} ( date, size, imps, clicks, revenue, revenue_usd, revenue_est_net, publisher_id, site_id, placement_id, buyer_member_id, brand_id) VALUES {1};'.format(self.reports_table, ','.join(self.insert_values))) # Upload to server self.put_file('insert.sql','insert.sql') try: if self.platform == 'all': for pl_name, pl_id in platforms_ids.items(): self.do_cmd(self.mysql_prefix + r'-e"DELETE from %s WHERE buyer_member_id=%s AND brand_id=17 AND date=\"%s 00:00:00\" " ' % (self.reports_table, pl_id, self.date,)) else: if self.platform in platforms_ids: self.do_cmd(self.mysql_prefix + r'-e"DELETE from %s WHERE buyer_member_id=%s AND brand_id=17 AND date=\"%s 00:00:00\" " ' % (self.reports_table, platforms_ids[self.platform], self.date,)) self.do_cmd(self.mysql_prefix + '< insert.sql') #self.do_cmd('/usr/bin/php /var/www/agora/public_html/sync/make_home_mv.php') except GracefulError: return False # Finally do the logging self.stream_logger.info('Ran for date %s and platform %s' % (self.date, self.platform)) self.stream_logger.info('Found %s rows in vps database' % self.row_len) if self.id_reject_count > 0: self.stream_logger.info('Rejected %s rows due to placement id parsing error.' % self.id_reject_count) if self.size_reject_count > 0: self.stream_logger.info('Rejected %s rows due to size parsing error.' % self.size_reject_count) if self.inventory_reject_count > 0: self.stream_logger.info('Rejected %s rows due to inventory table lookup error.' % self.inventory_reject_count) if self.publisher_fee_reject_count > 0: self.stream_logger.info('Rejected %s rows due to publisher fee lookup error.' % self.publisher_fee_reject_count) if self.revenue_reject_count > 0: self.stream_logger.info('Rejected %s rows due to revenue value error.' % self.revenue_reject_count) self.inserted_values_to_ui = len(self.insert_values)-self.insert_values_eurohoops self.stream_logger.info('Inserted %s rows in agora database' % str(self.inserted_values_to_ui)) self.stream_logger.info('Project Agora UI update log end') self.stream_logger.info('End push_dailies_to_ui') results = { 'found': self.row_len, 'inserted': self.inserted_values_to_ui, 'inserted_eurohoops': self.insert_values_eurohoops, 'rejected_id': self.id_reject_count, 'rejected_size': self.size_reject_count, 'rejected_inventory': self.inventory_reject_count, 'rejected_pub_fee': self.publisher_fee_reject_count, 'rejected_revenue': self.revenue_reject_count, 'messages':self.messages } monitor.data({'push_statistics': results}) monitor.log("End push for platform {0}".format(self.platform)) return results
class ReporterPuller(): def __init__(self, platform, startdate, enddate): self.startdate = startdate self.enddate = enddate self.platform = platform self.reports = MongoConnector('reports').collection self.stream_logger = StreamLogger.getLogger(__name__) self.reporter_client = ReporterClient(self.platform) def date_to_mongoid(self, date, daytime=(12,0,0)): hours, mins, secs = daytime datetimeobj = datetime.datetime.combine(date, datetime.time(hours, mins, secs)) mongo_id = ObjectId.generate_from_datetime(datetimeobj) return mongo_id @twpamonitorclient.context('Pull Data From Platform', 'pull_data', {}) def pull_data(self, monitor): ''' def jdefault(o): return o.__dict__ ''' if PullModeKeeper.isMuted(self.platform): return False monitor.log("Start pull from platform {0}".format(self.platform)) db_insert_entries = self.reporter_client.read(self.startdate, self.enddate) #self.stream_logger.info('len of db insert entries:{0}'.format(len(db_insert_entries))) monitor.data({'data_pulled_len': len(db_insert_entries)}) # just to produce the data to mock the above function ''' l = {str(k):v for k,v in db_insert_entries.items()} print(json.dumps(l, default=jdefault)) #print(db_insert_entries) exit() ''' # if db_insert_entries is empty means # an error occured if not db_insert_entries: return False start_mongo_id = self.date_to_mongoid(datetime.datetime.strptime(self.startdate, '%Y-%m-%d'), (0, 0, 0)) end_mongo_id = self.date_to_mongoid(datetime.datetime.strptime(self.enddate, '%Y-%m-%d'), (23, 0, 0)) # Get db dataset for same date reports_cursor = self.reports.find( {"$and": [ {"platform" : self.platform}, {"_id":{"$gte":start_mongo_id}}, {"_id":{"$lte":end_mongo_id}} ] } ) db_entries = {} for doc in reports_cursor: if self.platform == 'appnexus': key = ( doc['placement_name'], doc['date'], doc['buyer_member_id'], doc['buyer_member_name']) else: key = ( doc['placement_name'], doc['date'] ) db_entries[key] = doc #self.stream_logger.info('len of db entries:{0}'.format(len(db_entries))) # Insert rows into database monitor.data({'db_data_len': len(db_entries)}) entries_to_insert = [] new_entries_count = 0 updated_entries_count = 0 identical_entries_count = 0 deleted_entries_count = 0 bulk_update_requests = [] #self.stream_logger.info('db_insert_entries:{0}'.format(db_insert_entries)) # update the existing entries for key, entry in db_insert_entries.items(): # add custom mongo id if key not in db_entries: entry['_id'] = self.date_to_mongoid(datetime.datetime .strptime(entry['date'], '%Y-%m-%d')) new_entries_count += 1 entries_to_insert.append(entry if type(entry) is dict else entry.dict()) else: del db_entries[key]['_id'] #print('from platform:{0}, from db:{1}'.format(type(entry), type(db_entries[key]))) match_dict = entry if type(entry) is dict else entry.dict() if db_entries[key] == match_dict: #self.stream_logger.info('For identical db entry:{0}'.format(entry.dict())) identical_entries_count += 1 else: #self.stream_logger.info('For update db entry:{0}'.format(entry.dict())) updated_entries_count += 1 # update the entry in database if self.platform == 'appnexus': replace_filter = {'platform': self.platform, 'placement_name':key[0], 'date':key[1], 'buyer_member_id':key[2], 'buyer_member_name':key[3] } else: replace_filter = {'platform': self.platform, 'placement_name':key[0], 'date':key[1] } bulk_update_requests.append(ReplaceOne(replace_filter, entry if type(entry) is dict else entry.dict())) #reports_col.replace_one(replace_filter, entry) del db_entries[key] # update existing documents try: if bulk_update_requests: self.reports.bulk_write(bulk_update_requests) except BulkWriteError as bwe: exc_type, exc_value, exc_traceback = sys.exc_info() print('BulkWriteError: {0}, {1}, {2}'.format(exc_type, exc_value, exc_traceback)) monitor.log('BulkWriteError: {0}, {1}, {2}'.format(exc_type, exc_value, exc_traceback), MonitorClient.LOGSTATUS_ERROR) return False#pprint(bwe.details) # insert new entries if entries_to_insert: self.reports.insert_many(entries_to_insert) # delete the entries that are no more exist in the platform's statistics deleted_entries_count = len(db_entries) for key in db_entries: if self.platform == 'appnexus': delete_filter = {'platform': self.platform, 'placement_name':key[0], 'date':key[1], 'buyer_member_id':key[2], 'buyer_member_name':key[3] } else: delete_filter = {'platform': self.platform, 'placement_name':key[0], 'date':key[1] } self.reports.delete_one(delete_filter) results = { "new_entries_count": new_entries_count, "updated_entries_count": updated_entries_count, "identical_entries_count": identical_entries_count, "deleted_entries_count": deleted_entries_count } monitor.data({'pull_results': results}) #self.stream_logger.info('Results:{0}'.format(results)) monitor.log("End pull from platform {0}".format(self.platform)) return results
class PullDataTestCase(TestCase): use_case_valid_commands = [ 'pull_appnexus', 'pull_adsense', 'pull_adx', 'pull_teads', 'pull_criteo', 'pull_criteohb', 'pull_taboola', 'pull_smart', 'pull_smaato', 'pull_facebook', 'pull_rubicon', 'pull_pubmatic', ] expected_results_insert_new = { 'appnexus': { 'updated_entries_count': 0, 'new_entries_count': 27292, 'identical_entries_count': 0, 'deleted_entries_count': 0 }, 'adsense': { 'updated_entries_count': 0, 'new_entries_count': 21, 'identical_entries_count': 0, 'deleted_entries_count': 0 }, 'adx': { 'deleted_entries_count': 0, 'new_entries_count': 1333, 'updated_entries_count': 0, 'identical_entries_count': 0 }, 'teads': { 'new_entries_count': 69, 'deleted_entries_count': 0, 'identical_entries_count': 0, 'updated_entries_count': 0 }, 'criteo': { 'identical_entries_count': 0, 'deleted_entries_count': 0, 'new_entries_count': 3196, 'updated_entries_count': 0 }, 'criteohb': { 'new_entries_count': 23, 'deleted_entries_count': 0, 'identical_entries_count': 0, 'updated_entries_count': 0 }, 'taboola': { 'identical_entries_count': 0, 'deleted_entries_count': 0, 'new_entries_count': 264, 'updated_entries_count': 0 }, 'smart': { 'deleted_entries_count': 0, 'identical_entries_count': 0, 'new_entries_count': 422, 'updated_entries_count': 0 }, 'rubicon': { 'new_entries_count': 2871, 'identical_entries_count': 0, 'deleted_entries_count': 0, 'updated_entries_count': 0 }, 'smaato': { 'deleted_entries_count': 0, 'identical_entries_count': 0, 'updated_entries_count': 0, 'new_entries_count': 85 }, 'facebook': { 'new_entries_count': 5, 'updated_entries_count': 0, 'identical_entries_count': 0, 'deleted_entries_count': 0 }, 'pubmatic': { 'identical_entries_count': 0, 'updated_entries_count': 0, 'deleted_entries_count': 0, 'new_entries_count': 679 } } expected_results = { 'pull_appnexus': { 'deleted_entries_count': 1, 'new_entries_count': 28, 'updated_entries_count': 43, 'identical_entries_count': 27221 }, 'pull_adsense': { 'identical_entries_count': 17, 'new_entries_count': 2, 'deleted_entries_count': 1, 'updated_entries_count': 2 }, 'pull_adx': { 'new_entries_count': 2, 'updated_entries_count': 2, 'identical_entries_count': 2677, 'deleted_entries_count': 1 }, 'pull_teads': { 'updated_entries_count': 2, 'identical_entries_count': 128, 'new_entries_count': 2, 'deleted_entries_count': 1 }, 'pull_criteo': { 'deleted_entries_count': 1, 'identical_entries_count': 6331, 'new_entries_count': 2, 'updated_entries_count': 2 }, 'pull_criteohb': { 'deleted_entries_count': 1, 'updated_entries_count': 2, 'identical_entries_count': 42, 'new_entries_count': 2 }, 'pull_taboola': { 'identical_entries_count': 523, 'deleted_entries_count': 1, 'new_entries_count': 2, 'updated_entries_count': 2 }, 'pull_smart': { 'deleted_entries_count': 1, 'updated_entries_count': 2, 'new_entries_count': 2, 'identical_entries_count': 418 }, 'pull_rubicon': { 'deleted_entries_count': 1, 'updated_entries_count': 2, 'new_entries_count': 2, 'identical_entries_count': 2866 }, 'pull_smaato': { 'deleted_entries_count': 1, 'identical_entries_count': 168, 'updated_entries_count': 2, 'new_entries_count': 2 }, 'pull_facebook': { 'new_entries_count': 2, 'identical_entries_count': 1, 'updated_entries_count': 2, 'deleted_entries_count': 0 }, 'pull_pubmatic': { 'new_entries_count': 2, 'identical_entries_count': 1, 'updated_entries_count': 2, 'deleted_entries_count': 0 } } removed_placements = { 'pull_appnexus': [ '4157463_real.gr_politics_970x250', #17 '11766412_reader.gr_homepage-1_970x250' #11 ], 'pull_adsense': [ '12351524_LocaleNetwork_ros_970x250', '10011495_alwakeelnews.com_ros_970x250' ], 'pull_adx': [ '11974846_apkroids.com_ros_300x250', '12686534_babyradio.gr_adblock-1_300x600', ], 'pull_teads': [ '79096 - inread - meteorologos.gr_11550271', '85264 - inRead - themamagers.gr_12350620' ], 'pull_criteo': [ '12001599_gazzetta.gr_adblock-3_300x250', '12001600_gazzetta.gr_adblock-4_300x250' ], 'pull_criteohb': [ '12552348_merrjep.com_postBid_300x600', '12552372_merrjep.al_postBid_300x600' ], 'pull_taboola': ['ant1iwo#CY-p11169123', 'go4it#RO-p10884104'], 'pull_smart': [ '7491770_m.businessmagazin.ro_ros-2_300x250', '4964992_antenna.gr_ros_300x250' ], 'pull_rubicon': [ '9017352_go4it.ro_Homepage-branding_970x250', '8551373_newsbomb.gr_Autokinito-2_300x250' ], 'pull_smaato': [ '9905337_alphatv.gr_ros-4_300x250', '9905336_alphatv.gr_ros-3_300x250' ], 'pull_facebook': [ '5600778_m.antena3.ro_allsite_300x250', '5665857_m.kanald.ro_allsite_300x250' ], 'pull_pubmatic': ['10030247_skai.gr_ros-2_300x250', '10129032_tpu.ro_ros_970x250'], } changed_placements = { 'pull_appnexus': [ '12480112_frontpages.gr_homepage-ros-side-perf_300x250', #40 '11998822_jurnalmm.ro_inarticle_319x49' #3 ], 'pull_adsense': [ '12531406_savoirville.gr_ros-2_300x250', '12484008_LocaleNetwork_ros_970x90' ], 'pull_adx': [ '12659596_queen.gr_mobile-2_300x250', '6601364_lovecooking.gr_ros_728x90' ], 'pull_teads': [ '78426 - inRead - holdpont.hu_7198944', '78427 - inRead - beszeljukmac.com_6045377' ], 'pull_criteo': [ '4565457_skai.gr_localnews_300x600', '10030247_skai.gr_ROS-2_300x250' ], 'pull_criteohb': [ '12552413_mojtrg.rs_postBid_728x90', '12552406_mojtrg.rs_postBid_300x250' ], 'pull_taboola': ['webnyeremeny#HU-f11904363', 'royanews#MENA-p11308540'], 'pull_smart': [ '7052351_alphatv.gr_tvshows_970x250', '11837266_garbo.ro_ros-2_300x250' ], 'pull_rubicon': [ '11441382_xe.gr_automoto-ad-details_728x90', '9916795_queen.gr_mageirikh-1_728x90' ], 'pull_smaato': [ '7491935_m.csid.ro_ros-2_300x250', '11071848_m.observator.tv_homepage_300x250' ], 'pull_facebook': [ '3841343_m.alphatv.gr_mobile_300x250', '4068005_m.spynews.ro_allsite_300x250' ], 'pull_pubmatic': ['12112559_gustos.ro_ros_300x250', '12112570_gustos.ro_ros_728x90'], } request_body = '' expected_data = [] dict_expected_data = {} dict_nd_expected_data = {} dbdata = [] command = 'pull_appnexus' reports = MongoConnector('reports').collection @classmethod def date_to_mongoid(self, date, daytime=(0, 0, 0)): hours, mins, secs = daytime datetimeobj = datetime.datetime.combine( date, datetime.time(hours, mins, secs)) mongo_id = ObjectId.generate_from_datetime(datetimeobj) return mongo_id @classmethod def setCommand(cls, command): cls.command = command @classmethod def setInOutData(cls, command): cls.platform = cls.getPlatform(command) cls.dates = cls.getDates(command) cls.expected_data = cls.getExpected(command) cls.dict_expected_data = cls.getDictExpected(command) cls.dict_nd_expected_data = cls.getDictExpected(command, next_day=True) cls.dbdata = cls.getDbData() @classmethod def getSuiteOfValidTests(cls): suite = unittest.TestSuite() suite.addTest(cls('test_1_pull_data_insert')) suite.addTest(cls('test_2_pull_data_update')) suite.addTest(cls('test_3_pull_data_update_revenue')) suite.addTest(cls('test_4_pull_data_update_multiple_fields')) suite.addTest(cls('test_5_pull_data_insert_different_dates')) suite.addTest(cls('test_6_set_pull_mode_mute')) suite.addTest(cls('test_7_set_pull_mode_normal')) return suite @classmethod def getDbData(cls): #selected_placements = cls.selected_placements[command] start_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[0], '%Y-%m-%d')) end_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[1], '%Y-%m-%d'), (23, 0, 0)) ''' if cls.platform in ('appnexus', 'adsense', 'smart', 'facebook', 'rubicon'): end_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[1], '%Y-%m-%d') , (23, 0, 0)) else: end_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[1], '%Y-%m-%d') ) ''' # Get db dataset for same date reports_cursor = cls.reports.find( { "$and": [{ "platform": cls.platform }, { "_id": { "$gte": start_mongo_id } }, { "_id": { "$lt": end_mongo_id } }] }, { '_id': 0, 'placement_name': 0 }) db_entries = [] for doc in reports_cursor: if cls.platform == "appnexus": #print(doc) if doc['buyer_member_id'] == '2026': db_entries.append(doc) else: db_entries.append(doc) return db_entries @classmethod def insertDepricatedDbData(cls): platform = cls.getPlatform(cls.command) # Get db dataset for same date mongo_id = cls.date_to_mongoid( datetime.datetime.strptime("2018-01-20", '%Y-%m-%d'), (12, 0, 0)) result = cls.reports.insert({ "_id": mongo_id, "platform": platform, "date": "2018-01-20", "placement_id": 11974846, "placement_name": "11974846_depricated_apkroids.com_ros_300x250", "total_impressions": 21539, "resold_impressions": 127, "buyer_member_id": "2026", "buyer_member_name": " ", "revenue": 0.01, "revenue_dict": { "EUR": 0.01, "USD": 0.012221 }, "clicks": 0 }) #print('inserted_count:{0}'.format(result.inserted_count)) return result @classmethod def removeDepricatedDbData(cls): platform = cls.getPlatform(cls.command) # Get db dataset for same date result = cls.reports.delete_many({ "$and": [{ "platform": cls.platform }, { "placement_name": "11974846_depricated_apkroids.com_ros_300x250" }] }) return result @classmethod def removeDbData(cls): start_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[0], '%Y-%m-%d')) end_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[1], '%Y-%m-%d'), (23, 0, 0)) # Get db dataset for same date result = cls.reports.delete_many({ "$and": [{ "platform": cls.platform }, { "_id": { "$gte": start_mongo_id } }, { "_id": { "$lt": end_mongo_id } }] }) return result.deleted_count @classmethod def removeSelectedDbData(cls): selected_placements = cls.removed_placements[cls.command] start_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[0], '%Y-%m-%d')) end_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[1], '%Y-%m-%d'), (23, 0, 0)) # Get db dataset for same date result = cls.reports.delete_many({ "$and": [{ "platform": cls.platform }, { "_id": { "$gte": start_mongo_id } }, { "_id": { "$lt": end_mongo_id } }, { "placement_name": { "$in": selected_placements } }] }) return result.deleted_count @classmethod def updateSelectedDbDataRevenueField(cls): selected_placements = cls.changed_placements[cls.command] start_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[0], '%Y-%m-%d')) end_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[1], '%Y-%m-%d'), (23, 0, 0)) result = cls.reports.update_many( { "$and": [{ "platform": cls.platform }, { "_id": { "$gte": start_mongo_id } }, { "_id": { "$lt": end_mongo_id } }, { "placement_name": { "$in": selected_placements } }] }, { "$set": { "revenue": 100000.0, "revenue_dict.EUR": 100000.0, "revenue_dict.USD": 100000.0 } }) return result.modified_count @classmethod def updateSelectedDbDataMultipleFields(cls): selected_placements = cls.changed_placements[cls.command] start_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[0], '%Y-%m-%d')) end_mongo_id = cls.date_to_mongoid( datetime.datetime.strptime(cls.dates[1], '%Y-%m-%d'), (23, 0, 0)) result = cls.reports.update_many( { "$and": [{ "platform": cls.platform }, { "_id": { "$gte": start_mongo_id } }, { "_id": { "$lt": end_mongo_id } }, { "placement_name": { "$in": selected_placements } }] }, { "$set": { "revenue": 0.0, "revenue_dict.EUR": 0.0, "revenue_dict.USD": 0.0, "total_impressions": 0, "resold_impressions": 0 } }) return result.modified_count def setUp(self): pass self.maxDiff = None def tearDown(self): pass #@unittest.skip('just skip') def test_1_pull_data_insert(self): # for teads platform wait before the next request if self.platform in ('teads'): time.sleep(5) #self.maxDiff = None # delete data from database self.removeDbData() # set env variable for mongo host #os.environ['MONGO_HOST'] = '127.0.0.1' #os.environ['MONGO_HOST'] = 'mongodb' """ if self.platform == 'appnexus': puller = ReporterPuller(self.platform, self.dates[0], self.dates[1]) puller.reporter_client.read = MagicMock( return_value=self.dict_expected_data) ret_data = puller.pull_data() else: ret_data = JsonRPCClient("tcp://*****:*****@unittest.skip('just skip') def test_2_pull_data_update(self): #if self.platform == 'teads': # time.sleep(5) # get expected data # set env variable for mongo host #os.environ['MONGO_HOST'] = 'localhost' puller = ReporterPuller(self.platform, self.dates[0], self.dates[1]) puller.reporter_client.read = MagicMock( return_value=self.dict_expected_data) ret_data = puller.pull_data() #print(ret_data) self.dbdata = self.getDbData() #print(len(self.dbdata)) #print(len(self.expected_data)) #self.assertEqual(ret_data, True, # 'correct pull process') # read the data from the database and compare them with # expected data self.assertCountEqual(self.dbdata, self.expected_data, 'correct pulled data') #@unittest.skip('just skip') def test_3_pull_data_update_revenue(self): if self.platform in ('teads'): time.sleep(5) # for teads platform wait before the next request #self.maxDiff = None # delete data from database self.updateSelectedDbDataRevenueField() self.removeSelectedDbData() self.removeDepricatedDbData() self.insertDepricatedDbData() # set env variable for mongo host #os.environ['MONGO_HOST'] = '127.0.0.1' #os.environ['MONGO_HOST'] = 'mongodb' puller = ReporterPuller(self.platform, self.dates[0], self.dates[1]) puller.reporter_client.read = MagicMock( return_value=self.dict_expected_data) ret_data = puller.pull_data() #print(ret_data) expected_results = self.expected_results[self.command] #self.assertEqual(ret_data, expected_results, 'results are correct') self.dbdata = self.getDbData() self.assertCountEqual(self.dbdata, self.expected_data, 'pulled data are correct') #@unittest.skip('just skip') def test_4_pull_data_update_multiple_fields(self): # for teads platform wait before the next request if self.platform in ('teads'): time.sleep(5) self.maxDiff = None # delete data from database self.updateSelectedDbDataMultipleFields() self.removeSelectedDbData() self.removeDepricatedDbData() self.insertDepricatedDbData() # set env variable for mongo host #os.environ['MONGO_HOST'] = '127.0.0.1' #os.environ['MONGO_HOST'] = 'mongodb' puller = ReporterPuller(self.platform, self.dates[0], self.dates[1]) puller.reporter_client.read = MagicMock( return_value=self.dict_expected_data) ret_data = puller.pull_data() #print(ret_data) expected_results = self.expected_results[self.command] #self.assertEqual(ret_data, expected_results, 'results are correct') self.dbdata = self.getDbData() self.assertCountEqual(self.dbdata, self.expected_data, 'correct puremoveDepricatedDbDatalled data') #@unittest.skip('just skip') def test_5_pull_data_insert_different_dates(self): # for teads platform wait before the next request if self.platform in ('teads'): time.sleep(5) #self.maxDiff = None # delete data from database self.removeDbData() # set env variable for mongo host #os.environ['MONGO_HOST'] = '127.0.0.1' #os.environ['MONGO_HOST'] = 'mongodb' # pull data for 2018-01-20 puller = ReporterPuller(self.platform, self.dates[0], self.dates[1]) puller.reporter_client.read = MagicMock( return_value=self.dict_expected_data) ret_data = puller.pull_data() # pull data for next day puller = ReporterPuller(self.platform, self.dates[2], self.dates[3]) puller.reporter_client.read = MagicMock( return_value=self.dict_nd_expected_data) ret_data = puller.pull_data() # pull data for 2018-01-20 puller = ReporterPuller(self.platform, self.dates[0], self.dates[1]) puller.reporter_client.read = MagicMock( return_value=self.dict_expected_data) ret_data = puller.pull_data() #self.assertEqual(ret_data, True, 'correct pull process') self.dbdata = self.getDbData() #print(len(self.dbdata)) self.assertCountEqual(self.dbdata, self.expected_data, 'correct pulled data') #@unittest.skip('just skip') def test_6_set_pull_mode_mute(self): # for teads platform wait before the next request if self.platform in ('teads'): time.sleep(5) # set env variable for mongo host #os.environ['MONGO_HOST'] = '127.0.0.1' #os.environ['MONGO_HOST'] = 'mongodb' JsonRPCClient("tcp://*****:*****@unittest.skip('just skip') def test_7_set_pull_mode_normal(self): # for teads platform wait before the next request if self.platform in ('teads'): time.sleep(5) # set env variable for mongo host #os.environ['MONGO_HOST'] = '127.0.0.1' #os.environ['MONGO_HOST'] = 'mongodb' JsonRPCClient("tcp://localhost:5552").rpc_call('pull_mode_normal', platform=self.platform) puller = ReporterPuller(self.platform, self.dates[0], self.dates[1]) puller.reporter_client.read = MagicMock( return_value=self.dict_expected_data) ret_data = puller.pull_data() self.assertNotEqual(ret_data, False, 'correct pulled data') JsonRPCClient("tcp://localhost:5552").rpc_call('pull_mode_normal', platform=self.platform) puller = ReporterPuller(self.platform, self.dates[0], self.dates[1]) puller.reporter_client.read = MagicMock( return_value=self.dict_expected_data) ret_data = puller.pull_data() self.assertNotEqual(ret_data, False, 'correct pulled data')
def setUp(self): self.reporter_client = ReporterClient() self.reports = MongoConnector('reports').collection
class DataPusherTestCase(unittest.TestCase): def setUp(self): self.reporter_client = ReporterClient() self.reports = MongoConnector('reports').collection def tearDown(self): pass def updateReports(self): # update teads placement name self.reports.update_many( { "$and": [{ "platform": "teads" }, { "placement_name": "77422 - inRead - sportfm.gr_5687603" }] }, { "$set": { "placement_name": "77422 - inRead - sportfm.gr5687603" } }) # update adx size self.reports.update_many( { "$and": [{ "platform": "adx" }, { "placement_name": "11974846_apkroids.com_ros_300x250" }] }, {"$set": { "placement_name": "11974846_apkroids.com_ros300x250" }}) # update taboola size field in placement name self.reports.update_many( { "$and": [{ "platform": "taboola" }, { "placement_name": "insider#300x250#GR-r11409755" }] }, {"$set": { "placement_name": "insider#300250#GR-r11409755" }}) # update taboola revenue self.reports.update_many( { "$and": [{ "platform": "criteo" }, { "placement_name": "11777559_ma7room.com_ros_970x250" }] }, {"$set": { "revenue_dict.USD": "10,34" }}) # update criteo revenue self.reports.update_many( { "$and": [{ "platform": "criteo" }, { "placement_name": "12001599_gazzetta.gr_adblock-3_300x250" }] }, {"$set": { "revenue_dict.EUR": "10,35" }}) return True #@unittest.skip('just skip') def test_1_push_all(self): self.maxDiff = None expected = { 'found': 8285, 'inserted': 8267, 'inserted_eurohoops': 528, 'rejected_id': 8, 'rejected_size': 2, 'rejected_inventory': 6, 'rejected_pub_fee': 0, 'rejected_revenue': 2, 'messages': [ 'Platform adsense. Rejecting "TheFrog_300x250". Could_not_parse_placement_id', 'Platform adsense. Rejecting "Test_MSN_Remnant_300x250". Could_not_parse_placement_id', 'Platform adsense. Rejecting "Test_MSN_Remnant_728x90". Could_not_parse_placement_id', 'Platform adsense. Rejecting "Houselife Adsense 300x250". Could_not_parse_placement_id', 'Platform adsense. Rejecting "YL_300x250". Could_not_parse_placement_id', 'Platform adx. Rejecting "11974846_apkroids.com_ros300x250". Could_not_parse_placement_size', 'Platform adx. Rejecting "(No tag)". Could_not_parse_placement_id', 'Platform teads. Rejecting "77422 - inRead - sportfm.gr5687603". Could_not_parse_placement_id', 'Platform criteo. Rejecting "12001599_gazzetta.gr_adblock-3_300x250". revenue_is_not_float', 'Platform criteo. Rejecting "12341051_missbloom.bg_homepage-ros-b-perf_300x600". Could_not_find_it_in_inventory_table', 'Platform criteo. Rejecting "12341048_missbloom.bg_homepage-ros-b-perf_300x250". Could_not_find_it_in_inventory_table', 'Platform criteo. Rejecting "11777559_ma7room.com_ros_970x250". revenue_is_not_float', 'Platform criteo. Rejecting "12341055_missbloom.bg_homepage-ros-perf_970x250". Could_not_find_it_in_inventory_table', 'Platform taboola. Rejecting "mother#GR-p9964478_not_used - DEACTIVATED". Could_not_parse_placement_id', 'Platform taboola. Rejecting "insider#300250#GR-r11409755". Could_not_parse_placement_size', 'Platform rubicon. Rejecting "12341050_missbloom.bg_homepage-ros-a-perf_300x600". Could_not_find_it_in_inventory_table', 'Platform rubicon. Rejecting "12341056_missbloom.bg_homepage-ros-perf_728x90". Could_not_find_it_in_inventory_table', 'Platform rubicon. Rejecting "12341055_missbloom.bg_homepage-ros-perf_970x250". Could_not_find_it_in_inventory_table' ] } self.updateReports() #rs = self.reporter_client.push_to_ui('2018-01-20', 'test') ret_data = JsonRPCClient("tcp://*****:*****@unittest.skip('just skip') def test_2_push_adsense(self): self.maxDiff = None expected = { 'found': 21, 'inserted': 16, 'inserted_eurohoops': 1, 'rejected_id': 5, 'rejected_size': 0, 'rejected_inventory': 0, 'rejected_pub_fee': 0, 'rejected_revenue': 0, 'messages': [ 'Platform adsense. Rejecting "Houselife Adsense 300x250". Could_not_parse_placement_id', 'Platform adsense. Rejecting "Test_MSN_Remnant_300x250". Could_not_parse_placement_id', 'Platform adsense. Rejecting "Test_MSN_Remnant_728x90". Could_not_parse_placement_id', 'Platform adsense. Rejecting "TheFrog_300x250". Could_not_parse_placement_id', 'Platform adsense. Rejecting "YL_300x250". Could_not_parse_placement_id' ] } self.updateReports() #rs = self.reporter_client.push_to_ui('2018-01-20', 'test') ret_data = JsonRPCClient("tcp://localhost:5552").rpc_call( 'push_data', platform='adsense', date='2018-01-20', mode='test') # order messages in both dicts if 'messages' in expected: expected['messages'] = sorted(expected['messages']) if 'messages' in ret_data['result']: ret_data['result']['messages'] = sorted( ret_data['result']['messages']) print(ret_data['result']) self.assertDictEqual(ret_data['result'], expected, 'Push to UI is correct')