def main(self,start_id,thread_name,end_id): #import pdb;pdb.set_trace() self.mongo_connection() self.constant_param() self.get_env_url() self.logger=lib_common_modules().create_log(os.getcwd()+"/log/log.txt") result_sheet='/output/%s_Ingestion_checked_in_Px_%s_%s.csv'%(self.source,thread_name,datetime.date.today()) output_file=lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: self.writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n') self.writer.writerow(self.fieldnames) for id_ in range(start_id,end_id,100): self.total+=1 showtime_data = self.query_execute(id_) for data in showtime_data: self.getting_source_details(data) if self.show_type is not None: self.ingestion_checking(thread_name) else: self.writer.writerow([self.showtime_id,'',self.show_type,self.title,self.series_title,self.episode_title,self.showtime_show_id,self.season_number,self.episode_number,self.year,self.duration,self.updated_at,self.expiry_date]) self.logger.debug("\n") self.logger.debug([{"Total":self.total,"ingested_count":self.ingested_count, "not_ingested_count": self.not_ingested_count,"Thread_name":thread_name}]) self.logger.debug("\n") self.logger.debug(["date time:", datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")]) output_file.close() self.connection.close()
def main(self, start_id, thread_name, end_id): #import pdb;pdb.set_trace() self.get_env_url() self.mongo_mysql_connection() input_file = "/input/ids" input_data = lib_common_modules().read_csv(input_file) result_sheet = '/result/headrun_netflix_ott_%s_checking%s.csv' % ( thread_name, datetime.date.today()) output_file = lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: fieldnames = [ "%s_id" % self.source, "Series_id", "Projectx_id", "show_type", "px_video_link_present", "%s_link_present" % self.source, "ott_link_result", "mapping", "", "Expired" ] self.writer = csv.writer(mycsvfile, dialect="csv", lineterminator='\n') self.writer.writerow(fieldnames) for _id in range(start_id, end_id): self.headrun_netflix_id = str(input_data[_id][0]) query_headrun_netflix = self.sourcetable.find({ "service": "netflix", "id": "%s" % str(self.headrun_netflix_id) }) for data in query_headrun_netflix: if data.get("id") != "": if data.get("item_type") == "movie": self.show_type = "MO" else: self.show_type = "SE" self.count_headrun_netflix_id += 1 print("\n") print datetime.datetime.now() print("\n") print({ "count_headrun_netflix_id": self.count_headrun_netflix_id, "id": self.headrun_netflix_id, "thread_name": thread_name }) only_mapped_ids = ott_meta_data_validation_modules( ).getting_mapped_px_id_mapping_api( str(self.headrun_netflix_id), self.source_mapping_api, self.projectx_mapping_api, self.show_type, self.source, self.token) self.ott_checking(only_mapped_ids, thread_name, data) print("\n") print({ "count_headrun_netflix_id": self.count_headrun_netflix_id, "name": thread_name }) output_file.close() self.connection.close()
def season_0_checking(self, thread_name, projectx_sm_id, token): try: self.seasons_api_response = lib_common_modules( ).fetch_response_for_api_(self.seasons_api % projectx_sm_id, token) if self.seasons_api_response != []: for seasons in self.seasons_api_response: print("\n") print({"seasons": seasons}) if seasons["season_number"] == 0: self.season_number_0 = "Present" self.season_program_id = seasons["season_program_id"] self.season_name = unidecode.unidecode( pinyin.get(seasons["season_name"])) self.result = "Fail" self.comment = "Season number 0 present in Season API" self.all_episode_sequence_response = lib_common_modules( ).fetch_response_for_api_( self.all_episode_sequence_api % projectx_sm_id, token) if self.all_episode_sequence_response != []: for sequence in self.all_episode_sequence_response: if sequence[ "season_program_id"] == self.season_program_id: self.data_source = sequence["data_source"] self.season_served = sequence[ "season_number"] if { str(self.data_source): 'Season ' + str(self.season_served) } not in self.Serving_Sources_season_program_id: self.Serving_Sources_season_program_id.append( { str(self.data_source): 'Season ' + str(self.season_served) }) if self.Serving_Sources_season_program_id == []: self.Serving_Sources_season_program_id = [ "Season_program id not there in all episode sequence API" ] else: self.comment = "Empty_Season_response" self.result = "Fail" except (Exception, httplib.BadStatusLine, urllib2.HTTPError, urllib2.URLError) as e: self.retry_count += 1 print("Retrying...................................", self.retry_count) print("\n") print( "exception/error caught in season0_checking func...................", type(e), projectx_sm_id, thread_name) if self.retry_count <= 10: self.season_0_checking(thread_name, projectx_sm_id, token) else: self.retry_count = 0
def ott_checking(self,only_mapped_ids,appletv_id,show_type,thread_name,data): #import pdb;pdb.set_trace() try: if only_mapped_ids["source_flag"]=='True': self.total+=1 #import pdb;pdb.set_trace() projectx_id=only_mapped_ids["px_id"] source_id=only_mapped_ids["source_id"] self.logger.debug("\n") self.logger.debug ({"total_only_appletv_mapped":self.total,"%s_id"%self.source:appletv_id, "Px_id":projectx_id,"%s_id"%self.source:source_id,"thread_name":thread_name}) source_details=self.getting_source_details(source_id,show_type,data) projectx_details=self.getting_projectx_ott_details(projectx_id,show_type) if projectx_details!='Null': if data.get("purchase_info")!="": self.link_expired=lib_common_modules().link_expiry_check_(self.expired_api,self.projectx_domain,source_id,self.service,self.expired_token) ott_validation_result=ott_meta_data_validation_modules().ott_validation(projectx_details,source_id) self.writer.writerow([source_id,projectx_id,show_type,projectx_details["px_video_link_present"], source_details["source_link_present"],ott_validation_result,only_mapped_ids["source_flag"],'',self.link_expired]) else: self.writer.writerow([source_id,projectx_id,show_type,'','','', only_mapped_ids["source_flag"],'purchase_info_null']) else: self.writer.writerow([source_id,projectx_id,show_type,projectx_details, source_details["source_link_present"],'',only_mapped_ids["source_flag"],'Px_response_null']) elif only_mapped_ids["source_flag"]=='True(Rovi+others)': if show_type!='SM': projectx_id=only_mapped_ids["px_id"] source_id=only_mapped_ids["source_id"] self.logger.debug("\n") self.logger.debug ({"Px_id":projectx_id,"%s_id"%self.source:source_id,"thread_name":thread_name}) projectx_details=self.getting_projectx_ott_details(projectx_id,show_type) if projectx_details!='Null': if data.get("purchase_info")!="": self.link_expired=lib_common_modules().link_expiry_check_(self.expired_api,self.projectx_domain,source_id,self.service,self.expired_token) ott_validation_result=ott_meta_data_validation_modules().ott_validation(projectx_details,source_id) self.writer.writerow([source_id,projectx_id,show_type,'','',ott_validation_result, only_mapped_ids["source_flag"],'',self.link_expired]) else: self.writer.writerow([source_id,projectx_id,show_type,'','','', only_mapped_ids["source_flag"],'purchase_info_null']) else: self.writer.writerow([appletv_id,'',show_type,'','',only_mapped_ids,'Px_id_null']) except (Exception,httplib.BadStatusLine,urllib2.HTTPError,socket.error,urllib2.URLError,RuntimeError) as e: self.retry_count+=1 self.logger.debug("Retrying...................................",self.retry_count) self.logger.debug("\n") self.logger.debug ("exception/error caught in ott_checking func.........................",type(e),appletv_id,show_type,thread_name) if self.retry_count<=5: self.ott_checking(only_mapped_ids,appletv_id,show_type,thread_name,data) else: self.retry_count=0
def ingestion_checking(self,thread_name): #import pdb;pdb.set_trace() duplicate_px_id=[] try: showtime_mapping_api=self.source_mapping_api%(self.showtime_id,self.source,self.show_type.encode()) data_showtime_resp=lib_common_modules().fetch_response_for_api_(showtime_mapping_api,self.token) if data_showtime_resp!=[]: self.ingested_count+=1 #TODO: checking program response in Programs_Id search api for response in data_showtime_resp: self.projectx_id = response["projectx_id"] program_response=lib_common_modules().fetch_response_for_api_(self.programs_api%self.projectx_id,self.token) if program_response!=[]: self.px_response='True' self.writer.writerow([self.showtime_id,self.projectx_id,self.show_type,self.title,self.series_title,self.episode_title,self.showtime_show_id,self.season_number,self.episode_number,self.year,self.duration,self.updated_at,self.expiry_date,'','','True','',self.px_response]) # TODO : to check duplicate source_ids else: if self.show_type=='SM' or self.show_type=='MO': showtime_duplicate_api=self.source_duplicate_api%(self.showtime_id,self.source,self.show_type.encode()) else: #TODO: for episodes showtime_duplicate_api=self.source_duplicate_api%(self.showtime_show_id,self.source,'SM') data_showtime_resp_duplicate=lib_common_modules().fetch_response_for_api_(showtime_duplicate_api,self.token) #import pdb;pdb.set_trace() if data_showtime_resp_duplicate!=[]: self.ingested_count+=1 for px_id in data_showtime_resp_duplicate: duplicate_px_id.append(px_id.get("projectx_id")) if self.show_type=='MO' or self.show_type=='SM': source_id_duplicate=ingestion_script_modules().getting_duplicate_source_id(duplicate_px_id,self.px_mapping_api,self.show_type,self.token,self.source) else: #TODO : for episode source_id_duplicate=ingestion_script_modules().getting_duplicate_source_id(duplicate_px_id,self.px_mapping_api,'SM',self.token,self.source) self.writer.writerow([self.showtime_id,duplicate_px_id,self.show_type,self.title,self.series_title,self.episode_title,self.showtime_show_id,self.season_number,self.episode_number,self.year,self.duration,self.updated_at,self.expiry_date,'True',source_id_duplicate,'True']) else: self.not_ingested_count+=1 self.writer.writerow([self.showtime_id,'',self.show_type,self.title,self.series_title,self.episode_title,self.showtime_show_id,self.season_number,self.episode_number,self.year,self.duration,self.updated_at,self.expiry_date,'False','','','True',self.px_response]) self.logger.debug('\n') self.logger.debug(["%s_id:"%self.source,self.showtime_id,"show_type:",self.show_type,thread_name,"updated:","title:",self.title,"series_title: ",self.series_title,"season_no:",self.season_number,"episode_no:",self.episode_number,"ingested_count:",self.ingested_count, "not_ingested_count:", self.not_ingested_count]) except (Exception,httplib.BadStatusLine,urllib2.HTTPError,socket.error,URLError,RuntimeError,ValueError) as e: #import pdb;pdb.set_trace() self.retry_count+=1 self.logger.debug(["exception caught ingestion_checking func.........................",type(e),self.showtime_id,self.show_type,thread_name]) self.logger.debug(["Retrying.............",self.retry_count]) if self.retry_count<=5: self.ingestion_checking(thread_name) else: self.retry_count=0
def getting_source_ott_details(self, gracenote_id, show_type, details): #import pdb;pdb.set_trace() gracenote_link = [] expired_link = [] gracenote_link_present = 'Null' gracenote_link.append(details["Videos"]) if gracenote_link != []: gracenote_link_present = 'True' for links in gracenote_link: for key in links.keys(): for link_id in links[key]: service = self.to_check_service_proper_name(key) self.link_expired = lib_common_modules().link_expiry_check( self.expired_api, self.projectx_domain, link_id, service.lower(), self.expired_token, self.logger) if self.link_expired == 'True': expired_link.append({key: link_id}) self.logger.debug("\n") self.logger.debug({ "source_link_present": gracenote_link_present, "link_id": gracenote_link, "show_type": show_type, "expired_link": expired_link, "expired_status": self.link_expired }) return { "source_link_present": gracenote_link_present, "link_id": gracenote_link, "show_type": show_type, "expired_link": expired_link, "expired_status": self.link_expired }
def getting_projectx_ott_details(self, projectx_id, show_type): #import pdb;pdb.set_trace() px_video_link = [] px_video_link_present = 'False' px_response = 'Null' launch_id = [] fetched_from = [] appletv_id_format = "False" projectx_api = self.projectx_programs_api % (projectx_id, self.service) data_px_resp = lib_common_modules().fetch_response_for_api_( projectx_api, self.token) if data_px_resp != []: for data in data_px_resp: px_video_link = data.get("videos") if px_video_link: px_video_link_present = 'True' for linkid in px_video_link: launch_id.append(linkid.get("launch_id")) if linkid.get("fetched_from") not in fetched_from: fetched_from.append( linkid.get("fetched_from").encode("utf-8")) if linkid.get("appletv_id") == '{}{}'.format( "%s://www.%s.com/en-gb/video/" % (self.source.lower(), self.source.lower()), linkid.get("launch_id")): appletv_id_format = 'True' return { "px_video_link_present": px_video_link_present, "launch_id": launch_id, "fetched_from": fetched_from, "appletv_id_format": appletv_id_format } else: return px_response
def main(self,start_id,thread_name,end_id): #import pdb;pdb.set_trace() self.get_env_url() self.mongo_mysql_connection() result_sheet='/result/headrun_Showtime_ott_%s_checking%s.csv'%(thread_name,datetime.date.today()) output_file=lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: fieldnames = ["%s_id"%self.source,"Projectx_id","show_type","px_video_link_present","Fetch_from","Series_id","Expiry_date","%s_link_present"%self.source,"ott_link_result","mapping","","Expired"] self.writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n') self.writer.writerow(fieldnames) for _id in range(start_id,end_id,100): print({"skip":_id}) query_headrun_Showtime=self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":["movie","episode"]}},{"service":self.source.lower()}]}},{"$project":{"id":1,"_id":0,"item_type":1,"series_id":1,"title":1,"episode_title":1,"release_year":1,"episode_number":1,"season_number":1,"duration":1,"image_url":1,"url":1,"description":1,"cast":1,"directors":1,"writers":1,"categories":1,"genres":1,"maturity_ratings":1,"purchase_info":1,"service":1,"expiry_date":1}},{"$skip":_id},{"$limit":100}]) #query_headrun_Showtime=self.sourcetable.find({"service":"Showtime","item_type":"movie","id":"70301275"}) for data in query_headrun_Showtime: if data.get("id")!="": #import pdb;pdb.set_trace() self.headrun_Showtime_id=data.get("id").encode("ascii","ignore") if data.get("item_type")=="movie": self.show_type="MO" else: self.show_type="SE" self.count_headrun_Showtime_id+=1 print("\n") print datetime.datetime.now() print("\n") print ({"count_headrun_Showtime_id":self.count_headrun_Showtime_id, "id":self.headrun_Showtime_id,"thread_name":thread_name}) #import pdb;pdb.set_trace() only_mapped_ids=ott_meta_data_validation_modules().getting_mapped_px_id_mapping_api(str(self.headrun_Showtime_id),self.source_mapping_api,self.projectx_mapping_api,self.show_type,self.source,self.token) self.ott_checking(only_mapped_ids,thread_name,data) print("\n") print ({"count_headrun_Showtime_id":self.count_headrun_Showtime_id,"name":thread_name}) output_file.close() self.connection.close()
def ott_checking(self, only_mapped_ids, thread_name, data): try: if only_mapped_ids["source_flag"] == 'True' or only_mapped_ids[ "source_flag"] == 'True(Rovi+others)': self.total += 1 projectx_id = only_mapped_ids["px_id"] print("\n") print({ "total_only_headrun_HBOMAX_mapped": self.total, "%s_id" % self.source: self.headrun_HBOMAX_id, "Px_id": projectx_id, "thread_name": thread_name }) projectx_details = self.getting_projectx_ott_details( projectx_id, self.show_type) if projectx_details != 'Null': self.link_expired = lib_common_modules( ).link_expiry_check_(self.expired_api, self.projectx_domain, self.headrun_HBOMAX_id, self.source.lower(), self.expired_token) ott_validation_result = ott_meta_data_validation_modules( ).ott_validation(projectx_details, self.headrun_HBOMAX_id) self.writer.writerow([ self.headrun_HBOMAX_id, projectx_id, self.show_type, projectx_details["px_video_link_present"], data["series_id"], data["expiry_date"], '', ott_validation_result, only_mapped_ids["source_flag"], '', self.link_expired, projectx_details["appletv_id_format"], projectx_details["androidtv_id_format"], projectx_details["fetched_from"] ]) else: self.writer.writerow([ self.headrun_HBOMAX_id, projectx_id, self.show_type, projectx_details, data["series_id"], data["expiry_date"], '', '', only_mapped_ids["source_flag"], 'Px_response_null' ]) else: self.total += 1 self.writer.writerow([ self.headrun_HBOMAX_id, '', self.show_type, '', data["series_id"], data["expiry_date"], '', only_mapped_ids, 'Px_id_null' ]) except (Exception, httplib.BadStatusLine, urllib2.HTTPError, socket.error, urllib2.URLError, RuntimeError) as e: self.retry_count += 1 print("Retrying..............................", self.retry_count) print("\n") print( "exception/error caught in ott_checking func...................", type(e), self.headrun_HBOMAX_id, thread_name) if self.retry_count <= 5: self.ott_checking(only_mapped_ids, thread_name, data) else: self.retry_count = 0
def getting_source_ott_details(self, details): #import pdb;pdb.set_trace() gracenote_link = [] link_ids = [] expired_link = [] gracenote_link.append(details["Videos"]) for links in gracenote_link: for key in links.keys(): for link_id in links[key]: service = self.to_check_service_proper_name(key) link_ids.append(link_id) self.link_expired = lib_common_modules( ).link_expiry_check_(self.expire_api, self.preprod_domain_name, link_id, service.lower(), self.Token) if self.link_expired == 'True': expired_link.append({key: link_id}) else: break if link_ids: self.link_expired = "False" break self.logger.debug("\n") self.logger.debug({ "link_id": link_ids, "service": service.lower(), "expired_link": expired_link, "expired_status": self.link_expired }) return { "link_id": link_ids, "service": service.lower(), "expired_link": expired_link, "expired_status": self.link_expired }
def main(self,start_id,thread_name,end_id): self.get_env_url() self.default_param() self.mongo_connection() result_sheet_credit_match_false='/output_credit_match_false/duplicate_programs_credit_match_false_result_%s_%s.csv'%(thread_name,datetime.date.today()) output_file_credit_match_false=lib_common_modules().create_csv(result_sheet_credit_match_false) result_sheet='/output/duplicate_checked_in_search_api_%s_%s.csv'%(thread_name,datetime.date.today()) output_file=lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile, output_file_credit_match_false as mycsvfile1: writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n') writer.writerow(self.fieldnames_tag) writer_credit_match_false = csv.DictWriter(mycsvfile1,fieldnames=self.fieldnames_credit_match_false,dialect="csv",lineterminator = '\n') writer_credit_match_false.writeheader() for _id in range(start_id,end_id,100): try: query=self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":["movie","episode"]}},{"service":"hbomax"}]}},{"$project":{"id":1,"_id":0,"item_type":1,"series_id":1,"title":1,"episode_title":1,"release_year":1,"episode_number":1,"season_number":1,"duration":1,"image_url":1,"url":1,"description":1,"cast":1,"directors":1,"writers":1,"categories":1,"genres":1,"maturity_ratings":1,"purchase_info":1,"service":1}},{"$skip":_id},{"$limit":100}]) #query= self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":["movie","episode"]}},{"service":"hbomax","id":"80131281"}]}}]) #import pdb;pdb.set_trace() print ("\n") print({"start": start_id,"end":end_id}) for data in query: series_title='' _id=data.get("id") sm_id=data.get("series_id") show_type=data.get("item_type").encode('utf-8') if _id is not None and _id!="": if show_type=='episode': if sm_id not in self.sm_list: self.sm_list.append(sm_id) self.cleanup() self.duplicate_checking_series(data,writer,writer_credit_match_false,thread_name) elif show_type=='movie': #import pdb;pdb.set_trace() if _id not in self.mo_list: self.mo_list.append(_id) self.cleanup() self.duplicate_checking_movies(data,writer,writer_credit_match_false,thread_name) print("\n") print ({"Total SM":len(self.sm_list),"Total MO":len(self.mo_list),"Thread_name":thread_name}) print("\n") print (datetime.datetime.now()) except (pymongo.errors.CursorNotFound,Exception) as e: print ("exception caught in main func.............",type(e),thread_name) continue output_file.close() output_file_credit_match_false.close() self.connection.close()
def main(self, start_id, thread_name, end_id): #import pdb;pdb.set_trace() self.get_env_url() self.mongo_mysql_connection() input_file = "/input/hbogo_ids" input_data = lib_common_modules().read_csv(input_file) result_sheet = '/result/%s_ott_%s_checking%s.csv' % ( self.service, thread_name, datetime.date.today()) output_file = lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: self.writer = csv.writer(mycsvfile, dialect="csv", lineterminator='\n') self.writer.writerow(self.fieldnames) for _id in range(start_id, end_id): print({"skip": _id}) self.hbogo_id = str(input_data[_id][0]) self.show_type = str(input_data[_id][2]) self.show_type = 'MO' if self.show_type == 'OT' else self.show_type self.count_hbogo_id += 1 print("\n") print datetime.datetime.now() print("\n") print({ "count_hbogo_id": self.count_hbogo_id, "show_type": self.show_type, "id": self.hbogo_id, "thread_name": thread_name }) only_mapped_ids = ott_meta_data_validation_modules( ).getting_mapped_px_id_mapping_api(str(self.hbogo_id), self.source_mapping_api, self.projectx_mapping_api, self.show_type, self.source, self.token) self.ott_checking(only_mapped_ids, thread_name) print("\n") print({"count_hbogo_id": self.count_hbogo_id, "name": thread_name}) output_file.close() self.connection.close() self.cur.close()
def main(self,start_id,thread_name,end_id): #import pdb;pdb.set_trace() self.get_env_url() self.mongo_mysql_connection() result_sheet='/result/headrun_netflix_meta_data_checking%s_%s.csv'%(thread_name,datetime.date.today()) output_file=lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: fieldnames = ["%s_id"%self.source,"Projectx_id","show_type","%s_title"%self.source,"Px_title" ,"Px_episode_title","title_match","description_match","genres_match","aliases_match", "release_year_match","duration_match","season_number_match","episode_number_match", "px_video_link_present","%s_link_present"%self.source,"image_url_missing","Wrong_url", "credit_match","credit_mismatch"] self.writer = csv.writer(mycsvfile,dialect="excel",lineterminator = '\n') self.writer.writerow(fieldnames) projectx_id=0 source_id=0 #import pdb;pdb.set_trace() for _id in range(start_id,end_id,1000): query_headrun_netflix=self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":[ "movie","episode","tvshow" ]}},{"service":"netflix"}]}} ,{"$project":{"id":1,"_id":0,"item_type":1,"series_id":1,"title":1,"episode_title":1,"release_year":1, "episode_number":1,"season_number":1,"duration":1,"image_url":1,"url":1,"description":1,"cast":1,"directors":1,"writers":1, "categories":1,"genres":1,"maturity_ratings":1,"purchase_info":1,"service":1}},{"$skip":_id},{"$limit":1000}]) #query_headrun_netflix=self.sourcetable.find({"service":"netflix","item_type":"episode","id":"80192967"}) for data in query_headrun_netflix: if data.get("id")!="": #import pdb;pdb.set_trace() headrun_netflix_id=data.get("id") if data.get("item_type")=="movie": show_type="MO" elif data.get("item_type")=="tvshow": show_type="SM" else: show_type="SE" self.count_headrun_netflix_id+=1 print("\n") print datetime.datetime.now() print ("\n") print("%s_id:"%self.source,"id:",headrun_netflix_id,"count_headrun_netflix_id:" +str(self.count_headrun_netflix_id),"name:"+str(thread_name)) only_mapped_ids=ott_meta_data_validation_modules().getting_mapped_px_id_mapping_api(headrun_netflix_id,self.source_mapping_api ,self.projectx_mapping_api,show_type,self.source,self.token) if only_mapped_ids["source_flag"]=='True': self.total+=1 #import pdb;pdb.set_trace() projectx_id=only_mapped_ids["px_id"] source_id=only_mapped_ids["source_id"] print("\n") print ({"total":self.total,"id":headrun_netflix_id,"Px_id":projectx_id, "%s_id"%self.source:source_id,"thread_name":thread_name,"source_map":only_mapped_ids['source_map']}) self.meta_data_validation_(data,projectx_id,source_id,show_type,thread_name,only_mapped_ids) output_file.close() self.connection.close()
def main(self, start_id, thread_name, end_id): #import pdb;pdb.set_trace() self.get_env_url() self.mongo_mysql_connection() result_sheet = '/result/headrun_usanetwork_ott_%s_checking%s.csv' % ( thread_name, datetime.date.today()) output_file = lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: self.writer = csv.writer(mycsvfile, dialect="excel", lineterminator='\n') self.writer.writerow(self.fieldnames) for _id in range(start_id, end_id, 10): print({"skip": _id}) query_headrun_usanetwork = self.executed_query(_id) #query_headrun_usanetwork=self.sourcetable.find({"service":"nbc","item_type":"movie","id":"70301275"}) for data in query_headrun_usanetwork: if data.get("id") != "": #import pdb;pdb.set_trace() self.headrun_usanetwork_id = data.get("id").encode( "ascii", "ignore") if data.get("item_type") == "movie": self.show_type = "MO" #data.get("item_type") else: self.show_type = "SE" self.count_headrun_usanetwork_id += 1 print("\n") print datetime.datetime.now() print("\n") print({ "count_headrun_usanetwork_id": self.count_headrun_usanetwork_id, "show_type": self.show_type, "thread_name": thread_name, "nbc_id": self.headrun_usanetwork_id }) #import pdb;pdb.set_trace() only_mapped_ids = ott_meta_data_validation_modules( ).getting_mapped_px_id_mapping_api( str(self.headrun_usanetwork_id), self.source_mapping_api, self.projectx_mapping_api, self.show_type, self.source, self.token) self.ott_checking(only_mapped_ids, thread_name, data) print("\n") print({ "count_headrun_usanetwork_id": self.count_headrun_usanetwork_id, "name": thread_name }) output_file.close() self.connection.close()
def main(self, start_id, thread_name, end_id): self.get_env_url() input_file = "/input/projectx_sm_ids_season_0" input_data = lib_common_modules().read_csv(input_file) result_sheet = '/result/season_number_0_%s_checking%s.csv' % ( thread_name, datetime.date.today()) output_file = lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: self.writer = csv.writer(mycsvfile, dialect="csv", lineterminator='\n') self.writer.writerow(self.fieldnames) for _id in range(start_id, end_id): self.cleanup() self.projetcx_sm_id = str(input_data[_id][0]) self.total += 1 try: self.season_0_checking(thread_name, self.projetcx_sm_id, self.token) print("\n") print({ "projectx_sm_id": self.projetcx_sm_id, "season_number_0": self.season_number_0, "season_program_id": self.season_program_id, "result": self.result, "total": self.total, "thread": thread_name }) self.writer.writerow([ self.projetcx_sm_id, self.season_number_0, self.season_program_id, self.season_name, self.result, self.comment, self.Serving_Sources_season_program_id ]) except Exception as Error: print("Got exception in main............", self.projetcx_sm_id, Error) pass output_file.close()
def main(csv_file1, csv_file2): result_sheet = '/output/ids.csv' output_file = lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: writer = csv.writer(mycsvfile, dialect="csv", lineterminator='\n') writer.writerow(["ids"]) #import pdb;pdb.set_trace() print(input_file1) print("\n") print(input_file2) for id_ in input_file2: if id_ not in input_file1: print(id_) writer.writerow(id_)
def main(self, start_id, thread_name, end_id): #import pdb;pdb.set_trace() self.get_env_url() self.mongo_mysql_connection() result_sheet = '/result/gracenote_ott_%s_checking%s.csv' % ( thread_name, datetime.date.today()) output_file = lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: self.writer = csv.writer(mycsvfile, dialect="excel", lineterminator='\n') self.writer.writerow(self.fieldnames) for _id in range(start_id, end_id, 100): self.logger.debug({"skip": _id}) query_gracenote = self.executed_query(_id, 100) for data in query_gracenote: if data.get("Videos") != {}: #import pdb;pdb.set_trace() gracenote_id = data.get("sequence_id").encode( "ascii", "ignore") if data.get("show_type") == "SM": show_type = "SE" else: show_type = "MO" self.count_gracenote_id += 1 self.logger.debug("\n") self.logger.debug(datetime.datetime.now()) self.logger.debug("\n") self.logger.debug({ "count_gracenote_id": self.count_gracenote_id, "thread_name": thread_name, "details": data }) #import pdb;pdb.set_trace() only_mapped_ids = ott_meta_data_validation_modules( ).getting_mapped_px_id_mapping_api( str(gracenote_id), self.source_mapping_api, self.projectx_mapping_api, show_type, self.source, self.token) #TODO: Sub_main : here OTT validation result printed self.ott_checking(only_mapped_ids, gracenote_id, show_type, thread_name, data) self.logger.debug("\n") self.logger.debug({ "count_gracenote_id": self.count_gracenote_id, "name": thread_name }) output_file.close() self.connection.close()
def main(self, start_id, thread_name, end_id): #import pdb;pdb.set_trace() self.get_env_url() self.mongo_mysql_connection() result_sheet = '/result/%s_ott_%s_checking%s.csv' % ( self.service, thread_name, datetime.date.today()) output_file = lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: self.writer = csv.writer(mycsvfile, dialect="csv", lineterminator='\n') self.writer.writerow(self.fieldnames) for _id in range(start_id, end_id, 100): print({"skip": _id}) #import pdb;pdb.set_trace() query_starz = self.cur.execute( "SELECT * FROM starz_programs where item_type in ('episode','movie') limit %d,100" % (_id)) query_starz_result = self.cur.fetchall() for data in query_starz_result: if data.get("source_program_id") != "" and data.get( "source_program_id") is not None: #import pdb;pdb.set_trace() self.starz_id = data.get("source_program_id") self.show_type = data.get("item_type") self.show_type = 'MO' if self.show_type == 'movie' else self.show_type self.show_type = 'SE' if self.show_type == 'episode' else self.show_type self.count_starz_id += 1 print("\n") print datetime.datetime.now() print("\n") print({ "count_starz_id": self.count_starz_id, "show_type": self.show_type, "id": self.starz_id, "thread_name": thread_name }) #import pdb;pdb.set_trace() only_mapped_ids = ott_meta_data_validation_modules( ).getting_mapped_px_id_mapping_api( str(self.starz_id), self.source_mapping_api, self.projectx_mapping_api, self.show_type, self.source, self.token) self.ott_checking(only_mapped_ids, thread_name, data) print("\n") print({"count_starz_id": self.count_starz_id, "name": thread_name}) output_file.close() self.connection.close() self.cur.close()
def reverse_api_extract(self, link_details,source,show_type): #import pdb;pdb.set_trace() ## TODO: Only first index has been referred. reverse_api=self.reverse_api_domain%(link_details,self.service) reverse_api_resp=lib_common_modules().fetch_response_for_api_(reverse_api,self.token) for data in reverse_api_resp: if data.get("data_source")=='Rovi' and data.get("type")=='Program' and data.get("sub_type")=='ALL': if data.get("projectx_id") not in self.px_id: self.px_id.append(data.get("projectx_id")) if data.get("data_source")!=source and data.get("type")=='Program' and data.get("sub_type")==show_type: if data.get("projectx_id") not in self.px_id: self.px_id.append(data.get("projectx_id")) if data.get("data_source")==source and data.get("type")=='Program' and data.get("sub_type")==show_type: if data.get("projectx_id") not in self.px_id: self.px_id.append(data.get("projectx_id"))
def to_check_only_mapping_to_source(self, gracenote_id, show_type, thread_name): try: only_mapped_ids = ott_meta_data_validation_modules( ).getting_mapped_px_id_mapping_api(str(gracenote_id), self.source_mapping_api, self.projectx_mapping_api, show_type, self.source, self.token) self.logger.debug({ "%s_id" % self.source: gracenote_id, "count_gracenote_id": str(self.count_gracenote_id), "name": str(thread_name), "only_mapped_ids": only_mapped_ids["source_flag"] }) if only_mapped_ids["source_flag"] == 'True': source_details = lib_common_modules().fetch_response_for_api_( self.gracenote_api % (show_type, gracenote_id), self.token) self.total += 1 #import pdb;pdb.set_trace() projectx_id = only_mapped_ids["px_id"] source_id = only_mapped_ids["source_id"] self.logger.debug("\n") self.logger.debug({ "total": self.total, "MO_id": gracenote_id, "Px_id": projectx_id, "%s_id" % self.source: source_id, "thread_name": thread_name }) #self.meta_data_validation_(projectx_id,source_id,show_type,thread_name,only_mapped_ids,source_details) return ({ "projectx_id": projectx_id, "only_mapped_ids": only_mapped_ids, "source_details": source_details }) except (Exception, URLError, HTTPError, httplib.BadStatusLine) as error: self.logger.debug({ "exception caught in to_check_only_mapping_to_source func.. ": str(type(error), gracenote_id, show_type) }) self.to_check_only_mapping_to_source(gracenote_id, show_type, thread_name)
def check_mapping_px_id(self, mapping_from_source_api, token): try: mapping_from_source_api_response = lib_common_modules( ).fetch_response_for_api_(mapping_from_source_api, token) if mapping_from_source_api_response: return mapping_from_source_api_response[0]["projectx_id"] else: return "NA" except (Exception, URLError, HTTPError) as error: raise (error, error.reason, mapping_from_source_api, token) self.retry_count += 1 if self.retry_count <= 10: self.check_mapping_px_id(source_id, mapping_from_source_api, token) else: self.retry_count = 0
def main(self,start_id,thread_name,end_id): #import pdb;pdb.set_trace() self.get_env_url() self.mongo_mysql_connection() result_sheet='/result/hbogo_meta_data_checking%s_%s.csv'%(thread_name,datetime.date.today()) output_file=lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: self.writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n') self.writer.writerow(self.fieldnames) projectx_id=0 #import pdb;pdb.set_trace() for _id in range(start_id,end_id,1000): query_hbogo=self.cur.execute("SELECT * FROM hbogo_programs where (expired_at is null or expired_at > '%s') and expired='0' limit %d,1000 "%(self.running_datetime,_id)) #query_hbogo=self.cur.execute("SELECT * FROM hbogo_programs where launch_id='urn:hbo:feature:GVU4OxQ8lp47DwvwIAb24' and (expired_at is null or expired_at > '%s') "%(self.running_datetime)) query_hbogo_result=self.cur.fetchall() for data in query_hbogo_result: if data.get("launch_id")!="" and data.get("launch_id") is not None: #import pdb;pdb.set_trace() self.hbogo_id=data.get("launch_id") self.show_type=data.get("show_type") if self.show_type is not None and self.show_type!='SN': self.show_type='MO' if self.show_type=='OT' else self.show_type self.count_hbogo_id+=1 print("\n") print datetime.datetime.now() print ("\n") print("%s_id:"%self.source,"id:",self.hbogo_id,"count_hbogo_id:" +str(self.count_hbogo_id),"show_type:"+self.show_type,"name:"+str(thread_name)) only_mapped_ids=ott_meta_data_validation_modules().getting_mapped_px_id_mapping_api(self.hbogo_id,self.source_mapping_api,self.projectx_mapping_api,self.show_type,self.source,self.token) try: if only_mapped_ids["source_flag"]=='True': self.total+=1 #import pdb;pdb.set_trace() projectx_id=only_mapped_ids["px_id"] print("\n") print ({"total":self.total,"id":self.hbogo_id,"Px_id":projectx_id, "thread_name":thread_name,"source_map":only_mapped_ids['source_map']}) self.meta_data_validation_(data,projectx_id,thread_name,only_mapped_ids) except Exception as e: print ("got exception in main....",self.hbogo_id ,self.show_type,only_mapped_ids,type(e),thread_name) pass output_file.close() self.connection.close() self.cur.close()
def main(self,start_id,thread_name,end_id): #import pdb;pdb.set_trace() self.get_env_url() self.mongo_mysql_connection() result_sheet='/result/headrun_showtime_meta_data_checking%s_%s.csv'%(thread_name,datetime.date.today()) output_file=lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile: self.writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n') self.writer.writerow(self.fieldnames) projectx_id=0 for _id in range(start_id,end_id,100): query_headrun_showtime=self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":[ "movie","episode","tvshow" ]}},{"service":"showtime"}]}},{"$project":{"id":1,"_id":0,"item_type":1,"series_id":1,"title":1,"episode_title":1,"release_year":1,"episode_number":1,"season_number":1,"duration":1,"image_url":1,"url":1,"description":1,"cast":1,"directors":1,"writers":1, "categories":1,"genres":1,"maturity_ratings":1,"purchase_info":1,"service":1}},{"$skip":_id},{"$limit":100}]) #query_headrun_showtime=self.sourcetable.find({"service":"showtime","id":"3481853"}) for data in query_headrun_showtime: if data.get("id")!="": self.headrun_showtime_id=data.get("id") if data.get("item_type")=="movie": self.show_type="MO" elif data.get("item_type")=="tvshow": self.show_type="SM" else: self.show_type="SE" self.count_headrun_showtime_id+=1 print("\n") print(datetime.datetime.now()) print ("\n") print("%s_id:"%self.source,"id:",self.headrun_showtime_id,"count_headrun_showtime_id:" +str(self.count_headrun_showtime_id),"name:"+str(thread_name)) only_mapped_ids=ott_meta_data_validation_modules().getting_mapped_px_id_mapping_api(self.headrun_showtime_id,self.source_mapping_api,self.projectx_mapping_api,self.show_type,self.source,self.token) try: if only_mapped_ids["source_flag"]=='True': self.total+=1 projectx_id=only_mapped_ids["px_id"] print("\n") print ({"total":self.total,"id":self.headrun_showtime_id,"Px_id":projectx_id, "thread_name":thread_name,"source_map":only_mapped_ids['source_map']}) self.meta_data_validation_(data,projectx_id,thread_name,only_mapped_ids) except Exception as e: print ("got exception in main....",self.headrun_showtime_id ,self.show_type,only_mapped_ids,type(e),thread_name) pass output_file.close() self.connection.close()
def getting_projectx_ott_details(self, projectx_id, show_type): px_video_link = [] px_video_link_present = 'False' px_response = 'Null' launch_id = [] projectx_api = self.projectx_programs_api % (projectx_id, self.service) data_px_resp = lib_common_modules().fetch_response_for_api_( projectx_api, self.token) if data_px_resp != []: for data in data_px_resp: px_video_link = data.get("videos") if px_video_link: px_video_link_present = 'True' for linkid in px_video_link: launch_id.append(linkid.get("launch_id")) return { "px_video_link_present": px_video_link_present, "launch_id": launch_id } else: return px_response
def get_mappings_sources(self, projectx_id, mapping_api, token): try: response_array = [] mapping_response = lib_common_modules().fetch_response_for_api_( mapping_api, token) if mapping_response: for response in mapping_response: response_array.append([ response['data_source'] + " => " + response["source_id"] ]) return response_array else: return except (Exception, URLError, HTTPError) as error: raise (error, error.reason, mapping_api, token) self.retry_count += 1 if self.retry_count <= 10: self.get_mappings_sources(projectx_id, mapping_api, token) else: self.retry_count = 0
def getting_projectx_ott_details(self,projectx_id,show_type): px_video_link=[] px_video_link_present='False' px_response='Null' launch_id=[] fetch_from=[] #import pdb;pdb.set_trace() projectx_api=self.projectx_programs_api%(projectx_id,self.source.lower()) data_px_resp=lib_common_modules().fetch_response_for_api_(projectx_api,self.token) if data_px_resp!=[]: for data in data_px_resp: px_video_link= data.get("videos") if px_video_link: px_video_link_present='True' for linkid in px_video_link: launch_id.append(linkid.get("launch_id")) if linkid.get("fetched_from") not in fetch_from: fetch_from.append(linkid.get("fetched_from")) return {"px_video_link_present":px_video_link_present,"launch_id":launch_id,"fetch_from":fetch_from} else: return px_response
def getting_projectx_ott_details(self, projectx_id, show_type): px_video_link = [] fetched_from = [] px_video_link_present = 'False' androidtv_id_format = 'False' appletv_id_format = 'False' px_response = 'Null' launch_id = [] projectx_api = self.projectx_programs_api % (projectx_id, self.source.lower()) data_px_resp = lib_common_modules().fetch_response_for_api_( projectx_api, self.token) if data_px_resp != []: for data in data_px_resp: px_video_link = data.get("videos") if px_video_link: px_video_link_present = 'True' for linkid in px_video_link: launch_id.append(linkid.get("launch_id")) if linkid.get("fetched_from") not in fetched_from: fetched_from.append( linkid.get("fetched_from").encode("utf-8")) if linkid.get("appletv_id") == '{}{}{}'.format( self.source.lower(), '://', linkid.get("launch_id")): appletv_id_format = 'True' if linkid.get("androidtv_id") == '{}{}{}'.format( self.source.lower(), '://', linkid.get("launch_id")): androidtv_id_format = 'True' return { "px_video_link_present": px_video_link_present, "launch_id": launch_id, "appletv_id_format": appletv_id_format, "androidtv_id_format": androidtv_id_format, "fetched_from": fetched_from } else: return px_response
def duplicate_checking_series(self, series_data, source, writer,writer_credit_match_false,thread_name): #import pdb;pdb.set_trace() duplicate="" credit_array=[] print("\nFunction duplicate_checking_series_called.................") self.id=series_data.get("id").encode("ascii","ignore") self.show_id=series_data.get("series_id").encode("ascii","ignore") self.release_year=series_data.get("release_year").encode("ascii","ignore") self.series_title=unidecode.unidecode(pinyin.get(series_data.get("title"))) self.episode_title=unidecode.unidecode(pinyin.get(series_data.get("episode_title"))) if self.series_title is not None: data_expired_api_resp = lib_common_modules().link_expiry_check_(self.expire_api, self.preprod_domain_name,self.id,self.service,self.Token) if data_expired_api_resp: self.link_expired = "False" if data_expired_api_resp=='False' else "True" self.reverse_api_extract(self.id, source,'SE') print ("projectx_ids : {},fox_se_id: {}, fox_SM_id: {}".format(self.px_id,self.id,self.show_id),"thread_name:",thread_name) if len(self.px_id) > 1: px_link=self.projectx_preprod_api%'{}'.format(",".join([str(data) for data in self.px_id])) data_resp_link=lib_common_modules().fetch_response_for_api_(px_link,self.token) for id_ in data_resp_link: if id_.get("series_id") not in self.series_id_px: self.series_id_px.append(id_.get("series_id")) print ("projectx_ids_series : {0}".format(self.series_id_px),"thread:",thread_name) if len(self.series_id_px)>1: data_resp_search=duplicate_script_modules().search_api_call_response(self.series_title,self.projectx_preprod_search_api, self.projectx_domain,self.token) if data_resp_search is not None: result=duplicate_script_modules().search_api_response_validation(data_resp_search, source, self.series_id_px, duplicate, 'SM',self.token,self.projectx_preprod_api, self.projectx_mapping_api,self.beta_programs_api ,self.duplicate_api,self.credit_db_api) if (self.credit_match=='False' or self.credit_match=='') and len(result["search_px_id"])==2: #import pdb;pdb.set_trace() px_link=self.projectx_preprod_api %'{}'.format(",".join([str(i) for i in result["search_px_id"]])) data_resp_credits=lib_common_modules().fetch_response_for_api_(px_link,self.token) #import pdb;pdb.set_trace() for uu in data_resp_credits: if uu.get("credits"): for tt in uu.get("credits"): credit_array.append(unidecode.unidecode(tt.get("full_credit_name"))) if credit_array: for cc in credit_array: if credit_array.count(cc)>1: self.credit_match='True' break else: self.credit_match='False' result_credit_match_false=duplicate_script_modules().validation().meta_data_validation(result["search_px_id"],self.projectx_preprod_api ,self.credit_db_api,source,self.token) #import pdb;pdb.set_trace() writer_credit_match_false.writerow(result_credit_match_false) writer.writerow([source,self.show_id,self.id,'SE','',self.series_title,self.episode_title ,self.release_year,self.link_expired,self.series_id_px,'','','',result["comment"],result["comment"], result["duplicate"],result["search_px_id"],self.credit_match,result["count_rovi"],result["count_guidebox"],result["count_source"], result["count_hulu"],result["count_vudu"],result["rovi_mapping"],result["guidebox_mapping"],result["source_mapping"],result["hulu_mapping"],result["vudu_mapping"], result["comment_variant_parent_id_present"],result["comment_variant_parent_id"]]) else: duplicate_api=self.duplicate_api%(self.show_id,source,'SM') data_resp_duplicate=lib_common_modules().fetch_response_for_api_(duplicate_api,self.token) if data_resp_duplicate: duplicate='True' else: duplicate='False' self.comment="search_api_has_no_response" self.result="search_api_has_no_response" writer.writerow([source,self.show_id,self.id,'SE', '',self.series_title, self.episode_title, self.release_year, self.link_expired, self.series_id_px, '', '', '' ,self.comment,self.result,duplicate]) else: self.comment=('No multiple ids for this series',self.id,self.show_id) self.result="No multiple ids for this series" writer.writerow([source, self.show_id, self.id,'SE' ,'',self.series_title, self.episode_title, self.release_year, self.link_expired, self.series_id_px, '', '', '' ,self.comment,self.result]) else: self.comment=('No multiple ids for this episode',self.id,self.show_id) self.result="No multiple ids for this episode" writer.writerow([source, self.show_id, self.id,'SE', '',self.series_title, self.episode_title, self.release_year, self.link_expired, self.px_id, '', '', '' ,self.comment,self.result]) else: self.comment=('No series_title for this episode',self.id,self.show_id) self.result="No series_title for this episode" writer.writerow([source, self.show_id, self.id,'SE', '',self.series_title, self.episode_title, self.release_year, '', '', '', '', '' ,self.comment,self.result])
def main(self,start_id,thread_name,end_id): #import pdb;pdb.set_trace() inputFile="input/input_file" input_data=lib_common_modules().read_csv(inputFile) self.default_param() self.logger=lib_common_modules().create_log(os.getcwd()+'/logs/log_%s.txt'%thread_name) result_sheet='/output/output_file_%s_%s.csv'%(thread_name,datetime.date.today()) output_file=lib_common_modules().create_csv(result_sheet) self.column_fieldnames=["Service_name","Link_id","Rovi_id","Projectx_id","Rovi_id_form_reverse_api","Rovi_id_match","Link_expired", "Prod_link_status","Preprod_link_status","Link_fetched_from","Fetched_from_sources","Comment"] with output_file as mycsvfile: self.writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n') self.writer.writerow(self.column_fieldnames) for id_ in range(start_id,end_id): self.cleanup() self.getting_input_from_sheet(input_data,id_) self.total+=1 reverse_api_response=lib_common_modules().fetch_response_for_api(self.reverse_mapping_api%(self.host_IP, self.link_id,self.service),self.token,self.logger) if reverse_api_response: for data in reverse_api_response: if data["projectxId"] not in self.px_id and data["data_source"]=='Rovi' and data["type"]=='Program': self.px_id.append(data["projectxId"]) self.source_id.append(data["sourceId"]) if self.rovi_id in self.source_id: """comment""" self.same_rovi_id_present='True' if self.px_id: self.link_expired=lib_common_modules().link_expiry_check(self.expired_api,self.prod_domain,self.link_id,self.service,self.expired_token,self.logger) prod_api_response=lib_common_modules().fetch_response_for_api(self.programs_api%(self.prod_domain, self.source_id[0]),self.token,self.logger) self.prod_videos_response=[data['videos'] for data in prod_api_response] if self.prod_videos_response[0]: for data in self.prod_videos_response[0]: if data["source_id"]==self.service: self.prod_launch_id.append(data["launch_id"]) self.prod_video_link.append(data["link"]["uri"]) link_status_prod=self.link_ingestion_check(self.prod_launch_id,self.link_id, self.prod_video_link) if link_status_prod["link_id_present"]=="True": """comment""" self.prod_link_status="link_id_present_in_Prod" else: """comment""" self.prod_link_status='videos_not_available' preprod_api_response=lib_common_modules().fetch_response_for_api(self.programs_api%(self.preprod_domain, self.px_id[0]),self.token,self.logger) self.preprod_videos_response=[data['videos'] for data in preprod_api_response] if self.preprod_videos_response[0]: for data in self.preprod_videos_response[0]: if data["source_id"]==self.service: if data["launch_id"] not in self.preprod_launch_id: self.preprod_launch_id.append(data["launch_id"]) if data["link"]["uri"] not in self.preprod_video_link: self.preprod_video_link.append(data["link"]["uri"]) if data["fetched_from"] not in self.fetch_from: self.fetch_from.append(data["fetched_from"]) if self.preprod_launch_id or self.preprod_video_link: link_status_preprod= self.link_ingestion_check(self.preprod_launch_id,self.link_id, self.preprod_video_link) if link_status_preprod["link_id_present"]=='True': """comment""" self.preprod_link_status="link_id_present_in_Preprod" for source in self.fetch_from: if 'Rovi' in self.fetch_from and len(self.fetch_from)==1: self.fetched_source='Rovi' elif 'Rovi' in self.fetch_from and len(self.fetch_from)>1: self.fetched_source='Rovi+others' elif 'Rovi' not in self.fetch_from and len(self.fetch_from)>1: self.fetched_source='others' else: self.fetched_source='others' else: """comment""" self.preprod_link_status='videos_not_available' else: self.link_expired=lib_common_modules().link_expiry_check(self.expired_api,self.prod_domain,self.link_id,self.service,self.expired_token,self.logger) else: self.link_expired=lib_common_modules().link_expiry_check(self.expired_api,self.prod_domain,self.link_id,self.service,self.expired_token,self.logger) self.logger.debug("\n") if self.same_rovi_id_present=="True" and self.preprod_link_status=='link_id_present_in_Preprod': self.pass_count+=1 self.comment='Pass' else: self.fail_count+=1 self.comment='Fail' self.logger.debug (["date time:", datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")]) self.logger.debug("\n") self.logger.debug ([{"total":self.total,"pass":self.pass_count,"fail":self.fail_count,"Process":thread_name}]) self.logger.debug("\n") self.logger.debug([{"Service":self.service,"link_id":self.link_id,"Rovi_id":self.rovi_id, "px_id":self.px_id,"rovi_id_form_reverse_api":self.source_id,"rovi_id_status":self.same_rovi_id_present,"Link_expired":self.link_expired, "Prod_link_status":self.prod_link_status,"preprod_link_status":self.preprod_link_status,"link_fetched":self.fetched_source, "comment":self.comment,"Process":thread_name}]) self.writer.writerow([self.service,self.link_id,self.rovi_id,self.px_id,self.source_id,self.same_rovi_id_present, self.link_expired,self.prod_link_status,self.preprod_link_status,self.fetched_source,self.fetch_from, self.comment]) output_file.close()
def main(self,start_id,thread_name,end_id): #import pdb;pdb.set_trace() self.default_param() self.mongo_connection() fieldnames_tag = ["source","sm_id","id","show_type","movie_title","series_title","episode_title", "release_year","link_expired","Projectx_ids","Variant id present for episode", "Variant parent id for Episode","Projectx_series_id","comment","Result","Gb_id duplicate", 'Duplicate ids in search',"Credit_match","Rovi_id count for duplicate","Gb_id count for duplicate", "%s_id count for duplicate"%self.source,"Hulu_id count","vudu_id_count","Mapped rovi_id for duplicate","Mapped Gb_id for duplicate", "Mapped %s_id for duplicate"%self.source,"Mapped_hulu_id","Mapped_vudu_id","variant_parent_id_present","variant_parent_id","link_fetched_from", "Sources of link"] fieldnames_credit_match_false = ["px_id1","px_id1_show_type","px_id1_variant_parent_id","px_id1_is_group_language_primary", "px_id1_record_language","px_id2","px_id2_show_type","px_id2_variant_parent_id","px_id2_is_group_language_primary", "px_id2_record_language","px_id1_credits_null","px_id1_db_credit_present","px_id2_credits_null","px_id2_db_credit_present", "long_title_match","original_title_match","runtime_match","release_year_match","alias_title_match","comment"] result_sheet_credit_match_false='/output_credit_match_false/duplicate_programs_credit_match_false_result_%s_%s.csv'%(thread_name,datetime.date.today()) output_file_credit_match_false=lib_common_modules().create_csv(result_sheet_credit_match_false) result_sheet='/output/duplicate_checked_in_search_api_%s_%s.csv'%(thread_name,datetime.date.today()) output_file=lib_common_modules().create_csv(result_sheet) with output_file as mycsvfile, output_file_credit_match_false as mycsvfile1: writer = csv.writer(mycsvfile,dialect="excel",lineterminator = '\n') writer.writerow(fieldnames_tag) writer_credit_match_false = csv.DictWriter(mycsvfile1,fieldnames=fieldnames_credit_match_false,dialect="excel",lineterminator = '\n') writer_credit_match_false.writeheader() mo_list=[] sm_list=[] for _id in range(start_id,end_id,100): try: query=self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":["episode"]}},{"service":"fox"}]}} ,{"$project":{"id":1,"_id":0,"item_type":1,"series_id":1,"title":1,"episode_title":1,"release_year":1, "episode_number":1,"season_number":1,"duration":1,"image_url":1,"url":1,"description":1,"cast":1,"directors":1,"writers":1, "categories":1,"genres":1,"maturity_ratings":1,"purchase_info":1,"service":1}},{"$skip":_id},{"$limit":100}]) #query=self.sourcetable.find({"service":"netflix","item_type":"movie","id":"269880"}) print ("\n") print({"start": start_id,"end":end_id}) for data in query: series_title='' _id=data.get("id") sm_id=data.get("series_id") show_type=data.get("item_type").encode('utf-8') if _id is not None and _id!="": if show_type=='episode': if sm_id not in sm_list: sm_list.append(sm_id) self.cleanup() self.duplicate_checking_series(data,self.source,writer,writer_credit_match_false,thread_name) elif show_type=='movie': if _id not in mo_list: mo_list.append(_id) self.cleanup() self.duplicate_checking_movies(data,self.source,writer,writer_credit_match_false,thread_name) print("\n") print ({"Total SM":len(sm_list),"Total MO":len(mo_list),"Thread_name":thread_name}) print("\n") print datetime.datetime.now() except (pymongo.errors.CursorNotFound,Exception) as e: print ("exception caught in main func.............",type(e),thread_name) continue output_file.close() output_file_credit_match_false.close() self.connection.close()