def main(self,start_id,thread_name,end_id):
#import pdb;pdb.set_trace()
self.mongo_connection()
self.constant_param()
self.get_env_url()
self.logger=lib_common_modules().create_log(os.getcwd()+"/log/log.txt")
result_sheet='/output/%s_Ingestion_checked_in_Px_%s_%s.csv'%(self.source,thread_name,datetime.date.today())
output_file=lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
self.writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n')
self.writer.writerow(self.fieldnames)
for id_ in range(start_id,end_id,100):
self.total+=1
showtime_data = self.query_execute(id_)
for data in showtime_data:
self.getting_source_details(data)
if self.show_type is not None:
self.ingestion_checking(thread_name)
else:
self.writer.writerow([self.showtime_id,'',self.show_type,self.title,self.series_title,self.episode_title,self.showtime_show_id,self.season_number,self.episode_number,self.year,self.duration,self.updated_at,self.expiry_date])
self.logger.debug("\n")
self.logger.debug([{"Total":self.total,"ingested_count":self.ingested_count,
"not_ingested_count": self.not_ingested_count,"Thread_name":thread_name}])
self.logger.debug("\n")
self.logger.debug(["date time:", datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")])
output_file.close()
self.connection.close()
def main(self, start_id, thread_name, end_id):
#import pdb;pdb.set_trace()
self.get_env_url()
self.mongo_mysql_connection()
input_file = "/input/ids"
input_data = lib_common_modules().read_csv(input_file)
result_sheet = '/result/headrun_netflix_ott_%s_checking%s.csv' % (
thread_name, datetime.date.today())
output_file = lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
fieldnames = [
"%s_id" % self.source, "Series_id", "Projectx_id", "show_type",
"px_video_link_present",
"%s_link_present" % self.source, "ott_link_result", "mapping",
"", "Expired"
]
self.writer = csv.writer(mycsvfile,
dialect="csv",
lineterminator='\n')
self.writer.writerow(fieldnames)
for _id in range(start_id, end_id):
self.headrun_netflix_id = str(input_data[_id][0])
query_headrun_netflix = self.sourcetable.find({
"service":
"netflix",
"id":
"%s" % str(self.headrun_netflix_id)
})
for data in query_headrun_netflix:
if data.get("id") != "":
if data.get("item_type") == "movie":
self.show_type = "MO"
else:
self.show_type = "SE"
self.count_headrun_netflix_id += 1
print("\n")
print datetime.datetime.now()
print("\n")
print({
"count_headrun_netflix_id":
self.count_headrun_netflix_id,
"id": self.headrun_netflix_id,
"thread_name": thread_name
})
only_mapped_ids = ott_meta_data_validation_modules(
).getting_mapped_px_id_mapping_api(
str(self.headrun_netflix_id),
self.source_mapping_api, self.projectx_mapping_api,
self.show_type, self.source, self.token)
self.ott_checking(only_mapped_ids, thread_name, data)
print("\n")
print({
"count_headrun_netflix_id": self.count_headrun_netflix_id,
"name": thread_name
})
output_file.close()
self.connection.close()
def season_0_checking(self, thread_name, projectx_sm_id, token):
try:
self.seasons_api_response = lib_common_modules(
).fetch_response_for_api_(self.seasons_api % projectx_sm_id, token)
if self.seasons_api_response != []:
for seasons in self.seasons_api_response:
print("\n")
print({"seasons": seasons})
if seasons["season_number"] == 0:
self.season_number_0 = "Present"
self.season_program_id = seasons["season_program_id"]
self.season_name = unidecode.unidecode(
pinyin.get(seasons["season_name"]))
self.result = "Fail"
self.comment = "Season number 0 present in Season API"
self.all_episode_sequence_response = lib_common_modules(
).fetch_response_for_api_(
self.all_episode_sequence_api % projectx_sm_id,
token)
if self.all_episode_sequence_response != []:
for sequence in self.all_episode_sequence_response:
if sequence[
"season_program_id"] == self.season_program_id:
self.data_source = sequence["data_source"]
self.season_served = sequence[
"season_number"]
if {
str(self.data_source):
'Season ' + str(self.season_served)
} not in self.Serving_Sources_season_program_id:
self.Serving_Sources_season_program_id.append(
{
str(self.data_source):
'Season ' +
str(self.season_served)
})
if self.Serving_Sources_season_program_id == []:
self.Serving_Sources_season_program_id = [
"Season_program id not there in all episode sequence API"
]
else:
self.comment = "Empty_Season_response"
self.result = "Fail"
except (Exception, httplib.BadStatusLine, urllib2.HTTPError,
urllib2.URLError) as e:
self.retry_count += 1
print("Retrying...................................",
self.retry_count)
print("\n")
print(
"exception/error caught in season0_checking func...................",
type(e), projectx_sm_id, thread_name)
if self.retry_count <= 10:
self.season_0_checking(thread_name, projectx_sm_id, token)
else:
self.retry_count = 0
def ott_checking(self,only_mapped_ids,appletv_id,show_type,thread_name,data):
#import pdb;pdb.set_trace()
try:
if only_mapped_ids["source_flag"]=='True':
self.total+=1
#import pdb;pdb.set_trace()
projectx_id=only_mapped_ids["px_id"]
source_id=only_mapped_ids["source_id"]
self.logger.debug("\n")
self.logger.debug ({"total_only_appletv_mapped":self.total,"%s_id"%self.source:appletv_id,
"Px_id":projectx_id,"%s_id"%self.source:source_id,"thread_name":thread_name})
source_details=self.getting_source_details(source_id,show_type,data)
projectx_details=self.getting_projectx_ott_details(projectx_id,show_type)
if projectx_details!='Null':
if data.get("purchase_info")!="":
self.link_expired=lib_common_modules().link_expiry_check_(self.expired_api,self.projectx_domain,source_id,self.service,self.expired_token)
ott_validation_result=ott_meta_data_validation_modules().ott_validation(projectx_details,source_id)
self.writer.writerow([source_id,projectx_id,show_type,projectx_details["px_video_link_present"],
source_details["source_link_present"],ott_validation_result,only_mapped_ids["source_flag"],'',self.link_expired])
else:
self.writer.writerow([source_id,projectx_id,show_type,'','','',
only_mapped_ids["source_flag"],'purchase_info_null'])
else:
self.writer.writerow([source_id,projectx_id,show_type,projectx_details,
source_details["source_link_present"],'',only_mapped_ids["source_flag"],'Px_response_null'])
elif only_mapped_ids["source_flag"]=='True(Rovi+others)':
if show_type!='SM':
projectx_id=only_mapped_ids["px_id"]
source_id=only_mapped_ids["source_id"]
self.logger.debug("\n")
self.logger.debug ({"Px_id":projectx_id,"%s_id"%self.source:source_id,"thread_name":thread_name})
projectx_details=self.getting_projectx_ott_details(projectx_id,show_type)
if projectx_details!='Null':
if data.get("purchase_info")!="":
self.link_expired=lib_common_modules().link_expiry_check_(self.expired_api,self.projectx_domain,source_id,self.service,self.expired_token)
ott_validation_result=ott_meta_data_validation_modules().ott_validation(projectx_details,source_id)
self.writer.writerow([source_id,projectx_id,show_type,'','',ott_validation_result,
only_mapped_ids["source_flag"],'',self.link_expired])
else:
self.writer.writerow([source_id,projectx_id,show_type,'','','',
only_mapped_ids["source_flag"],'purchase_info_null'])
else:
self.writer.writerow([appletv_id,'',show_type,'','',only_mapped_ids,'Px_id_null'])
except (Exception,httplib.BadStatusLine,urllib2.HTTPError,socket.error,urllib2.URLError,RuntimeError) as e:
self.retry_count+=1
self.logger.debug("Retrying...................................",self.retry_count)
self.logger.debug("\n")
self.logger.debug ("exception/error caught in ott_checking func.........................",type(e),appletv_id,show_type,thread_name)
if self.retry_count<=5:
self.ott_checking(only_mapped_ids,appletv_id,show_type,thread_name,data)
else:
self.retry_count=0
def ingestion_checking(self,thread_name):
#import pdb;pdb.set_trace()
duplicate_px_id=[]
try:
showtime_mapping_api=self.source_mapping_api%(self.showtime_id,self.source,self.show_type.encode())
data_showtime_resp=lib_common_modules().fetch_response_for_api_(showtime_mapping_api,self.token)
if data_showtime_resp!=[]:
self.ingested_count+=1
#TODO: checking program response in Programs_Id search api
for response in data_showtime_resp:
self.projectx_id = response["projectx_id"]
program_response=lib_common_modules().fetch_response_for_api_(self.programs_api%self.projectx_id,self.token)
if program_response!=[]:
self.px_response='True'
self.writer.writerow([self.showtime_id,self.projectx_id,self.show_type,self.title,self.series_title,self.episode_title,self.showtime_show_id,self.season_number,self.episode_number,self.year,self.duration,self.updated_at,self.expiry_date,'','','True','',self.px_response])
# TODO : to check duplicate source_ids
else:
if self.show_type=='SM' or self.show_type=='MO':
showtime_duplicate_api=self.source_duplicate_api%(self.showtime_id,self.source,self.show_type.encode())
else:
#TODO: for episodes
showtime_duplicate_api=self.source_duplicate_api%(self.showtime_show_id,self.source,'SM')
data_showtime_resp_duplicate=lib_common_modules().fetch_response_for_api_(showtime_duplicate_api,self.token)
#import pdb;pdb.set_trace()
if data_showtime_resp_duplicate!=[]:
self.ingested_count+=1
for px_id in data_showtime_resp_duplicate:
duplicate_px_id.append(px_id.get("projectx_id"))
if self.show_type=='MO' or self.show_type=='SM':
source_id_duplicate=ingestion_script_modules().getting_duplicate_source_id(duplicate_px_id,self.px_mapping_api,self.show_type,self.token,self.source)
else:
#TODO : for episode
source_id_duplicate=ingestion_script_modules().getting_duplicate_source_id(duplicate_px_id,self.px_mapping_api,'SM',self.token,self.source)
self.writer.writerow([self.showtime_id,duplicate_px_id,self.show_type,self.title,self.series_title,self.episode_title,self.showtime_show_id,self.season_number,self.episode_number,self.year,self.duration,self.updated_at,self.expiry_date,'True',source_id_duplicate,'True'])
else:
self.not_ingested_count+=1
self.writer.writerow([self.showtime_id,'',self.show_type,self.title,self.series_title,self.episode_title,self.showtime_show_id,self.season_number,self.episode_number,self.year,self.duration,self.updated_at,self.expiry_date,'False','','','True',self.px_response])
self.logger.debug('\n')
self.logger.debug(["%s_id:"%self.source,self.showtime_id,"show_type:",self.show_type,thread_name,"updated:","title:",self.title,"series_title: ",self.series_title,"season_no:",self.season_number,"episode_no:",self.episode_number,"ingested_count:",self.ingested_count,
"not_ingested_count:", self.not_ingested_count])
except (Exception,httplib.BadStatusLine,urllib2.HTTPError,socket.error,URLError,RuntimeError,ValueError) as e:
#import pdb;pdb.set_trace()
self.retry_count+=1
self.logger.debug(["exception caught ingestion_checking func.........................",type(e),self.showtime_id,self.show_type,thread_name])
self.logger.debug(["Retrying.............",self.retry_count])
if self.retry_count<=5:
self.ingestion_checking(thread_name)
else:
self.retry_count=0
def getting_source_ott_details(self, gracenote_id, show_type, details):
#import pdb;pdb.set_trace()
gracenote_link = []
expired_link = []
gracenote_link_present = 'Null'
gracenote_link.append(details["Videos"])
if gracenote_link != []:
gracenote_link_present = 'True'
for links in gracenote_link:
for key in links.keys():
for link_id in links[key]:
service = self.to_check_service_proper_name(key)
self.link_expired = lib_common_modules().link_expiry_check(
self.expired_api, self.projectx_domain, link_id,
service.lower(), self.expired_token, self.logger)
if self.link_expired == 'True':
expired_link.append({key: link_id})
self.logger.debug("\n")
self.logger.debug({
"source_link_present": gracenote_link_present,
"link_id": gracenote_link,
"show_type": show_type,
"expired_link": expired_link,
"expired_status": self.link_expired
})
return {
"source_link_present": gracenote_link_present,
"link_id": gracenote_link,
"show_type": show_type,
"expired_link": expired_link,
"expired_status": self.link_expired
}
def getting_projectx_ott_details(self, projectx_id, show_type):
#import pdb;pdb.set_trace()
px_video_link = []
px_video_link_present = 'False'
px_response = 'Null'
launch_id = []
fetched_from = []
appletv_id_format = "False"
projectx_api = self.projectx_programs_api % (projectx_id, self.service)
data_px_resp = lib_common_modules().fetch_response_for_api_(
projectx_api, self.token)
if data_px_resp != []:
for data in data_px_resp:
px_video_link = data.get("videos")
if px_video_link:
px_video_link_present = 'True'
for linkid in px_video_link:
launch_id.append(linkid.get("launch_id"))
if linkid.get("fetched_from") not in fetched_from:
fetched_from.append(
linkid.get("fetched_from").encode("utf-8"))
if linkid.get("appletv_id") == '{}{}'.format(
"%s://www.%s.com/en-gb/video/" %
(self.source.lower(), self.source.lower()),
linkid.get("launch_id")):
appletv_id_format = 'True'
return {
"px_video_link_present": px_video_link_present,
"launch_id": launch_id,
"fetched_from": fetched_from,
"appletv_id_format": appletv_id_format
}
else:
return px_response
def main(self,start_id,thread_name,end_id):
#import pdb;pdb.set_trace()
self.get_env_url()
self.mongo_mysql_connection()
result_sheet='/result/headrun_Showtime_ott_%s_checking%s.csv'%(thread_name,datetime.date.today())
output_file=lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
fieldnames = ["%s_id"%self.source,"Projectx_id","show_type","px_video_link_present","Fetch_from","Series_id","Expiry_date","%s_link_present"%self.source,"ott_link_result","mapping","","Expired"]
self.writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n')
self.writer.writerow(fieldnames)
for _id in range(start_id,end_id,100):
print({"skip":_id})
query_headrun_Showtime=self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":["movie","episode"]}},{"service":self.source.lower()}]}},{"$project":{"id":1,"_id":0,"item_type":1,"series_id":1,"title":1,"episode_title":1,"release_year":1,"episode_number":1,"season_number":1,"duration":1,"image_url":1,"url":1,"description":1,"cast":1,"directors":1,"writers":1,"categories":1,"genres":1,"maturity_ratings":1,"purchase_info":1,"service":1,"expiry_date":1}},{"$skip":_id},{"$limit":100}])
#query_headrun_Showtime=self.sourcetable.find({"service":"Showtime","item_type":"movie","id":"70301275"})
for data in query_headrun_Showtime:
if data.get("id")!="":
#import pdb;pdb.set_trace()
self.headrun_Showtime_id=data.get("id").encode("ascii","ignore")
if data.get("item_type")=="movie":
self.show_type="MO"
else:
self.show_type="SE"
self.count_headrun_Showtime_id+=1
print("\n")
print datetime.datetime.now()
print("\n")
print ({"count_headrun_Showtime_id":self.count_headrun_Showtime_id,
"id":self.headrun_Showtime_id,"thread_name":thread_name})
#import pdb;pdb.set_trace()
only_mapped_ids=ott_meta_data_validation_modules().getting_mapped_px_id_mapping_api(str(self.headrun_Showtime_id),self.source_mapping_api,self.projectx_mapping_api,self.show_type,self.source,self.token)
self.ott_checking(only_mapped_ids,thread_name,data)
print("\n")
print ({"count_headrun_Showtime_id":self.count_headrun_Showtime_id,"name":thread_name})
output_file.close()
self.connection.close()
def ott_checking(self, only_mapped_ids, thread_name, data):
try:
if only_mapped_ids["source_flag"] == 'True' or only_mapped_ids[
"source_flag"] == 'True(Rovi+others)':
self.total += 1
projectx_id = only_mapped_ids["px_id"]
print("\n")
print({
"total_only_headrun_HBOMAX_mapped": self.total,
"%s_id" % self.source: self.headrun_HBOMAX_id,
"Px_id": projectx_id,
"thread_name": thread_name
})
projectx_details = self.getting_projectx_ott_details(
projectx_id, self.show_type)
if projectx_details != 'Null':
self.link_expired = lib_common_modules(
).link_expiry_check_(self.expired_api,
self.projectx_domain,
self.headrun_HBOMAX_id,
self.source.lower(),
self.expired_token)
ott_validation_result = ott_meta_data_validation_modules(
).ott_validation(projectx_details, self.headrun_HBOMAX_id)
self.writer.writerow([
self.headrun_HBOMAX_id, projectx_id, self.show_type,
projectx_details["px_video_link_present"],
data["series_id"], data["expiry_date"], '',
ott_validation_result, only_mapped_ids["source_flag"],
'', self.link_expired,
projectx_details["appletv_id_format"],
projectx_details["androidtv_id_format"],
projectx_details["fetched_from"]
])
else:
self.writer.writerow([
self.headrun_HBOMAX_id, projectx_id, self.show_type,
projectx_details, data["series_id"],
data["expiry_date"], '', '',
only_mapped_ids["source_flag"], 'Px_response_null'
])
else:
self.total += 1
self.writer.writerow([
self.headrun_HBOMAX_id, '', self.show_type, '',
data["series_id"], data["expiry_date"], '',
only_mapped_ids, 'Px_id_null'
])
except (Exception, httplib.BadStatusLine, urllib2.HTTPError,
socket.error, urllib2.URLError, RuntimeError) as e:
self.retry_count += 1
print("Retrying..............................", self.retry_count)
print("\n")
print(
"exception/error caught in ott_checking func...................",
type(e), self.headrun_HBOMAX_id, thread_name)
if self.retry_count <= 5:
self.ott_checking(only_mapped_ids, thread_name, data)
else:
self.retry_count = 0
def getting_source_ott_details(self, details):
#import pdb;pdb.set_trace()
gracenote_link = []
link_ids = []
expired_link = []
gracenote_link.append(details["Videos"])
for links in gracenote_link:
for key in links.keys():
for link_id in links[key]:
service = self.to_check_service_proper_name(key)
link_ids.append(link_id)
self.link_expired = lib_common_modules(
).link_expiry_check_(self.expire_api,
self.preprod_domain_name, link_id,
service.lower(), self.Token)
if self.link_expired == 'True':
expired_link.append({key: link_id})
else:
break
if link_ids:
self.link_expired = "False"
break
self.logger.debug("\n")
self.logger.debug({
"link_id": link_ids,
"service": service.lower(),
"expired_link": expired_link,
"expired_status": self.link_expired
})
return {
"link_id": link_ids,
"service": service.lower(),
"expired_link": expired_link,
"expired_status": self.link_expired
}
def main(self,start_id,thread_name,end_id):
self.get_env_url()
self.default_param()
self.mongo_connection()
result_sheet_credit_match_false='/output_credit_match_false/duplicate_programs_credit_match_false_result_%s_%s.csv'%(thread_name,datetime.date.today())
output_file_credit_match_false=lib_common_modules().create_csv(result_sheet_credit_match_false)
result_sheet='/output/duplicate_checked_in_search_api_%s_%s.csv'%(thread_name,datetime.date.today())
output_file=lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile, output_file_credit_match_false as mycsvfile1:
writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n')
writer.writerow(self.fieldnames_tag)
writer_credit_match_false = csv.DictWriter(mycsvfile1,fieldnames=self.fieldnames_credit_match_false,dialect="csv",lineterminator = '\n')
writer_credit_match_false.writeheader()
for _id in range(start_id,end_id,100):
try:
query=self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":["movie","episode"]}},{"service":"hbomax"}]}},{"$project":{"id":1,"_id":0,"item_type":1,"series_id":1,"title":1,"episode_title":1,"release_year":1,"episode_number":1,"season_number":1,"duration":1,"image_url":1,"url":1,"description":1,"cast":1,"directors":1,"writers":1,"categories":1,"genres":1,"maturity_ratings":1,"purchase_info":1,"service":1}},{"$skip":_id},{"$limit":100}])
#query= self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":["movie","episode"]}},{"service":"hbomax","id":"80131281"}]}}])
#import pdb;pdb.set_trace()
print ("\n")
print({"start": start_id,"end":end_id})
for data in query:
series_title=''
_id=data.get("id")
sm_id=data.get("series_id")
show_type=data.get("item_type").encode('utf-8')
if _id is not None and _id!="":
if show_type=='episode':
if sm_id not in self.sm_list:
self.sm_list.append(sm_id)
self.cleanup()
self.duplicate_checking_series(data,writer,writer_credit_match_false,thread_name)
elif show_type=='movie':
#import pdb;pdb.set_trace()
if _id not in self.mo_list:
self.mo_list.append(_id)
self.cleanup()
self.duplicate_checking_movies(data,writer,writer_credit_match_false,thread_name)
print("\n")
print ({"Total SM":len(self.sm_list),"Total MO":len(self.mo_list),"Thread_name":thread_name})
print("\n")
print (datetime.datetime.now())
except (pymongo.errors.CursorNotFound,Exception) as e:
print ("exception caught in main func.............",type(e),thread_name)
continue
output_file.close()
output_file_credit_match_false.close()
self.connection.close()
def main(self, start_id, thread_name, end_id):
#import pdb;pdb.set_trace()
self.get_env_url()
self.mongo_mysql_connection()
input_file = "/input/hbogo_ids"
input_data = lib_common_modules().read_csv(input_file)
result_sheet = '/result/%s_ott_%s_checking%s.csv' % (
self.service, thread_name, datetime.date.today())
output_file = lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
self.writer = csv.writer(mycsvfile,
dialect="csv",
lineterminator='\n')
self.writer.writerow(self.fieldnames)
for _id in range(start_id, end_id):
print({"skip": _id})
self.hbogo_id = str(input_data[_id][0])
self.show_type = str(input_data[_id][2])
self.show_type = 'MO' if self.show_type == 'OT' else self.show_type
self.count_hbogo_id += 1
print("\n")
print datetime.datetime.now()
print("\n")
print({
"count_hbogo_id": self.count_hbogo_id,
"show_type": self.show_type,
"id": self.hbogo_id,
"thread_name": thread_name
})
only_mapped_ids = ott_meta_data_validation_modules(
).getting_mapped_px_id_mapping_api(str(self.hbogo_id),
self.source_mapping_api,
self.projectx_mapping_api,
self.show_type, self.source,
self.token)
self.ott_checking(only_mapped_ids, thread_name)
print("\n")
print({"count_hbogo_id": self.count_hbogo_id, "name": thread_name})
output_file.close()
self.connection.close()
self.cur.close()
def main(self,start_id,thread_name,end_id):
#import pdb;pdb.set_trace()
self.get_env_url()
self.mongo_mysql_connection()
result_sheet='/result/headrun_netflix_meta_data_checking%s_%s.csv'%(thread_name,datetime.date.today())
output_file=lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
fieldnames = ["%s_id"%self.source,"Projectx_id","show_type","%s_title"%self.source,"Px_title"
,"Px_episode_title","title_match","description_match","genres_match","aliases_match",
"release_year_match","duration_match","season_number_match","episode_number_match",
"px_video_link_present","%s_link_present"%self.source,"image_url_missing","Wrong_url",
"credit_match","credit_mismatch"]
self.writer = csv.writer(mycsvfile,dialect="excel",lineterminator = '\n')
self.writer.writerow(fieldnames)
projectx_id=0
source_id=0
#import pdb;pdb.set_trace()
for _id in range(start_id,end_id,1000):
query_headrun_netflix=self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":[ "movie","episode","tvshow" ]}},{"service":"netflix"}]}}
,{"$project":{"id":1,"_id":0,"item_type":1,"series_id":1,"title":1,"episode_title":1,"release_year":1,
"episode_number":1,"season_number":1,"duration":1,"image_url":1,"url":1,"description":1,"cast":1,"directors":1,"writers":1,
"categories":1,"genres":1,"maturity_ratings":1,"purchase_info":1,"service":1}},{"$skip":_id},{"$limit":1000}])
#query_headrun_netflix=self.sourcetable.find({"service":"netflix","item_type":"episode","id":"80192967"})
for data in query_headrun_netflix:
if data.get("id")!="":
#import pdb;pdb.set_trace()
headrun_netflix_id=data.get("id")
if data.get("item_type")=="movie":
show_type="MO"
elif data.get("item_type")=="tvshow":
show_type="SM"
else:
show_type="SE"
self.count_headrun_netflix_id+=1
print("\n")
print datetime.datetime.now()
print ("\n")
print("%s_id:"%self.source,"id:",headrun_netflix_id,"count_headrun_netflix_id:"
+str(self.count_headrun_netflix_id),"name:"+str(thread_name))
only_mapped_ids=ott_meta_data_validation_modules().getting_mapped_px_id_mapping_api(headrun_netflix_id,self.source_mapping_api
,self.projectx_mapping_api,show_type,self.source,self.token)
if only_mapped_ids["source_flag"]=='True':
self.total+=1
#import pdb;pdb.set_trace()
projectx_id=only_mapped_ids["px_id"]
source_id=only_mapped_ids["source_id"]
print("\n")
print ({"total":self.total,"id":headrun_netflix_id,"Px_id":projectx_id,
"%s_id"%self.source:source_id,"thread_name":thread_name,"source_map":only_mapped_ids['source_map']})
self.meta_data_validation_(data,projectx_id,source_id,show_type,thread_name,only_mapped_ids)
output_file.close()
self.connection.close()
def main(self, start_id, thread_name, end_id):
#import pdb;pdb.set_trace()
self.get_env_url()
self.mongo_mysql_connection()
result_sheet = '/result/headrun_usanetwork_ott_%s_checking%s.csv' % (
thread_name, datetime.date.today())
output_file = lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
self.writer = csv.writer(mycsvfile,
dialect="excel",
lineterminator='\n')
self.writer.writerow(self.fieldnames)
for _id in range(start_id, end_id, 10):
print({"skip": _id})
query_headrun_usanetwork = self.executed_query(_id)
#query_headrun_usanetwork=self.sourcetable.find({"service":"nbc","item_type":"movie","id":"70301275"})
for data in query_headrun_usanetwork:
if data.get("id") != "":
#import pdb;pdb.set_trace()
self.headrun_usanetwork_id = data.get("id").encode(
"ascii", "ignore")
if data.get("item_type") == "movie":
self.show_type = "MO" #data.get("item_type")
else:
self.show_type = "SE"
self.count_headrun_usanetwork_id += 1
print("\n")
print datetime.datetime.now()
print("\n")
print({
"count_headrun_usanetwork_id":
self.count_headrun_usanetwork_id,
"show_type": self.show_type,
"thread_name": thread_name,
"nbc_id": self.headrun_usanetwork_id
})
#import pdb;pdb.set_trace()
only_mapped_ids = ott_meta_data_validation_modules(
).getting_mapped_px_id_mapping_api(
str(self.headrun_usanetwork_id),
self.source_mapping_api, self.projectx_mapping_api,
self.show_type, self.source, self.token)
self.ott_checking(only_mapped_ids, thread_name, data)
print("\n")
print({
"count_headrun_usanetwork_id":
self.count_headrun_usanetwork_id,
"name": thread_name
})
output_file.close()
self.connection.close()
def main(self, start_id, thread_name, end_id):
self.get_env_url()
input_file = "/input/projectx_sm_ids_season_0"
input_data = lib_common_modules().read_csv(input_file)
result_sheet = '/result/season_number_0_%s_checking%s.csv' % (
thread_name, datetime.date.today())
output_file = lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
self.writer = csv.writer(mycsvfile,
dialect="csv",
lineterminator='\n')
self.writer.writerow(self.fieldnames)
for _id in range(start_id, end_id):
self.cleanup()
self.projetcx_sm_id = str(input_data[_id][0])
self.total += 1
try:
self.season_0_checking(thread_name, self.projetcx_sm_id,
self.token)
print("\n")
print({
"projectx_sm_id": self.projetcx_sm_id,
"season_number_0": self.season_number_0,
"season_program_id": self.season_program_id,
"result": self.result,
"total": self.total,
"thread": thread_name
})
self.writer.writerow([
self.projetcx_sm_id, self.season_number_0,
self.season_program_id, self.season_name, self.result,
self.comment, self.Serving_Sources_season_program_id
])
except Exception as Error:
print("Got exception in main............",
self.projetcx_sm_id, Error)
pass
output_file.close()
def main(csv_file1, csv_file2):
result_sheet = '/output/ids.csv'
output_file = lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
writer = csv.writer(mycsvfile, dialect="csv", lineterminator='\n')
writer.writerow(["ids"])
#import pdb;pdb.set_trace()
print(input_file1)
print("\n")
print(input_file2)
for id_ in input_file2:
if id_ not in input_file1:
print(id_)
writer.writerow(id_)
def main(self, start_id, thread_name, end_id):
#import pdb;pdb.set_trace()
self.get_env_url()
self.mongo_mysql_connection()
result_sheet = '/result/gracenote_ott_%s_checking%s.csv' % (
thread_name, datetime.date.today())
output_file = lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
self.writer = csv.writer(mycsvfile,
dialect="excel",
lineterminator='\n')
self.writer.writerow(self.fieldnames)
for _id in range(start_id, end_id, 100):
self.logger.debug({"skip": _id})
query_gracenote = self.executed_query(_id, 100)
for data in query_gracenote:
if data.get("Videos") != {}:
#import pdb;pdb.set_trace()
gracenote_id = data.get("sequence_id").encode(
"ascii", "ignore")
if data.get("show_type") == "SM":
show_type = "SE"
else:
show_type = "MO"
self.count_gracenote_id += 1
self.logger.debug("\n")
self.logger.debug(datetime.datetime.now())
self.logger.debug("\n")
self.logger.debug({
"count_gracenote_id": self.count_gracenote_id,
"thread_name": thread_name,
"details": data
})
#import pdb;pdb.set_trace()
only_mapped_ids = ott_meta_data_validation_modules(
).getting_mapped_px_id_mapping_api(
str(gracenote_id), self.source_mapping_api,
self.projectx_mapping_api, show_type, self.source,
self.token)
#TODO: Sub_main : here OTT validation result printed
self.ott_checking(only_mapped_ids, gracenote_id,
show_type, thread_name, data)
self.logger.debug("\n")
self.logger.debug({
"count_gracenote_id": self.count_gracenote_id,
"name": thread_name
})
output_file.close()
self.connection.close()
def main(self, start_id, thread_name, end_id):
#import pdb;pdb.set_trace()
self.get_env_url()
self.mongo_mysql_connection()
result_sheet = '/result/%s_ott_%s_checking%s.csv' % (
self.service, thread_name, datetime.date.today())
output_file = lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
self.writer = csv.writer(mycsvfile,
dialect="csv",
lineterminator='\n')
self.writer.writerow(self.fieldnames)
for _id in range(start_id, end_id, 100):
print({"skip": _id})
#import pdb;pdb.set_trace()
query_starz = self.cur.execute(
"SELECT * FROM starz_programs where item_type in ('episode','movie') limit %d,100"
% (_id))
query_starz_result = self.cur.fetchall()
for data in query_starz_result:
if data.get("source_program_id") != "" and data.get(
"source_program_id") is not None:
#import pdb;pdb.set_trace()
self.starz_id = data.get("source_program_id")
self.show_type = data.get("item_type")
self.show_type = 'MO' if self.show_type == 'movie' else self.show_type
self.show_type = 'SE' if self.show_type == 'episode' else self.show_type
self.count_starz_id += 1
print("\n")
print datetime.datetime.now()
print("\n")
print({
"count_starz_id": self.count_starz_id,
"show_type": self.show_type,
"id": self.starz_id,
"thread_name": thread_name
})
#import pdb;pdb.set_trace()
only_mapped_ids = ott_meta_data_validation_modules(
).getting_mapped_px_id_mapping_api(
str(self.starz_id), self.source_mapping_api,
self.projectx_mapping_api, self.show_type,
self.source, self.token)
self.ott_checking(only_mapped_ids, thread_name, data)
print("\n")
print({"count_starz_id": self.count_starz_id, "name": thread_name})
output_file.close()
self.connection.close()
self.cur.close()
def reverse_api_extract(self, link_details,source,show_type):
#import pdb;pdb.set_trace()
## TODO: Only first index has been referred.
reverse_api=self.reverse_api_domain%(link_details,self.service)
reverse_api_resp=lib_common_modules().fetch_response_for_api_(reverse_api,self.token)
for data in reverse_api_resp:
if data.get("data_source")=='Rovi' and data.get("type")=='Program' and data.get("sub_type")=='ALL':
if data.get("projectx_id") not in self.px_id:
self.px_id.append(data.get("projectx_id"))
if data.get("data_source")!=source and data.get("type")=='Program' and data.get("sub_type")==show_type:
if data.get("projectx_id") not in self.px_id:
self.px_id.append(data.get("projectx_id"))
if data.get("data_source")==source and data.get("type")=='Program' and data.get("sub_type")==show_type:
if data.get("projectx_id") not in self.px_id:
self.px_id.append(data.get("projectx_id"))
def to_check_only_mapping_to_source(self, gracenote_id, show_type,
thread_name):
try:
only_mapped_ids = ott_meta_data_validation_modules(
).getting_mapped_px_id_mapping_api(str(gracenote_id),
self.source_mapping_api,
self.projectx_mapping_api,
show_type, self.source,
self.token)
self.logger.debug({
"%s_id" % self.source:
gracenote_id,
"count_gracenote_id":
str(self.count_gracenote_id),
"name":
str(thread_name),
"only_mapped_ids":
only_mapped_ids["source_flag"]
})
if only_mapped_ids["source_flag"] == 'True':
source_details = lib_common_modules().fetch_response_for_api_(
self.gracenote_api % (show_type, gracenote_id), self.token)
self.total += 1
#import pdb;pdb.set_trace()
projectx_id = only_mapped_ids["px_id"]
source_id = only_mapped_ids["source_id"]
self.logger.debug("\n")
self.logger.debug({
"total": self.total,
"MO_id": gracenote_id,
"Px_id": projectx_id,
"%s_id" % self.source: source_id,
"thread_name": thread_name
})
#self.meta_data_validation_(projectx_id,source_id,show_type,thread_name,only_mapped_ids,source_details)
return ({
"projectx_id": projectx_id,
"only_mapped_ids": only_mapped_ids,
"source_details": source_details
})
except (Exception, URLError, HTTPError,
httplib.BadStatusLine) as error:
self.logger.debug({
"exception caught in to_check_only_mapping_to_source func.. ":
str(type(error), gracenote_id, show_type)
})
self.to_check_only_mapping_to_source(gracenote_id, show_type,
thread_name)
def check_mapping_px_id(self, mapping_from_source_api, token):
try:
mapping_from_source_api_response = lib_common_modules(
).fetch_response_for_api_(mapping_from_source_api, token)
if mapping_from_source_api_response:
return mapping_from_source_api_response[0]["projectx_id"]
else:
return "NA"
except (Exception, URLError, HTTPError) as error:
raise (error, error.reason, mapping_from_source_api, token)
self.retry_count += 1
if self.retry_count <= 10:
self.check_mapping_px_id(source_id, mapping_from_source_api,
token)
else:
self.retry_count = 0
def main(self,start_id,thread_name,end_id):
#import pdb;pdb.set_trace()
self.get_env_url()
self.mongo_mysql_connection()
result_sheet='/result/hbogo_meta_data_checking%s_%s.csv'%(thread_name,datetime.date.today())
output_file=lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
self.writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n')
self.writer.writerow(self.fieldnames)
projectx_id=0
#import pdb;pdb.set_trace()
for _id in range(start_id,end_id,1000):
query_hbogo=self.cur.execute("SELECT * FROM hbogo_programs where (expired_at is null or expired_at > '%s') and expired='0' limit %d,1000 "%(self.running_datetime,_id))
#query_hbogo=self.cur.execute("SELECT * FROM hbogo_programs where launch_id='urn:hbo:feature:GVU4OxQ8lp47DwvwIAb24' and (expired_at is null or expired_at > '%s') "%(self.running_datetime))
query_hbogo_result=self.cur.fetchall()
for data in query_hbogo_result:
if data.get("launch_id")!="" and data.get("launch_id") is not None:
#import pdb;pdb.set_trace()
self.hbogo_id=data.get("launch_id")
self.show_type=data.get("show_type")
if self.show_type is not None and self.show_type!='SN':
self.show_type='MO' if self.show_type=='OT' else self.show_type
self.count_hbogo_id+=1
print("\n")
print datetime.datetime.now()
print ("\n")
print("%s_id:"%self.source,"id:",self.hbogo_id,"count_hbogo_id:"
+str(self.count_hbogo_id),"show_type:"+self.show_type,"name:"+str(thread_name))
only_mapped_ids=ott_meta_data_validation_modules().getting_mapped_px_id_mapping_api(self.hbogo_id,self.source_mapping_api,self.projectx_mapping_api,self.show_type,self.source,self.token)
try:
if only_mapped_ids["source_flag"]=='True':
self.total+=1
#import pdb;pdb.set_trace()
projectx_id=only_mapped_ids["px_id"]
print("\n")
print ({"total":self.total,"id":self.hbogo_id,"Px_id":projectx_id,
"thread_name":thread_name,"source_map":only_mapped_ids['source_map']})
self.meta_data_validation_(data,projectx_id,thread_name,only_mapped_ids)
except Exception as e:
print ("got exception in main....",self.hbogo_id
,self.show_type,only_mapped_ids,type(e),thread_name)
pass
output_file.close()
self.connection.close()
self.cur.close()
def main(self,start_id,thread_name,end_id):
#import pdb;pdb.set_trace()
self.get_env_url()
self.mongo_mysql_connection()
result_sheet='/result/headrun_showtime_meta_data_checking%s_%s.csv'%(thread_name,datetime.date.today())
output_file=lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile:
self.writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n')
self.writer.writerow(self.fieldnames)
projectx_id=0
for _id in range(start_id,end_id,100):
query_headrun_showtime=self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":[ "movie","episode","tvshow" ]}},{"service":"showtime"}]}},{"$project":{"id":1,"_id":0,"item_type":1,"series_id":1,"title":1,"episode_title":1,"release_year":1,"episode_number":1,"season_number":1,"duration":1,"image_url":1,"url":1,"description":1,"cast":1,"directors":1,"writers":1,
"categories":1,"genres":1,"maturity_ratings":1,"purchase_info":1,"service":1}},{"$skip":_id},{"$limit":100}])
#query_headrun_showtime=self.sourcetable.find({"service":"showtime","id":"3481853"})
for data in query_headrun_showtime:
if data.get("id")!="":
self.headrun_showtime_id=data.get("id")
if data.get("item_type")=="movie":
self.show_type="MO"
elif data.get("item_type")=="tvshow":
self.show_type="SM"
else:
self.show_type="SE"
self.count_headrun_showtime_id+=1
print("\n")
print(datetime.datetime.now())
print ("\n")
print("%s_id:"%self.source,"id:",self.headrun_showtime_id,"count_headrun_showtime_id:"
+str(self.count_headrun_showtime_id),"name:"+str(thread_name))
only_mapped_ids=ott_meta_data_validation_modules().getting_mapped_px_id_mapping_api(self.headrun_showtime_id,self.source_mapping_api,self.projectx_mapping_api,self.show_type,self.source,self.token)
try:
if only_mapped_ids["source_flag"]=='True':
self.total+=1
projectx_id=only_mapped_ids["px_id"]
print("\n")
print ({"total":self.total,"id":self.headrun_showtime_id,"Px_id":projectx_id,
"thread_name":thread_name,"source_map":only_mapped_ids['source_map']})
self.meta_data_validation_(data,projectx_id,thread_name,only_mapped_ids)
except Exception as e:
print ("got exception in main....",self.headrun_showtime_id
,self.show_type,only_mapped_ids,type(e),thread_name)
pass
output_file.close()
self.connection.close()
def getting_projectx_ott_details(self, projectx_id, show_type):
px_video_link = []
px_video_link_present = 'False'
px_response = 'Null'
launch_id = []
projectx_api = self.projectx_programs_api % (projectx_id, self.service)
data_px_resp = lib_common_modules().fetch_response_for_api_(
projectx_api, self.token)
if data_px_resp != []:
for data in data_px_resp:
px_video_link = data.get("videos")
if px_video_link:
px_video_link_present = 'True'
for linkid in px_video_link:
launch_id.append(linkid.get("launch_id"))
return {
"px_video_link_present": px_video_link_present,
"launch_id": launch_id
}
else:
return px_response
def get_mappings_sources(self, projectx_id, mapping_api, token):
try:
response_array = []
mapping_response = lib_common_modules().fetch_response_for_api_(
mapping_api, token)
if mapping_response:
for response in mapping_response:
response_array.append([
response['data_source'] + " => " +
response["source_id"]
])
return response_array
else:
return
except (Exception, URLError, HTTPError) as error:
raise (error, error.reason, mapping_api, token)
self.retry_count += 1
if self.retry_count <= 10:
self.get_mappings_sources(projectx_id, mapping_api, token)
else:
self.retry_count = 0
def getting_projectx_ott_details(self,projectx_id,show_type):
px_video_link=[]
px_video_link_present='False'
px_response='Null'
launch_id=[]
fetch_from=[]
#import pdb;pdb.set_trace()
projectx_api=self.projectx_programs_api%(projectx_id,self.source.lower())
data_px_resp=lib_common_modules().fetch_response_for_api_(projectx_api,self.token)
if data_px_resp!=[]:
for data in data_px_resp:
px_video_link= data.get("videos")
if px_video_link:
px_video_link_present='True'
for linkid in px_video_link:
launch_id.append(linkid.get("launch_id"))
if linkid.get("fetched_from") not in fetch_from:
fetch_from.append(linkid.get("fetched_from"))
return {"px_video_link_present":px_video_link_present,"launch_id":launch_id,"fetch_from":fetch_from}
else:
return px_response
def getting_projectx_ott_details(self, projectx_id, show_type):
px_video_link = []
fetched_from = []
px_video_link_present = 'False'
androidtv_id_format = 'False'
appletv_id_format = 'False'
px_response = 'Null'
launch_id = []
projectx_api = self.projectx_programs_api % (projectx_id,
self.source.lower())
data_px_resp = lib_common_modules().fetch_response_for_api_(
projectx_api, self.token)
if data_px_resp != []:
for data in data_px_resp:
px_video_link = data.get("videos")
if px_video_link:
px_video_link_present = 'True'
for linkid in px_video_link:
launch_id.append(linkid.get("launch_id"))
if linkid.get("fetched_from") not in fetched_from:
fetched_from.append(
linkid.get("fetched_from").encode("utf-8"))
if linkid.get("appletv_id") == '{}{}{}'.format(
self.source.lower(), '://',
linkid.get("launch_id")):
appletv_id_format = 'True'
if linkid.get("androidtv_id") == '{}{}{}'.format(
self.source.lower(), '://',
linkid.get("launch_id")):
androidtv_id_format = 'True'
return {
"px_video_link_present": px_video_link_present,
"launch_id": launch_id,
"appletv_id_format": appletv_id_format,
"androidtv_id_format": androidtv_id_format,
"fetched_from": fetched_from
}
else:
return px_response
def duplicate_checking_series(self, series_data, source, writer,writer_credit_match_false,thread_name):
#import pdb;pdb.set_trace()
duplicate=""
credit_array=[]
print("\nFunction duplicate_checking_series_called.................")
self.id=series_data.get("id").encode("ascii","ignore")
self.show_id=series_data.get("series_id").encode("ascii","ignore")
self.release_year=series_data.get("release_year").encode("ascii","ignore")
self.series_title=unidecode.unidecode(pinyin.get(series_data.get("title")))
self.episode_title=unidecode.unidecode(pinyin.get(series_data.get("episode_title")))
if self.series_title is not None:
data_expired_api_resp = lib_common_modules().link_expiry_check_(self.expire_api,
self.preprod_domain_name,self.id,self.service,self.Token)
if data_expired_api_resp:
self.link_expired = "False" if data_expired_api_resp=='False' else "True"
self.reverse_api_extract(self.id, source,'SE')
print ("projectx_ids : {},fox_se_id: {}, fox_SM_id: {}".format(self.px_id,self.id,self.show_id),"thread_name:",thread_name)
if len(self.px_id) > 1:
px_link=self.projectx_preprod_api%'{}'.format(",".join([str(data) for data in self.px_id]))
data_resp_link=lib_common_modules().fetch_response_for_api_(px_link,self.token)
for id_ in data_resp_link:
if id_.get("series_id") not in self.series_id_px:
self.series_id_px.append(id_.get("series_id"))
print ("projectx_ids_series : {0}".format(self.series_id_px),"thread:",thread_name)
if len(self.series_id_px)>1:
data_resp_search=duplicate_script_modules().search_api_call_response(self.series_title,self.projectx_preprod_search_api,
self.projectx_domain,self.token)
if data_resp_search is not None:
result=duplicate_script_modules().search_api_response_validation(data_resp_search, source, self.series_id_px, duplicate,
'SM',self.token,self.projectx_preprod_api,
self.projectx_mapping_api,self.beta_programs_api
,self.duplicate_api,self.credit_db_api)
if (self.credit_match=='False' or self.credit_match=='') and len(result["search_px_id"])==2:
#import pdb;pdb.set_trace()
px_link=self.projectx_preprod_api %'{}'.format(",".join([str(i) for i in result["search_px_id"]]))
data_resp_credits=lib_common_modules().fetch_response_for_api_(px_link,self.token)
#import pdb;pdb.set_trace()
for uu in data_resp_credits:
if uu.get("credits"):
for tt in uu.get("credits"):
credit_array.append(unidecode.unidecode(tt.get("full_credit_name")))
if credit_array:
for cc in credit_array:
if credit_array.count(cc)>1:
self.credit_match='True'
break
else:
self.credit_match='False'
result_credit_match_false=duplicate_script_modules().validation().meta_data_validation(result["search_px_id"],self.projectx_preprod_api
,self.credit_db_api,source,self.token)
#import pdb;pdb.set_trace()
writer_credit_match_false.writerow(result_credit_match_false)
writer.writerow([source,self.show_id,self.id,'SE','',self.series_title,self.episode_title
,self.release_year,self.link_expired,self.series_id_px,'','','',result["comment"],result["comment"],
result["duplicate"],result["search_px_id"],self.credit_match,result["count_rovi"],result["count_guidebox"],result["count_source"],
result["count_hulu"],result["count_vudu"],result["rovi_mapping"],result["guidebox_mapping"],result["source_mapping"],result["hulu_mapping"],result["vudu_mapping"],
result["comment_variant_parent_id_present"],result["comment_variant_parent_id"]])
else:
duplicate_api=self.duplicate_api%(self.show_id,source,'SM')
data_resp_duplicate=lib_common_modules().fetch_response_for_api_(duplicate_api,self.token)
if data_resp_duplicate:
duplicate='True'
else:
duplicate='False'
self.comment="search_api_has_no_response"
self.result="search_api_has_no_response"
writer.writerow([source,self.show_id,self.id,'SE', '',self.series_title,
self.episode_title, self.release_year, self.link_expired, self.series_id_px,
'', '', '' ,self.comment,self.result,duplicate])
else:
self.comment=('No multiple ids for this series',self.id,self.show_id)
self.result="No multiple ids for this series"
writer.writerow([source, self.show_id, self.id,'SE' ,'',self.series_title,
self.episode_title, self.release_year, self.link_expired, self.series_id_px,
'', '', '' ,self.comment,self.result])
else:
self.comment=('No multiple ids for this episode',self.id,self.show_id)
self.result="No multiple ids for this episode"
writer.writerow([source, self.show_id, self.id,'SE', '',self.series_title,
self.episode_title, self.release_year, self.link_expired, self.px_id,
'', '', '' ,self.comment,self.result])
else:
self.comment=('No series_title for this episode',self.id,self.show_id)
self.result="No series_title for this episode"
writer.writerow([source, self.show_id, self.id,'SE', '',self.series_title,
self.episode_title, self.release_year, '', '',
'', '', '' ,self.comment,self.result])
def main(self,start_id,thread_name,end_id):
#import pdb;pdb.set_trace()
inputFile="input/input_file"
input_data=lib_common_modules().read_csv(inputFile)
self.default_param()
self.logger=lib_common_modules().create_log(os.getcwd()+'/logs/log_%s.txt'%thread_name)
result_sheet='/output/output_file_%s_%s.csv'%(thread_name,datetime.date.today())
output_file=lib_common_modules().create_csv(result_sheet)
self.column_fieldnames=["Service_name","Link_id","Rovi_id","Projectx_id","Rovi_id_form_reverse_api","Rovi_id_match","Link_expired",
"Prod_link_status","Preprod_link_status","Link_fetched_from","Fetched_from_sources","Comment"]
with output_file as mycsvfile:
self.writer = csv.writer(mycsvfile,dialect="csv",lineterminator = '\n')
self.writer.writerow(self.column_fieldnames)
for id_ in range(start_id,end_id):
self.cleanup()
self.getting_input_from_sheet(input_data,id_)
self.total+=1
reverse_api_response=lib_common_modules().fetch_response_for_api(self.reverse_mapping_api%(self.host_IP,
self.link_id,self.service),self.token,self.logger)
if reverse_api_response:
for data in reverse_api_response:
if data["projectxId"] not in self.px_id and data["data_source"]=='Rovi' and data["type"]=='Program':
self.px_id.append(data["projectxId"])
self.source_id.append(data["sourceId"])
if self.rovi_id in self.source_id:
"""comment"""
self.same_rovi_id_present='True'
if self.px_id:
self.link_expired=lib_common_modules().link_expiry_check(self.expired_api,self.prod_domain,self.link_id,self.service,self.expired_token,self.logger)
prod_api_response=lib_common_modules().fetch_response_for_api(self.programs_api%(self.prod_domain,
self.source_id[0]),self.token,self.logger)
self.prod_videos_response=[data['videos'] for data in prod_api_response]
if self.prod_videos_response[0]:
for data in self.prod_videos_response[0]:
if data["source_id"]==self.service:
self.prod_launch_id.append(data["launch_id"])
self.prod_video_link.append(data["link"]["uri"])
link_status_prod=self.link_ingestion_check(self.prod_launch_id,self.link_id,
self.prod_video_link)
if link_status_prod["link_id_present"]=="True":
"""comment"""
self.prod_link_status="link_id_present_in_Prod"
else:
"""comment"""
self.prod_link_status='videos_not_available'
preprod_api_response=lib_common_modules().fetch_response_for_api(self.programs_api%(self.preprod_domain,
self.px_id[0]),self.token,self.logger)
self.preprod_videos_response=[data['videos'] for data in preprod_api_response]
if self.preprod_videos_response[0]:
for data in self.preprod_videos_response[0]:
if data["source_id"]==self.service:
if data["launch_id"] not in self.preprod_launch_id:
self.preprod_launch_id.append(data["launch_id"])
if data["link"]["uri"] not in self.preprod_video_link:
self.preprod_video_link.append(data["link"]["uri"])
if data["fetched_from"] not in self.fetch_from:
self.fetch_from.append(data["fetched_from"])
if self.preprod_launch_id or self.preprod_video_link:
link_status_preprod= self.link_ingestion_check(self.preprod_launch_id,self.link_id,
self.preprod_video_link)
if link_status_preprod["link_id_present"]=='True':
"""comment"""
self.preprod_link_status="link_id_present_in_Preprod"
for source in self.fetch_from:
if 'Rovi' in self.fetch_from and len(self.fetch_from)==1:
self.fetched_source='Rovi'
elif 'Rovi' in self.fetch_from and len(self.fetch_from)>1:
self.fetched_source='Rovi+others'
elif 'Rovi' not in self.fetch_from and len(self.fetch_from)>1:
self.fetched_source='others'
else:
self.fetched_source='others'
else:
"""comment"""
self.preprod_link_status='videos_not_available'
else:
self.link_expired=lib_common_modules().link_expiry_check(self.expired_api,self.prod_domain,self.link_id,self.service,self.expired_token,self.logger)
else:
self.link_expired=lib_common_modules().link_expiry_check(self.expired_api,self.prod_domain,self.link_id,self.service,self.expired_token,self.logger)
self.logger.debug("\n")
if self.same_rovi_id_present=="True" and self.preprod_link_status=='link_id_present_in_Preprod':
self.pass_count+=1
self.comment='Pass'
else:
self.fail_count+=1
self.comment='Fail'
self.logger.debug (["date time:", datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")])
self.logger.debug("\n")
self.logger.debug ([{"total":self.total,"pass":self.pass_count,"fail":self.fail_count,"Process":thread_name}])
self.logger.debug("\n")
self.logger.debug([{"Service":self.service,"link_id":self.link_id,"Rovi_id":self.rovi_id,
"px_id":self.px_id,"rovi_id_form_reverse_api":self.source_id,"rovi_id_status":self.same_rovi_id_present,"Link_expired":self.link_expired,
"Prod_link_status":self.prod_link_status,"preprod_link_status":self.preprod_link_status,"link_fetched":self.fetched_source,
"comment":self.comment,"Process":thread_name}])
self.writer.writerow([self.service,self.link_id,self.rovi_id,self.px_id,self.source_id,self.same_rovi_id_present,
self.link_expired,self.prod_link_status,self.preprod_link_status,self.fetched_source,self.fetch_from,
self.comment])
output_file.close()
def main(self,start_id,thread_name,end_id):
#import pdb;pdb.set_trace()
self.default_param()
self.mongo_connection()
fieldnames_tag = ["source","sm_id","id","show_type","movie_title","series_title","episode_title",
"release_year","link_expired","Projectx_ids","Variant id present for episode",
"Variant parent id for Episode","Projectx_series_id","comment","Result","Gb_id duplicate",
'Duplicate ids in search',"Credit_match","Rovi_id count for duplicate","Gb_id count for duplicate",
"%s_id count for duplicate"%self.source,"Hulu_id count","vudu_id_count","Mapped rovi_id for duplicate","Mapped Gb_id for duplicate",
"Mapped %s_id for duplicate"%self.source,"Mapped_hulu_id","Mapped_vudu_id","variant_parent_id_present","variant_parent_id","link_fetched_from",
"Sources of link"]
fieldnames_credit_match_false = ["px_id1","px_id1_show_type","px_id1_variant_parent_id","px_id1_is_group_language_primary",
"px_id1_record_language","px_id2","px_id2_show_type","px_id2_variant_parent_id","px_id2_is_group_language_primary",
"px_id2_record_language","px_id1_credits_null","px_id1_db_credit_present","px_id2_credits_null","px_id2_db_credit_present",
"long_title_match","original_title_match","runtime_match","release_year_match","alias_title_match","comment"]
result_sheet_credit_match_false='/output_credit_match_false/duplicate_programs_credit_match_false_result_%s_%s.csv'%(thread_name,datetime.date.today())
output_file_credit_match_false=lib_common_modules().create_csv(result_sheet_credit_match_false)
result_sheet='/output/duplicate_checked_in_search_api_%s_%s.csv'%(thread_name,datetime.date.today())
output_file=lib_common_modules().create_csv(result_sheet)
with output_file as mycsvfile, output_file_credit_match_false as mycsvfile1:
writer = csv.writer(mycsvfile,dialect="excel",lineterminator = '\n')
writer.writerow(fieldnames_tag)
writer_credit_match_false = csv.DictWriter(mycsvfile1,fieldnames=fieldnames_credit_match_false,dialect="excel",lineterminator = '\n')
writer_credit_match_false.writeheader()
mo_list=[]
sm_list=[]
for _id in range(start_id,end_id,100):
try:
query=self.sourcetable.aggregate([{"$match":{"$and":[{"item_type":{"$in":["episode"]}},{"service":"fox"}]}}
,{"$project":{"id":1,"_id":0,"item_type":1,"series_id":1,"title":1,"episode_title":1,"release_year":1,
"episode_number":1,"season_number":1,"duration":1,"image_url":1,"url":1,"description":1,"cast":1,"directors":1,"writers":1,
"categories":1,"genres":1,"maturity_ratings":1,"purchase_info":1,"service":1}},{"$skip":_id},{"$limit":100}])
#query=self.sourcetable.find({"service":"netflix","item_type":"movie","id":"269880"})
print ("\n")
print({"start": start_id,"end":end_id})
for data in query:
series_title=''
_id=data.get("id")
sm_id=data.get("series_id")
show_type=data.get("item_type").encode('utf-8')
if _id is not None and _id!="":
if show_type=='episode':
if sm_id not in sm_list:
sm_list.append(sm_id)
self.cleanup()
self.duplicate_checking_series(data,self.source,writer,writer_credit_match_false,thread_name)
elif show_type=='movie':
if _id not in mo_list:
mo_list.append(_id)
self.cleanup()
self.duplicate_checking_movies(data,self.source,writer,writer_credit_match_false,thread_name)
print("\n")
print ({"Total SM":len(sm_list),"Total MO":len(mo_list),"Thread_name":thread_name})
print("\n")
print datetime.datetime.now()
except (pymongo.errors.CursorNotFound,Exception) as e:
print ("exception caught in main func.............",type(e),thread_name)
continue
output_file.close()
output_file_credit_match_false.close()
self.connection.close()
