def downloadLogoAndThumbnail(self):
if self._logo != "" and self._logo.startswith("http://"):
logoFilename = urlparse.urlsplit(self._logo)[2].split('/')[-1]
logoPath = os.path.join(DATA_ROOT, "assets/images/channels/logos",
logoFilename)
filename, _, _ = getPage(self._logo, logoPath)
if filename:
self._logo = logoFilename
self._logger.info(
"Channel.downloadLogoAndThumbnail: downloaded logo to %s" %
(logoPath))
else:
self._logger.info(
"Channel.downloadLogoAndThumbnail: could not download logo from %s"
% (self._logo))
self._logo = ""
if self._thumbnail != "" and self._thumbnail.startswith("http://"):
thumbnailFilename = urlparse.urlsplit(
self._thumbnail)[2].split('/')[-1]
thumbnailPath = os.path.join(DATA_ROOT,
"assets/images/channels/thumbnails",
thumbnailFilename)
filename, _, _ = getPage(self._thumbnail, thumbnailPath)
if filename:
self._thumbnail = thumbnailFilename
self._logger.info(
"Channel.downloadLogoAndThumbnail: downloaded thumbnail to %s"
% (thumbnailPath))
else:
self._logger.info(
"Channel.downloadLogoAndThumbnail: could not download thumbnail from %s"
% (self._thumbnail))
self._thumbnail = ""
def _getStyleCss(self, styleCssPath):
content, _, _ = getPage(styleCssPath)
if content:
fileHandle = gzip.GzipFile(fileobj=StringIO(content))
fileContent = fileHandle.read()
return fileContent
return None
def _getStyleCss( self, styleCssPath ):
content, _, _ = getPage( styleCssPath )
if content:
fileHandle = gzip.GzipFile( fileobj=StringIO( content ) )
fileContent = fileHandle.read()
return fileContent
return None
def downloadLogoAndThumbnail( self ):
if self._logo != "" and self._logo.startswith( "http://" ):
logoFilename = urlparse.urlsplit( self._logo )[2].split( '/' )[-1]
logoPath = os.path.join( DATA_ROOT, "assets/images/channels/logos", logoFilename )
filename, _, _ = getPage( self._logo, logoPath )
if filename:
self._logo = logoFilename
self._logger.info( "Channel.downloadLogoAndThumbnail: downloaded logo to %s" % ( logoPath ) )
else:
self._logger.info( "Channel.downloadLogoAndThumbnail: could not download logo from %s" % ( self._logo ) )
self._logo = ""
if self._thumbnail != "" and self._thumbnail.startswith( "http://" ):
thumbnailFilename = urlparse.urlsplit( self._thumbnail )[2].split( '/' )[-1]
thumbnailPath = os.path.join( DATA_ROOT, "assets/images/channels/thumbnails", thumbnailFilename )
filename, _, _ = getPage( self._thumbnail, thumbnailPath )
if filename:
self._thumbnail = thumbnailFilename
self._logger.info( "Channel.downloadLogoAndThumbnail: downloaded thumbnail to %s" % ( thumbnailPath ) )
else:
self._logger.info( "Channel.downloadLogoAndThumbnail: could not download thumbnail from %s" % ( self._thumbnail ) )
self._thumbnail = ""
def _grabDetailedEpgForProgram( self, program, epgId=None ):
grabbed = True
# Fetch detailed information. http://w.zt6.nl/epgdata/xx/xxxxxx.json
detailsFilename = "/%s/%s.json" % ( program.originalId[-2:], program.originalId )
detailsUrl = self._glashartConfig.epgDataPath + detailsFilename
detailsPage, _, _ = getPage( detailsUrl, None )
if detailsPage and len( detailsPage ) > 0:
detailsData = json.loads( detailsPage )
program = self._getDetailedProgramFromJson( program, detailsData )
else:
grabbed = False
return program, grabbed
def _grabDetailedEpgForProgram(self, program, epgId=None):
grabbed = True
# Fetch detailed information. http://w.zt6.nl/epgdata/xx/xxxxxx.json
detailsFilename = "/%s/%s.json" % (program.originalId[-2:],
program.originalId)
detailsUrl = self._glashartConfig.epgDataPath + detailsFilename
detailsPage, _, _ = getPage(detailsUrl, None)
if detailsPage and len(detailsPage) > 0:
detailsData = json.loads(detailsPage)
program = self._getDetailedProgramFromJson(program, detailsData)
else:
grabbed = False
return program, grabbed
def _parseIndexPage(self):
content, _, _ = getPage(self._glashartConfig.tvmenuIndexPath)
if content:
title = None
codeJsPath = None
styleCssPath = None
fileHandle = gzip.GzipFile(fileobj=StringIO(content))
fileContent = fileHandle.read()
regExp = {
"TITLE":
r"\<title\>(?P<page_title>.*)\<\/title\>",
"STYLE":
r"\<link rel='stylesheet' type='text\/css' href='(?P<css_filename>.*)'\>\<\/link\>",
"JAVASCRIPT":
r"\<script type='text\/javascript' src='(?P<js_filename>.*)'\>\<\/script\>"
}
titleMatch = re.compile(regExp["TITLE"]).search(fileContent)
if titleMatch:
title = titleMatch.group("page_title")
else:
self._logger.error("_parseIndexPage: no match for TITLE")
return (None, None, None, None)
codeJsMatch = re.compile(regExp["JAVASCRIPT"]).search(fileContent)
if codeJsMatch:
codeJsPath = self._glashartConfig.iptvBaseUrl + "/" + self._glashartConfig.tvmenuPath + "/" + codeJsMatch.group(
"js_filename")
else:
self._logger.error("_parseIndexPage: no match for JAVASCRIPT")
return (None, None, None, None)
styleCssMatch = re.compile(regExp["STYLE"]).search(fileContent)
if styleCssMatch:
styleCssPath = self._glashartConfig.iptvBaseUrl + "/" + self._glashartConfig.tvmenuPath + "/" + styleCssMatch.group(
"css_filename")
else:
self._logger.error("_parseIndexPage: no match for STYLE")
return (None, None, None, None)
return (fileContent, title, codeJsPath, styleCssPath)
else:
return (None, None, None, None)
def _parseIndexPage( self ):
content, _, _ = getPage( self._glashartConfig.tvmenuIndexPath )
if content:
title = None
codeJsPath = None
styleCssPath = None
fileHandle = gzip.GzipFile( fileobj=StringIO( content ) )
fileContent = fileHandle.read()
regExp = {
"TITLE": r"\<title\>(?P<page_title>.*)\<\/title\>",
"STYLE": r"\<link rel='stylesheet' type='text\/css' href='(?P<css_filename>.*)'\>\<\/link\>",
"JAVASCRIPT": r"\<script type='text\/javascript' src='(?P<js_filename>.*)'\>\<\/script\>"
}
titleMatch = re.compile( regExp["TITLE"] ).search( fileContent )
if titleMatch:
title = titleMatch.group( "page_title" )
else:
self._logger.error( "_parseIndexPage: no match for TITLE" )
return ( None, None, None, None )
codeJsMatch = re.compile( regExp["JAVASCRIPT"] ).search( fileContent )
if codeJsMatch:
codeJsPath = self._glashartConfig.iptvBaseUrl + "/" + self._glashartConfig.tvmenuPath + "/" + codeJsMatch.group( "js_filename" )
else:
self._logger.error( "_parseIndexPage: no match for JAVASCRIPT" )
return ( None, None, None, None )
styleCssMatch = re.compile( regExp["STYLE"] ).search( fileContent )
if styleCssMatch:
styleCssPath = self._glashartConfig.iptvBaseUrl + "/" + self._glashartConfig.tvmenuPath + "/" + styleCssMatch.group( "css_filename" )
else:
self._logger.error( "_parseIndexPage: no match for STYLE" )
return ( None, None, None, None )
return ( fileContent, title, codeJsPath, styleCssPath )
else:
return ( None, None, None, None )
def _parseCodeJs(self, codeJsPath):
content, _, _ = getPage(codeJsPath)
if content:
fileHandle = gzip.GzipFile(fileobj=StringIO(content))
fileContent = fileHandle.read()
parseOk = True
regExp = {
# H [ b ] = { k :b , c :{ " default" :" NPO 1" } , r :{ " default" :" NPO 1" } , l :" ned1" , n :" ned1" , u :" npotv1.png" , v :" npotv1.png" , p :" npotv1.png" , o :a , e :[ ] , f :[ ] , g :[ ] } ;
"CHANNEL_LIST_1":
r"(?P<channel_list>[A-Za-z]{1})\[[a-z]{1}\]=\s*\{(?P<channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_name_long>[A-Za-z]{1}):\{\"default\":\"NPO 1\"\},\s*(?P<channel_name_short>[A-Za-z]{1}):\{\"default\":\"NPO 1\"\},\s*(?P<channel_id_1>[A-Za-z]{1}):\"ned1\",\s*(?P<channel_id_2>[A-Za-z]{1}):\"ned1\",\s*(?P<channel_logo_1>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<channel_logo_2>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<channel_logo_3>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<prev_channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_streams>[A-Za-z]{1}):\[\],\s*(?P<channel_unknown_1>[A-Za-z]{1}):\[\],\s*(?P<channel_unknown_2>[A-Za-z]{1}):\[\]\};",
# a = { } ; a .c ={ " default" :"HD+ " }; a .h = " igmp:/ / 224 .0 .251 .124 :8248;rtpskip=yes " ; a .C = 1; a .M = 1;
# a = { } ; a .c ={ " default" :"HD ( NOB) " }; a .h = " igmp:/ / 224 .1 .3 .1 :12110 " ; a .Ba = [{Oa: "103"}]; a .C = 1; a .M = 1;
"CHANNEL_LIST_2":
r"[A-Za-z]{1}=\s*\{\};\s*[A-Za-z]{1}.(?P<channel_metadata>[A-Za-z]{1})=\{\"default\":\"[A-Za-z \(\)\+]{2,10}\"\};\s*[A-Za-z]{1}.(?P<channel_url>[A-Za-z]{1,2})=\s*\"igmp:\/\/[0-9]{3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}:[0-9]{1,5}(;rtpskip=yes)?\";\s*([A-Za-z]{1}.[A-Za-z]{1,2}=\s*\[([^\]]*)\];\s*)?[A-Za-z]{1}.[A-Za-z]{1}=\s*1;\s*[A-Za-z]{1}.(?P<channel_url_hd>[A-Za-z]{1})=\s*1;",
# for( $ .tv.q .push( a ) ;f .length;) E [ f .pop( ) ] = a ;
"CHANNEL_LIST_3":
r"for\([\$A-Za-z]{1,2}.tv.[A-Za-z]{1,2}.push\([A-Za-z]{1,2}\);[A-Za-z]{1,2}.length;\)(?P<channel_id_list>[A-Za-z]{1,2})\[[A-Za-z]{1,2}.pop\(\)\]=\s*[A-Za-z]{1,2};",
# z [ b ] = { k :b , c :{ " default" :" NPO 3FM" } , q :{ " default" :" NPO 3FM" } , j :" radio3" , m :" radio3" , s :" nporadio3.png" , t :" nporadio3.png" , o :" nporadio3.png" , u :1, n :a ,d:[],e:[]};
"CHANNEL_LIST_4":
r"(?P<channel_list>[A-Za-z]{1})\[[A-Za-z]{1}\]=\s*\{(?P<channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_name_long>[A-Za-z]{1}):\{\"default\":\"NPO 3FM\"\},\s*(?P<channel_name_short>[A-Za-z]{1}):\{\"default\":\"NPO 3FM\"\},\s*(?P<channel_id_1>[A-Za-z]{1}):\"radio3\",\s*(?P<channel_id_2>[A-Za-z]{1}):\"radio3\",\s*(?P<channel_logo_1>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_logo_2>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_logo_3>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_radio>[A-Za-z]{1}):1,\s*(?P<prev_channel_number>[A-Za-z]{1}):[a-z]{1}([^\}]+)\};",
# Gb = f ;B [ a ] .i = f ; for( B [ f ] .k = a ;e .length;) w [ e .pop( ) ] = f ;J .root
"CHANNEL_LIST_5":
r"(?P<last_channel_number>[A-Za-z]{1,2})=\s*[A-Za-z]{1};[A-Za-z]{1}\[[A-Za-z]{1}\].(?P<next_channel_number>[A-Za-z]{1})=\s*[A-Za-z]{1};\s*for\([A-Za-z]{1}\[[A-Za-z]{1}\].[A-Za-z]{1}=\s*[A-Za-z]{1};[A-Za-z]{1}.length;\)[A-Za-z]{1}\[[A-Za-z]{1}.pop\(\)\]=\s*[A-Za-z]{1};[A-Za-z]{1}.root",
# function( ) { window.clearInterval (g ) ;u ( $a ) ;var e = new ab ( t .dj ) ;e .B =c ( ) ;v ( e ) ;window.setTimeout( function( ) { ga ( ) } ,50) }
"PLAY_STREAM_ACTIONS":
r"function\(\)\{window.clearInterval\([A-Za-z]{1,2}\);(?P<play_action_1>[A-Za-z]{1,2})\([$A-Za-z]{1,2}\);var [A-Za-z]{1,2}=\s*new [A-Za-z]{1,2}\([A-Za-z]{1,2}.[A-Za-z]{1,2}\);[A-Za-z]{1,2}.[A-Za-z]{1,2}=[A-Za-z]{1,2}\(\);(?P<play_action_2>[A-Za-z]{1,2})\([A-Za-z]{1,2}\);window.setTimeout\(function\(\)\{[A-Za-z]{1,2}\(\)\},50\)\}",
# function Ue ( a ,b ,f ,g ,e ) {L .call( this) ;if( b ===undefined) b = " " ;
"PLAY_STREAM_CLASS":
r"function (?P<play_stream_class>[A-Za-z]{1,2})\((?P<play_stream_url>[A-Za-z]{1,2}),[A-Za-z]{1,2},[A-Za-z]{1,2},[A-Za-z]{1,2},[A-Za-z]{1,2}\){[A-Za-z]{1,2}.call\(this\);if\([A-Za-z]{1,2}===undefined\)[A-Za-z]{1,2}=\s*\"\";",
"SET_CHANNEL_FUNCTION":
r"[A-Za-z]{1,2}.prototype.(?P<set_channel_function>[A-Za-z]{1,2})=\s*function\([A-Za-z]{1,2},[A-Za-z]{1,2}\){if\(this.[A-Za-z]{1,2}\){window.clearTimeout\(this.[A-Za-z]{1,2}\);",
# this.l .appendChild( b ) ;b = new O ( o .fh ) ;b .ua = function( ) {G .ka(
"SET_CHANNEL_INSTANCE":
r"this.[A-Za-z]{1,2}.appendChild\([A-Za-z]{1,2}\);[A-Za-z]{1,2}=\s*new [A-Za-z]{1,2}\([A-Za-z]{1,2}.[A-Za-z]{1,2}\);[A-Za-z]{1,2}.[A-Za-z]{1,2}=\s*function\(\){(?P<set_channel_instance>[A-Za-z]{1,2}).%s\(",
"CHANNEL_OBJECT":
r"(document.cookie=\"channel=\"\+encodeURIComponent\(this.(?P<channel_object_1>[A-Za-z]{1,2}).(?P<channel_object_2>[A-Za-z]{1,2})\);)",
"DEBUG_FUNCTION":
r"function (?P<debug_function>[A-Za-z]{1,2})\([A-Za-z]{1,2}\){if\(window.console\)window.console.log\(",
# for( o ( " doRunAfterLoad( ) : " + Kb .length) ;Kb .length;) { var a =Kb .pop( ) ;try{ a ( ) }
"INIT_FUNCTION":
r"for\(%s\(\"doRunAfterLoad\(\): \"\+[A-Za-z]{1,2}.length\);[A-Za-z]{1,2}.length;\)\{var [A-Za-z]{1,2}=[A-Za-z]{1,2}.pop\(\);try\{[A-Za-z]{1,2}\(\)\}",
"LAST_JS_LINE":
r"([A-Za-z]{1,2}\(\);\s*%s\(\"initial buildMenu\(\) done.\"\);)",
# function oc ( b ) { ia ( ) ;$a &&clearTimeout( $a ) ;b = b .keyCode| | b .charCode;
"KEY_EVENT_FUNCTION":
r"function (?P<key_event_function>[A-Za-z]{1,2})\([A-Za-z]{1,2}\)\{[A-Za-z]{1,2}\(\);[$A-Za-z]{1,2}&&clearTimeout\([$A-Za-z]{1,2}\);[A-Za-z]{1,2}=\s*[A-Za-z]{1,2}.keyCode\|\|[A-Za-z]{1,2}.charCode;",
}
symbolNames = {}
symbolNames[
"channel_logo_path"] = self._glashartConfig.channelLogoPath
symbolNames[
"channel_thumb_path"] = self._glashartConfig.channelThumbPath
channelList1Match = re.compile(
regExp["CHANNEL_LIST_1"]).search(fileContent)
if channelList1Match:
symbolNames["channel_list"] = channelList1Match.group(
"channel_list")
symbolNames["channel_number"] = channelList1Match.group(
"channel_number")
symbolNames["channel_name_long"] = channelList1Match.group(
"channel_name_long")
symbolNames["channel_name_short"] = channelList1Match.group(
"channel_name_short")
symbolNames["channel_id_1"] = channelList1Match.group(
"channel_id_1")
symbolNames["channel_id_2"] = channelList1Match.group(
"channel_id_2")
symbolNames["channel_logo_1"] = channelList1Match.group(
"channel_logo_1")
symbolNames["channel_logo_2"] = channelList1Match.group(
"channel_logo_2")
symbolNames["channel_logo_3"] = channelList1Match.group(
"channel_logo_3")
symbolNames["prev_channel_number"] = channelList1Match.group(
"prev_channel_number")
symbolNames["channel_streams"] = channelList1Match.group(
"channel_streams")
# For the folloing matches I only need about 3000 characters starting from the start of this match
channelListString = fileContent[channelList1Match.start(
):channelList1Match.start() + 3000]
channelList2Match = re.compile(
regExp["CHANNEL_LIST_2"]).search(channelListString)
if channelList2Match:
symbolNames["channel_metadata"] = channelList2Match.group(
"channel_metadata")
symbolNames["channel_url"] = channelList2Match.group(
"channel_url")
symbolNames["channel_url_hd"] = channelList2Match.group(
"channel_url_hd")
else:
self._logger.error(
"_parseCodeJs: no match for CHANNEL_LIST_2")
parseOk = False
channelList3Match = re.compile(
regExp["CHANNEL_LIST_3"]).search(channelListString)
if channelList3Match:
symbolNames["channel_id_list"] = channelList3Match.group(
"channel_id_list")
else:
self._logger.error(
"_parseCodeJs: no match for CHANNEL_LIST_3")
parseOk = False
# Take the rest of the file starting from the first match
channelListString = fileContent[channelList1Match.start():]
channelList4Match = re.compile(
regExp["CHANNEL_LIST_4"]).search(channelListString)
if channelList4Match:
if (channelList4Match.group("channel_list")
== symbolNames["channel_list"]
and channelList4Match.group("channel_number")
== symbolNames["channel_number"]
and channelList4Match.group("channel_name_long")
== symbolNames["channel_name_long"]
and channelList4Match.group("channel_name_short")
== symbolNames["channel_name_short"]
and channelList4Match.group("channel_id_1")
== symbolNames["channel_id_1"]
and channelList4Match.group("channel_id_1")
== symbolNames["channel_id_1"]
and channelList4Match.group("channel_logo_1")
== symbolNames["channel_logo_1"]
and channelList4Match.group("channel_logo_2")
== symbolNames["channel_logo_2"]
and channelList4Match.group("channel_logo_3")
== symbolNames["channel_logo_3"]
and channelList4Match.group("prev_channel_number")
== symbolNames["prev_channel_number"]):
symbolNames["channel_radio"] = channelList4Match.group(
"channel_radio")
else:
self._logger.error(
"_parseCodeJs: mismatch in CHANNEL_LIST_4 with earlier matches!"
)
parseOk = False
else:
_logger.error("_parseCodeJs: no match for CHANNEL_LIST_4")
parseOk = False
channelList5Match = re.compile(
regExp["CHANNEL_LIST_5"]).search(channelListString)
if channelList5Match:
symbolNames[
"last_channel_number"] = channelList5Match.group(
"last_channel_number")
symbolNames[
"next_channel_number"] = channelList5Match.group(
"next_channel_number")
else:
self._logger.error(
"_parseCodeJs: no match for CHANNEL_LIST_5")
parseOk = False
else:
self._logger.error("_parseCodeJs: no match for CHANNEL_LIST_1")
parseOk = False
playStreamActionsMatch = re.compile(
regExp["PLAY_STREAM_ACTIONS"]).search(fileContent)
if playStreamActionsMatch:
symbolNames["play_action_1"] = playStreamActionsMatch.group(
"play_action_1")
symbolNames["play_action_2"] = playStreamActionsMatch.group(
"play_action_2")
else:
self._logger.error(
"_parseCodeJs: no match for PLAY_STREAM_ACTIONS")
parseOk = False
playStreamClassMatch = re.compile(
regExp["PLAY_STREAM_CLASS"]).search(fileContent)
if playStreamClassMatch:
symbolNames["play_stream_class"] = playStreamClassMatch.group(
"play_stream_class")
symbolNames["play_stream_url"] = playStreamClassMatch.group(
"play_stream_url")
else:
self._logger.error(
"_parseCodeJs: no match for PLAY_STREAM_CLASS")
parseOk = False
setChannelFunctionMatch = re.compile(
regExp["SET_CHANNEL_FUNCTION"]).search(fileContent)
if setChannelFunctionMatch:
symbolNames[
"set_channel_function"] = setChannelFunctionMatch.group(
"set_channel_function")
setChannelInstanceMatch = re.compile(
regExp["SET_CHANNEL_INSTANCE"] %
(symbolNames["set_channel_function"])).search(fileContent)
if setChannelInstanceMatch:
symbolNames[
"set_channel_instance"] = setChannelInstanceMatch.group(
"set_channel_instance")
else:
self._logger.error(
"_parseCodeJs: no match for SET_CHANNEL_FUNCTION")
parseOk = False
else:
self._logger.error(
"_parseCodeJs: no match for SET_CHANNEL_FUNCTION")
parseOk = False
channelObjectMatch = re.compile(
regExp["CHANNEL_OBJECT"]).search(fileContent)
if channelObjectMatch:
symbolNames["channel_object_1"] = channelObjectMatch.group(
"channel_object_1")
symbolNames["channel_object_2"] = channelObjectMatch.group(
"channel_object_2")
else:
self._logger.error("_parseCodeJs: no match for CHANNEL_OBJECT")
parseOk = False
debugFunctionMatch = re.compile(
regExp["DEBUG_FUNCTION"]).search(fileContent)
if debugFunctionMatch:
symbolNames["debug_function"] = debugFunctionMatch.group(
"debug_function")
initFunctionMatch = re.compile(
regExp["INIT_FUNCTION"] %
(symbolNames["debug_function"])).search(fileContent)
if not initFunctionMatch:
_logger.error("_parseCodeJs: no match for INIT_FUNCTION")
parseOk = False
lastJSLineMatch = re.compile(
regExp["LAST_JS_LINE"] %
(symbolNames["debug_function"])).search(fileContent)
if not lastJSLineMatch:
self._logger.error(
"_parseCodeJs: no match for LAST_JS_LINE")
parseOk = False
else:
self._logger.error("_parseCodeJs: no match for DEBUG_FUNCTION")
parseOk = False
keyEventFunctionMatch = re.compile(
regExp["KEY_EVENT_FUNCTION"]).search(fileContent)
if keyEventFunctionMatch:
symbolNames[
"key_event_function"] = keyEventFunctionMatch.group(
"key_event_function")
else:
_logger.error("_parseCodeJs: no match for KEY_EVENT_FUNCTION")
parseOk = False
if parseOk:
return (fileContent, symbolNames)
else:
return (None, None)
else:
self._logger.error("_parseCodeJs: no file content!")
return (None, None)
def _parseCodeJs( self, codeJsPath ):
content, _, _ = getPage( codeJsPath )
if content:
fileHandle = gzip.GzipFile( fileobj=StringIO( content ) )
fileContent = fileHandle.read()
parseOk = True
regExp = {
# H [ b ] = { k :b , c :{ " default" :" NPO 1" } , r :{ " default" :" NPO 1" } , l :" ned1" , n :" ned1" , u :" npotv1.png" , v :" npotv1.png" , p :" npotv1.png" , o :a , e :[ ] , f :[ ] , g :[ ] } ;
"CHANNEL_LIST_1": r"(?P<channel_list>[A-Za-z]{1})\[[a-z]{1}\]=\s*\{(?P<channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_name_long>[A-Za-z]{1}):\{\"default\":\"NPO 1\"\},\s*(?P<channel_name_short>[A-Za-z]{1}):\{\"default\":\"NPO 1\"\},\s*(?P<channel_id_1>[A-Za-z]{1}):\"ned1\",\s*(?P<channel_id_2>[A-Za-z]{1}):\"ned1\",\s*(?P<channel_logo_1>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<channel_logo_2>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<channel_logo_3>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<prev_channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_streams>[A-Za-z]{1}):\[\],\s*(?P<channel_unknown_1>[A-Za-z]{1}):\[\],\s*(?P<channel_unknown_2>[A-Za-z]{1}):\[\]\};",
# a = { } ; a .c ={ " default" :"HD+ " }; a .h = " igmp:/ / 224 .0 .251 .124 :8248;rtpskip=yes " ; a .C = 1; a .M = 1;
# a = { } ; a .c ={ " default" :"HD ( NOB) " }; a .h = " igmp:/ / 224 .1 .3 .1 :12110 " ; a .Ba = [{Oa: "103"}]; a .C = 1; a .M = 1;
"CHANNEL_LIST_2": r"[A-Za-z]{1}=\s*\{\};\s*[A-Za-z]{1}.(?P<channel_metadata>[A-Za-z]{1})=\{\"default\":\"[A-Za-z \(\)\+]{2,10}\"\};\s*[A-Za-z]{1}.(?P<channel_url>[A-Za-z]{1,2})=\s*\"igmp:\/\/[0-9]{3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}:[0-9]{1,5}(;rtpskip=yes)?\";\s*([A-Za-z]{1}.[A-Za-z]{1,2}=\s*\[([^\]]*)\];\s*)?[A-Za-z]{1}.[A-Za-z]{1}=\s*1;\s*[A-Za-z]{1}.(?P<channel_url_hd>[A-Za-z]{1})=\s*1;",
# for( $ .tv.q .push( a ) ;f .length;) E [ f .pop( ) ] = a ;
"CHANNEL_LIST_3": r"for\([\$A-Za-z]{1,2}.tv.[A-Za-z]{1,2}.push\([A-Za-z]{1,2}\);[A-Za-z]{1,2}.length;\)(?P<channel_id_list>[A-Za-z]{1,2})\[[A-Za-z]{1,2}.pop\(\)\]=\s*[A-Za-z]{1,2};",
# z [ b ] = { k :b , c :{ " default" :" NPO 3FM" } , q :{ " default" :" NPO 3FM" } , j :" radio3" , m :" radio3" , s :" nporadio3.png" , t :" nporadio3.png" , o :" nporadio3.png" , u :1, n :a ,d:[],e:[]};
"CHANNEL_LIST_4": r"(?P<channel_list>[A-Za-z]{1})\[[A-Za-z]{1}\]=\s*\{(?P<channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_name_long>[A-Za-z]{1}):\{\"default\":\"NPO 3FM\"\},\s*(?P<channel_name_short>[A-Za-z]{1}):\{\"default\":\"NPO 3FM\"\},\s*(?P<channel_id_1>[A-Za-z]{1}):\"radio3\",\s*(?P<channel_id_2>[A-Za-z]{1}):\"radio3\",\s*(?P<channel_logo_1>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_logo_2>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_logo_3>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_radio>[A-Za-z]{1}):1,\s*(?P<prev_channel_number>[A-Za-z]{1}):[a-z]{1}([^\}]+)\};",
# Gb = f ;B [ a ] .i = f ; for( B [ f ] .k = a ;e .length;) w [ e .pop( ) ] = f ;J .root
"CHANNEL_LIST_5": r"(?P<last_channel_number>[A-Za-z]{1,2})=\s*[A-Za-z]{1};[A-Za-z]{1}\[[A-Za-z]{1}\].(?P<next_channel_number>[A-Za-z]{1})=\s*[A-Za-z]{1};\s*for\([A-Za-z]{1}\[[A-Za-z]{1}\].[A-Za-z]{1}=\s*[A-Za-z]{1};[A-Za-z]{1}.length;\)[A-Za-z]{1}\[[A-Za-z]{1}.pop\(\)\]=\s*[A-Za-z]{1};[A-Za-z]{1}.root",
# function( ) { window.clearInterval (g ) ;u ( $a ) ;var e = new ab ( t .dj ) ;e .B =c ( ) ;v ( e ) ;window.setTimeout( function( ) { ga ( ) } ,50) }
"PLAY_STREAM_ACTIONS": r"function\(\)\{window.clearInterval\([A-Za-z]{1,2}\);(?P<play_action_1>[A-Za-z]{1,2})\([$A-Za-z]{1,2}\);var [A-Za-z]{1,2}=\s*new [A-Za-z]{1,2}\([A-Za-z]{1,2}.[A-Za-z]{1,2}\);[A-Za-z]{1,2}.[A-Za-z]{1,2}=[A-Za-z]{1,2}\(\);(?P<play_action_2>[A-Za-z]{1,2})\([A-Za-z]{1,2}\);window.setTimeout\(function\(\)\{[A-Za-z]{1,2}\(\)\},50\)\}",
# function Ue ( a ,b ,f ,g ,e ) {L .call( this) ;if( b ===undefined) b = " " ;
"PLAY_STREAM_CLASS": r"function (?P<play_stream_class>[A-Za-z]{1,2})\((?P<play_stream_url>[A-Za-z]{1,2}),[A-Za-z]{1,2},[A-Za-z]{1,2},[A-Za-z]{1,2},[A-Za-z]{1,2}\){[A-Za-z]{1,2}.call\(this\);if\([A-Za-z]{1,2}===undefined\)[A-Za-z]{1,2}=\s*\"\";",
"SET_CHANNEL_FUNCTION": r"[A-Za-z]{1,2}.prototype.(?P<set_channel_function>[A-Za-z]{1,2})=\s*function\([A-Za-z]{1,2},[A-Za-z]{1,2}\){if\(this.[A-Za-z]{1,2}\){window.clearTimeout\(this.[A-Za-z]{1,2}\);",
# this.l .appendChild( b ) ;b = new O ( o .fh ) ;b .ua = function( ) {G .ka(
"SET_CHANNEL_INSTANCE": r"this.[A-Za-z]{1,2}.appendChild\([A-Za-z]{1,2}\);[A-Za-z]{1,2}=\s*new [A-Za-z]{1,2}\([A-Za-z]{1,2}.[A-Za-z]{1,2}\);[A-Za-z]{1,2}.[A-Za-z]{1,2}=\s*function\(\){(?P<set_channel_instance>[A-Za-z]{1,2}).%s\(",
"CHANNEL_OBJECT": r"(document.cookie=\"channel=\"\+encodeURIComponent\(this.(?P<channel_object_1>[A-Za-z]{1,2}).(?P<channel_object_2>[A-Za-z]{1,2})\);)",
"DEBUG_FUNCTION": r"function (?P<debug_function>[A-Za-z]{1,2})\([A-Za-z]{1,2}\){if\(window.console\)window.console.log\(",
# for( o ( " doRunAfterLoad( ) : " + Kb .length) ;Kb .length;) { var a =Kb .pop( ) ;try{ a ( ) }
"INIT_FUNCTION": r"for\(%s\(\"doRunAfterLoad\(\): \"\+[A-Za-z]{1,2}.length\);[A-Za-z]{1,2}.length;\)\{var [A-Za-z]{1,2}=[A-Za-z]{1,2}.pop\(\);try\{[A-Za-z]{1,2}\(\)\}",
"LAST_JS_LINE": r"([A-Za-z]{1,2}\(\);\s*%s\(\"initial buildMenu\(\) done.\"\);)",
# function oc ( b ) { ia ( ) ;$a &&clearTimeout( $a ) ;b = b .keyCode| | b .charCode;
"KEY_EVENT_FUNCTION": r"function (?P<key_event_function>[A-Za-z]{1,2})\([A-Za-z]{1,2}\)\{[A-Za-z]{1,2}\(\);[$A-Za-z]{1,2}&&clearTimeout\([$A-Za-z]{1,2}\);[A-Za-z]{1,2}=\s*[A-Za-z]{1,2}.keyCode\|\|[A-Za-z]{1,2}.charCode;",
}
symbolNames = {}
symbolNames["channel_logo_path"] = self._glashartConfig.channelLogoPath
symbolNames["channel_thumb_path"] = self._glashartConfig.channelThumbPath
channelList1Match = re.compile( regExp["CHANNEL_LIST_1"] ).search( fileContent )
if channelList1Match:
symbolNames["channel_list"] = channelList1Match.group( "channel_list" )
symbolNames["channel_number"] = channelList1Match.group( "channel_number" )
symbolNames["channel_name_long"] = channelList1Match.group( "channel_name_long" )
symbolNames["channel_name_short"] = channelList1Match.group( "channel_name_short" )
symbolNames["channel_id_1"] = channelList1Match.group( "channel_id_1" )
symbolNames["channel_id_2"] = channelList1Match.group( "channel_id_2" )
symbolNames["channel_logo_1"] = channelList1Match.group( "channel_logo_1" )
symbolNames["channel_logo_2"] = channelList1Match.group( "channel_logo_2" )
symbolNames["channel_logo_3"] = channelList1Match.group( "channel_logo_3" )
symbolNames["prev_channel_number"] = channelList1Match.group( "prev_channel_number" )
symbolNames["channel_streams"] = channelList1Match.group( "channel_streams" )
# For the folloing matches I only need about 3000 characters starting from the start of this match
channelListString = fileContent[channelList1Match.start():channelList1Match.start() + 3000]
channelList2Match = re.compile( regExp["CHANNEL_LIST_2"] ).search( channelListString )
if channelList2Match:
symbolNames["channel_metadata"] = channelList2Match.group( "channel_metadata" )
symbolNames["channel_url"] = channelList2Match.group( "channel_url" )
symbolNames["channel_url_hd"] = channelList2Match.group( "channel_url_hd" )
else:
self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_2" )
parseOk = False
channelList3Match = re.compile( regExp["CHANNEL_LIST_3"] ).search( channelListString )
if channelList3Match:
symbolNames["channel_id_list"] = channelList3Match.group( "channel_id_list" )
else:
self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_3" )
parseOk = False
# Take the rest of the file starting from the first match
channelListString = fileContent[channelList1Match.start():]
channelList4Match = re.compile( regExp["CHANNEL_LIST_4"] ).search( channelListString )
if channelList4Match:
if ( channelList4Match.group( "channel_list" ) == symbolNames["channel_list"] and
channelList4Match.group( "channel_number" ) == symbolNames["channel_number"] and
channelList4Match.group( "channel_name_long" ) == symbolNames["channel_name_long"] and
channelList4Match.group( "channel_name_short" ) == symbolNames["channel_name_short"] and
channelList4Match.group( "channel_id_1" ) == symbolNames["channel_id_1"] and
channelList4Match.group( "channel_id_1" ) == symbolNames["channel_id_1"] and
channelList4Match.group( "channel_logo_1" ) == symbolNames["channel_logo_1"] and
channelList4Match.group( "channel_logo_2" ) == symbolNames["channel_logo_2"] and
channelList4Match.group( "channel_logo_3" ) == symbolNames["channel_logo_3"] and
channelList4Match.group( "prev_channel_number" ) == symbolNames["prev_channel_number"] ):
symbolNames["channel_radio"] = channelList4Match.group( "channel_radio" )
else:
self._logger.error( "_parseCodeJs: mismatch in CHANNEL_LIST_4 with earlier matches!" )
parseOk = False
else:
_logger.error( "_parseCodeJs: no match for CHANNEL_LIST_4" )
parseOk = False
channelList5Match = re.compile( regExp["CHANNEL_LIST_5"] ).search( channelListString )
if channelList5Match:
symbolNames["last_channel_number"] = channelList5Match.group( "last_channel_number" )
symbolNames["next_channel_number"] = channelList5Match.group( "next_channel_number" )
else:
self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_5" )
parseOk = False
else:
self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_1" )
parseOk = False
playStreamActionsMatch = re.compile( regExp["PLAY_STREAM_ACTIONS"] ).search( fileContent )
if playStreamActionsMatch:
symbolNames["play_action_1"] = playStreamActionsMatch.group( "play_action_1" );
symbolNames["play_action_2"] = playStreamActionsMatch.group( "play_action_2" );
else:
self._logger.error( "_parseCodeJs: no match for PLAY_STREAM_ACTIONS" )
parseOk = False
playStreamClassMatch = re.compile( regExp["PLAY_STREAM_CLASS"] ).search( fileContent )
if playStreamClassMatch:
symbolNames["play_stream_class"] = playStreamClassMatch.group( "play_stream_class" );
symbolNames["play_stream_url"] = playStreamClassMatch.group( "play_stream_url" );
else:
self._logger.error( "_parseCodeJs: no match for PLAY_STREAM_CLASS" )
parseOk = False
setChannelFunctionMatch = re.compile( regExp["SET_CHANNEL_FUNCTION"] ).search( fileContent )
if setChannelFunctionMatch:
symbolNames["set_channel_function"] = setChannelFunctionMatch.group( "set_channel_function" );
setChannelInstanceMatch = re.compile( regExp["SET_CHANNEL_INSTANCE"] % ( symbolNames["set_channel_function"] ) ).search( fileContent )
if setChannelInstanceMatch:
symbolNames["set_channel_instance"] = setChannelInstanceMatch.group( "set_channel_instance" );
else:
self._logger.error( "_parseCodeJs: no match for SET_CHANNEL_FUNCTION" )
parseOk = False
else:
self._logger.error( "_parseCodeJs: no match for SET_CHANNEL_FUNCTION" )
parseOk = False
channelObjectMatch = re.compile( regExp["CHANNEL_OBJECT"] ).search( fileContent )
if channelObjectMatch:
symbolNames["channel_object_1"] = channelObjectMatch.group( "channel_object_1" );
symbolNames["channel_object_2"] = channelObjectMatch.group( "channel_object_2" );
else:
self._logger.error( "_parseCodeJs: no match for CHANNEL_OBJECT" )
parseOk = False
debugFunctionMatch = re.compile( regExp["DEBUG_FUNCTION"] ).search( fileContent )
if debugFunctionMatch:
symbolNames["debug_function"] = debugFunctionMatch.group( "debug_function" );
initFunctionMatch = re.compile( regExp["INIT_FUNCTION"] % ( symbolNames["debug_function"] ) ).search( fileContent )
if not initFunctionMatch:
_logger.error( "_parseCodeJs: no match for INIT_FUNCTION" )
parseOk = False
lastJSLineMatch = re.compile( regExp["LAST_JS_LINE"] % ( symbolNames["debug_function"] ) ).search( fileContent )
if not lastJSLineMatch:
self._logger.error( "_parseCodeJs: no match for LAST_JS_LINE" )
parseOk = False
else:
self._logger.error( "_parseCodeJs: no match for DEBUG_FUNCTION" )
parseOk = False
keyEventFunctionMatch = re.compile( regExp["KEY_EVENT_FUNCTION"] ).search( fileContent )
if keyEventFunctionMatch:
symbolNames["key_event_function"] = keyEventFunctionMatch.group( "key_event_function" );
else:
_logger.error( "_parseCodeJs: no match for KEY_EVENT_FUNCTION" )
parseOk = False
if parseOk:
return ( fileContent, symbolNames )
else:
return ( None, None )
else:
self._logger.error( "_parseCodeJs: no file content!" )
return ( None, None )
def _grabEpgForChannel( self, channel=None, epgId=None ):
conn = DBConnection()
if channel:
epgId = EpgId.getFromDb( conn, channel.epgId )
self._logger.info( "Grabbing EPG for channel: %s (%s; method: %s)" % ( channel.name, channel.epgId, epgId.strategy ) )
if not epgId:
return
else:
self._logger.info( "Grabbing EPG for epgId: %s (method: %s)" % ( epgId.epgId, epgId.strategy ) )
# Check if _fail_# is behind strategy
# This is to indicate epg grabbing for this epgId failed previously
strategy = epgId.strategy
strategyRe = re.compile( r'_fail_(?P<fail>\d+)' )
failMatch = strategyRe.search( strategy )
failCount = 0
if failMatch:
failCount = int( failMatch.group( "fail" ) )
strategy = epgId.strategy.split( '_' )[0]
# We're going to attempt to grab EPG information for this channel 5 times
# before we stop grabbing this epgId in the future.
if failCount < 5:
now = time.localtime()
nowDay = datetime.datetime( now[0], now[1], now[2] )
daysDetailDelta = datetime.timedelta( days = 3 )
epgFilename = "/%s.json.gz" % ( epgId.epgId )
epgUrl = self._glashartConfig.epgChannelsPath + epgFilename
currentPrograms = EpgProgram.getAllByEpgIdFromDb( conn, epgId.epgId )
currentProgramsDict = { currProgram.originalId: currProgram for currProgram in currentPrograms }
newProgramsDict = {}
content, _, _ = getPage( epgUrl )
if content:
fileHandle = gzip.GzipFile( fileobj=StringIO( content ) )
epgData = json.loads( fileHandle.read() )
# If strategy has changed (working after (a few) failed attempts)
if epgId.strategy != strategy:
epgId.strategy = strategy
epgId.addToDb( conn )
numPrograms = 0
numProgramsDetail = 0
numProgramsDetailFailed = 0
numProgramsNew = 0
numProgramsUpdated = 0
for program in epgData:
if not self._running:
break
numPrograms += 1
programNew = self._getProgramFromJson( epgId.epgId, program )
if programNew.originalId in newProgramsDict:
self._logger.warning( "Program with originalId %d already in newProgramsDict" % ( programNew.originalId ) )
newProgramsDict[programNew.originalId] = programNew
updateDetailedData = True
programOld = None
if programNew.originalId in currentProgramsDict:
programOld = currentProgramsDict[programNew.originalId]
# If the old program has detailed info, copy those fields
# TODO: do this somewhere else
if programOld and programOld.detailed:
programNew.subtitle = programOld.subtitle
programNew.description = programOld.description
programNew.aspectRatio = programOld.aspectRatio
programNew.parentalRating = programOld.parentalRating
programNew.genres = programOld.genres
programNew.actors = programOld.actors
programNew.directors = programOld.directors
programNew.presenters = programOld.presenters
programNew.ratings = programOld.ratings
programNew.detailed = programOld.detailed
# Now, compare the old program and the new program
# Are they the same, then we don't need to download detailed information
if programOld and programOld.detailed and programNew == programOld:
programNew = programOld
updateDetailedData = False
if updateDetailedData:
if ( ( epgId.strategy == "default" and (nowDay + daysDetailDelta) > datetime.datetime.fromtimestamp( programNew.startTime ) ) or
( epgId.strategy == "full" ) ):
time.sleep( random.uniform( 0.5, 1.0 ) )
programNew, grabbed = self._grabDetailedEpgForProgram( programNew )
if grabbed:
numProgramsDetail += 1
else:
# if more than 10 detailed program information grabs failed, set strategy to none.
numProgramsDetailFailed += 1
if numProgramsDetailFailed == 10:
self._logger.error( "Couldn't download at least 10 detailed program information files, so setting strategy to 'none', but do not store" )
epgId.strategy = "none"
conn.delayCommit( True )
for programId in newProgramsDict:
programNew = newProgramsDict[programId]
programOld = None
if programNew.originalId in currentProgramsDict:
programOld = currentProgramsDict[programNew.originalId]
if not programOld or programNew != programOld:
if programOld:
self._logger.debug( "Updated program: id = %s" % ( programNew.originalId ) )
self._logger.debug( "Start time: %s > %s" % ( str( programOld.startTime ), str( programNew.startTime ) ) )
self._logger.debug( "End time: %s > %s" % ( str( programOld.endTime ), str( programNew.endTime ) ) )
self._logger.debug( "Name: %s > %s" % ( repr( programOld.title ), repr( programNew.title ) ) )
programNew.id = programOld.id
numProgramsUpdated += 1
else:
numProgramsNew += 1
try:
programNew.addToDb( conn )
except:
self._logger.exception( programNew.dump() )
conn.delayCommit( False )
if self._running:
self._logger.debug( "Num programs: %i" % ( numPrograms ) )
self._logger.debug( "Num program details: %i" % ( numProgramsDetail ) )
self._logger.info( "Num new programs: %i" % ( numProgramsNew ) )
self._logger.info( "Num updated programs: %i" % ( numProgramsUpdated ) )
if numProgramsNew == 0:
self._logger.warning( "No new programs were added for epgId: %s" % ( epgId.epgId ) )
else:
self._logger.warning( "Unable to download EPG information for epgId: %s" % ( epgId.epgId ) )
failCount += 1
epgId.strategy = "%s_fail_%d" % ( strategy, failCount )
epgId.addToDb( conn )
else:
self._logger.info( "Downloading of EPG information for epgId: %s skipped becaused it failed too many times" % ( epgId.epgId ) )
def _grabEpgForChannel(self, channel=None, epgId=None):
conn = DBConnection()
if channel:
epgId = EpgId.getFromDb(conn, channel.epgId)
self._logger.info("Grabbing EPG for channel: %s (%s; method: %s)" %
(channel.name, channel.epgId, epgId.strategy))
if not epgId:
return
else:
self._logger.info("Grabbing EPG for epgId: %s (method: %s)" %
(epgId.epgId, epgId.strategy))
# Check if _fail_# is behind strategy
# This is to indicate epg grabbing for this epgId failed previously
strategy = epgId.strategy
strategyRe = re.compile(r'_fail_(?P<fail>\d+)')
failMatch = strategyRe.search(strategy)
failCount = 0
if failMatch:
failCount = int(failMatch.group("fail"))
strategy = epgId.strategy.split('_')[0]
# We're going to attempt to grab EPG information for this channel 5 times
# before we stop grabbing this epgId in the future.
if failCount < 5:
now = time.localtime()
nowDay = datetime.datetime(now[0], now[1], now[2])
daysDetailDelta = datetime.timedelta(days=3)
epgFilename = "/%s.json.gz" % (epgId.epgId)
epgUrl = self._glashartConfig.epgChannelsPath + epgFilename
currentPrograms = EpgProgram.getAllByEpgIdFromDb(conn, epgId.epgId)
currentProgramsDict = {
currProgram.originalId: currProgram
for currProgram in currentPrograms
}
newProgramsDict = {}
content, _, _ = getPage(epgUrl)
if content:
fileHandle = gzip.GzipFile(fileobj=StringIO(content))
epgData = json.loads(fileHandle.read())
# If strategy has changed (working after (a few) failed attempts)
if epgId.strategy != strategy:
epgId.strategy = strategy
epgId.addToDb(conn)
numPrograms = 0
numProgramsDetail = 0
numProgramsDetailFailed = 0
numProgramsNew = 0
numProgramsUpdated = 0
for program in epgData:
if not self._running:
break
numPrograms += 1
programNew = self._getProgramFromJson(epgId.epgId, program)
if programNew.originalId in newProgramsDict:
self._logger.warning(
"Program with originalId %d already in newProgramsDict"
% (programNew.originalId))
newProgramsDict[programNew.originalId] = programNew
updateDetailedData = True
programOld = None
if programNew.originalId in currentProgramsDict:
programOld = currentProgramsDict[programNew.originalId]
# If the old program has detailed info, copy those fields
# TODO: do this somewhere else
if programOld and programOld.detailed:
programNew.subtitle = programOld.subtitle
programNew.description = programOld.description
programNew.aspectRatio = programOld.aspectRatio
programNew.parentalRating = programOld.parentalRating
programNew.genres = programOld.genres
programNew.actors = programOld.actors
programNew.directors = programOld.directors
programNew.presenters = programOld.presenters
programNew.ratings = programOld.ratings
programNew.detailed = programOld.detailed
# Now, compare the old program and the new program
# Are they the same, then we don't need to download detailed information
if programOld and programOld.detailed and programNew == programOld:
programNew = programOld
updateDetailedData = False
if updateDetailedData:
if ((epgId.strategy == "default" and
(nowDay + daysDetailDelta) >
datetime.datetime.fromtimestamp(
programNew.startTime))
or (epgId.strategy == "full")):
time.sleep(random.uniform(0.5, 1.0))
programNew, grabbed = self._grabDetailedEpgForProgram(
programNew)
if grabbed:
numProgramsDetail += 1
else:
# if more than 10 detailed program information grabs failed, set strategy to none.
numProgramsDetailFailed += 1
if numProgramsDetailFailed == 10:
self._logger.error(
"Couldn't download at least 10 detailed program information files, so setting strategy to 'none', but do not store"
)
epgId.strategy = "none"
conn.delayCommit(True)
for programId in newProgramsDict:
programNew = newProgramsDict[programId]
programOld = None
if programNew.originalId in currentProgramsDict:
programOld = currentProgramsDict[programNew.originalId]
if not programOld or programNew != programOld:
if programOld:
self._logger.debug("Updated program: id = %s" %
(programNew.originalId))
self._logger.debug("Start time: %s > %s" %
(str(programOld.startTime),
str(programNew.startTime)))
self._logger.debug("End time: %s > %s" % (str(
programOld.endTime), str(programNew.endTime)))
self._logger.debug("Name: %s > %s" % (repr(
programOld.title), repr(programNew.title)))
programNew.id = programOld.id
numProgramsUpdated += 1
else:
numProgramsNew += 1
try:
programNew.addToDb(conn)
except:
self._logger.exception(programNew.dump())
conn.delayCommit(False)
if self._running:
self._logger.debug("Num programs: %i" %
(numPrograms))
self._logger.debug("Num program details: %i" %
(numProgramsDetail))
self._logger.info("Num new programs: %i" %
(numProgramsNew))
self._logger.info("Num updated programs: %i" %
(numProgramsUpdated))
if numProgramsNew == 0:
self._logger.warning(
"No new programs were added for epgId: %s" %
(epgId.epgId))
else:
self._logger.warning(
"Unable to download EPG information for epgId: %s" %
(epgId.epgId))
failCount += 1
epgId.strategy = "%s_fail_%d" % (strategy, failCount)
epgId.addToDb(conn)
else:
self._logger.info(
"Downloading of EPG information for epgId: %s skipped becaused it failed too many times"
% (epgId.epgId))