def downloadLogoAndThumbnail(self): if self._logo != "" and self._logo.startswith("http://"): logoFilename = urlparse.urlsplit(self._logo)[2].split('/')[-1] logoPath = os.path.join(DATA_ROOT, "assets/images/channels/logos", logoFilename) filename, _, _ = getPage(self._logo, logoPath) if filename: self._logo = logoFilename self._logger.info( "Channel.downloadLogoAndThumbnail: downloaded logo to %s" % (logoPath)) else: self._logger.info( "Channel.downloadLogoAndThumbnail: could not download logo from %s" % (self._logo)) self._logo = "" if self._thumbnail != "" and self._thumbnail.startswith("http://"): thumbnailFilename = urlparse.urlsplit( self._thumbnail)[2].split('/')[-1] thumbnailPath = os.path.join(DATA_ROOT, "assets/images/channels/thumbnails", thumbnailFilename) filename, _, _ = getPage(self._thumbnail, thumbnailPath) if filename: self._thumbnail = thumbnailFilename self._logger.info( "Channel.downloadLogoAndThumbnail: downloaded thumbnail to %s" % (thumbnailPath)) else: self._logger.info( "Channel.downloadLogoAndThumbnail: could not download thumbnail from %s" % (self._thumbnail)) self._thumbnail = ""
def _getStyleCss(self, styleCssPath): content, _, _ = getPage(styleCssPath) if content: fileHandle = gzip.GzipFile(fileobj=StringIO(content)) fileContent = fileHandle.read() return fileContent return None
def _getStyleCss( self, styleCssPath ): content, _, _ = getPage( styleCssPath ) if content: fileHandle = gzip.GzipFile( fileobj=StringIO( content ) ) fileContent = fileHandle.read() return fileContent return None
def downloadLogoAndThumbnail( self ): if self._logo != "" and self._logo.startswith( "http://" ): logoFilename = urlparse.urlsplit( self._logo )[2].split( '/' )[-1] logoPath = os.path.join( DATA_ROOT, "assets/images/channels/logos", logoFilename ) filename, _, _ = getPage( self._logo, logoPath ) if filename: self._logo = logoFilename self._logger.info( "Channel.downloadLogoAndThumbnail: downloaded logo to %s" % ( logoPath ) ) else: self._logger.info( "Channel.downloadLogoAndThumbnail: could not download logo from %s" % ( self._logo ) ) self._logo = "" if self._thumbnail != "" and self._thumbnail.startswith( "http://" ): thumbnailFilename = urlparse.urlsplit( self._thumbnail )[2].split( '/' )[-1] thumbnailPath = os.path.join( DATA_ROOT, "assets/images/channels/thumbnails", thumbnailFilename ) filename, _, _ = getPage( self._thumbnail, thumbnailPath ) if filename: self._thumbnail = thumbnailFilename self._logger.info( "Channel.downloadLogoAndThumbnail: downloaded thumbnail to %s" % ( thumbnailPath ) ) else: self._logger.info( "Channel.downloadLogoAndThumbnail: could not download thumbnail from %s" % ( self._thumbnail ) ) self._thumbnail = ""
def _grabDetailedEpgForProgram( self, program, epgId=None ): grabbed = True # Fetch detailed information. http://w.zt6.nl/epgdata/xx/xxxxxx.json detailsFilename = "/%s/%s.json" % ( program.originalId[-2:], program.originalId ) detailsUrl = self._glashartConfig.epgDataPath + detailsFilename detailsPage, _, _ = getPage( detailsUrl, None ) if detailsPage and len( detailsPage ) > 0: detailsData = json.loads( detailsPage ) program = self._getDetailedProgramFromJson( program, detailsData ) else: grabbed = False return program, grabbed
def _grabDetailedEpgForProgram(self, program, epgId=None): grabbed = True # Fetch detailed information. http://w.zt6.nl/epgdata/xx/xxxxxx.json detailsFilename = "/%s/%s.json" % (program.originalId[-2:], program.originalId) detailsUrl = self._glashartConfig.epgDataPath + detailsFilename detailsPage, _, _ = getPage(detailsUrl, None) if detailsPage and len(detailsPage) > 0: detailsData = json.loads(detailsPage) program = self._getDetailedProgramFromJson(program, detailsData) else: grabbed = False return program, grabbed
def _parseIndexPage(self): content, _, _ = getPage(self._glashartConfig.tvmenuIndexPath) if content: title = None codeJsPath = None styleCssPath = None fileHandle = gzip.GzipFile(fileobj=StringIO(content)) fileContent = fileHandle.read() regExp = { "TITLE": r"\<title\>(?P<page_title>.*)\<\/title\>", "STYLE": r"\<link rel='stylesheet' type='text\/css' href='(?P<css_filename>.*)'\>\<\/link\>", "JAVASCRIPT": r"\<script type='text\/javascript' src='(?P<js_filename>.*)'\>\<\/script\>" } titleMatch = re.compile(regExp["TITLE"]).search(fileContent) if titleMatch: title = titleMatch.group("page_title") else: self._logger.error("_parseIndexPage: no match for TITLE") return (None, None, None, None) codeJsMatch = re.compile(regExp["JAVASCRIPT"]).search(fileContent) if codeJsMatch: codeJsPath = self._glashartConfig.iptvBaseUrl + "/" + self._glashartConfig.tvmenuPath + "/" + codeJsMatch.group( "js_filename") else: self._logger.error("_parseIndexPage: no match for JAVASCRIPT") return (None, None, None, None) styleCssMatch = re.compile(regExp["STYLE"]).search(fileContent) if styleCssMatch: styleCssPath = self._glashartConfig.iptvBaseUrl + "/" + self._glashartConfig.tvmenuPath + "/" + styleCssMatch.group( "css_filename") else: self._logger.error("_parseIndexPage: no match for STYLE") return (None, None, None, None) return (fileContent, title, codeJsPath, styleCssPath) else: return (None, None, None, None)
def _parseIndexPage( self ): content, _, _ = getPage( self._glashartConfig.tvmenuIndexPath ) if content: title = None codeJsPath = None styleCssPath = None fileHandle = gzip.GzipFile( fileobj=StringIO( content ) ) fileContent = fileHandle.read() regExp = { "TITLE": r"\<title\>(?P<page_title>.*)\<\/title\>", "STYLE": r"\<link rel='stylesheet' type='text\/css' href='(?P<css_filename>.*)'\>\<\/link\>", "JAVASCRIPT": r"\<script type='text\/javascript' src='(?P<js_filename>.*)'\>\<\/script\>" } titleMatch = re.compile( regExp["TITLE"] ).search( fileContent ) if titleMatch: title = titleMatch.group( "page_title" ) else: self._logger.error( "_parseIndexPage: no match for TITLE" ) return ( None, None, None, None ) codeJsMatch = re.compile( regExp["JAVASCRIPT"] ).search( fileContent ) if codeJsMatch: codeJsPath = self._glashartConfig.iptvBaseUrl + "/" + self._glashartConfig.tvmenuPath + "/" + codeJsMatch.group( "js_filename" ) else: self._logger.error( "_parseIndexPage: no match for JAVASCRIPT" ) return ( None, None, None, None ) styleCssMatch = re.compile( regExp["STYLE"] ).search( fileContent ) if styleCssMatch: styleCssPath = self._glashartConfig.iptvBaseUrl + "/" + self._glashartConfig.tvmenuPath + "/" + styleCssMatch.group( "css_filename" ) else: self._logger.error( "_parseIndexPage: no match for STYLE" ) return ( None, None, None, None ) return ( fileContent, title, codeJsPath, styleCssPath ) else: return ( None, None, None, None )
def _parseCodeJs(self, codeJsPath): content, _, _ = getPage(codeJsPath) if content: fileHandle = gzip.GzipFile(fileobj=StringIO(content)) fileContent = fileHandle.read() parseOk = True regExp = { # H [ b ] = { k :b , c :{ " default" :" NPO 1" } , r :{ " default" :" NPO 1" } , l :" ned1" , n :" ned1" , u :" npotv1.png" , v :" npotv1.png" , p :" npotv1.png" , o :a , e :[ ] , f :[ ] , g :[ ] } ; "CHANNEL_LIST_1": r"(?P<channel_list>[A-Za-z]{1})\[[a-z]{1}\]=\s*\{(?P<channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_name_long>[A-Za-z]{1}):\{\"default\":\"NPO 1\"\},\s*(?P<channel_name_short>[A-Za-z]{1}):\{\"default\":\"NPO 1\"\},\s*(?P<channel_id_1>[A-Za-z]{1}):\"ned1\",\s*(?P<channel_id_2>[A-Za-z]{1}):\"ned1\",\s*(?P<channel_logo_1>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<channel_logo_2>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<channel_logo_3>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<prev_channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_streams>[A-Za-z]{1}):\[\],\s*(?P<channel_unknown_1>[A-Za-z]{1}):\[\],\s*(?P<channel_unknown_2>[A-Za-z]{1}):\[\]\};", # a = { } ; a .c ={ " default" :"HD+ " }; a .h = " igmp:/ / 224 .0 .251 .124 :8248;rtpskip=yes " ; a .C = 1; a .M = 1; # a = { } ; a .c ={ " default" :"HD ( NOB) " }; a .h = " igmp:/ / 224 .1 .3 .1 :12110 " ; a .Ba = [{Oa: "103"}]; a .C = 1; a .M = 1; "CHANNEL_LIST_2": r"[A-Za-z]{1}=\s*\{\};\s*[A-Za-z]{1}.(?P<channel_metadata>[A-Za-z]{1})=\{\"default\":\"[A-Za-z \(\)\+]{2,10}\"\};\s*[A-Za-z]{1}.(?P<channel_url>[A-Za-z]{1,2})=\s*\"igmp:\/\/[0-9]{3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}:[0-9]{1,5}(;rtpskip=yes)?\";\s*([A-Za-z]{1}.[A-Za-z]{1,2}=\s*\[([^\]]*)\];\s*)?[A-Za-z]{1}.[A-Za-z]{1}=\s*1;\s*[A-Za-z]{1}.(?P<channel_url_hd>[A-Za-z]{1})=\s*1;", # for( $ .tv.q .push( a ) ;f .length;) E [ f .pop( ) ] = a ; "CHANNEL_LIST_3": r"for\([\$A-Za-z]{1,2}.tv.[A-Za-z]{1,2}.push\([A-Za-z]{1,2}\);[A-Za-z]{1,2}.length;\)(?P<channel_id_list>[A-Za-z]{1,2})\[[A-Za-z]{1,2}.pop\(\)\]=\s*[A-Za-z]{1,2};", # z [ b ] = { k :b , c :{ " default" :" NPO 3FM" } , q :{ " default" :" NPO 3FM" } , j :" radio3" , m :" radio3" , s :" nporadio3.png" , t :" nporadio3.png" , o :" nporadio3.png" , u :1, n :a ,d:[],e:[]}; "CHANNEL_LIST_4": r"(?P<channel_list>[A-Za-z]{1})\[[A-Za-z]{1}\]=\s*\{(?P<channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_name_long>[A-Za-z]{1}):\{\"default\":\"NPO 3FM\"\},\s*(?P<channel_name_short>[A-Za-z]{1}):\{\"default\":\"NPO 3FM\"\},\s*(?P<channel_id_1>[A-Za-z]{1}):\"radio3\",\s*(?P<channel_id_2>[A-Za-z]{1}):\"radio3\",\s*(?P<channel_logo_1>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_logo_2>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_logo_3>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_radio>[A-Za-z]{1}):1,\s*(?P<prev_channel_number>[A-Za-z]{1}):[a-z]{1}([^\}]+)\};", # Gb = f ;B [ a ] .i = f ; for( B [ f ] .k = a ;e .length;) w [ e .pop( ) ] = f ;J .root "CHANNEL_LIST_5": r"(?P<last_channel_number>[A-Za-z]{1,2})=\s*[A-Za-z]{1};[A-Za-z]{1}\[[A-Za-z]{1}\].(?P<next_channel_number>[A-Za-z]{1})=\s*[A-Za-z]{1};\s*for\([A-Za-z]{1}\[[A-Za-z]{1}\].[A-Za-z]{1}=\s*[A-Za-z]{1};[A-Za-z]{1}.length;\)[A-Za-z]{1}\[[A-Za-z]{1}.pop\(\)\]=\s*[A-Za-z]{1};[A-Za-z]{1}.root", # function( ) { window.clearInterval (g ) ;u ( $a ) ;var e = new ab ( t .dj ) ;e .B =c ( ) ;v ( e ) ;window.setTimeout( function( ) { ga ( ) } ,50) } "PLAY_STREAM_ACTIONS": r"function\(\)\{window.clearInterval\([A-Za-z]{1,2}\);(?P<play_action_1>[A-Za-z]{1,2})\([$A-Za-z]{1,2}\);var [A-Za-z]{1,2}=\s*new [A-Za-z]{1,2}\([A-Za-z]{1,2}.[A-Za-z]{1,2}\);[A-Za-z]{1,2}.[A-Za-z]{1,2}=[A-Za-z]{1,2}\(\);(?P<play_action_2>[A-Za-z]{1,2})\([A-Za-z]{1,2}\);window.setTimeout\(function\(\)\{[A-Za-z]{1,2}\(\)\},50\)\}", # function Ue ( a ,b ,f ,g ,e ) {L .call( this) ;if( b ===undefined) b = " " ; "PLAY_STREAM_CLASS": r"function (?P<play_stream_class>[A-Za-z]{1,2})\((?P<play_stream_url>[A-Za-z]{1,2}),[A-Za-z]{1,2},[A-Za-z]{1,2},[A-Za-z]{1,2},[A-Za-z]{1,2}\){[A-Za-z]{1,2}.call\(this\);if\([A-Za-z]{1,2}===undefined\)[A-Za-z]{1,2}=\s*\"\";", "SET_CHANNEL_FUNCTION": r"[A-Za-z]{1,2}.prototype.(?P<set_channel_function>[A-Za-z]{1,2})=\s*function\([A-Za-z]{1,2},[A-Za-z]{1,2}\){if\(this.[A-Za-z]{1,2}\){window.clearTimeout\(this.[A-Za-z]{1,2}\);", # this.l .appendChild( b ) ;b = new O ( o .fh ) ;b .ua = function( ) {G .ka( "SET_CHANNEL_INSTANCE": r"this.[A-Za-z]{1,2}.appendChild\([A-Za-z]{1,2}\);[A-Za-z]{1,2}=\s*new [A-Za-z]{1,2}\([A-Za-z]{1,2}.[A-Za-z]{1,2}\);[A-Za-z]{1,2}.[A-Za-z]{1,2}=\s*function\(\){(?P<set_channel_instance>[A-Za-z]{1,2}).%s\(", "CHANNEL_OBJECT": r"(document.cookie=\"channel=\"\+encodeURIComponent\(this.(?P<channel_object_1>[A-Za-z]{1,2}).(?P<channel_object_2>[A-Za-z]{1,2})\);)", "DEBUG_FUNCTION": r"function (?P<debug_function>[A-Za-z]{1,2})\([A-Za-z]{1,2}\){if\(window.console\)window.console.log\(", # for( o ( " doRunAfterLoad( ) : " + Kb .length) ;Kb .length;) { var a =Kb .pop( ) ;try{ a ( ) } "INIT_FUNCTION": r"for\(%s\(\"doRunAfterLoad\(\): \"\+[A-Za-z]{1,2}.length\);[A-Za-z]{1,2}.length;\)\{var [A-Za-z]{1,2}=[A-Za-z]{1,2}.pop\(\);try\{[A-Za-z]{1,2}\(\)\}", "LAST_JS_LINE": r"([A-Za-z]{1,2}\(\);\s*%s\(\"initial buildMenu\(\) done.\"\);)", # function oc ( b ) { ia ( ) ;$a &&clearTimeout( $a ) ;b = b .keyCode| | b .charCode; "KEY_EVENT_FUNCTION": r"function (?P<key_event_function>[A-Za-z]{1,2})\([A-Za-z]{1,2}\)\{[A-Za-z]{1,2}\(\);[$A-Za-z]{1,2}&&clearTimeout\([$A-Za-z]{1,2}\);[A-Za-z]{1,2}=\s*[A-Za-z]{1,2}.keyCode\|\|[A-Za-z]{1,2}.charCode;", } symbolNames = {} symbolNames[ "channel_logo_path"] = self._glashartConfig.channelLogoPath symbolNames[ "channel_thumb_path"] = self._glashartConfig.channelThumbPath channelList1Match = re.compile( regExp["CHANNEL_LIST_1"]).search(fileContent) if channelList1Match: symbolNames["channel_list"] = channelList1Match.group( "channel_list") symbolNames["channel_number"] = channelList1Match.group( "channel_number") symbolNames["channel_name_long"] = channelList1Match.group( "channel_name_long") symbolNames["channel_name_short"] = channelList1Match.group( "channel_name_short") symbolNames["channel_id_1"] = channelList1Match.group( "channel_id_1") symbolNames["channel_id_2"] = channelList1Match.group( "channel_id_2") symbolNames["channel_logo_1"] = channelList1Match.group( "channel_logo_1") symbolNames["channel_logo_2"] = channelList1Match.group( "channel_logo_2") symbolNames["channel_logo_3"] = channelList1Match.group( "channel_logo_3") symbolNames["prev_channel_number"] = channelList1Match.group( "prev_channel_number") symbolNames["channel_streams"] = channelList1Match.group( "channel_streams") # For the folloing matches I only need about 3000 characters starting from the start of this match channelListString = fileContent[channelList1Match.start( ):channelList1Match.start() + 3000] channelList2Match = re.compile( regExp["CHANNEL_LIST_2"]).search(channelListString) if channelList2Match: symbolNames["channel_metadata"] = channelList2Match.group( "channel_metadata") symbolNames["channel_url"] = channelList2Match.group( "channel_url") symbolNames["channel_url_hd"] = channelList2Match.group( "channel_url_hd") else: self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_2") parseOk = False channelList3Match = re.compile( regExp["CHANNEL_LIST_3"]).search(channelListString) if channelList3Match: symbolNames["channel_id_list"] = channelList3Match.group( "channel_id_list") else: self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_3") parseOk = False # Take the rest of the file starting from the first match channelListString = fileContent[channelList1Match.start():] channelList4Match = re.compile( regExp["CHANNEL_LIST_4"]).search(channelListString) if channelList4Match: if (channelList4Match.group("channel_list") == symbolNames["channel_list"] and channelList4Match.group("channel_number") == symbolNames["channel_number"] and channelList4Match.group("channel_name_long") == symbolNames["channel_name_long"] and channelList4Match.group("channel_name_short") == symbolNames["channel_name_short"] and channelList4Match.group("channel_id_1") == symbolNames["channel_id_1"] and channelList4Match.group("channel_id_1") == symbolNames["channel_id_1"] and channelList4Match.group("channel_logo_1") == symbolNames["channel_logo_1"] and channelList4Match.group("channel_logo_2") == symbolNames["channel_logo_2"] and channelList4Match.group("channel_logo_3") == symbolNames["channel_logo_3"] and channelList4Match.group("prev_channel_number") == symbolNames["prev_channel_number"]): symbolNames["channel_radio"] = channelList4Match.group( "channel_radio") else: self._logger.error( "_parseCodeJs: mismatch in CHANNEL_LIST_4 with earlier matches!" ) parseOk = False else: _logger.error("_parseCodeJs: no match for CHANNEL_LIST_4") parseOk = False channelList5Match = re.compile( regExp["CHANNEL_LIST_5"]).search(channelListString) if channelList5Match: symbolNames[ "last_channel_number"] = channelList5Match.group( "last_channel_number") symbolNames[ "next_channel_number"] = channelList5Match.group( "next_channel_number") else: self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_5") parseOk = False else: self._logger.error("_parseCodeJs: no match for CHANNEL_LIST_1") parseOk = False playStreamActionsMatch = re.compile( regExp["PLAY_STREAM_ACTIONS"]).search(fileContent) if playStreamActionsMatch: symbolNames["play_action_1"] = playStreamActionsMatch.group( "play_action_1") symbolNames["play_action_2"] = playStreamActionsMatch.group( "play_action_2") else: self._logger.error( "_parseCodeJs: no match for PLAY_STREAM_ACTIONS") parseOk = False playStreamClassMatch = re.compile( regExp["PLAY_STREAM_CLASS"]).search(fileContent) if playStreamClassMatch: symbolNames["play_stream_class"] = playStreamClassMatch.group( "play_stream_class") symbolNames["play_stream_url"] = playStreamClassMatch.group( "play_stream_url") else: self._logger.error( "_parseCodeJs: no match for PLAY_STREAM_CLASS") parseOk = False setChannelFunctionMatch = re.compile( regExp["SET_CHANNEL_FUNCTION"]).search(fileContent) if setChannelFunctionMatch: symbolNames[ "set_channel_function"] = setChannelFunctionMatch.group( "set_channel_function") setChannelInstanceMatch = re.compile( regExp["SET_CHANNEL_INSTANCE"] % (symbolNames["set_channel_function"])).search(fileContent) if setChannelInstanceMatch: symbolNames[ "set_channel_instance"] = setChannelInstanceMatch.group( "set_channel_instance") else: self._logger.error( "_parseCodeJs: no match for SET_CHANNEL_FUNCTION") parseOk = False else: self._logger.error( "_parseCodeJs: no match for SET_CHANNEL_FUNCTION") parseOk = False channelObjectMatch = re.compile( regExp["CHANNEL_OBJECT"]).search(fileContent) if channelObjectMatch: symbolNames["channel_object_1"] = channelObjectMatch.group( "channel_object_1") symbolNames["channel_object_2"] = channelObjectMatch.group( "channel_object_2") else: self._logger.error("_parseCodeJs: no match for CHANNEL_OBJECT") parseOk = False debugFunctionMatch = re.compile( regExp["DEBUG_FUNCTION"]).search(fileContent) if debugFunctionMatch: symbolNames["debug_function"] = debugFunctionMatch.group( "debug_function") initFunctionMatch = re.compile( regExp["INIT_FUNCTION"] % (symbolNames["debug_function"])).search(fileContent) if not initFunctionMatch: _logger.error("_parseCodeJs: no match for INIT_FUNCTION") parseOk = False lastJSLineMatch = re.compile( regExp["LAST_JS_LINE"] % (symbolNames["debug_function"])).search(fileContent) if not lastJSLineMatch: self._logger.error( "_parseCodeJs: no match for LAST_JS_LINE") parseOk = False else: self._logger.error("_parseCodeJs: no match for DEBUG_FUNCTION") parseOk = False keyEventFunctionMatch = re.compile( regExp["KEY_EVENT_FUNCTION"]).search(fileContent) if keyEventFunctionMatch: symbolNames[ "key_event_function"] = keyEventFunctionMatch.group( "key_event_function") else: _logger.error("_parseCodeJs: no match for KEY_EVENT_FUNCTION") parseOk = False if parseOk: return (fileContent, symbolNames) else: return (None, None) else: self._logger.error("_parseCodeJs: no file content!") return (None, None)
def _parseCodeJs( self, codeJsPath ): content, _, _ = getPage( codeJsPath ) if content: fileHandle = gzip.GzipFile( fileobj=StringIO( content ) ) fileContent = fileHandle.read() parseOk = True regExp = { # H [ b ] = { k :b , c :{ " default" :" NPO 1" } , r :{ " default" :" NPO 1" } , l :" ned1" , n :" ned1" , u :" npotv1.png" , v :" npotv1.png" , p :" npotv1.png" , o :a , e :[ ] , f :[ ] , g :[ ] } ; "CHANNEL_LIST_1": r"(?P<channel_list>[A-Za-z]{1})\[[a-z]{1}\]=\s*\{(?P<channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_name_long>[A-Za-z]{1}):\{\"default\":\"NPO 1\"\},\s*(?P<channel_name_short>[A-Za-z]{1}):\{\"default\":\"NPO 1\"\},\s*(?P<channel_id_1>[A-Za-z]{1}):\"ned1\",\s*(?P<channel_id_2>[A-Za-z]{1}):\"ned1\",\s*(?P<channel_logo_1>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<channel_logo_2>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<channel_logo_3>[A-Za-z]{1}):\"npotv1.png\",\s*(?P<prev_channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_streams>[A-Za-z]{1}):\[\],\s*(?P<channel_unknown_1>[A-Za-z]{1}):\[\],\s*(?P<channel_unknown_2>[A-Za-z]{1}):\[\]\};", # a = { } ; a .c ={ " default" :"HD+ " }; a .h = " igmp:/ / 224 .0 .251 .124 :8248;rtpskip=yes " ; a .C = 1; a .M = 1; # a = { } ; a .c ={ " default" :"HD ( NOB) " }; a .h = " igmp:/ / 224 .1 .3 .1 :12110 " ; a .Ba = [{Oa: "103"}]; a .C = 1; a .M = 1; "CHANNEL_LIST_2": r"[A-Za-z]{1}=\s*\{\};\s*[A-Za-z]{1}.(?P<channel_metadata>[A-Za-z]{1})=\{\"default\":\"[A-Za-z \(\)\+]{2,10}\"\};\s*[A-Za-z]{1}.(?P<channel_url>[A-Za-z]{1,2})=\s*\"igmp:\/\/[0-9]{3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}:[0-9]{1,5}(;rtpskip=yes)?\";\s*([A-Za-z]{1}.[A-Za-z]{1,2}=\s*\[([^\]]*)\];\s*)?[A-Za-z]{1}.[A-Za-z]{1}=\s*1;\s*[A-Za-z]{1}.(?P<channel_url_hd>[A-Za-z]{1})=\s*1;", # for( $ .tv.q .push( a ) ;f .length;) E [ f .pop( ) ] = a ; "CHANNEL_LIST_3": r"for\([\$A-Za-z]{1,2}.tv.[A-Za-z]{1,2}.push\([A-Za-z]{1,2}\);[A-Za-z]{1,2}.length;\)(?P<channel_id_list>[A-Za-z]{1,2})\[[A-Za-z]{1,2}.pop\(\)\]=\s*[A-Za-z]{1,2};", # z [ b ] = { k :b , c :{ " default" :" NPO 3FM" } , q :{ " default" :" NPO 3FM" } , j :" radio3" , m :" radio3" , s :" nporadio3.png" , t :" nporadio3.png" , o :" nporadio3.png" , u :1, n :a ,d:[],e:[]}; "CHANNEL_LIST_4": r"(?P<channel_list>[A-Za-z]{1})\[[A-Za-z]{1}\]=\s*\{(?P<channel_number>[A-Za-z]{1}):[A-Za-z]{1},\s*(?P<channel_name_long>[A-Za-z]{1}):\{\"default\":\"NPO 3FM\"\},\s*(?P<channel_name_short>[A-Za-z]{1}):\{\"default\":\"NPO 3FM\"\},\s*(?P<channel_id_1>[A-Za-z]{1}):\"radio3\",\s*(?P<channel_id_2>[A-Za-z]{1}):\"radio3\",\s*(?P<channel_logo_1>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_logo_2>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_logo_3>[A-Za-z]{1}):\"nporadio3.png\",\s*(?P<channel_radio>[A-Za-z]{1}):1,\s*(?P<prev_channel_number>[A-Za-z]{1}):[a-z]{1}([^\}]+)\};", # Gb = f ;B [ a ] .i = f ; for( B [ f ] .k = a ;e .length;) w [ e .pop( ) ] = f ;J .root "CHANNEL_LIST_5": r"(?P<last_channel_number>[A-Za-z]{1,2})=\s*[A-Za-z]{1};[A-Za-z]{1}\[[A-Za-z]{1}\].(?P<next_channel_number>[A-Za-z]{1})=\s*[A-Za-z]{1};\s*for\([A-Za-z]{1}\[[A-Za-z]{1}\].[A-Za-z]{1}=\s*[A-Za-z]{1};[A-Za-z]{1}.length;\)[A-Za-z]{1}\[[A-Za-z]{1}.pop\(\)\]=\s*[A-Za-z]{1};[A-Za-z]{1}.root", # function( ) { window.clearInterval (g ) ;u ( $a ) ;var e = new ab ( t .dj ) ;e .B =c ( ) ;v ( e ) ;window.setTimeout( function( ) { ga ( ) } ,50) } "PLAY_STREAM_ACTIONS": r"function\(\)\{window.clearInterval\([A-Za-z]{1,2}\);(?P<play_action_1>[A-Za-z]{1,2})\([$A-Za-z]{1,2}\);var [A-Za-z]{1,2}=\s*new [A-Za-z]{1,2}\([A-Za-z]{1,2}.[A-Za-z]{1,2}\);[A-Za-z]{1,2}.[A-Za-z]{1,2}=[A-Za-z]{1,2}\(\);(?P<play_action_2>[A-Za-z]{1,2})\([A-Za-z]{1,2}\);window.setTimeout\(function\(\)\{[A-Za-z]{1,2}\(\)\},50\)\}", # function Ue ( a ,b ,f ,g ,e ) {L .call( this) ;if( b ===undefined) b = " " ; "PLAY_STREAM_CLASS": r"function (?P<play_stream_class>[A-Za-z]{1,2})\((?P<play_stream_url>[A-Za-z]{1,2}),[A-Za-z]{1,2},[A-Za-z]{1,2},[A-Za-z]{1,2},[A-Za-z]{1,2}\){[A-Za-z]{1,2}.call\(this\);if\([A-Za-z]{1,2}===undefined\)[A-Za-z]{1,2}=\s*\"\";", "SET_CHANNEL_FUNCTION": r"[A-Za-z]{1,2}.prototype.(?P<set_channel_function>[A-Za-z]{1,2})=\s*function\([A-Za-z]{1,2},[A-Za-z]{1,2}\){if\(this.[A-Za-z]{1,2}\){window.clearTimeout\(this.[A-Za-z]{1,2}\);", # this.l .appendChild( b ) ;b = new O ( o .fh ) ;b .ua = function( ) {G .ka( "SET_CHANNEL_INSTANCE": r"this.[A-Za-z]{1,2}.appendChild\([A-Za-z]{1,2}\);[A-Za-z]{1,2}=\s*new [A-Za-z]{1,2}\([A-Za-z]{1,2}.[A-Za-z]{1,2}\);[A-Za-z]{1,2}.[A-Za-z]{1,2}=\s*function\(\){(?P<set_channel_instance>[A-Za-z]{1,2}).%s\(", "CHANNEL_OBJECT": r"(document.cookie=\"channel=\"\+encodeURIComponent\(this.(?P<channel_object_1>[A-Za-z]{1,2}).(?P<channel_object_2>[A-Za-z]{1,2})\);)", "DEBUG_FUNCTION": r"function (?P<debug_function>[A-Za-z]{1,2})\([A-Za-z]{1,2}\){if\(window.console\)window.console.log\(", # for( o ( " doRunAfterLoad( ) : " + Kb .length) ;Kb .length;) { var a =Kb .pop( ) ;try{ a ( ) } "INIT_FUNCTION": r"for\(%s\(\"doRunAfterLoad\(\): \"\+[A-Za-z]{1,2}.length\);[A-Za-z]{1,2}.length;\)\{var [A-Za-z]{1,2}=[A-Za-z]{1,2}.pop\(\);try\{[A-Za-z]{1,2}\(\)\}", "LAST_JS_LINE": r"([A-Za-z]{1,2}\(\);\s*%s\(\"initial buildMenu\(\) done.\"\);)", # function oc ( b ) { ia ( ) ;$a &&clearTimeout( $a ) ;b = b .keyCode| | b .charCode; "KEY_EVENT_FUNCTION": r"function (?P<key_event_function>[A-Za-z]{1,2})\([A-Za-z]{1,2}\)\{[A-Za-z]{1,2}\(\);[$A-Za-z]{1,2}&&clearTimeout\([$A-Za-z]{1,2}\);[A-Za-z]{1,2}=\s*[A-Za-z]{1,2}.keyCode\|\|[A-Za-z]{1,2}.charCode;", } symbolNames = {} symbolNames["channel_logo_path"] = self._glashartConfig.channelLogoPath symbolNames["channel_thumb_path"] = self._glashartConfig.channelThumbPath channelList1Match = re.compile( regExp["CHANNEL_LIST_1"] ).search( fileContent ) if channelList1Match: symbolNames["channel_list"] = channelList1Match.group( "channel_list" ) symbolNames["channel_number"] = channelList1Match.group( "channel_number" ) symbolNames["channel_name_long"] = channelList1Match.group( "channel_name_long" ) symbolNames["channel_name_short"] = channelList1Match.group( "channel_name_short" ) symbolNames["channel_id_1"] = channelList1Match.group( "channel_id_1" ) symbolNames["channel_id_2"] = channelList1Match.group( "channel_id_2" ) symbolNames["channel_logo_1"] = channelList1Match.group( "channel_logo_1" ) symbolNames["channel_logo_2"] = channelList1Match.group( "channel_logo_2" ) symbolNames["channel_logo_3"] = channelList1Match.group( "channel_logo_3" ) symbolNames["prev_channel_number"] = channelList1Match.group( "prev_channel_number" ) symbolNames["channel_streams"] = channelList1Match.group( "channel_streams" ) # For the folloing matches I only need about 3000 characters starting from the start of this match channelListString = fileContent[channelList1Match.start():channelList1Match.start() + 3000] channelList2Match = re.compile( regExp["CHANNEL_LIST_2"] ).search( channelListString ) if channelList2Match: symbolNames["channel_metadata"] = channelList2Match.group( "channel_metadata" ) symbolNames["channel_url"] = channelList2Match.group( "channel_url" ) symbolNames["channel_url_hd"] = channelList2Match.group( "channel_url_hd" ) else: self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_2" ) parseOk = False channelList3Match = re.compile( regExp["CHANNEL_LIST_3"] ).search( channelListString ) if channelList3Match: symbolNames["channel_id_list"] = channelList3Match.group( "channel_id_list" ) else: self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_3" ) parseOk = False # Take the rest of the file starting from the first match channelListString = fileContent[channelList1Match.start():] channelList4Match = re.compile( regExp["CHANNEL_LIST_4"] ).search( channelListString ) if channelList4Match: if ( channelList4Match.group( "channel_list" ) == symbolNames["channel_list"] and channelList4Match.group( "channel_number" ) == symbolNames["channel_number"] and channelList4Match.group( "channel_name_long" ) == symbolNames["channel_name_long"] and channelList4Match.group( "channel_name_short" ) == symbolNames["channel_name_short"] and channelList4Match.group( "channel_id_1" ) == symbolNames["channel_id_1"] and channelList4Match.group( "channel_id_1" ) == symbolNames["channel_id_1"] and channelList4Match.group( "channel_logo_1" ) == symbolNames["channel_logo_1"] and channelList4Match.group( "channel_logo_2" ) == symbolNames["channel_logo_2"] and channelList4Match.group( "channel_logo_3" ) == symbolNames["channel_logo_3"] and channelList4Match.group( "prev_channel_number" ) == symbolNames["prev_channel_number"] ): symbolNames["channel_radio"] = channelList4Match.group( "channel_radio" ) else: self._logger.error( "_parseCodeJs: mismatch in CHANNEL_LIST_4 with earlier matches!" ) parseOk = False else: _logger.error( "_parseCodeJs: no match for CHANNEL_LIST_4" ) parseOk = False channelList5Match = re.compile( regExp["CHANNEL_LIST_5"] ).search( channelListString ) if channelList5Match: symbolNames["last_channel_number"] = channelList5Match.group( "last_channel_number" ) symbolNames["next_channel_number"] = channelList5Match.group( "next_channel_number" ) else: self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_5" ) parseOk = False else: self._logger.error( "_parseCodeJs: no match for CHANNEL_LIST_1" ) parseOk = False playStreamActionsMatch = re.compile( regExp["PLAY_STREAM_ACTIONS"] ).search( fileContent ) if playStreamActionsMatch: symbolNames["play_action_1"] = playStreamActionsMatch.group( "play_action_1" ); symbolNames["play_action_2"] = playStreamActionsMatch.group( "play_action_2" ); else: self._logger.error( "_parseCodeJs: no match for PLAY_STREAM_ACTIONS" ) parseOk = False playStreamClassMatch = re.compile( regExp["PLAY_STREAM_CLASS"] ).search( fileContent ) if playStreamClassMatch: symbolNames["play_stream_class"] = playStreamClassMatch.group( "play_stream_class" ); symbolNames["play_stream_url"] = playStreamClassMatch.group( "play_stream_url" ); else: self._logger.error( "_parseCodeJs: no match for PLAY_STREAM_CLASS" ) parseOk = False setChannelFunctionMatch = re.compile( regExp["SET_CHANNEL_FUNCTION"] ).search( fileContent ) if setChannelFunctionMatch: symbolNames["set_channel_function"] = setChannelFunctionMatch.group( "set_channel_function" ); setChannelInstanceMatch = re.compile( regExp["SET_CHANNEL_INSTANCE"] % ( symbolNames["set_channel_function"] ) ).search( fileContent ) if setChannelInstanceMatch: symbolNames["set_channel_instance"] = setChannelInstanceMatch.group( "set_channel_instance" ); else: self._logger.error( "_parseCodeJs: no match for SET_CHANNEL_FUNCTION" ) parseOk = False else: self._logger.error( "_parseCodeJs: no match for SET_CHANNEL_FUNCTION" ) parseOk = False channelObjectMatch = re.compile( regExp["CHANNEL_OBJECT"] ).search( fileContent ) if channelObjectMatch: symbolNames["channel_object_1"] = channelObjectMatch.group( "channel_object_1" ); symbolNames["channel_object_2"] = channelObjectMatch.group( "channel_object_2" ); else: self._logger.error( "_parseCodeJs: no match for CHANNEL_OBJECT" ) parseOk = False debugFunctionMatch = re.compile( regExp["DEBUG_FUNCTION"] ).search( fileContent ) if debugFunctionMatch: symbolNames["debug_function"] = debugFunctionMatch.group( "debug_function" ); initFunctionMatch = re.compile( regExp["INIT_FUNCTION"] % ( symbolNames["debug_function"] ) ).search( fileContent ) if not initFunctionMatch: _logger.error( "_parseCodeJs: no match for INIT_FUNCTION" ) parseOk = False lastJSLineMatch = re.compile( regExp["LAST_JS_LINE"] % ( symbolNames["debug_function"] ) ).search( fileContent ) if not lastJSLineMatch: self._logger.error( "_parseCodeJs: no match for LAST_JS_LINE" ) parseOk = False else: self._logger.error( "_parseCodeJs: no match for DEBUG_FUNCTION" ) parseOk = False keyEventFunctionMatch = re.compile( regExp["KEY_EVENT_FUNCTION"] ).search( fileContent ) if keyEventFunctionMatch: symbolNames["key_event_function"] = keyEventFunctionMatch.group( "key_event_function" ); else: _logger.error( "_parseCodeJs: no match for KEY_EVENT_FUNCTION" ) parseOk = False if parseOk: return ( fileContent, symbolNames ) else: return ( None, None ) else: self._logger.error( "_parseCodeJs: no file content!" ) return ( None, None )
def _grabEpgForChannel( self, channel=None, epgId=None ): conn = DBConnection() if channel: epgId = EpgId.getFromDb( conn, channel.epgId ) self._logger.info( "Grabbing EPG for channel: %s (%s; method: %s)" % ( channel.name, channel.epgId, epgId.strategy ) ) if not epgId: return else: self._logger.info( "Grabbing EPG for epgId: %s (method: %s)" % ( epgId.epgId, epgId.strategy ) ) # Check if _fail_# is behind strategy # This is to indicate epg grabbing for this epgId failed previously strategy = epgId.strategy strategyRe = re.compile( r'_fail_(?P<fail>\d+)' ) failMatch = strategyRe.search( strategy ) failCount = 0 if failMatch: failCount = int( failMatch.group( "fail" ) ) strategy = epgId.strategy.split( '_' )[0] # We're going to attempt to grab EPG information for this channel 5 times # before we stop grabbing this epgId in the future. if failCount < 5: now = time.localtime() nowDay = datetime.datetime( now[0], now[1], now[2] ) daysDetailDelta = datetime.timedelta( days = 3 ) epgFilename = "/%s.json.gz" % ( epgId.epgId ) epgUrl = self._glashartConfig.epgChannelsPath + epgFilename currentPrograms = EpgProgram.getAllByEpgIdFromDb( conn, epgId.epgId ) currentProgramsDict = { currProgram.originalId: currProgram for currProgram in currentPrograms } newProgramsDict = {} content, _, _ = getPage( epgUrl ) if content: fileHandle = gzip.GzipFile( fileobj=StringIO( content ) ) epgData = json.loads( fileHandle.read() ) # If strategy has changed (working after (a few) failed attempts) if epgId.strategy != strategy: epgId.strategy = strategy epgId.addToDb( conn ) numPrograms = 0 numProgramsDetail = 0 numProgramsDetailFailed = 0 numProgramsNew = 0 numProgramsUpdated = 0 for program in epgData: if not self._running: break numPrograms += 1 programNew = self._getProgramFromJson( epgId.epgId, program ) if programNew.originalId in newProgramsDict: self._logger.warning( "Program with originalId %d already in newProgramsDict" % ( programNew.originalId ) ) newProgramsDict[programNew.originalId] = programNew updateDetailedData = True programOld = None if programNew.originalId in currentProgramsDict: programOld = currentProgramsDict[programNew.originalId] # If the old program has detailed info, copy those fields # TODO: do this somewhere else if programOld and programOld.detailed: programNew.subtitle = programOld.subtitle programNew.description = programOld.description programNew.aspectRatio = programOld.aspectRatio programNew.parentalRating = programOld.parentalRating programNew.genres = programOld.genres programNew.actors = programOld.actors programNew.directors = programOld.directors programNew.presenters = programOld.presenters programNew.ratings = programOld.ratings programNew.detailed = programOld.detailed # Now, compare the old program and the new program # Are they the same, then we don't need to download detailed information if programOld and programOld.detailed and programNew == programOld: programNew = programOld updateDetailedData = False if updateDetailedData: if ( ( epgId.strategy == "default" and (nowDay + daysDetailDelta) > datetime.datetime.fromtimestamp( programNew.startTime ) ) or ( epgId.strategy == "full" ) ): time.sleep( random.uniform( 0.5, 1.0 ) ) programNew, grabbed = self._grabDetailedEpgForProgram( programNew ) if grabbed: numProgramsDetail += 1 else: # if more than 10 detailed program information grabs failed, set strategy to none. numProgramsDetailFailed += 1 if numProgramsDetailFailed == 10: self._logger.error( "Couldn't download at least 10 detailed program information files, so setting strategy to 'none', but do not store" ) epgId.strategy = "none" conn.delayCommit( True ) for programId in newProgramsDict: programNew = newProgramsDict[programId] programOld = None if programNew.originalId in currentProgramsDict: programOld = currentProgramsDict[programNew.originalId] if not programOld or programNew != programOld: if programOld: self._logger.debug( "Updated program: id = %s" % ( programNew.originalId ) ) self._logger.debug( "Start time: %s > %s" % ( str( programOld.startTime ), str( programNew.startTime ) ) ) self._logger.debug( "End time: %s > %s" % ( str( programOld.endTime ), str( programNew.endTime ) ) ) self._logger.debug( "Name: %s > %s" % ( repr( programOld.title ), repr( programNew.title ) ) ) programNew.id = programOld.id numProgramsUpdated += 1 else: numProgramsNew += 1 try: programNew.addToDb( conn ) except: self._logger.exception( programNew.dump() ) conn.delayCommit( False ) if self._running: self._logger.debug( "Num programs: %i" % ( numPrograms ) ) self._logger.debug( "Num program details: %i" % ( numProgramsDetail ) ) self._logger.info( "Num new programs: %i" % ( numProgramsNew ) ) self._logger.info( "Num updated programs: %i" % ( numProgramsUpdated ) ) if numProgramsNew == 0: self._logger.warning( "No new programs were added for epgId: %s" % ( epgId.epgId ) ) else: self._logger.warning( "Unable to download EPG information for epgId: %s" % ( epgId.epgId ) ) failCount += 1 epgId.strategy = "%s_fail_%d" % ( strategy, failCount ) epgId.addToDb( conn ) else: self._logger.info( "Downloading of EPG information for epgId: %s skipped becaused it failed too many times" % ( epgId.epgId ) )
def _grabEpgForChannel(self, channel=None, epgId=None): conn = DBConnection() if channel: epgId = EpgId.getFromDb(conn, channel.epgId) self._logger.info("Grabbing EPG for channel: %s (%s; method: %s)" % (channel.name, channel.epgId, epgId.strategy)) if not epgId: return else: self._logger.info("Grabbing EPG for epgId: %s (method: %s)" % (epgId.epgId, epgId.strategy)) # Check if _fail_# is behind strategy # This is to indicate epg grabbing for this epgId failed previously strategy = epgId.strategy strategyRe = re.compile(r'_fail_(?P<fail>\d+)') failMatch = strategyRe.search(strategy) failCount = 0 if failMatch: failCount = int(failMatch.group("fail")) strategy = epgId.strategy.split('_')[0] # We're going to attempt to grab EPG information for this channel 5 times # before we stop grabbing this epgId in the future. if failCount < 5: now = time.localtime() nowDay = datetime.datetime(now[0], now[1], now[2]) daysDetailDelta = datetime.timedelta(days=3) epgFilename = "/%s.json.gz" % (epgId.epgId) epgUrl = self._glashartConfig.epgChannelsPath + epgFilename currentPrograms = EpgProgram.getAllByEpgIdFromDb(conn, epgId.epgId) currentProgramsDict = { currProgram.originalId: currProgram for currProgram in currentPrograms } newProgramsDict = {} content, _, _ = getPage(epgUrl) if content: fileHandle = gzip.GzipFile(fileobj=StringIO(content)) epgData = json.loads(fileHandle.read()) # If strategy has changed (working after (a few) failed attempts) if epgId.strategy != strategy: epgId.strategy = strategy epgId.addToDb(conn) numPrograms = 0 numProgramsDetail = 0 numProgramsDetailFailed = 0 numProgramsNew = 0 numProgramsUpdated = 0 for program in epgData: if not self._running: break numPrograms += 1 programNew = self._getProgramFromJson(epgId.epgId, program) if programNew.originalId in newProgramsDict: self._logger.warning( "Program with originalId %d already in newProgramsDict" % (programNew.originalId)) newProgramsDict[programNew.originalId] = programNew updateDetailedData = True programOld = None if programNew.originalId in currentProgramsDict: programOld = currentProgramsDict[programNew.originalId] # If the old program has detailed info, copy those fields # TODO: do this somewhere else if programOld and programOld.detailed: programNew.subtitle = programOld.subtitle programNew.description = programOld.description programNew.aspectRatio = programOld.aspectRatio programNew.parentalRating = programOld.parentalRating programNew.genres = programOld.genres programNew.actors = programOld.actors programNew.directors = programOld.directors programNew.presenters = programOld.presenters programNew.ratings = programOld.ratings programNew.detailed = programOld.detailed # Now, compare the old program and the new program # Are they the same, then we don't need to download detailed information if programOld and programOld.detailed and programNew == programOld: programNew = programOld updateDetailedData = False if updateDetailedData: if ((epgId.strategy == "default" and (nowDay + daysDetailDelta) > datetime.datetime.fromtimestamp( programNew.startTime)) or (epgId.strategy == "full")): time.sleep(random.uniform(0.5, 1.0)) programNew, grabbed = self._grabDetailedEpgForProgram( programNew) if grabbed: numProgramsDetail += 1 else: # if more than 10 detailed program information grabs failed, set strategy to none. numProgramsDetailFailed += 1 if numProgramsDetailFailed == 10: self._logger.error( "Couldn't download at least 10 detailed program information files, so setting strategy to 'none', but do not store" ) epgId.strategy = "none" conn.delayCommit(True) for programId in newProgramsDict: programNew = newProgramsDict[programId] programOld = None if programNew.originalId in currentProgramsDict: programOld = currentProgramsDict[programNew.originalId] if not programOld or programNew != programOld: if programOld: self._logger.debug("Updated program: id = %s" % (programNew.originalId)) self._logger.debug("Start time: %s > %s" % (str(programOld.startTime), str(programNew.startTime))) self._logger.debug("End time: %s > %s" % (str( programOld.endTime), str(programNew.endTime))) self._logger.debug("Name: %s > %s" % (repr( programOld.title), repr(programNew.title))) programNew.id = programOld.id numProgramsUpdated += 1 else: numProgramsNew += 1 try: programNew.addToDb(conn) except: self._logger.exception(programNew.dump()) conn.delayCommit(False) if self._running: self._logger.debug("Num programs: %i" % (numPrograms)) self._logger.debug("Num program details: %i" % (numProgramsDetail)) self._logger.info("Num new programs: %i" % (numProgramsNew)) self._logger.info("Num updated programs: %i" % (numProgramsUpdated)) if numProgramsNew == 0: self._logger.warning( "No new programs were added for epgId: %s" % (epgId.epgId)) else: self._logger.warning( "Unable to download EPG information for epgId: %s" % (epgId.epgId)) failCount += 1 epgId.strategy = "%s_fail_%d" % (strategy, failCount) epgId.addToDb(conn) else: self._logger.info( "Downloading of EPG information for epgId: %s skipped becaused it failed too many times" % (epgId.epgId))