def __init__(self):
super(MediathekViewPlugin, self).__init__()
self.settings = Settings()
self.notifier = Notifier()
self.database = Store(self.get_new_logger('Store'), self.notifier,
self.settings)
self.unicodePath = unicode(self.path, 'utf-8')
class MediathekViewPlugin(KodiPlugin):
""" The main plugin class """
def __init__(self):
super(MediathekViewPlugin, self).__init__()
self.settings = Settings()
self.notifier = Notifier()
self.database = Store(self.get_new_logger('Store'), self.notifier,
self.settings)
def show_main_menu(self):
""" Creates the main menu of the plugin """
# Search
self.add_folder_item(30901, {
'mode': "search",
'extendedsearch': False
},
icon=os.path.join(self.path, 'resources', 'icons',
'search-m.png'))
# Search all
self.add_folder_item(30902, {
'mode': "search",
'extendedsearch': True
},
icon=os.path.join(self.path, 'resources', 'icons',
'search-m.png'))
# Browse livestreams
self.add_folder_item(30903, {'mode': "livestreams"},
icon=os.path.join(self.path, 'resources', 'icons',
'live2-m.png'))
# Browse recently added
self.add_folder_item(30904, {
'mode': "recent",
'channel': 0
},
icon=os.path.join(self.path, 'resources', 'icons',
'new-m.png'))
# Browse recently added by channel
self.add_folder_item(30905, {'mode': "recentchannels"},
icon=os.path.join(self.path, 'resources', 'icons',
'new-m.png'))
# Browse by Initial->Show
self.add_folder_item(30906, {
'mode': "initial",
'channel': 0
},
icon=os.path.join(self.path, 'resources', 'icons',
'movie-m.png'))
# Browse by Channel->Initial->Shows
self.add_folder_item(30907, {'mode': "channels"},
icon=os.path.join(self.path, 'resources', 'icons',
'movie-m.png'))
# Database Information
self.add_action_item(30908, {'mode': "action-dbinfo"},
icon=os.path.join(self.path, 'resources', 'icons',
'dbinfo-m.png'))
# Manual database update
if self.settings.updmode == 1 or self.settings.updmode == 2:
self.add_action_item(30909, {'mode': "action-dbupdate"})
self.end_of_directory()
self._check_outdate()
def show_searches(self, extendedsearch=False):
"""
Fill the search screen with "New Search..." and the
list of recent searches
Args:
extendedsearch(bool, optionsl): If `True`, the searches
are performed both in show title and description.
Default is `False`
"""
self.add_folder_item(30931, {
'mode': "newsearch",
'extendedsearch': extendedsearch
},
icon=os.path.join(self.path, 'resources', 'icons',
'search-m.png'))
RecentSearches(self, extendedsearch).load().populate()
self.end_of_directory()
def new_search(self, extendedsearch=False):
"""
Asks the user to enter his search terms and then
performs the search and displays the results.
Args:
extendedsearch(bool, optionsl): If `True`, the searches
are performed both in show title and description.
Default is `False`
"""
settingid = 'lastsearch2' if extendedsearch is True else 'lastsearch1'
headingid = 30902 if extendedsearch is True else 30901
# are we returning from playback ?
search = self.get_setting(settingid)
if search:
# restore previous search
self.database.search(search, FilmUI(self), extendedsearch)
else:
# enter search term
(search, confirmed) = self.notifier.get_entered_text('', headingid)
if len(search) > 2 and confirmed is True:
RecentSearches(self, extendedsearch).load().add(search).save()
if self.database.search(search, FilmUI(self),
extendedsearch) > 0:
self.set_setting(settingid, search)
else:
# pylint: disable=line-too-long
self.info(
'The following ERROR can be ignored. It is caused by the architecture of the Kodi Plugin Engine'
)
self.end_of_directory(False, cache_to_disc=True)
def show_db_info(self):
""" Displays current information about the database """
info = self.database.get_status()
heading = self.language(30908)
infostr = self.language({
'NONE': 30941,
'UNINIT': 30942,
'IDLE': 30943,
'UPDATING': 30944,
'ABORTED': 30945
}.get(info['status'], 30941))
infostr = self.language(30965) % infostr
totinfo = self.language(30971) % (info['tot_chn'], info['tot_shw'],
info['tot_mov'])
updatetype = self.language(30972 if info['fullupdate'] > 0 else 30973)
if info['status'] == 'UPDATING' and info['filmupdate'] > 0:
updinfo = self.language(30967) % (
updatetype, datetime.datetime.fromtimestamp(
info['filmupdate']).strftime('%Y-%m-%d %H:%M:%S'),
info['add_chn'], info['add_shw'], info['add_mov'])
elif info['status'] == 'UPDATING':
updinfo = self.language(30968) % (updatetype, info['add_chn'],
info['add_shw'], info['add_mov'])
elif info['lastupdate'] > 0 and info['filmupdate'] > 0:
updinfo = self.language(30969) % (
updatetype, datetime.datetime.fromtimestamp(
info['lastupdate']).strftime('%Y-%m-%d %H:%M:%S'),
datetime.datetime.fromtimestamp(
info['filmupdate']).strftime('%Y-%m-%d %H:%M:%S'),
info['add_chn'], info['add_shw'], info['add_mov'],
info['del_chn'], info['del_shw'], info['del_mov'])
elif info['lastupdate'] > 0:
updinfo = self.language(30970) % (
updatetype, datetime.datetime.fromtimestamp(
info['lastupdate']).strftime('%Y-%m-%d %H:%M:%S'),
info['add_chn'], info['add_shw'], info['add_mov'],
info['del_chn'], info['del_shw'], info['del_mov'])
else:
updinfo = self.language(30966)
xbmcgui.Dialog().textviewer(
heading, infostr + '\n\n' + totinfo + '\n\n' + updinfo)
def _check_outdate(self, maxage=172800):
if self.settings.updmode != 1 and self.settings.updmode != 2:
# no check with update disabled or update automatic
return
if self.database is None:
# should never happen
self.notifier.show_outdated_unknown()
return
status = self.database.get_status()
if status['status'] == 'NONE' or status['status'] == 'UNINIT':
# should never happen
self.notifier.show_outdated_unknown()
return
elif status['status'] == 'UPDATING':
# great... we are updating. nuthin to show
return
# lets check how old we are
tsnow = int(time.time())
tsold = int(status['lastupdate'])
if tsnow - tsold > maxage:
self.notifier.show_outdated_known(status)
def init(self):
""" Initialisation of the plugin """
if self.database.init():
if self.settings.handle_first_run():
pass
self.settings.handle_update_on_start()
def run(self):
""" Execution of the plugin """
# save last activity timestamp
self.settings.reset_user_activity()
# process operation
self.info("Plugin invoked with parameters {}", self.args)
mode = self.get_arg('mode', None)
if mode is None:
self.show_main_menu()
elif mode == 'search':
extendedsearch = self.get_arg('extendedsearch', 'False') == 'True'
self.show_searches(extendedsearch)
elif mode == 'newsearch':
self.new_search(self.get_arg('extendedsearch', 'False') == 'True')
elif mode == 'research':
search = self.get_arg('search', '')
extendedsearch = self.get_arg('extendedsearch', 'False') == 'True'
self.database.search(search, FilmUI(self), extendedsearch)
RecentSearches(self, extendedsearch).load().add(search).save()
elif mode == 'delsearch':
search = self.get_arg('search', '')
extendedsearch = self.get_arg('extendedsearch', 'False') == 'True'
RecentSearches(
self, extendedsearch).load().delete(search).save().populate()
self.run_builtin('Container.Refresh')
elif mode == 'livestreams':
self.database.get_live_streams(
FilmUI(self, [xbmcplugin.SORT_METHOD_LABEL]))
elif mode == 'recent':
channel = self.get_arg('channel', 0)
self.database.get_recents(channel, FilmUI(self))
elif mode == 'recentchannels':
self.database.get_recent_channels(ChannelUI(self,
nextdir='recent'))
elif mode == 'channels':
self.database.get_channels(ChannelUI(self, nextdir='shows'))
elif mode == 'action-dbinfo':
self.show_db_info()
elif mode == 'action-dbupdate':
self.settings.trigger_update()
self.notifier.show_notification(30963, 30964, time=10000)
elif mode == 'initial':
channel = self.get_arg('channel', 0)
self.database.get_initials(channel, InitialUI(self))
elif mode == 'shows':
channel = self.get_arg('channel', 0)
initial = self.get_arg('initial', None)
self.database.get_shows(channel, initial, ShowUI(self))
elif mode == 'films':
show = self.get_arg('show', 0)
self.database.get_films(show, FilmUI(self))
elif mode == 'downloadmv':
filmid = self.get_arg('id', 0)
quality = self.get_arg('quality', 1)
Downloader(self).download_movie(filmid, quality)
elif mode == 'downloadep':
filmid = self.get_arg('id', 0)
quality = self.get_arg('quality', 1)
Downloader(self).download_episode(filmid, quality)
elif mode == 'playwithsrt':
filmid = self.get_arg('id', 0)
Downloader(self).play_movie_with_subs(filmid)
# cleanup saved searches
if mode is None or mode != 'newsearch':
self.set_setting('lastsearch1', '')
self.set_setting('lastsearch2', '')
def exit(self):
""" Shutdown of the application """
self.database.exit()
def Init( self ): if self.db is not None: self.Exit() self.db = Store( self.logger, self.notifier, self.settings ) self.db.Init()
class MediathekViewUpdater( object ):
def __init__( self, logger, notifier, settings, monitor = None ):
self.logger = logger
self.notifier = notifier
self.settings = settings
self.monitor = monitor
self.db = None
self.use_xz = mvutils.find_xz() is not None
def Init( self ):
if self.db is not None:
self.Exit()
self.db = Store( self.logger, self.notifier, self.settings )
self.db.Init()
def Exit( self ):
if self.db is not None:
self.db.Exit()
del self.db
self.db = None
def IsEnabled( self ):
return self.settings.updenabled
def GetCurrentUpdateOperation( self ):
if not self.IsEnabled() or self.db is None:
# update disabled or not possible
self.logger.info( 'update disabled or not possible' )
return 0
status = self.db.GetStatus()
tsnow = int( time.time() )
tsold = status['lastupdate']
dtnow = datetime.datetime.fromtimestamp( tsnow ).date()
dtold = datetime.datetime.fromtimestamp( tsold ).date()
if status['status'] == 'UNINIT':
# database not initialized
self.logger.debug( 'database not initialized' )
return 0
elif status['status'] == "UPDATING" and tsnow - tsold > 10800:
# process was probably killed during update
self.logger.info( 'Stuck update pretending to run since epoch {} reset', tsold )
self.db.UpdateStatus( 'ABORTED' )
return 0
elif status['status'] == "UPDATING":
# already updating
self.logger.debug( 'already updating' )
return 0
elif tsnow - tsold < self.settings.updinterval:
# last update less than the configured update interval. do nothing
self.logger.debug( 'last update less than the configured update interval. do nothing' )
return 0
elif dtnow != dtold:
# last update was not today. do full update once a day
self.logger.debug( 'last update was not today. do full update once a day' )
return 1
elif status['status'] == "ABORTED" and status['fullupdate'] == 1:
# last full update was aborted - full update needed
self.logger.debug( 'last full update was aborted - full update needed' )
return 1
else:
# do differential update
self.logger.debug( 'do differential update' )
return 2
def Update( self, full ):
if self.db is None:
return
if self.db.SupportsUpdate():
if self.GetNewestList( full ):
self.Import( full )
def Import( self, full ):
( _, _, destfile, avgrecsize ) = self._get_update_info( full )
if not mvutils.file_exists( destfile ):
self.logger.error( 'File {} does not exists', destfile )
return False
# estimate number of records in update file
records = int( mvutils.file_size( destfile ) / avgrecsize )
if not self.db.ftInit():
self.logger.warn( 'Failed to initialize update. Maybe a concurrency problem?' )
return False
try:
self.logger.info( 'Starting import of approx. {} records from {}', records, destfile )
with open( destfile, 'r' ) as file:
parser = ijson.parse( file )
flsm = 0
flts = 0
( self.tot_chn, self.tot_shw, self.tot_mov ) = self._update_start( full )
self.notifier.ShowUpdateProgress()
for prefix, event, value in parser:
if ( prefix, event ) == ( "X", "start_array" ):
self._init_record()
elif ( prefix, event ) == ( "X", "end_array" ):
self._end_record( records )
if self.count % 100 == 0 and self.monitor.abortRequested():
# kodi is shutting down. Close all
self._update_end( full, 'ABORTED' )
self.notifier.CloseUpdateProgress()
return True
elif ( prefix, event ) == ( "X.item", "string" ):
if value is not None:
# self._add_value( value.strip().encode('utf-8') )
self._add_value( value.strip() )
else:
self._add_value( "" )
elif ( prefix, event ) == ( "Filmliste", "start_array" ):
flsm += 1
elif ( prefix, event ) == ( "Filmliste.item", "string" ):
flsm += 1
if flsm == 2 and value is not None:
# this is the timestmap of this database update
try:
fldt = datetime.datetime.strptime( value.strip(), "%d.%m.%Y, %H:%M" )
flts = int( time.mktime( fldt.timetuple() ) )
self.db.UpdateStatus( filmupdate = flts )
self.logger.info( 'Filmliste dated {}', value.strip() )
except TypeError:
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int( time.mktime( time.strptime( value.strip(), "%d.%m.%Y, %H:%M" ) ) )
self.db.UpdateStatus( filmupdate = flts )
self.logger.info( 'Filmliste dated {}', value.strip() )
except Exception as err:
# If the universe hates us...
self.logger.debug( 'Could not determine date "{}" of filmliste: {}', value.strip(), err )
except ValueError as err:
pass
self._update_end( full, 'IDLE' )
self.logger.info( 'Import of {} finished', destfile )
self.notifier.CloseUpdateProgress()
return True
except KeyboardInterrupt:
self._update_end( full, 'ABORTED' )
self.logger.info( 'Interrupted by user' )
self.notifier.CloseUpdateProgress()
return True
except DatabaseCorrupted as err:
self.logger.error( '{}', err )
self.notifier.CloseUpdateProgress()
except DatabaseLost as err:
self.logger.error( '{}', err )
self.notifier.CloseUpdateProgress()
except Exception as err:
self.logger.error( 'Error {} wile processing {}', err, destfile )
self._update_end( full, 'ABORTED' )
self.notifier.CloseUpdateProgress()
return False
def GetNewestList( self, full ):
( url, compfile, destfile, _ ) = self._get_update_info( full )
if url is None:
self.logger.error( 'No suitable archive extractor available for this system' )
self.notifier.ShowMissingExtractorError()
return False
# get mirrorlist
self.logger.info( 'Opening {}', url )
try:
data = urllib2.urlopen( url ).read()
except urllib2.URLError as err:
self.logger.error( 'Failure opening {}', url )
self.notifier.ShowDownloadError( url, err )
return False
root = etree.fromstring ( data )
urls = []
for server in root.findall( 'Server' ):
try:
URL = server.find( 'URL' ).text
Prio = server.find( 'Prio' ).text
urls.append( ( self._get_update_url( URL ), float( Prio ) + random.random() * 1.2 ) )
self.logger.info( 'Found mirror {} (Priority {})', URL, Prio )
except AttributeError:
pass
urls = sorted( urls, key = itemgetter( 1 ) )
urls = [ url[0] for url in urls ]
# cleanup downloads
self.logger.info( 'Cleaning up old downloads...' )
self._file_remove( compfile )
self._file_remove( destfile )
# download filmliste
self.notifier.ShowDownloadProgress()
lasturl = ''
for url in urls:
try:
lasturl = url
self.logger.info( 'Trying to download {} from {}...', os.path.basename( compfile ), url )
self.notifier.UpdateDownloadProgress( 0, url )
mvutils.url_retrieve( url, filename = compfile, reporthook = self.notifier.HookDownloadProgress, aborthook = self.monitor.abortRequested )
break
except urllib2.URLError as err:
self.logger.error( 'Failure downloading {}', url )
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError( lasturl, err )
return False
except ExitRequested as err:
self.logger.error( 'Immediate exit requested. Aborting download of {}', url )
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError( lasturl, err )
return False
except Exception as err:
self.logger.error( 'Failure writng {}', url )
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError( lasturl, err )
return False
# decompress filmliste
if self.use_xz is True:
self.logger.info( 'Trying to decompress xz file...' )
retval = subprocess.call( [ mvutils.find_xz(), '-d', compfile ] )
self.logger.info( 'Return {}', retval )
elif upd_can_bz2 is True:
self.logger.info( 'Trying to decompress bz2 file...' )
retval = self._decompress_bz2( compfile, destfile )
self.logger.info( 'Return {}', retval )
elif upd_can_gz is True:
self.logger.info( 'Trying to decompress gz file...' )
retval = self._decompress_gz( compfile, destfile )
self.logger.info( 'Return {}', retval )
else:
# should nebver reach
pass
self.notifier.CloseDownloadProgress()
return retval == 0 and mvutils.file_exists( destfile )
def _get_update_info( self, full ):
if self.use_xz is True:
ext = 'xz'
elif upd_can_bz2 is True:
ext = 'bz2'
elif upd_can_gz is True:
ext = 'gz'
else:
return ( None, None, None, 0, )
if full:
return (
FILMLISTE_AKT_URL,
os.path.join( self.settings.datapath, 'Filmliste-akt.' + ext ),
os.path.join( self.settings.datapath, 'Filmliste-akt' ),
600,
)
else:
return (
FILMLISTE_DIF_URL,
os.path.join( self.settings.datapath, 'Filmliste-diff.' + ext ),
os.path.join( self.settings.datapath, 'Filmliste-diff' ),
700,
)
def _get_update_url( self, url ):
if self.use_xz is True:
return url
elif upd_can_bz2 is True:
return os.path.splitext( url )[0] + '.bz2'
elif upd_can_gz is True:
return os.path.splitext( url )[0] + '.gz'
else:
# should never happen since it will not be called
return None
def _file_remove( self, name ):
if mvutils.file_exists( name ):
try:
os.remove( name )
return True
except OSError as err:
self.logger.error( 'Failed to remove {}: error {}', name, err )
return False
def _update_start( self, full ):
self.logger.info( 'Initializing update...' )
self.add_chn = 0
self.add_shw = 0
self.add_mov = 0
self.add_chn = 0
self.add_shw = 0
self.add_mov = 0
self.del_chn = 0
self.del_shw = 0
self.del_mov = 0
self.index = 0
self.count = 0
self.film = {
"channel": "",
"show": "",
"title": "",
"aired": "1980-01-01 00:00:00",
"duration": "00:00:00",
"size": 0,
"description": "",
"website": "",
"url_sub": "",
"url_video": "",
"url_video_sd": "",
"url_video_hd": "",
"airedepoch": 0,
"geo": ""
}
return self.db.ftUpdateStart( full )
def _update_end( self, full, status ):
self.logger.info( 'Added: channels:%d, shows:%d, movies:%d ...' % ( self.add_chn, self.add_shw, self.add_mov ) )
( self.del_chn, self.del_shw, self.del_mov, self.tot_chn, self.tot_shw, self.tot_mov ) = self.db.ftUpdateEnd( full and status == 'IDLE' )
self.logger.info( 'Deleted: channels:%d, shows:%d, movies:%d' % ( self.del_chn, self.del_shw, self.del_mov ) )
self.logger.info( 'Total: channels:%d, shows:%d, movies:%d' % ( self.tot_chn, self.tot_shw, self.tot_mov ) )
self.db.UpdateStatus(
status,
int( time.time() ) if status != 'ABORTED' else None,
None,
1 if full else 0,
self.add_chn, self.add_shw, self.add_mov,
self.del_chn, self.del_shw, self.del_mov,
self.tot_chn, self.tot_shw, self.tot_mov
)
def _init_record( self ):
self.index = 0
self.film["title"] = ""
self.film["aired"] = "1980-01-01 00:00:00"
self.film["duration"] = "00:00:00"
self.film["size"] = 0
self.film["description"] = ""
self.film["website"] = ""
self.film["url_sub"] = ""
self.film["url_video"] = ""
self.film["url_video_sd"] = ""
self.film["url_video_hd"] = ""
self.film["airedepoch"] = 0
self.film["geo"] = ""
def _end_record( self, records ):
if self.count % 1000 == 0:
percent = int( self.count * 100 / records )
self.logger.info( 'In progress (%d%%): channels:%d, shows:%d, movies:%d ...' % ( percent, self.add_chn, self.add_shw, self.add_mov ) )
self.notifier.UpdateUpdateProgress( percent if percent <= 100 else 100, self.count, self.add_chn, self.add_shw, self.add_mov )
self.db.UpdateStatus(
add_chn = self.add_chn,
add_shw = self.add_shw,
add_mov = self.add_mov,
tot_chn = self.tot_chn + self.add_chn,
tot_shw = self.tot_shw + self.add_shw,
tot_mov = self.tot_mov + self.add_mov
)
self.count = self.count + 1
( _, cnt_chn, cnt_shw, cnt_mov ) = self.db.ftInsertFilm( self.film, True )
else:
self.count = self.count + 1
( _, cnt_chn, cnt_shw, cnt_mov ) = self.db.ftInsertFilm( self.film, False )
self.add_chn += cnt_chn
self.add_shw += cnt_shw
self.add_mov += cnt_mov
def _add_value( self, val ):
if self.index == 0:
if val != "":
self.film["channel"] = val
elif self.index == 1:
if val != "":
self.film["show"] = val[:255]
elif self.index == 2:
self.film["title"] = val[:255]
elif self.index == 3:
if len(val) == 10:
self.film["aired"] = val[6:] + '-' + val[3:5] + '-' + val[:2]
elif self.index == 4:
if ( self.film["aired"] != "1980-01-01 00:00:00" ) and ( len(val) == 8 ):
self.film["aired"] = self.film["aired"] + " " + val
elif self.index == 5:
if len(val) == 8:
self.film["duration"] = val
elif self.index == 6:
if val != "":
self.film["size"] = int(val)
elif self.index == 7:
self.film["description"] = val
elif self.index == 8:
self.film["url_video"] = val
elif self.index == 9:
self.film["website"] = val
elif self.index == 10:
self.film["url_sub"] = val
elif self.index == 12:
self.film["url_video_sd"] = self._make_url(val)
elif self.index == 14:
self.film["url_video_hd"] = self._make_url(val)
elif self.index == 16:
if val != "":
self.film["airedepoch"] = int(val)
elif self.index == 18:
self.film["geo"] = val
self.index = self.index + 1
def _make_url( self, val ):
x = val.split( '|' )
if len( x ) == 2:
cnt = int( x[0] )
return self.film["url_video"][:cnt] + x[1]
else:
return val
def _decompress_bz2( self, sourcefile, destfile ):
blocksize = 8192
try:
with open( destfile, 'wb' ) as df, open( sourcefile, 'rb' ) as sf:
decompressor = bz2.BZ2Decompressor()
for data in iter( lambda : sf.read( blocksize ), b'' ):
df.write( decompressor.decompress( data ) )
except Exception as err:
self.logger.error( 'bz2 decompression failed: {}'.format( err ) )
return -1
return 0
def _decompress_gz( self, sourcefile, destfile ):
blocksize = 8192
try:
with open( destfile, 'wb' ) as df, gzip.open( sourcefile ) as sf:
for data in iter( lambda : sf.read( blocksize ), b'' ):
df.write( data )
except Exception as err:
self.logger.error( 'gz decompression failed: {}'.format( err ) )
return -1
return 0
def init(self, convert=False):
""" Initializes the updater """
if self.database is not None:
self.exit()
self.database = Store(self.logger, self.notifier, self.settings)
self.database.init(convert=convert)
class MediathekViewUpdater(object):
""" The database updator class """
def __init__(self, logger, notifier, settings, monitor=None):
self.logger = logger
self.notifier = notifier
self.settings = settings
self.monitor = monitor
self.database = None
self.use_xz = mvutils.find_xz() is not None
self.cycle = 0
self.add_chn = 0
self.add_shw = 0
self.add_mov = 0
self.del_chn = 0
self.del_shw = 0
self.del_mov = 0
self.tot_chn = 0
self.tot_shw = 0
self.tot_mov = 0
self.index = 0
self.count = 0
self.film = {}
def init(self, convert=False):
""" Initializes the updater """
if self.database is not None:
self.exit()
self.database = Store(self.logger, self.notifier, self.settings)
self.database.init(convert=convert)
def exit(self):
""" Resets the updater """
if self.database is not None:
self.database.exit()
del self.database
self.database = None
def reload(self):
""" Reloads the updater """
self.exit()
self.init()
def is_enabled(self):
""" Returns if the updater is enabled """
return self.settings.updenabled
def get_current_update_operation(self, force=False, full=False):
"""
Determines which update operation should be done. Returns
one of these values:
0 - no update operation pending
1 - full update
2 - differential update
Args:
force(bool, optional): if `True` a full update
is always returned. Default is `False`
full(book, optional): if `True` a full update
is always returned. Default is `False`
"""
if self.database is None:
# db not available - no update
self.logger.info('Update disabled since database not available')
return 0
elif self.settings.updmode == 0:
# update disabled - no update
return 0
elif self.settings.updmode == 1 or self.settings.updmode == 2:
# manual update or update on first start
if self.settings.is_update_triggered() is True:
return self._get_next_update_operation(True, False)
else:
# no update on all subsequent calls
return 0
elif self.settings.updmode == 3:
# automatic update
if self.settings.is_user_alive():
return self._get_next_update_operation(force, full)
else:
# no update if user is idle for more than 2 hours
return 0
elif self.settings.updmode == 4:
# continous update
return self._get_next_update_operation(force, full)
def _get_next_update_operation(self, force=False, full=False):
status = self.database.get_status()
tsnow = int(time.time())
tsold = status['lastupdate']
dtnow = datetime.datetime.fromtimestamp(tsnow).date()
dtold = datetime.datetime.fromtimestamp(tsold).date()
if status['status'] == 'UNINIT':
# database not initialized - no update
self.logger.debug('database not initialized')
return 0
elif status['status'] == "UPDATING" and tsnow - tsold > 10800:
# process was probably killed during update - no update
self.logger.info(
'Stuck update pretending to run since epoch {} reset', tsold)
self.database.update_status('ABORTED')
return 0
elif status['status'] == "UPDATING":
# already updating - no update
self.logger.debug('Already updating')
return 0
elif not full and not force and tsnow - tsold < self.settings.updinterval:
# last update less than the configured update interval - no update
self.logger.debug(
'Last update less than the configured update interval. do nothing')
return 0
elif dtnow != dtold:
# last update was not today. do full update once a day
self.logger.debug(
'Last update was not today. do full update once a day')
return 1
elif status['status'] == "ABORTED" and status['fullupdate'] == 1:
# last full update was aborted - full update needed
self.logger.debug(
'Last full update was aborted - full update needed')
return 1
elif full is True:
# full update requested
self.logger.info('Full update requested')
return 1
else:
# do differential update
self.logger.debug('Do differential update')
return 2
def update(self, full):
"""
Downloads the database update file and
then performs a database update
Args:
full(bool): Perform full update if `True`
"""
if self.database is None:
return
elif self.database.supports_native_update(full):
if self.get_newest_list(full):
if self.database.native_update(full):
self.cycle += 1
self.delete_list(full)
elif self.database.supports_update():
if self.get_newest_list(full):
if self.import_database(full):
self.cycle += 1
self.delete_list(full)
def _object_pairs_hook(self, inputArg):
"""
We need this because the default handler would convert everything to dict
and the same key would be overwritten and lost (e.g. X)
"""
return inputArg
def import_database(self, full):
"""
Performs a database update when a
downloaded update file is available
Args:
full(bool): Perform full update if `True`
"""
(_, _, destfile, avgrecsize) = self._get_update_info(full)
if not mvutils.file_exists(destfile):
self.logger.error('File {} does not exists', destfile)
return False
# estimate number of records in update file
records = int(mvutils.file_size(destfile) / avgrecsize)
if not self.database.ft_init():
self.logger.warn(
'Failed to initialize update. Maybe a concurrency problem?')
return False
# pylint: disable=broad-except
try:
starttime = time.time()
with closing( open(destfile, 'r', encoding="utf-8") ) as updatefile:
jsonDoc = json.load( updatefile, object_pairs_hook=self._object_pairs_hook )
self.logger.info( 'Starting import of {} records from {}', (len(jsonDoc)-2), destfile )
flsm = 0
flts = 0
(self.tot_chn, self.tot_shw, self.tot_mov) = self._update_start(full)
self.notifier.show_update_progress()
####
flsm = 0
sender = ""
thema = ""
### ROOT LIST
for atuple in jsonDoc:
if (atuple[0] == 'Filmliste' and flsm == 0):
### META
### "Filmliste":["23.04.2020, 18:23","23.04.2020, 16:23","3","MSearch [Vers.: 3.1.129]","3c90946f05eb1e2fa6cf2327cca4f1d4"],
flsm +=1
# this is the timestamp of this database update
value = atuple[1][0]
try:
fldt = datetime.datetime.strptime(
value.strip(), "%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.database.update_status(filmupdate=flts)
self.logger.info(
'Filmliste dated {}', value.strip())
except TypeError:
# pylint: disable=line-too-long
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(time.mktime(time.strptime(
value.strip(), "%d.%m.%Y, %H:%M")))
self.database.update_status(
filmupdate=flts)
self.logger.info(
'Filmliste dated {}', value.strip())
# pylint: disable=broad-except
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}', value.strip(), err)
except ValueError as err:
pass
elif (atuple[0] == 'filmliste' and flsm == 1):
flsm +=1
# VOID - we do not need column names
# "Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"],
elif (atuple[0] == 'X'):
self._init_record()
# behaviour of the update list
if (len(atuple[1][0]) > 0):
sender = atuple[1][0]
else:
atuple[1][0] = sender
# same for thema
if (len(atuple[1][1]) > 0):
thema = atuple[1][1]
else:
atuple[1][1] = thema
##
self._add_value( atuple[1] )
self._end_record(records)
if self.count % 100 == 0 and self.monitor.abort_requested():
# kodi is shutting down. Close all
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return True
self._update_end(full, 'IDLE')
self.logger.info('{} records processed',self.count)
self.logger.info(
'Import of {} in update cycle {} finished. Duration: {} seconds',
destfile,
self.cycle,
int(time.time() - starttime)
)
self.notifier.close_update_progress()
return True
except KeyboardInterrupt:
self._update_end(full, 'ABORTED')
self.logger.info('Update cycle {} interrupted by user', self.cycle)
self.notifier.close_update_progress()
return False
except DatabaseCorrupted as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except DatabaseLost as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except Exception as err:
self.logger.error(
'Error {} while processing {} on update cycle {}', err, destfile, self.cycle)
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return False
def get_newest_list(self, full):
"""
Downloads the database update file
Args:
full(bool): Downloads the full list if `True`
"""
(url, compfile, destfile, _) = self._get_update_info(full)
if url is None:
self.logger.error(
'No suitable archive extractor available for this system')
self.notifier.show_missing_extractor_error()
return False
# cleanup downloads
self.logger.info('Cleaning up old downloads...')
mvutils.file_remove(compfile)
mvutils.file_remove(destfile)
# download filmliste
self.notifier.show_download_progress()
# pylint: disable=broad-except
try:
self.logger.info('Trying to download {} from {}...',
os.path.basename(compfile), url)
self.notifier.update_download_progress(0, url)
mvutils.url_retrieve(
url,
filename=compfile,
reporthook=self.notifier.hook_download_progress,
aborthook=self.monitor.abort_requested
)
except URLError as err:
self.logger.error('Failure downloading {} - {}', url, err)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
return False
except ExitRequested as err:
self.logger.error(
'Immediate exit requested. Aborting download of {}', url)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
return False
except Exception as err:
self.logger.error('Failure writing {}', url)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
return False
# decompress filmliste
if self.use_xz is True:
self.logger.info('Trying to decompress xz file...')
retval = subprocess.call([mvutils.find_xz(), '-d', compfile])
self.logger.info('Return {}', retval)
elif UPD_CAN_BZ2 is True:
self.logger.info('Trying to decompress bz2 file...')
retval = self._decompress_bz2(compfile, destfile)
self.logger.info('Return {}', retval)
elif UPD_CAN_GZ is True:
self.logger.info('Trying to decompress gz file...')
retval = self._decompress_gz(compfile, destfile)
self.logger.info('Return {}', retval)
else:
# should never reach
pass
self.notifier.close_download_progress()
return retval == 0 and mvutils.file_exists(destfile)
def delete_list(self, full):
"""
Deletes locally stored database update files
Args:
full(bool): Deletes the full lists if `True`
"""
(_, compfile, destfile, _) = self._get_update_info(full)
self.logger.info('Cleaning up downloads...')
mvutils.file_remove(compfile)
mvutils.file_remove(destfile)
def _get_update_info(self, full):
if self.use_xz is True:
ext = '.xz'
elif UPD_CAN_BZ2 is True:
ext = '.bz2'
elif UPD_CAN_GZ is True:
ext = '.gz'
else:
return (None, None, None, 0, )
info = self.database.get_native_info(full)
if info is not None:
return (
self._get_update_url(info[0]),
os.path.join(self.settings.datapath, info[1] + ext),
os.path.join(self.settings.datapath, info[1]),
500
)
if full:
return (
FILMLISTE_URL + FILMLISTE_AKT + ext,
os.path.join(self.settings.datapath, FILMLISTE_AKT + ext),
os.path.join(self.settings.datapath, FILMLISTE_AKT),
600,
)
else:
return (
FILMLISTE_URL + FILMLISTE_DIF + ext,
os.path.join(self.settings.datapath, FILMLISTE_DIF + ext),
os.path.join(self.settings.datapath, FILMLISTE_DIF),
700,
)
def _get_update_url(self, url):
if self.use_xz is True:
return url
elif UPD_CAN_BZ2 is True:
return os.path.splitext(url)[0] + '.bz2'
elif UPD_CAN_GZ is True:
return os.path.splitext(url)[0] + '.gz'
else:
# should never happen since it will not be called
return None
def _update_start(self, full):
self.logger.info('Initializing update...')
self.add_chn = 0
self.add_shw = 0
self.add_mov = 0
self.del_chn = 0
self.del_shw = 0
self.del_mov = 0
self.index = 0
self.count = 0
self.film = {
"channel": "",
"show": "",
"title": "",
"aired": "1980-01-01 00:00:00",
"duration": "00:00:00",
"size": 0,
"description": "",
"website": "",
"url_sub": "",
"url_video": "",
"url_video_sd": "",
"url_video_hd": "",
"airedepoch": 0,
"geo": ""
}
return self.database.ft_update_start(full)
def _update_end(self, full, status):
self.logger.info('Added: channels:%d, shows:%d, movies:%d ...' % (
self.add_chn, self.add_shw, self.add_mov))
(self.del_chn, self.del_shw, self.del_mov, self.tot_chn, self.tot_shw,
self.tot_mov) = self.database.ft_update_end(full and status == 'IDLE')
self.logger.info('Deleted: channels:%d, shows:%d, movies:%d' %
(self.del_chn, self.del_shw, self.del_mov))
self.logger.info('Total: channels:%d, shows:%d, movies:%d' %
(self.tot_chn, self.tot_shw, self.tot_mov))
self.database.update_status(
status,
int(time.time()) if status != 'ABORTED' else None,
None,
1 if full else 0,
self.add_chn, self.add_shw, self.add_mov,
self.del_chn, self.del_shw, self.del_mov,
self.tot_chn, self.tot_shw, self.tot_mov
)
def _init_record(self):
self.index = 0
self.film["title"] = ""
self.film["aired"] = "1980-01-01 00:00:00"
self.film["duration"] = "00:00:00"
self.film["size"] = 0
self.film["description"] = ""
self.film["website"] = ""
self.film["url_sub"] = ""
self.film["url_video"] = ""
self.film["url_video_sd"] = ""
self.film["url_video_hd"] = ""
self.film["airedepoch"] = 0
self.film["geo"] = ""
def _end_record(self, records):
if self.count % 1000 == 0:
# pylint: disable=line-too-long
percent = int(self.count * 100 / records)
self.logger.info('In progress (%d%%): channels:%d, shows:%d, movies:%d ...' % (
percent, self.add_chn, self.add_shw, self.add_mov))
self.notifier.update_update_progress(
percent if percent <= 100 else 100, self.count, self.add_chn, self.add_shw, self.add_mov)
self.database.update_status(
add_chn=self.add_chn,
add_shw=self.add_shw,
add_mov=self.add_mov,
tot_chn=self.tot_chn + self.add_chn,
tot_shw=self.tot_shw + self.add_shw,
tot_mov=self.tot_mov + self.add_mov
)
self.count = self.count + 1
(_, cnt_chn, cnt_shw, cnt_mov) = self.database.ft_insert_film(
self.film,
True
)
else:
self.count = self.count + 1
(_, cnt_chn, cnt_shw, cnt_mov) = self.database.ft_insert_film(
self.film,
False
)
self.add_chn += cnt_chn
self.add_shw += cnt_shw
self.add_mov += cnt_mov
def _add_value(self, valueArray):
self.film["channel"] = valueArray[0]
self.film["show"] = valueArray[1][:255]
self.film["title"] = valueArray[2][:255]
##
if len(valueArray[3]) == 10:
self.film["aired"] = valueArray[3][6:] + '-' + valueArray[3][3:5] + '-' + valueArray[3][:2]
if (len(valueArray[4]) == 8):
self.film["aired"] = self.film["aired"] + " " + valueArray[4]
##
if len(valueArray[5]) > 0:
self.film["duration"] = valueArray[5]
if len(valueArray[6]) > 0:
self.film["size"] = int(valueArray[6])
if len(valueArray[7]) > 0:
self.film["description"] = valueArray[7]
self.film["url_video"] = valueArray[8]
self.film["website"] = valueArray[9]
self.film["url_sub"] = valueArray[10]
self.film["url_video_sd"] = self._make_url(valueArray[12])
self.film["url_video_hd"] = self._make_url(valueArray[14])
if len(valueArray[16]) > 0:
self.film["airedepoch"] = int(valueArray[16])
self.film["geo"] = valueArray[18]
def _make_url(self, val):
parts = val.split('|')
if len(parts) == 2:
cnt = int(parts[0])
return self.film["url_video"][:cnt] + parts[1]
else:
return val
def _decompress_bz2(self, sourcefile, destfile):
blocksize = 8192
try:
with open(destfile, 'wb') as dstfile, open(sourcefile, 'rb') as srcfile:
decompressor = bz2.BZ2Decompressor()
for data in iter(lambda: srcfile.read(blocksize), b''):
dstfile.write(decompressor.decompress(data))
# pylint: disable=broad-except
except Exception as err:
self.logger.error('bz2 decompression failed: {}'.format(err))
return -1
return 0
def _decompress_gz(self, sourcefile, destfile):
blocksize = 8192
# pylint: disable=broad-except
try:
with open(destfile, 'wb') as dstfile, gzip.open(sourcefile) as srcfile:
for data in iter(lambda: srcfile.read(blocksize), b''):
dstfile.write(data)
except Exception as err:
self.logger.error(
'gz decompression of "{}" to "{}" failed: {}', sourcefile, destfile, err)
if mvutils.find_gzip() is not None:
gzip_binary = mvutils.find_gzip()
self.logger.info(
'Trying to decompress gzip file "{}" using {}...', sourcefile, gzip_binary)
try:
mvutils.file_remove(destfile)
retval = subprocess.call([gzip_binary, '-d', sourcefile])
self.logger.info('Calling {} -d {} returned {}',
gzip_binary, sourcefile, retval)
return retval
except Exception as err:
self.logger.error(
'gz commandline decompression of "{}" to "{}" failed: {}',
sourcefile, destfile, err)
return -1
return 0
# pylint: disable=pointless-string-statement
"""
def Init(self, convert=False):
if self.db is not None:
self.Exit()
self.db = Store(self.logger, self.notifier, self.settings)
self.db.Init(convert=convert)
class MediathekViewUpdater(object):
def __init__(self, logger, notifier, settings, monitor=None):
self.logger = logger
self.notifier = notifier
self.settings = settings
self.monitor = monitor
self.db = None
self.cycle = 0
self.use_xz = mvutils.find_xz() is not None
def Init(self, convert=False):
if self.db is not None:
self.Exit()
self.db = Store(self.logger, self.notifier, self.settings)
self.db.Init(convert=convert)
def Exit(self):
if self.db is not None:
self.db.Exit()
del self.db
self.db = None
def Reload(self):
self.Exit()
self.Init()
def IsEnabled(self):
return self.settings.updenabled
def GetCurrentUpdateOperation(self, force=False):
if self.db is None:
# db not available - no update
self.logger.info('Update disabled since database not available')
return 0
elif self.settings.updmode == 0:
# update disabled - no update
return 0
elif self.settings.updmode == 1 or self.settings.updmode == 2:
# manual update or update on first start
if self.settings.IsUpdateTriggered() is True:
return self._getNextUpdateOperation(True)
else:
# no update on all subsequent calls
return 0
elif self.settings.updmode == 3:
# automatic update
if self.settings.IsUserAlive():
return self._getNextUpdateOperation(force)
else:
# no update of user is idle for more than 2 hours
return 0
def _getNextUpdateOperation(self, force=False):
status = self.db.GetStatus()
tsnow = int(time.time())
tsold = status['lastupdate']
dtnow = datetime.datetime.fromtimestamp(tsnow).date()
dtold = datetime.datetime.fromtimestamp(tsold).date()
if status['status'] == 'UNINIT':
# database not initialized - no update
self.logger.debug('database not initialized')
return 0
elif status['status'] == "UPDATING" and tsnow - tsold > 10800:
# process was probably killed during update - no update
self.logger.info(
'Stuck update pretending to run since epoch {} reset', tsold)
self.db.UpdateStatus('ABORTED')
return 0
elif status['status'] == "UPDATING":
# already updating - no update
self.logger.debug('Already updating')
return 0
elif not force and tsnow - tsold < self.settings.updinterval:
# last update less than the configured update interval - no update
self.logger.debug(
'Last update less than the configured update interval. do nothing'
)
return 0
elif dtnow != dtold:
# last update was not today. do full update once a day
self.logger.debug(
'Last update was not today. do full update once a day')
return 1
elif status['status'] == "ABORTED" and status['fullupdate'] == 1:
# last full update was aborted - full update needed
self.logger.debug(
'Last full update was aborted - full update needed')
return 1
else:
# do differential update
self.logger.debug('Do differential update')
return 2
def Update(self, full):
if self.db is None:
return
if self.db.SupportsUpdate():
if self.GetNewestList(full):
if self.Import(full):
self.cycle += 1
self.DeleteList(full)
def Import(self, full):
(_, _, destfile, avgrecsize) = self._get_update_info(full)
if not mvutils.file_exists(destfile):
self.logger.error('File {} does not exists', destfile)
return False
# estimate number of records in update file
records = int(mvutils.file_size(destfile) / avgrecsize)
if not self.db.ftInit():
self.logger.warn(
'Failed to initialize update. Maybe a concurrency problem?')
return False
try:
self.logger.info('Starting import of approx. {} records from {}',
records, destfile)
with open(destfile, 'r') as file:
parser = ijson.parse(file)
flsm = 0
flts = 0
(self.tot_chn, self.tot_shw,
self.tot_mov) = self._update_start(full)
self.notifier.ShowUpdateProgress()
for prefix, event, value in parser:
if (prefix, event) == ("X", "start_array"):
self._init_record()
elif (prefix, event) == ("X", "end_array"):
self._end_record(records)
if self.count % 100 == 0 and self.monitor.abortRequested(
):
# kodi is shutting down. Close all
self._update_end(full, 'ABORTED')
self.notifier.CloseUpdateProgress()
return True
elif (prefix, event) == ("X.item", "string"):
if value is not None:
self._add_value(value.strip())
else:
self._add_value("")
elif (prefix, event) == ("Filmliste", "start_array"):
flsm += 1
elif (prefix, event) == ("Filmliste.item", "string"):
flsm += 1
if flsm == 2 and value is not None:
# this is the timestmap of this database update
try:
fldt = datetime.datetime.strptime(
value.strip(), "%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.db.UpdateStatus(filmupdate=flts)
self.logger.info('Filmliste dated {}',
value.strip())
except TypeError:
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(
time.mktime(
time.strptime(
value.strip(),
"%d.%m.%Y, %H:%M")))
self.db.UpdateStatus(filmupdate=flts)
self.logger.info('Filmliste dated {}',
value.strip())
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}',
value.strip(), err)
except ValueError as err:
pass
self.db.ftFlushInsert()
self._update_end(full, 'IDLE')
self.logger.info('Import of {} in update cycle {} finished',
destfile, self.cycle)
self.notifier.CloseUpdateProgress()
return True
except KeyboardInterrupt:
self._update_end(full, 'ABORTED')
self.logger.info('Update cycle {} interrupted by user', self.cycle)
self.notifier.CloseUpdateProgress()
return False
except DatabaseCorrupted as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.CloseUpdateProgress()
except DatabaseLost as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.CloseUpdateProgress()
except Exception as err:
self.logger.error(
'Error {} while processing {} on update cycle {}', err,
destfile, self.cycle)
self._update_end(full, 'ABORTED')
self.notifier.CloseUpdateProgress()
return False
def GetNewestList(self, full):
(url, compfile, destfile, _) = self._get_update_info(full)
if url is None:
self.logger.error(
'No suitable archive extractor available for this system')
self.notifier.ShowMissingExtractorError()
return False
# get mirrorlist
self.logger.info('Opening {}', url)
try:
data = urllib2.urlopen(url).read()
except urllib2.URLError as err:
self.logger.error('Failure opening {}', url)
self.notifier.ShowDownloadError(url, err)
return False
root = etree.fromstring(data)
urls = []
for server in root.findall('Server'):
try:
URL = server.find('URL').text
Prio = server.find('Prio').text
urls.append((self._get_update_url(URL),
float(Prio) + random.random() * 1.2))
self.logger.info('Found mirror {} (Priority {})', URL, Prio)
except AttributeError:
pass
urls = sorted(urls, key=itemgetter(1))
urls = [url[0] for url in urls]
# cleanup downloads
self.logger.info('Cleaning up old downloads...')
mvutils.file_remove(compfile)
mvutils.file_remove(destfile)
# download filmliste
self.notifier.ShowDownloadProgress()
lasturl = ''
for url in urls:
try:
lasturl = url
self.logger.info('Trying to download {} from {}...',
os.path.basename(compfile), url)
self.notifier.UpdateDownloadProgress(0, url)
mvutils.url_retrieve(
url,
filename=compfile,
reporthook=self.notifier.HookDownloadProgress,
aborthook=self.monitor.abortRequested)
break
except urllib2.URLError as err:
self.logger.error('Failure downloading {}', url)
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError(lasturl, err)
return False
except ExitRequested as err:
self.logger.error(
'Immediate exit requested. Aborting download of {}', url)
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError(lasturl, err)
return False
except Exception as err:
self.logger.error('Failure writng {}', url)
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError(lasturl, err)
return False
# decompress filmliste
if self.use_xz is True:
self.logger.info('Trying to decompress xz file...')
retval = subprocess.call([mvutils.find_xz(), '-d', compfile])
self.logger.info('Return {}', retval)
elif upd_can_bz2 is True:
self.logger.info('Trying to decompress bz2 file...')
retval = self._decompress_bz2(compfile, destfile)
self.logger.info('Return {}', retval)
elif upd_can_gz is True:
self.logger.info('Trying to decompress gz file...')
retval = self._decompress_gz(compfile, destfile)
self.logger.info('Return {}', retval)
else:
# should nebver reach
pass
self.notifier.CloseDownloadProgress()
return retval == 0 and mvutils.file_exists(destfile)
def DeleteList(self, full):
(_, compfile, destfile, _) = self._get_update_info(full)
self.logger.info('Cleaning up downloads...')
mvutils.file_remove(compfile)
mvutils.file_remove(destfile)
def _get_update_info(self, full):
if self.use_xz is True:
ext = 'xz'
elif upd_can_bz2 is True:
ext = 'bz2'
elif upd_can_gz is True:
ext = 'gz'
else:
return (
None,
None,
None,
0,
)
if full:
return (
FILMLISTE_AKT_URL,
os.path.join(self.settings.datapath, 'Filmliste-akt.' + ext),
os.path.join(self.settings.datapath, 'Filmliste-akt'),
600,
)
else:
return (
FILMLISTE_DIF_URL,
os.path.join(self.settings.datapath, 'Filmliste-diff.' + ext),
os.path.join(self.settings.datapath, 'Filmliste-diff'),
700,
)
def _get_update_url(self, url):
if self.use_xz is True:
return url
elif upd_can_bz2 is True:
return os.path.splitext(url)[0] + '.bz2'
elif upd_can_gz is True:
return os.path.splitext(url)[0] + '.gz'
else:
# should never happen since it will not be called
return None
def _update_start(self, full):
self.logger.info('Initializing update...')
self.add_chn = 0
self.add_shw = 0
self.add_mov = 0
self.del_chn = 0
self.del_shw = 0
self.del_mov = 0
self.index = 0
self.count = 0
self.film = {
"channel": "",
"show": "",
"title": "",
"aired": "1980-01-01 00:00:00",
"duration": "00:00:00",
"size": 0,
"description": "",
"website": "",
"url_sub": "",
"url_video": "",
"url_video_sd": "",
"url_video_hd": "",
"airedepoch": 0,
"geo": ""
}
return self.db.ftUpdateStart(full)
def _update_end(self, full, status):
self.logger.info('Added: channels:%d, shows:%d, movies:%d ...' %
(self.add_chn, self.add_shw, self.add_mov))
(self.del_chn, self.del_shw, self.del_mov, self.tot_chn, self.tot_shw,
self.tot_mov) = self.db.ftUpdateEnd(full and status == 'IDLE')
self.logger.info('Deleted: channels:%d, shows:%d, movies:%d' %
(self.del_chn, self.del_shw, self.del_mov))
self.logger.info('Total: channels:%d, shows:%d, movies:%d' %
(self.tot_chn, self.tot_shw, self.tot_mov))
self.db.UpdateStatus(status,
int(time.time()) if status != 'ABORTED' else None,
None, 1 if full else 0, self.add_chn,
self.add_shw, self.add_mov, self.del_chn,
self.del_shw, self.del_mov, self.tot_chn,
self.tot_shw, self.tot_mov)
def _init_record(self):
self.index = 0
self.film["title"] = ""
self.film["aired"] = "1980-01-01 00:00:00"
self.film["duration"] = "00:00:00"
self.film["size"] = 0
self.film["description"] = ""
self.film["website"] = ""
self.film["url_sub"] = ""
self.film["url_video"] = ""
self.film["url_video_sd"] = ""
self.film["url_video_hd"] = ""
self.film["airedepoch"] = 0
self.film["geo"] = ""
def _end_record(self, records):
self.count = self.count + 1
if self.count % self.db.flushBlockSize() == 0:
#add 10% to record for final db update time in update_end
percent = int(self.count * 100 / (records * 1.1))
self.logger.info(
'In progress (%d%%): channels:%d, shows:%d, movies:%d ...' %
(percent, self.add_chn, self.add_shw, self.add_mov))
self.notifier.UpdateUpdateProgress(
percent if percent <= 100 else 100, self.count, self.add_chn,
self.add_shw, self.add_mov)
self.db.UpdateStatus(add_chn=self.add_chn,
add_shw=self.add_shw,
add_mov=self.add_mov,
tot_chn=self.tot_chn + self.add_chn,
tot_shw=self.tot_shw + self.add_shw,
tot_mov=self.tot_mov + self.add_mov)
(_, cnt_chn, cnt_shw,
cnt_mov) = self.db.ftInsertFilm(self.film, True)
self.db.ftFlushInsert()
else:
(_, cnt_chn, cnt_shw,
cnt_mov) = self.db.ftInsertFilm(self.film, False)
self.add_chn += cnt_chn
self.add_shw += cnt_shw
self.add_mov += cnt_mov
def _add_value(self, val):
if self.index == 0:
if val != "":
self.film["channel"] = val
elif self.index == 1:
if val != "":
self.film["show"] = val[:255]
elif self.index == 2:
self.film["title"] = val[:255]
elif self.index == 3:
if len(val) == 10:
self.film["aired"] = val[6:] + '-' + val[3:5] + '-' + val[:2]
elif self.index == 4:
if (self.film["aired"] != "1980-01-01 00:00:00") and (len(val)
== 8):
self.film["aired"] = self.film["aired"] + " " + val
elif self.index == 5:
if len(val) == 8:
self.film["duration"] = val
elif self.index == 6:
if val != "":
self.film["size"] = int(val)
elif self.index == 7:
self.film["description"] = val
elif self.index == 8:
self.film["url_video"] = val
elif self.index == 9:
self.film["website"] = val
elif self.index == 10:
self.film["url_sub"] = val
elif self.index == 12:
self.film["url_video_sd"] = self._make_url(val)
elif self.index == 14:
self.film["url_video_hd"] = self._make_url(val)
elif self.index == 16:
if val != "":
self.film["airedepoch"] = int(val)
elif self.index == 18:
self.film["geo"] = val
self.index = self.index + 1
def _make_url(self, val):
x = val.split('|')
if len(x) == 2:
cnt = int(x[0])
return self.film["url_video"][:cnt] + x[1]
else:
return val
def _decompress_bz2(self, sourcefile, destfile):
blocksize = 8192
try:
with open(destfile, 'wb') as df, open(sourcefile, 'rb') as sf:
decompressor = bz2.BZ2Decompressor()
for data in iter(lambda: sf.read(blocksize), b''):
df.write(decompressor.decompress(data))
except Exception as err:
self.logger.error('bz2 decompression failed: {}'.format(err))
return -1
return 0
def _decompress_gz(self, sourcefile, destfile):
blocksize = 8192
try:
with open(destfile, 'wb') as df, gzip.open(sourcefile) as sf:
for data in iter(lambda: sf.read(blocksize), b''):
df.write(data)
except Exception as err:
self.logger.error('gz decompression failed: {}'.format(err))
return -1
return 0
def Init(self):
if self.db is not None:
self.Exit()
self.db = Store(self.logger, self.notifier, self.settings)
self.db.Init()
def __init__(self):
super(MediathekView, self).__init__()
self.settings = Settings()
self.notifier = Notifier()
self.database = Store(self.getNewLogger('Store'), self.notifier,
self.settings)
class MediathekView(KodiPlugin):
def __init__(self):
super(MediathekView, self).__init__()
self.settings = Settings()
self.notifier = Notifier()
self.database = Store(self.getNewLogger('Store'), self.notifier,
self.settings)
def show_main_menu(self):
# Search
self.addFolderItem(30901, {'mode': "search", 'extendedsearch': False})
# Search all
self.addFolderItem(30902, {'mode': "search", 'extendedsearch': True})
# Browse livestreams
self.addFolderItem(30903, {'mode': "livestreams"})
# Browse recently added
self.addFolderItem(30904, {'mode': "recent", 'channel': 0})
# Browse recently added by channel
self.addFolderItem(30905, {'mode': "recentchannels"})
# Browse by Initial->Show
self.addFolderItem(30906, {'mode': "initial", 'channel': 0})
# Browse by Channel->Initial->Shows
self.addFolderItem(30907, {'mode': "channels"})
# Database Information
self.addActionItem(30908, {'mode': "action-dbinfo"})
# Manual database update
if self.settings.updmode == 1 or self.settings.updmode == 2:
self.addActionItem(30909, {'mode': "action-dbupdate"})
self.endOfDirectory()
self._check_outdate()
def show_searches(self, extendedsearch=False):
self.addFolderItem(30931, {
'mode': "newsearch",
'extendedsearch': extendedsearch
})
RecentSearches(self, extendedsearch).load().populate()
self.endOfDirectory()
def new_search(self, extendedsearch=False):
settingid = 'lastsearch2' if extendedsearch is True else 'lastsearch1'
headingid = 30902 if extendedsearch is True else 30901
# are we returning from playback ?
search = self.addon.getSetting(settingid)
if search:
# restore previous search
self.database.Search(search, FilmUI(self), extendedsearch)
else:
# enter search term
(search, confirmed) = self.notifier.GetEnteredText('', headingid)
if len(search) > 2 and confirmed is True:
RecentSearches(self, extendedsearch).load().add(search).save()
if self.database.Search(search, FilmUI(self),
extendedsearch) > 0:
self.addon.setSetting(settingid, search)
else:
# pylint: disable=line-too-long
self.info(
'The following ERROR can be ignored. It is caused by the architecture of the Kodi Plugin Engine'
)
self.endOfDirectory(False, cacheToDisc=True)
# self.show_searches( extendedsearch )
def show_db_info(self):
info = self.database.GetStatus()
heading = self.language(30907)
infostr = self.language({
'NONE': 30941,
'UNINIT': 30942,
'IDLE': 30943,
'UPDATING': 30944,
'ABORTED': 30945
}.get(info['status'], 30941))
infostr = self.language(30965) % infostr
totinfo = self.language(30971) % (info['tot_chn'], info['tot_shw'],
info['tot_mov'])
updatetype = self.language(30972 if info['fullupdate'] > 0 else 30973)
if info['status'] == 'UPDATING' and info['filmupdate'] > 0:
updinfo = self.language(30967) % (
updatetype, datetime.datetime.fromtimestamp(
info['filmupdate']).strftime('%Y-%m-%d %H:%M:%S'),
info['add_chn'], info['add_shw'], info['add_mov'])
elif info['status'] == 'UPDATING':
updinfo = self.language(30968) % (updatetype, info['add_chn'],
info['add_shw'], info['add_mov'])
elif info['lastupdate'] > 0 and info['filmupdate'] > 0:
updinfo = self.language(30969) % (
updatetype, datetime.datetime.fromtimestamp(
info['lastupdate']).strftime('%Y-%m-%d %H:%M:%S'),
datetime.datetime.fromtimestamp(
info['filmupdate']).strftime('%Y-%m-%d %H:%M:%S'),
info['add_chn'], info['add_shw'], info['add_mov'],
info['del_chn'], info['del_shw'], info['del_mov'])
elif info['lastupdate'] > 0:
updinfo = self.language(30970) % (
updatetype, datetime.datetime.fromtimestamp(
info['lastupdate']).strftime('%Y-%m-%d %H:%M:%S'),
info['add_chn'], info['add_shw'], info['add_mov'],
info['del_chn'], info['del_shw'], info['del_mov'])
else:
updinfo = self.language(30966)
xbmcgui.Dialog().textviewer(
heading, infostr + '\n\n' + totinfo + '\n\n' + updinfo)
def _check_outdate(self, maxage=172800):
if self.settings.updmode != 1 and self.settings.updmode != 2:
# no check with update disabled or update automatic
return
if self.database is None:
# should never happen
self.notifier.ShowOutdatedUnknown()
return
status = self.database.GetStatus()
if status['status'] == 'NONE' or status['status'] == 'UNINIT':
# should never happen
self.notifier.ShowOutdatedUnknown()
return
elif status['status'] == 'UPDATING':
# great... we are updating. nuthin to show
return
# lets check how old we are
tsnow = int(time.time())
tsold = int(status['lastupdate'])
if tsnow - tsold > maxage:
self.notifier.ShowOutdatedKnown(status)
def init(self):
if self.database.Init():
if self.settings.HandleFirstRun():
pass
self.settings.HandleUpdateOnStart()
def run(self):
# save last activity timestamp
self.settings.ResetUserActivity()
# process operation
mode = self.get_arg('mode', None)
if mode is None:
self.show_main_menu()
elif mode == 'search':
extendedsearch = self.get_arg('extendedsearch', 'False') == 'True'
self.show_searches(extendedsearch)
elif mode == 'newsearch':
self.new_search(self.get_arg('extendedsearch', 'False') == 'True')
elif mode == 'research':
search = self.get_arg('search', '')
extendedsearch = self.get_arg('extendedsearch', 'False') == 'True'
self.database.Search(search, FilmUI(self), extendedsearch)
RecentSearches(self, extendedsearch).load().add(search).save()
elif mode == 'delsearch':
search = self.get_arg('search', '')
extendedsearch = self.get_arg('extendedsearch', 'False') == 'True'
RecentSearches(
self, extendedsearch).load().delete(search).save().populate()
self.runBuiltin('Container.Refresh')
elif mode == 'livestreams':
self.database.GetLiveStreams(
FilmUI(self, [xbmcplugin.SORT_METHOD_LABEL]))
elif mode == 'recent':
channel = self.get_arg('channel', 0)
self.database.GetRecents(channel, FilmUI(self))
elif mode == 'recentchannels':
self.database.GetRecentChannels(ChannelUI(self, nextdir='recent'))
elif mode == 'channels':
self.database.GetChannels(ChannelUI(self, nextdir='shows'))
elif mode == 'action-dbinfo':
self.show_db_info()
elif mode == 'action-dbupdate':
self.settings.TriggerUpdate()
self.notifier.ShowNotification(30963, 30964, time=10000)
elif mode == 'initial':
channel = self.get_arg('channel', 0)
self.database.GetInitials(channel, InitialUI(self))
elif mode == 'shows':
channel = self.get_arg('channel', 0)
initial = self.get_arg('initial', None)
self.database.GetShows(channel, initial, ShowUI(self))
elif mode == 'films':
show = self.get_arg('show', 0)
self.database.GetFilms(show, FilmUI(self))
elif mode == 'downloadmv':
filmid = self.get_arg('id', 0)
quality = self.get_arg('quality', 1)
Downloader(self).download_movie(filmid, quality)
elif mode == 'downloadep':
filmid = self.get_arg('id', 0)
quality = self.get_arg('quality', 1)
Downloader(self).download_episode(filmid, quality)
elif mode == 'playwithsrt':
filmid = self.get_arg('id', 0)
only_sru = self.get_arg('only_set_resolved_url', 'False') == 'True'
Downloader(self).play_movie_with_subs(filmid, only_sru)
# cleanup saved searches
if mode is None or mode != 'search':
self.addon.setSetting('lastsearch1', '')
if mode is None or mode != 'searchall':
self.addon.setSetting('lastsearch2', '')
def exit(self):
self.database.Exit()
class MediathekViewUpdater(object):
""" The database updator class """
def __init__(self, logger, notifier, settings, monitor=None):
self.logger = logger
self.notifier = notifier
self.settings = settings
self.monitor = monitor
self.database = None
self.use_xz = mvutils.find_xz() is not None
self.cycle = 0
self.add_chn = 0
self.add_shw = 0
self.add_mov = 0
self.del_chn = 0
self.del_shw = 0
self.del_mov = 0
self.tot_chn = 0
self.tot_shw = 0
self.tot_mov = 0
self.index = 0
self.count = 0
self.film = {}
def init(self, convert=False):
""" Initializes the updater """
if self.database is not None:
self.exit()
self.database = Store(self.logger, self.notifier, self.settings)
self.database.init(convert=convert)
def exit(self):
""" Resets the updater """
if self.database is not None:
self.database.exit()
del self.database
self.database = None
def reload(self):
""" Reloads the updater """
self.exit()
self.init()
def is_enabled(self):
""" Returns if the updater is enabled """
return self.settings.updenabled
def get_current_update_operation(self, force=False, full=False):
"""
Determines which update operation should be done. Returns
one of these values:
0 - no update operation pending
1 - full update
2 - differential update
Args:
force(bool, optional): if `True` a full update
is always returned. Default is `False`
full(book, optional): if `True` a full update
is always returned. Default is `False`
"""
if self.database is None:
# db not available - no update
self.logger.info('Update disabled since database not available')
return 0
elif self.settings.updmode == 0:
# update disabled - no update
return 0
elif self.settings.updmode == 1 or self.settings.updmode == 2:
# manual update or update on first start
if self.settings.is_update_triggered() is True:
return self._get_next_update_operation(True, False)
else:
# no update on all subsequent calls
return 0
elif self.settings.updmode == 3:
# automatic update
if self.settings.is_user_alive():
return self._get_next_update_operation(force, full)
else:
# no update if user is idle for more than 2 hours
return 0
elif self.settings.updmode == 4:
# continous update
return self._get_next_update_operation(force, full)
def _get_next_update_operation(self, force=False, full=False):
status = self.database.get_status()
tsnow = int(time.time())
tsold = status['lastupdate']
dtnow = datetime.datetime.fromtimestamp(tsnow).date()
dtold = datetime.datetime.fromtimestamp(tsold).date()
if status['status'] == 'UNINIT':
# database not initialized - no update
self.logger.debug('database not initialized')
return 0
elif status['status'] == "UPDATING" and tsnow - tsold > 10800:
# process was probably killed during update - no update
self.logger.info(
'Stuck update pretending to run since epoch {} reset', tsold)
self.database.update_status('ABORTED')
return 0
elif status['status'] == "UPDATING":
# already updating - no update
self.logger.debug('Already updating')
return 0
elif not full and not force and tsnow - tsold < self.settings.updinterval:
# last update less than the configured update interval - no update
self.logger.debug(
'Last update less than the configured update interval. do nothing'
)
return 0
elif dtnow != dtold:
# last update was not today. do full update once a day
self.logger.debug(
'Last update was not today. do full update once a day')
return 1
elif status['status'] == "ABORTED" and status['fullupdate'] == 1:
# last full update was aborted - full update needed
self.logger.debug(
'Last full update was aborted - full update needed')
return 1
elif full is True:
# full update requested
self.logger.info('Full update requested')
return 1
else:
# do differential update
self.logger.debug('Do differential update')
return 2
def update(self, full):
"""
Downloads the database update file and
then performs a database update
Args:
full(bool): Perform full update if `True`
"""
if self.database is None:
return
elif self.database.supports_native_update(full):
if self.get_newest_list(full):
if self.database.native_update(full):
self.cycle += 1
self.delete_list(full)
elif self.database.supports_update():
if self.get_newest_list(full):
if self.import_database(full):
self.cycle += 1
self.delete_list(full)
def import_database(self, full):
"""
Performs a database update when a
downloaded update file is available
Args:
full(bool): Perform full update if `True`
"""
(_, _, destfile, avgrecsize) = self._get_update_info(full)
if not mvutils.file_exists(destfile):
self.logger.error('File {} does not exists', destfile)
return False
# estimate number of records in update file
records = int(mvutils.file_size(destfile) / avgrecsize)
if not self.database.ft_init():
self.logger.warn(
'Failed to initialize update. Maybe a concurrency problem?')
return False
# pylint: disable=broad-except
try:
starttime = time.time()
self.logger.info('Starting import of approx. {} records from {}',
records, destfile)
with closing(open(destfile, 'r')) as updatefile:
parser = ijson.parse(updatefile)
flsm = 0
flts = 0
(self.tot_chn, self.tot_shw,
self.tot_mov) = self._update_start(full)
self.notifier.show_update_progress()
for prefix, event, value in parser:
if (prefix, event) == ("X", "start_array"):
self._init_record()
elif (prefix, event) == ("X", "end_array"):
self._end_record(records)
if self.count % 100 == 0 and self.monitor.abort_requested(
):
# kodi is shutting down. Close all
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return True
elif (prefix, event) == ("X.item", "string"):
if value is not None:
# self._add_value( value.strip().encode('utf-8') )
self._add_value(value.strip())
else:
self._add_value("")
elif (prefix, event) == ("Filmliste", "start_array"):
flsm += 1
elif (prefix, event) == ("Filmliste.item", "string"):
flsm += 1
if flsm == 2 and value is not None:
# this is the timestmap of this database update
try:
fldt = datetime.datetime.strptime(
value.strip(), "%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.database.update_status(filmupdate=flts)
self.logger.info('Filmliste dated {}',
value.strip())
except TypeError:
# pylint: disable=line-too-long
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(
time.mktime(
time.strptime(
value.strip(),
"%d.%m.%Y, %H:%M")))
self.database.update_status(
filmupdate=flts)
self.logger.info('Filmliste dated {}',
value.strip())
# pylint: disable=broad-except
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}',
value.strip(), err)
except ValueError as err:
pass
self._update_end(full, 'IDLE')
self.logger.info(
'Import of {} in update cycle {} finished. Duration: {} seconds',
destfile, self.cycle, int(time.time() - starttime))
self.notifier.close_update_progress()
return True
except KeyboardInterrupt:
self._update_end(full, 'ABORTED')
self.logger.info('Update cycle {} interrupted by user', self.cycle)
self.notifier.close_update_progress()
return False
except DatabaseCorrupted as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except DatabaseLost as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except Exception as err:
self.logger.error(
'Error {} while processing {} on update cycle {}', err,
destfile, self.cycle)
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return False
def get_newest_list(self, full):
"""
Downloads the database update file
Args:
full(bool): Downloads the full list if `True`
"""
(url, compfile, destfile, _) = self._get_update_info(full)
if url is None:
self.logger.error(
'No suitable archive extractor available for this system')
self.notifier.show_missing_extractor_error()
return False
# cleanup downloads
self.logger.info('Cleaning up old downloads...')
mvutils.file_remove(compfile)
mvutils.file_remove(destfile)
# download filmliste
self.notifier.show_download_progress()
# pylint: disable=broad-except
try:
self.logger.info('Trying to download {} from {}...',
os.path.basename(compfile), url)
self.notifier.update_download_progress(0, url)
mvutils.url_retrieve(
url,
filename=compfile,
reporthook=self.notifier.hook_download_progress,
aborthook=self.monitor.abort_requested)
except urllib2.URLError as err:
self.logger.error('Failure downloading {} - {}', url, err)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
return False
except ExitRequested as err:
self.logger.error(
'Immediate exit requested. Aborting download of {}', url)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
return False
except Exception as err:
self.logger.error('Failure writng {}', url)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
return False
# decompress filmliste
if self.use_xz is True:
self.logger.info('Trying to decompress xz file...')
retval = subprocess.call([mvutils.find_xz(), '-d', compfile])
self.logger.info('Return {}', retval)
elif UPD_CAN_BZ2 is True:
self.logger.info('Trying to decompress bz2 file...')
retval = self._decompress_bz2(compfile, destfile)
self.logger.info('Return {}', retval)
elif UPD_CAN_GZ is True:
self.logger.info('Trying to decompress gz file...')
retval = self._decompress_gz(compfile, destfile)
self.logger.info('Return {}', retval)
else:
# should nebver reach
pass
self.notifier.close_download_progress()
return retval == 0 and mvutils.file_exists(destfile)
def delete_list(self, full):
"""
Deletes locally stored database update files
Args:
full(bool): Deletes the full lists if `True`
"""
(_, compfile, destfile, _) = self._get_update_info(full)
self.logger.info('Cleaning up downloads...')
mvutils.file_remove(compfile)
mvutils.file_remove(destfile)
def _get_update_info(self, full):
if self.use_xz is True:
ext = '.xz'
elif UPD_CAN_BZ2 is True:
ext = '.bz2'
elif UPD_CAN_GZ is True:
ext = '.gz'
else:
return (
None,
None,
None,
0,
)
info = self.database.get_native_info(full)
if info is not None:
return (self._get_update_url(info[0]),
os.path.join(self.settings.datapath, info[1] + ext),
os.path.join(self.settings.datapath, info[1]), 500)
if full:
return (
FILMLISTE_URL + FILMLISTE_AKT + ext,
os.path.join(self.settings.datapath, FILMLISTE_AKT + ext),
os.path.join(self.settings.datapath, FILMLISTE_AKT),
600,
)
else:
return (
FILMLISTE_URL + FILMLISTE_DIF + ext,
os.path.join(self.settings.datapath, FILMLISTE_DIF + ext),
os.path.join(self.settings.datapath, FILMLISTE_DIF),
700,
)
def _get_update_url(self, url):
if self.use_xz is True:
return url
elif UPD_CAN_BZ2 is True:
return os.path.splitext(url)[0] + '.bz2'
elif UPD_CAN_GZ is True:
return os.path.splitext(url)[0] + '.gz'
else:
# should never happen since it will not be called
return None
def _update_start(self, full):
self.logger.info('Initializing update...')
self.add_chn = 0
self.add_shw = 0
self.add_mov = 0
self.del_chn = 0
self.del_shw = 0
self.del_mov = 0
self.index = 0
self.count = 0
self.film = {
"channel": "",
"show": "",
"title": "",
"aired": "1980-01-01 00:00:00",
"duration": "00:00:00",
"size": 0,
"description": "",
"website": "",
"url_sub": "",
"url_video": "",
"url_video_sd": "",
"url_video_hd": "",
"airedepoch": 0,
"geo": ""
}
return self.database.ft_update_start(full)
def _update_end(self, full, status):
self.logger.info('Added: channels:%d, shows:%d, movies:%d ...' %
(self.add_chn, self.add_shw, self.add_mov))
(self.del_chn, self.del_shw, self.del_mov, self.tot_chn, self.tot_shw,
self.tot_mov) = self.database.ft_update_end(full and status == 'IDLE')
self.logger.info('Deleted: channels:%d, shows:%d, movies:%d' %
(self.del_chn, self.del_shw, self.del_mov))
self.logger.info('Total: channels:%d, shows:%d, movies:%d' %
(self.tot_chn, self.tot_shw, self.tot_mov))
self.database.update_status(
status,
int(time.time()) if status != 'ABORTED' else None, None,
1 if full else 0, self.add_chn, self.add_shw, self.add_mov,
self.del_chn, self.del_shw, self.del_mov, self.tot_chn,
self.tot_shw, self.tot_mov)
def _init_record(self):
self.index = 0
self.film["title"] = ""
self.film["aired"] = "1980-01-01 00:00:00"
self.film["duration"] = "00:00:00"
self.film["size"] = 0
self.film["description"] = ""
self.film["website"] = ""
self.film["url_sub"] = ""
self.film["url_video"] = ""
self.film["url_video_sd"] = ""
self.film["url_video_hd"] = ""
self.film["airedepoch"] = 0
self.film["geo"] = ""
def _end_record(self, records):
if self.count % 1000 == 0:
# pylint: disable=line-too-long
percent = int(self.count * 100 / records)
self.logger.info(
'In progress (%d%%): channels:%d, shows:%d, movies:%d ...' %
(percent, self.add_chn, self.add_shw, self.add_mov))
self.notifier.update_update_progress(
percent if percent <= 100 else 100, self.count, self.add_chn,
self.add_shw, self.add_mov)
self.database.update_status(add_chn=self.add_chn,
add_shw=self.add_shw,
add_mov=self.add_mov,
tot_chn=self.tot_chn + self.add_chn,
tot_shw=self.tot_shw + self.add_shw,
tot_mov=self.tot_mov + self.add_mov)
self.count = self.count + 1
(_, cnt_chn, cnt_shw,
cnt_mov) = self.database.ft_insert_film(self.film, True)
else:
self.count = self.count + 1
(_, cnt_chn, cnt_shw,
cnt_mov) = self.database.ft_insert_film(self.film, False)
self.add_chn += cnt_chn
self.add_shw += cnt_shw
self.add_mov += cnt_mov
def _add_value(self, val):
if self.index == 0:
if val != "":
self.film["channel"] = val
elif self.index == 1:
if val != "":
self.film["show"] = val[:255]
elif self.index == 2:
self.film["title"] = val[:255]
elif self.index == 3:
if len(val) == 10:
self.film["aired"] = val[6:] + '-' + val[3:5] + '-' + val[:2]
elif self.index == 4:
if (self.film["aired"] != "1980-01-01 00:00:00") and (len(val)
== 8):
self.film["aired"] = self.film["aired"] + " " + val
elif self.index == 5:
if len(val) == 8:
self.film["duration"] = val
elif self.index == 6:
if val != "":
self.film["size"] = int(val)
elif self.index == 7:
self.film["description"] = val
elif self.index == 8:
self.film["url_video"] = val
elif self.index == 9:
self.film["website"] = val
elif self.index == 10:
self.film["url_sub"] = val
elif self.index == 12:
self.film["url_video_sd"] = self._make_url(val)
elif self.index == 14:
self.film["url_video_hd"] = self._make_url(val)
elif self.index == 16:
if val != "":
self.film["airedepoch"] = int(val)
elif self.index == 18:
self.film["geo"] = val
self.index = self.index + 1
def _make_url(self, val):
parts = val.split('|')
if len(parts) == 2:
cnt = int(parts[0])
return self.film["url_video"][:cnt] + parts[1]
else:
return val
def _decompress_bz2(self, sourcefile, destfile):
blocksize = 8192
try:
with open(destfile, 'wb') as dstfile, open(sourcefile,
'rb') as srcfile:
decompressor = bz2.BZ2Decompressor()
for data in iter(lambda: srcfile.read(blocksize), b''):
dstfile.write(decompressor.decompress(data))
# pylint: disable=broad-except
except Exception as err:
self.logger.error('bz2 decompression failed: {}'.format(err))
return -1
return 0
def _decompress_gz(self, sourcefile, destfile):
"""
blocksize = 8192
try:
with open(destfile, 'wb') as dstfile, gzip.open(sourcefile) as srcfile:
for data in iter(lambda: srcfile.read(blocksize), b''):
dstfile.write(data)
# pylint: disable=broad-except
except Exception as err:
self.logger.error('gz decompression of "{}" to "{}" failed: {}'.format(
sourcefile, destfile, err))
return -1
return 0
"""
blocksize = 8192
# pylint: disable=broad-except,line-too-long
try:
srcfile = gzip.open(sourcefile)
except Exception as err:
self.logger.error(
'gz decompression of "{}" to "{}" failed on opening gz file: {}'
.format(sourcefile, destfile, err))
return -1
try:
dstfile = open(destfile, 'wb')
except Exception as err:
self.logger.error(
'gz decompression of "{}" to "{}" failed on opening destination file: {}'
.format(sourcefile, destfile, err))
return -1
try:
for data in iter(lambda: srcfile.read(blocksize), b''):
try:
dstfile.write(data)
except Exception as err:
self.logger.error(
'gz decompression of "{}" to "{}" failed on writing destination file: {}'
.format(sourcefile, destfile, err))
return -1
except Exception as err:
self.logger.error(
'gz decompression of "{}" to "{}" failed on reading gz file: {}'
.format(sourcefile, destfile, err))
return -1
return 0
class MediathekView(KodiPlugin):
def __init__(self):
super(MediathekView, self).__init__()
self.settings = Settings()
self.notifier = Notifier()
self.db = Store(self.getNewLogger('Store'), self.notifier,
self.settings)
def showMainMenu(self):
# Search
self.addFolderItem(30901, {'mode': "search"})
# Search all
self.addFolderItem(30902, {'mode': "searchall"})
# Browse livestreams
self.addFolderItem(30903, {'mode': "livestreams"})
# Browse recently added
self.addFolderItem(30904, {'mode': "recent", 'channel': 0})
# Browse recently added by channel
self.addFolderItem(30905, {'mode': "recentchannels"})
# Browse by Initial->Show
self.addFolderItem(30906, {'mode': "initial", 'channel': 0})
# Browse by Channel->Initial->Shows
self.addFolderItem(30907, {'mode': "channels"})
# Database Information
self.addActionItem(30908, {'mode': "action-dbinfo"})
# Manual database update
if self.settings.updmode == 1 or self.settings.updmode == 2:
self.addActionItem(30909, {'mode': "action-dbupdate"})
self.endOfDirectory()
self._check_outdate()
def showSearch(self, extendedsearch=False):
settingid = 'lastsearch2' if extendedsearch is True else 'lastsearch1'
headingid = 30902 if extendedsearch is True else 30901
# are we returning from playback ?
searchText = self.addon.getSetting(settingid)
if len(searchText) > 0:
# restore previous search
self.db.Search(searchText, FilmUI(self), extendedsearch)
else:
# enter search term
searchText = self.notifier.GetEnteredText('', headingid)
if len(searchText) > 2:
if self.db.Search(searchText, FilmUI(self),
extendedsearch) > 0:
self.addon.setSetting(settingid, searchText)
else:
self.info(
'The following ERROR can be ignored. It is caused by the architecture of the Kodi Plugin Engine'
)
self.endOfDirectory(False, cacheToDisc=True)
# self.showMainMenu()
def showDbInfo(self):
info = self.db.GetStatus()
heading = self.language(30907)
infostr = self.language({
'NONE': 30941,
'UNINIT': 30942,
'IDLE': 30943,
'UPDATING': 30944,
'ABORTED': 30945
}.get(info['status'], 30941))
infostr = self.language(30965) % infostr
totinfo = self.language(30971) % (info['tot_chn'], info['tot_shw'],
info['tot_mov'])
updatetype = self.language(30972 if info['fullupdate'] > 0 else 30973)
if info['status'] == 'UPDATING' and info['filmupdate'] > 0:
updinfo = self.language(30967) % (
updatetype, datetime.datetime.fromtimestamp(
info['filmupdate']).strftime('%Y-%m-%d %H:%M:%S'),
info['add_chn'], info['add_shw'], info['add_mov'])
elif info['status'] == 'UPDATING':
updinfo = self.language(30968) % (updatetype, info['add_chn'],
info['add_shw'], info['add_mov'])
elif info['lastupdate'] > 0 and info['filmupdate'] > 0:
updinfo = self.language(30969) % (
updatetype, datetime.datetime.fromtimestamp(
info['lastupdate']).strftime('%Y-%m-%d %H:%M:%S'),
datetime.datetime.fromtimestamp(
info['filmupdate']).strftime('%Y-%m-%d %H:%M:%S'),
info['add_chn'], info['add_shw'], info['add_mov'],
info['del_chn'], info['del_shw'], info['del_mov'])
elif info['lastupdate'] > 0:
updinfo = self.language(30970) % (
updatetype, datetime.datetime.fromtimestamp(
info['lastupdate']).strftime('%Y-%m-%d %H:%M:%S'),
info['add_chn'], info['add_shw'], info['add_mov'],
info['del_chn'], info['del_shw'], info['del_mov'])
else:
updinfo = self.language(30966)
xbmcgui.Dialog().textviewer(
heading, infostr + '\n\n' + totinfo + '\n\n' + updinfo)
def doDownloadFilm(self, filmid, quality):
if self.settings.downloadpath:
film = self.db.RetrieveFilmInfo(filmid)
if film is None:
# film not found - should never happen
return
# check if the download path is reachable
if not xbmcvfs.exists(self.settings.downloadpath):
self.notifier.ShowError(self.language(30952),
self.language(30979))
return
# get the best url
if quality == '0' and film.url_video_sd:
videourl = film.url_video_sd
elif quality == '2' and film.url_video_hd:
videourl = film.url_video_hd
else:
videourl = film.url_video
# prepare names
showname = mvutils.cleanup_filename(film.show)[:64]
filestem = mvutils.cleanup_filename(film.title)[:64]
extension = os.path.splitext(videourl)[1]
if not extension:
extension = u'.mp4'
if not filestem:
filestem = u'Film-{}'.format(film.id)
if not showname:
showname = filestem
# prepare download directory and determine episode number
dirname = self.settings.downloadpath + showname + '/'
episode = 1
if xbmcvfs.exists(dirname):
(
_,
epfiles,
) = xbmcvfs.listdir(dirname)
for epfile in epfiles:
match = re.search('^.* [eE][pP]([0-9]*)\.[^/]*$', epfile)
if match and len(match.groups()) > 0:
if episode <= int(match.group(1)):
episode = int(match.group(1)) + 1
else:
xbmcvfs.mkdir(dirname)
# prepare resulting filenames
fileepi = filestem + u' - EP%04d' % episode
movname = dirname + fileepi + extension
srtname = dirname + fileepi + u'.srt'
ttmname = dirname + fileepi + u'.ttml'
nfoname = dirname + fileepi + u'.nfo'
# download video
bgd = KodiBGDialog()
bgd.Create(self.language(30974), fileepi + extension)
try:
bgd.Update(0)
mvutils.url_retrieve_vfs(videourl, movname,
bgd.UrlRetrieveHook)
bgd.Close()
self.notifier.ShowNotification(
30960,
self.language(30976).format(videourl))
except Exception as err:
bgd.Close()
self.error('Failure downloading {}: {}', videourl, err)
self.notifier.ShowError(
30952,
self.language(30975).format(videourl, err))
# download subtitles
if film.url_sub:
bgd = KodiBGDialog()
bgd.Create(30978, fileepi + u'.ttml')
try:
bgd.Update(0)
mvutils.url_retrieve_vfs(film.url_sub, ttmname,
bgd.UrlRetrieveHook)
try:
ttml2srt(xbmcvfs.File(ttmname, 'r'),
xbmcvfs.File(srtname, 'w'))
except Exception as err:
self.info('Failed to convert to srt: {}', err)
bgd.Close()
except Exception as err:
bgd.Close()
self.error('Failure downloading {}: {}', film.url_sub, err)
# create NFO Files
self._make_nfo_files(film, episode, dirname, nfoname, videourl)
else:
self.notifier.ShowError(30952, 30958)
def doEnqueueFilm(self, filmid):
self.info('Enqueue {}', filmid)
def _check_outdate(self, maxage=172800):
if self.settings.updmode != 1 and self.settings.updmode != 2:
# no check with update disabled or update automatic
return
if self.db is None:
# should never happen
self.notifier.ShowOutdatedUnknown()
return
status = self.db.GetStatus()
if status['status'] == 'NONE' or status['status'] == 'UNINIT':
# should never happen
self.notifier.ShowOutdatedUnknown()
return
elif status['status'] == 'UPDATING':
# great... we are updating. nuthin to show
return
# lets check how old we are
tsnow = int(time.time())
tsold = int(status['lastupdate'])
if tsnow - tsold > maxage:
self.notifier.ShowOutdatedKnown(status)
def _make_nfo_files(self, film, episode, dirname, filename, videourl):
# create NFO files
if not xbmcvfs.exists(dirname + 'tvshow.nfo'):
try:
with closing(xbmcvfs.File(dirname + 'tvshow.nfo',
'w')) as file:
file.write(b'<tvshow>\n')
file.write(b'<id></id>\n')
file.write(
bytearray('\t<title>{}</title>\n'.format(film.show),
'utf-8'))
file.write(
bytearray(
'\t<sorttitle>{}</sorttitle>\n'.format(film.show),
'utf-8'))
# TODO: file.write( bytearray( '\t<year>{}</year>\n'.format( 2018 ), 'utf-8' ) )
file.write(
bytearray(
'\t<studio>{}</studio>\n'.format(film.channel),
'utf-8'))
file.write(b'</tvshow>\n')
except Exception as err:
self.error('Failure creating show NFO file for {}: {}',
videourl, err)
try:
with closing(xbmcvfs.File(filename, 'w')) as file:
file.write(b'<episodedetails>\n')
file.write(
bytearray('\t<title>{}</title>\n'.format(film.title),
'utf-8'))
file.write(b'\t<season>1</season>\n')
file.write(b'\t<autonumber>1</autonumber>\n')
file.write(
bytearray('\t<episode>{}</episode>\n'.format(episode),
'utf-8'))
file.write(
bytearray(
'\t<showtitle>{}</showtitle>\n'.format(film.show),
'utf-8'))
file.write(
bytearray('\t<plot>{}</plot>\n'.format(film.description),
'utf-8'))
file.write(
bytearray('\t<aired>{}</aired>\n'.format(film.aired),
'utf-8'))
if film.seconds > 60:
file.write(
bytearray(
'\t<runtime>{}</runtime>\n'.format(
int(film.seconds / 60)), 'utf-8'))
file.write(
bytearray('\t<studio>{}</studio\n'.format(film.channel),
'utf-8'))
file.write(b'</episodedetails>\n')
except Exception as err:
self.error('Failure creating episode NFO file for {}: {}',
videourl, err)
def Init(self):
self.args = urlparse.parse_qs(sys.argv[2][1:])
self.db.Init()
if self.settings.HandleFirstRun():
# TODO: Implement Issue #16
pass
def Do(self):
# save last activity timestamp
self.settings.ResetUserActivity()
# process operation
mode = self.args.get('mode', None)
if mode is None:
self.showMainMenu()
elif mode[0] == 'search':
self.showSearch()
elif mode[0] == 'searchall':
self.showSearch(extendedsearch=True)
elif mode[0] == 'livestreams':
self.db.GetLiveStreams(FilmUI(self,
[xbmcplugin.SORT_METHOD_LABEL]))
elif mode[0] == 'recent':
channel = self.args.get('channel', [0])
self.db.GetRecents(channel[0], FilmUI(self))
elif mode[0] == 'recentchannels':
self.db.GetRecentChannels(ChannelUI(self, nextdir='recent'))
elif mode[0] == 'channels':
self.db.GetChannels(ChannelUI(self, nextdir='shows'))
elif mode[0] == 'action-dbinfo':
self.showDbInfo()
elif mode[0] == 'action-dbupdate':
self.settings.TriggerUpdate()
self.notifier.ShowNotification(30963, 30964, time=10000)
elif mode[0] == 'initial':
channel = self.args.get('channel', [0])
self.db.GetInitials(channel[0], InitialUI(self))
elif mode[0] == 'shows':
channel = self.args.get('channel', [0])
initial = self.args.get('initial', [None])
self.db.GetShows(channel[0], initial[0], ShowUI(self))
elif mode[0] == 'films':
show = self.args.get('show', [0])
self.db.GetFilms(show[0], FilmUI(self))
elif mode[0] == 'download':
filmid = self.args.get('id', [0])
quality = self.args.get('quality', [1])
self.doDownloadFilm(filmid[0], quality[0])
elif mode[0] == 'enqueue':
self.doEnqueueFilm(self.args.get('id', [0])[0])
# cleanup saved searches
if mode is None or mode[0] != 'search':
self.addon.setSetting('lastsearch1', '')
if mode is None or mode[0] != 'searchall':
self.addon.setSetting('lastsearch2', '')
def Exit(self):
self.db.Exit()