def migrateExtendedSearch(self):
import resources.lib.mvutils as mvutils
oldExtSearchFilename = os.path.join(
appContext.MVSETTINGS.getDatapath(), 'recent_ext_searches.json')
self.logger.debug("migrateExtendedSearch {}",
mvutils.file_exists(oldExtSearchFilename))
if mvutils.file_exists(oldExtSearchFilename):
oldData = mvutils.loadJsonFile(oldExtSearchFilename)
self.logger.debug("Found legacy ext search entries to be migrated")
if (oldData != None):
newData = []
lid = int(time.time())
for entry in oldData:
esm = ExtendedSearchModel.ExtendedSearchModel(
entry.get('search'))
esm.setId(lid)
lid += 1
esm.setTitle(entry.get('search'))
esm.setDescription(entry.get('search'))
esm.setWhen(int(entry.get('when')))
newData.append(esm.toDict())
#
newExtSearchFilename = os.path.join(
appContext.MVSETTINGS.getDatapath(), 'searchConfig.json')
mvutils.saveJsonFile(newExtSearchFilename, newData)
self.logger.debug(
"Migrated {} legacy ext search entries to new format",
len(newData))
#
oldExtSearchFilenameBackup = os.path.join(
appContext.MVSETTINGS.getDatapath(),
'recent_ext_searches.json.bk')
mvutils.file_rename(oldExtSearchFilename,
oldExtSearchFilenameBackup)
def Init(self, reset, convert):
self.logger.info(
'Using SQLite version {}, python library sqlite3 version {}',
sqlite3.sqlite_version, sqlite3.version)
if not mvutils.dir_exists(self.settings.datapath):
os.mkdir(self.settings.datapath)
# remove old versions
mvutils.file_remove(
os.path.join(self.settings.datapath, 'filmliste-v1.db'))
if reset == True or not mvutils.file_exists(self.dbfile):
self.logger.info(
'===== RESET: Database will be deleted and regenerated =====')
mvutils.file_remove(self.dbfile)
self.conn = sqlite3.connect(self.dbfile, timeout=60)
self._handle_database_initialization()
else:
try:
self.conn = sqlite3.connect(self.dbfile, timeout=60)
except sqlite3.DatabaseError as err:
self.logger.error(
'Error while opening database: {}. trying to fully reset the Database...',
err)
return self.Init(reset=True, convert=convert)
self.conn.execute(
'pragma journal_mode=off') # 3x speed-up, check mode 'WAL'
self.conn.execute('pragma synchronous=off'
) # that is a bit dangerous :-) but faaaast
self.conn.create_function('UNIX_TIMESTAMP', 0, UNIX_TIMESTAMP)
self.conn.create_aggregate('GROUP_CONCAT', 1, GROUP_CONCAT)
return True
def _handle_update_substitution(self):
updfile = os.path.join(self.settings.datapath, DATABASE_AKT)
sqlfile = os.path.join(self.settings.datapath, 'filmliste-v2.db')
if mvutils.file_exists(updfile):
self.logger.info('Native update file found. Updating database...')
return mvutils.file_rename(updfile, sqlfile)
return False
def _file_remove( self, name ):
if mvutils.file_exists( name ):
try:
os.remove( name )
return True
except OSError as err:
self.logger.error( 'Failed to remove {}: error {}', name, err )
return False
def _file_remove(self, name):
if mvutils.file_exists(name):
try:
os.remove(name)
return True
except OSError as err:
self.logger.error('Failed to remove {}: error {}', name, err)
return False
def init(self, reset=False, convert=False):
"""
Startup of the database system
Args:
reset(bool, optional): if `True` the database
will be cleaned up and recreated. Default
is `False`
convert(bool, optional): if `True` the database
will be converted in case it is older than
the supported version. If `False` a UI message
will be displayed to the user informing that
the database will be converted. Default is
`False`
"""
self.logger.info(
'Using SQLite version {}, python library sqlite3 version {}',
sqlite3.sqlite_version, sqlite3.version)
if not mvutils.dir_exists(self.settings.datapath):
os.mkdir(self.settings.datapath)
# remove old versions
mvutils.file_remove(
os.path.join(self.settings.datapath, 'filmliste-v1.db'))
if reset is True or not mvutils.file_exists(self.dbfile):
self.logger.info(
'===== RESET: Database will be deleted and regenerated =====')
self.exit()
mvutils.file_remove(self.dbfile)
if self._handle_update_substitution():
self.conn = sqlite3.connect(self.dbfile, timeout=60)
else:
self.conn = sqlite3.connect(self.dbfile, timeout=60)
self._handle_database_initialization()
else:
self._handle_update_substitution()
try:
self.conn = sqlite3.connect(self.dbfile, timeout=60)
except sqlite3.DatabaseError as err:
self.logger.error(
'Error while opening database: {}. trying to fully reset the Database...',
err)
return self.init(reset=True, convert=convert)
# 3x speed-up, check mode 'WAL'
self.conn.execute('pragma journal_mode=off')
# that is a bit dangerous :-) but faaaast
self.conn.execute('pragma synchronous=off')
self.conn.create_function('UNIX_TIMESTAMP', 0, get_unix_timestamp)
self.conn.create_aggregate('GROUP_CONCAT', 1, GroupConcatClass)
return True
def getConnection(self):
if self.conn is None:
if (not mvutils.file_exists(self.dbfile)):
self.settings.setDatabaseStatus('UNINIT')
self.logger.debug('Missing StoreSQLite DBFile: {}',
self.dbfile)
self.conn = sqlite3.connect(self.dbfile, timeout=60)
self.conn.execute('pragma synchronous=off')
self.conn.execute('pragma journal_mode=off')
self.conn.execute('pragma page_size=16384')
self.conn.execute('pragma encoding="UTF-8"')
return self.conn
def getConnection(self):
if self.conn is None:
if (not mvutils.file_exists(self.dbfile)):
self.settings.setDatabaseStatus('UNINIT')
self.logger.debug('Missing StoreSQLite DBFile: {}',
self.dbfile)
self.conn = sqlite3.connect(self.dbfile, timeout=60)
self.conn.execute('pragma synchronous=off')
self.conn.execute('pragma journal_mode=off')
self.conn.execute('pragma page_size=65536')
self.conn.execute('pragma encoding="UTF-8"')
self.conn.create_function('UNIX_TIMESTAMP', 0, get_unix_timestamp)
self.conn.create_aggregate('GROUP_CONCAT', 1, GroupConcatClass)
return self.conn
def load_cache(self, reqtype, condition):
start = time.time()
if not self.settings.getCaching():
self.logger.debug('loading cache is disabled')
return None
#
filename = os.path.join(self.settings.getDatapath(),
reqtype + '.cache')
if not mvutils.file_exists(filename):
self.logger.debug('no cache file request "{}" and condition "{}"',
reqtype, condition)
return None
#
dbLastUpdate = self.settings.getLastUpdate()
try:
with closing(open(filename, encoding='utf-8')) as json_file:
data = json.load(json_file)
if isinstance(data, dict):
if data.get('type', '') != reqtype:
self.logger.debug(
'no matching cache for type {} vs {}',
data.get('type', ''), reqtype)
return None
if data.get('condition', '') != condition:
self.logger.debug(
'no matching cache for condition {} vs {}',
data.get('condition', ''), condition)
return None
if int(dbLastUpdate) != data.get('time', 0):
self.logger.debug('outdated cache')
return None
data = data.get('data', [])
if isinstance(data, list):
self.logger.debug(
'return cache after {} sec for request "{}" and condition "{}"',
(time.time() - start), reqtype, condition)
return data
# pylint: disable=broad-except
except Exception as err:
self.logger.error('Failed to load cache file {}: {}', filename,
err)
mvutils.file_remove(filename)
raise
self.logger.debug('no cache found')
return None
def Init( self, reset = False ):
self.logger.info( 'Using SQLite version {}, python library sqlite3 version {}', sqlite3.sqlite_version, sqlite3.version )
if not mvutils.dir_exists( self.settings.datapath ):
os.mkdir( self.settings.datapath )
if reset == True or not mvutils.file_exists( self.dbfile ):
self.logger.info( '===== RESET: Database will be deleted and regenerated =====' )
self._file_remove( self.dbfile )
self.conn = sqlite3.connect( self.dbfile, timeout = 60 )
self._handle_database_initialization()
else:
try:
self.conn = sqlite3.connect( self.dbfile, timeout = 60 )
except sqlite3.DatabaseError as err:
self.logger.error( 'Error while opening database: {}. trying to fully reset the Database...', err )
self.Init( reset = True )
self.conn.execute( 'pragma journal_mode=off' ) # 3x speed-up, check mode 'WAL'
self.conn.execute( 'pragma synchronous=off' ) # that is a bit dangerous :-) but faaaast
self.conn.create_function( 'UNIX_TIMESTAMP', 0, UNIX_TIMESTAMP )
self.conn.create_aggregate( 'GROUP_CONCAT', 1, GROUP_CONCAT )
def import_database(self, full):
"""
Performs a database update when a
downloaded update file is available
Args:
full(bool): Perform full update if `True`
"""
(_, _, destfile, avgrecsize) = self._get_update_info(full)
if not mvutils.file_exists(destfile):
self.logger.error('File {} does not exists', destfile)
return False
# estimate number of records in update file
fileSizeInByte = mvutils.file_size(destfile)
records = int(fileSizeInByte / avgrecsize)
self.logger.info( 'Starting import of {} records from {}', records, destfile )
if not self.database.ft_init():
self.logger.warn(
'Failed to initialize update. Maybe a concurrency problem?')
return False
# pylint: disable=broad-except
try:
starttime = time.time()
flsm = 0
flts = 0
####
flsm = 0
sender = ""
thema = ""
self.notifier.show_update_progress()
(self.tot_chn, self.tot_shw, self.tot_mov) = self._update_start(full)
ufp = UpdateFileParser.UpdateFileParser(self.logger, 512000, destfile)
ufp.init()
fileHeader = ufp.next(',"X":');
### META
## {"Filmliste":["30.08.2020, 11:13","30.08.2020, 09:13","3","MSearch [Vers.: 3.1.139]","d93c9794acaf3e482d42c24e513f78a8"],"Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"]
# this is the timestamp of this database update
#value = jsonDoc['Filmliste'][0]
value = fileHeader[15:32]
#self.logger.info( 'update date ' + value )
try:
fldt = datetime.datetime.strptime(
value.strip(), "%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.database.update_status(filmupdate=flts)
self.logger.info(
'Filmliste dated {}', value.strip())
except TypeError:
# pylint: disable=line-too-long
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(time.mktime(time.strptime(
value.strip(), "%d.%m.%Y, %H:%M")))
self.database.update_status(
filmupdate=flts)
self.logger.info(
'Filmliste dated {}', value.strip())
# pylint: disable=broad-except
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}', value.strip(), err)
except ValueError as err:
pass
###
while (True):
aPart = ufp.next(',"X":');
if (len(aPart) == 0):
break;
##
aPart = '{"X":' + aPart;
if (not(aPart.endswith("}"))):
aPart = aPart + "}";
##
jsonDoc = json.loads(aPart)
jsonDoc = jsonDoc['X']
self._init_record()
# behaviour of the update list
if (len(jsonDoc[0]) > 0):
sender = jsonDoc[0]
else:
jsonDoc[0] = sender
# same for thema
if (len(jsonDoc[1]) > 0):
thema = jsonDoc[1]
else:
jsonDoc[1] = thema
##
self._add_value( jsonDoc )
self._end_record(records)
if self.count % 100 == 0 and self.monitor.abort_requested():
# kodi is shutting down. Close all
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return True
ufp.close()
self._update_end(full, 'IDLE')
self.logger.info('{} records processed',self.count)
self.logger.info(
'Import of {} in update cycle {} finished. Duration: {} seconds',
destfile,
self.cycle,
int(time.time() - starttime)
)
self.notifier.close_update_progress()
return True
except KeyboardInterrupt:
self._update_end(full, 'ABORTED')
self.logger.info('Update cycle {} interrupted by user', self.cycle)
self.notifier.close_update_progress()
return False
except DatabaseCorrupted as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except DatabaseLost as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except Exception as err:
self.logger.error(
'Error {} while processing {} on update cycle {}', err, destfile, self.cycle)
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return False
def Import( self, full ):
( _, _, destfile, avgrecsize ) = self._get_update_info( full )
if not mvutils.file_exists( destfile ):
self.logger.error( 'File {} does not exists', destfile )
return False
# estimate number of records in update file
records = int( mvutils.file_size( destfile ) / avgrecsize )
if not self.db.ftInit():
self.logger.warn( 'Failed to initialize update. Maybe a concurrency problem?' )
return False
try:
self.logger.info( 'Starting import of approx. {} records from {}', records, destfile )
with open( destfile, 'r' ) as file:
parser = ijson.parse( file )
flsm = 0
flts = 0
( self.tot_chn, self.tot_shw, self.tot_mov ) = self._update_start( full )
self.notifier.ShowUpdateProgress()
for prefix, event, value in parser:
if ( prefix, event ) == ( "X", "start_array" ):
self._init_record()
elif ( prefix, event ) == ( "X", "end_array" ):
self._end_record( records )
if self.count % 100 == 0 and self.monitor.abortRequested():
# kodi is shutting down. Close all
self._update_end( full, 'ABORTED' )
self.notifier.CloseUpdateProgress()
return True
elif ( prefix, event ) == ( "X.item", "string" ):
if value is not None:
# self._add_value( value.strip().encode('utf-8') )
self._add_value( value.strip() )
else:
self._add_value( "" )
elif ( prefix, event ) == ( "Filmliste", "start_array" ):
flsm += 1
elif ( prefix, event ) == ( "Filmliste.item", "string" ):
flsm += 1
if flsm == 2 and value is not None:
# this is the timestmap of this database update
try:
fldt = datetime.datetime.strptime( value.strip(), "%d.%m.%Y, %H:%M" )
flts = int( time.mktime( fldt.timetuple() ) )
self.db.UpdateStatus( filmupdate = flts )
self.logger.info( 'Filmliste dated {}', value.strip() )
except TypeError:
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int( time.mktime( time.strptime( value.strip(), "%d.%m.%Y, %H:%M" ) ) )
self.db.UpdateStatus( filmupdate = flts )
self.logger.info( 'Filmliste dated {}', value.strip() )
except Exception as err:
# If the universe hates us...
self.logger.debug( 'Could not determine date "{}" of filmliste: {}', value.strip(), err )
except ValueError as err:
pass
self._update_end( full, 'IDLE' )
self.logger.info( 'Import of {} finished', destfile )
self.notifier.CloseUpdateProgress()
return True
except KeyboardInterrupt:
self._update_end( full, 'ABORTED' )
self.logger.info( 'Interrupted by user' )
self.notifier.CloseUpdateProgress()
return True
except DatabaseCorrupted as err:
self.logger.error( '{}', err )
self.notifier.CloseUpdateProgress()
except DatabaseLost as err:
self.logger.error( '{}', err )
self.notifier.CloseUpdateProgress()
except Exception as err:
self.logger.error( 'Error {} wile processing {}', err, destfile )
self._update_end( full, 'ABORTED' )
self.notifier.CloseUpdateProgress()
return False
def GetNewestList( self, full ):
( url, compfile, destfile, _ ) = self._get_update_info( full )
if url is None:
self.logger.error( 'No suitable archive extractor available for this system' )
self.notifier.ShowMissingExtractorError()
return False
# get mirrorlist
self.logger.info( 'Opening {}', url )
try:
data = urllib2.urlopen( url ).read()
except urllib2.URLError as err:
self.logger.error( 'Failure opening {}', url )
self.notifier.ShowDownloadError( url, err )
return False
root = etree.fromstring ( data )
urls = []
for server in root.findall( 'Server' ):
try:
URL = server.find( 'URL' ).text
Prio = server.find( 'Prio' ).text
urls.append( ( self._get_update_url( URL ), float( Prio ) + random.random() * 1.2 ) )
self.logger.info( 'Found mirror {} (Priority {})', URL, Prio )
except AttributeError:
pass
urls = sorted( urls, key = itemgetter( 1 ) )
urls = [ url[0] for url in urls ]
# cleanup downloads
self.logger.info( 'Cleaning up old downloads...' )
self._file_remove( compfile )
self._file_remove( destfile )
# download filmliste
self.notifier.ShowDownloadProgress()
lasturl = ''
for url in urls:
try:
lasturl = url
self.logger.info( 'Trying to download {} from {}...', os.path.basename( compfile ), url )
self.notifier.UpdateDownloadProgress( 0, url )
mvutils.url_retrieve( url, filename = compfile, reporthook = self.notifier.HookDownloadProgress, aborthook = self.monitor.abortRequested )
break
except urllib2.URLError as err:
self.logger.error( 'Failure downloading {}', url )
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError( lasturl, err )
return False
except ExitRequested as err:
self.logger.error( 'Immediate exit requested. Aborting download of {}', url )
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError( lasturl, err )
return False
except Exception as err:
self.logger.error( 'Failure writng {}', url )
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError( lasturl, err )
return False
# decompress filmliste
if self.use_xz is True:
self.logger.info( 'Trying to decompress xz file...' )
retval = subprocess.call( [ mvutils.find_xz(), '-d', compfile ] )
self.logger.info( 'Return {}', retval )
elif upd_can_bz2 is True:
self.logger.info( 'Trying to decompress bz2 file...' )
retval = self._decompress_bz2( compfile, destfile )
self.logger.info( 'Return {}', retval )
elif upd_can_gz is True:
self.logger.info( 'Trying to decompress gz file...' )
retval = self._decompress_gz( compfile, destfile )
self.logger.info( 'Return {}', retval )
else:
# should nebver reach
pass
self.notifier.CloseDownloadProgress()
return retval == 0 and mvutils.file_exists( destfile )
def get_newest_list(self, full):
"""
Downloads the database update file
Args:
full(bool): Downloads the full list if `True`
"""
(url, compfile, destfile, _) = self._get_update_info(full)
if url is None:
self.logger.error(
'No suitable archive extractor available for this system')
self.notifier.show_missing_extractor_error()
return False
# cleanup downloads
self.logger.info('Cleaning up old downloads...')
mvutils.file_remove(compfile)
mvutils.file_remove(destfile)
# download filmliste
self.notifier.show_download_progress()
# pylint: disable=broad-except
try:
self.logger.info('Trying to download {} from {}...',
os.path.basename(compfile), url)
self.notifier.update_download_progress(0, url)
mvutils.url_retrieve(
url,
filename=compfile,
reporthook=self.notifier.hook_download_progress,
aborthook=self.monitor.abort_requested
)
except URLError as err:
self.logger.error('Failure downloading {} - {}', url, err)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
return False
except ExitRequested as err:
self.logger.error(
'Immediate exit requested. Aborting download of {}', url)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
return False
except Exception as err:
self.logger.error('Failure writing {}', url)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
return False
# decompress filmliste
if self.use_xz is True:
self.logger.info('Trying to decompress xz file...')
retval = subprocess.call([mvutils.find_xz(), '-d', compfile])
self.logger.info('Return {}', retval)
elif UPD_CAN_BZ2 is True:
self.logger.info('Trying to decompress bz2 file...')
retval = self._decompress_bz2(compfile, destfile)
self.logger.info('Return {}', retval)
elif UPD_CAN_GZ is True:
self.logger.info('Trying to decompress gz file...')
retval = self._decompress_gz(compfile, destfile)
self.logger.info('Return {}', retval)
else:
# should never reach
pass
self.notifier.close_download_progress()
return retval == 0 and mvutils.file_exists(destfile)
def import_database(self, full):
"""
Performs a database update when a
downloaded update file is available
Args:
full(bool): Perform full update if `True`
"""
(_, _, destfile, avgrecsize) = self._get_update_info(full)
if not mvutils.file_exists(destfile):
self.logger.error('File {} does not exists', destfile)
return False
# estimate number of records in update file
records = int(mvutils.file_size(destfile) / avgrecsize)
if not self.database.ft_init():
self.logger.warn(
'Failed to initialize update. Maybe a concurrency problem?')
return False
# pylint: disable=broad-except
try:
starttime = time.time()
with closing( open(destfile, 'r', encoding="utf-8") ) as updatefile:
jsonDoc = json.load( updatefile, object_pairs_hook=self._object_pairs_hook )
self.logger.info( 'Starting import of {} records from {}', (len(jsonDoc)-2), destfile )
flsm = 0
flts = 0
(self.tot_chn, self.tot_shw, self.tot_mov) = self._update_start(full)
self.notifier.show_update_progress()
####
flsm = 0
sender = ""
thema = ""
### ROOT LIST
for atuple in jsonDoc:
if (atuple[0] == 'Filmliste' and flsm == 0):
### META
### "Filmliste":["23.04.2020, 18:23","23.04.2020, 16:23","3","MSearch [Vers.: 3.1.129]","3c90946f05eb1e2fa6cf2327cca4f1d4"],
flsm +=1
# this is the timestamp of this database update
value = atuple[1][0]
try:
fldt = datetime.datetime.strptime(
value.strip(), "%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.database.update_status(filmupdate=flts)
self.logger.info(
'Filmliste dated {}', value.strip())
except TypeError:
# pylint: disable=line-too-long
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(time.mktime(time.strptime(
value.strip(), "%d.%m.%Y, %H:%M")))
self.database.update_status(
filmupdate=flts)
self.logger.info(
'Filmliste dated {}', value.strip())
# pylint: disable=broad-except
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}', value.strip(), err)
except ValueError as err:
pass
elif (atuple[0] == 'filmliste' and flsm == 1):
flsm +=1
# VOID - we do not need column names
# "Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"],
elif (atuple[0] == 'X'):
self._init_record()
# behaviour of the update list
if (len(atuple[1][0]) > 0):
sender = atuple[1][0]
else:
atuple[1][0] = sender
# same for thema
if (len(atuple[1][1]) > 0):
thema = atuple[1][1]
else:
atuple[1][1] = thema
##
self._add_value( atuple[1] )
self._end_record(records)
if self.count % 100 == 0 and self.monitor.abort_requested():
# kodi is shutting down. Close all
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return True
self._update_end(full, 'IDLE')
self.logger.info('{} records processed',self.count)
self.logger.info(
'Import of {} in update cycle {} finished. Duration: {} seconds',
destfile,
self.cycle,
int(time.time() - starttime)
)
self.notifier.close_update_progress()
return True
except KeyboardInterrupt:
self._update_end(full, 'ABORTED')
self.logger.info('Update cycle {} interrupted by user', self.cycle)
self.notifier.close_update_progress()
return False
except DatabaseCorrupted as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except DatabaseLost as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except Exception as err:
self.logger.error(
'Error {} while processing {} on update cycle {}', err, destfile, self.cycle)
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return False
def Import(self, full):
(_, _, destfile, avgrecsize) = self._get_update_info(full)
if not mvutils.file_exists(destfile):
self.logger.error('File {} does not exists', destfile)
return False
# estimate number of records in update file
records = int(mvutils.file_size(destfile) / avgrecsize)
if not self.db.ftInit():
self.logger.warn(
'Failed to initialize update. Maybe a concurrency problem?')
return False
try:
self.logger.info('Starting import of approx. {} records from {}',
records, destfile)
with open(destfile, 'r') as file:
parser = ijson.parse(file)
flsm = 0
flts = 0
(self.tot_chn, self.tot_shw,
self.tot_mov) = self._update_start(full)
self.notifier.ShowUpdateProgress()
for prefix, event, value in parser:
if (prefix, event) == ("X", "start_array"):
self._init_record()
elif (prefix, event) == ("X", "end_array"):
self._end_record(records)
if self.count % 100 == 0 and self.monitor.abortRequested(
):
# kodi is shutting down. Close all
self._update_end(full, 'ABORTED')
self.notifier.CloseUpdateProgress()
return True
elif (prefix, event) == ("X.item", "string"):
if value is not None:
self._add_value(value.strip())
else:
self._add_value("")
elif (prefix, event) == ("Filmliste", "start_array"):
flsm += 1
elif (prefix, event) == ("Filmliste.item", "string"):
flsm += 1
if flsm == 2 and value is not None:
# this is the timestmap of this database update
try:
fldt = datetime.datetime.strptime(
value.strip(), "%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.db.UpdateStatus(filmupdate=flts)
self.logger.info('Filmliste dated {}',
value.strip())
except TypeError:
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(
time.mktime(
time.strptime(
value.strip(),
"%d.%m.%Y, %H:%M")))
self.db.UpdateStatus(filmupdate=flts)
self.logger.info('Filmliste dated {}',
value.strip())
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}',
value.strip(), err)
except ValueError as err:
pass
self.db.ftFlushInsert()
self._update_end(full, 'IDLE')
self.logger.info('Import of {} in update cycle {} finished',
destfile, self.cycle)
self.notifier.CloseUpdateProgress()
return True
except KeyboardInterrupt:
self._update_end(full, 'ABORTED')
self.logger.info('Update cycle {} interrupted by user', self.cycle)
self.notifier.CloseUpdateProgress()
return False
except DatabaseCorrupted as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.CloseUpdateProgress()
except DatabaseLost as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.CloseUpdateProgress()
except Exception as err:
self.logger.error(
'Error {} while processing {} on update cycle {}', err,
destfile, self.cycle)
self._update_end(full, 'ABORTED')
self.notifier.CloseUpdateProgress()
return False
def _importFile(self, targetFilename):
#
if not mvutils.file_exists(targetFilename):
self.logger.error('File {} does not exists', targetFilename)
return False
# estimate number of records in update file
fileSizeInByte = mvutils.file_size(targetFilename)
records = int(fileSizeInByte / 600)
self.logger.info('Starting import of approximately {} records from {}',
records, targetFilename)
#
# pylint: disable=broad-except
try:
flsm = 0
flts = 0
#
sender = ""
thema = ""
self.notifier.show_update_progress()
#
ufp = UpdateFileParser.UpdateFileParser(self.logger, 512000,
targetFilename)
ufp.init()
fileHeader = ufp.next(',"X":')
# META
# {"Filmliste":["30.08.2020, 11:13","30.08.2020, 09:13","3","MSearch [Vers.: 3.1.139]","d93c9794acaf3e482d42c24e513f78a8"],"Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"]
# this is the timestamp of this database update
# value = jsonDoc['Filmliste'][0]
value = fileHeader[15:32]
# self.logger.debug( 'update date ' + value )
try:
fldt = datetime.datetime.strptime(value.strip(),
"%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.logger.debug('Filmliste dated {}', value.strip())
self.database.set_status('UPDATING', pFilmupdate=flts)
except TypeError:
# pylint: disable=line-too-long
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(
time.mktime(
time.strptime(value.strip(), "%d.%m.%Y, %H:%M")))
self.database.set_status('UPDATING', pFilmupdate=flts)
self.logger.debug('Filmliste dated {}', value.strip())
# pylint: disable=broad-except
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}',
value.strip(), err)
except ValueError as err:
pass
#
recordArray = []
#
while (True):
aPart = ufp.next(',"X":')
if (len(aPart) == 0):
break
#
aPart = '{"X":' + aPart
if (not (aPart.endswith("}"))):
aPart = aPart + "}"
#
jsonDoc = json.loads(aPart)
jsonDoc = jsonDoc['X']
self._init_record()
# behaviour of the update list
if (len(jsonDoc[0]) > 0):
sender = jsonDoc[0][:32]
else:
jsonDoc[0] = sender
# same for thema
if (len(jsonDoc[1]) > 0):
thema = jsonDoc[1][:128]
else:
jsonDoc[1] = thema
#
self.film['channel'] = sender
self.film['show'] = thema
self.film["title"] = jsonDoc[2][:128]
#
if len(jsonDoc[3]) == 10:
self.film["aired"] = jsonDoc[3][6:] + '-' + jsonDoc[3][
3:5] + '-' + jsonDoc[3][:2]
if (len(jsonDoc[4]) == 8):
self.film[
"aired"] = self.film["aired"] + " " + jsonDoc[4]
#
if len(jsonDoc[5]) > 0:
self.film["duration"] = jsonDoc[5]
if len(jsonDoc[7]) > 0:
self.film["description"] = jsonDoc[7][:1024]
self.film["url_video"] = jsonDoc[8]
self.film["website"] = jsonDoc[9]
self.film["url_sub"] = jsonDoc[10]
self.film["url_video_sd"] = self._make_url(jsonDoc[12])
self.film["url_video_hd"] = self._make_url(jsonDoc[14])
if len(jsonDoc[16]) > 0:
self.film["airedepoch"] = int(jsonDoc[16])
self.film["geo"] = jsonDoc[18]
#
# check if the movie is there
#
checkString = sender + thema + self.film["title"] + self.film[
'url_video']
idhash = hashlib.md5(checkString.encode('utf-8')).hexdigest()
#
showid = hashlib.md5(thema.encode('utf-8')).hexdigest()
showid = showid[:8]
#
recordArray.append(
(idhash, int(time.time()), self.film['channel'], showid,
self.film['show'], self.film["title"],
self.film['airedepoch'],
mvutils.make_duration(self.film['duration']),
self.film['description'], self.film['url_sub'],
self.film['url_video'], self.film['url_video_sd'],
self.film['url_video_hd']))
self.count = self.count + 1
# check
if self.count % self.settings.getDatabaseImportBatchSize(
) == 0:
if self.monitor.abort_requested():
# kodi is shutting down. Close all
self._update_end()
self.notifier.close_update_progress()
raise Exception('User requested Abort')
else:
# run insert
try:
(ai, au) = self.database.import_films(recordArray)
self.insertCount += ai
self.updateCount += au
except Exception as err:
self.logger.error('Error in data import: {}', err)
self.errorCount = self.errorCount + 1
recordArray = []
# update status
percent = int(self.count * 100 / records)
percent = percent if percent <= 100 else 100
self.logger.debug(
'In progress (%d%%): insert:%d, update:%d' %
(percent, self.insertCount, self.updateCount))
self.notifier.update_update_progress(
percent, self.count, self.insertCount,
self.updateCount)
if len(recordArray) > 0:
try:
(ai, au) = self.database.import_films(recordArray)
self.insertCount += ai
self.updateCount += au
except Exception as err:
self.logger.error('Error in data import: {}', err)
self.errorCount = self.errorCount + 1
#
ufp.close()
self.notifier.close_update_progress()
if self.errorCount > 0:
self.logger.warn('Update finished with error(s)')
except Exception as err:
self.logger.error('Error {} while processing {}', err,
targetFilename)
self._update_end()
self.database.set_status('ABORTED')
self.notifier.close_update_progress()
raise
def GetNewestList(self, full):
(url, compfile, destfile, _) = self._get_update_info(full)
if url is None:
self.logger.error(
'No suitable archive extractor available for this system')
self.notifier.ShowMissingExtractorError()
return False
# get mirrorlist
self.logger.info('Opening {}', url)
try:
data = urllib2.urlopen(url).read()
except urllib2.URLError as err:
self.logger.error('Failure opening {}', url)
self.notifier.ShowDownloadError(url, err)
return False
root = etree.fromstring(data)
urls = []
for server in root.findall('Server'):
try:
URL = server.find('URL').text
Prio = server.find('Prio').text
urls.append((self._get_update_url(URL),
float(Prio) + random.random() * 1.2))
self.logger.info('Found mirror {} (Priority {})', URL, Prio)
except AttributeError:
pass
urls = sorted(urls, key=itemgetter(1))
urls = [url[0] for url in urls]
# cleanup downloads
self.logger.info('Cleaning up old downloads...')
mvutils.file_remove(compfile)
mvutils.file_remove(destfile)
# download filmliste
self.notifier.ShowDownloadProgress()
lasturl = ''
for url in urls:
try:
lasturl = url
self.logger.info('Trying to download {} from {}...',
os.path.basename(compfile), url)
self.notifier.UpdateDownloadProgress(0, url)
mvutils.url_retrieve(
url,
filename=compfile,
reporthook=self.notifier.HookDownloadProgress,
aborthook=self.monitor.abortRequested)
break
except urllib2.URLError as err:
self.logger.error('Failure downloading {}', url)
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError(lasturl, err)
return False
except ExitRequested as err:
self.logger.error(
'Immediate exit requested. Aborting download of {}', url)
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError(lasturl, err)
return False
except Exception as err:
self.logger.error('Failure writng {}', url)
self.notifier.CloseDownloadProgress()
self.notifier.ShowDownloadError(lasturl, err)
return False
# decompress filmliste
if self.use_xz is True:
self.logger.info('Trying to decompress xz file...')
retval = subprocess.call([mvutils.find_xz(), '-d', compfile])
self.logger.info('Return {}', retval)
elif upd_can_bz2 is True:
self.logger.info('Trying to decompress bz2 file...')
retval = self._decompress_bz2(compfile, destfile)
self.logger.info('Return {}', retval)
elif upd_can_gz is True:
self.logger.info('Trying to decompress gz file...')
retval = self._decompress_gz(compfile, destfile)
self.logger.info('Return {}', retval)
else:
# should nebver reach
pass
self.notifier.CloseDownloadProgress()
return retval == 0 and mvutils.file_exists(destfile)
def _download(self, url, compressedFilename, targetFilename):
# cleanup downloads
start = time.time()
self.logger.debug('Cleaning up old downloads...')
mvutils.file_remove(compressedFilename)
mvutils.file_remove(targetFilename)
#
# download filmliste
self.notifier.show_download_progress()
# pylint: disable=broad-except
try:
self.logger.debug('Trying to download {} from {}...',
os.path.basename(compressedFilename), url)
self.notifier.update_download_progress(0, url)
mvutils.url_retrieve(
url,
filename=compressedFilename,
reporthook=self.notifier.hook_download_progress,
aborthook=self.monitor.abort_requested)
self.logger.debug('downloaded {} in {} sec', compressedFilename,
(time.time() - start))
except URLError as err:
self.logger.error('Failure downloading {} - {}', url, err)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
raise
except ExitRequested as err:
self.logger.error(
'Immediate exit requested. Aborting download of {}', url)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
raise
except Exception as err:
self.logger.error('Failure writing {}', url)
self.notifier.close_download_progress()
self.notifier.show_download_error(url, err)
raise
# decompress filmliste
start = time.time()
try:
if self.use_xz is True:
self.logger.debug('Trying to decompress xz file...')
retval = subprocess.call(
[mvutils.find_xz(), '-d', compressedFilename])
self.logger.debug('decompress xz {} in {} sec', retval,
(time.time() - start))
elif UPD_CAN_BZ2 is True:
self.logger.debug('Trying to decompress bz2 file...')
retval = self._decompress_bz2(compressedFilename,
targetFilename)
self.logger.debug('decompress bz2 {} in {} sec', retval,
(time.time() - start))
elif UPD_CAN_GZ is True:
self.logger.debug('Trying to decompress gz file...')
retval = self._decompress_gz(compressedFilename,
targetFilename)
self.logger.debug('decompress gz {} in {} sec', retval,
(time.time() - start))
else:
# should never reach
pass
except Exception as err:
self.logger.error('Failure decompress {}', err)
self.notifier.close_download_progress()
self.notifier.show_download_error('decompress failed', err)
raise
self.notifier.close_download_progress()
return retval == 0 and mvutils.file_exists(targetFilename)
def init(self, reset=False, convert=False, failedCount=0):
"""
Startup of the database system
Args:
reset(bool, optional): if `True` the database
will be cleaned up and recreated. Default
is `False`
convert(bool, optional): if `True` the database
will be converted in case it is older than
the supported version. If `False` a UI message
will be displayed to the user informing that
the database will be converted. Default is
`False`
"""
self.logger.info(
'Using SQLite version {}, python library sqlite3 version {}',
sqlite3.sqlite_version, sqlite3.version)
if not mvutils.dir_exists(self.settings.datapath):
os.mkdir(self.settings.datapath)
# remove old versions
mvutils.file_remove(
os.path.join(self.settings.datapath, 'filmliste-v1.db'))
if reset is True or not mvutils.file_exists(self.dbfile):
self.logger.info(
'===== RESET: Database will be deleted and regenerated =====')
self.exit()
mvutils.file_remove(self.dbfile)
if self._handle_update_substitution():
self.conn = sqlite3.connect(self.dbfile, timeout=60)
else:
self.conn = sqlite3.connect(self.dbfile, timeout=60)
self._handle_database_initialization()
else:
try:
if self._handle_update_substitution():
self._handle_not_update_to_date_dbfile()
self.conn = sqlite3.connect(self.dbfile, timeout=60)
except sqlite3.DatabaseError as err:
self.logger.error(
'Error while opening database: {}. trying to fully reset the Database...',
err)
return self.init(reset=True, convert=convert)
try:
# 3x speed-up, check mode 'WAL'
self.conn.execute('pragma journal_mode=off')
# check if DB is ready or broken
cursor = self.conn.cursor()
cursor.execute('SELECT * FROM `status` LIMIT 1')
rs = cursor.fetchall()
##
self.logger.info('Current DB Status Last modified {} ({})',
time.ctime(rs[0][0]), rs[0][0])
self.logger.info('Current DB Status Last lastupdate {} ({})',
time.ctime(rs[0][2]), rs[0][2])
self.logger.info('Current DB Status Last filmupdate {} ({})',
time.ctime(rs[0][3]), rs[0][3])
self.logger.info('Current DB Status Last fullupdate {}', rs[0][4])
##
cursor.close()
except sqlite3.DatabaseError as err:
failedCount += 1
if (failedCount > 3):
self.logger.error(
'Failed to restore database, please uninstall plugin, delete user profile and reinstall'
)
raise err
self.logger.error(
'Error on first query: {}. trying to fully reset the Database...trying {} times',
err, failedCount)
return self.init(reset=True,
convert=convert,
failedCount=failedCount)
# that is a bit dangerous :-) but faaaast
self.conn.execute('pragma synchronous=off')
self.conn.create_function('UNIX_TIMESTAMP', 0, get_unix_timestamp)
self.conn.create_aggregate('GROUP_CONCAT', 1, GroupConcatClass)
return True
