def downloadFullUpdateFile(self):
#
ext = self._getExtension()
downloadUrl = FILMLISTE_URL + FILMLISTE_AKT + ext
self._compressedFilename = os.path.join(self.settings.getDatapath(),
FILMLISTE_AKT + ext)
self._filename = os.path.join(self.settings.getDatapath(),
FILMLISTE_AKT)
#
check = self._download(downloadUrl, self._compressedFilename,
self._filename)
#
if check:
filesize = mvutils.file_size(self._filename)
if filesize < 200000000:
raise Exception(
'FullUpdate file size {} smaller than allowed (200MB)'.
format(filesize))
#
return check
def downloadSqliteDb(self):
ext = self._getExtension()
downloadUrl = DATABASE_URL + DATABASE_DBF + ext
self._compressedFilename = os.path.join(self.settings.getDatapath(),
'tmp_' + DATABASE_DBF + ext)
self._filename = os.path.join(self.settings.getDatapath(),
'tmp_' + DATABASE_DBF)
self._Dbfilename = os.path.join(self.settings.getDatapath(),
DATABASE_DBF)
#
check = self._download(downloadUrl, self._compressedFilename,
self._filename)
#
if check:
filesize = mvutils.file_size(self._filename)
if filesize < 200000000:
raise Exception(
'FullUpdate file size {} smaller than allowed (200MB)'.
format(filesize))
#
return check
def Import( self, full ):
( _, _, destfile, avgrecsize ) = self._get_update_info( full )
if not mvutils.file_exists( destfile ):
self.logger.error( 'File {} does not exists', destfile )
return False
# estimate number of records in update file
records = int( mvutils.file_size( destfile ) / avgrecsize )
if not self.db.ftInit():
self.logger.warn( 'Failed to initialize update. Maybe a concurrency problem?' )
return False
try:
self.logger.info( 'Starting import of approx. {} records from {}', records, destfile )
with open( destfile, 'r' ) as file:
parser = ijson.parse( file )
flsm = 0
flts = 0
( self.tot_chn, self.tot_shw, self.tot_mov ) = self._update_start( full )
self.notifier.ShowUpdateProgress()
for prefix, event, value in parser:
if ( prefix, event ) == ( "X", "start_array" ):
self._init_record()
elif ( prefix, event ) == ( "X", "end_array" ):
self._end_record( records )
if self.count % 100 == 0 and self.monitor.abortRequested():
# kodi is shutting down. Close all
self._update_end( full, 'ABORTED' )
self.notifier.CloseUpdateProgress()
return True
elif ( prefix, event ) == ( "X.item", "string" ):
if value is not None:
# self._add_value( value.strip().encode('utf-8') )
self._add_value( value.strip() )
else:
self._add_value( "" )
elif ( prefix, event ) == ( "Filmliste", "start_array" ):
flsm += 1
elif ( prefix, event ) == ( "Filmliste.item", "string" ):
flsm += 1
if flsm == 2 and value is not None:
# this is the timestmap of this database update
try:
fldt = datetime.datetime.strptime( value.strip(), "%d.%m.%Y, %H:%M" )
flts = int( time.mktime( fldt.timetuple() ) )
self.db.UpdateStatus( filmupdate = flts )
self.logger.info( 'Filmliste dated {}', value.strip() )
except TypeError:
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int( time.mktime( time.strptime( value.strip(), "%d.%m.%Y, %H:%M" ) ) )
self.db.UpdateStatus( filmupdate = flts )
self.logger.info( 'Filmliste dated {}', value.strip() )
except Exception as err:
# If the universe hates us...
self.logger.debug( 'Could not determine date "{}" of filmliste: {}', value.strip(), err )
except ValueError as err:
pass
self._update_end( full, 'IDLE' )
self.logger.info( 'Import of {} finished', destfile )
self.notifier.CloseUpdateProgress()
return True
except KeyboardInterrupt:
self._update_end( full, 'ABORTED' )
self.logger.info( 'Interrupted by user' )
self.notifier.CloseUpdateProgress()
return True
except DatabaseCorrupted as err:
self.logger.error( '{}', err )
self.notifier.CloseUpdateProgress()
except DatabaseLost as err:
self.logger.error( '{}', err )
self.notifier.CloseUpdateProgress()
except Exception as err:
self.logger.error( 'Error {} wile processing {}', err, destfile )
self._update_end( full, 'ABORTED' )
self.notifier.CloseUpdateProgress()
return False
def import_database(self, full):
"""
Performs a database update when a
downloaded update file is available
Args:
full(bool): Perform full update if `True`
"""
(_, _, destfile, avgrecsize) = self._get_update_info(full)
if not mvutils.file_exists(destfile):
self.logger.error('File {} does not exists', destfile)
return False
# estimate number of records in update file
records = int(mvutils.file_size(destfile) / avgrecsize)
if not self.database.ft_init():
self.logger.warn(
'Failed to initialize update. Maybe a concurrency problem?')
return False
# pylint: disable=broad-except
try:
starttime = time.time()
with closing( open(destfile, 'r', encoding="utf-8") ) as updatefile:
jsonDoc = json.load( updatefile, object_pairs_hook=self._object_pairs_hook )
self.logger.info( 'Starting import of {} records from {}', (len(jsonDoc)-2), destfile )
flsm = 0
flts = 0
(self.tot_chn, self.tot_shw, self.tot_mov) = self._update_start(full)
self.notifier.show_update_progress()
####
flsm = 0
sender = ""
thema = ""
### ROOT LIST
for atuple in jsonDoc:
if (atuple[0] == 'Filmliste' and flsm == 0):
### META
### "Filmliste":["23.04.2020, 18:23","23.04.2020, 16:23","3","MSearch [Vers.: 3.1.129]","3c90946f05eb1e2fa6cf2327cca4f1d4"],
flsm +=1
# this is the timestamp of this database update
value = atuple[1][0]
try:
fldt = datetime.datetime.strptime(
value.strip(), "%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.database.update_status(filmupdate=flts)
self.logger.info(
'Filmliste dated {}', value.strip())
except TypeError:
# pylint: disable=line-too-long
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(time.mktime(time.strptime(
value.strip(), "%d.%m.%Y, %H:%M")))
self.database.update_status(
filmupdate=flts)
self.logger.info(
'Filmliste dated {}', value.strip())
# pylint: disable=broad-except
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}', value.strip(), err)
except ValueError as err:
pass
elif (atuple[0] == 'filmliste' and flsm == 1):
flsm +=1
# VOID - we do not need column names
# "Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"],
elif (atuple[0] == 'X'):
self._init_record()
# behaviour of the update list
if (len(atuple[1][0]) > 0):
sender = atuple[1][0]
else:
atuple[1][0] = sender
# same for thema
if (len(atuple[1][1]) > 0):
thema = atuple[1][1]
else:
atuple[1][1] = thema
##
self._add_value( atuple[1] )
self._end_record(records)
if self.count % 100 == 0 and self.monitor.abort_requested():
# kodi is shutting down. Close all
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return True
self._update_end(full, 'IDLE')
self.logger.info('{} records processed',self.count)
self.logger.info(
'Import of {} in update cycle {} finished. Duration: {} seconds',
destfile,
self.cycle,
int(time.time() - starttime)
)
self.notifier.close_update_progress()
return True
except KeyboardInterrupt:
self._update_end(full, 'ABORTED')
self.logger.info('Update cycle {} interrupted by user', self.cycle)
self.notifier.close_update_progress()
return False
except DatabaseCorrupted as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except DatabaseLost as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except Exception as err:
self.logger.error(
'Error {} while processing {} on update cycle {}', err, destfile, self.cycle)
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return False
def import_database(self, full):
"""
Performs a database update when a
downloaded update file is available
Args:
full(bool): Perform full update if `True`
"""
(_, _, destfile, avgrecsize) = self._get_update_info(full)
if not mvutils.file_exists(destfile):
self.logger.error('File {} does not exists', destfile)
return False
# estimate number of records in update file
fileSizeInByte = mvutils.file_size(destfile)
records = int(fileSizeInByte / avgrecsize)
self.logger.info( 'Starting import of {} records from {}', records, destfile )
if not self.database.ft_init():
self.logger.warn(
'Failed to initialize update. Maybe a concurrency problem?')
return False
# pylint: disable=broad-except
try:
starttime = time.time()
flsm = 0
flts = 0
####
flsm = 0
sender = ""
thema = ""
self.notifier.show_update_progress()
(self.tot_chn, self.tot_shw, self.tot_mov) = self._update_start(full)
ufp = UpdateFileParser.UpdateFileParser(self.logger, 512000, destfile)
ufp.init()
fileHeader = ufp.next(',"X":');
### META
## {"Filmliste":["30.08.2020, 11:13","30.08.2020, 09:13","3","MSearch [Vers.: 3.1.139]","d93c9794acaf3e482d42c24e513f78a8"],"Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"]
# this is the timestamp of this database update
#value = jsonDoc['Filmliste'][0]
value = fileHeader[15:32]
#self.logger.info( 'update date ' + value )
try:
fldt = datetime.datetime.strptime(
value.strip(), "%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.database.update_status(filmupdate=flts)
self.logger.info(
'Filmliste dated {}', value.strip())
except TypeError:
# pylint: disable=line-too-long
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(time.mktime(time.strptime(
value.strip(), "%d.%m.%Y, %H:%M")))
self.database.update_status(
filmupdate=flts)
self.logger.info(
'Filmliste dated {}', value.strip())
# pylint: disable=broad-except
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}', value.strip(), err)
except ValueError as err:
pass
###
while (True):
aPart = ufp.next(',"X":');
if (len(aPart) == 0):
break;
##
aPart = '{"X":' + aPart;
if (not(aPart.endswith("}"))):
aPart = aPart + "}";
##
jsonDoc = json.loads(aPart)
jsonDoc = jsonDoc['X']
self._init_record()
# behaviour of the update list
if (len(jsonDoc[0]) > 0):
sender = jsonDoc[0]
else:
jsonDoc[0] = sender
# same for thema
if (len(jsonDoc[1]) > 0):
thema = jsonDoc[1]
else:
jsonDoc[1] = thema
##
self._add_value( jsonDoc )
self._end_record(records)
if self.count % 100 == 0 and self.monitor.abort_requested():
# kodi is shutting down. Close all
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return True
ufp.close()
self._update_end(full, 'IDLE')
self.logger.info('{} records processed',self.count)
self.logger.info(
'Import of {} in update cycle {} finished. Duration: {} seconds',
destfile,
self.cycle,
int(time.time() - starttime)
)
self.notifier.close_update_progress()
return True
except KeyboardInterrupt:
self._update_end(full, 'ABORTED')
self.logger.info('Update cycle {} interrupted by user', self.cycle)
self.notifier.close_update_progress()
return False
except DatabaseCorrupted as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except DatabaseLost as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.close_update_progress()
except Exception as err:
self.logger.error(
'Error {} while processing {} on update cycle {}', err, destfile, self.cycle)
self._update_end(full, 'ABORTED')
self.notifier.close_update_progress()
return False
def Import(self, full):
(_, _, destfile, avgrecsize) = self._get_update_info(full)
if not mvutils.file_exists(destfile):
self.logger.error('File {} does not exists', destfile)
return False
# estimate number of records in update file
records = int(mvutils.file_size(destfile) / avgrecsize)
if not self.db.ftInit():
self.logger.warn(
'Failed to initialize update. Maybe a concurrency problem?')
return False
try:
self.logger.info('Starting import of approx. {} records from {}',
records, destfile)
with open(destfile, 'r') as file:
parser = ijson.parse(file)
flsm = 0
flts = 0
(self.tot_chn, self.tot_shw,
self.tot_mov) = self._update_start(full)
self.notifier.ShowUpdateProgress()
for prefix, event, value in parser:
if (prefix, event) == ("X", "start_array"):
self._init_record()
elif (prefix, event) == ("X", "end_array"):
self._end_record(records)
if self.count % 100 == 0 and self.monitor.abortRequested(
):
# kodi is shutting down. Close all
self._update_end(full, 'ABORTED')
self.notifier.CloseUpdateProgress()
return True
elif (prefix, event) == ("X.item", "string"):
if value is not None:
self._add_value(value.strip())
else:
self._add_value("")
elif (prefix, event) == ("Filmliste", "start_array"):
flsm += 1
elif (prefix, event) == ("Filmliste.item", "string"):
flsm += 1
if flsm == 2 and value is not None:
# this is the timestmap of this database update
try:
fldt = datetime.datetime.strptime(
value.strip(), "%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.db.UpdateStatus(filmupdate=flts)
self.logger.info('Filmliste dated {}',
value.strip())
except TypeError:
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(
time.mktime(
time.strptime(
value.strip(),
"%d.%m.%Y, %H:%M")))
self.db.UpdateStatus(filmupdate=flts)
self.logger.info('Filmliste dated {}',
value.strip())
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}',
value.strip(), err)
except ValueError as err:
pass
self.db.ftFlushInsert()
self._update_end(full, 'IDLE')
self.logger.info('Import of {} in update cycle {} finished',
destfile, self.cycle)
self.notifier.CloseUpdateProgress()
return True
except KeyboardInterrupt:
self._update_end(full, 'ABORTED')
self.logger.info('Update cycle {} interrupted by user', self.cycle)
self.notifier.CloseUpdateProgress()
return False
except DatabaseCorrupted as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.CloseUpdateProgress()
except DatabaseLost as err:
self.logger.error('{} on update cycle {}', err, self.cycle)
self.notifier.CloseUpdateProgress()
except Exception as err:
self.logger.error(
'Error {} while processing {} on update cycle {}', err,
destfile, self.cycle)
self._update_end(full, 'ABORTED')
self.notifier.CloseUpdateProgress()
return False
def _importFile(self, targetFilename):
#
if not mvutils.file_exists(targetFilename):
self.logger.error('File {} does not exists', targetFilename)
return False
# estimate number of records in update file
fileSizeInByte = mvutils.file_size(targetFilename)
records = int(fileSizeInByte / 600)
self.logger.info('Starting import of approximately {} records from {}',
records, targetFilename)
#
# pylint: disable=broad-except
try:
flsm = 0
flts = 0
#
sender = ""
thema = ""
self.notifier.show_update_progress()
#
ufp = UpdateFileParser.UpdateFileParser(self.logger, 512000,
targetFilename)
ufp.init()
fileHeader = ufp.next(',"X":')
# META
# {"Filmliste":["30.08.2020, 11:13","30.08.2020, 09:13","3","MSearch [Vers.: 3.1.139]","d93c9794acaf3e482d42c24e513f78a8"],"Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"]
# this is the timestamp of this database update
# value = jsonDoc['Filmliste'][0]
value = fileHeader[15:32]
# self.logger.debug( 'update date ' + value )
try:
fldt = datetime.datetime.strptime(value.strip(),
"%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.logger.debug('Filmliste dated {}', value.strip())
self.database.set_status('UPDATING', pFilmupdate=flts)
except TypeError:
# pylint: disable=line-too-long
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(
time.mktime(
time.strptime(value.strip(), "%d.%m.%Y, %H:%M")))
self.database.set_status('UPDATING', pFilmupdate=flts)
self.logger.debug('Filmliste dated {}', value.strip())
# pylint: disable=broad-except
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}',
value.strip(), err)
except ValueError as err:
pass
#
recordArray = []
#
while (True):
aPart = ufp.next(',"X":')
if (len(aPart) == 0):
break
#
aPart = '{"X":' + aPart
if (not (aPart.endswith("}"))):
aPart = aPart + "}"
#
jsonDoc = json.loads(aPart)
jsonDoc = jsonDoc['X']
self._init_record()
# behaviour of the update list
if (len(jsonDoc[0]) > 0):
sender = jsonDoc[0][:32]
else:
jsonDoc[0] = sender
# same for thema
if (len(jsonDoc[1]) > 0):
thema = jsonDoc[1][:128]
else:
jsonDoc[1] = thema
#
self.film['channel'] = sender
self.film['show'] = thema
self.film["title"] = jsonDoc[2][:128]
#
if len(jsonDoc[3]) == 10:
self.film["aired"] = jsonDoc[3][6:] + '-' + jsonDoc[3][
3:5] + '-' + jsonDoc[3][:2]
if (len(jsonDoc[4]) == 8):
self.film[
"aired"] = self.film["aired"] + " " + jsonDoc[4]
#
if len(jsonDoc[5]) > 0:
self.film["duration"] = jsonDoc[5]
if len(jsonDoc[7]) > 0:
self.film["description"] = jsonDoc[7][:1024]
self.film["url_video"] = jsonDoc[8]
self.film["website"] = jsonDoc[9]
self.film["url_sub"] = jsonDoc[10]
self.film["url_video_sd"] = self._make_url(jsonDoc[12])
self.film["url_video_hd"] = self._make_url(jsonDoc[14])
if len(jsonDoc[16]) > 0:
self.film["airedepoch"] = int(jsonDoc[16])
self.film["geo"] = jsonDoc[18]
#
# check if the movie is there
#
checkString = sender + thema + self.film["title"] + self.film[
'url_video']
idhash = hashlib.md5(checkString.encode('utf-8')).hexdigest()
#
showid = hashlib.md5(thema.encode('utf-8')).hexdigest()
showid = showid[:8]
#
recordArray.append(
(idhash, int(time.time()), self.film['channel'], showid,
self.film['show'], self.film["title"],
self.film['airedepoch'],
mvutils.make_duration(self.film['duration']),
self.film['description'], self.film['url_sub'],
self.film['url_video'], self.film['url_video_sd'],
self.film['url_video_hd']))
self.count = self.count + 1
# check
if self.count % self.settings.getDatabaseImportBatchSize(
) == 0:
if self.monitor.abort_requested():
# kodi is shutting down. Close all
self._update_end()
self.notifier.close_update_progress()
raise Exception('User requested Abort')
else:
# run insert
try:
(ai, au) = self.database.import_films(recordArray)
self.insertCount += ai
self.updateCount += au
except Exception as err:
self.logger.error('Error in data import: {}', err)
self.errorCount = self.errorCount + 1
recordArray = []
# update status
percent = int(self.count * 100 / records)
percent = percent if percent <= 100 else 100
self.logger.debug(
'In progress (%d%%): insert:%d, update:%d' %
(percent, self.insertCount, self.updateCount))
self.notifier.update_update_progress(
percent, self.count, self.insertCount,
self.updateCount)
if len(recordArray) > 0:
try:
(ai, au) = self.database.import_films(recordArray)
self.insertCount += ai
self.updateCount += au
except Exception as err:
self.logger.error('Error in data import: {}', err)
self.errorCount = self.errorCount + 1
#
ufp.close()
self.notifier.close_update_progress()
if self.errorCount > 0:
self.logger.warn('Update finished with error(s)')
except Exception as err:
self.logger.error('Error {} while processing {}', err,
targetFilename)
self._update_end()
self.database.set_status('ABORTED')
self.notifier.close_update_progress()
raise