def ftInsertFilm( self, film, commit ):
try:
cursor = self.conn.cursor()
newchn = False
inschn = 0
insshw = 0
insmov = 0
# handle channel
if self.ft_channel != film['channel']:
# process changed channel
newchn = True
cursor.execute( 'SELECT `id`,`touched` FROM `channel` WHERE channel.channel=?', ( film['channel'], ) )
r = cursor.fetchall()
if len( r ) > 0:
# get the channel data
self.ft_channel = film['channel']
self.ft_channelid = r[0][0]
if r[0][1] == 0:
# updated touched
cursor.execute( 'UPDATE `channel` SET `touched`=1 WHERE ( channel.id=? )', ( self.ft_channelid, ) )
else:
# insert the new channel
inschn = 1
cursor.execute( 'INSERT INTO `channel` ( `dtCreated`,`channel` ) VALUES ( ?,? )', ( int( time.time() ), film['channel'] ) )
self.ft_channel = film['channel']
self.ft_channelid = cursor.lastrowid
# handle show
if newchn or self.ft_show != film['show']:
# process changed show
cursor.execute( 'SELECT `id`,`touched` FROM `show` WHERE ( show.channelid=? ) AND ( show.show=? )', ( self.ft_channelid, film['show'] ) )
r = cursor.fetchall()
if len( r ) > 0:
# get the show data
self.ft_show = film['show']
self.ft_showid = r[0][0]
if r[0][1] == 0:
# updated touched
cursor.execute( 'UPDATE `show` SET `touched`=1 WHERE ( show.id=? )', ( self.ft_showid, ) )
else:
# insert the new show
insshw = 1
cursor.execute(
"""
INSERT INTO `show` (
`dtCreated`,
`channelid`,
`show`,
`search`
)
VALUES (
?,
?,
?,
?
)
""", (
int( time.time() ),
self.ft_channelid, film['show'],
mvutils.make_search_string( film['show'] )
)
)
self.ft_show = film['show']
self.ft_showid = cursor.lastrowid
# check if the movie is there
cursor.execute( """
SELECT `id`,
`touched`
FROM `film`
WHERE ( film.channelid = ? )
AND
( film.showid = ? )
AND
( film.url_video = ? )
""", ( self.ft_channelid, self.ft_showid, film['url_video'] ) )
r = cursor.fetchall()
if len( r ) > 0:
# film found
filmid = r[0][0]
if r[0][1] == 0:
# update touched
cursor.execute( 'UPDATE `film` SET `touched`=1 WHERE ( film.id=? )', ( filmid, ) )
else:
# insert the new film
insmov = 1
cursor.execute(
"""
INSERT INTO `film` (
`dtCreated`,
`channelid`,
`showid`,
`title`,
`search`,
`aired`,
`duration`,
`size`,
`description`,
`website`,
`url_sub`,
`url_video`,
`url_video_sd`,
`url_video_hd`
)
VALUES (
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?
)
""", (
int( time.time() ),
self.ft_channelid,
self.ft_showid,
film['title'],
mvutils.make_search_string( film['title'] ),
film['airedepoch'],
mvutils.make_duration( film['duration'] ),
film['size'],
film['description'],
film['website'],
film['url_sub'],
film['url_video'],
film['url_video_sd'],
film['url_video_hd']
)
)
filmid = cursor.lastrowid
if commit:
self.conn.commit()
cursor.close()
return ( filmid, inschn, insshw, insmov )
except sqlite3.DatabaseError as err:
self._handle_database_corruption( err )
raise DatabaseCorrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) )
def ftInsertFilm(self, film, commit):
try:
cursor = self.conn.cursor()
newchn = False
inschn = 0
insshw = 0
insmov = 0
channel = film['channel'][:64]
show = film['show'][:128]
title = film['title'][:128]
# handle channel
if self.ft_channel != channel:
# process changed channel
newchn = True
cursor.execute(
'SELECT `id`,`touched` FROM `channel` WHERE channel.channel=?',
(channel, ))
r = cursor.fetchall()
if len(r) > 0:
# get the channel data
self.ft_channel = channel
self.ft_channelid = r[0][0]
if r[0][1] == 0:
# updated touched
cursor.execute(
'UPDATE `channel` SET `touched`=1 WHERE ( channel.id=? )',
(self.ft_channelid, ))
else:
# insert the new channel
inschn = 1
cursor.execute(
'INSERT INTO `channel` ( `dtCreated`,`channel` ) VALUES ( ?,? )',
(int(time.time()), channel))
self.ft_channel = channel
self.ft_channelid = cursor.lastrowid
# handle show
if newchn or self.ft_show != show:
# process changed show
cursor.execute(
'SELECT `id`,`touched` FROM `show` WHERE ( show.channelid=? ) AND ( show.show=? )',
(self.ft_channelid, show))
r = cursor.fetchall()
if len(r) > 0:
# get the show data
self.ft_show = show
self.ft_showid = r[0][0]
if r[0][1] == 0:
# updated touched
cursor.execute(
'UPDATE `show` SET `touched`=1 WHERE ( show.id=? )',
(self.ft_showid, ))
else:
# insert the new show
insshw = 1
cursor.execute(
"""
INSERT INTO `show` (
`dtCreated`,
`channelid`,
`show`,
`search`
)
VALUES (
?,
?,
?,
?
)
""", (int(time.time()), self.ft_channelid, show,
mvutils.make_search_string(show)))
self.ft_show = show
self.ft_showid = cursor.lastrowid
# check if the movie is there
idhash = hashlib.md5(
"{}:{}:{}".format(self.ft_channelid, self.ft_showid,
film['url_video'])).hexdigest()
cursor.execute(
"""
SELECT `id`,
`touched`
FROM `film`
WHERE ( film.idhash = ? )
""", (idhash, ))
r = cursor.fetchall()
if len(r) > 0:
# film found
filmid = r[0][0]
if r[0][1] == 0:
# update touched
cursor.execute(
'UPDATE `film` SET `touched`=1 WHERE ( film.id=? )',
(filmid, ))
else:
# insert the new film
insmov = 1
cursor.execute(
"""
INSERT INTO `film` (
`idhash`,
`dtCreated`,
`channelid`,
`showid`,
`title`,
`search`,
`aired`,
`duration`,
`size`,
`description`,
`website`,
`url_sub`,
`url_video`,
`url_video_sd`,
`url_video_hd`
)
VALUES (
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?
)
""", (idhash, int(time.time()), self.ft_channelid, self.ft_showid, title,
mvutils.make_search_string(film['title']), film['airedepoch'],
mvutils.make_duration(film['duration']), film['size'],
film['description'], film['website'], film['url_sub'],
film['url_video'], film['url_video_sd'], film['url_video_hd']))
filmid = cursor.lastrowid
if commit:
self.conn.commit()
cursor.close()
return (filmid, inschn, insshw, insmov)
except sqlite3.DatabaseError as err:
self._handle_database_corruption(err)
raise DatabaseCorrupted(
'Database error during critical operation: {} - Database will be rebuilt from scratch.'
.format(err))
def _importFile(self, targetFilename):
#
if not mvutils.file_exists(targetFilename):
self.logger.error('File {} does not exists', targetFilename)
return False
# estimate number of records in update file
fileSizeInByte = mvutils.file_size(targetFilename)
records = int(fileSizeInByte / 600)
self.logger.info('Starting import of approximately {} records from {}',
records, targetFilename)
#
# pylint: disable=broad-except
try:
flsm = 0
flts = 0
#
sender = ""
thema = ""
self.notifier.show_update_progress()
#
ufp = UpdateFileParser.UpdateFileParser(self.logger, 512000,
targetFilename)
ufp.init()
fileHeader = ufp.next(',"X":')
# META
# {"Filmliste":["30.08.2020, 11:13","30.08.2020, 09:13","3","MSearch [Vers.: 3.1.139]","d93c9794acaf3e482d42c24e513f78a8"],"Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"]
# this is the timestamp of this database update
# value = jsonDoc['Filmliste'][0]
value = fileHeader[15:32]
# self.logger.debug( 'update date ' + value )
try:
fldt = datetime.datetime.strptime(value.strip(),
"%d.%m.%Y, %H:%M")
flts = int(time.mktime(fldt.timetuple()))
self.logger.debug('Filmliste dated {}', value.strip())
self.database.set_status('UPDATING', pFilmupdate=flts)
except TypeError:
# pylint: disable=line-too-long
# SEE: https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507
# Wonderful. His name is also Leopold
try:
flts = int(
time.mktime(
time.strptime(value.strip(), "%d.%m.%Y, %H:%M")))
self.database.set_status('UPDATING', pFilmupdate=flts)
self.logger.debug('Filmliste dated {}', value.strip())
# pylint: disable=broad-except
except Exception as err:
# If the universe hates us...
self.logger.debug(
'Could not determine date "{}" of filmliste: {}',
value.strip(), err)
except ValueError as err:
pass
#
recordArray = []
#
while (True):
aPart = ufp.next(',"X":')
if (len(aPart) == 0):
break
#
aPart = '{"X":' + aPart
if (not (aPart.endswith("}"))):
aPart = aPart + "}"
#
jsonDoc = json.loads(aPart)
jsonDoc = jsonDoc['X']
self._init_record()
# behaviour of the update list
if (len(jsonDoc[0]) > 0):
sender = jsonDoc[0][:32]
else:
jsonDoc[0] = sender
# same for thema
if (len(jsonDoc[1]) > 0):
thema = jsonDoc[1][:128]
else:
jsonDoc[1] = thema
#
self.film['channel'] = sender
self.film['show'] = thema
self.film["title"] = jsonDoc[2][:128]
#
if len(jsonDoc[3]) == 10:
self.film["aired"] = jsonDoc[3][6:] + '-' + jsonDoc[3][
3:5] + '-' + jsonDoc[3][:2]
if (len(jsonDoc[4]) == 8):
self.film[
"aired"] = self.film["aired"] + " " + jsonDoc[4]
#
if len(jsonDoc[5]) > 0:
self.film["duration"] = jsonDoc[5]
if len(jsonDoc[7]) > 0:
self.film["description"] = jsonDoc[7][:1024]
self.film["url_video"] = jsonDoc[8]
self.film["website"] = jsonDoc[9]
self.film["url_sub"] = jsonDoc[10]
self.film["url_video_sd"] = self._make_url(jsonDoc[12])
self.film["url_video_hd"] = self._make_url(jsonDoc[14])
if len(jsonDoc[16]) > 0:
self.film["airedepoch"] = int(jsonDoc[16])
self.film["geo"] = jsonDoc[18]
#
# check if the movie is there
#
checkString = sender + thema + self.film["title"] + self.film[
'url_video']
idhash = hashlib.md5(checkString.encode('utf-8')).hexdigest()
#
showid = hashlib.md5(thema.encode('utf-8')).hexdigest()
showid = showid[:8]
#
recordArray.append(
(idhash, int(time.time()), self.film['channel'], showid,
self.film['show'], self.film["title"],
self.film['airedepoch'],
mvutils.make_duration(self.film['duration']),
self.film['description'], self.film['url_sub'],
self.film['url_video'], self.film['url_video_sd'],
self.film['url_video_hd']))
self.count = self.count + 1
# check
if self.count % self.settings.getDatabaseImportBatchSize(
) == 0:
if self.monitor.abort_requested():
# kodi is shutting down. Close all
self._update_end()
self.notifier.close_update_progress()
raise Exception('User requested Abort')
else:
# run insert
try:
(ai, au) = self.database.import_films(recordArray)
self.insertCount += ai
self.updateCount += au
except Exception as err:
self.logger.error('Error in data import: {}', err)
self.errorCount = self.errorCount + 1
recordArray = []
# update status
percent = int(self.count * 100 / records)
percent = percent if percent <= 100 else 100
self.logger.debug(
'In progress (%d%%): insert:%d, update:%d' %
(percent, self.insertCount, self.updateCount))
self.notifier.update_update_progress(
percent, self.count, self.insertCount,
self.updateCount)
if len(recordArray) > 0:
try:
(ai, au) = self.database.import_films(recordArray)
self.insertCount += ai
self.updateCount += au
except Exception as err:
self.logger.error('Error in data import: {}', err)
self.errorCount = self.errorCount + 1
#
ufp.close()
self.notifier.close_update_progress()
if self.errorCount > 0:
self.logger.warn('Update finished with error(s)')
except Exception as err:
self.logger.error('Error {} while processing {}', err,
targetFilename)
self._update_end()
self.database.set_status('ABORTED')
self.notifier.close_update_progress()
raise
