def run(self, name):
"""Import movies, function called in a loop over source files"""
from add import validate_details, edit_movie
from gutils import find_next_available
from sqlalchemy import Select, func
import gtk
if not self.set_source(name):
self.debug.show("Can't read data from file %s" % name)
return False
self.widgets['pwindow'].show()
while gtk.events_pending(): # give GTK some time for updates
gtk.main_iteration()
# progressbar
update_on = []
count = self.count_movies()
if count > 0:
for i in range(0,100):
update_on.append(int(float(i)/100*count))
statement = Select( [ func.max(self.db.Movie.c.number) ] )
number = statement.execute().fetchone()[0]
if number is None:
number = 1
else:
number += 1
statement = Select([self.db.Movie.c.number])
processed = 0
while self._abort is False:
details = self.get_movie_details()
if details is None:
break
processed += 1
if processed in update_on:
self.widgets['progressbar'].set_fraction(float(processed)/float(count))
gtk.main_iteration()
self.widgets['progressbar'].set_text("%s (%s/%s)" % (str(self.imported), str(processed), str(count)))
gtk.main_iteration()
gtk.main_iteration() # extra iteration for abort button
if (details.has_key('o_title') and details['o_title']) or (details.has_key('title') and details['title']):
if details.has_key('o_title') and details['o_title']:
statement.whereclause = func.lower(self.db.Movie.c.o_title)==details['o_title'].lower()
if details.has_key('title') and details['title']:
statement.append_whereclause( func.lower(self.db.Movie.c.title)==details['title'].lower() )
tmp = statement.execute().fetchone()
if tmp is not None:
self.debug.show("movie already exists (number=%s, o_title=%s)" % (tmp.number, details['o_title']))
continue
elif details.has_key('title') and details['title']:
statement.whereclause = func.lower(self.db.Movie.c.title)==details['title'].lower()
tmp = statement.execute().fetchone()
if tmp is not None:
self.debug.show("movie already exists (number=%s, title=%s)" % (tmp.number, details['title']))
continue
validate_details(details, self.fields_to_import)
if self.edit is True:
response = edit_movie(self.parent, details) # FIXME: wait until save or cancel button pressed
if response == 1:
self.imported += 1
else:
if not details.has_key('number') or (details.has_key('number') and details['number'] is None):
#details['number'] = find_next_available(self.db)
details['number'] = number
number += 1
#movie = self.db.Movie()
#movie.add_to_db(details)
try:
self.db.Movie.mapper.mapped_table.insert().execute(details)
self.imported += 1
except Exception, e:
self.debug.show("movie details are not unique, skipping: %s" % str(e))
else:
self.debug.show('skipping movie without title or original title')
def run(self, name):
"""Import movies, function called in a loop over source files"""
from add import validate_details, edit_movie
from sqlalchemy import select
import gtk
name = name.decode('utf-8')
if not self.set_source(name):
log.info("Can't read data from file %s", name)
return False
self.widgets['pwindow'].show()
while gtk.events_pending(): # give GTK some time for updates
gtk.main_iteration()
# progressbar
update_on = []
count = self.count_movies()
if count > 0:
for i in range(0, 100):
update_on.append(int(float(i) / 100 * count))
session = self.db.Session()
session.bind = self.db.session.bind
# move some stuff outside the loop to speed it up
set_fraction = self.widgets['progressbar'].set_fraction
set_text = self.widgets['progressbar'].set_text
main_iteration = gtk.main_iteration
# get some values from DB to avoid queries in the loop
statement = select([db.Movie.number, db.Movie.title, db.Movie.o_title])
data = session.execute(statement).fetchall()
numbers = set(i[0] for i in data)
titles = set(i[1].lower() for i in data if i[1])
o_titles = set(i[2].lower() for i in data if i[2])
gc_was_enabled = gc.isenabled()
if gc_was_enabled:
gc.collect()
gc.disable()
begin = time.time()
processed = 0
try:
while self._continue:
details = self.get_movie_details()
if details is None:
break
processed += 1
if processed in update_on:
set_fraction(float(processed) / count)
main_iteration()
set_text("%s (%s/%s)" % (self.imported, processed, count))
main_iteration()
main_iteration() # extra iteration for abort button
o_title_lower = details.get('o_title', '').lower()
title_lower = details.get('title', '').lower()
if o_title_lower or title_lower:
if o_title_lower and o_title_lower in o_titles:
if title_lower and title_lower in titles:
log.info(
"movie already exists (o_title=%s, title=%s)",
details['o_title'], details['title'])
continue
elif title_lower and title_lower in titles: # o_title is not available so lets check title only
log.info("movie already exists (title=%s)",
details['title'])
continue
if self.edit: # XXX: not used for now
validate_details(details, self.fields_to_import)
response = edit_movie(
self.parent, details
) # FIXME: wait until save or cancel button pressed
if response == 1:
self.imported += 1
else:
# get the number and make it unique. number is needed later
number = details.get('number', 1)
while number in numbers:
number += 1
if 'tags' in details:
tags = details.pop('tags')
else:
tags = None
poster = None
if 'poster' in details:
poster = details.pop('poster')
elif 'image' in details:
poster = details.pop('image')
try:
# optional: do mapping of lookup data
if not self.foreignkeymaps:
self._loadmappings()
for fkcolumnname in self.foreignkeymaps:
try:
if fkcolumnname in details and details[
fkcolumnname]:
fkcolumn_id = int(
details[fkcolumnname])
except:
try:
fkcolumn_id = self.normalizename(
details[fkcolumnname])
currentmap = self.foreignkeymaps[
fkcolumnname]
if fkcolumn_id in currentmap:
details[fkcolumnname] = currentmap[
fkcolumn_id]
else:
details[
fkcolumnname] = self._addmappingvalue(
fkcolumnname,
details[fkcolumnname])
except:
log.exception(fkcolumnname)
details[fkcolumnname] = None
# validation before insertion
validate_details(details, self.fields_to_import)
# set number here again because if it is not selected validate_details will remove it
details['number'] = number
# insert the movie in the database
movie = db.tables.movies.insert(
bind=self.db.session.bind).execute(details)
self.imported += 1
# optional: adding tags
if tags:
if self.tagmap is None:
self.loadmappings()
for tag in tags:
try:
if isinstance(tag, (str, unicode)):
# TODO: adding new tag names?
tag_id = self.tagmap[tag.lower()]
else:
tag_id = int(tag)
db.tables.movie_tag.insert(
bind=self.db.session.bind).execute(
{
'movie_id':
movie.lastrowid,
'tag_id': tag_id
})
except:
pass
self.db.session.commit()
# adding poster
if poster:
if len(poster) > 4:
# check for JPEG/PNG header otherwise it should be a filename
header = struct.unpack_from('4s',
poster)[0]
if header == '\xff\xd8\xff\xe0' or \
header == '\x89\x50\x4e\x47':
# make a temporary file
try:
posterfilefd, posterfilename = mkstemp(
'.img')
try:
os.write(posterfilefd, poster)
finally:
os.close(posterfilefd)
edit.update_image(
self.parent, number,
posterfilename)
finally:
if os.path.isfile(posterfilename):
os.remove(posterfilename)
else:
edit.update_image(
self.parent, number, poster)
except Exception:
log.exception(
"movie details are not unique, skipping")
numbers.add(number)
else:
log.info('skipping movie without title and original title')
finally:
log.info("Import process took %s s; %s/%s movies imported",
(time.time() - begin), processed, count)
if gc_was_enabled:
gc.enable()
self.widgets['pwindow'].hide()
return True
def run(self, name):
"""Import movies, function called in a loop over source files"""
from add import validate_details, edit_movie
from gutils import find_next_available
from sqlalchemy import select, func
import gtk
if not self.set_source(name):
log.info("Can't read data from file %s", name)
return False
self.widgets['pwindow'].show()
while gtk.events_pending(): # give GTK some time for updates
gtk.main_iteration()
# progressbar
update_on = []
count = self.count_movies()
if count > 0:
for i in range(0,100):
update_on.append(int(float(i)/100*count))
session = self.db.Session()
session.bind = self.db.session.bind
# move some stuff outside the loop to speed it up
set_fraction = self.widgets['progressbar'].set_fraction
set_text = self.widgets['progressbar'].set_text
main_iteration = gtk.main_iteration
# get some values from DB to avoid queries in the loop
statement = select([db.Movie.number, db.Movie.title, db.Movie.o_title])
data = session.execute(statement).fetchall()
numbers = set(i[0] for i in data)
titles = set(i[1].lower() for i in data)
o_titles = set(i[2].lower() for i in data)
gc_was_enabled = gc.isenabled()
if gc_was_enabled:
gc.collect()
gc.disable()
begin = time.time()
processed = 0
while self._continue:
details = self.get_movie_details()
if details is None:
break
processed += 1
if processed in update_on:
set_fraction(float(processed)/count)
main_iteration()
set_text("%s (%s/%s)" % (self.imported, processed, count))
main_iteration()
main_iteration() # extra iteration for abort button
o_title_avail = 'o_title' in details
title_avail = 'title' in details
if (o_title_avail and details['o_title']) or (title_avail and details['title']):
if o_title_avail and details['o_title'].lower() in o_titles:
if title_avail and details['title'].lower() in titles:
log.info("movie already exists (o_title=%s, title=%s)", details['o_title'], details['title'])
continue
elif title_avail and details['title'].lower() in titles: # o_title is not available so lets check title only
log.info("movie already exists (title=%s)", details['title'])
continue
validate_details(details, self.fields_to_import)
if self.edit: # XXX: not used for now
response = edit_movie(self.parent, details) # FIXME: wait until save or cancel button pressed
if response == 1:
self.imported += 1
else:
number = details.get('number', 1)
while number in numbers:
number += 1
details['number'] = number
#movie = db.Movie()
#movie.add_to_db(details)
try:
db.movies_table.insert(bind=self.db.session.bind).execute(details)
self.imported += 1
except Exception, e:
log.info("movie details are not unique, skipping: %s", e)
numbers.add(number)
else:
log.info('skipping movie without title and original title')
def run(self, name):
"""Import movies, function called in a loop over source files"""
from add import validate_details, edit_movie
from sqlalchemy import select
import gtk
if not self.set_source(name):
log.info("Can't read data from file %s", name)
return False
self.widgets['pwindow'].show()
while gtk.events_pending(): # give GTK some time for updates
gtk.main_iteration()
# progressbar
update_on = []
count = self.count_movies()
if count > 0:
for i in range(0, 100):
update_on.append(int(float(i) / 100 * count))
session = self.db.Session()
session.bind = self.db.session.bind
# move some stuff outside the loop to speed it up
set_fraction = self.widgets['progressbar'].set_fraction
set_text = self.widgets['progressbar'].set_text
main_iteration = gtk.main_iteration
# get some values from DB to avoid queries in the loop
statement = select([db.Movie.number, db.Movie.title, db.Movie.o_title])
data = session.execute(statement).fetchall()
numbers = set(i[0] for i in data)
titles = set(i[1].lower() for i in data if i[1])
o_titles = set(i[2].lower() for i in data if i[2])
gc_was_enabled = gc.isenabled()
if gc_was_enabled:
gc.collect()
gc.disable()
begin = time.time()
processed = 0
try:
while self._continue:
details = self.get_movie_details()
if details is None:
break
processed += 1
if processed in update_on:
set_fraction(float(processed) / count)
main_iteration()
set_text("%s (%s/%s)" % (self.imported, processed, count))
main_iteration()
main_iteration() # extra iteration for abort button
o_title_lower = details.get('o_title', '').lower()
title_lower = details.get('title', '').lower()
if o_title_lower or title_lower:
if o_title_lower and o_title_lower in o_titles:
if title_lower and title_lower in titles:
log.info("movie already exists (o_title=%s, title=%s)", details['o_title'], details['title'])
continue
elif title_lower and title_lower in titles: # o_title is not available so lets check title only
log.info("movie already exists (title=%s)", details['title'])
continue
if self.edit: # XXX: not used for now
validate_details(details, self.fields_to_import)
response = edit_movie(self.parent, details) # FIXME: wait until save or cancel button pressed
if response == 1:
self.imported += 1
else:
number = details.get('number', 1)
while number in numbers:
number += 1
details['number'] = number
if 'tags' in details:
tags = details.pop('tags')
else:
tags = None
if 'poster' in details:
poster = details.pop('poster')
else:
poster = None
try:
# optional: do mapping of lookup data
if not self.foreignkeymaps:
self._loadmappings()
for fkcolumnname in self.foreignkeymaps:
try:
if fkcolumnname in details and details[fkcolumnname]:
fkcolumn_id = int(details[fkcolumnname])
except:
try:
fkcolumn_id = self.normalizename(details[fkcolumnname])
currentmap = self.foreignkeymaps[fkcolumnname]
if fkcolumn_id in currentmap:
details[fkcolumnname] = currentmap[fkcolumn_id]
else:
details[fkcolumnname] = self._addmappingvalue(fkcolumnname, details[fkcolumnname])
except:
log.exception(fkcolumnname)
details[fkcolumnname] = None
# validation before insertion
validate_details(details, self.fields_to_import)
# insert the movie in the database
movie = db.tables.movies.insert(bind=self.db.session.bind).execute(details)
self.imported += 1
# optional: adding tags
if tags:
if self.tagmap is None:
self.loadmappings()
for tag in tags:
try:
if isinstance(tag, (str, unicode)):
# TODO: adding new tag names?
tag_id = self.tagmap[tag.lower()]
else:
tag_id = int(tag)
db.tables.movie_tag.insert(bind=self.db.session.bind).execute({'movie_id': movie.lastrowid, 'tag_id': tag_id})
except:
pass
# adding poster
if poster:
if len(poster) > 4:
# check for JPEG/PNG header otherwise it should be a filename
header = struct.unpack_from('4s', poster)[0]
if header == '\xff\xd8\xff\xe0' or \
header == '\x89\x50\x4e\x47':
# make a temporary file
try:
posterfilefd, posterfilename = mkstemp('.img')
try:
os.write(posterfilefd, poster)
finally:
os.close(posterfilefd)
edit.update_image(self.parent, number, posterfilename)
finally:
if os.path.isfile(posterfilename):
os.remove(posterfilename)
else:
edit.update_image(self.parent, number, poster)
except Exception:
log.exception("movie details are not unique, skipping")
numbers.add(number)
else:
log.info('skipping movie without title and original title')
finally:
log.info("Import process took %s s; %s/%s movies imported", (time.time() - begin), processed, count)
if gc_was_enabled:
gc.enable()
self.widgets['pwindow'].hide()
return True
