예제 #1
0
	def run(self, name):
		"""Import movies, function called in a loop over source files"""
		from add import validate_details, edit_movie
		from gutils import find_next_available
		from sqlalchemy import Select, func
		import gtk
		
		if not self.set_source(name):
			self.debug.show("Can't read data from file %s" % name)
			return False
		
		self.widgets['pwindow'].show()
		while gtk.events_pending():	# give GTK some time for updates
			gtk.main_iteration()

		# progressbar
		update_on = []
		count = self.count_movies()
		if count > 0:
			for i in range(0,100):
				update_on.append(int(float(i)/100*count))

		statement = Select( [ func.max(self.db.Movie.c.number) ] )
		number = statement.execute().fetchone()[0]
		if number is None:
			number = 1
		else:
			number += 1

		statement = Select([self.db.Movie.c.number])

		processed = 0
		while self._abort is False:
			details = self.get_movie_details()
			if details is None:
				break

			processed += 1
			if processed in update_on:
				self.widgets['progressbar'].set_fraction(float(processed)/float(count))
				gtk.main_iteration()
				self.widgets['progressbar'].set_text("%s (%s/%s)" % (str(self.imported), str(processed), str(count)))
				gtk.main_iteration()
				gtk.main_iteration() # extra iteration for abort button

			if (details.has_key('o_title') and details['o_title']) or (details.has_key('title') and details['title']):
				if details.has_key('o_title') and details['o_title']:
					statement.whereclause = func.lower(self.db.Movie.c.o_title)==details['o_title'].lower()
					if details.has_key('title') and details['title']:
						statement.append_whereclause( func.lower(self.db.Movie.c.title)==details['title'].lower() )
					tmp = statement.execute().fetchone()
					if tmp is not None:
						self.debug.show("movie already exists (number=%s, o_title=%s)" % (tmp.number, details['o_title']))
						continue
				elif details.has_key('title') and details['title']:
					statement.whereclause = func.lower(self.db.Movie.c.title)==details['title'].lower()
					tmp = statement.execute().fetchone()
					if tmp is not None:
						self.debug.show("movie already exists (number=%s, title=%s)" % (tmp.number, details['title']))
						continue
				validate_details(details, self.fields_to_import)
				if self.edit is True:
					response = edit_movie(self.parent, details)	# FIXME: wait until save or cancel button pressed
					if response == 1:
						self.imported += 1
				else:
					if not details.has_key('number') or (details.has_key('number') and details['number'] is None):
						#details['number'] = find_next_available(self.db)
						details['number'] = number
						number += 1
					#movie = self.db.Movie()
					#movie.add_to_db(details)
					try:
						self.db.Movie.mapper.mapped_table.insert().execute(details)
						self.imported += 1
					except Exception, e:
						self.debug.show("movie details are not unique, skipping: %s" % str(e))
			else:
				self.debug.show('skipping movie without title or original title')
예제 #2
0
파일: __init__.py 프로젝트: glensc/griffith
    def run(self, name):
        """Import movies, function called in a loop over source files"""
        from add import validate_details, edit_movie
        from sqlalchemy import select
        import gtk

        name = name.decode('utf-8')
        if not self.set_source(name):
            log.info("Can't read data from file %s", name)
            return False

        self.widgets['pwindow'].show()
        while gtk.events_pending():  # give GTK some time for updates
            gtk.main_iteration()

        # progressbar
        update_on = []
        count = self.count_movies()
        if count > 0:
            for i in range(0, 100):
                update_on.append(int(float(i) / 100 * count))

        session = self.db.Session()
        session.bind = self.db.session.bind

        # move some stuff outside the loop to speed it up
        set_fraction = self.widgets['progressbar'].set_fraction
        set_text = self.widgets['progressbar'].set_text
        main_iteration = gtk.main_iteration

        # get some values from DB to avoid queries in the loop
        statement = select([db.Movie.number, db.Movie.title, db.Movie.o_title])
        data = session.execute(statement).fetchall()
        numbers = set(i[0] for i in data)
        titles = set(i[1].lower() for i in data if i[1])
        o_titles = set(i[2].lower() for i in data if i[2])

        gc_was_enabled = gc.isenabled()
        if gc_was_enabled:
            gc.collect()
            gc.disable()

        begin = time.time()
        processed = 0
        try:
            while self._continue:
                details = self.get_movie_details()
                if details is None:
                    break

                processed += 1
                if processed in update_on:
                    set_fraction(float(processed) / count)
                    main_iteration()
                    set_text("%s (%s/%s)" % (self.imported, processed, count))
                    main_iteration()
                    main_iteration()  # extra iteration for abort button

                o_title_lower = details.get('o_title', '').lower()
                title_lower = details.get('title', '').lower()

                if o_title_lower or title_lower:
                    if o_title_lower and o_title_lower in o_titles:
                        if title_lower and title_lower in titles:
                            log.info(
                                "movie already exists (o_title=%s, title=%s)",
                                details['o_title'], details['title'])
                            continue
                    elif title_lower and title_lower in titles:  # o_title is not available so lets check title only
                        log.info("movie already exists (title=%s)",
                                 details['title'])
                        continue
                    if self.edit:  # XXX: not used for now
                        validate_details(details, self.fields_to_import)
                        response = edit_movie(
                            self.parent, details
                        )  # FIXME: wait until save or cancel button pressed
                        if response == 1:
                            self.imported += 1
                    else:
                        # get the number and make it unique. number is needed later
                        number = details.get('number', 1)
                        while number in numbers:
                            number += 1
                        if 'tags' in details:
                            tags = details.pop('tags')
                        else:
                            tags = None
                        poster = None
                        if 'poster' in details:
                            poster = details.pop('poster')
                        elif 'image' in details:
                            poster = details.pop('image')
                        try:
                            # optional: do mapping of lookup data
                            if not self.foreignkeymaps:
                                self._loadmappings()
                            for fkcolumnname in self.foreignkeymaps:
                                try:
                                    if fkcolumnname in details and details[
                                            fkcolumnname]:
                                        fkcolumn_id = int(
                                            details[fkcolumnname])
                                except:
                                    try:
                                        fkcolumn_id = self.normalizename(
                                            details[fkcolumnname])
                                        currentmap = self.foreignkeymaps[
                                            fkcolumnname]
                                        if fkcolumn_id in currentmap:
                                            details[fkcolumnname] = currentmap[
                                                fkcolumn_id]
                                        else:
                                            details[
                                                fkcolumnname] = self._addmappingvalue(
                                                    fkcolumnname,
                                                    details[fkcolumnname])
                                    except:
                                        log.exception(fkcolumnname)
                                        details[fkcolumnname] = None
                            # validation before insertion
                            validate_details(details, self.fields_to_import)
                            # set number here again because if it is not selected validate_details will remove it
                            details['number'] = number
                            # insert the movie in the database
                            movie = db.tables.movies.insert(
                                bind=self.db.session.bind).execute(details)
                            self.imported += 1
                            # optional: adding tags
                            if tags:
                                if self.tagmap is None:
                                    self.loadmappings()
                                for tag in tags:
                                    try:
                                        if isinstance(tag, (str, unicode)):
                                            # TODO: adding new tag names?
                                            tag_id = self.tagmap[tag.lower()]
                                        else:
                                            tag_id = int(tag)
                                        db.tables.movie_tag.insert(
                                            bind=self.db.session.bind).execute(
                                                {
                                                    'movie_id':
                                                    movie.lastrowid,
                                                    'tag_id': tag_id
                                                })
                                    except:
                                        pass
                            self.db.session.commit()
                            # adding poster
                            if poster:
                                if len(poster) > 4:
                                    # check for JPEG/PNG header otherwise it should be a filename
                                    header = struct.unpack_from('4s',
                                                                poster)[0]
                                    if header == '\xff\xd8\xff\xe0' or \
                                       header == '\x89\x50\x4e\x47':
                                        # make a temporary file
                                        try:
                                            posterfilefd, posterfilename = mkstemp(
                                                '.img')
                                            try:
                                                os.write(posterfilefd, poster)
                                            finally:
                                                os.close(posterfilefd)
                                            edit.update_image(
                                                self.parent, number,
                                                posterfilename)
                                        finally:
                                            if os.path.isfile(posterfilename):
                                                os.remove(posterfilename)
                                    else:
                                        edit.update_image(
                                            self.parent, number, poster)
                        except Exception:
                            log.exception(
                                "movie details are not unique, skipping")
                        numbers.add(number)
                else:
                    log.info('skipping movie without title and original title')
        finally:
            log.info("Import process took %s s; %s/%s movies imported",
                     (time.time() - begin), processed, count)
            if gc_was_enabled:
                gc.enable()
            self.widgets['pwindow'].hide()
        return True
예제 #3
0
    def run(self, name):
        """Import movies, function called in a loop over source files"""
        from add import validate_details, edit_movie
        from gutils import find_next_available
        from sqlalchemy import select, func
        import gtk
        
        if not self.set_source(name):
            log.info("Can't read data from file %s", name)
            return False
        
        self.widgets['pwindow'].show()
        while gtk.events_pending():    # give GTK some time for updates
            gtk.main_iteration()

        # progressbar
        update_on = []
        count = self.count_movies()
        if count > 0:
            for i in range(0,100):
                update_on.append(int(float(i)/100*count))

        session = self.db.Session()
        session.bind = self.db.session.bind
        
        # move some stuff outside the loop to speed it up
        set_fraction = self.widgets['progressbar'].set_fraction
        set_text = self.widgets['progressbar'].set_text
        main_iteration = gtk.main_iteration

        # get some values from DB to avoid queries in the loop
        statement = select([db.Movie.number, db.Movie.title, db.Movie.o_title])
        data = session.execute(statement).fetchall()
        numbers = set(i[0] for i in data)
        titles = set(i[1].lower() for i in data)
        o_titles = set(i[2].lower() for i in data)

        gc_was_enabled = gc.isenabled()
        if gc_was_enabled:
            gc.collect()
            gc.disable()

        begin = time.time()
        processed = 0
        while self._continue:
            details = self.get_movie_details()
            if details is None:
                break
            
            processed += 1
            if processed in update_on:
                set_fraction(float(processed)/count)
                main_iteration()
                set_text("%s (%s/%s)" % (self.imported, processed, count))
                main_iteration()
                main_iteration() # extra iteration for abort button

            o_title_avail = 'o_title' in details
            title_avail = 'title' in details

            if (o_title_avail and details['o_title']) or (title_avail and details['title']):
                if o_title_avail and details['o_title'].lower() in o_titles:
                    if title_avail and details['title'].lower() in titles:
                        log.info("movie already exists (o_title=%s, title=%s)", details['o_title'], details['title'])
                        continue
                elif title_avail and details['title'].lower() in titles: # o_title is not available so lets check title only
                    log.info("movie already exists (title=%s)", details['title'])
                    continue
                validate_details(details, self.fields_to_import)
                if self.edit: # XXX: not used for now
                    response = edit_movie(self.parent, details)    # FIXME: wait until save or cancel button pressed
                    if response == 1:
                        self.imported += 1
                else:
                    number = details.get('number', 1)
                    while number in numbers:
                        number += 1
                    details['number'] = number
                    #movie = db.Movie()
                    #movie.add_to_db(details)
                    try:
                        db.movies_table.insert(bind=self.db.session.bind).execute(details)
                        self.imported += 1
                    except Exception, e:
                        log.info("movie details are not unique, skipping: %s", e)
                    numbers.add(number)
            else:
                log.info('skipping movie without title and original title')
예제 #4
0
    def run(self, name):
        """Import movies, function called in a loop over source files"""
        from add import validate_details, edit_movie
        from sqlalchemy import select
        import gtk

        if not self.set_source(name):
            log.info("Can't read data from file %s", name)
            return False

        self.widgets['pwindow'].show()
        while gtk.events_pending():    # give GTK some time for updates
            gtk.main_iteration()

        # progressbar
        update_on = []
        count = self.count_movies()
        if count > 0:
            for i in range(0, 100):
                update_on.append(int(float(i) / 100 * count))

        session = self.db.Session()
        session.bind = self.db.session.bind

        # move some stuff outside the loop to speed it up
        set_fraction = self.widgets['progressbar'].set_fraction
        set_text = self.widgets['progressbar'].set_text
        main_iteration = gtk.main_iteration

        # get some values from DB to avoid queries in the loop
        statement = select([db.Movie.number, db.Movie.title, db.Movie.o_title])
        data = session.execute(statement).fetchall()
        numbers = set(i[0] for i in data)
        titles = set(i[1].lower() for i in data if i[1])
        o_titles = set(i[2].lower() for i in data if i[2])

        gc_was_enabled = gc.isenabled()
        if gc_was_enabled:
            gc.collect()
            gc.disable()

        begin = time.time()
        processed = 0
        try:
            while self._continue:
                details = self.get_movie_details()
                if details is None:
                    break

                processed += 1
                if processed in update_on:
                    set_fraction(float(processed) / count)
                    main_iteration()
                    set_text("%s (%s/%s)" % (self.imported, processed, count))
                    main_iteration()
                    main_iteration() # extra iteration for abort button

                o_title_lower = details.get('o_title', '').lower()
                title_lower = details.get('title', '').lower()

                if o_title_lower or title_lower:
                    if o_title_lower and o_title_lower in o_titles:
                        if title_lower and title_lower in titles:
                            log.info("movie already exists (o_title=%s, title=%s)", details['o_title'], details['title'])
                            continue
                    elif title_lower and title_lower in titles: # o_title is not available so lets check title only
                        log.info("movie already exists (title=%s)", details['title'])
                        continue
                    if self.edit: # XXX: not used for now
                        validate_details(details, self.fields_to_import)
                        response = edit_movie(self.parent, details)    # FIXME: wait until save or cancel button pressed
                        if response == 1:
                            self.imported += 1
                    else:
                        number = details.get('number', 1)
                        while number in numbers:
                            number += 1
                        details['number'] = number
                        if 'tags' in details:
                            tags = details.pop('tags')
                        else:
                            tags = None
                        if 'poster' in details:
                            poster = details.pop('poster')
                        else:
                            poster = None
                        try:
                            # optional: do mapping of lookup data
                            if not self.foreignkeymaps:
                                self._loadmappings()
                            for fkcolumnname in self.foreignkeymaps:
                                try:
                                    if fkcolumnname in details and details[fkcolumnname]:
                                        fkcolumn_id = int(details[fkcolumnname])
                                except:
                                    try:
                                        fkcolumn_id = self.normalizename(details[fkcolumnname])
                                        currentmap = self.foreignkeymaps[fkcolumnname]
                                        if fkcolumn_id in currentmap:
                                            details[fkcolumnname] = currentmap[fkcolumn_id]
                                        else:
                                            details[fkcolumnname] = self._addmappingvalue(fkcolumnname, details[fkcolumnname])
                                    except:
                                        log.exception(fkcolumnname)
                                        details[fkcolumnname] = None
                            # validation before insertion
                            validate_details(details, self.fields_to_import)
                            # insert the movie in the database
                            movie = db.tables.movies.insert(bind=self.db.session.bind).execute(details)
                            self.imported += 1
                            # optional: adding tags
                            if tags:
                                if self.tagmap is None:
                                    self.loadmappings()
                                for tag in tags:
                                    try:
                                        if isinstance(tag, (str, unicode)):
                                            # TODO: adding new tag names?
                                            tag_id = self.tagmap[tag.lower()]
                                        else:
                                            tag_id = int(tag)
                                        db.tables.movie_tag.insert(bind=self.db.session.bind).execute({'movie_id': movie.lastrowid, 'tag_id': tag_id})
                                    except:
                                        pass
                            # adding poster
                            if poster:
                                if len(poster) > 4:
                                    # check for JPEG/PNG header otherwise it should be a filename
                                    header = struct.unpack_from('4s', poster)[0]
                                    if header == '\xff\xd8\xff\xe0' or \
                                       header == '\x89\x50\x4e\x47':
                                        # make a temporary file
                                        try:
                                            posterfilefd, posterfilename = mkstemp('.img')
                                            try:
                                                os.write(posterfilefd, poster)
                                            finally:
                                                os.close(posterfilefd)
                                            edit.update_image(self.parent, number, posterfilename)
                                        finally:
                                            if os.path.isfile(posterfilename):
                                                os.remove(posterfilename)
                                    else:
                                        edit.update_image(self.parent, number, poster)
                        except Exception:
                            log.exception("movie details are not unique, skipping")
                        numbers.add(number)
                else:
                    log.info('skipping movie without title and original title')
        finally:
            log.info("Import process took %s s; %s/%s movies imported", (time.time() - begin), processed, count)
            if gc_was_enabled:
                gc.enable()
            self.widgets['pwindow'].hide()
        return True