def install(): """Install dependencies and tables""" FileModel.install() q = DBHelper.quote_identifier thumb_table = q(ImageHelper._thumbnail_table) file_table = q(FileModel._table) file_table_pk = q(FileModel._pk) InstallHelper.install('image', ( lambda: ( DBHelper().query(""" CREATE TABLE %s ( "_id" INTEGER PRIMARY KEY AUTOINCREMENT, "file" INTEGER NOT NULL DEFAULT 0, "width" INTEGER, "height" INTEGER, "thumbnail" TEXT, FOREIGN KEY ("file") REFERENCES %s(%s) ON DELETE CASCADE ON UPDATE CASCADE ) """ % (thumb_table, file_table, file_table_pk)), DBHelper().query(""" CREATE UNIQUE INDEX "UNQ_FILE_THUMB_SIZE" ON %s("file", "width", "height") """ % thumb_table), DBHelper().query(""" CREATE INDEX "IDX_FILE_THUMB_WIDTH" ON %s("width") """ % thumb_table), DBHelper().query(""" CREATE INDEX "IDX_FILE_THUMB_HEIGHT" ON %s("height") """ % thumb_table) ), ))
def setUp(self): """ExtendedModel test set up""" if os.path.isfile("/tmp/box.db"): os.unlink("/tmp/box.db") DBHelper().set_db("/tmp/box.db") InstallHelper.reset() FileModel.install()
def test_image_type(self): """Test image specific functionality""" filename = self._get_file("jpg") pk = FileModel.factory(filename).save().id() model = FileModel().load(pk) self.assertGreater(model.width(), 0) self.assertGreater(model.height(), 0)
def test_factory(self): """Test file factory function""" filename = self._get_file("txt") pk = FileModel.factory(filename).save().id() self.assertIsNotNone(pk) model = FileModel().load(pk) self.assertEqual(model.name(), os.path.basename(filename)) self.assertEqual(model.extension(), "txt") self.assertEqual(model.abspath(), os.path.dirname(os.path.abspath(filename)))
def index_device(self, device): """Index all new files for a given device""" transfer_dirs = device.get_transfer_dirs() for transfer_dir in transfer_dirs: abstrans = os.path.join(self._basedir, transfer_dir) flagfile = os.path.join(abstrans, '.__indexed__') if os.path.isfile(flagfile): continue for abspath, dirs, files in os.walk(abstrans): for filename in files: devpath = re.sub(r'^%s' % re.escape(abstrans), '', abspath) absfile = os.path.join(abspath, filename) file_model = FileModel.factory(absfile) file_model.add_data({ 'devpath': devpath, 'device': device.id() }) try: file_model.save() except sqlite3.IntegrityError: # unique index on "name", "devpath" and "checksum" duplicates = FileModel.all().where( 'name = ?', file_model.name()).where( 'devpath = ?', file_model.devpath()).where( 'checksum = ?', file_model.checksum()).limit(1) if (len(duplicates) > 0 and duplicates[0].abspath() != file_model.abspath()): # the new file is identical to an old one but not # the same file, lets unlik it. duplicate = os.path.join( file_model.abspath(), file_model.name()) try: os.unlink(duplicate) logger.notice('Removed duplicate %s' % duplicate) try: os.rmdir(file_model.abspath()) except OSError: pass # dir is not empty except OSError: logger.error('Unable to remove duplicate %s' % duplicate) logger.info('%s already exists, skipping..' % os.path.join(devpath, filename)) open(flagfile, 'w').close() # touch indexed flag
def test_guesswork(self): """Test file factory helper functions""" file_type = FileModel.guess_type("bild.JPG") self.assertEqual(file_type[0], "image/jpeg") extension = FileModel.guess_extension("bild.JPG") self.assertEqual(extension, "jpg") extension = FileModel.guess_extension("bild") self.assertIsNone(extension) extension = FileModel.guess_extension("bild", file_type[0]) self.assertEqual(extension, "jpe") vis = FileModel.guess_visibility self.assertEqual(vis("/path/to/visible"), VISIBILITY_VISIBLE) self.assertEqual(vis("/path/to/.hidden"), VISIBILITY_HIDDEN) self.assertEqual(vis("/path/.to/hidden"), VISIBILITY_IN_HIDDEN_DIR) self.assertEqual(vis("/.path/to/.hidden"), VISIBILITY_IN_HIDDEN_DIR | VISIBILITY_HIDDEN)
def files_action(): data = [] args = request.args pixel_ratio = 1 models = FileModel.all().limit(240) for arg in args.keys(): if arg == 'retina': val = args.get(arg) if val and val.lower() != 'false': pixel_ratio = 2 continue vals = args.get(arg).split(',') if arg == 'device': models.add_filter(arg, {'in': vals}) else: FilterHelper.apply_filter(arg, vals, models) ImageHelper().join_file_thumbnails( models, 'm.%s' % FileModel._pk, 260*pixel_ratio, 260*pixel_ratio ) ImageHelper().add_file_icons(models, 48*pixel_ratio, 128*pixel_ratio) for model in models: data.append(model.get_data()) return jsonify({'files': data, 'sql': models.render()})
def test_checksum(self): """Test file checksum calculator""" filename = self._get_file("txt") contents = "" with open(filename) as fh: contents = fh.read() hasher = hashlib.sha256() hasher.update(contents) self.assertEqual(FileModel.calculate_checksum(filename, 1), hasher.hexdigest())
def get_time_attribute(cls): """Return the exif DateTime corresponding attribute""" if not hasattr(cls, '_time_attr'): for attr in FileModel.get_all_attributes(): if attr['code'] == 'timestamp': cls._time_attr = attr break return cls._time_attr
def test_checksum(self): """Test file checksum calculator""" filename = self._get_file('txt') contents = '' with open(filename) as fh: contents = fh.read() hasher = hashlib.sha256() hasher.update(contents) self.assertEqual(FileModel.calculate_checksum(filename, 1), hasher.hexdigest())
def test_factory(self): """Test file factory function""" filename = self._get_file('txt') pk = FileModel.factory(filename).save().id() self.assertIsNotNone(pk) model = FileModel().load(pk) self.assertEqual(model.name(), os.path.basename(filename)) self.assertEqual(model.extension(), 'txt') self.assertEqual(model.abspath(), os.path.dirname(os.path.abspath(filename)))
def file_stream_action(file_id=None, display_name=None): if not file_id: abort(404) model = FileModel().load(file_id) if not model.id(): abort(404) filename = '%s/%s' % (model.abspath(), model.name()) mimetype = '%s/%s' % (model.type(), model.subtype()) if not os.path.isfile(filename): abort(404) return Response(file(filename), direct_passthrough=True, content_type=mimetype)
def file_stream_action(file_id=None, display_name=None): if not file_id: abort(404) model = FileModel().load(file_id) if not model.id(): abort(404) filename = '%s/%s' % (model.abspath(), model.name()) mimetype = '%s/%s' % (model.type(), model.subtype()) if not os.path.isfile(filename): abort(404) return Response( file(filename), direct_passthrough=True, content_type=mimetype)
def install(): """Install dependencies""" FileModel.install() FilterHelper.install()
def file_details_action(): model = FileModel().load(request.args.get('id')) return jsonify(model.get_data())
try: os.rmdir(file_model.abspath()) except OSError: pass # dir is not empty except OSError: logger.error('Unable to remove duplicate %s' % duplicate) logger.info('%s already exists, skipping..' % os.path.join(devpath, filename)) open(flagfile, 'w').close() # touch indexed flag if (__name__ == '__main__'): """~$ python file.py path/to/database path/to/basedir""" if len(sys.argv) < 3: exit(1) database = sys.argv[1] basedir = '%s/devices' % sys.argv[2] #TODO: read from config if not os.path.isdir(basedir): exit(2) DBHelper(database) DeviceModel.install() FileModel.install() FileIndexer(basedir).index_all_devices()
def test_filter(self): """Test FilterHelper functions""" # create a type filter type_filter = FilterHelper.create_filter('type', 'Type') FilterHelper.add_filter_option(type_filter, 1, 'Text') FilterHelper.add_filter_option(type_filter, 2, 'Audio') FilterHelper.add_filter_option(type_filter, 3, 'Image') # create a size filter size_filter = FilterHelper.create_filter('size', 'Size') FilterHelper.add_filter_option(size_filter, 1, 'Small') FilterHelper.add_filter_option(size_filter, 2, 'Medium') FilterHelper.add_filter_option(size_filter, 3, 'Large') f = FileModel({ 'name': 'textfile', 'type': 'text', 'size': '10' }).save() FilterHelper.set_filter_values(f, 'type', 1) FilterHelper.set_filter_values(f, 'size', 1) f = FileModel({ 'name': 'audiofile', 'type': 'audio', 'size': '100' }).save() FilterHelper.set_filter_values(f, 'type', 2) FilterHelper.set_filter_values(f, 'size', 2) f = FileModel({ 'name': 'hybridfile', 'type': 'hybrid', 'size': '1000' }).save() FilterHelper.set_filter_values(f, 'type', (1, 3)) FilterHelper.set_filter_values(f, 'size', (2, 3)) fileset = FileModel.all() FilterHelper.apply_filter('type', 1, fileset) # fileset should contain 'textfile' and 'hybridfile' self.assertEqual(len(fileset), 2) self.assertIn(fileset[0].name(), ('textfile', 'hybridfile')) self.assertIn(fileset[1].name(), ('textfile', 'hybridfile')) fileset = FileModel.all() FilterHelper.apply_filter('size', 2, fileset) # fileset should contain 'audiofile' and 'hybridfile' self.assertEqual(len(fileset), 2) self.assertIn(fileset[0].name(), ('audiofile', 'hybridfile')) self.assertIn(fileset[1].name(), ('audiofile', 'hybridfile')) fileset = FileModel.all() FilterHelper.apply_filter('type', 1, fileset) FilterHelper.apply_filter('size', 2, fileset) # fileset should only contain 'hybridfile' self.assertEqual(len(fileset), 1) self.assertEqual(fileset[0].name(), 'hybridfile')
def index_file_thumbnails(cls, basedir, width, height): """Generate thumbnails for image files""" ImageHelper.install() insert = {} #models = FileModel.all().add_filter('extension', {'in': ( # 'bmp', 'gif', 'im', 'jpg', 'jpe', 'jpeg', 'msp', # 'pcx', 'png', 'ppm', 'tiff', 'xpm', 'mov' #)}).add_filter( # ('orientation', 'orientation'), # ({'null': True}, {'in': range(1, 9)}) #) models = FileModel.all().add_filter('extension', {'in': ( 'mov' )}).add_filter( ('orientation', 'orientation'), ({'null': True}, {'in': range(1, 9)}) ) ImageHelper.join_file_thumbnails( models, 'm.%s' % FileModel._pk, width, height, ()) models.where('tt.thumbnail IS NULL').limit(5) for model in models: filename = os.path.join(model.abspath(), model.name()) extension = os.path.splitext(filename)[1][1:] thumbname = os.path.join( 'thumbnails', '%sx%s' % (width if width else '', height if height else ''), model.name()[0] if len(model.name()) > 0 else '_', model.name()[1] if len(model.name()) > 1 else '_', model.name() ) if extension == "MOV": thumbfile = os.path.join(basedir, thumbname+".jpg") print "Found Movie: "+filename if not os.path.isfile(thumbfile): directory = os.path.dirname(thumbfile) if not os.path.isdir(directory): os.makedirs(directory) cmdline = [ 'avconv', '-itsoffset', '-4', '-i', filename, '-vcodec', 'mjpeg', '-vframes', '1', '-an', '-f', 'rawvideo', '-s', str(width)+"x"+str(height), thumbfile, ] subprocess.call(cmdline) print "generated thumbfile:"+thumbfile insert[model.id()] = thumbname+".jpg" else: thumbfile = os.path.join(basedir, thumbname) if not os.path.isfile(filename): insert[model.id()] = None image = Image.open(filename) if not image: insert[model.id()] = None if not os.path.isfile(thumbfile): directory = os.path.dirname(thumbfile) if not os.path.isdir(directory): os.makedirs(directory) rotation = 0. if model.orientation() == 3: rotation = -180. elif model.orientation() == 6: rotation = -90. elif model.orientation() == 8: rotation = -270. if rotation: image = image.rotate(rotation, expand = 1) image = ImageHelper.resize(image, width, height) print (thumbfile) image.save(thumbfile) insert[model.id()] = thumbname print "Inserted file:"+thumbname+str(model.id()) for file_id in insert.keys(): ImageHelper.add_file_thumbnail( file_id, width, height, insert[file_id])
def install(): """Install filter tables""" FileModel.install() q = DBHelper.quote_identifier filter_table = q(FilterHelper._filter_table) option_table = q(FilterHelper._filter_option_table) value_table = q(FilterHelper._filter_value_table) file_table = q(FileModel._table) file_table_pk = q(FileModel._pk) InstallHelper.install('filter', ( lambda: ( DBHelper().query(""" CREATE TABLE %s ( "_id" INTEGER PRIMARY KEY AUTOINCREMENT, "code" TEXT NOT NULL DEFAULT '', "label" TEXT NOT NULL DEFAULT '' ) """ % filter_table), DBHelper().query(""" CREATE UNIQUE INDEX "UNQ_FILE_FILTER_CODE" ON %s ("code") """ % filter_table), DBHelper().query(""" CREATE TABLE %s ( "filter" INTEGER NOT NULL DEFAULT 0, "value" INTEGER NOT NULL DEFAULT 0, "label" TEXT NOT NULL DEFAULT '', FOREIGN KEY ("filter") REFERENCES %s("_id") ON DELETE CASCADE ON UPDATE CASCADE ) """ % (option_table, filter_table)), DBHelper().query(""" CREATE UNIQUE INDEX "UNQ_FILE_FILTER_OPTION_FILTER_VALUE" ON %s("filter", "value") """ % option_table), DBHelper().query(""" CREATE TABLE %s ( "file" INTEGER NOT NULL DEFAULT 0, "filter" INTEGER NOT NULL DEFAULT 0, "value" INTEGER NOT NULL DEFAULT 0, FOREIGN KEY ("file") REFERENCES %s(%s) ON DELETE CASCADE ON UPDATE CASCADE, FOREIGN KEY ("filter") REFERENCES %s("_id") ON DELETE CASCADE ON UPDATE CASCADE ) """ % (value_table, file_table, file_table_pk, filter_table)), DBHelper().query(""" CREATE UNIQUE INDEX "UNQ_FILE_FILTER_VALUE" ON %s("file", "filter", "value") """ % value_table), DBHelper().query(""" CREATE INDEX "IDX_FILE_FILTER_VALUE_FILE" ON %s("file") """ % value_table), DBHelper().query(""" CREATE INDEX "IDX_FILE_FILTER_VALUE_FILTER" ON %s("filter") """ % value_table), DBHelper().query(""" CREATE INDEX "IDX_FILE_FILTER_VALUE_VALUE" ON %s("value") """ % value_table) ), ))