def install():
"""Install dependencies and tables"""
FileModel.install()
q = DBHelper.quote_identifier
thumb_table = q(ImageHelper._thumbnail_table)
file_table = q(FileModel._table)
file_table_pk = q(FileModel._pk)
InstallHelper.install('image', (
lambda: (
DBHelper().query("""
CREATE TABLE %s (
"_id" INTEGER PRIMARY KEY AUTOINCREMENT,
"file" INTEGER NOT NULL DEFAULT 0,
"width" INTEGER,
"height" INTEGER,
"thumbnail" TEXT,
FOREIGN KEY ("file") REFERENCES %s(%s)
ON DELETE CASCADE ON UPDATE CASCADE
)
""" % (thumb_table, file_table, file_table_pk)),
DBHelper().query("""
CREATE UNIQUE INDEX "UNQ_FILE_THUMB_SIZE"
ON %s("file", "width", "height")
""" % thumb_table),
DBHelper().query("""
CREATE INDEX "IDX_FILE_THUMB_WIDTH" ON %s("width")
""" % thumb_table),
DBHelper().query("""
CREATE INDEX "IDX_FILE_THUMB_HEIGHT" ON %s("height")
""" % thumb_table)
),
))
def setUp(self):
"""ExtendedModel test set up"""
if os.path.isfile("/tmp/box.db"):
os.unlink("/tmp/box.db")
DBHelper().set_db("/tmp/box.db")
InstallHelper.reset()
FileModel.install()
def test_image_type(self):
"""Test image specific functionality"""
filename = self._get_file("jpg")
pk = FileModel.factory(filename).save().id()
model = FileModel().load(pk)
self.assertGreater(model.width(), 0)
self.assertGreater(model.height(), 0)
def test_factory(self):
"""Test file factory function"""
filename = self._get_file("txt")
pk = FileModel.factory(filename).save().id()
self.assertIsNotNone(pk)
model = FileModel().load(pk)
self.assertEqual(model.name(), os.path.basename(filename))
self.assertEqual(model.extension(), "txt")
self.assertEqual(model.abspath(), os.path.dirname(os.path.abspath(filename)))
def index_device(self, device):
"""Index all new files for a given device"""
transfer_dirs = device.get_transfer_dirs()
for transfer_dir in transfer_dirs:
abstrans = os.path.join(self._basedir, transfer_dir)
flagfile = os.path.join(abstrans, '.__indexed__')
if os.path.isfile(flagfile):
continue
for abspath, dirs, files in os.walk(abstrans):
for filename in files:
devpath = re.sub(r'^%s' % re.escape(abstrans), '', abspath)
absfile = os.path.join(abspath, filename)
file_model = FileModel.factory(absfile)
file_model.add_data({
'devpath': devpath,
'device': device.id()
})
try:
file_model.save()
except sqlite3.IntegrityError:
# unique index on "name", "devpath" and "checksum"
duplicates = FileModel.all().where(
'name = ?', file_model.name()).where(
'devpath = ?', file_model.devpath()).where(
'checksum = ?', file_model.checksum()).limit(1)
if (len(duplicates) > 0
and duplicates[0].abspath()
!= file_model.abspath()):
# the new file is identical to an old one but not
# the same file, lets unlik it.
duplicate = os.path.join(
file_model.abspath(),
file_model.name())
try:
os.unlink(duplicate)
logger.notice('Removed duplicate %s'
% duplicate)
try:
os.rmdir(file_model.abspath())
except OSError:
pass # dir is not empty
except OSError:
logger.error('Unable to remove duplicate %s'
% duplicate)
logger.info('%s already exists, skipping..'
% os.path.join(devpath, filename))
open(flagfile, 'w').close() # touch indexed flag
def test_guesswork(self):
"""Test file factory helper functions"""
file_type = FileModel.guess_type("bild.JPG")
self.assertEqual(file_type[0], "image/jpeg")
extension = FileModel.guess_extension("bild.JPG")
self.assertEqual(extension, "jpg")
extension = FileModel.guess_extension("bild")
self.assertIsNone(extension)
extension = FileModel.guess_extension("bild", file_type[0])
self.assertEqual(extension, "jpe")
vis = FileModel.guess_visibility
self.assertEqual(vis("/path/to/visible"), VISIBILITY_VISIBLE)
self.assertEqual(vis("/path/to/.hidden"), VISIBILITY_HIDDEN)
self.assertEqual(vis("/path/.to/hidden"), VISIBILITY_IN_HIDDEN_DIR)
self.assertEqual(vis("/.path/to/.hidden"), VISIBILITY_IN_HIDDEN_DIR | VISIBILITY_HIDDEN)
def files_action():
data = []
args = request.args
pixel_ratio = 1
models = FileModel.all().limit(240)
for arg in args.keys():
if arg == 'retina':
val = args.get(arg)
if val and val.lower() != 'false':
pixel_ratio = 2
continue
vals = args.get(arg).split(',')
if arg == 'device':
models.add_filter(arg, {'in': vals})
else:
FilterHelper.apply_filter(arg, vals, models)
ImageHelper().join_file_thumbnails(
models,
'm.%s' % FileModel._pk,
260*pixel_ratio,
260*pixel_ratio
)
ImageHelper().add_file_icons(models, 48*pixel_ratio, 128*pixel_ratio)
for model in models:
data.append(model.get_data())
return jsonify({'files': data, 'sql': models.render()})
def test_checksum(self):
"""Test file checksum calculator"""
filename = self._get_file("txt")
contents = ""
with open(filename) as fh:
contents = fh.read()
hasher = hashlib.sha256()
hasher.update(contents)
self.assertEqual(FileModel.calculate_checksum(filename, 1), hasher.hexdigest())
def get_time_attribute(cls):
"""Return the exif DateTime corresponding attribute"""
if not hasattr(cls, '_time_attr'):
for attr in FileModel.get_all_attributes():
if attr['code'] == 'timestamp':
cls._time_attr = attr
break
return cls._time_attr
def test_checksum(self):
"""Test file checksum calculator"""
filename = self._get_file('txt')
contents = ''
with open(filename) as fh:
contents = fh.read()
hasher = hashlib.sha256()
hasher.update(contents)
self.assertEqual(FileModel.calculate_checksum(filename, 1),
hasher.hexdigest())
def test_factory(self):
"""Test file factory function"""
filename = self._get_file('txt')
pk = FileModel.factory(filename).save().id()
self.assertIsNotNone(pk)
model = FileModel().load(pk)
self.assertEqual(model.name(), os.path.basename(filename))
self.assertEqual(model.extension(), 'txt')
self.assertEqual(model.abspath(),
os.path.dirname(os.path.abspath(filename)))
def file_stream_action(file_id=None, display_name=None):
if not file_id:
abort(404)
model = FileModel().load(file_id)
if not model.id():
abort(404)
filename = '%s/%s' % (model.abspath(), model.name())
mimetype = '%s/%s' % (model.type(), model.subtype())
if not os.path.isfile(filename):
abort(404)
return Response(file(filename),
direct_passthrough=True,
content_type=mimetype)
def file_stream_action(file_id=None, display_name=None):
if not file_id:
abort(404)
model = FileModel().load(file_id)
if not model.id():
abort(404)
filename = '%s/%s' % (model.abspath(), model.name())
mimetype = '%s/%s' % (model.type(), model.subtype())
if not os.path.isfile(filename):
abort(404)
return Response(
file(filename),
direct_passthrough=True,
content_type=mimetype)
def install():
"""Install dependencies"""
FileModel.install()
FilterHelper.install()
def file_details_action():
model = FileModel().load(request.args.get('id'))
return jsonify(model.get_data())
try:
os.rmdir(file_model.abspath())
except OSError:
pass # dir is not empty
except OSError:
logger.error('Unable to remove duplicate %s'
% duplicate)
logger.info('%s already exists, skipping..'
% os.path.join(devpath, filename))
open(flagfile, 'w').close() # touch indexed flag
if (__name__ == '__main__'):
"""~$ python file.py path/to/database path/to/basedir"""
if len(sys.argv) < 3:
exit(1)
database = sys.argv[1]
basedir = '%s/devices' % sys.argv[2] #TODO: read from config
if not os.path.isdir(basedir):
exit(2)
DBHelper(database)
DeviceModel.install()
FileModel.install()
FileIndexer(basedir).index_all_devices()
def test_filter(self):
"""Test FilterHelper functions"""
# create a type filter
type_filter = FilterHelper.create_filter('type', 'Type')
FilterHelper.add_filter_option(type_filter, 1, 'Text')
FilterHelper.add_filter_option(type_filter, 2, 'Audio')
FilterHelper.add_filter_option(type_filter, 3, 'Image')
# create a size filter
size_filter = FilterHelper.create_filter('size', 'Size')
FilterHelper.add_filter_option(size_filter, 1, 'Small')
FilterHelper.add_filter_option(size_filter, 2, 'Medium')
FilterHelper.add_filter_option(size_filter, 3, 'Large')
f = FileModel({
'name': 'textfile',
'type': 'text',
'size': '10'
}).save()
FilterHelper.set_filter_values(f, 'type', 1)
FilterHelper.set_filter_values(f, 'size', 1)
f = FileModel({
'name': 'audiofile',
'type': 'audio',
'size': '100'
}).save()
FilterHelper.set_filter_values(f, 'type', 2)
FilterHelper.set_filter_values(f, 'size', 2)
f = FileModel({
'name': 'hybridfile',
'type': 'hybrid',
'size': '1000'
}).save()
FilterHelper.set_filter_values(f, 'type', (1, 3))
FilterHelper.set_filter_values(f, 'size', (2, 3))
fileset = FileModel.all()
FilterHelper.apply_filter('type', 1, fileset)
# fileset should contain 'textfile' and 'hybridfile'
self.assertEqual(len(fileset), 2)
self.assertIn(fileset[0].name(), ('textfile', 'hybridfile'))
self.assertIn(fileset[1].name(), ('textfile', 'hybridfile'))
fileset = FileModel.all()
FilterHelper.apply_filter('size', 2, fileset)
# fileset should contain 'audiofile' and 'hybridfile'
self.assertEqual(len(fileset), 2)
self.assertIn(fileset[0].name(), ('audiofile', 'hybridfile'))
self.assertIn(fileset[1].name(), ('audiofile', 'hybridfile'))
fileset = FileModel.all()
FilterHelper.apply_filter('type', 1, fileset)
FilterHelper.apply_filter('size', 2, fileset)
# fileset should only contain 'hybridfile'
self.assertEqual(len(fileset), 1)
self.assertEqual(fileset[0].name(), 'hybridfile')
def install():
"""Install dependencies"""
FileModel.install()
FilterHelper.install()
try:
os.rmdir(file_model.abspath())
except OSError:
pass # dir is not empty
except OSError:
logger.error('Unable to remove duplicate %s' %
duplicate)
logger.info('%s already exists, skipping..' %
os.path.join(devpath, filename))
open(flagfile, 'w').close() # touch indexed flag
if (__name__ == '__main__'):
"""~$ python file.py path/to/database path/to/basedir"""
if len(sys.argv) < 3:
exit(1)
database = sys.argv[1]
basedir = '%s/devices' % sys.argv[2] #TODO: read from config
if not os.path.isdir(basedir):
exit(2)
DBHelper(database)
DeviceModel.install()
FileModel.install()
FileIndexer(basedir).index_all_devices()
def file_details_action():
model = FileModel().load(request.args.get('id'))
return jsonify(model.get_data())
def index_file_thumbnails(cls, basedir, width, height):
"""Generate thumbnails for image files"""
ImageHelper.install()
insert = {}
#models = FileModel.all().add_filter('extension', {'in': (
# 'bmp', 'gif', 'im', 'jpg', 'jpe', 'jpeg', 'msp',
# 'pcx', 'png', 'ppm', 'tiff', 'xpm', 'mov'
#)}).add_filter(
# ('orientation', 'orientation'),
# ({'null': True}, {'in': range(1, 9)})
#)
models = FileModel.all().add_filter('extension', {'in': (
'mov'
)}).add_filter(
('orientation', 'orientation'),
({'null': True}, {'in': range(1, 9)})
)
ImageHelper.join_file_thumbnails(
models, 'm.%s' % FileModel._pk, width, height, ())
models.where('tt.thumbnail IS NULL').limit(5)
for model in models:
filename = os.path.join(model.abspath(), model.name())
extension = os.path.splitext(filename)[1][1:]
thumbname = os.path.join(
'thumbnails',
'%sx%s' % (width if width else '', height if height else ''),
model.name()[0] if len(model.name()) > 0 else '_',
model.name()[1] if len(model.name()) > 1 else '_',
model.name()
)
if extension == "MOV":
thumbfile = os.path.join(basedir, thumbname+".jpg")
print "Found Movie: "+filename
if not os.path.isfile(thumbfile):
directory = os.path.dirname(thumbfile)
if not os.path.isdir(directory):
os.makedirs(directory)
cmdline = [
'avconv',
'-itsoffset',
'-4',
'-i',
filename,
'-vcodec',
'mjpeg',
'-vframes',
'1',
'-an',
'-f',
'rawvideo',
'-s',
str(width)+"x"+str(height),
thumbfile,
]
subprocess.call(cmdline)
print "generated thumbfile:"+thumbfile
insert[model.id()] = thumbname+".jpg"
else:
thumbfile = os.path.join(basedir, thumbname)
if not os.path.isfile(filename):
insert[model.id()] = None
image = Image.open(filename)
if not image:
insert[model.id()] = None
if not os.path.isfile(thumbfile):
directory = os.path.dirname(thumbfile)
if not os.path.isdir(directory):
os.makedirs(directory)
rotation = 0.
if model.orientation() == 3:
rotation = -180.
elif model.orientation() == 6:
rotation = -90.
elif model.orientation() == 8:
rotation = -270.
if rotation:
image = image.rotate(rotation, expand = 1)
image = ImageHelper.resize(image, width, height)
print (thumbfile)
image.save(thumbfile)
insert[model.id()] = thumbname
print "Inserted file:"+thumbname+str(model.id())
for file_id in insert.keys():
ImageHelper.add_file_thumbnail(
file_id, width, height, insert[file_id])
def install():
"""Install filter tables"""
FileModel.install()
q = DBHelper.quote_identifier
filter_table = q(FilterHelper._filter_table)
option_table = q(FilterHelper._filter_option_table)
value_table = q(FilterHelper._filter_value_table)
file_table = q(FileModel._table)
file_table_pk = q(FileModel._pk)
InstallHelper.install('filter', (
lambda: (
DBHelper().query("""
CREATE TABLE %s (
"_id" INTEGER PRIMARY KEY AUTOINCREMENT,
"code" TEXT NOT NULL DEFAULT '',
"label" TEXT NOT NULL DEFAULT ''
)
""" % filter_table),
DBHelper().query("""
CREATE UNIQUE INDEX "UNQ_FILE_FILTER_CODE"
ON %s ("code")
""" % filter_table),
DBHelper().query("""
CREATE TABLE %s (
"filter" INTEGER NOT NULL DEFAULT 0,
"value" INTEGER NOT NULL DEFAULT 0,
"label" TEXT NOT NULL DEFAULT '',
FOREIGN KEY ("filter") REFERENCES %s("_id")
ON DELETE CASCADE ON UPDATE CASCADE
)
""" % (option_table, filter_table)),
DBHelper().query("""
CREATE UNIQUE INDEX "UNQ_FILE_FILTER_OPTION_FILTER_VALUE"
ON %s("filter", "value")
""" % option_table),
DBHelper().query("""
CREATE TABLE %s (
"file" INTEGER NOT NULL DEFAULT 0,
"filter" INTEGER NOT NULL DEFAULT 0,
"value" INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY ("file") REFERENCES %s(%s)
ON DELETE CASCADE ON UPDATE CASCADE,
FOREIGN KEY ("filter") REFERENCES %s("_id")
ON DELETE CASCADE ON UPDATE CASCADE
)
""" % (value_table, file_table, file_table_pk, filter_table)),
DBHelper().query("""
CREATE UNIQUE INDEX "UNQ_FILE_FILTER_VALUE"
ON %s("file", "filter", "value")
""" % value_table),
DBHelper().query("""
CREATE INDEX "IDX_FILE_FILTER_VALUE_FILE" ON %s("file")
""" % value_table),
DBHelper().query("""
CREATE INDEX "IDX_FILE_FILTER_VALUE_FILTER" ON %s("filter")
""" % value_table),
DBHelper().query("""
CREATE INDEX "IDX_FILE_FILTER_VALUE_VALUE" ON %s("value")
""" % value_table)
),
))
def test_filter(self):
"""Test FilterHelper functions"""
# create a type filter
type_filter = FilterHelper.create_filter('type', 'Type')
FilterHelper.add_filter_option(type_filter, 1, 'Text')
FilterHelper.add_filter_option(type_filter, 2, 'Audio')
FilterHelper.add_filter_option(type_filter, 3, 'Image')
# create a size filter
size_filter = FilterHelper.create_filter('size', 'Size')
FilterHelper.add_filter_option(size_filter, 1, 'Small')
FilterHelper.add_filter_option(size_filter, 2, 'Medium')
FilterHelper.add_filter_option(size_filter, 3, 'Large')
f = FileModel({
'name': 'textfile',
'type': 'text',
'size': '10'
}).save()
FilterHelper.set_filter_values(f, 'type', 1)
FilterHelper.set_filter_values(f, 'size', 1)
f = FileModel({
'name': 'audiofile',
'type': 'audio',
'size': '100'
}).save()
FilterHelper.set_filter_values(f, 'type', 2)
FilterHelper.set_filter_values(f, 'size', 2)
f = FileModel({
'name': 'hybridfile',
'type': 'hybrid',
'size': '1000'
}).save()
FilterHelper.set_filter_values(f, 'type', (1, 3))
FilterHelper.set_filter_values(f, 'size', (2, 3))
fileset = FileModel.all()
FilterHelper.apply_filter('type', 1, fileset)
# fileset should contain 'textfile' and 'hybridfile'
self.assertEqual(len(fileset), 2)
self.assertIn(fileset[0].name(), ('textfile', 'hybridfile'))
self.assertIn(fileset[1].name(), ('textfile', 'hybridfile'))
fileset = FileModel.all()
FilterHelper.apply_filter('size', 2, fileset)
# fileset should contain 'audiofile' and 'hybridfile'
self.assertEqual(len(fileset), 2)
self.assertIn(fileset[0].name(), ('audiofile', 'hybridfile'))
self.assertIn(fileset[1].name(), ('audiofile', 'hybridfile'))
fileset = FileModel.all()
FilterHelper.apply_filter('type', 1, fileset)
FilterHelper.apply_filter('size', 2, fileset)
# fileset should only contain 'hybridfile'
self.assertEqual(len(fileset), 1)
self.assertEqual(fileset[0].name(), 'hybridfile')
