def test_image_good(self):
write_image = Image.open(
os.path.join(os.path.dirname(__file__), 'rainier.jpg'))
t = Thumbnailer()
thumb_image = t.make_thumbnail(write_image)
assert thumb_image.size == (100, 100)
write_image.show()
thumb_image.show()
def download(file=None, url=None, thumbnail_size=None, download_dir=None):
download_dir = get_download_dir(download_dir)
global downloader, thumbnailer
downloader = ImageDownloader()
thumbnailer = None
if thumbnail_size:
width = int(thumbnail_size[0])
height = int(thumbnail_size[1])
thumbnailer = Thumbnailer(width, height)
if file:
result_list = list()
f = codecs.open(file, 'r', 'utf-8')
url_list = f.readlines()
for url in url_list:
result_list.append(
do_work(url=url.strip(), download_dir=download_dir))
return result_list
elif url:
return do_work(url=url, download_dir=download_dir)
else:
print "no arguments"
def main(cache_dir):
okapi = Okapi(OC_OKAPI_KEY, user_agent=USER_AGENT)
os.makedirs(cache_dir, exist_ok=True)
os.makedirs(os.path.join(cache_dir, "json"), exist_ok=True)
os.makedirs(os.path.join(cache_dir, "orig"), exist_ok=True)
os.makedirs(os.path.join(cache_dir, "small"), exist_ok=True)
os.makedirs(os.path.join(cache_dir, "big"), exist_ok=True)
file_name = os.path.join(cache_dir, "json", "caches.json")
if os.path.isfile(file_name):
json_data = load_json(file_name)
else:
print("-- downloading query...")
oc_codes = download_query(OC_USERNAME, OC_PASSWORD, OC_QUERYID)
try:
with open(MANUAL_CACHES_FILE, "r") as f:
for oc_code in f:
oc_code = oc_code.strip()
if oc_code.startswith("OC"):
print("-- adding manual code {}".format(oc_code))
oc_codes.append(oc_code)
except IOError:
pass
print("-> codes: {}".format(len(oc_codes)))
fields = [
'code', 'name', 'location', 'status', 'url', 'owner', 'founds',
'date_hidden', 'date_created', 'short_description', 'description',
'images', 'preview_image', 'internal_id'
]
json_data = okapi.get_caches(oc_codes, fields)
store_json(file_name, json_data)
print("-- analyzing cache data...")
caches = load_caches(json_data)
caches = sorted(caches, key=lambda c: c._date, reverse=True)
print("-> caches: {}".format(len(caches)))
print("-- analyzing log data...")
total_logs = 0
logs_without_coords = 0
for cache in caches:
file_name = os.path.join(cache_dir, "json", f"{cache._code}-logs.json")
if os.path.isfile(file_name):
json_data = load_json(file_name)
else:
fields = [
'uuid', 'date', 'user', 'type', 'comment', 'images',
'internal_id'
]
json_data = okapi.get_logs(cache._code, fields)
store_json(file_name, json_data)
cache._logs = load_logs(json_data)
for log in cache._logs:
total_logs += 1
if log._coordinates is None:
logs_without_coords += 1
print("-- logs without coordinates: {}/{}".format(logs_without_coords,
total_logs))
print("-- downloading missing images...")
downloader = Downloader(threads=4, user_agent=USER_AGENT)
thumbnailer = Thumbnailer(threads=4)
for cache in caches:
if cache._preview_image is not None:
extension = 'noext'
m = re.match('^.*\.([^.\?]+)(\?.*)?$', cache._preview_image)
if m:
extension = m.group(1)
raw_image = '{}/{}/{}.{}'.format(cache_dir, "orig", cache._code,
extension)
downloader.add_job(cache._preview_image, raw_image)
thumb_small = '{}/{}/{}.jpg'.format(cache_dir, "small",
cache._code)
thumbnailer.add_job(raw_image, thumb_small, SIZE_SMALL)
thumb_big = '{}/{}/{}.jpg'.format(cache_dir, "big", cache._code)
thumbnailer.add_job(raw_image, thumb_big, SIZE_BIG)
downloader.run()
print("-- scaling images...")
thumbnailer.run()
print("-- creating files...")
create_db(caches, os.path.join(cache_dir, "safari.sqlite"))
collect_logs(caches, os.path.join(cache_dir, "log-data.js"))
createlist(caches, 30, cache_dir)
create_feed(caches, os.path.join(cache_dir, "feed.xml"))
create_sidebar(caches, "static/index.html",
os.path.join(cache_dir, "index.html"), cache_dir)
def __init__(self, parent = None):
QMainWindow.__init__(self, parent)
self.settings = QSettings(' '.join([self.name, self.version]), 'Nikita Nikishin')
self.ui = gui.Ui_MainWindow()
self.ui.setupUi(self)
self.setWindowTitle(self.full_name)
self.previews = True
self.projectFiles = QDialog(self)
self.projectFiles.ui = project_files.Ui_ProjectFilesDialog()
self.projectFiles.ui.setupUi(self.projectFiles)
self.projectFiles.ui.browseButton1.clicked.connect(self.showFileDialog)
self.projectFiles.ui.browseButton2.clicked.connect(self.showSaveDialog)
self.projectFiles.ui.addToProjectButton.clicked.connect(self.addToProject)
self.projectFiles.ui.removeFromProjectButton.clicked.connect(self.removeFromProject)
self.projectFiles.ui.okButton.clicked.connect(self.projectFilesAccepted)
self.projectFiles.ui.addToProjectButton.setIcon(self.QIconFromTheme('go-next'))
self.projectFiles.ui.removeFromProjectButton.setIcon(self.QIconFromTheme('go-previous'))
self.projectFiles.ui.browseButton1.setIcon(self.QIconFromTheme('document-save'))
self.projectFiles.ui.browseButton2.setIcon(self.QIconFromTheme('document-save'))
self.projectFiles.ui.okButton.setIcon(self.QIconFromTheme('dialog-ok'))
self.projectFiles.ui.cancelButton.setIcon(self.QIconFromTheme('process-stop'))
self.projectFiles.ui.selectAllButton1.setIcon(self.QIconFromTheme('gtk-select-all'))
self.projectFiles.ui.selectAllButton2.setIcon(self.QIconFromTheme('gtk-select-all'))
self.ui.startButton.setIcon(self.QIconFromTheme('media-playback-start'))
self.ui.addPageButton.setIcon(self.QIconFromTheme('list-add'))
self.ui.removePageButton.setIcon(self.QIconFromTheme('list-remove'))
self.ui.newMenuItem.setIcon(self.QIconFromTheme('document-new'))
self.ui.openMenuItem.setIcon(self.QIconFromTheme('document-open'))
self.ui.saveMenuItem.setIcon(self.QIconFromTheme('document-save'))
self.ui.saveAsMenuItem.setIcon(self.QIconFromTheme('document-save-as'))
self.ui.quitMenuItem.setIcon(self.QIconFromTheme('exit'))
self.ui.helpMenuItem.setIcon(self.QIconFromTheme('help'))
self.ui.aboutMenuItem.setIcon(self.QIconFromTheme('gtk-about'))
self.ui.startBindingMenuItem.setIcon(self.QIconFromTheme('media-playback-start'))
self.ui.addPageMenuItem.setIcon(self.QIconFromTheme('list-add'))
self.ui.removePageMenuItem.setIcon(self.QIconFromTheme('list-remove'))
self.ui.moveToTopButton.setIcon(self.QIconFromTheme('go-top'))
self.ui.moveUpButton.setIcon(self.QIconFromTheme('go-up'))
self.ui.moveDownButton.setIcon(self.QIconFromTheme('go-down'))
self.ui.moveToBottomButton.setIcon(self.QIconFromTheme('go-bottom'))
self.ui.clearLogButton.setIcon(self.QIconFromTheme('gtk-clear'))
self.ui.saveLogButton.setIcon(self.QIconFromTheme('document-save'))
self.ui.outputFileBrowseButton.setIcon(self.QIconFromTheme('document-save'))
self.ui.actionReload_Thumbnails.setIcon(self.QIconFromTheme('reload'))
self.ui.actionAbout_Qt4.setIcon(self.QIconFromTheme('gtk-about'))
self.ui.saveMenuItem.setEnabled(False)
self.ui.startBindingMenuItem.setEnabled(False)
self.ui.removePageMenuItem.setEnabled(False)
self.thumbnailer = Thumbnailer(self.ui.pageList)
self.previewer = Previewer()
self.binder = Binder()
self.connect(self.thumbnailer, SIGNAL('makeIcon(int, QImage)'), self.makeIcon)
self.connect(self.previewer, SIGNAL('previewPage(QImage)'), self.previewPage)
self.connect(self.binder, SIGNAL('updateProgress(int, QString)'), self.updateProgress)
self.connect(self.binder, SIGNAL('updateBackground(int, QColor)'), self.updateBackground)
self.connect(self.binder, SIGNAL('finishedBinding'), self.finishedBinding)
self.connect(self.binder, SIGNAL('error(QString)'), self.error)
self.checkDependencies()
if '--test' in sys.argv:
for image in sorted(glob.glob('tests/*.tif')):
self.addFile(image)
self.ui.outputFile.setText('/tmp/output.djvu')
for widget in [self.ui.startButton, self.ui.startBindingMenuItem]:
widget.setEnabled(self.ui.pageList.count() > 0)
self.itemSelectionChanged()
self.hideBackground()
self.thumbnailer.start()
class Bindery(sorting.Sorting, dialogs.Dialogs, error.Error, functions.Bindery, QMainWindow):
name = 'Bindery'
version = '2.7.5'
caption = ''
full_name = ' '.join([name, version, caption])
description = 'Bindery is a cross-platform solution to binding scanned pages into PDF and DjVu documents.'
def __init__(self, parent = None):
QMainWindow.__init__(self, parent)
self.settings = QSettings(' '.join([self.name, self.version]), 'Nikita Nikishin')
self.ui = gui.Ui_MainWindow()
self.ui.setupUi(self)
self.setWindowTitle(self.full_name)
self.previews = True
self.projectFiles = QDialog(self)
self.projectFiles.ui = project_files.Ui_ProjectFilesDialog()
self.projectFiles.ui.setupUi(self.projectFiles)
self.projectFiles.ui.browseButton1.clicked.connect(self.showFileDialog)
self.projectFiles.ui.browseButton2.clicked.connect(self.showSaveDialog)
self.projectFiles.ui.addToProjectButton.clicked.connect(self.addToProject)
self.projectFiles.ui.removeFromProjectButton.clicked.connect(self.removeFromProject)
self.projectFiles.ui.okButton.clicked.connect(self.projectFilesAccepted)
self.projectFiles.ui.addToProjectButton.setIcon(self.QIconFromTheme('go-next'))
self.projectFiles.ui.removeFromProjectButton.setIcon(self.QIconFromTheme('go-previous'))
self.projectFiles.ui.browseButton1.setIcon(self.QIconFromTheme('document-save'))
self.projectFiles.ui.browseButton2.setIcon(self.QIconFromTheme('document-save'))
self.projectFiles.ui.okButton.setIcon(self.QIconFromTheme('dialog-ok'))
self.projectFiles.ui.cancelButton.setIcon(self.QIconFromTheme('process-stop'))
self.projectFiles.ui.selectAllButton1.setIcon(self.QIconFromTheme('gtk-select-all'))
self.projectFiles.ui.selectAllButton2.setIcon(self.QIconFromTheme('gtk-select-all'))
self.ui.startButton.setIcon(self.QIconFromTheme('media-playback-start'))
self.ui.addPageButton.setIcon(self.QIconFromTheme('list-add'))
self.ui.removePageButton.setIcon(self.QIconFromTheme('list-remove'))
self.ui.newMenuItem.setIcon(self.QIconFromTheme('document-new'))
self.ui.openMenuItem.setIcon(self.QIconFromTheme('document-open'))
self.ui.saveMenuItem.setIcon(self.QIconFromTheme('document-save'))
self.ui.saveAsMenuItem.setIcon(self.QIconFromTheme('document-save-as'))
self.ui.quitMenuItem.setIcon(self.QIconFromTheme('exit'))
self.ui.helpMenuItem.setIcon(self.QIconFromTheme('help'))
self.ui.aboutMenuItem.setIcon(self.QIconFromTheme('gtk-about'))
self.ui.startBindingMenuItem.setIcon(self.QIconFromTheme('media-playback-start'))
self.ui.addPageMenuItem.setIcon(self.QIconFromTheme('list-add'))
self.ui.removePageMenuItem.setIcon(self.QIconFromTheme('list-remove'))
self.ui.moveToTopButton.setIcon(self.QIconFromTheme('go-top'))
self.ui.moveUpButton.setIcon(self.QIconFromTheme('go-up'))
self.ui.moveDownButton.setIcon(self.QIconFromTheme('go-down'))
self.ui.moveToBottomButton.setIcon(self.QIconFromTheme('go-bottom'))
self.ui.clearLogButton.setIcon(self.QIconFromTheme('gtk-clear'))
self.ui.saveLogButton.setIcon(self.QIconFromTheme('document-save'))
self.ui.outputFileBrowseButton.setIcon(self.QIconFromTheme('document-save'))
self.ui.actionReload_Thumbnails.setIcon(self.QIconFromTheme('reload'))
self.ui.actionAbout_Qt4.setIcon(self.QIconFromTheme('gtk-about'))
self.ui.saveMenuItem.setEnabled(False)
self.ui.startBindingMenuItem.setEnabled(False)
self.ui.removePageMenuItem.setEnabled(False)
self.thumbnailer = Thumbnailer(self.ui.pageList)
self.previewer = Previewer()
self.binder = Binder()
self.connect(self.thumbnailer, SIGNAL('makeIcon(int, QImage)'), self.makeIcon)
self.connect(self.previewer, SIGNAL('previewPage(QImage)'), self.previewPage)
self.connect(self.binder, SIGNAL('updateProgress(int, QString)'), self.updateProgress)
self.connect(self.binder, SIGNAL('updateBackground(int, QColor)'), self.updateBackground)
self.connect(self.binder, SIGNAL('finishedBinding'), self.finishedBinding)
self.connect(self.binder, SIGNAL('error(QString)'), self.error)
self.checkDependencies()
if '--test' in sys.argv:
for image in sorted(glob.glob('tests/*.tif')):
self.addFile(image)
self.ui.outputFile.setText('/tmp/output.djvu')
for widget in [self.ui.startButton, self.ui.startBindingMenuItem]:
widget.setEnabled(self.ui.pageList.count() > 0)
self.itemSelectionChanged()
self.hideBackground()
self.thumbnailer.start()
def closeEvent(self, event):
if self.binder.isRunning():
if QMessageBox.question(self, 'Bindery', 'A book is currently binding. Are you sure you want to exit?', QMessageBox.Yes, QMessageBox.No) == QMessageBox.Yes:
event.accept()
else:
event.ignore()
def QIconFromTheme(self, name):
if QIcon.hasThemeIcon(name):
return QIcon.fromTheme(name)
else:
return QIcon(':/icons/{0}.png'.format(name))
def main():
okapi = Okapi(OC_OKAPI_KEY, user_agent=USER_AGENT)
mkdir(CACHE_DIR)
mkdir("{}/json".format(CACHE_DIR))
mkdir("{}/orig".format(CACHE_DIR))
mkdir("{}/small".format(CACHE_DIR))
mkdir("{}/big".format(CACHE_DIR))
file_name = "{}/json/caches.json".format(CACHE_DIR)
if os.path.isfile(file_name):
json_data = load_json(file_name)
else:
print("-- downloading query...")
oc_codes = download_query(OC_USERNAME, OC_PASSWORD, OC_QUERYID)
try:
with open(MANUAL_CACHES_FILE, "r") as f:
for oc_code in f:
oc_code = oc_code.strip()
if oc_code.startswith("OC"):
print("-- adding manual code {}".format(oc_code))
oc_codes.append(oc_code)
except IOError:
pass
print("-> codes: {}".format(len(oc_codes)))
fields = ['code', 'name', 'location', 'status', 'url', 'owner', 'founds', 'date_hidden', 'date_created',
'short_description', 'description', 'images', 'preview_image', 'internal_id']
json_data = okapi.get_caches(oc_codes, fields)
store_json(file_name, json_data)
print("-- analyzing cache data...")
caches = load_caches(json_data)
caches = sorted(caches, key=lambda c: c._date, reverse=True)
print("-> caches: {}".format(len(caches)))
print("-- analyzing log data...")
total_logs = 0
logs_without_coords = 0
for cache in caches:
file_name = "{}/json/{}-logs.json".format(CACHE_DIR, cache._code)
if os.path.isfile(file_name):
json_data = load_json(file_name)
else:
fields = ['uuid', 'date', 'user', 'type', 'comment', 'images', 'internal_id']
json_data = okapi.get_logs(cache._code, fields)
store_json(file_name, json_data)
cache._logs = load_logs(json_data)
for log in cache._logs:
total_logs += 1
if log._coordinates is None:
logs_without_coords += 1
print("-- logs without coordinates: {}/{}".format(logs_without_coords, total_logs))
print("-- downloading missing images...")
downloader = Downloader(threads=4, user_agent=USER_AGENT)
thumbnailer = Thumbnailer(threads=4)
for cache in caches:
if cache._preview_image is not None:
extension = 'noext'
m = re.match('^.*\.([^.\?]+)(\?.*)?$', cache._preview_image)
if m:
extension = m.group(1)
raw_image = '{}/{}/{}.{}'.format(CACHE_DIR, "orig", cache._code, extension)
downloader.add_job(cache._preview_image, raw_image)
thumb_small = '{}/{}/{}.jpg'.format(CACHE_DIR, "small", cache._code)
thumbnailer.add_job(raw_image, thumb_small, SIZE_SMALL)
thumb_big = '{}/{}/{}.jpg'.format(CACHE_DIR, "big", cache._code)
thumbnailer.add_job(raw_image, thumb_big, SIZE_BIG)
downloader.run()
print("-- scaling images...")
thumbnailer.run()
print("-- creating files...")
create_db(caches, ".cache/safari.sqlite")
collect_logs(caches, ".cache/log-data.js")
createlist(caches, 30)
create_feed(caches, ".cache/feed.xml")
create_sidebar(caches, "static/index.html", ".cache/index.html")
def test_image_bad(self):
t = Thumbnailer()
with pytest.raises(Exception):
thumb = t.make_thumbnail("Not an image")
from flask import Flask, send_file
from flask_restful import Resource, Api, reqparse
from flask_cors import CORS
import numpy as np
from data_backend import Dataset as HDF_Dataset
from dataset_manager import DatasetManager
from thumbnailer import Thumbnailer
from utils import merge_overlapping_filters
DATASET_PATH = "./datasets"
dataset_manager = DatasetManager(DATASET_PATH)
API_BASE_STR = "/api/v1"
# Init thumbnails (clean directory)
thumbnailer = Thumbnailer("./thumbnails")
thumbnailer.clean();
dataset_list = []
for dset_index, name in enumerate(dataset_manager.get_dataset_names()):
dset = HDF_Dataset(DATASET_PATH, name)
dataset_list.append({
"id": dset_index,
"name": name,
"device": {
"name": dset.device.name,
"version": dset.device.version
},
"subsets": [
{
"id": subset_index,
"name": subset,
