def test_image_good(self):
        write_image = Image.open(
            os.path.join(os.path.dirname(__file__), 'rainier.jpg'))
        t = Thumbnailer()
        thumb_image = t.make_thumbnail(write_image)

        assert thumb_image.size == (100, 100)
        write_image.show()
        thumb_image.show()
Exemple #2
0
def download(file=None, url=None, thumbnail_size=None, download_dir=None):

    download_dir = get_download_dir(download_dir)

    global downloader, thumbnailer
    downloader = ImageDownloader()
    thumbnailer = None

    if thumbnail_size:
        width = int(thumbnail_size[0])
        height = int(thumbnail_size[1])
        thumbnailer = Thumbnailer(width, height)

    if file:
        result_list = list()
        f = codecs.open(file, 'r', 'utf-8')
        url_list = f.readlines()

        for url in url_list:
            result_list.append(
                do_work(url=url.strip(), download_dir=download_dir))

        return result_list

    elif url:
        return do_work(url=url, download_dir=download_dir)
    else:
        print "no arguments"
Exemple #3
0
def main(cache_dir):
    okapi = Okapi(OC_OKAPI_KEY, user_agent=USER_AGENT)

    os.makedirs(cache_dir, exist_ok=True)
    os.makedirs(os.path.join(cache_dir, "json"), exist_ok=True)
    os.makedirs(os.path.join(cache_dir, "orig"), exist_ok=True)
    os.makedirs(os.path.join(cache_dir, "small"), exist_ok=True)
    os.makedirs(os.path.join(cache_dir, "big"), exist_ok=True)

    file_name = os.path.join(cache_dir, "json", "caches.json")
    if os.path.isfile(file_name):
        json_data = load_json(file_name)
    else:
        print("-- downloading query...")
        oc_codes = download_query(OC_USERNAME, OC_PASSWORD, OC_QUERYID)
        try:
            with open(MANUAL_CACHES_FILE, "r") as f:
                for oc_code in f:
                    oc_code = oc_code.strip()
                    if oc_code.startswith("OC"):
                        print("-- adding manual code {}".format(oc_code))
                        oc_codes.append(oc_code)
        except IOError:
            pass

        print("-> codes: {}".format(len(oc_codes)))
        fields = [
            'code', 'name', 'location', 'status', 'url', 'owner', 'founds',
            'date_hidden', 'date_created', 'short_description', 'description',
            'images', 'preview_image', 'internal_id'
        ]
        json_data = okapi.get_caches(oc_codes, fields)
        store_json(file_name, json_data)

    print("-- analyzing cache data...")
    caches = load_caches(json_data)
    caches = sorted(caches, key=lambda c: c._date, reverse=True)
    print("-> caches: {}".format(len(caches)))

    print("-- analyzing log data...")
    total_logs = 0
    logs_without_coords = 0
    for cache in caches:
        file_name = os.path.join(cache_dir, "json", f"{cache._code}-logs.json")
        if os.path.isfile(file_name):
            json_data = load_json(file_name)
        else:
            fields = [
                'uuid', 'date', 'user', 'type', 'comment', 'images',
                'internal_id'
            ]
            json_data = okapi.get_logs(cache._code, fields)
            store_json(file_name, json_data)
        cache._logs = load_logs(json_data)

        for log in cache._logs:
            total_logs += 1
            if log._coordinates is None:
                logs_without_coords += 1
    print("-- logs without coordinates: {}/{}".format(logs_without_coords,
                                                      total_logs))

    print("-- downloading missing images...")
    downloader = Downloader(threads=4, user_agent=USER_AGENT)
    thumbnailer = Thumbnailer(threads=4)
    for cache in caches:
        if cache._preview_image is not None:
            extension = 'noext'
            m = re.match('^.*\.([^.\?]+)(\?.*)?$', cache._preview_image)
            if m:
                extension = m.group(1)
            raw_image = '{}/{}/{}.{}'.format(cache_dir, "orig", cache._code,
                                             extension)
            downloader.add_job(cache._preview_image, raw_image)
            thumb_small = '{}/{}/{}.jpg'.format(cache_dir, "small",
                                                cache._code)
            thumbnailer.add_job(raw_image, thumb_small, SIZE_SMALL)
            thumb_big = '{}/{}/{}.jpg'.format(cache_dir, "big", cache._code)
            thumbnailer.add_job(raw_image, thumb_big, SIZE_BIG)
    downloader.run()

    print("-- scaling images...")
    thumbnailer.run()

    print("-- creating files...")
    create_db(caches, os.path.join(cache_dir, "safari.sqlite"))
    collect_logs(caches, os.path.join(cache_dir, "log-data.js"))
    createlist(caches, 30, cache_dir)
    create_feed(caches, os.path.join(cache_dir, "feed.xml"))
    create_sidebar(caches, "static/index.html",
                   os.path.join(cache_dir, "index.html"), cache_dir)
Exemple #4
0
  def __init__(self, parent = None):
    QMainWindow.__init__(self, parent)
    
    self.settings = QSettings(' '.join([self.name, self.version]), 'Nikita Nikishin')
    
    self.ui = gui.Ui_MainWindow()
    self.ui.setupUi(self)
    
    self.setWindowTitle(self.full_name)
    self.previews = True
    
    self.projectFiles = QDialog(self)
    self.projectFiles.ui = project_files.Ui_ProjectFilesDialog()
    self.projectFiles.ui.setupUi(self.projectFiles)

    self.projectFiles.ui.browseButton1.clicked.connect(self.showFileDialog)
    self.projectFiles.ui.browseButton2.clicked.connect(self.showSaveDialog)
    self.projectFiles.ui.addToProjectButton.clicked.connect(self.addToProject)
    self.projectFiles.ui.removeFromProjectButton.clicked.connect(self.removeFromProject)
    self.projectFiles.ui.okButton.clicked.connect(self.projectFilesAccepted)
    
    self.projectFiles.ui.addToProjectButton.setIcon(self.QIconFromTheme('go-next'))
    self.projectFiles.ui.removeFromProjectButton.setIcon(self.QIconFromTheme('go-previous'))
    
    self.projectFiles.ui.browseButton1.setIcon(self.QIconFromTheme('document-save'))
    self.projectFiles.ui.browseButton2.setIcon(self.QIconFromTheme('document-save'))

    self.projectFiles.ui.okButton.setIcon(self.QIconFromTheme('dialog-ok'))
    self.projectFiles.ui.cancelButton.setIcon(self.QIconFromTheme('process-stop'))
    
    self.projectFiles.ui.selectAllButton1.setIcon(self.QIconFromTheme('gtk-select-all'))
    self.projectFiles.ui.selectAllButton2.setIcon(self.QIconFromTheme('gtk-select-all'))
    
    self.ui.startButton.setIcon(self.QIconFromTheme('media-playback-start'))
    self.ui.addPageButton.setIcon(self.QIconFromTheme('list-add'))
    self.ui.removePageButton.setIcon(self.QIconFromTheme('list-remove'))
    
    self.ui.newMenuItem.setIcon(self.QIconFromTheme('document-new'))
    self.ui.openMenuItem.setIcon(self.QIconFromTheme('document-open'))
    self.ui.saveMenuItem.setIcon(self.QIconFromTheme('document-save'))
    self.ui.saveAsMenuItem.setIcon(self.QIconFromTheme('document-save-as'))
    self.ui.quitMenuItem.setIcon(self.QIconFromTheme('exit'))
    self.ui.helpMenuItem.setIcon(self.QIconFromTheme('help'))
    self.ui.aboutMenuItem.setIcon(self.QIconFromTheme('gtk-about'))
    
    self.ui.startBindingMenuItem.setIcon(self.QIconFromTheme('media-playback-start'))
    self.ui.addPageMenuItem.setIcon(self.QIconFromTheme('list-add'))
    self.ui.removePageMenuItem.setIcon(self.QIconFromTheme('list-remove'))
    
    self.ui.moveToTopButton.setIcon(self.QIconFromTheme('go-top'))
    self.ui.moveUpButton.setIcon(self.QIconFromTheme('go-up'))
    self.ui.moveDownButton.setIcon(self.QIconFromTheme('go-down'))
    self.ui.moveToBottomButton.setIcon(self.QIconFromTheme('go-bottom'))
    
    self.ui.clearLogButton.setIcon(self.QIconFromTheme('gtk-clear'))
    self.ui.saveLogButton.setIcon(self.QIconFromTheme('document-save'))
    
    self.ui.outputFileBrowseButton.setIcon(self.QIconFromTheme('document-save'))
    
    self.ui.actionReload_Thumbnails.setIcon(self.QIconFromTheme('reload'))
    self.ui.actionAbout_Qt4.setIcon(self.QIconFromTheme('gtk-about'))
    
    self.ui.saveMenuItem.setEnabled(False)
    self.ui.startBindingMenuItem.setEnabled(False)
    self.ui.removePageMenuItem.setEnabled(False)
    
    
    self.thumbnailer = Thumbnailer(self.ui.pageList)
    self.previewer = Previewer()
    self.binder = Binder()
    
    self.connect(self.thumbnailer, SIGNAL('makeIcon(int, QImage)'), self.makeIcon)
    self.connect(self.previewer, SIGNAL('previewPage(QImage)'), self.previewPage)    
    
    self.connect(self.binder, SIGNAL('updateProgress(int, QString)'), self.updateProgress)
    self.connect(self.binder, SIGNAL('updateBackground(int, QColor)'), self.updateBackground)
    self.connect(self.binder, SIGNAL('finishedBinding'), self.finishedBinding)
    self.connect(self.binder, SIGNAL('error(QString)'), self.error)
    
    self.checkDependencies()
    
    if '--test' in sys.argv:
      for image in sorted(glob.glob('tests/*.tif')):
        self.addFile(image)
        
        self.ui.outputFile.setText('/tmp/output.djvu')
    
    for widget in [self.ui.startButton, self.ui.startBindingMenuItem]:
      widget.setEnabled(self.ui.pageList.count() > 0)
    
    self.itemSelectionChanged()
    self.hideBackground()
    self.thumbnailer.start()
Exemple #5
0
class Bindery(sorting.Sorting, dialogs.Dialogs, error.Error, functions.Bindery, QMainWindow):
  name = 'Bindery'
  version = '2.7.5'
  caption = ''
  full_name = ' '.join([name, version, caption])
  description = 'Bindery is a cross-platform solution to binding scanned pages into PDF and DjVu documents.'
  
  def __init__(self, parent = None):
    QMainWindow.__init__(self, parent)
    
    self.settings = QSettings(' '.join([self.name, self.version]), 'Nikita Nikishin')
    
    self.ui = gui.Ui_MainWindow()
    self.ui.setupUi(self)
    
    self.setWindowTitle(self.full_name)
    self.previews = True
    
    self.projectFiles = QDialog(self)
    self.projectFiles.ui = project_files.Ui_ProjectFilesDialog()
    self.projectFiles.ui.setupUi(self.projectFiles)

    self.projectFiles.ui.browseButton1.clicked.connect(self.showFileDialog)
    self.projectFiles.ui.browseButton2.clicked.connect(self.showSaveDialog)
    self.projectFiles.ui.addToProjectButton.clicked.connect(self.addToProject)
    self.projectFiles.ui.removeFromProjectButton.clicked.connect(self.removeFromProject)
    self.projectFiles.ui.okButton.clicked.connect(self.projectFilesAccepted)
    
    self.projectFiles.ui.addToProjectButton.setIcon(self.QIconFromTheme('go-next'))
    self.projectFiles.ui.removeFromProjectButton.setIcon(self.QIconFromTheme('go-previous'))
    
    self.projectFiles.ui.browseButton1.setIcon(self.QIconFromTheme('document-save'))
    self.projectFiles.ui.browseButton2.setIcon(self.QIconFromTheme('document-save'))

    self.projectFiles.ui.okButton.setIcon(self.QIconFromTheme('dialog-ok'))
    self.projectFiles.ui.cancelButton.setIcon(self.QIconFromTheme('process-stop'))
    
    self.projectFiles.ui.selectAllButton1.setIcon(self.QIconFromTheme('gtk-select-all'))
    self.projectFiles.ui.selectAllButton2.setIcon(self.QIconFromTheme('gtk-select-all'))
    
    self.ui.startButton.setIcon(self.QIconFromTheme('media-playback-start'))
    self.ui.addPageButton.setIcon(self.QIconFromTheme('list-add'))
    self.ui.removePageButton.setIcon(self.QIconFromTheme('list-remove'))
    
    self.ui.newMenuItem.setIcon(self.QIconFromTheme('document-new'))
    self.ui.openMenuItem.setIcon(self.QIconFromTheme('document-open'))
    self.ui.saveMenuItem.setIcon(self.QIconFromTheme('document-save'))
    self.ui.saveAsMenuItem.setIcon(self.QIconFromTheme('document-save-as'))
    self.ui.quitMenuItem.setIcon(self.QIconFromTheme('exit'))
    self.ui.helpMenuItem.setIcon(self.QIconFromTheme('help'))
    self.ui.aboutMenuItem.setIcon(self.QIconFromTheme('gtk-about'))
    
    self.ui.startBindingMenuItem.setIcon(self.QIconFromTheme('media-playback-start'))
    self.ui.addPageMenuItem.setIcon(self.QIconFromTheme('list-add'))
    self.ui.removePageMenuItem.setIcon(self.QIconFromTheme('list-remove'))
    
    self.ui.moveToTopButton.setIcon(self.QIconFromTheme('go-top'))
    self.ui.moveUpButton.setIcon(self.QIconFromTheme('go-up'))
    self.ui.moveDownButton.setIcon(self.QIconFromTheme('go-down'))
    self.ui.moveToBottomButton.setIcon(self.QIconFromTheme('go-bottom'))
    
    self.ui.clearLogButton.setIcon(self.QIconFromTheme('gtk-clear'))
    self.ui.saveLogButton.setIcon(self.QIconFromTheme('document-save'))
    
    self.ui.outputFileBrowseButton.setIcon(self.QIconFromTheme('document-save'))
    
    self.ui.actionReload_Thumbnails.setIcon(self.QIconFromTheme('reload'))
    self.ui.actionAbout_Qt4.setIcon(self.QIconFromTheme('gtk-about'))
    
    self.ui.saveMenuItem.setEnabled(False)
    self.ui.startBindingMenuItem.setEnabled(False)
    self.ui.removePageMenuItem.setEnabled(False)
    
    
    self.thumbnailer = Thumbnailer(self.ui.pageList)
    self.previewer = Previewer()
    self.binder = Binder()
    
    self.connect(self.thumbnailer, SIGNAL('makeIcon(int, QImage)'), self.makeIcon)
    self.connect(self.previewer, SIGNAL('previewPage(QImage)'), self.previewPage)    
    
    self.connect(self.binder, SIGNAL('updateProgress(int, QString)'), self.updateProgress)
    self.connect(self.binder, SIGNAL('updateBackground(int, QColor)'), self.updateBackground)
    self.connect(self.binder, SIGNAL('finishedBinding'), self.finishedBinding)
    self.connect(self.binder, SIGNAL('error(QString)'), self.error)
    
    self.checkDependencies()
    
    if '--test' in sys.argv:
      for image in sorted(glob.glob('tests/*.tif')):
        self.addFile(image)
        
        self.ui.outputFile.setText('/tmp/output.djvu')
    
    for widget in [self.ui.startButton, self.ui.startBindingMenuItem]:
      widget.setEnabled(self.ui.pageList.count() > 0)
    
    self.itemSelectionChanged()
    self.hideBackground()
    self.thumbnailer.start()
  
  def closeEvent(self, event):
    if self.binder.isRunning():
      if QMessageBox.question(self, 'Bindery', 'A book is currently binding. Are you sure you want to exit?', QMessageBox.Yes, QMessageBox.No) == QMessageBox.Yes:
        event.accept()
      else:
        event.ignore()
  
  def QIconFromTheme(self, name):
    if QIcon.hasThemeIcon(name):
      return QIcon.fromTheme(name)
    else:
      return QIcon(':/icons/{0}.png'.format(name))
Exemple #6
0
def main():
    okapi = Okapi(OC_OKAPI_KEY, user_agent=USER_AGENT)

    mkdir(CACHE_DIR)
    mkdir("{}/json".format(CACHE_DIR))
    mkdir("{}/orig".format(CACHE_DIR))
    mkdir("{}/small".format(CACHE_DIR))
    mkdir("{}/big".format(CACHE_DIR))

    file_name = "{}/json/caches.json".format(CACHE_DIR)
    if os.path.isfile(file_name):
        json_data = load_json(file_name)
    else:
        print("-- downloading query...")
        oc_codes = download_query(OC_USERNAME, OC_PASSWORD, OC_QUERYID)
        try:
            with open(MANUAL_CACHES_FILE, "r") as f:
                for oc_code in f:
                    oc_code = oc_code.strip()
                    if oc_code.startswith("OC"):
                        print("-- adding manual code {}".format(oc_code))
                        oc_codes.append(oc_code)
        except IOError:
            pass

        print("-> codes: {}".format(len(oc_codes)))
        fields = ['code', 'name', 'location', 'status', 'url', 'owner', 'founds', 'date_hidden', 'date_created',
                  'short_description', 'description', 'images', 'preview_image', 'internal_id']
        json_data = okapi.get_caches(oc_codes, fields)
        store_json(file_name, json_data)

    print("-- analyzing cache data...")
    caches = load_caches(json_data)
    caches = sorted(caches, key=lambda c: c._date, reverse=True)
    print("-> caches: {}".format(len(caches)))

    print("-- analyzing log data...")
    total_logs = 0
    logs_without_coords = 0
    for cache in caches:
        file_name = "{}/json/{}-logs.json".format(CACHE_DIR, cache._code)
        if os.path.isfile(file_name):
            json_data = load_json(file_name)
        else:
            fields = ['uuid', 'date', 'user', 'type', 'comment', 'images', 'internal_id']
            json_data = okapi.get_logs(cache._code, fields)
            store_json(file_name, json_data)
        cache._logs = load_logs(json_data)

        for log in cache._logs:
            total_logs += 1
            if log._coordinates is None:
                logs_without_coords += 1
    print("-- logs without coordinates: {}/{}".format(logs_without_coords, total_logs))

    print("-- downloading missing images...")
    downloader = Downloader(threads=4, user_agent=USER_AGENT)
    thumbnailer = Thumbnailer(threads=4)
    for cache in caches:
        if cache._preview_image is not None:
            extension = 'noext'
            m = re.match('^.*\.([^.\?]+)(\?.*)?$', cache._preview_image)
            if m:
                extension = m.group(1)
            raw_image = '{}/{}/{}.{}'.format(CACHE_DIR, "orig", cache._code, extension)
            downloader.add_job(cache._preview_image, raw_image)
            thumb_small = '{}/{}/{}.jpg'.format(CACHE_DIR, "small", cache._code)
            thumbnailer.add_job(raw_image, thumb_small, SIZE_SMALL)
            thumb_big = '{}/{}/{}.jpg'.format(CACHE_DIR, "big", cache._code)
            thumbnailer.add_job(raw_image, thumb_big, SIZE_BIG)
    downloader.run()

    print("-- scaling images...")
    thumbnailer.run()

    print("-- creating files...")
    create_db(caches, ".cache/safari.sqlite")
    collect_logs(caches, ".cache/log-data.js")
    createlist(caches, 30)
    create_feed(caches, ".cache/feed.xml")
    create_sidebar(caches, "static/index.html", ".cache/index.html")
 def test_image_bad(self):
     t = Thumbnailer()
     with pytest.raises(Exception):
         thumb = t.make_thumbnail("Not an image")
Exemple #8
0
from flask import Flask, send_file
from flask_restful import Resource, Api, reqparse
from flask_cors import CORS
import numpy as np
from data_backend import Dataset as HDF_Dataset
from dataset_manager import DatasetManager
from thumbnailer import Thumbnailer
from utils import merge_overlapping_filters

DATASET_PATH = "./datasets"
dataset_manager = DatasetManager(DATASET_PATH)
API_BASE_STR = "/api/v1"

# Init thumbnails (clean directory)
thumbnailer = Thumbnailer("./thumbnails")
thumbnailer.clean();

dataset_list = []
for dset_index, name in enumerate(dataset_manager.get_dataset_names()):
    dset = HDF_Dataset(DATASET_PATH, name)
    dataset_list.append({
        "id": dset_index,
        "name": name,
        "device": {
            "name": dset.device.name,
            "version": dset.device.version
        },
        "subsets": [
            {
                "id": subset_index,
                "name": subset,