def preconfigure_settings(self, page_url, settings):
init_gamera()
task_image = load_image(page_url)
miyao_settings = settings.copy()
del miyao_settings['image_width']
return {'image_width': task_image.ncols}
def test_textline_reading_order():
from gamera.core import load_image, init_gamera
init_gamera()
from gamera.plugins.pagesegmentation import textline_reading_order
correct_orders = {
"data/reading_order_2.png":
[(42, 40, 1462, 114), (42, 158, 683, 232), (42, 276, 683, 350),
(42, 418, 683, 492), (42, 560, 683, 633), (822, 158, 1462, 633),
(42, 701, 1462, 775), (42, 843, 494, 917), (562, 843, 1132, 917),
(42, 985, 683, 1059), (822, 985, 1132, 1059),
(1200, 843, 1462, 1059)],
"data/reading_order.png": [(51, 56, 1471, 130), (51, 174, 691, 248),
(51, 292, 691, 366), (51, 434, 691, 508),
(51, 576, 691, 649), (830, 174, 1471, 508),
(830, 576, 1471, 649), (51, 717, 1471, 791),
(51, 859, 691, 933), (51, 1001, 691, 1075),
(830, 859, 1471, 933),
(830, 1001, 1471, 1075)]
}
for file, correct in list(correct_orders.items()):
img = load_image(file)
ccs = img.cc_analysis()
ro = textline_reading_order(ccs)
result = [(a.ul_x, a.ul_y, a.lr_x, a.lr_y) for a in ro]
assert result == correct
del ro
del ccs
del img
def preconfigure_settings(self, page_url, settings):
init_gamera()
task_image = load_image(page_url)
miyao_settings = settings.copy()
del miyao_settings['image_width']
return {'image_width': task_image.ncols}
def run_task(self, result_id, runjob_id, *args, **kwargs):
"""Note: Even if you decide to override the entire run method,
make sure you load and save images with gamera methods, and not PIL methods.
Otherwise the loaded/saved images often don't have the proper pixel types,
and becomes unsuitable for use in a workflow with other gamera-based jobs."""
runjob = RunJob.objects.get(pk=runjob_id)
if runjob.needs_input:
if runjob.status == RunJobStatus.RUN_ONCE_WAITING:
self.retry(args=[result_id, runjob_id], *args, countdown=10, **kwargs)
else:
# This is the first time the job is running.
taskutil.set_running(runjob)
page_url = taskutil.get_page_url(runjob, result_id)
settings = taskutil.get_settings(runjob)
updates = self.preconfigure_settings(page_url, settings)
taskutil.apply_updates(runjob, updates)
taskutil.set_run_once_waiting(runjob)
self.retry(args=[result_id, runjob_id], *args, countdown=10, **kwargs)
else:
taskutil.set_running(runjob)
page_url = taskutil.get_page_url(runjob, result_id)
settings = taskutil.get_settings(runjob)
init_gamera()
task_image = load_image(page_url)
result_image = self.process_image(task_image, settings)
result = taskutil.save_result_as_png(result_image, runjob)
return str(result.uuid)
def main():
gam.init_gamera()
tornado.options.parse_command_line()
application = tornado.web.Application([(r"/", RootHandler), (r"/imagemask(.*?)", ImageMaskHandler)], **settings)
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(9000)
tornado.ioloop.IOLoop.instance().start()
def main():
gam.init_gamera()
tornado.options.parse_command_line()
application = tornado.web.Application([
(r"/", RootHandler),
(r"/imagemask(.*?)", ImageMaskHandler),
], **settings)
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
def main():
gam.init_gamera()
tornado.options.parse_command_line()
application = tornado.web.Application([
(r"/", MainHandler),
(r"/binarize", BinarizationHandler)
], **settings)
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
def get(self, thresh_method):
#retrieve all relevant info
t_value = self.get_argument("thresh_value", None)
img_url_value = self.get_argument("img_url", None)
gam.init_gamera()
simple_thresh_obj = threshold.threshold()
output_img = simple_thresh_obj(gam.load_image(img_url_value),int(t_value))
gam.save_image(output_img,"./static/resultimages/AntA_1520_1520.1_D.Mu_01_006_simpletresh" + t_value +".tiff")
self.write(thresh_method)
def get(self, thresh_method):
#retrieve all relevant info
t_value = self.get_argument("thresh_value", None)
img_url_value = self.get_argument("img_url", None)
gam.init_gamera()
simple_thresh_obj = threshold.threshold()
output_img = simple_thresh_obj(gam.load_image(img_url_value),int(t_value))
gam.save_image(output_img,"./static/resultimages/AntA_1520_1520.1_D.Mu_01_006_simpletresh" + t_value +".tiff")
self.write(thresh_method)
def run_task(self, result_id, runjob_id, *args, **kwargs):
runjob = RunJob.objects.get(pk=runjob_id)
taskutil.set_runjob_status(runjob, RunJobStatus.RUNNING)
# fall through to retrying if we're waiting for input
if runjob.needs_input:
runjob = taskutil.set_runjob_status(runjob,
RunJobStatus.WAITING_FOR_INPUT)
self.retry(args=[result_id, runjob_id],
*args,
countdown=10,
**kwargs)
if runjob.status == RunJobStatus.WAITING_FOR_INPUT:
runjob = taskutil.set_runjob_status(runjob, RunJobStatus.RUNNING)
if result_id is None:
# this is the first job in a run
page = runjob.page.compat_file_path
else:
# we take the page image we want to operate on from the previous result object
result = Result.objects.get(pk=result_id)
page = result.result.path
new_result = Result(run_job=runjob)
taskutil.save_instance(new_result)
result_save_path = new_result.result_path
settings = {}
for s in runjob.job_settings:
setting_name = "_".join(s['name'].split(" "))
setting_value = argconvert.convert_to_arg_type(
s['type'], s['default'])
settings[setting_name] = setting_value
init_gamera()
task_image = load_image(page)
tdir = tempfile.mkdtemp()
task_function = self.name.split(".")[-1]
result_image = getattr(task_image, task_function)(**settings)
result_file = "{0}.png".format(str(uuid.uuid4()))
result_image.save_image(os.path.join(tdir, result_file))
f = open(os.path.join(tdir, result_file))
taskutil.save_file_field(new_result.result,
os.path.join(result_save_path, result_file),
File(f))
f.close()
shutil.rmtree(tdir)
return str(new_result.uuid)
def preconfigure_settings(self, page_url, settings):
init_gamera()
task_image = load_image(page_url)
ranked_page = task_image.rank(9, 9, 0)
miyao_settings = settings.copy()
del miyao_settings['polygon_outer_points'], miyao_settings['image_width']
staff_finder = StaffFinder_miyao(ranked_page, 0, 0)
staff_finder.find_staves(**miyao_settings)
poly_list = staff_finder.get_polygon()
poly_list = fix_poly_point_list(poly_list, staff_finder.staffspace_height)
poly_json_list = create_polygon_outer_points_json_dict(poly_list)
return {'polygon_outer_points': poly_json_list, 'image_width': task_image.ncols}
def home(request):
init_gamera()
encoded_glyphs = []
gamera_glyphs = gamera_xml.glyphs_from_xml(
'/Volumes/Shared/LU-OMR/Liber_Usualis_NO_ST/Processed_Pages/1234/classifier_glyphs_1234_654.xml'
)
for gamera_glyph in gamera_glyphs:
glyph = gamera_glyph.to_rgb().to_pil()
buf = StringIO.StringIO()
glyph.save(buf, format='PNG')
png = buf.getvalue()
encoded_png = base64.b64encode(png)
encoded_glyphs.append(encoded_png)
for glyph in encoded_glyphs:
print glyph
print ""
return render(request, 'index.html', {'encoded_glyphs': encoded_glyphs})
def home(request):
init_gamera()
encoded_glyphs = []
gamera_glyphs = gamera_xml.glyphs_from_xml('/Volumes/Shared/LU-OMR/Liber_Usualis_NO_ST/Processed_Pages/1234/classifier_glyphs_1234_654.xml')
for gamera_glyph in gamera_glyphs:
glyph = gamera_glyph.to_rgb().to_pil()
buf = StringIO.StringIO()
glyph.save(buf, format='PNG')
png = buf.getvalue()
encoded_png = base64.b64encode(png)
encoded_glyphs.append(encoded_png)
for glyph in encoded_glyphs:
print glyph
print ""
return render(request, 'index.html', {
'encoded_glyphs': encoded_glyphs
})
def post(self, thresh_method):
#retrieve all relevant info
json_value = self.get_argument("JSON", None)
img_url_value = "./static/images/testim.tiff"
json_data = json.loads(json_value)
im = Image.open(img_url_value)
mask = Image.new('1', im.size)
mdraw = ImageDraw.Draw(mask)
for poly in json_data:
flatPoly = [j for i in poly for j in i]
mdraw.polygon(flatPoly, outline=1, fill=1)
out = ImageMath.eval("~(a - b)", a=im, b=mask)
out.save("./static/resultimages/testout.tiff")
#print out
#print thresh_method
gam.init_gamera()
def run_task(self, result_id, runjob_id, *args, **kwargs):
"""Note: Even if you decide to override the entire run method,
make sure you load and save images with gamera methods, and not PIL methods.
Otherwise the loaded/saved images often don't have the proper pixel types,
and becomes unsuitable for use in a workflow with other gamera-based jobs."""
runjob = RunJob.objects.get(pk=runjob_id)
if runjob.needs_input:
if runjob.status == RunJobStatus.RUN_ONCE_WAITING:
self.retry(args=[result_id, runjob_id],
*args,
countdown=10,
**kwargs)
else:
# This is the first time the job is running.
taskutil.set_running(runjob)
page_url = taskutil.get_page_url(runjob, result_id)
settings = taskutil.get_settings(runjob)
updates = self.preconfigure_settings(page_url, settings)
taskutil.apply_updates(runjob, updates)
taskutil.set_run_once_waiting(runjob)
self.retry(args=[result_id, runjob_id],
*args,
countdown=10,
**kwargs)
else:
taskutil.set_running(runjob)
page_url = taskutil.get_page_url(runjob, result_id)
settings = taskutil.get_settings(runjob)
init_gamera()
task_image = load_image(page_url)
result_image = self.process_image(task_image, settings)
result = taskutil.save_result_as_png(result_image, runjob)
return str(result.uuid)
def process_image(self, task_image, settings):
"""
Note that if the incoming image is onebit, it will convert it to greyscale,
do the cropping, and then convert it back to onebit. This can sometimes
lead to unexpected behavior, especially because converting to greyscale
can do some interpolation, and converting back to onebit uses a default
binarizaiton algorithm (the otsu threshold). These can change the image.
"""
init_gamera()
need_to_change_back = False
if task_image.data.pixel_type == 0:
task_image = task_image.to_greyscale()
need_to_change_back = True
crop_mask = CropBorderRemovalTask.create_mask(task_image, **settings)
result_image = task_image.mask(crop_mask)
if need_to_change_back:
result_image = result_image.to_onebit()
return result_image
from os.path import isfile, join
import numpy as np
import gamera.core as gc
gc.init_gamera()
import matplotlib.pyplot as plt
from gamera.plugins.image_utilities import union_images
import pickle
import itertools as iter
import os
import re
# PARAMETERS FOR PREPROCESSING
saturation_thresh = 0.9
sat_area_thresh = 150
despeckle_amt = 100 # an int in [1,100]: ignore ccs with area smaller than this
noise_area_thresh = 100 # an int in : ignore ccs with area smaller than this
# PARAMETERS FOR TEXT LINE SEGMENTATION
filter_size = 30 # size of moving-average filter used to smooth projection
prominence_tolerance = 0.70 # log-projection peaks must be at least this prominent
collision_strip_scale = 1 # in [0,inf]; amt of each cc to consider when clipping
remove_capitals_scale = 10000 # removes large ccs. turned off for now
# CC GROUPING (BLOBS)
cc_group_gap_min = 20 # any gap at least this wide will be assumed to be a space between words!
max_distance_to_staff = 200
# i need a custom set of colors when working on cc analysis because i'm too colorblind for the default :(
colors = [
gc.RGBPixel(150, 0, 0),
gc.RGBPixel(0, 100, 0),
from gamera.core import init_gamera, load_image, RGBPixel
from gamera import gamera_xml
import numpy as np
from SliceFinding import SliceFinder
import sys
import datetime
import os
import glob
init_gamera()
class ProjectionSplitter(object):
# Given an image of a neume, should return
# a list of separated neume components
def __init__(self, **kwargs):
self.kwargs = kwargs
# Things to use later
self.image = None
self.analysis_image = None
self.sf = None
self.min_glyph_size = kwargs[
'min_glyph_size'] # min number of x or y pixels for a glyph
self.max_recursive_cuts = kwargs[
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
from gamera import core, knn, graph, gamera_xml
core.init_gamera()
def get_lengths(node, depth, lengths, cur_depth=0, path = {}):
if cur_depth >= depth:
return
for edge in node.edges:
if path.has_key(edge):
continue
path[edge] = None
lengths.append(edge.cost)
get_lengths(edge.traverse(node), depth, lengths, cur_depth + 1, path)
def label(graph, node, label_start, label):
for node in graph.DFS(node):
node().classify_automatic(label_start + str(label))
def run(self, *args, **kwargs):
"""
This runs the optimization on all the classifiers in the project.
The optimization can be run with many different settings. For
convenience, I've included all the settings in this script and
commented out the ones that I'm not currently using. Feel free
to tweak with them.
For most of the arguments, I hope it is implicitly clear which
ones are int and which ones are float/double. For example,
arg_one=3 means arg_one should be an int, while arg_two=3.0 means
arg_two should be a double/float.
Later on, we can build a client interface to tweak all these settings.
Detailed documentation can be found at http://gamera.sourceforge.net/doc/html/ga_optimization.html
Code template taken from http://gamera.sourceforge.net/doc/html/ga_optimization.html#script-usage
"""
init_gamera()
classifiers = Classifier.objects.all()
for classifier in classifiers:
project = classifier.project
optimization_start = timezone.now()
print "Optimizing classifier {0}".format(classifier.name)
cknn = knn.kNNNonInteractive(classifier.file_path,
features = 'all',
normalize = False)
print "Setting base settings"
baseSettings = knnga.GABaseSetting()
baseSettings.opMode = knnga.GA_WEIGHTING # Or knnga.GA_SELECTION
baseSettings.popSize = 75
baseSettings.crossRate = 0.95
baseSettings.mutRate = 0.05
print "Settings selection options"
selection = knnga.GASelection()
selection.setRoulettWheelScaled(2.0)
#selection.setRoulettWheelScaled(double pressure=2.0) # Pressure \in [1,2]
#selection.setRandomSelection()
#selection.setRankSelection(pressure=2.0, exponent=1.0)
#selection.setStochUniSampling()
#selection.setRoulettWheel()
#selection.setTournamentSelection(tSize=3)
print "Setting crossover settings"
crossover = knnga.GACrossover()
crossover.setUniformCrossover(0.5)
#crossover.setUniformCrossover(double preference = 0.5)
#crossover.setNPointCrossover(n=1)
#crossover.setHypercubeCrossover(int numFeatures, double min, double max, alpha=0.0)
#crossover.setSBXcrossover(int numFeatures, double min, double max, eta=0.0)
#crossover.setSegmentCrossover(int numFeatures, double min, double max, alpha=0.0)
print "Setting Mutation settings"
mutation = knnga.GAMutation()
#mutation.setShiftMutation()
mutation.setSwapMutation()
#mutation.setBinaryMutation(rate=0.05, normalize=False)
mutation.setBinaryMutation(0.05, False)
#mutation.setGaussMutation(int numFeatures, double min, double max, double sigma, double rate)
#mutation.setInversionMutation()
print "Setting replacement settings"
replacement = knnga.GAReplacement()
replacement.setSSGAdetTournament(3)
#replacement.setSSGAdetTournament(int tSize=3)
#replacement.setGenerationalReplacement()
#replacement.setSSGAworse()
print "Setting stop criteria"
stop = knnga.GAStopCriteria()
#stop.setSteadyStateStop(int minGens=100, int noChangeGens=20)
stop.setSteadyStateStop(100, 20)
#stop.setBestFitnessStop(optimum=1.0)
#stop.setMaxFitnessEvals(n=5000)
#stop.setMaxGenerations(100)
print "Setting parallelization settings"
parallel = knnga.GAParallelization()
parallel.mode = True
parallel.thredNum = 4
# Combine each setting object into one main object
ga = knnga.GAOptimization(cknn, baseSettings, selection,
crossover, mutation, replacement,
stop, parallel)
print "Beginning calculation..."
ga.startCalculation()
print "Done! Saving the produced settings."
optimization_end = timezone.now()
# Choosing a name for the new classifier_setting
date_string = datetime.now().strftime("%Y_%m_%d_%I%M%p")
setting_name = "{0}:{1} Periodic Optimization".format(classifier.name[:200], date_string,)
tdir = tempfile.mkdtemp()
temp_xml_filepath = os.path.join(tdir, str(classifier.uuid) + '.xml')
cknn.save_settings(temp_xml_filepath)
try:
classifier = refetch_from_db(classifier)
except ObjectDeletedError:
print "Sadly classifier {0} was deleted.".format(classifier.name)
# Now this is a lonely setting file with no classifier to hang out with.
classifier = None
classifier_setting_instance = ClassifierSetting.objects.create(name=setting_name,
project=project,
fitness=ga.bestFitness,
producer=classifier,
optimization_started_at = optimization_start,
optimization_finished_at = optimization_end)
with open(temp_xml_filepath, 'rb') as f:
taskutil.save_file_field(classifier_setting_instance.settings_file, 'settings_xml', File(f))
shutil.rmtree(tdir)
__version__ = "0.1.0"
from gamera.core import init_gamera
init_gamera()
from rodan.jobs import module_loader
module_loader('rodan.jobs.gamera_rodan.wrappers.classification')
module_loader('rodan.jobs.gamera_rodan.wrappers.masking')
module_loader('rodan.jobs.gamera_rodan.wrappers.plugins.binarization')
module_loader('rodan.jobs.gamera_rodan.wrappers.plugins.cc_analysis')
module_loader('rodan.jobs.gamera_rodan.wrappers.plugins.image_conversion')
module_loader('rodan.jobs.gamera_rodan.wrappers.plugins.morphology')
module_loader('rodan.jobs.gamera_rodan.wrappers.plugins.threshold')
module_loader('rodan.jobs.gamera_rodan.wrappers.plugins.image_utilities')
module_loader('rodan.jobs.gamera_rodan.wrappers.toolkits.music-staves.miyao')
module_loader('rodan.jobs.gamera_rodan.wrappers.toolkits.music-staves.roach_tatem')
module_loader('rodan.jobs.gamera_rodan.wrappers.toolkits.document-preprocessing-toolkit.stable_paths')
module_loader('rodan.jobs.gamera_rodan.wrappers.toolkits.custom.poly_mask')
def run(self, *args, **kwargs):
"""
This runs the optimization on all the classifiers in the project.
The optimization can be run with many different settings. For
convenience, I've included all the settings in this script and
commented out the ones that I'm not currently using. Feel free
to tweak with them.
For most of the arguments, I hope it is implicitly clear which
ones are int and which ones are float/double. For example,
arg_one=3 means arg_one should be an int, while arg_two=3.0 means
arg_two should be a double/float.
Later on, we can build a client interface to tweak all these settings.
Detailed documentation can be found at http://gamera.sourceforge.net/doc/html/ga_optimization.html
Code template taken from http://gamera.sourceforge.net/doc/html/ga_optimization.html#script-usage
"""
init_gamera()
classifiers = Classifier.objects.all()
for classifier in classifiers:
project = classifier.project
optimization_start = timezone.now()
print "Optimizing classifier {0}".format(classifier.name)
cknn = knn.kNNNonInteractive(classifier.file_path,
features='all',
normalize=False)
print "Setting base settings"
baseSettings = knnga.GABaseSetting()
baseSettings.opMode = knnga.GA_WEIGHTING # Or knnga.GA_SELECTION
baseSettings.popSize = 75
baseSettings.crossRate = 0.95
baseSettings.mutRate = 0.05
print "Settings selection options"
selection = knnga.GASelection()
selection.setRoulettWheelScaled(2.0)
#selection.setRoulettWheelScaled(double pressure=2.0) # Pressure \in [1,2]
#selection.setRandomSelection()
#selection.setRankSelection(pressure=2.0, exponent=1.0)
#selection.setStochUniSampling()
#selection.setRoulettWheel()
#selection.setTournamentSelection(tSize=3)
print "Setting crossover settings"
crossover = knnga.GACrossover()
crossover.setUniformCrossover(0.5)
#crossover.setUniformCrossover(double preference = 0.5)
#crossover.setNPointCrossover(n=1)
#crossover.setHypercubeCrossover(int numFeatures, double min, double max, alpha=0.0)
#crossover.setSBXcrossover(int numFeatures, double min, double max, eta=0.0)
#crossover.setSegmentCrossover(int numFeatures, double min, double max, alpha=0.0)
print "Setting Mutation settings"
mutation = knnga.GAMutation()
#mutation.setShiftMutation()
mutation.setSwapMutation()
#mutation.setBinaryMutation(rate=0.05, normalize=False)
mutation.setBinaryMutation(0.05, False)
#mutation.setGaussMutation(int numFeatures, double min, double max, double sigma, double rate)
#mutation.setInversionMutation()
print "Setting replacement settings"
replacement = knnga.GAReplacement()
replacement.setSSGAdetTournament(3)
#replacement.setSSGAdetTournament(int tSize=3)
#replacement.setGenerationalReplacement()
#replacement.setSSGAworse()
print "Setting stop criteria"
stop = knnga.GAStopCriteria()
#stop.setSteadyStateStop(int minGens=100, int noChangeGens=20)
stop.setSteadyStateStop(100, 20)
#stop.setBestFitnessStop(optimum=1.0)
#stop.setMaxFitnessEvals(n=5000)
#stop.setMaxGenerations(100)
print "Setting parallelization settings"
parallel = knnga.GAParallelization()
parallel.mode = True
parallel.thredNum = 4
# Combine each setting object into one main object
ga = knnga.GAOptimization(cknn, baseSettings, selection, crossover,
mutation, replacement, stop, parallel)
print "Beginning calculation..."
ga.startCalculation()
print "Done! Saving the produced settings."
optimization_end = timezone.now()
# Choosing a name for the new classifier_setting
date_string = datetime.now().strftime("%Y_%m_%d_%I%M%p")
setting_name = "{0}:{1} Periodic Optimization".format(
classifier.name[:200],
date_string,
)
tdir = tempfile.mkdtemp()
temp_xml_filepath = os.path.join(tdir,
str(classifier.uuid) + '.xml')
cknn.save_settings(temp_xml_filepath)
try:
classifier = refetch_from_db(classifier)
except ObjectDeletedError:
print "Sadly classifier {0} was deleted.".format(
classifier.name)
# Now this is a lonely setting file with no classifier to hang out with.
classifier = None
classifier_setting_instance = ClassifierSetting.objects.create(
name=setting_name,
project=project,
fitness=ga.bestFitness,
producer=classifier,
optimization_started_at=optimization_start,
optimization_finished_at=optimization_end)
with open(temp_xml_filepath, 'rb') as f:
taskutil.save_file_field(
classifier_setting_instance.settings_file, 'settings_xml',
File(f))
shutil.rmtree(tdir)
