def test_bake(self):
options = cake.setup(['bake', '-n', '-d',
local_file("Dockerfile.1"),
local_file("cake.conf")])
df = cake.bake_main(options)
assert "disco" in df.entrypoint['args'][0]
assert df[-1]['args'] == ["cake", "layer", "test",
"-d", constants.LAYERS_HOME]
def convert_to_netlist(filename):
utils.spawn(['bash', utils.local_file('eeschema-to-netlist.sh'), filename])
netlist = filename.replace('.sch', '.cir')
if path.exists(netlist):
return (filename, netlist)
else:
return (filename, None)
def test_discovery(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self.loop.set_debug(False)
with Environ(DISCO_CFG="flat.file={}".format(
local_file("mysql.yaml"))):
d = discovery.Discover(configure_from_env())
# low-level API to directly populate
kb = Knowledge()
self.loop.run_until_complete(d.populate(kb))
self.assertEqual(kb['mysql.host'], "localhost:3306")
def test_discovery(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self.loop.set_debug(False)
with Environ(
DISCO_CFG="flat.file={}".format(local_file("mysql.yaml"))):
d = discovery.Discover(configure_from_env())
# low-level API to directly populate
kb = Knowledge()
self.loop.run_until_complete(d.populate(kb))
self.assertEqual(kb['mysql.host'], "localhost:3306")
def load_cls(name):
return joblib.load(local_file(name))
from transitions import transition_features
from feature_extraction import extract_features
import logging
import datetime
import os
from numpy import uint8
from sklearn import svm
from sklearn.linear_model import LogisticRegression
from sklearn.utils import shuffle
from cv2 import GaussianBlur
from sobel_features import sobel_features
import glob
TIMEFORMAT = '%Y%m%d%H%M%S'
allchars = shelve.open(local_file('allchars_dict2'))
label_chars = allchars['label_chars']
chars_label = allchars['allchars']
label_chars = allchars['label_chars']
PCA_PICKLE = 'pca.pkl'
allchars.close()
class TrainingData(object):
def __init__(self, scaled=False, normed=False):
self.scaler = None
self.normalizer = None
self.scaled = scaled
cls = load_cls('logistic-cls')
## Ignore warnings. THis is mostlu in response to incessant sklearn
## warnings about passing in 1d arrays
warnings.filterwarnings("ignore")
print 'ignoring all warnings'
###
rbfcls = load_cls('rbf-cls')
predict_log_proba = cls.predict_log_proba
predict_proba = cls.predict_proba
# Trained characters are labeled by number. Open the shelve that contains
# the mappings between the Unicode character and its number label.
allchars = shelve.open(local_file('allchars_dict2'))
char_to_dig = allchars['allchars']
dig_to_char = allchars['label_chars']
allchars.close()
## Uncomment the line below when enabling viterbi_hidden_tsek
gram3 = pickle.load(open(local_file('3gram_stack_dict.pkl'),'rb'))
word_parts = set(word_parts)
PCA_TRANS = False
trs_prob = np.load(open(local_file('stack_bigram_mat.npz'),'rb'))
trs_prob = trs_prob[trs_prob.files[0]]
cdmap = pickle.load(open(local_file('extended_char_dig.pkl'),'rb'))
from page_elements2 import PageElements
from sklearn.externals import joblib
from tempfile import mkdtemp
from transitions import horizontal_transitions
from utils import add_padding, local_file, check_for_overlap
from viterbi_cython import viterbi_cython
cls = load_cls('logistic-cls')
## commonly called functions
GaussianBlur = cv.GaussianBlur
predict_log_proba = cls.predict_log_proba
boundingRect = cv.boundingRect
char_gaussians = PageElements.char_gaussians
trans_p = load(open(local_file('stack_bigram_logprob32.npz')))
trans_p = trans_p[trans_p.files[0]].transpose()
start_p = load(open(local_file('stack_start_logprob32.npz')))
start_p = start_p[start_p.files[0]]
n_states = trans_p.shape[0]
def combine_many_boxes(bxs):
'''Return the largest bounding box using max height and width from all boxes
bxs is a list of boxes
returns (x,y,w,h) of the new box
'''
if not bxs:
def save_configs(confs):
'''Save collection of configurations'''
pickle.dump(confs, open(local_file(CONF_FILE), 'wb'))
import cPickle as pickle
import uuid
from utils import local_file
import codecs
import json
CONF_FILE = local_file('configs.pkl')
defaults = {
'too_small': 7,
'small_contour_thresh': 2, # times small std dev
'viterbi_post_process': True,
'break_threshold': 2.0,
'hang_off_amount': .4,
'segmenter': 'experimental',
'line_breaker': 'line_cut',
'detect_special_notation': False,
'feature_transform': None, # pca pickle or whatever
}
def save_config(config, path):
'''Save a single configuration'''
return json.dump(config, codecs.open(path, 'w', 'utf-8'))
def load_config(path):
'''Load a single configuration'''
return json.load(codecs.open(path, 'r', 'utf-8'))
def test_layer(self):
layer = cake.Layer.from_path(local_file('disco_layer'))
assert layer.name == "disco-layer"
assert layer.config['name'] == "disco-layer"
assert layer.config['author'] == "bcsaller"
def test_babel(self):
b = babel.Babel()
loop = asyncio.get_event_loop()
output = loop.run_until_complete(b(local_file("test.jsx")))
self.assertIn("createElement", output)
def test_dockerfile(self):
df = Dockerfile(local_file("Dockerfile.1"))
assert df.cmd['args'] == "ps aux"
assert df.entrypoint['args'] == ["/bin/bash"]
assert df['MAINTAINER'] == ["none"]
assert df['LABEL'] == ['version="1.0"', 'description="Multi line"']
def test_dockerfile_mutation(self):
df = Dockerfile(local_file("Dockerfile.1"))
df.cmd = df.entrypoint['args']
df.entrypoint = ["/usr/bin/disco"]
assert df.entrypoint['args'] == ["/usr/bin/disco"]
assert df.cmd['args'] == ["/bin/bash"]
from cv2 import HuMoments, moments, GaussianBlur
from fast_utils import fnormalize, scale_transform
import numpy as np
from sklearn.externals import joblib
from sklearn.mixture import GMM
from sobel_features import sobel_features
from transitions import transition_features
from zernike_moments import zernike_features
import os
from utils import local_file
import platform
SCALER_PATH = 'zernike_scaler-latest'
if os.path.exists(SCALER_PATH):
scaler = joblib.load(local_file(SCALER_PATH))
transform = scaler.transform
try:
sc_o_std = 1.0 / scaler.scale_
except AttributeError:
sc_o_std = 1.0 / scaler.std_
sc_mean = scaler.mean_
SCALER_DEFINED = True
else:
SCALER_DEFINED = False
FEAT_SIZE = 346
hstack = np.hstack
NORM_SIZE = 32
