def install():
"""
Install mercurial in a python virtual environment.
:Example:
fab --config=config.conf mercurial.install
"""
# Python build headers.
packages = [
'python-setuptools',
'python-virtualenv',
'python-dev',
'python-tk',
]
# Install python build headers.
sudo('apt-get -y install {}'.format(' '.join(packages)))
# Put all virtual environments in one directory.
run('mkdir -p {}'.format(utils.home('venvs')))
# Create the virtual environment.
venvs = utils.home('venvs')
with cd(venvs):
run('virtualenv mercurial')
# Activate the virtual environment.
with prefix('source {}/mercurial/bin/activate'.format(venvs)):
run('pip install mercurial')
def public():
"""
Create the public directories.
:Example:
fab --config=config.conf nginx.public
"""
require('PROJECT_NAME')
media_dir = utils.home('public', env.PROJECT_NAME, 'media')
static_dir = utils.home('public', env.PROJECT_NAME, 'static')
run('mkdir -p {}'.format(media_dir))
run('mkdir -p {}'.format(static_dir))
def migrate():
"""
Run database migrations.
:Example:
fab --config=config.conf project.migrate
"""
require('PROJECT_VENV')
require('PROJECT_NAME')
venv = utils.home('venvs', env.PROJECT_VENV)
with prefix('source {0}/bin/activate'.format(venv)):
with cd(utils.home('apps', env.PROJECT_NAME)):
run('python manage.py migrate --settings=config.settings.prod')
def helper(tree=None, source='src', options=lambda _: []):
"""Run main in clean environments, with various '--source' options.
Each invocation is done against a separate, temporary directory.
main is currently run 3 times:
- from the current working directory, with `--source tmpdir/src`;
- from `tmpdir`, with `--source src`;
- from `tmpdir/src`, without --source argument.
"""
homedir = Path(tempfile.mkdtemp())
with home(homedir):
with chdir(homedir):
with directory_tree(tree) as tmpdir:
main('--source', tmpdir / source, *options(tmpdir))
yield tmpdir
with directory_tree(tree) as tmpdir:
with chdir(tmpdir):
main('--source', source, *options(tmpdir))
yield tmpdir
with directory_tree(tree) as tmpdir:
with chdir(tmpdir / source):
main(*options(tmpdir))
yield tmpdir
# Implicitely asserts that `homedir` == `cwd` is empty
homedir.rmdir()
def helper(tree=None, source='src', options=lambda _: []):
"""Run main in clean environments, with various '--source' options.
Each invocation is done against a separate, temporary directory.
main is currently run 3 times:
- from the current working directory, with `--source tmpdir/src`;
- from `tmpdir`, with `--source src`;
- from `tmpdir/src`, without --source argument.
"""
homedir = Path(tempfile.mkdtemp())
with home(homedir):
with chdir(homedir):
with directory_tree(tree) as tmpdir:
main('--source', tmpdir / source, *options(tmpdir))
yield tmpdir
with directory_tree(tree) as tmpdir:
with chdir(tmpdir):
main('--source', source, *options(tmpdir))
yield tmpdir
with directory_tree(tree) as tmpdir:
with chdir(tmpdir / source):
main(*options(tmpdir))
yield tmpdir
# Implicitely asserts that `homedir` == `cwd` is empty
homedir.rmdir()
def collectstatic():
"""
Deploy static files.
:Example:
fab --config=config.conf project.collectstatic
"""
require('PROJECT_VENV')
require('PROJECT_NAME')
venv = utils.home('venvs', env.PROJECT_VENV)
with prefix('source {0}/bin/activate'.format(venv)):
with cd(utils.home('apps', env.PROJECT_NAME)):
run('python manage.py collectstatic --noinput --settings=config.settings.prod')
def config():
"""
Upload a custom configuration file.
:Example:
fab --config=config.conf nginx.config
"""
local = utils.file_path('nginx', 'default')
remote = '/etc/nginx/sites-available/default'
target = '/etc/nginx/sites-enabled/default'
put(local, remote, mode='0644', use_sudo=True)
# Change owner to "root".
sudo('chown root:root {}'.format(remote))
# Remove old symlink.
sudo('rm {}'.format(target))
# Create new symlink.
sudo('ln -s {} {}'.format(remote, target))
# Create the nginx log directory.
run('mkdir -p {}'.format(utils.home('logs', 'nginx')))
def virtualenv_setup():
"""
Creates a new virtualenv that will hold the NGAS installation
"""
ngasInstallDir = ngas_install_dir()
if check_dir(ngasInstallDir):
overwrite = ngas_overwrite_installation()
if not overwrite:
msg = (
"%s exists already. Specify NGAS_OVERWRITE_INSTALLATION to overwrite, "
"or a different NGAS_INSTALL_DIR location")
abort(msg % (ngasInstallDir, ))
run("rm -rf %s" % (ngasInstallDir, ))
# Check which python will be bound to the virtualenv
ppath = check_python()
if not ppath:
ppath = python_setup(os.path.join(home(), 'python'))
# Use our create_venv.sh script to create the virtualenv
# It already handles the download automatically if no virtualenv command is
# found in the system, and also allows to specify a python executable path
with cd(ngas_source_dir()):
run("./create_venv.sh -p {0} {1}".format(ppath, ngasInstallDir))
# Download this particular certifcate; otherwise pip complains
# in some platforms
if ngas_use_custom_pip_cert():
if not (check_dir('~/.pip')):
run('mkdir ~/.pip')
with cd('~/.pip'):
download('http://curl.haxx.se/ca/cacert.pem')
run('echo "[global]" > ~/.pip/pip.conf; echo "cert = {0}/.pip/cacert.pem" >> ~/.pip/pip.conf;'
.format(home()))
# Update pip and install wheel; this way we can install binary wheels from
# PyPI if available (like astropy)
# TODO: setuptools and python-daemon are here only because
# python-daemon 2.1.2 is having a problem to install via setuptools
# but not via pip (see https://pagure.io/python-daemon/issue/2 and
# https://pagure.io/python-daemon/issue/3).
# When this problem is fixed we'll fix our dependency on python-daemo
# to avoid this issue entirely
virtualenv('pip install -U pip wheel setuptools python-daemon')
success("Virtualenv setup completed")
def update():
"""
Pull the latest changes from the remote repo.
"""
require('PROJECT_NAME')
with cd(utils.home('apps', env.PROJECT_NAME)):
run('hg pull')
run('hg up')
def conda_list_environments():
"""
List all conda virtual environments.
:Example:
fab --config=config.conf python.conda_list_environments
"""
conda = '{0}/bin/conda'.format(utils.home('apps', 'miniconda'))
run('{conda} info --envs'.format(conda=conda))
def config_django():
require('PROJECT_NAME')
project = env.PROJECT_NAME
filename = 'prod.py'
local_file = utils.file_path('project', filename)
remote_file = utils.home('apps', project, 'config', 'settings', filename)
put(local_file, remote_file)
def test_defaults():
assert config.defaults().destination == Path.home()
assert config.defaults().source == Path.cwd()
with directory_tree({}) as tmpdir:
with chdir(tmpdir):
assert config.defaults().source.samefile(tmpdir)
assert Path.cwd().samefile(tmpdir)
with home(tmpdir):
assert config.defaults().destination.samefile(tmpdir)
assert Path.home().samefile(tmpdir)
def test_defaults():
assert config.defaults().destination == Path.home()
assert config.defaults().source == Path.cwd()
with directory_tree({}) as tmpdir:
with chdir(tmpdir):
assert config.defaults().source.samefile(tmpdir)
assert Path.cwd().samefile(tmpdir)
with home(tmpdir):
assert config.defaults().destination.samefile(tmpdir)
assert Path.home().samefile(tmpdir)
def clone():
"""
Clone the project repository.
:Example:
fab --config=config.conf project.clone
"""
require('PROJECT_NAME')
require('PROJECT_REPO')
require('MERCURIAL_BIN')
# Create the "apps" directory if it does not exist.
run('mkdir -p {}'.format(utils.home('apps')))
if files.exists(utils.home('apps', env.PROJECT_NAME)):
delete()
with cd(utils.home('apps')):
run('{0} clone {1} {2}'.format(env.MERCURIAL_BIN,
env.PROJECT_REPO,
env.PROJECT_NAME))
def conda_install():
"""
Download and install "miniconda".
:Example:
fab --config=config.conf python.conda_install
"""
require('MINICONDA_NAME')
require('MINICONDA_FILE')
with cd(utils.home()):
if not files.exists(env.MINICONDA_NAME):
# Download the miniconda installer.
run('wget {0}'.format(env.MINICONDA_FILE))
# Give permission to execute installer.
run('chmod u+x {0}'.format(env.MINICONDA_NAME))
# Put miniconda in the "apps" directory.
apps = utils.home('apps')
run('mkdir -p {0}'.format(apps))
# OPTIONS:
#
# -b Run in silent mode.
# -p The path where miniconda will be installed.
run('bash {0} -b -p {1}/miniconda'.format(env.MINICONDA_NAME, apps))
# Add the executables to the system path.
export = 'export PATH="{0}/miniconda/bin:$PATH"'.format(apps)
files.append('.bashrc', '###########')
files.append('.bashrc', '# MINICONDA')
files.append('.bashrc', export)
# Remove the installer after installation.
run('rm {0}'.format(env.MINICONDA_NAME))
def test_defaults():
for path in (Path.cwd(), Path.home()):
default = Config.defaults(path)
assert default.destination == Path.home()
assert 'source' not in default
with directory_tree({}) as tmpdir:
with home(tmpdir):
default = Config.defaults(tmpdir)
# The home directory should have changed.
assert Path.home().samefile(tmpdir)
# Emanate's default destination should be the new home directory.
assert default.destination.samefile(tmpdir)
assert 'source' not in default
def conda_create_environment(name, python='3'):
"""
Create a new virtual environment. Installs "Python 3" by default.
:Example:
fab --config=config.conf python.conda_create_environment:name=bugtrax
"""
conda = '{0}/bin/conda'.format(utils.home('apps', 'miniconda'))
run('{conda} create --name {name} python={python} --yes'.format(
name=name,
conda=conda,
python=python))
def conda_install_requirements(venv):
"""
Run "pip install -r" on the requirements file.
:Example:
fab --config=config.conf python.conda_install_requirements:venv=bugtrax
"""
# Upload the requirements file.
put(utils.files('requirements', 'base.txt'), utils.home('base.txt'))
put(utils.files('requirements', 'prod.txt'), utils.home('prod.txt'))
# Activate the virtual environment.
activate = '{0}/bin/activate'.format(utils.home('apps', 'miniconda'))
with prefix('source {activate} {venv}'.format(venv=venv, activate=activate)):
run('pip install -r {0}'.format(utils.home('prod.txt')))
# Remove the uploaded files.
with cd(utils.home()):
run('rm {0}'.format(utils.home('base.txt')))
run('rm {0}'.format(utils.home('prod.txt')))
def ngas_root_dir():
key = 'NGAS_ROOT_DIR'
if key not in env:
env[key] = os.path.abspath(os.path.join(home(), NGAS_ROOT_DIR_NAME))
return env[key]
def ngas_install_dir():
key = 'NGAS_INSTALL_DIR'
if key not in env:
env[key] = os.path.abspath(os.path.join(home(), NGAS_INSTALL_DIR_NAME))
return env[key]
def APP_source_dir():
key = 'APP_SRC_DIR'
if key not in env:
env[key] = os.path.abspath(os.path.join(home(), APP_SRC_DIR_NAME))
return env[key]
def ngas_source_dir():
key = 'NGAS_SRC_DIR'
if key not in env:
env[key] = os.path.abspath(os.path.join(home(), NGAS_SRC_DIR_NAME))
return env[key]
def _main(args):
model_path = os.path.expanduser(args.model_path)
assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.'
anchors_path = os.path.expanduser(args.anchors_path)
classes_path = os.path.expanduser(args.classes_path)
test_path = os.path.expanduser(args.test_path)
output_path = os.path.expanduser(args.output_path)
label_file = args.label_file
if not os.path.exists(output_path):
print('Creating output path {}'.format(output_path))
os.mkdir(output_path)
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
# TODO: USE THIS INSTEAD
# with open(anchors_path) as f:
# anchors = f.readline()
# anchors = [float(x) for x in anchors.split(',')]
# anchors = np.array(anchors).reshape(-1, 2)
# yolo_model = load_model(model_path)
# ALEXANDER HACK
anchors = np.array(
((0.57273, 0.677385), (1.87446, 2.06253), (3.33843, 5.47434),
(7.88282, 3.52778), (9.77052, 9.16828)))
yolo_model, model = create_model(anchors,
class_names,
load_pretrained=False,
freeze_body=True)
model.load_weights(model_path)
# Verify model, anchors, and classes are compatible
num_classes = len(class_names)
num_anchors = len(anchors)
# TODO: Assumes dim ordering is channel last
model_output_channels = yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == num_anchors * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes. ' \
'Specify matching anchors and classes with --anchors_path and ' \
'--classes_path flags.'
print('{} model, anchors, and classes loaded.'.format(model_path))
# Check if model is fully convolutional, assuming channel last order.
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(class_names), 1., 1.)
for x in range(len(class_names))]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
# TODO: Wrap these backend operations with Keras layers.
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=args.score_threshold,
iou_threshold=args.iou_threshold)
lf = open(label_file)
num_tests = 10
count = 0
while (count < num_tests):
count += 1
line = lf.readline()
line = line.split(' ')
image_file = os.path.join(utils.home(), 'data', line[0])
image = Image.open(image_file)
image_file = line[0]
pts = line[1:]
pts = np.asarray(pts, dtype=np.float)
image = np.array(image)
image = utils.imwarp(image, pts, sz=(416, 416))
image_data = image
image_data = np.expand_dims(image_data, 0)
image = Image.fromarray(image)
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
yolo_model.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), image_file))
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print(label, (left, top), (right, bottom))
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
image.save(os.path.join(output_path, image_file), quality=90)
sess.close()
def delete():
"""
Delete the cloned repo from the server.
"""
run('rm -r {}'.format(utils.home('apps', env.PROJECT_NAME)))
def APP_install_dir():
key = 'APP_INSTALL_DIR'
if key not in env:
env[key] = os.path.abspath(os.path.join(home(), APP_INSTALL_DIR_NAME))
return env[key]
if (t == ''):
return BLANK_LABEL
# capitalize letter
t = t.capitalize()
return t
def black_pixel_percentage(img):
w = img.shape[1]
return float(1 - cv2.countNonZero(img) / w**2)
'''Main Function'''
# get the path of the labels text file
path = os.path.join(os.path.join(utils.home(), 'labels'), 'labels.txt')
instructions = "Press the letter on the keyboard that represents the letter shown. " \
"\nIf no letter is show, press the spacebar to see the next tile."
print(instructions)
txtfile = os.path.join(utils.home(), 'labels', 'testing.txt')
f = open(txtfile, "a+")
if (not os.stat(txtfile).st_size == 0):
exit()
# get the image of the board
img = cv2.imread("/Users/Alex/Desktop/Summer-2019/scrabble/data/testing.png")
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
w, h = img.shape[0], img.shape[1]
print("full image: width: {}, height: {}".format(w, h))
# get the individual tiles from the image
sqs = utils.squares_from_img(img)
# reshape to 4 dimensions so that x and y position can be tracked
def APP_root_dir():
key = 'APP_ROOT_DIR'
if key not in env:
env[key] = os.path.abspath(os.path.join(home(), APP_ROOT_DIR_NAME))
return env[key]
import cv2
import utils
import ipdb
import os
import numpy as np
import argparse
parser = argparse.ArgumentParser(description='Input a text file and an image to overlay labels on the image')
parser.add_argument('-ld', '--labeldirectory', type=str, help='the directory containing the label text file',
default=os.path.join(utils.home(), 'labels'))
parser.add_argument('-d', '--datadirectory', type=str, help='the directory containing the image',
default=os.path.join(utils.home(), 'data'))
parser.add_argument('-n', '--name', type=str, help='the name of an image with no extension ex: Photo_2005-08-20_006 '
'leave empty to cycle through a directory')
def overlay_text(img, lf):
for line in lf.readlines():
if(line.split(' ')[0] == 'NONE'):
continue
points = line.split(' ')
x, y = float(points[1]), float(points[2])
x, y = np.float32(x * img.shape[0]), np.float32(y * img.shape[1])
cv2.putText(img=img, org=(x,y), text=points[0], fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=10,
color=(0, 0, 255), thickness=7)
cv2.namedWindow("img", cv2.WINDOW_NORMAL)
cv2.resizeWindow("img", 1000, 1000)
cv2.imshow("img", img)
cv2.waitKey(0)
def index():
'''root home page'''
return utils.home(CHANNELS.channels)
parser.add_argument('-s',
'--score_threshold',
type=float,
help='threshold for bounding box scores, default .3',
default=.3)
parser.add_argument('-iou',
'--iou_threshold',
type=float,
help='threshold for non max suppression IOU, default .5',
default=.5)
parser.add_argument(
'-l',
'--label_file',
type=str,
help='Label file containing file names and corners of board.',
default=os.path.join(utils.home(), 'labels', 'labels.txt'))
def _main(args):
model_path = os.path.expanduser(args.model_path)
assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.'
anchors_path = os.path.expanduser(args.anchors_path)
classes_path = os.path.expanduser(args.classes_path)
test_path = os.path.expanduser(args.test_path)
output_path = os.path.expanduser(args.output_path)
label_file = args.label_file
if not os.path.exists(output_path):
print('Creating output path {}'.format(output_path))
os.mkdir(output_path)
