def Run(args):
utils.setup_environment()
os.environ["OMP_NUM_THREADS"] = "1"
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
os.environ["NUMEXPR_NUM_THREADS"] = "1"
if args.threads == None:
if args.tensor_fn == "PIPE":
param.NUM_THREADS = 4
else:
param.NUM_THREADS = args.threads
param.NUM_THREADS -= 1
if param.NUM_THREADS < 1: param.NUM_THREADS = 1
m = cv.Clair()
m.init()
m.restore_parameters(os.path.abspath(args.chkpnt_fn))
if args.activation_only:
log_activation(args, m, utils)
else:
Test(args, m, utils)
def Prepare(args):
utils.setup_environment()
m = cv.Clair()
m.init()
m.restore_parameters(args.chkpnt_fn)
return m, utils, 1
def setup_class(cls):
'''ensure that we have a clean environment
before running any tests
'''
_ = setup_environment()
app = create_app(TEST_CONFIG_PATH)
cls.api_client = app.test_client()
def setup_class(cls):
'''ensure that we have a clean environment
before running any tests
'''
_ = setup_environment()
_ = _stamp_notes(CONFIG['notes_directory'],
stamp_todos=False,
stamp_today=False,
stamp_questions=True,
stamp_answers=True,
grep_path=CONFIG['grep_path'])
def setup_class(cls):
# ensure that we have a clean environment before running any tests
_ = setup_environment()
_ = _stamp_notes(CONFIG['notes_directory'],
stamp_todos=True,
stamp_today=True,
stamp_questions=False,
stamp_answers=False,
grep_path=CONFIG['grep_path'])
app = create_app(TEST_CONFIG_PATH)
cls.api_client = app.test_client()
def setup_class(self):
'''ensure that we have a clean environment
before running any tests
'''
_ = setup_environment()
response = _stamp_notes(CONFIG['notes_directory'],
stamp_todos=True,
stamp_today=True,
stamp_questions=True,
stamp_answers=True,
grep_path=CONFIG['grep_path'])
assert response.keys()
def test_stamp_call(capfd, caplog):
_ = setup_environment()
_ = cli_stamp_notes(ORIGINAL_CONFIG)
# Validate stdout
captured = capfd.readouterr()
old_line_count, new_line_count, file_count = 0, 0, 0
for line in captured.out.split('\n'):
if '(old):' in line:
old_line_count += 1
if '(new):' in line:
new_line_count += 1
if line.startswith('<<--') and line.endswith('-->>'):
file_count += 1
assert new_line_count == old_line_count
assert file_count == 4
# Validate that we only get DEBUG and INFO log messages
for record in caplog.records:
assert record.levelname in ['DEBUG', 'INFO']
teardown_environment()
def setup_class(cls):
# ensure that we have a clean environment before running any tests
_ = setup_environment()
_ = _update_ngram_database(CONFIG['notes_directory'],
CONFIG['cache_directory'])
def first_run_configuration(request):
setup_environment(request.user)
return HttpResponse('Ok')
def main():
setup_environment()
app.run()
def seed_db():
setup_environment()
store.seed_db(seed.seeds())
def setup_class(cls):
# ensure that we have a clean environment before running any tests
_ = setup_environment()
import os
import json
import copy
import logging
import pytest
import unittest
from shorthand import ShorthandServer
from shorthand.utils.config import clean_and_validate_config
from utils import setup_environment, TEMP_DIR, validate_setup, setup_logging
ORIGINAL_CONFIG = setup_environment()
setup_logging(ORIGINAL_CONFIG)
log = logging.getLogger(__name__)
server_config_path = TEMP_DIR + '/server_config.json'
server = None
def write_config(config):
with open(server_config_path, 'w') as config_file_object:
json.dump(config, config_file_object)
class TestServerClass(unittest.TestCase):
"""Test the implementation of the ShorthandServer Class"""
@classmethod
def setup_class(cls):
def setup_method(self, method):
'''Validate that the environment has been set up correctly
Re-do the setup every time because we are modifying notes in most tests
'''
_ = setup_environment()
validate_setup()
def main():
setup_environment()
aggregate_leads()
def main():
parser = argparse.ArgumentParser(
description="Used to train TensorFlow model")
parser.add_argument(
"train_files_path",
metavar="path",
help="Path to the training files.",
)
parser.add_argument(
"test_files_path",
metavar="path",
help="Path to the test files",
)
parser.add_argument(
"--checkpoints_save_path",
dest="checkpoints_save_path",
metavar="path",
type=str,
help="Path where the checkpoints should be saved.",
)
parser.add_argument(
"--last_checkpoint_path",
dest="last_checkpoint_path",
metavar="path",
type=str,
help="Path to the last checkpoint to continue training.",
)
parser.add_argument(
"--plot_history",
dest="plot_history",
metavar="boolean (default: false)",
type=bool,
help="Plots the model training history",
)
plot_history = False
args = parser.parse_args()
if args.plot_history:
plot_history = args.plot_history
setup_environment()
train_files_path = args.train_files_path
eval_files_path = args.test_files_path
input_shape = (240, 240, 1)
generator_config = ImageGeneratorConfig()
generator_config.loop_count = 10
generator_config.horizontal_flip = True
generator_config.zoom_range = 0.3
generator_config.width_shift_range = 0.3
generator_config.height_shift_range = 0.3
generator_config.rotation_range = 10
train_x, train_y, eval_x, eval_y = load_dataset(train_files_path,
input_shape,
validation_split=0)
eval_x, eval_y, _, _, = load_dataset(eval_files_path,
input_shape,
validation_split=0)
backbone = backbones.SegmentationVanillaUnet(input_shape)
# optimizer = SGD(lr=0.0001, momentum=0.9, decay=0.0005)
optimizer = Adam(lr=0.00001)
train_engine = TrainEngine(
input_shape,
backbone.model,
optimizer,
loss="binary_crossentropy",
checkpoints_save_path=args.checkpoints_save_path,
checkpoint_save_period=100,
last_checkpoint_path=args.last_checkpoint_path,
)
loss, acc, val_loss, val_acc = train_engine.train(
train_x,
train_y,
eval_x,
eval_y,
epochs=50,
batch_size=10,
image_generator_config=generator_config,
)
if plot_history:
plots.plot_history(loss, acc, val_loss, val_acc)
for idx in range(len(eval_x[:2])):
predictions = train_engine.model.predict(np.array(
[eval_x[idx]], dtype=np.float32),
batch_size=1)
plots.plot_prediction(predictions, [eval_x[idx]], input_shape)
K.clear_session()
import re
import json
import logging
import unittest
import pytest
from shorthand.elements.record_sets import _get_record_set, _get_record_sets
from shorthand.utils.rec import load_from_string
from utils import setup_environment, validate_setup, setup_logging
CONFIG = setup_environment()
setup_logging(CONFIG)
log = logging.getLogger(__name__)
class TestRecConfig(unittest.TestCase):
'''Basic tests for recfile config parsing
'''
def test_valid_config_parsing(self):
with open('rec_data/valid_config.rec', 'r') as f:
valid_config_data = f.read()
split_config = re.split(r'#.*?\n', valid_config_data)
split_config = [config for config in split_config if config.strip()]
for valid_config in split_config:
record_set = load_from_string(valid_config)
assert record_set.config.keys()
def test_invalid_config_parsing(self):
with open('rec_data/invalid_config.rec', 'r') as f:
def first_run_configuration(request):
setup_environment(request.user)
return HttpResponse('Ok')
"Prefix for checkpoint outputs at each learning rate change, REQUIRED")
parser.add_argument('--olog_dir',
type=str,
default=None,
help="Directory for tensorboard log outputs, optional")
args = parser.parse_args()
if len(sys.argv[1:]) == 0:
parser.print_help()
sys.exit(1)
# initialize
logging.info("[INFO] Initializing")
utils.setup_environment()
m = cv.Clair()
m.init()
dataset_info = utils.dataset_info_from(binary_file_path=args.bin_fn,
tensor_file_path=args.tensor_fn,
variant_file_path=args.var_fn,
bed_file_path=args.bed_fn)
training_config = dict(
dataset_info=dataset_info,
learning_rate=args.learning_rate,
l2_regularization_lambda=args.lambd,
output_file_path_prefix=args.ochk_prefix,
model_initalization_file_path=args.chkpnt_fn,
summary_writer=m.get_summary_file_writer(args.olog_dir)
def initialize():
setup_environment()
store.initialize(seeds=seed.seeds())
