def main():
sys.path.append('.')
from source.tool import downloader
from source.tool import tuner
from source.tool import config_parser
from source.config.image_classification_config import \
ImageClassificationInputterConfig, \
ImageClassificationModelerConfig
parser = config_parser.default_parser()
parser.add_argument("--num_classes",
help="Number of classes.",
type=int,
default=10)
parser.add_argument("--image_height",
help="Image height.",
type=int,
default=32)
parser.add_argument("--image_width",
help="Image width.",
type=int,
default=32)
parser.add_argument("--image_depth",
help="Number of color channels.",
type=int,
default=3)
config = parser.parse_args()
config = config_parser.prepare(config)
# public_props = (
# name for name in dir(config) if not name.startswith('_'))
# for props_name in public_props:
# print(props_name + ": " + str(getattr(config, props_name)))
# Download data if necessary
if config.mode != "infer":
if hasattr(config, "dataset_meta"):
if not os.path.exists(config.dataset_meta):
downloader.download_and_extract(config.dataset_meta,
config.dataset_url, False)
else:
print("Found " + config.dataset_meta + ".")
elif hasattr(config, "train_dataset_meta"):
if not os.path.exists(config.train_dataset_meta):
print(config.train_dataset_meta)
downloader.download_and_extract(config.train_dataset_meta,
config.dataset_url, False)
else:
print("Found " + config.train_dataset_meta + ".")
else:
assert False, "A meta data must be provided."
# Generate config
runner_config, callback_config, inputter_config, modeler_config = \
config_parser.default_config(config)
inputter_config = ImageClassificationInputterConfig(
inputter_config,
image_height=config.image_height,
image_width=config.image_width,
image_depth=config.image_depth,
num_classes=config.num_classes)
modeler_config = ImageClassificationModelerConfig(
modeler_config, num_classes=config.num_classes)
if config.mode == "tune":
inputter_module = importlib.import_module(
"source.inputter.image_classification_csv_inputter")
modeler_module = importlib.import_module(
"source.modeler.image_classification_modeler")
runner_module = importlib.import_module(
"source.runner.parameter_server_runner")
tuner.tune(config, runner_config, callback_config, inputter_config,
modeler_config, inputter_module, modeler_module,
runner_module)
else:
"""
An application owns a runner.
Runner: Distributes a job across devices, schedules the excution.
It owns an inputter and a modeler.
Inputter: Handles the data pipeline.
It (optionally) owns a data augmenter.
Modeler: Creates functions for network, loss, optimization and evaluation.
It owns a network and a list of callbacks as inputs.
"""
augmenter = (None if not config.augmenter else
importlib.import_module("source.augmenter." +
config.augmenter))
net = getattr(
importlib.import_module("source.network." + config.network), "net")
callbacks = []
for name in config.callbacks:
callback = importlib.import_module("source.callback." +
name).build(callback_config)
callbacks.append(callback)
inputter = importlib.import_module(
"source.inputter.image_classification_csv_inputter").build(
inputter_config, augmenter)
modeler = importlib.import_module(
"source.modeler.image_classification_modeler").build(
modeler_config, net)
runner = importlib.import_module(
"source.runner.parameter_server_runner").build(
runner_config, inputter, modeler, callbacks)
# Run application
runner.run()
def main():
sys.path.append('.')
from source.tool import downloader
from source.tool import tuner
from source.tool import config_parser
from source.config.text_generation_config import \
TextGenerationInputterConfig, TextGenerationCallbackConfig
parser = config_parser.default_parser()
app_parser = parser.add_argument_group('app')
app_parser.add_argument("--vocab_file",
help="Path of the vocabulary file.",
type=str,
default="")
app_parser.add_argument(
"--vocab_top_k",
help="Number of words kept in the vocab. set to -1 to use all words.",
type=int,
default=-1)
app_parser.add_argument("--vocab_format",
help="Format of vocabulary.",
type=str,
default="pickle",
choices=["pickle", "txt"])
app_parser.add_argument("--encode_method",
help="Name of the method to encode text.",
type=str,
default="basic")
app_parser.add_argument(
"--unit",
choices=["char", "word"],
help="Type of unit. Must be chosen from char or word.",
type=str,
default="word")
app_parser.add_argument(
"--starter",
help=
"The starting token(s) to generate the text, splited by the splitter (default #)",
type=str,
default="T")
app_parser.add_argument("--softmax_temperature",
help="Control the randomness during generation.",
type=float,
default=1.0)
# Default configs
runner_config, callback_config, inputter_config, modeler_config, app_config = \
config_parser.default_config(parser)
inputter_config = TextGenerationInputterConfig(
inputter_config,
vocab_file=app_config.vocab_file,
vocab_top_k=app_config.vocab_top_k,
vocab_format=app_config.vocab_format,
encode_method=app_config.encode_method,
unit=app_config.unit,
starter=app_config.starter)
callback_config = TextGenerationCallbackConfig(
callback_config,
unit=app_config.unit,
softmax_temperature=app_config.softmax_temperature)
# Check if data is available
downloader.check_data(inputter_config)
if runner_config.mode == "tune":
inputter_module = importlib.import_module(
"source.inputter.text_generation_inputter")
modeler_module = importlib.import_module(
"source.modeler.text_generation_modeler")
runner_module = importlib.import_module(
"source.runner.parameter_server_runner")
tuner.tune(app_config, runner_config, callback_config, inputter_config,
modeler_config, inputter_module, modeler_module,
runner_module)
else:
"""
An application owns a runner.
Runner: Distributes a job across devices, schedules the excution.
It owns an inputter and a modeler.
Inputter: Handles the data pipeline.
It (optionally) owns a data augmenter.
Modeler: Creates functions for network, loss, optimization and evaluation.
It owns a network and a list of callbacks as inputs.
"""
augmenter = (None if not inputter_config.augmenter else
importlib.import_module("source.augmenter." +
inputter_config.augmenter))
encoder = (None if not inputter_config.encode_method else
importlib.import_module("source.network.encoder." +
inputter_config.encode_method))
net = importlib.import_module("source.network." +
modeler_config.network)
callbacks = []
for name in callback_config.callbacks:
callback = importlib.import_module("source.callback." +
name).build(callback_config)
callbacks.append(callback)
inputter = importlib.import_module(
"source.inputter.text_generation_inputter").build(
inputter_config, augmenter, encoder)
modeler = importlib.import_module(
"source.modeler.text_generation_modeler").build(
modeler_config, net)
runner = importlib.import_module(
"source.runner.parameter_server_runner").build(
runner_config, inputter, modeler, callbacks)
# Run application
runner.run()
def main():
sys.path.append('.')
from source.tool import downloader
from source.tool import tuner
from source.tool import config_parser
from source.config.image_classification_config import \
ImageClassificationInputterConfig, \
ImageClassificationModelerConfig
parser = config_parser.default_parser()
app_parser = parser.add_argument_group('app')
app_parser.add_argument("--num_classes",
help="Number of classes.",
type=int,
default=10)
app_parser.add_argument("--image_height",
help="Image height.",
type=int,
default=32)
app_parser.add_argument("--image_width",
help="Image width.",
type=int,
default=32)
app_parser.add_argument("--image_depth",
help="Number of color channels.",
type=int,
default=3)
# Default configs
runner_config, callback_config, inputter_config, modeler_config, app_config = \
config_parser.default_config(parser)
# Application dependent configs
inputter_config = ImageClassificationInputterConfig(
inputter_config,
image_height=app_config.image_height,
image_width=app_config.image_width,
image_depth=app_config.image_depth,
num_classes=app_config.num_classes)
modeler_config = ImageClassificationModelerConfig(
modeler_config, num_classes=app_config.num_classes)
# Check if data is available
downloader.check_data(inputter_config)
if runner_config.mode == "tune":
inputter_module = importlib.import_module(
"source.inputter.image_classification_csv_inputter")
modeler_module = importlib.import_module(
"source.modeler.image_classification_modeler")
runner_module = importlib.import_module(
"source.runner.parameter_server_runner")
tuner.tune(app_config, runner_config, callback_config, inputter_config,
modeler_config, inputter_module, modeler_module,
runner_module)
else:
"""
An application owns a runner.
Runner: Distributes a job across devices, schedules the excution.
It owns an inputter and a modeler.
Inputter: Handles the data pipeline.
It (optionally) owns a data augmenter.
Modeler: Creates functions for network, loss, optimization and evaluation.
It owns a network and a list of callbacks as inputs.
"""
augmenter = (None if not inputter_config.augmenter else
importlib.import_module("source.augmenter." +
inputter_config.augmenter))
net = importlib.import_module("source.network." +
modeler_config.network)
callbacks = []
for name in callback_config.callbacks:
callback = importlib.import_module("source.callback." +
name).build(callback_config)
callbacks.append(callback)
inputter = importlib.import_module(
"source.inputter.image_classification_csv_inputter").build(
inputter_config, augmenter)
modeler = importlib.import_module(
"source.modeler.image_classification_modeler").build(
modeler_config, net)
runner = importlib.import_module(
"source.runner.parameter_server_runner").build(
runner_config, inputter, modeler, callbacks)
# Run application
runner.run()
def main():
sys.path.append('.')
from source.tool import downloader
from source.tool import tuner
from source.tool import config_parser
from source.config.style_transfer_config import \
StyleTransferInputterConfig, StyleTransferModelerConfig
parser = config_parser.default_parser()
app_parser = parser.add_argument_group('app')
app_parser.add_argument("--style_weight",
help="Weight for style loss",
default=100)
app_parser.add_argument("--content_weight",
help="Weight for content loss",
default=15)
app_parser.add_argument("--tv_weight",
help="Weight for tv loss",
default=200)
app_parser.add_argument("--image_height",
help="Image height.",
type=int,
default=256)
app_parser.add_argument("--image_width",
help="Image width.",
type=int,
default=256)
app_parser.add_argument("--resize_side_min",
help="The minimal image size in augmentation.",
type=int,
default=400)
app_parser.add_argument("--resize_side_max",
help="The maximul image size in augmentation.",
type=int,
default=600)
app_parser.add_argument("--image_depth",
help="Number of color channels.",
type=int,
default=3)
app_parser.add_argument("--feature_net",
help="Name of feature net",
default="vgg_19_conv")
app_parser.add_argument("--feature_net_path",
help="Path to pre-trained vgg model.",
default=os.path.join(
os.environ['HOME'],
"demo/model/vgg_19_2016_08_28/vgg_19.ckpt"))
app_parser.add_argument("--style_image_path",
help="Path to style image",
default=os.path.join(
os.environ['HOME'],
"demo/data/mscoco_fns/gothic.jpg"))
app_parser.add_argument("--data_format",
help="channels_first or channels_last",
choices=["channels_first", "channels_last"],
default="channels_last")
app_parser.add_argument(
"--feature_net_url",
help="URL for downloading pre-trained feature_net",
default="http://download.tensorflow.org/models/vgg_19_2016_08_28.tar.gz"
)
# Default configs
runner_config, callback_config, inputter_config, modeler_config, app_config = \
config_parser.default_config(parser)
# Application dependent configs
inputter_config = StyleTransferInputterConfig(
inputter_config,
image_height=app_config.image_height,
image_width=app_config.image_width,
image_depth=app_config.image_depth,
resize_side_min=app_config.resize_side_min,
resize_side_max=app_config.resize_side_max)
modeler_config = StyleTransferModelerConfig(
modeler_config,
data_format=app_config.data_format,
image_depth=app_config.image_depth,
style_weight=app_config.style_weight,
content_weight=app_config.content_weight,
tv_weight=app_config.tv_weight,
feature_net=app_config.feature_net,
feature_net_path=app_config.feature_net_path,
style_image_path=app_config.style_image_path)
# Download data if necessary
downloader.check_and_download(inputter_config)
if runner_config.mode == "tune":
inputter_module = importlib.import_module(
"source.inputter.style_transfer_csv_inputter")
modeler_module = importlib.import_module(
"source.modeler.style_transfer_modeler")
runner_module = importlib.import_module(
"source.runner.parameter_server_runner")
tuner.tune(app_config, runner_config, callback_config, inputter_config,
modeler_config, inputter_module, modeler_module,
runner_module)
else:
"""
An application owns a runner.
Runner: Distributes a job across devices, schedules the excution.
It owns an inputter and a modeler.
Inputter: Handles the data pipeline.
It (optionally) owns a data augmenter.
Modeler: Creates functions for network, loss, optimization and evaluation.
It owns a network and a list of callbacks as inputs.
"""
augmenter = (None if not inputter_config.augmenter else
importlib.import_module("source.augmenter." +
inputter_config.augmenter))
net = importlib.import_module("source.network." +
modeler_config.network)
callbacks = []
for name in callback_config.callbacks:
callback = importlib.import_module("source.callback." +
name).build(callback_config)
callbacks.append(callback)
inputter = importlib.import_module(
"source.inputter.style_transfer_csv_inputter").build(
inputter_config, augmenter)
modeler = importlib.import_module(
"source.modeler.style_transfer_modeler").build(
modeler_config, net)
runner = importlib.import_module(
"source.runner.parameter_server_runner").build(
runner_config, inputter, modeler, callbacks)
# Run application
runner.run()
def main():
sys.path.append('.')
from source.tool import downloader
from source.tool import tuner
from source.tool import config_parser
from source.config.text_classification_config import \
TextClassificationInputterConfig, TextClassificationModelerConfig
parser = config_parser.default_parser()
app_parser = parser.add_argument_group('app')
app_parser.add_argument("--vocab_file",
help="Path of the vocabulary file.",
type=str,
default="")
app_parser.add_argument("--encode_method",
help="Name of the method to encode text.",
type=str,
default="basic")
app_parser.add_argument(
"--vocab_top_k",
help="Number of words kept in the vocab. set to -1 to use all words.",
type=int,
default=-1)
app_parser.add_argument("--num_classes",
help="Number of classes.",
type=int,
default=2)
app_parser.add_argument("--lr_method",
choices=["step", "linear_plus_warmup"],
help="Name of the learning rate scheduling method",
type=str,
default="step")
# Default configs
runner_config, callback_config, inputter_config, modeler_config, app_config = \
config_parser.default_config(parser)
inputter_config = TextClassificationInputterConfig(
inputter_config,
vocab_file=app_config.vocab_file,
vocab_top_k=app_config.vocab_top_k,
encode_method=app_config.encode_method)
modeler_config = TextClassificationModelerConfig(
modeler_config,
num_classes=app_config.num_classes,
lr_method=app_config.lr_method)
# Download data if necessary
downloader.check_and_download(inputter_config)
if runner_config.mode == "tune":
inputter_module = importlib.import_module(
"source.inputter.text_classification_inputter")
modeler_module = importlib.import_module(
"source.modeler.text_classification_modeler")
runner_module = importlib.import_module(
"source.runner.parameter_server_runner")
tuner.tune(app_config, runner_config, callback_config, inputter_config,
modeler_config, inputter_module, modeler_module,
runner_module)
else:
"""
An application owns a runner.
Runner: Distributes a job across devices, schedules the excution.
It owns an inputter and a modeler.
Inputter: Handles the data pipeline.
It (optionally) owns a data augmenter.
Modeler: Creates functions for network, loss, optimization and evaluation.
It owns a network and a list of callbacks as inputs.
"""
augmenter = (None if not inputter_config.augmenter else
importlib.import_module("source.augmenter." +
inputter_config.augmenter))
encoder = (None if not inputter_config.encode_method else
importlib.import_module("source.network.encoder." +
inputter_config.encode_method))
net = importlib.import_module("source.network." +
modeler_config.network)
callbacks = []
for name in callback_config.callbacks:
callback = importlib.import_module("source.callback." +
name).build(callback_config)
callbacks.append(callback)
inputter = importlib.import_module(
"source.inputter.text_classification_inputter").build(
inputter_config, augmenter, encoder)
modeler = importlib.import_module(
"source.modeler.text_classification_modeler").build(
modeler_config, net)
runner = importlib.import_module(
"source.runner.parameter_server_runner").build(
runner_config, inputter, modeler, callbacks)
# Run application
runner.run()
def main():
sys.path.append('.')
from source.tool import tuner
from source.tool import config_parser
from source.config.object_detection_config import \
ObjectDetectionInputterConfig, \
ObjectDetectionModelerConfig
parser = config_parser.default_parser()
app_parser = parser.add_argument_group('app')
app_parser.add_argument("--num_classes",
help="Number of classes.",
type=int,
default=81)
app_parser.add_argument("--resolution",
help="Image resolution used for detectoin.",
type=int,
default=512)
app_parser.add_argument("--dataset_dir",
help="Path to dataset.",
type=str,
default="/mnt/data/data/mscoco")
app_parser.add_argument("--feature_net",
help="Name of feature net",
default="vgg_16_reduced")
app_parser.add_argument("--feature_net_path",
help="Path to pre-trained vgg model.",
default=os.path.join(
os.environ['HOME'],
"demo/model/VGG_16_reduce/VGG_16_reduce.p"))
app_parser.add_argument("--data_format",
help="channels_first or channels_last",
choices=["channels_first", "channels_last"],
default="channels_last")
app_parser.add_argument("--confidence_threshold",
help="threshold to remove weak detection",
type=float,
default=0.5)
# Default configs
runner_config, callback_config, inputter_config, modeler_config, app_config = \
config_parser.default_config(parser)
inputter_config = ObjectDetectionInputterConfig(
inputter_config,
dataset_dir=app_config.dataset_dir,
num_classes=app_config.num_classes,
resolution=app_config.resolution)
modeler_config = ObjectDetectionModelerConfig(
modeler_config,
num_classes=app_config.num_classes,
data_format=app_config.data_format,
feature_net=app_config.feature_net,
feature_net_path=app_config.feature_net_path,
confidence_threshold=app_config.confidence_threshold)
if runner_config.mode == "tune":
inputter_module = importlib.import_module(
"source.inputter.object_detection_mscoco_inputter")
modeler_module = importlib.import_module(
"source.modeler.object_detection_modeler")
runner_module = importlib.import_module(
"source.runner.parameter_server_runner")
tuner.tune(app_config, runner_config, callback_config, inputter_config,
modeler_config, inputter_module, modeler_module,
runner_module)
else:
"""
An application owns a runner.
Runner: Distributes a job across devices, schedules the excution.
It owns an inputter and a modeler.
Inputter: Handles the data pipeline.
It (optionally) owns a data augmenter.
Modeler: Creates functions for network, loss, optimization and evaluation.
It owns a network and a list of callbacks as inputs.
"""
augmenter = (None if not inputter_config.augmenter else
importlib.import_module("source.augmenter." +
inputter_config.augmenter))
net = importlib.import_module("source.network." +
modeler_config.network)
callbacks = []
for name in callback_config.callbacks:
callback = importlib.import_module("source.callback." +
name).build(callback_config)
callbacks.append(callback)
inputter = importlib.import_module(
"source.inputter.object_detection_mscoco_inputter").build(
inputter_config, augmenter)
modeler = importlib.import_module(
"source.modeler.object_detection_modeler").build(
modeler_config, net)
runner = importlib.import_module(
"source.runner.parameter_server_runner").build(
runner_config, inputter, modeler, callbacks)
# Run application
runner.run()
def main():
sys.path.append('.')
from source.tool import downloader
from source.tool import tuner
from source.tool import config_parser
from source.config.image_segmentation_config import \
ImageSegmentationCallbackConfig, ImageSegmentationInputterConfig, \
ImageSegmentationModelerConfig
parser = config_parser.default_parser()
parser.add_argument("--num_classes",
help="Number of classes.",
type=int,
default=12)
parser.add_argument("--image_height",
help="Image height.",
type=int,
default=360)
parser.add_argument("--image_width",
help="Image width.",
type=int,
default=480)
parser.add_argument("--image_depth",
help="Number of color channels.",
type=int,
default=3)
parser.add_argument("--output_height",
help="Output height.",
type=int,
default=368)
parser.add_argument("--output_width",
help="Output width.",
type=int,
default=480)
parser.add_argument("--resize_side_min",
help="The minimal image size in augmentation.",
type=int,
default=400)
parser.add_argument("--resize_side_max",
help="The maximul image size in augmentation.",
type=int,
default=600)
parser.add_argument("--data_format",
help="channels_first or channels_last",
default="channels_first")
config = parser.parse_args()
config = config_parser.prepare(config)
# Download data if necessary
if config.mode != "infer":
if hasattr(config, "dataset_meta"):
if not os.path.exists(config.dataset_meta):
downloader.download_and_extract(config.dataset_meta,
config.dataset_url, False)
else:
print("Found " + config.dataset_meta + ".")
elif hasattr(config, "train_dataset_meta"):
if not os.path.exists(config.train_dataset_meta):
print(config.train_dataset_meta)
downloader.download_and_extract(config.train_dataset_meta,
config.dataset_url, False)
else:
print("Found " + config.train_dataset_meta + ".")
else:
assert False, "A meta data must be provided."
# Generate config
runner_config, callback_config, inputter_config, modeler_config = \
config_parser.default_config(config)
callback_config = ImageSegmentationCallbackConfig(
callback_config, num_classes=config.num_classes)
inputter_config = ImageSegmentationInputterConfig(
inputter_config,
image_height=config.image_height,
image_width=config.image_width,
image_depth=config.image_depth,
output_height=config.output_height,
output_width=config.output_width,
resize_side_min=config.resize_side_min,
resize_side_max=config.resize_side_max,
num_classes=config.num_classes)
modeler_config = ImageSegmentationModelerConfig(
modeler_config,
num_classes=config.num_classes,
data_format=config.data_format)
if config.mode == "tune":
inputter_module = importlib.import_module(
"source.inputter.image_segmentation_csv_inputter")
modeler_module = importlib.import_module(
"source.modeler.image_segmentation_modeler")
runner_module = importlib.import_module(
"source.runner.parameter_server_runner")
tuner.tune(config, runner_config, callback_config, inputter_config,
modeler_config, inputter_module, modeler_module,
runner_module)
else:
"""
An application owns a runner.
Runner: Distributes a job across devices, schedules the excution.
It owns an inputter and a modeler.
Inputter: Handles the data pipeline.
It (optionally) owns a data augmenter.
Modeler: Creates functions for network, loss, optimization and evaluation.
It owns a network and a list of callbacks as inputs.
"""
augmenter = (None if not config.augmenter else
importlib.import_module("source.augmenter." +
config.augmenter))
net = getattr(
importlib.import_module("source.network." + config.network), "net")
callbacks = []
for name in config.callbacks:
callback = importlib.import_module("source.callback." +
name).build(callback_config)
callbacks.append(callback)
inputter = importlib.import_module(
"source.inputter.image_segmentation_csv_inputter").build(
inputter_config, augmenter)
modeler = importlib.import_module(
"source.modeler.image_segmentation_modeler").build(
modeler_config, net)
runner = importlib.import_module(
"source.runner.parameter_server_runner").build(
runner_config, inputter, modeler, callbacks)
# Run application
runner.run()
def main():
sys.path.append('.')
from source.tool import downloader
from source.tool import tuner
from source.tool import config_parser
from source.config.image_segmentation_config import \
ImageSegmentationCallbackConfig, ImageSegmentationInputterConfig, \
ImageSegmentationModelerConfig
parser = config_parser.default_parser()
app_parser = parser.add_argument_group('app')
app_parser.add_argument("--num_classes",
help="Number of classes.",
type=int,
default=12)
app_parser.add_argument("--image_height",
help="Image height.",
type=int,
default=360)
app_parser.add_argument("--image_width",
help="Image width.",
type=int,
default=480)
app_parser.add_argument("--image_depth",
help="Number of color channels.",
type=int,
default=3)
app_parser.add_argument("--output_height",
help="Output height.",
type=int,
default=368)
app_parser.add_argument("--output_width",
help="Output width.",
type=int,
default=480)
app_parser.add_argument("--resize_side_min",
help="The minimal image size in augmentation.",
type=int,
default=400)
app_parser.add_argument("--resize_side_max",
help="The maximul image size in augmentation.",
type=int,
default=600)
app_parser.add_argument("--data_format",
help="channels_first or channels_last",
default="channels_first")
# Default configs
runner_config, callback_config, inputter_config, modeler_config, app_config = \
config_parser.default_config(parser)
# Application dependent configs
callback_config = ImageSegmentationCallbackConfig(
callback_config, num_classes=app_config.num_classes)
inputter_config = ImageSegmentationInputterConfig(
inputter_config,
image_height=app_config.image_height,
image_width=app_config.image_width,
image_depth=app_config.image_depth,
output_height=app_config.output_height,
output_width=app_config.output_width,
resize_side_min=app_config.resize_side_min,
resize_side_max=app_config.resize_side_max,
num_classes=app_config.num_classes)
modeler_config = ImageSegmentationModelerConfig(
modeler_config,
num_classes=app_config.num_classes,
data_format=app_config.data_format)
# Download data if necessary
downloader.check_and_download(inputter_config)
if runner_config.mode == "tune":
inputter_module = importlib.import_module(
"source.inputter.image_segmentation_csv_inputter")
modeler_module = importlib.import_module(
"source.modeler.image_segmentation_modeler")
runner_module = importlib.import_module(
"source.runner.parameter_server_runner")
tuner.tune(app_config, runner_config, callback_config, inputter_config,
modeler_config, inputter_module, modeler_module,
runner_module)
else:
"""
An application owns a runner.
Runner: Distributes a job across devices, schedules the excution.
It owns an inputter and a modeler.
Inputter: Handles the data pipeline.
It (optionally) owns a data augmenter.
Modeler: Creates functions for network, loss, optimization and evaluation.
It owns a network and a list of callbacks as inputs.
"""
augmenter = (None if not inputter_config.augmenter else
importlib.import_module("source.augmenter." +
inputter_config.augmenter))
net = importlib.import_module("source.network." +
modeler_config.network)
callbacks = []
for name in callback_config.callbacks:
callback = importlib.import_module("source.callback." +
name).build(callback_config)
callbacks.append(callback)
inputter = importlib.import_module(
"source.inputter.image_segmentation_csv_inputter").build(
inputter_config, augmenter)
modeler = importlib.import_module(
"source.modeler.image_segmentation_modeler").build(
modeler_config, net)
runner = importlib.import_module(
"source.runner.parameter_server_runner").build(
runner_config, inputter, modeler, callbacks)
# Run application
runner.run()
def main():
sys.path.append('.')
from source.tool import downloader
from source.tool import tuner
from source.tool import config_parser
parser = config_parser.default_parser()
config = parser.parse_args()
config = config_parser.prepare(config)
# Download data if necessary
if config.mode != "infer":
if hasattr(config, "dataset_meta"):
if not os.path.exists(config.dataset_meta):
downloader.download_and_extract(config.dataset_meta,
config.dataset_url, False)
else:
print("Found " + config.dataset_meta + ".")
elif hasattr(config, "train_dataset_meta"):
if not os.path.exists(config.train_dataset_meta):
print(config.train_dataset_meta)
downloader.download_and_extract(config.train_dataset_meta,
config.dataset_url, False)
else:
print("Found " + config.train_dataset_meta + ".")
else:
assert False, "A meta data must be provided."
# Generate config
runner_config, callback_config, inputter_config, modeler_config = \
config_parser.default_config(config)
if config.mode == "tune":
inputter_module = importlib.import_module(
"source.inputter.text_generation_txt_inputter")
modeler_module = importlib.import_module(
"source.modeler.text_generation_modeler")
runner_module = importlib.import_module(
"source.runner.parameter_server_runner")
tuner.tune(config, runner_config, callback_config, inputter_config,
modeler_config, inputter_module, modeler_module,
runner_module)
else:
"""
An application owns a runner.
Runner: Distributes a job across devices, schedules the excution.
It owns an inputter and a modeler.
Inputter: Handles the data pipeline.
It (optionally) owns a data augmenter.
Modeler: Creates functions for network, loss, optimization and evaluation.
It owns a network and a list of callbacks as inputs.
"""
augmenter = (None if not config.augmenter else
importlib.import_module("source.augmenter." +
config.augmenter))
net = getattr(
importlib.import_module("source.network." + config.network), "net")
callbacks = []
for name in config.callbacks:
callback = importlib.import_module("source.callback." +
name).build(callback_config)
callbacks.append(callback)
inputter = importlib.import_module(
"source.inputter.text_generation_txt_inputter").build(
inputter_config, augmenter)
modeler = importlib.import_module(
"source.modeler.text_generation_modeler").build(
modeler_config, net)
runner = importlib.import_module(
"source.runner.parameter_server_runner").build(
runner_config, inputter, modeler, callbacks)
# Run application
runner.run()
def main():
sys.path.append('.')
from source.tool import downloader
from source.tool import tuner
from source.tool import config_parser
parser = config_parser.default_parser()
app_parser = parser.add_argument_group('app')
# Default configs
runner_config, callback_config, inputter_config, modeler_config, app_config = \
config_parser.default_config(parser)
# Download data if necessary
downloader.check_and_download(inputter_config)
if runner_config.mode == "tune":
inputter_module = importlib.import_module(
"source.inputter.text_generation_txt_inputter")
modeler_module = importlib.import_module(
"source.modeler.text_generation_modeler")
runner_module = importlib.import_module(
"source.runner.parameter_server_runner")
tuner.tune(app_config, runner_config, callback_config, inputter_config,
modeler_config, inputter_module, modeler_module,
runner_module)
else:
"""
An application owns a runner.
Runner: Distributes a job across devices, schedules the excution.
It owns an inputter and a modeler.
Inputter: Handles the data pipeline.
It (optionally) owns a data augmenter.
Modeler: Creates functions for network, loss, optimization and evaluation.
It owns a network and a list of callbacks as inputs.
"""
augmenter = (None if not inputter_config.augmenter else
importlib.import_module("source.augmenter." +
inputter_config.augmenter))
net = importlib.import_module("source.network." +
modeler_config.network)
callbacks = []
for name in callback_config.callbacks:
callback = importlib.import_module("source.callback." +
name).build(callback_config)
callbacks.append(callback)
inputter = importlib.import_module(
"source.inputter.text_generation_txt_inputter").build(
inputter_config, augmenter)
modeler = importlib.import_module(
"source.modeler.text_generation_modeler").build(
modeler_config, net)
runner = importlib.import_module(
"source.runner.parameter_server_runner").build(
runner_config, inputter, modeler, callbacks)
# Run application
runner.run()
