def test_xshards_symbol(self):
# prepare data
resource_path = os.path.join(
os.path.split(__file__)[0], "../../../resources")
self.ray_ctx = get_ray_ctx()
train_file_path = os.path.join(resource_path,
"orca/learn/train_data.json")
train_data_shard = zoo.orca.data.pandas.read_json(train_file_path,
self.ray_ctx,
orient='records',
lines=False)
train_data_shard.transform_shard(prepare_data_symbol)
test_file_path = os.path.join(resource_path,
"orca/learn/test_data.json")
test_data_shard = zoo.orca.data.pandas.read_json(test_file_path,
self.ray_ctx,
orient='records',
lines=False)
test_data_shard.transform_shard(prepare_data_symbol)
config = create_trainer_config(batch_size=32, log_interval=1, seed=42)
trainer = MXNetTrainer(config,
train_data_shard,
get_symbol_model,
validation_metrics_creator=get_metrics,
test_data=test_data_shard,
eval_metrics_creator=get_metrics)
trainer.train(nb_epoch=2)
def test_xshards_symbol(self):
# prepare data
resource_path = os.path.join(
os.path.split(__file__)[0], "../../../resources")
self.ray_ctx = get_ray_ctx()
train_file_path = os.path.join(resource_path,
"orca/learn/single_input_json/train")
train_data_shard = zoo.orca.data.pandas.read_json(train_file_path,
self.ray_ctx,
orient='records',
lines=False)
train_data_shard.transform_shard(prepare_data_symbol)
test_file_path = os.path.join(resource_path,
"orca/learn/single_input_json/test")
test_data_shard = zoo.orca.data.pandas.read_json(test_file_path,
self.ray_ctx,
orient='records',
lines=False)
test_data_shard.transform_shard(prepare_data_symbol)
config = create_config(batch_size=32, log_interval=1, seed=42)
estimator = Estimator(config,
get_symbol_model,
validation_metrics_creator=get_metrics,
eval_metrics_creator=get_metrics,
num_workers=2)
estimator.fit(train_data_shard, test_data_shard, nb_epoch=2)
estimator.shutdown()