def test_simple(self):
with rflow.begin_graph("reentrancy", HERE) as g:
g.t1 = T1()
g.t1.args.v1 = 5
g.t1.resource = rflow.FSResource("T1.pickle")
g.t2 = T2()
g.t2.args.x2 = g.t1
g.t2.resource = rflow.FSResource("T2.pickle")
# First call, all evals must be 1
self.assertEqual(4000, g.t2.call())
self.assertEqual(1, T1.eval_count)
self.assertEqual(1, T2.eval_count)
self.assertEqual(0, T1.load_count)
self.assertEqual(0, T2.load_count)
# Next test will forget the values, and expects only t2 be
# loaded. Because t1 didn't change values and t2 has a
# resource.
# forget values
# pylint: disable=protected-access
g.t1._is_dirty = True
g.t2._is_dirty = True
g.t1.value = Uninit
g.t2.value = Uninit
self.assertEqual(4000, g.t2.call())
self.assertEqual(1, T1.eval_count)
self.assertEqual(1, T2.eval_count)
self.assertEqual(0, T1.load_count)
self.assertEqual(1, T2.load_count)
def test_depends_on_file(self):
with rflow.begin_graph("depends_on_file", HERE) as g:
t = g.t = DependsOnFile()
t.resource = rflow.FSResource(os.path.join(
resources.HERE, "sample-lines.txt"))
self.assertEqual(["line 1", "line 2", "line 3"], t.call())
def workflow1(g):
g.add = Add()
g.add.args.a = 1
g.add.args.b = 2
g.sub = Sub(rflow.FSResource('sub.pkl'))
g.sub.args.a = 8
g.sub.args.b = g.add
def _make_experiment(g, model_node,
crop_width, crop_height):
data_g = rflow.open_graph(".", "data")
g.train_dataset = AugmentDataset(show=False)
with g.train_dataset as args:
args.dataset = data_g.dataset_split[0]
args.crop_width = crop_width
args.crop_height = crop_height
g.train_dataset_view = ViewSegmentationDataset()
g.train_dataset_view.args.dataset = g.train_dataset
val_dataset = data_g.dataset_split[1]
test_dataset = data_g.dataset_split[2]
model = model_node[0]
preprocessing = model_node[1]
g.untrain_test = ViewDatasetPredictions()
with g.untrain_test as args:
args.dataset = test_dataset
args.model = model
args.preprocessing = preprocessing
g.train = Train(rflow.FSResource(f"{g.name}.torch"))
with g.train as args:
args.model = model
args.preprocessing = preprocessing
args.train_dataset = g.train_dataset
args.val_dataset = val_dataset
args.batch_size = 8
args.learning_rate = 0.001
args.max_epochs = 50
args.num_workers = rflow.UserArgument(
"--num_workers", default=4, type=int)
g.predict_test_set = ViewDatasetPredictions()
with g.predict_test_set as args:
args.dataset = test_dataset
args.model = g.train
args.preprocessing = preprocessing
g.predict_image = ViewImagePrediction()
with g.predict_image as args:
args.image_path = rflow.UserArgument(
"--sample-image",
default="dataset/resources/640px-Arabidopsis_Thaliana_planted_in_Laboratory.jpeg")
args.model = g.train
args.preprocessing = preprocessing
g.metrics = EvaluateMetrics()
with g.metrics as args:
args.dataset = data_g.dataset_split[2]
args.model = g.train
args.preprocessing = preprocessing
args.name = g.name
def mnist_train(g):
g.dataset = LoadDataset(rflow.FSResource("data"))
g.train = Train(rflow.FSResource("model.torch"))
with g.train as args:
args.train_dataset = g.dataset[0]
args.test_dataset = g.dataset[1]
args.batch_size = 64
args.test_batch_size = 1000
args.epochs = 14
args.learning_rate = 1.0
args.gamma = 0.1
args.device = "cuda:0"
g.test = Test()
with g.test as args:
args.model = g.train
args.test_dataset = g.dataset[1]
args.test_batch_size = 1000
args.device = "cuda:0"
def data(g):
"""
Graph width dataset loading and split.
"""
g.dataset = LoadDataset(rflow.FSResource(
"dataset/CVPPP2017_LCC_training/training"))
g.dataset_split = SplitDataset(
rflow.FSResource("dataset/dataset-split.pkl"))
with g.dataset_split as args:
args.dataset = g.dataset
args.train_size = 0.9
args.val_size = 0.05
args.test_size = 0.05
g.train_dataset_view = ViewSegmentationDataset()
g.train_dataset_view.args.dataset = g.dataset_split[0]
g.test_dataset_view = ViewSegmentationDataset()
g.test_dataset_view.args.dataset = g.dataset_split[2]
def test_resource_link(self):
with rflow.begin_graph('explict', HERE) as gh:
gh.g = GenFile()
gh.g.resource = rflow.FSResource(DependencyTest.GENFILE_PATH)
gh.g.args.p1 = 4
gh.g.args.p2 = 'Hello'
gh.u = ExplictUseFile()
gh.u.args.input_resource = gh.g.resource
self.assertEqual(['4 - Hello\n'], gh.u.call())
def test_task(self):
with rflow.begin_graph("task", HERE) as g:
t1 = g.t1 = T()
t1.args.v1 = 4
t1.args.v2 = 3
t1.args.v3 = 2
# R = 4 * 3 + 2 = 14
t1.resource = rflow.FSResource("T1.pickle")
t2 = g.t2 = T()
t2.args.v1 = t1
t2.args.v2 = 5
t2.args.v3 = t1
# 14 * 5 + 14 = 84
t2.resource = rflow.FSResource("T2.pickle")
c = g.c = C()
c.args.v1 = t2
c.args.v2 = t2
self.assertEqual(84, c.call()[0])
self.assertEqual(84, c.call()[1])
def exp2(g):
"""
Sample random text counting experiment.
Args:
g (:obj:`shrkit.workflow.Graph`): Experiment's initialized DAG.
"""
g.random_text = RandomText(rflow.FSResource('random-text.txt'))
with g.random_text as args:
args.num_of_words = 50
args.selected_words = [
'lorem', 'ipsum', 'dolor', 'sit', 'amet', 'consectetur',
'adipiscing', 'elit', 'vestibulum', 'non', 'feugiat', 'felis'
]
g.word_freq = WordFrequency(rflow.FSResource('random-text-freq.txt'))
g.word_freq.args.input_resource = g.random_text.resource
g.kmost_freq = PrintKMost()
with g.kmost_freq as args:
args.word_freq = g.word_freq
args.K = 10
def test_implicit(self):
with rflow.begin_graph('implicit', HERE) as gh:
gh.g = GenFile()
gh.g.resource = rflow.FSResource(DependencyTest.GENFILE_PATH)
gh.g.args.p1 = 5
gh.g.args.p2 = 'bye'
gh.u = ImplicitUseFile()
gh.u.args.input_filename = DependencyTest.GENFILE_PATH
with self.assertRaises(IOError):
gh.u.call()
gh.u.require(gh.g)
self.assertEqual(['5 - bye\n'], gh.u.call())
def test_template(self):
g = rflow.get_graph('test_template')
g.template = rflow.shell.TemplateFile(['HELLO', 'WHAT'])
with g.template as args:
args.HELLO = "Hello"
args.WHAT = "World"
args.template_resource = rflow.FSResource(RESOURCES /
'txt1.txt.template')
g.template.resource = rflow.FSResource(RESOURCES / 'txt1.txt')
g.template.call()
self._textfile_equals(RESOURCES / 'txt1.txt',
'Hello this is a World ?')
g.template.resource = rflow.FSResource(RESOURCES / 'txt2.txt')
g.template.call()
self._textfile_equals(RESOURCES / 'txt2.txt',
'Hello this is a World ?')
g.template.args.template_resource = rflow.FSResource(
RESOURCES / 'txt2.txt.template')
g.template.call()
self._textfile_equals(RESOURCES / 'txt2.txt', 'World, Hello ?')
def test_non_collateral(self):
with rflow.begin_graph("collateral-test", HERE) as g:
g.a = A()
g.a.resource = rflow.FSResource("collateral-v1.pkl")
g.a.args.v1 = 1
g.a.args.v2 = True
g.a.call()
g.a.args.v2 = False
g.a._is_dirty = True
g.a.value = rflow.Uninit
g.a.call()
g.a.args.v1 = 2
g.a._is_dirty = True
g.a.value = rflow.Uninit
g.a.call()
self.assertEqual(1, A.load_count)
self.assertEqual(2, A.eval_count)
