def test_param_grid_list():
first = {'a': [1, 2], 'b': [1, 2]}
second = {'c': [3, 4], 'd': [3, 4]}
pg = ParamGrid([first, second])
assert list(pg.product()) == [{
'a': 1,
'b': 1
}, {
'a': 1,
'b': 2
}, {
'a': 2,
'b': 1
}, {
'a': 2,
'b': 2
}, {
'c': 3,
'd': 3
}, {
'c': 3,
'd': 4
}, {
'c': 4,
'd': 3
}, {
'c': 4,
'd': 4
}]
def from_grid(cls,
task_class,
product_class,
product_primitive,
task_kwargs,
dag,
grid,
name=None,
namer=None,
resolve_relative_to=None):
"""
Build a group of tasks of the same class from an grid of parameters
using the same source.
Parameters
----------
grid : dict or list of dicts
If dict, all combinations of individual parameters are generated.
If list of dicts, each dict is processed individually, then
concatenated to generate the final set.
Notes
-----
All parameters, except for grid are the same as in .from_params
"""
params_array = ParamGrid(grid).product()
return cls.from_params(task_class=task_class,
product_class=product_class,
product_primitive=product_primitive,
task_kwargs=task_kwargs,
dag=dag,
name=name,
params_array=params_array,
namer=namer,
resolve_relative_to=resolve_relative_to)
def test_param_grid_w_interval():
pg = ParamGrid({'a': Interval(0, 10, 2), 'b': [2, 4, 6, 8, 10]})
assert compare(list(pg.zip()), [{
'a': (0, 2),
'b': 2
}, {
'a': (2, 4),
'b': 4
}, {
'a': (4, 6),
'b': 6
}, {
'a': (6, 8),
'b': 8
}, {
'a': (8, 10),
'b': 10
}])
def test_param_grid():
pg = ParamGrid({'a': [1, 2, 3], 'b': [2, 4, 6]})
assert compare(list(pg.zip()), [{
'a': 1,
'b': 2
}, {
'a': 2,
'b': 4
}, {
'a': 3,
'b': 6
}])
assert compare(list(pg.product()), [{
'a': 1,
'b': 2
}, {
'a': 1,
'b': 4
}, {
'a': 1,
'b': 6
}, {
'a': 2,
'b': 2
}, {
'a': 2,
'b': 4
}, {
'a': 2,
'b': 6
}, {
'a': 3,
'b': 2
}, {
'a': 3,
'b': 4
}, {
'a': 3,
'b': 6
}])
def test_param_grid_zip_with_single_value(val):
pg = ParamGrid({'a': val, 'b': ['more']})
assert len(list(pg.zip())) == 1
def test_param_grid_product_with_single_value(val):
pg = ParamGrid({'a': val, 'b': ['more', 'final']})
assert len(list(pg.product())) == 2
def test_param_grid_with_str_list():
pg = ParamGrid({
'a': ['one', 'another'],
'b': ['more', 'final'],
})
assert len(list(pg.product())) == 4
closed='left',
freq='D')
values = np.random.rand(dates_series.shape[0])
df = pd.DataFrame({'dates': dates_series, 'values': values})
df.to_parquet(str(product))
dag = DAG()
product = File('{{name}}.parquet')
start_date = date(year=2010, month=1, day=1)
end_date = date(year=2019, month=6, day=1)
delta = relativedelta(years=1)
params_array = ParamGrid({
'dates': Interval(start_date, end_date, delta)
}).zip()
def namer(params):
s = str(params['dates'][0]).replace('-', '_')
e = str(params['dates'][1]).replace('-', '_')
return 'get_data_{}_{}'.format(s, e)
make_task_group(task_class=PythonCallable,
task_kwargs={
'source': get_data,
'product': product
},
dag=dag,