def _(points, grid, snap_fn):
"""Returns grid coordinates contained within a dict of points.
Args:
points (dict): {'ulx': 0, 'uly': 15.5, 'lrx': 55.3, 'lry':-1000.12}
grid (dict): The target grid: {'name': 'chip', 'sx': 3000, 'sy': 3000, 'rx': 1, 'ry': -1}
snap_fn (func): A function that accepts x, y and returns a snapped x, y
Returns:
tuple: tuple of tuples of grid coordinates ((x1,y1), (x2,y1) ...)
This example assumes a grid size of 500x500 pixels.
Example:
>>> coordinates = coordinates({ulx: -1001, uly: 1000, lrx: -500, lry: 500},
grid={'name': 'chip', 'sx': 500, 'sy': 500, 'rx': 1, 'ry': -1},
snap_fn=some_func)
((-1000, 500), (-500, 500), (-1000, -500), (-500, -500))
"""
# snap start/end x & y
start_x, start_y = get_in([grid.get('name'), 'proj-pt'],
snap_fn(x=points.get('ulx'),
y=points.get('uly')))
end_x, end_y = get_in([grid.get('name'), 'proj-pt'],
snap_fn(x=points.get('lrx'), y=points.get('lry')))
# get x and y scale factors multiplied by reflection
x_interval = grid.get('sx') * grid.get('rx')
y_interval = grid.get('sy') * grid.get('ry')
return tuple((x, y)
for x in np.arange(start_x, end_x + x_interval, x_interval)
for y in np.arange(start_y, end_y + y_interval, y_interval))
def block_num_from_jsonrpc_response(jsonrpc_response: dict = None) -> int:
# pylint: disable=no-member
# for get_block
block_id = cytoolz.get_in(['result', 'block_id'], jsonrpc_response)
if block_id:
return block_num_from_id(block_id)
# for get_block_header
previous = cytoolz.get_in(['result', 'previous'], jsonrpc_response)
return block_num_from_id(previous) + 1
def extract_serialize(x):
""" Pull out Serialize objects from message
This also remove large bytestrings from the message into a second
dictionary.
Examples
--------
>>> from distributed.protocol import to_serialize
>>> msg = {'op': 'update', 'data': to_serialize(123)}
>>> extract_serialize(msg)
({'op': 'update'}, {('data',): <Serialize: 123>}, set())
"""
ser = {}
_extract_serialize(x, ser)
if ser:
x = container_copy(x)
for path in ser:
t = get_in(path[:-1], x)
if isinstance(t, dict):
del t[path[-1]]
else:
t[path[-1]] = None
bytestrings = set()
for k, v in ser.items():
if type(v) in (bytes, bytearray):
ser[k] = to_serialize(v)
bytestrings.add(k)
return x, ser, bytestrings
async def test_rpc_against_fixtures(chain, ipc_server, chain_fixture, fixture_data):
rpc = RPCServer(MainnetFullChain(None))
setup_result, setup_error = await call_rpc(rpc, 'evm_resetToGenesisFixture', [chain_fixture])
assert setup_error is None and setup_result is True, "cannot load chain for {0}".format(fixture_data) # noqa: E501
await validate_accounts(rpc, chain_fixture['pre'])
for block_fixture in chain_fixture['blocks']:
should_be_good_block = 'blockHeader' in block_fixture
if 'rlp_error' in block_fixture:
assert not should_be_good_block
continue
block_result, block_error = await call_rpc(rpc, 'evm_applyBlockFixture', [block_fixture])
if should_be_good_block:
assert block_error is None
assert block_result == block_fixture['rlp']
await validate_block(rpc, block_fixture, block_fixture['blockHeader']['hash'])
else:
assert block_error is not None
if chain_fixture.get('lastblockhash', None):
for block_fixture in chain_fixture['blocks']:
if get_in(['blockHeader', 'hash'], block_fixture) == chain_fixture['lastblockhash']:
await validate_last_block(rpc, block_fixture)
await validate_accounts(rpc, chain_fixture['postState'])
await validate_accounts(rpc, chain_fixture['pre'], 'earliest')
await validate_accounts(rpc, chain_fixture['pre'], 0)
def train(ctx, cfg):
'''Train an xgboost model'''
itrain, itest, dtrain, dtest = train_test_split(ctx['independent'],
ctx['dependent'],
test_size=get_in(['xgboost', 'test_size'], cfg))
train_matrix = xgb.DMatrix(data=itrain, label=dtrain)
test_matrix = xgb.DMatrix(data=itest, label=dtest)
return assoc(ctx, 'model', xgb.train(params=get_in(['xgboost', 'parameters'], cfg),
dtrain=train_matrix,
num_boost_round=get_in(['xgboost', 'num_round'], cfg),
evals=watchlist(train_matrix, test_matrix),
early_stopping_rounds=get_in(['xgboost', 'early_stopping_rounds'], cfg),
verbose_eval=get_in(['xgboost', 'verbose_eval'], cfg)))
def relevant_markets_near(
location,
commodity_filter=None,
radius=30,
):
results = []
if commodity_filter is None:
return markets_near(location, radius)
else:
systems = edsm.systems_in_sphere(location, radius)
system_names = [system["name"] for system in systems]
with ThreadPoolExecutor(max_workers=6) as exe:
all_markets = list(exe.map(edsm.markets_in_system, system_names))
for (system, markets) in zip(systems, all_markets):
stations = [market["station"] for market in markets]
all_commodities = [
get_in(["market", "commodities"], market, default=[])
for market in markets
]
relevant_commodities = [
commodity_filter(market) for market in markets
]
results.extend([{
"system": system,
"station": station,
"commodities": commodities,
"relevant": relevant,
} for (station, commodities, relevant
) in zip(stations, all_commodities, relevant_commodities)
if relevant])
return results
def test_rpc_against_fixtures(ipc_server, chain_fixture, fixture_data):
rpc = RPCServer(None)
setup_result, setup_error = call_rpc(rpc, 'evm_resetToGenesisFixture', [chain_fixture])
assert setup_error is None and setup_result is True, "cannot load chain for %r" % fixture_data
validate_accounts(rpc, chain_fixture['pre'])
for block_fixture in chain_fixture['blocks']:
should_be_good_block = 'blockHeader' in block_fixture
if 'rlp_error' in block_fixture:
assert not should_be_good_block
continue
block_result, block_error = call_rpc(rpc, 'evm_applyBlockFixture', [block_fixture])
if should_be_good_block:
assert block_error is None
assert block_result == block_fixture['rlp']
validate_block(rpc, block_fixture, block_fixture['blockHeader']['hash'])
else:
assert block_error is not None
if chain_fixture.get('lastblockhash', None):
for block_fixture in chain_fixture['blocks']:
if get_in(['blockHeader', 'hash'], block_fixture) == chain_fixture['lastblockhash']:
validate_last_block(rpc, block_fixture)
validate_accounts(rpc, chain_fixture['postState'])
validate_accounts(rpc, chain_fixture['pre'], 'earliest')
validate_accounts(rpc, chain_fixture['pre'], 0)
def fetch_swept_parameters(self, config, sweep_paths,
swept_parameter_paths):
"""
Construct named tuples for all the parameters being swept.
"""
# Filter sweep paths to all 3 sweep types
disjoint_sweep_paths = list(
filter(lambda s: 'disjoint' in last(s), sweep_paths))
product_sweep_paths = list(
filter(lambda s: 'product' in last(s), sweep_paths))
default_sweep_paths = list(
filter(lambda s: 'default' in last(s), sweep_paths))
# Construct SweptParameter and Swept__Parameter namedtuples,
# making it
# consisting of the path to the parameter and its sweep configuration
construct_swept_parameters = lambda subtype, paths, wrapper: list(
map(lambda p: subtype(wrapper(p), tz.get_in(p, coll=config)), paths
))
swept_parameters = construct_swept_parameters(SweptParameter,
swept_parameter_paths,
identity)
swept_disjoint_parameters = construct_swept_parameters(
SweptDisjointParameter, disjoint_sweep_paths,
QuinSweep.get_parameter_path)
swept_product_parameters = construct_swept_parameters(
SweptProductParameter, product_sweep_paths,
QuinSweep.get_parameter_path)
swept_default_parameters = construct_swept_parameters(
SweptDefaultParameter, default_sweep_paths,
QuinSweep.get_parameter_path)
return swept_parameters, swept_disjoint_parameters, swept_product_parameters, swept_default_parameters
def __init__(self, schema=None, **kwargs):
super(QuinineArgumentParser, self).__init__(**kwargs)
# Add a default argument for the path to the YAML configuration file that will be passed in
self.add_argument('--config',
type=str,
required=True,
help="YAML configuration file.")
self.schema = schema
if self.schema is not None:
# Populate the argument parser with arguments from the schema
paths_to_type = list(
filter(lambda l: l[-1] == 'type',
get_all_leaf_paths(self.schema)))
type_lookup = dict([(tuple(filter(lambda e: e != 'schema',
e[:-1])), tz.get_in(e, schema))
for e in paths_to_type])
valid_params = self.get_all_params(schema)
for param in valid_params:
self.add_argument(f'--{".".join(param)}',
type=self.types[type_lookup[param]])
self.schema = merge(self.schema, {
'config': tstring,
'inherit': tlist
})
def extract_serialize(x):
""" Pull out Serialize objects from message
This also remove large bytestrings from the message into a second
dictionary.
Examples
--------
>>> from distributed.protocol import to_serialize
>>> msg = {'op': 'update', 'data': to_serialize(123)}
>>> extract_serialize(msg)
({'op': 'update'}, {('data',): <Serialize: 123>}, set())
"""
ser = {}
_extract_serialize(x, ser)
if ser:
x = container_copy(x)
for path in ser:
t = get_in(path[:-1], x)
if isinstance(t, dict):
del t[path[-1]]
else:
t[path[-1]] = None
bytestrings = set()
for k, v in ser.items():
if type(v) is bytes:
ser[k] = to_serialize(v)
bytestrings.add(k)
return x, ser, bytestrings
def cascading_lookup(paths, data):
NOT_FOUND = object()
for path in paths:
result = get_in(path, data, default=NOT_FOUND)
if result is not NOT_FOUND:
return result
else:
return None
def fetch_tape(self, path: List[str]) -> InteractionTape:
"""Fetch an InteractionTape.
Args:
path:
Returns:
"""
return tz.get_in(path, self.interactions)
def train(ctx, cfg):
'''Train an xgboost model'''
logger.info("training model")
itrain, itest, dtrain, dtest = train_test_split(
ctx['independent'],
ctx['dependent'],
test_size=get_in(['xgboost', 'test_size'], cfg))
train_matrix = xgb.DMatrix(data=itrain, label=dtrain)
test_matrix = xgb.DMatrix(data=itest, label=dtest)
watch_list = watchlist(train_matrix, test_matrix)
model = xgb.train(params=get_in(['xgboost', 'parameters'], cfg),
dtrain=train_matrix,
num_boost_round=get_in(['xgboost', 'num_round'], cfg),
evals=watch_list,
early_stopping_rounds=get_in(
['xgboost', 'early_stopping_rounds'], cfg),
verbose_eval=get_in(['xgboost', 'verbose_eval'], cfg))
ctx['independent'] = None
ctx['dependent'] = None
itrain = None
itest = None
dtrain = None
dtest = None
train_matrix = None
test_matrix = None
watch_list = None
del ctx['independent']
del ctx['dependent']
del itrain
del itest
del dtrain
del dtest
del train_matrix
del test_matrix
del watch_list
return assoc(ctx, 'model', model)
def test_get_in_doctest():
# Original doctest:
# >>> get_in(['y'], {}, no_default=True)
# Traceback (most recent call last):
# ...
# KeyError: 'y'
# cytoolz result:
# KeyError:
raises(KeyError, lambda: cytoolz.get_in(['y'], {}, no_default=True))
def parse_quinfig(self):
# Parse all the arguments from the command line, overriding defaults in the argparse
args = self.parse_args()
cli_keys = []
for cli_arg in _sys.argv[1:]:
if cli_arg.startswith('--'):
if str(cli_arg) == '--config':
continue
cli_keys.append(cli_arg[2:].replace("-", "_"))
elif cli_arg.startswith('-'):
raise NotImplementedError(
"QuinineArgumentParser doesn't support abbreviated arguments."
)
else:
continue
# Get parameters which need to be overridden from command line
override_args = project(args.__dict__, cli_keys)
# Trick: first load the config without a schema as a base config
# quinfig = Quinfig(config_path=args.config)
# Override all the defaults using the yaml config
# quinfig = rmerge(Quinfig(config=args.__dict__), quinfig)
# print(quinfig)
# Use all the defaults in args to populate a dictionary
quinfig = {}
for param, val in args.__dict__.items():
param_path = param.split(".")
quinfig = tz.assoc_in(quinfig, param_path, val)
# Override all the defaults using the yaml config
quinfig = rmerge(quinfig, Quinfig(config_path=args.config))
# Replace all the arguments passed into command line
if len(override_args) > 0:
print(
f"\n(quinine) Overriding parameters in {args.config} from command line (___ is unspecified)."
)
for param, val in override_args.items():
param_path = param.split(".")
old_val = tz.get_in(param_path, quinfig)
if old_val != val:
print(f"> ({param}): {old_val} --> {val}")
else:
print(f"> ({param}): ___ --> {val}")
quinfig = tz.assoc_in(quinfig, param_path, val)
# Load the config again
return Quinfig(config=quinfig, schema=self.schema)
def test_get_in_doctest():
# Original doctest:
# >>> get_in(['y'], {}, no_default=True)
# Traceback (most recent call last):
# ...
# KeyError: 'y'
# cytoolz result:
# KeyError:
raises(KeyError, lambda: cytoolz.get_in(['y'], {}, no_default=True))
def filter_markets(
markets,
desired_commodities,
min_price=1,
min_demand=1,
max_update_seconds=24 * 3600,
):
result = []
if not markets:
return []
for market in markets:
commodities1 = (get_in(["market", "commodities"], market, default=None)
or [])
commodities = [
c for c in commodities1 if c["name"] in desired_commodities
]
price_demand_ok = any(
(c["name"] in desired_commodities and c["sellPrice"] >= min_price
and c["demand"] >= min_demand) for c in commodities)
if not price_demand_ok:
continue
market_update = get_in(
["station", "updateTime", "market"],
market,
default=None,
)
if market_update is None:
continue
delta = time_since(market_update)
update_ok = delta.total_seconds() < max_update_seconds
if not update_ok:
continue
# otherwise
result.append(market)
return result
def _filter_sell_commodity(market):
commodities = get_in(
["market", "commodities"],
market,
default=None,
)
commodities = commodities or []
return next(
(c for c in commodities if c["name"].lower() == name.lower()
and c["demand"] >= min_demand and c["sellPrice"] >= min_price),
None,
)
def get_bygeom(self, geometry: GTYPE, geo_crs: str) -> gpd.GeoDataFrame:
"""Retrieve NID data within a geometry.
Parameters
----------
geometry : Polygon, MultiPolygon, or tuple of length 4
Geometry or bounding box (west, south, east, north) for extracting the data.
geo_crs : list of str
The CRS of the input geometry, defaults to epsg:4326.
Returns
-------
geopandas.GeoDataFrame
GeoDataFrame of NID data
Examples
--------
>>> from pygeohydro import NID
>>> nid = NID()
>>> dams = nid.get_bygeom((-69.77, 45.07, -69.31, 45.45), "epsg:4326")
>>> print(dams.name.iloc[0])
Little Moose
"""
_geometry = geoutils.geo2polygon(geometry, geo_crs, DEF_CRS)
wbd = ArcGISRESTful(
ServiceURL().restful.wbd,
4,
outformat="json",
outfields="huc8",
expire_after=self.expire_after,
disable_caching=self.disable_caching,
)
resp = wbd.get_features(wbd.oids_bygeom(_geometry), return_geom=False)
huc_ids = [
tlz.get_in(["attributes", "huc8"], i) for r in resp
for i in tlz.get_in(["features"], r)
]
dams = self.get_byfilter([{"huc8": huc_ids}])[0]
return dams[dams.within(_geometry)].copy()
def combine(ctx):
'''Combine segments with matching aux entry'''
data = []
for s in ctx['segments']:
key = (s.cx, s.cy, s.px, s.py)
a = get_in(['aux', key], ctx, None)
if a is not None:
data.append(merge(a, s._asdict()))
return assoc(ctx, 'data', data)
def combine(ctx):
'''Combine segments with matching aux entry'''
data = []
for s in ctx['segments']:
key = (s['cx'], s['cy'], s['px'], s['py'])
a = get_in(['aux', key], ctx, None)
if a is not None:
data.append(merge(a, s))
return assoc(ctx, 'data', data)
def network_id(self) -> int:
if self._network_id is not None:
return self._network_id
try:
return get_in(
[self.chain_identifier, 'network_id'],
CHAIN_CONFIG_DEFAULTS,
no_default=True,
)
except KeyError:
raise ValueError(
"The network_id for the chain '{0}' was not explicitely set and "
"is not for a known network. Please specify a network_id")
def create(x, y, acquired, cfg):
"""Create a timeseries.
Args:
x (int): x coordinate
y (int): y coordinate
acquired (string): iso8601 date range
cfg (dict): A Merlin configuration
Returns:
tuple - Results of format_fn applied to results of chips_fn
"""
x, y = get_in(['chip', 'proj-pt'], cfg['snap_fn'](x=x, y=y))
# get specs
specmap = cfg['specs_fn'](specs=cfg['registry_fn']())
# get function that will return chipmap.
# Don't create state with a realized variable to preserve memory
chipmap = partial(chips.mapped,
x=x,
y=y,
acquired=acquired,
specmap=specmap,
chips_fn=cfg['chips_fn'])
# calculate locations chip. There's another function
# here to be split out and organized.
grid = first(filter(lambda x: x['name'] == 'chip', cfg['grid_fn']()))
cw, ch = specs.refspec(specmap).get('data_shape')
locations = partial(chips.locations,
x=x,
y=y,
cw=cw,
ch=ch,
rx=grid.get('rx'),
ry=grid.get('ry'),
sx=grid.get('sx'),
sy=grid.get('sy'))
return cfg['format_fn'](x=x,
y=y,
locations=locations(),
dates_fn=cfg['dates_fn'],
specmap=specmap,
chipmap=chipmap())
def test_pyccd():
c = cfg.get('chipmunk-ard', env=test.env)
x, y = get_in(['chip', 'proj-pt'], c['snap_fn'](x=test.x, y=test.y))
# get specs
specmap = c['specs_fn'](specs=c['registry_fn']())
# get function that will return chipmap.
# Don't create state with a realized variable to preserve memory
chipmap = partial(chips.mapped,
x=test.x,
y=test.y,
acquired=test.acquired,
specmap=specmap,
chips_fn=c['chips_fn'])
# calculate locations chip. There's another function
# here to be split out and organized.
grid = first(filter(lambda x: x['name'] == 'chip', c['grid_fn']()))
cw, ch = specs.refspec(specmap).get('data_shape')
locations = chips.locations(x=x,
y=y,
cw=cw,
ch=ch,
rx=grid.get('rx'),
ry=grid.get('ry'),
sx=grid.get('sx'),
sy=grid.get('sy'))
data = c['format_fn'](x=x,
y=y,
locations=locations,
dates_fn=c['dates_fn'],
specmap=specmap,
chipmap=chipmap())
# we are only testing the structure of the response here.
# Full data validation is being done in the test for merlin.create()
assert type(data) is tuple
assert len(data) == 10000
assert type(first(data)) is tuple
assert type(first(first(data))) is tuple
assert type(second(first(data))) is dict
assert type(second(second(first(data)))) is tuple or list
assert len(second(second(first(data)))) > 0
def hypothetical_sale(commodities, market):
mkt_commodities = get_in(["market", "commodities"], market, default=[])
mkt = {c["name"]: c for c in mkt_commodities}
matches = [{
"name": name,
"sellPrice": mkt[name]["sellPrice"],
"revenue": mkt[name]["sellPrice"] * quantity,
} for (name, quantity) in commodities.items() if name in mkt]
return {
"total":
sum(m["revenue"] for m in matches),
"matched":
matches,
"missing":
[name for (name, quantity) in commodities.items() if name not in mkt],
}
def loads(frames, deserialize=True):
""" Transform bytestream back into Python value """
frames = frames[::-1] # reverse order to improve pop efficiency
if not isinstance(frames, list):
frames = list(frames)
try:
small_header = frames.pop()
small_payload = frames.pop()
msg = loads_msgpack(small_header, small_payload)
if not frames:
return msg
header = frames.pop()
header = msgpack.loads(header, encoding='utf8', use_list=False)
keys = header['keys']
headers = header['headers']
bytestrings = set(header['bytestrings'])
for key in keys:
head = headers[key]
count = head['count']
if count:
fs = frames[-count::][::-1]
del frames[-count:]
else:
fs = []
if deserialize or key in bytestrings:
if 'compression' in head:
fs = decompress(head, fs)
fs = merge_frames(head, fs)
value = _deserialize(head, fs)
else:
value = Serialized(head, fs)
get_in(key[:-1], msg)[key[-1]] = value
return msg
except Exception:
logger.critical("Failed to deserialize", exc_info=True)
raise
def parse_quinfig(self):
# Parse all the arguments
args = self.parse_args()
override_args = dict(
select_values(lambda v: v is not None,
omit(args.__dict__, ['config'])))
# Trick: first load the config without a schema
quinfig = Quinfig(config_path=args.config)
# Replace all the arguments passed into command line
if len(override_args) > 0:
print(f"Overriding arguments in {args.config} from command line.")
for param, val in override_args.items():
param_path = param.split(".")
old_val = tz.get_in(param_path, quinfig)
print(f"> ({param}): {old_val} --> {val}")
quinfig = tz.assoc_in(quinfig, param_path, val)
# Load the config again, this time with the schema
return Quinfig(config=quinfig.__dict__, schema=self.schema)
def loads(frames, deserialize=True):
""" Transform bytestream back into Python value """
frames = frames[::-1] # reverse order to improve pop efficiency
if not isinstance(frames, list):
frames = list(frames)
try:
small_header = frames.pop()
small_payload = frames.pop()
msg = loads_msgpack(small_header, small_payload)
if not frames:
return msg
header = frames.pop()
header = msgpack.loads(header, encoding='utf8', use_list=False)
keys = header['keys']
headers = header['headers']
bytestrings = set(header['bytestrings'])
for key in keys:
head = headers[key]
lengths = head['lengths']
count = head['count']
fs = frames[-count::][::-1]
del frames[-count:]
if deserialize or key in bytestrings:
fs = decompress(head, fs)
fs = merge_frames(head, fs)
value = _deserialize(head, fs)
else:
value = Serialized(head, fs)
get_in(key[:-1], msg)[key[-1]] = value
return msg
except Exception as e:
logger.critical("Failed to deserialize", exc_info=True)
raise
def difference(*colls):
"""
Find the keys that have different values in an arbitrary number of (nested) collections. Any key
that differs in at least 2 collections is considered to fit this criterion.
"""
# Get all the leaf paths for each collection: make each path a tuple
leaf_paths_by_coll = list(
map(lambda c: list(map(tuple, get_all_leaf_paths(c))), colls))
# Find the union of all leaf paths: merge all the paths and keep only the unique paths
union_leaf_paths = list(distinct(concat(*leaf_paths_by_coll)))
# Get the values corresponding to these leaf paths in every collection: if a leaf path doesn't exist, assumes None
values_by_coll = list(
map(lambda lp: list(map(lambda coll: tz.get_in(lp, coll), colls)),
union_leaf_paths))
# Filter out the leaf paths that have identical values across the collections
keep_leaf_paths = list(
map(
0,
filter(lambda t: not allequal(t[1]),
zip(union_leaf_paths, values_by_coll))))
keep_values = list(
map(
1,
filter(lambda t: not allequal(t[1]),
zip(union_leaf_paths, values_by_coll))))
# Rearrange to construct a list of dictionaries -- one per original collection.
# Each of these dictionaries maps a 'kept' leaf path to its corresponding
# value in the collection
differences = list(
map(lambda vals: dict(zip(keep_leaf_paths, vals)),
list(zip(*keep_values))))
return differences
def booster(cfg, model_bytes):
return segaux.booster_from_bytes(
model_bytes,
{'nthread': get_in(['xgboost', 'parameters', 'nthread'], cfg)})
def replace_underscores(self, swept_parameter):
"""
Replace all the underscore references in sweep of swept_parameter.
"""
# Find all the references (i.e. dependencies) made by the swept_parameter
references = []
for token in QuinSweep.SWEEP_TOKENS[:
-1]: # omit default since it's value is never a dict
if f"~{token}" in swept_parameter.sweep and is_mapping(
swept_parameter.sweep[f"~{token}"]):
references.extend(
list(swept_parameter.sweep[f"~{token}"].keys()))
# Find all the referred parameters
parsed_references = list(map(QuinSweep.parse_ref_dotpath, references))
dotpaths = list(cat(parsed_references))
ref_dict = merge_with(compose(list, cat),
*list(map(lambda e: dict([e]), dotpaths)))
# TODO: there's a bug here potentially
assert all(map(lambda l: len(l) == len(set(l)), list(itervalues(ref_dict)))), \
'All conditions must be distinct.'
ref_dict_no_underscores = walk_values(
compose(set,
autocurry(map)(int),
autocurry(filter)(lambda e: e != '_')), ref_dict)
if not references:
return swept_parameter
def compute_possibilities(full_dotpath, reference):
# Look up the parameter using the dotpath
parameter = self.swept_parameters_dict[full_dotpath]
# Use the reference to figure out how many possiblities exist for the underscore
if len(reference) > 0:
# Merge all the sweeps performed for this parameter
merged_sweep = merge(
*list(filter(is_mapping, itervalues(parameter.sweep))))
# Look up the reference
return len(merged_sweep[reference])
assert len(
parameter.sweep
) == 1, 'If no reference, must be a single unconditional sweep.'
# The number of possibilities is simply the number of values specified
# in the (product/disjoint) unconditional sweep
return len(list(parameter.sweep.values())[0])
# Update the sweep by replacing underscores
updated_sweep = swept_parameter.sweep
# Loop over all the parsed references
for parsed_ref in parsed_references:
# Expand all the partial dotpaths
# TODO: remove? expanding all the partial dotpaths in the beginning?
parsed_ref = list(
map(lambda t: (self.expand_partial_dotpath(t[0]), t[1]),
parsed_ref))
# For each parsed reference, there will be multiple (dotpath, idx) pairs
for i, (full_dotpath, ref_idx) in enumerate(parsed_ref):
# If the reference index is not an underscore, continue
if not ref_idx == '_':
continue
# Compute the prefix reference
prefix_reference = ".".join(list(cat(parsed_ref[:i])))
# Compute the number of possible ways to replace the underscore
n_possibilities = compute_possibilities(
full_dotpath, prefix_reference)
replacements = set(range(
n_possibilities)) - ref_dict_no_underscores[full_dotpath]
# Find the path to the underscore condition
path_to_condition = get_only_paths(
updated_sweep,
lambda p: any(lambda e: '_' in e, p),
stop_at=full_dotpath)[0]
# Find the value of the underscore condition
value = tz.get_in(path_to_condition, updated_sweep)
# Construct keys that are subtitutes for the underscore
keys = list(map(lambda s: f'{full_dotpath}.{s}', replacements))
keys = list(map(lambda k: path_to_condition[:-1] + [k], keys))
# Update by adding those keys in
for k in keys:
updated_sweep = tz.assoc_in(updated_sweep, k, value)
# Create a new swept parameter with the updated sweep
swept_parameter = SweptParameter(
swept_parameter.path,
walk_values(
iffy(is_mapping,
autocurry(select_keys)(lambda k: '_' not in k)),
updated_sweep))
return swept_parameter
def block_num_from_jsonrpc_response(
jsonrpc_response: SingleJsonRpcRequest = None) -> int:
# pylint: disable=no-member
block_id = cytoolz.get_in(['result', 'block_id'], jsonrpc_response)
return block_num_from_id(block_id)
def format(cx, cy, px, py, dates, ccdresult):
return [{
'cx': int(cx),
'cy': int(cy),
'px': int(px),
'py': int(py),
'sday': date.fromordinal(get('start_day', cm, 1)).isoformat(),
'eday': date.fromordinal(get('end_day', cm, 1)).isoformat(),
'bday': date.fromordinal(get('break_day', cm, 1)).isoformat(),
'chprob': get('change_probability', cm, 0.0),
'curqa': get('curve_qa', cm, 0),
'blmag': get_in(['blue', 'magnitude'], cm, 0.0),
'grmag': get_in(['green', 'magnitude'], cm, 0.0),
'remag': get_in(['red', 'magnitude'], cm, 0.0),
'nimag': get_in(['nir', 'magnitude'], cm, 0.0),
's1mag': get_in(['swir1', 'magnitude'], cm, 0.0),
's2mag': get_in(['swir2', 'magnitude'], cm, 0.0),
'thmag': get_in(['thermal', 'magnitude'], cm, 0.0),
'blrmse': get_in(['blue', 'rmse'], cm, 0.0),
'grrmse': get_in(['green', 'rmse'], cm, 0.0),
'rermse': get_in(['red', 'rmse'], cm, 0.0),
'nirmse': get_in(['nir', 'rmse'], cm, 0.0),
's1rmse': get_in(['swir1', 'rmse'], cm, 0.0),
's2rmse': get_in(['swir2', 'rmse'], cm, 0.0),
'thrmse': get_in(['thermal', 'rmse'], cm, 0.0),
'blcoef': coefficients(cm, 'blue'),
'grcoef': coefficients(cm, 'green'),
'recoef': coefficients(cm, 'red'),
'nicoef': coefficients(cm, 'nir'),
's1coef': coefficients(cm, 'swir1'),
's2coef': coefficients(cm, 'swir2'),
'thcoef': coefficients(cm, 'thermal'),
'blint': get_in(['blue', 'intercept'], cm, 0.0),
'grint': get_in(['green', 'intercept'], cm, 0.0),
'reint': get_in(['red', 'intercept'], cm, 0.0),
'niint': get_in(['nir', 'intercept'], cm, 0.0),
's1int': get_in(['swir1', 'intercept'], cm, 0.0),
's2int': get_in(['swir2', 'intercept'], cm, 0.0),
'thint': get_in(['thermal', 'intercept'], cm, 0.0),
'dates': [date.fromordinal(o).isoformat() for o in dates],
'mask': get('processing_mask', ccdresult)
} for cm in defaults(get('change_models', ccdresult, None))]
def coefficients(change_model, spectra):
coefs = get_in([spectra, 'coefficients'], change_model)
return list(coefs) if coefs else []