def tags(beamline, runs):
hi_tags = fromiter(map(read_hightagnumber(beamline), runs), 'int')
if not hi_tags.all():
raise ValueError('Not all the runs have a single high tag!')
hi_tag = hi_tags[0]
low_tags = concat(map(read_taglist_byrun(beamline), runs))
return hi_tag, low_tags
def data_generator_fn():
data_generator = dataset.training_set.random_batch_arrays_generator(
batch_size)
data_generator = cz.map(
Dict(image=P[0], labels=P[1], keras_training=False),
data_generator)
# data_generator = cz.map(utils.get_processed_image, data_generator)
return data_generator
def __getitem__(self, key: str) -> iter:
if key not in self.map:
raise ValueError(
dedent("""\
Key '{}' is invalid!
Valid keys: {}
""".format(
key,
reduce(lambda k1, k2: '{}, {}'.format(k1, k2),
map(lambda k: "'{}'".format(k), self.map)))))
ref = self.map[key]
if 'api' not in ref:
ref['api'] = 'dbpy' # default api
api = ref['api']
# load reader
if key not in self.cache:
print("Loading '{}' reader...".format(key))
if api not in ('dbpy', 'stpy'):
raise ValueError("Invalid api type '{}'!".format(api))
if 'id' not in ref:
ref['id'] = key # default id
id = ref['id']
if api == 'dbpy':
self.cache[key] = fromiter(
read_syncdatalist_float(id, self.hi_tag,
tuple(map(int,
self.low_tags))),
'float')
if api == 'stpy':
self.cache[key] = StorageWrapper(*map(int, self.runs),
beamline=self.beamline,
id=id)
if 'deco' not in ref:
ref['deco'] = identity # default deco
print('Loaded!')
data = self.cache[key]
deco = ref['deco'] if hasattr(ref['deco'], '__call__') else eval(
ref['deco'])
if api == 'dbpy':
return map(deco, data)
if api == 'stpy':
return map(compose(deco, data.__getitem__), self.low_tags)
def make_dict_and_corpus(tokensets):
dictionary = corpora.Dictionary(tokensets)
dictionary.filter_extremes(no_below=5, no_above=0.5, keep_n=100000)
dictionary.compactify()
corpus = list(tlz.map(dictionary.doc2bow, tokensets))
return (dictionary, corpus)
def csv(specs):
"""Converts a spec dict to a csv entry
Args:
specs (sequence): spec dicts
Returns:
sequence: csv records
"""
return map(lambda spec: '{x},{y},{value}\n'.format(**spec), specs)
def write(values, filepath):
"""Writes a sequence of values to a filepath
Args:
filepath (str): Full path of raster file to write
values (sequence): Values to write
Returns:
tuple: (filepath, bytes written)
"""
with open(filepath, 'w+') as handle:
return (filepath, reduce(add, map(handle.write, values)))
def treemap(tree, mapfn, key=None, path=()):
res = tree
if isinstance(tree, dict):
res = keyfilter(
identity,
itemmap(
lambda item: treemap(item[1], mapfn, item[0], path +
(item[0], )), tree),
)
elif isinstance(tree, list):
res = list(map(lambda t: treemap(t, mapfn, None, path), tree))
return xform(key, res, mapfn, path)
def locations(spec):
"""Generator for all locations represented by a spec
Args:
spec (dict): xsize, ysize, pixel_x, pixel_y, ulx, uly keys
Returns:
Generator yielding spec plus x, y, x_index and y_index keys
"""
locator = partial(locate, spec=spec)
indices = ((y, x) for y in range(spec['ysize'])
for x in range(spec['xsize']))
return map(locator, indices)
def map_delayed(func, mps, **kwargs):
func = partial(func, **kwargs)
delayed_func = delayed(func)
return map(delayed_func, mps)
def main(device, epochs, batch_size):
# seed: resultados repetibles
seed = 32
np.random.seed(seed=seed)
random.seed(seed)
# dataget
dataset = data("german-traffic-signs").get()
# data_generator
data_generator = dataset.training_set.random_batch_arrays_generator(
batch_size)
data_generator = utils.batch_random_image_rotation(data_generator, 15.0)
data_generator = cz.map(Dict(features=P[0], labels=P[1]), data_generator)
graph = tf.Graph()
sess = tf.Session(graph=graph)
# inputs
inputs = SupervisedInputs(
name=network_name + "_inputs",
graph=graph,
sess=sess,
# tensors
features=dict(shape=(None, 32, 32, 3)),
labels=dict(shape=(None, ), dtype=tf.uint8))
# create model template
template = Model(
n_classes=43,
name=network_name,
model_path=model_path,
graph=graph,
sess=sess,
seed=seed,
optimizer=tf.train.AdamOptimizer,
)
# model
with tf.device(device):
inputs = inputs()
model = template(inputs)
# initialize variables
model.initialize()
# fit
print("training")
model.fit(
data_generator=data_generator,
epochs=epochs,
log_summaries=True,
log_interval=10,
print_test_info=True,
)
# save
print("saving model")
model.save()
def stage_data(self) -> pd.DataFrame:
"""Stage the NHDPlus Attributes database and save to nhdplus_attrs.feather."""
r = self.get_children(self.nhd_attr_item)
titles = tlz.pluck("title", r["items"])
titles = tlz.concat(
tlz.map(tlz.partial(re.findall, "Select(.*?)Attributes"), titles))
titles = tlz.map(str.strip, titles)
main_items = dict(zip(titles, tlz.pluck("id", r["items"])))
files = {}
soil = main_items.pop("Soil")
for i, item in main_items.items():
r = self.get_children(item)
titles = tlz.pluck("title", r["items"])
titles = tlz.map(
lambda s: s.split(":")[1].strip() if ":" in s else s, titles)
child_items = dict(zip(titles, tlz.pluck("id", r["items"])))
files[i] = {t: self.get_files(c) for t, c in child_items.items()}
r = self.get_children(soil)
titles = tlz.pluck("title", r["items"])
titles = tlz.map(lambda s: s.split(":")[1].strip()
if ":" in s else s, titles)
child_items = dict(zip(titles, tlz.pluck("id", r["items"])))
stat = child_items.pop("STATSGO Soil Characteristics")
ssur = child_items.pop("SSURGO Soil Characteristics")
files["Soil"] = {t: self.get_files(c) for t, c in child_items.items()}
r = self.get_children(stat)
titles = tlz.pluck("title", r["items"])
titles = tlz.map(lambda s: s.split(":")[1].split(",")[1].strip(),
titles)
child_items = dict(zip(titles, tlz.pluck("id", r["items"])))
files["STATSGO"] = {
t: self.get_files(c)
for t, c in child_items.items()
}
r = self.get_children(ssur)
titles = tlz.pluck("title", r["items"])
titles = tlz.map(lambda s: s.split(":")[1].strip(), titles)
child_items = dict(zip(titles, tlz.pluck("id", r["items"])))
files["SSURGO"] = {
t: self.get_files(c)
for t, c in child_items.items()
}
chars = []
types = {"CAT": "local", "TOT": "upstream_acc", "ACC": "div_routing"}
for t, dd in files.items():
for d, fd in dd.items():
for f, u in fd.items():
chars.append({
"name": f,
"type": types.get(f[-3:], "other"),
"theme": t,
"description": d,
"url": u[0],
"meta": u[1],
})
char_df = pd.DataFrame(chars, dtype="category")
char_df.to_feather(self.char_feather)
return char_df
# instead of modifying/using global state, choosing to pass in
# the updated request as a param means that the handler functions
# are all pure functions of their input params.
#
# This should make testing them easier - it's one less thing to mock.
return req_fun(t.merge(opts, r), *args, **kwargs)
return requirejson_wrapper
return reqjson
# converts a dictionary to flat list of key/value pairs.
# each key can have multiple values and they will all be unpacked accordingly.
multipairs=lambda d: list(t.concat(t.map(
lambda i: (lambda k,v: t.concat((k,e) for e in v)
if isinstance(v,list)
else (k,v))(i[0],i[1]),
d.items())))
# --------------------------------------------------------------------------
# REST API
# --------------------------------------------------------------------------
@app.get('/')
def default(message=''):
return template('signin', message=message)
@app.post('/signin')
@params(keys=['barcode'])
def signin(p):
u = filter(lambda v: v.id == p['barcode'], data['users'].values())
if len(u) > 0: