def to_xarray( self, r_dict: Dict[str, bytes], geometry: Union[Polygon, MultiPolygon, None] = None) -> xr.Dataset: """Convert response to xarray.DataArray.""" if isinstance(geometry, (Polygon, MultiPolygon)): gtiff2xarray = tlz.partial(geoutils.gtiff2xarray, geometry=geometry, geo_crs=self.crs) else: gtiff2xarray = tlz.partial(geoutils.gtiff2xarray) try: _ds = gtiff2xarray(r_dict=r_dict) except rio.RasterioIOError as ex: raise ServiceUnavailable(self.wms.url) from ex ds: xr.Dataset = _ds.to_dataset() if isinstance(_ds, xr.DataArray) else _ds ds.attrs = _ds.attrs for lyr in self.layers: name = [n for n in self.units if n in lyr.lower()][-1] lyr_name = f"{name}_{lyr.split('_')[1]}" ds = ds.rename({lyr: lyr_name}) ds[lyr_name].attrs["units"] = self.units[name] ds[lyr_name] = ds[lyr_name].astype(self.types[name]) ds[lyr_name].attrs["nodatavals"] = (self.nodata[name], ) return ds
def pipeline(chip, tx, ty, date, acquired, cfg): ctx = { 'tx': tx, 'ty': ty, 'cx': first(chip), 'cy': second(chip), 'date': date, 'acquired': acquired } return thread_first( ctx, partial(segments, cfg=cfg), segments_filter, partial(segaux.aux, cfg=cfg), segaux.aux_filter, segaux.combine, segaux.unload_segments, segaux.unload_aux, segaux.add_training_dates, add_average_reflectance, segaux.training_format, #segaux.log_chip, segaux.exit_pipeline)
def create_routes(app, redis): route = partial(add_route, app) handler = lambda fn: partial(fn, redis) route('GET', '/api/services', handler(get_services)) route('POST', '/api/services', handler(create_service)) route('GET', '/api/services/{s_id}', handler(get_service)) route('DELETE', '/api/services/{s_id}', handler(delete_service))
def ssebopeta_bygeom( geometry: GTYPE, dates: Union[Tuple[str, str], Union[int, List[int]]], geo_crs: str = DEF_CRS, ) -> xr.DataArray: """Get daily actual ET for a region from SSEBop database. Notes ----- Since there's still no web service available for subsetting SSEBop, the data first needs to be downloaded for the requested period then it is masked by the region of interest locally. Therefore, it's not as fast as other functions and the bottleneck could be the download speed. Parameters ---------- geometry : shapely.geometry.Polygon or tuple The geometry for downloading clipping the data. For a tuple bbox, the order should be (west, south, east, north). dates : tuple or list, optional Start and end dates as a tuple (start, end) or a list of years [2001, 2010, ...]. geo_crs : str, optional The CRS of the input geometry, defaults to epsg:4326. Returns ------- xarray.DataArray Daily actual ET within a geometry in mm/day at 1 km resolution """ f_list = helpers.get_ssebopeta_urls(dates) if isinstance(geometry, (Polygon, MultiPolygon)): gtiff2xarray = tlz.partial(geoutils.gtiff2xarray, geometry=geometry, geo_crs=geo_crs) else: gtiff2xarray = tlz.partial(geoutils.gtiff2xarray) session = RetrySession() with patch("socket.has_ipv6", False): def _ssebop(t: pd.Timestamp, url: str) -> xr.DataArray: resp = session.get(url) zfile = zipfile.ZipFile(io.BytesIO(resp.content)) content = zfile.read(zfile.filelist[0].filename) ds: xr.DataArray = gtiff2xarray(r_dict={"eta": content}) return ds.expand_dims({"time": [t]}) data = xr.merge(_ssebop(t, url) for t, url in f_list) eta: xr.DataArray = data.where( data.eta < data.eta.nodatavals[0]).eta.copy() * 1e-3 eta.attrs.update({ "units": "mm/day", "nodatavals": (np.nan, ), "crs": DEF_CRS, "long_name": "Actual ET" }) return eta
def load_data(ctx, cfg): return assoc( ctx, 'data', thread_first( ctx, partial(segments, cfg=cfg), partial(segaux.aux, cfg=cfg), segaux.combine, segaux.unload_segments, segaux.unload_aux, extract_segments, partial(segaux.prediction_dates, month=get("month", ctx), day=get("day", ctx)), segaux.average_reflectance, reformat))
def test_gaussian_GFE_entropy_gradient(): num_units = 5 lay = layers.GaussianLayer(num_units) lay.params.loc[:] = be.rand_like(lay.params.loc) lay.params.log_var[:] = be.randn(be.shape(lay.params.loc)) from cytoolz import compose sum_square = compose(be.tsum, be.square) for itr in range(10): mag = lay.get_random_magnetization() lms = lay.lagrange_multipliers_analytic(mag) entropy = lay.TAP_entropy(mag) lr = 0.001 gogogo = True grad = lay.TAP_magnetization_grad(mag, [], [], []) grad_mag = math.sqrt(be.float_scalar(be.accumulate(sum_square, grad))) normit = partial(be.tmul_, be.float_scalar(1.0/grad_mag)) be.apply_(normit, grad) rand_grad = lay.get_random_magnetization() grad_mag = math.sqrt(be.float_scalar(be.accumulate(sum_square, rand_grad))) normit = partial(be.tmul_, be.float_scalar(1.0/grad_mag)) be.apply_(normit, rand_grad) while gogogo: cop1_mag = deepcopy(mag) cop1_lms = deepcopy(lms) cop2_mag = deepcopy(mag) cop2_lms = deepcopy(lms) cop1_mag.mean[:] = mag.mean + lr * grad.mean cop2_mag.mean[:] = mag.mean + lr * rand_grad.mean cop1_mag.variance[:] = mag.variance + lr * grad.variance cop2_mag.variance[:] = mag.variance + lr * rand_grad.variance lay.clip_magnetization_(cop1_mag) lay.clip_magnetization_(cop2_mag) cop1_lms = lay.lagrange_multipliers_analytic(cop1_mag) cop2_lms = lay.lagrange_multipliers_analytic(cop2_mag) entropy_1 = lay.TAP_entropy(cop1_mag) entropy_2 = lay.TAP_entropy(cop2_mag) regress = entropy_1 - entropy_2 < 0.0 #print(itr, "[",lr, "] ", entropy, entropy_1, entropy_2, regress) if regress: #print(grad, rand_grad) if lr < 1e-6: assert False,\ "Gaussian GFE magnetization gradient is wrong" break else: lr *= 0.5 else: break
def tf_ngrams_pipeline(input_col: str, output_col: str, df: DataFrame, n=3): """Calculate the term frequency vectors for the character-wise trigrams of an input string""" filled_col = input_col + "_filled" trigrams_col = input_col + "_character_ngrams" return pipe( df, partial(fill_nulls_with_empty_string, input_col, filled_col), partial(character_ngrams, filled_col, trigrams_col, n=n), partial(rm_empty_strings_from_tokens, trigrams_col, trigrams_col), partial(term_frequency_vectors, trigrams_col, output_col), partial(drop_cols, [filled_col, trigrams_col]), )
def tfidf_vectors_pipeline(input_col: str, output_col: str, df: DataFrame, n=3): """Calculate the tfidf vectors for the character-wise trigrams of an input string""" tf_vectors = input_col + "_tf_vectors" tfidf_col = input_col + "_tfifd_vectors" return pipe( tf_ngrams_pipeline(input_col, tf_vectors, df), partial(tfidf_vectors, tf_vectors, tfidf_col), partial(normalize_vectors, tfidf_col, output_col), partial(drop_cols, [tf_vectors, tfidf_col]), )
def normalize_compilation_result(compilation_result): """ Take the result from the --standard-json compilation and flatten it into an iterable of contract data dictionaries. """ for source_path, file_contracts in compilation_result['contracts'].items(): for contract_name, raw_contract_data in file_contracts.items(): contract_data = normalize_standard_json_contract_data( raw_contract_data) yield pipe( contract_data, partial(assoc, key='source_path', value=source_path), partial(assoc, key='name', value=contract_name), )
def create(x, y, acquired, cfg): """Create a timeseries. Args: x (int): x coordinate y (int): y coordinate acquired (string): iso8601 date range cfg (dict): A Merlin configuration Returns: tuple - Results of format_fn applied to results of chips_fn """ x, y = get_in(['chip', 'proj-pt'], cfg['snap_fn'](x=x, y=y)) # get specs specmap = cfg['specs_fn'](specs=cfg['registry_fn']()) # get function that will return chipmap. # Don't create state with a realized variable to preserve memory chipmap = partial(chips.mapped, x=x, y=y, acquired=acquired, specmap=specmap, chips_fn=cfg['chips_fn']) # calculate locations chip. There's another function # here to be split out and organized. grid = first(filter(lambda x: x['name'] == 'chip', cfg['grid_fn']())) cw, ch = specs.refspec(specmap).get('data_shape') locations = partial(chips.locations, x=x, y=y, cw=cw, ch=ch, rx=grid.get('rx'), ry=grid.get('ry'), sx=grid.get('sx'), sy=grid.get('sy')) return cfg['format_fn'](x=x, y=y, locations=locations(), dates_fn=cfg['dates_fn'], specmap=specmap, chipmap=chipmap())
def map_abi_data(normalizers, types, data): ''' This function will apply normalizers to your data, in the context of the relevant types. Each normalizer is in the format: def normalizer(datatype, data): # Conditionally modify data return (datatype, data) Where datatype is a valid ABI type string, like "uint". In case of an array, like "bool[2]", normalizer will receive `data` as an iterable of typed data, like `[("bool", True), ("bool", False)]`. Internals --- This is accomplished by: 1. Decorating the data tree with types 2. Recursively mapping each of the normalizers to the data 3. Stripping the types back out of the tree ''' pipeline = itertools.chain( [abi_data_tree(types)], map(data_tree_map, normalizers), [partial(recursive_map, strip_abi_type)], ) return pipe(data, *pipeline)
def paths_ratio(graph, links_map, ppp, pool_size=4): p = Pool(pool_size) pmap = p.map pairs = [(n1, n2) for n1 in graph.nodes() for n2 in graph.nodes() if n1 != n2] fn = partial(_all_simple_paths_dfs, graph, links_map, ppp) simple_paths = merge(pmap(fn, pairs)) all_paths = 0.0 ok_paths = 0.0 count = 0.0 summ = 0.0 averages = [] for path_key, path_tuples in simple_paths.iteritems(): count += 1 ok_paths += len(path_tuples[0]) all_paths += path_tuples[3] summ += len(path_tuples[0]) / float(path_tuples[3]) print "--", path_key, len(path_tuples[0]) / float(path_tuples[3]) averages.append(len(path_tuples[0]) / float(path_tuples[3])) print ">>>>>>>>", ok_paths / all_paths print "Average of average", summ / count print "MEAN", statistics.mean(averages) print "variance", statistics.variance(averages) p.close() return simple_paths
def chexists(dictionary, keys, check_fn): """applies check_fn against dictionary minus keys then ensures the items returned from check_fn exist in dictionary[keys] Args: dictionary (dict): {key: [v1, v3, v2]} keys (sequence): A sequence of keys in dictionary check_fn (function): Function that accepts dict and returns sequence of items or Exception Returns: A sequence of items that are returned from check_fn and exist in dictionary[keys] or Exception """ def exists_in(superset, subset): if issubset(subset, second(superset)): return True else: msg = '{} is missing data.'.format(first(superset)) msg2 = '{} is not a subset of {}'.format(subset, second(superset)) raise Exception('\n\n'.join([msg, msg2])) popped = {k: dictionary[k] for k in keys} checked = check_fn({k: dictionary[k] for k in difference(dictionary, keys)}) all(map(partial(exists_in, subset=checked), popped.items())) return checked
def map_abi_data(normalizers, types, data): ''' This function will apply normalizers to your data, in the context of the relevant types. Each normalizer is in the format: def normalizer(datatype, data): # Conditionally modify data return (datatype, data) Where datatype is a valid ABI type string, like "uint". In case of an array, like "bool[2]", normalizer will receive `data` as an iterable of typed data, like `[("bool", True), ("bool", False)]`. Internals --- This is accomplished by: 1. Decorating the data tree with types 2. Recursively mapping each of the normalizers to the data 3. Stripping the types back out of the tree ''' pipeline = itertools.chain( [abi_data_tree(types)], map(data_tree_map, normalizers), [partial(recursive_map, strip_abi_type)], ) return pipe(data, *pipeline)
def update(self, model, v_data, v_model, epoch): """ Update the model parameters with a gradient step. Notes: Changes parameters of model in place. Args: model: a Model object to optimize v_data (tensor): observations v_mdoel (tensor): samples from the model epoch (int): the current epoch Returns: None """ self.scheduler.increment(epoch) lr_ = partial(be.tmul_, be.float_scalar(self.scheduler.get_lr() * self.stepsize)) grad = model.gradient(v_data, v_model) self.memory.update(grad) self.delta = self.memory.normalize(self.memory.mean_gradient, unbiased=True) hidden.grad_apply_(lr_, self.delta) model.parameter_update(self.delta)
def commodity_in_sphere(center_system, commodity, radius=10): systems = systems_in_sphere(center_system, radius) system_names = [system["name"] for system in systems] this_commodity_from_system = partial(commodity_from_system, commodity=commodity) with ThreadPoolExecutor(max_workers=16) as exe: commodities_batched = exe.map(this_commodity_from_system, system_names) return list(itertools.chain.from_iterable(commodities_batched))
def generate_data(n, k, d, std_noise): X = np.random.normal(size=(n, d)) betas = np.random.normal(size=(d, k)) gate_betas = np.random.normal(size=(d, k)) noise = np.random.normal(size=n) * std_noise true_model = tz.partial(predict_noiseless, betas=betas, gate_betas=gate_betas) y = true_model(X) + noise return X, y, true_model
def token_vectors_pipeline(input_col: str, output_col: str, df: DataFrame, stemmer_func=None): """Convert a string into an array of integer token ids""" filled_col = input_col + "_filled" tokenised_col = input_col + "_tokenised" tf_vectors = input_col + "_tf_vectors" transforms = [ # note that the tokenizer completely breaks given null input values partial(fill_nulls_with_empty_string, input_col, filled_col), partial(tokenize_words, filled_col, tokenised_col), ] # optionally stem the tokens if stemmer_func: transforms += [partial(stemmer_func, tokenised_col, tokenised_col)] transforms += [ partial(rm_empty_strings_from_tokens, tokenised_col, tokenised_col), partial(term_frequency_vectors, tokenised_col, tf_vectors), partial(sparse_vector_indices, tf_vectors, output_col), partial(drop_cols, [filled_col, tokenised_col, tf_vectors]), ] return pipe(df, *transforms)
def hash_key(args, kwargs): # return (args, hash(frozenset(kwargs.items()))) # return (map(make_hashable, args), frozenset(kwargs.items())) args = tuple(map(make_hashable, args)) kwargs = frozenset( map(compose(tuple, partial(map, make_hashable)), kwargs.items())) # print('args', args) # print('kwargs', kwargs) return (args, kwargs)
def segments(): return thread_first(request.json, partial(exception_handler, http_status=500, name='log_request', fn=log_request), partial(exception_handler, http_status=400, name='parameters', fn=parameters), partial(exception_handler, http_status=500, name='timeseries', fn=partial(timeseries, cfg=cfg)), partial(exception_handler, http_status=500, name='nodata', fn=partial(nodata, cfg=cfg)), partial(exception_handler, http_status=500, name='detection', fn=partial(detection, cfg=cfg)), partial(exception_handler, http_status=500, name='delete', fn=partial(delete, cfg=cfg)), partial(exception_handler, http_status=500, name='save', fn=partial(save, cfg=cfg)), respond)
def tags_at(run: int, *other_runs: int, beamline: int = None) -> Tuple[int, Sequence[int]]: """ Example: hightag, tags = tags_at(509700, beamline=3) # from single run hightag, tags = tags_at(509700, 509701, 509702, beamline=3) # from multiple runs """ if beamline is None: raise ValueError("Keyword argument 'beamline' must be given!") runs = run, *other_runs hightag_at_the_beamline = partial(hightag, beamline) taglist_at_the_beamline = partial(taglist, beamline) hightags: ndarray = pipe(runs, partial(map, hightag_at_the_beamline), partial(fromiter, dtype='int')) if not (hightags == hightags[0]).all(): raise ValueError('Not all the runs have a single hightag!') tags = pipe(runs, partial(map, taglist_at_the_beamline), concat, tuple) return hightags[0], tags
def find_background_illumination(fns, radius=None, input_bitdepth=None, quantile=0.5, stretch_quantile=0.): """Use a set of related images to find uneven background illumination. Parameters ---------- fns : list of string A list of image file names radius : int, optional The radius of the structuring element used to find background. default: The width or height of the input images divided by 4, whichever is smaller. input_bitdepth : int, optional The bit-depth of the input images. Should be specified if non-standard bitdepth images are used in a 16-bit image file, e.g. 12-bit images. Default is the dtype of the input image. quantile : float in [0, 1], optional The desired quantile to find background. default: 0.5 (median) stretch_quantile : float in [0, 1], optional Stretch image to full dtype limit, saturating above this quantile. Returns ------- illum : np.ndarray, float, shape (M, N) The estimated illumination over the image field. See Also -------- `correct_image_illumination`, `correct_multiimage_illumination`. """ # this function follows the "PyToolz" streaming data model to # obtain the illumination estimate. # first, define the functions for each individual step: in_range = ('image' if input_bitdepth is None else (0, 2**input_bitdepth - 1)) rescale = tz.curry(exposure.rescale_intensity) normalize = (tz.partial(stretchlim, bottom=stretch_quantile) if stretch_quantile > 0 else skimage.img_as_float) # produce a stream of properly-scaled images ims = (tz.pipe(fn, io.imread, rescale(in_range=in_range), normalize) for fn in fns) # take the mean of that stream mean_image = mean(ims) # return the median filter of that mean radius = radius or min(mean_image.shape) // 4 illum = ndi.percentile_filter(mean_image, percentile=(quantile * 100), footprint=morphology.disk(radius)) return illum
def pyccd(x, y, locations, dates_fn, specmap, chipmap): """Builds inputs for the pyccd algorithm. Args: x: x projection coordinate of chip y: y projection coordinate of chip locations: chip shaped 2d array of projection coordinates dates_fn (fn): returns dates that should be included in time series specmap (dict): mapping of keys to specs chipmap (dict): mapping of keys to chips Returns: A tuple of tuples. The pyccd format key is ```(chip_x, chip_y, x, y)``` with a dictionary of sorted numpy arrays representing each spectra plus an additional sorted dates array. >>> pyccd_format(*args) (((chip_x, chip_y, x1, y1), {"dates": [], "reds": [], "greens": [], "blues": [], "nirs1": [], "swir1s": [], "swir2s": [], "thermals": [], "qas": []}), ((chip_x, chip_y, x1, y2), {"dates": [], "reds": [], "greens": [], "blues": [], "nirs1": [], "swir1s": [], "swir2s": [], "thermals": [], "qas": []})) ... """ _index = specs.index(list(functions.flatten(specmap.values()))) _dates = dates_fn(datemap=dates.mapped(chipmap)) _creator = partial(rods.create, x=x, y=y, dateseq=_dates, locations=locations, spec_index=_index) _flipped = partial(functions.flip_keys, {k: _creator(chipseq=v) for k, v in chipmap.items()}) _rods = functions.insert_into_every(key='dates', value=list(map(dates.to_ordinal, dates.rsort(_dates))), dods=_flipped()) return tuple((k, v) for k, v in _rods.items())
def grad_normalize_(grad): """ Normalize the gradient vector with respect to the L2 norm Args: grad (Gradient) Return: None """ nrm = grad_norm(grad) grad_apply_(partial(be.tmul_, be.float_scalar(1.0/nrm)), grad)
def find_background_illumination(fns, radius=None, input_bitdepth=None, quantile=0.5, stretch_quantile=0.): """Use a set of related images to find uneven background illumination. Parameters ---------- fns : list of string A list of image file names radius : int, optional The radius of the structuring element used to find background. default: The width or height of the input images divided by 4, whichever is smaller. input_bitdepth : int, optional The bit-depth of the input images. Should be specified if non-standard bitdepth images are used in a 16-bit image file, e.g. 12-bit images. Default is the dtype of the input image. quantile : float in [0, 1], optional The desired quantile to find background. default: 0.5 (median) stretch_quantile : float in [0, 1], optional Stretch image to full dtype limit, saturating above this quantile. Returns ------- illum : np.ndarray, float, shape (M, N) The estimated illumination over the image field. See Also -------- `correct_image_illumination`, `correct_multiimage_illumination`. """ # this function follows the "PyToolz" streaming data model to # obtain the illumination estimate. # first, define the functions for each individual step: in_range = ('image' if input_bitdepth is None else (0, 2**input_bitdepth - 1)) rescale = tz.curry(exposure.rescale_intensity) normalize = (tz.partial(stretchlim, bottom=stretch_quantile) if stretch_quantile > 0 else skimage.img_as_float) # produce a stream of properly-scaled images ims = (tz.pipe(fn, io.imread, rescale(in_range=in_range), normalize) for fn in fns) # take the mean of that stream mean_image = mean(ims) # return the median filter of that mean radius = radius or min(mean_image.shape) // 4 mean_image = img_as_ubyte(stretchlim(mean_image)) illum = imfilter.rank.median(mean_image, selem=morphology.disk(radius)) return illum
def data(ctx, cfg): '''Retrieve training data for all chips in parallel''' p = partial(pipeline, tx=ctx['tx'], ty=ctx['ty'], date=ctx['date'], acquired=ctx['acquired'], cfg=cfg) with workers(cfg) as w: return assoc(ctx, 'data', numpy.array(list(flatten(w.map(p, ctx['chips']))), dtype=numpy.float32))
def create_routes(app): route = partial(add_route, app) route('GET', '/', handlers.index) route('GET', '/api/recipes', handlers.get_recipes) route('GET', '/api/recipes/{r_id}', handlers.get_recipe) route('POST', '/api/recipes', handlers.create_recipe) route('DELETE', '/api/recipes/{r_id}', handlers.delete_recipe) route('GET', '/api/ingredients', handlers.get_ingredients) route('GET', '/api/ingredients/{keyword}', handlers.get_ingredient) route('POST', '/api/ingredients', handlers.create_ingredient) route('DELETE', '/api/ingredients/{keyword}', handlers.delete_ingredient)
def serializable_unsigned_transaction_from_dict(transaction_dict): assert_valid_fields(transaction_dict) filled_transaction = pipe( transaction_dict, dict, partial(merge, TRANSACTION_DEFAULTS), chain_id_to_v, apply_formatters_to_dict(TRANSACTION_FORMATTERS), ) if 'v' in filled_transaction: serializer = Transaction else: serializer = UnsignedTransaction return serializer.from_dict(filled_transaction)
def tif_to_json(tifpath, outpath, nodata=None, fill=None, partition_size=10000): """Converts TIF files to JSON suitable for reading as DataFrames Args: tifpath (str): Full path to input tif file jsonpath (str): Full path to output json file fill: Fill value. Used to exclude values from the conversion nodata: Nodata value. Used to exclude values which shouldnt be included in the conversion. partition_size (int): How big should each file partition be Returns: str: Full path of output json file """ reader = partial(read, filepath=tifpath) writer = partial(write, filepath=outpath) return pipe(tifpath, genspec, locations, reader, compact, csv, writer)
def locations(spec): """Generator for all locations represented by a spec Args: spec (dict): xsize, ysize, pixel_x, pixel_y, ulx, uly keys Returns: Generator yielding spec plus x, y, x_index and y_index keys """ locator = partial(locate, spec=spec) indices = ((y, x) for y in range(spec['ysize']) for x in range(spec['xsize'])) return map(locator, indices)
def __init__(self, mean_weight=0.9, mean_square_weight=0.0): """ Create a gradient memory object to keep track of the first two moments of the gradient. Args: mean_weight (float \in (0,1); optional): how strongly to weight the previous gradient mean_square_weight (float \in (0,1); optional) how strongly to weight the square of the previous gradient Returns: GradientMemory """ self.mean_weight = be.float_scalar(mean_weight) self.mean_square_weight = be.float_scalar(mean_square_weight) self.mean_gradient = None self.mean_square_gradient = None self.mixer_ = partial(be.mix_, self.mean_weight) self.square_mixer_ = partial(be.square_mix_, self.mean_square_weight)
def test_pyccd(): c = cfg.get('chipmunk-ard', env=test.env) x, y = get_in(['chip', 'proj-pt'], c['snap_fn'](x=test.x, y=test.y)) # get specs specmap = c['specs_fn'](specs=c['registry_fn']()) # get function that will return chipmap. # Don't create state with a realized variable to preserve memory chipmap = partial(chips.mapped, x=test.x, y=test.y, acquired=test.acquired, specmap=specmap, chips_fn=c['chips_fn']) # calculate locations chip. There's another function # here to be split out and organized. grid = first(filter(lambda x: x['name'] == 'chip', c['grid_fn']())) cw, ch = specs.refspec(specmap).get('data_shape') locations = chips.locations(x=x, y=y, cw=cw, ch=ch, rx=grid.get('rx'), ry=grid.get('ry'), sx=grid.get('sx'), sy=grid.get('sy')) data = c['format_fn'](x=x, y=y, locations=locations, dates_fn=c['dates_fn'], specmap=specmap, chipmap=chipmap()) # we are only testing the structure of the response here. # Full data validation is being done in the test for merlin.create() assert type(data) is tuple assert len(data) == 10000 assert type(first(data)) is tuple assert type(first(first(data))) is tuple assert type(second(first(data))) is dict assert type(second(second(first(data)))) is tuple or list assert len(second(second(first(data)))) > 0
def find_background_illumination(fns, radius=51, quantile=0.05, stretch_quantile=0., method='mean'): """Use a set of related images to find uneven background illumination. Parameters ---------- fns : list of string A list of image file names radius : int, optional The radius of the structuring element used to find background. default: 51 quantile : float in [0, 1], optional The desired quantile to find background. default: 0.05 stretch_quantile : float in [0, 1], optional Stretch image to full dtype limit, saturating above this quantile. method : 'mean', 'average', 'median', or 'histogram', optional How to use combine the smoothed intensities of the input images to infer the illumination field: - 'mean' or 'average': Use the mean value of the smoothed images at each pixel as the illumination field. - 'median': use the median value. Since all images need to be in-memory to compute this, use only for small sets of images. - 'histogram': use the median value approximated by a histogram. This can be computed on-line for large sets of images. Returns ------- illum : np.ndarray, float, shape (M, N) The estimated illumination over the image field. See Also -------- ``correct_image_illumination``. """ # This function follows the "PyToolz" streaming data model to # obtain the illumination estimate. First, define each processing # step: read = io.imread normalize = (tlz.partial(stretchlim, bottom=stretch_quantile) if stretch_quantile > 0 else skimage.img_as_float) rescale = rescale_to_11bits pad = fun.partial(skimage.util.pad, pad_width=radius, mode='reflect') rank_filter = fun.partial(rank.percentile, selem=skmorph.disk(radius), p0=quantile) _unpad = fun.partial(unpad, pad_width=radius) unscale = rescale_from_11bits # Next, compose all the steps, apply to all images (streaming) bg = (tlz.pipe(fn, read, normalize, rescale, pad, rank_filter, _unpad, unscale) for fn in fns) # Finally, reduce all the images and compute the estimate if method == 'mean' or method == 'average': illum, count = _reduce_with_count(np.add, bg) illum = skimage.img_as_float(illum) / count elif method == 'median': illum = np.median(list(bg), axis=0) elif method == 'histogram': raise NotImplementedError('histogram background illumination method ' 'not yet implemented.') else: raise ValueError('Method "%s" of background illumination finding ' 'not recognised.' % method) return illum
} filter_params_remapper = apply_key_map(FILTER_PARAMS_MAPPINGS) FILTER_PARAMS_FORMATTERS = { 'fromBlock': to_integer_if_hex, 'toBlock': to_integer_if_hex, } filter_params_formatter = apply_formatters_to_dict(FILTER_PARAMS_FORMATTERS) filter_params_transformer = compose(filter_params_remapper, filter_params_formatter) TRANSACTION_FORMATTERS = { 'to': apply_formatter_if(partial(operator.eq, b''), static_return(None)), } transaction_formatter = apply_formatters_to_dict(TRANSACTION_FORMATTERS) RECEIPT_FORMATTERS = { 'logs': apply_formatter_to_array(log_key_remapper), } receipt_formatter = apply_formatters_to_dict(RECEIPT_FORMATTERS) transaction_params_transformer = compose(transaction_params_remapper, transaction_params_formatter)