Ejemplo n.º 1
0
def _(points, grid, snap_fn):
    """Returns grid coordinates contained within a dict of points.

    Args:
        points  (dict): {'ulx': 0, 'uly': 15.5, 'lrx': 55.3, 'lry':-1000.12}
        grid    (dict): The target grid: {'name': 'chip', 'sx': 3000, 'sy': 3000, 'rx': 1, 'ry': -1}
        snap_fn (func): A function that accepts x, y and returns a snapped x, y

    Returns:
        tuple: tuple of tuples of grid coordinates ((x1,y1), (x2,y1) ...)

    This example assumes a grid size of 500x500 pixels.

    Example:
        >>> coordinates = coordinates({ulx: -1001, uly: 1000, lrx: -500, lry: 500},
                                      grid={'name': 'chip', 'sx': 500, 'sy': 500, 'rx': 1, 'ry': -1}, 
                                      snap_fn=some_func)

        ((-1000, 500), (-500, 500), (-1000, -500), (-500, -500))
    """

    # snap start/end x & y
    start_x, start_y = get_in([grid.get('name'), 'proj-pt'],
                              snap_fn(x=points.get('ulx'),
                                      y=points.get('uly')))
    end_x, end_y = get_in([grid.get('name'), 'proj-pt'],
                          snap_fn(x=points.get('lrx'), y=points.get('lry')))

    # get x and y scale factors multiplied by reflection
    x_interval = grid.get('sx') * grid.get('rx')
    y_interval = grid.get('sy') * grid.get('ry')

    return tuple((x, y)
                 for x in np.arange(start_x, end_x + x_interval, x_interval)
                 for y in np.arange(start_y, end_y + y_interval, y_interval))
Ejemplo n.º 2
0
def block_num_from_jsonrpc_response(jsonrpc_response: dict = None) -> int:
    # pylint: disable=no-member
    # for get_block
    block_id = cytoolz.get_in(['result', 'block_id'], jsonrpc_response)
    if block_id:
        return block_num_from_id(block_id)

    # for get_block_header
    previous = cytoolz.get_in(['result', 'previous'], jsonrpc_response)
    return block_num_from_id(previous) + 1
Ejemplo n.º 3
0
def extract_serialize(x):
    """ Pull out Serialize objects from message

    This also remove large bytestrings from the message into a second
    dictionary.

    Examples
    --------
    >>> from distributed.protocol import to_serialize
    >>> msg = {'op': 'update', 'data': to_serialize(123)}
    >>> extract_serialize(msg)
    ({'op': 'update'}, {('data',): <Serialize: 123>}, set())
    """
    ser = {}
    _extract_serialize(x, ser)
    if ser:
        x = container_copy(x)
        for path in ser:
            t = get_in(path[:-1], x)
            if isinstance(t, dict):
                del t[path[-1]]
            else:
                t[path[-1]] = None

    bytestrings = set()
    for k, v in ser.items():
        if type(v) in (bytes, bytearray):
            ser[k] = to_serialize(v)
            bytestrings.add(k)
    return x, ser, bytestrings
Ejemplo n.º 4
0
async def test_rpc_against_fixtures(chain, ipc_server, chain_fixture, fixture_data):
    rpc = RPCServer(MainnetFullChain(None))

    setup_result, setup_error = await call_rpc(rpc, 'evm_resetToGenesisFixture', [chain_fixture])
    assert setup_error is None and setup_result is True, "cannot load chain for {0}".format(fixture_data)  # noqa: E501

    await validate_accounts(rpc, chain_fixture['pre'])

    for block_fixture in chain_fixture['blocks']:
        should_be_good_block = 'blockHeader' in block_fixture

        if 'rlp_error' in block_fixture:
            assert not should_be_good_block
            continue

        block_result, block_error = await call_rpc(rpc, 'evm_applyBlockFixture', [block_fixture])

        if should_be_good_block:
            assert block_error is None
            assert block_result == block_fixture['rlp']

            await validate_block(rpc, block_fixture, block_fixture['blockHeader']['hash'])
        else:
            assert block_error is not None

    if chain_fixture.get('lastblockhash', None):
        for block_fixture in chain_fixture['blocks']:
            if get_in(['blockHeader', 'hash'], block_fixture) == chain_fixture['lastblockhash']:
                await validate_last_block(rpc, block_fixture)

    await validate_accounts(rpc, chain_fixture['postState'])
    await validate_accounts(rpc, chain_fixture['pre'], 'earliest')
    await validate_accounts(rpc, chain_fixture['pre'], 0)
Ejemplo n.º 5
0
def train(ctx, cfg):
    '''Train an xgboost model'''
    
    itrain, itest, dtrain, dtest = train_test_split(ctx['independent'],
                                                    ctx['dependent'],
                                                    test_size=get_in(['xgboost', 'test_size'], cfg))
    
    train_matrix = xgb.DMatrix(data=itrain, label=dtrain)
    test_matrix  = xgb.DMatrix(data=itest, label=dtest)
    
    return assoc(ctx, 'model', xgb.train(params=get_in(['xgboost', 'parameters'], cfg),
                                         dtrain=train_matrix,
                                         num_boost_round=get_in(['xgboost', 'num_round'], cfg),
                                         evals=watchlist(train_matrix, test_matrix),
                                         early_stopping_rounds=get_in(['xgboost', 'early_stopping_rounds'], cfg),
                                         verbose_eval=get_in(['xgboost', 'verbose_eval'], cfg)))
Ejemplo n.º 6
0
def relevant_markets_near(
    location,
    commodity_filter=None,
    radius=30,
):
    results = []
    if commodity_filter is None:
        return markets_near(location, radius)
    else:
        systems = edsm.systems_in_sphere(location, radius)
        system_names = [system["name"] for system in systems]
        with ThreadPoolExecutor(max_workers=6) as exe:
            all_markets = list(exe.map(edsm.markets_in_system, system_names))
        for (system, markets) in zip(systems, all_markets):
            stations = [market["station"] for market in markets]
            all_commodities = [
                get_in(["market", "commodities"], market, default=[])
                for market in markets
            ]
            relevant_commodities = [
                commodity_filter(market) for market in markets
            ]
            results.extend([{
                "system": system,
                "station": station,
                "commodities": commodities,
                "relevant": relevant,
            } for (station, commodities, relevant
                   ) in zip(stations, all_commodities, relevant_commodities)
                            if relevant])
        return results
Ejemplo n.º 7
0
def test_rpc_against_fixtures(ipc_server, chain_fixture, fixture_data):
    rpc = RPCServer(None)

    setup_result, setup_error = call_rpc(rpc, 'evm_resetToGenesisFixture', [chain_fixture])
    assert setup_error is None and setup_result is True, "cannot load chain for %r" % fixture_data

    validate_accounts(rpc, chain_fixture['pre'])

    for block_fixture in chain_fixture['blocks']:
        should_be_good_block = 'blockHeader' in block_fixture

        if 'rlp_error' in block_fixture:
            assert not should_be_good_block
            continue

        block_result, block_error = call_rpc(rpc, 'evm_applyBlockFixture', [block_fixture])

        if should_be_good_block:
            assert block_error is None
            assert block_result == block_fixture['rlp']

            validate_block(rpc, block_fixture, block_fixture['blockHeader']['hash'])
        else:
            assert block_error is not None

    if chain_fixture.get('lastblockhash', None):
        for block_fixture in chain_fixture['blocks']:
            if get_in(['blockHeader', 'hash'], block_fixture) == chain_fixture['lastblockhash']:
                validate_last_block(rpc, block_fixture)

    validate_accounts(rpc, chain_fixture['postState'])
    validate_accounts(rpc, chain_fixture['pre'], 'earliest')
    validate_accounts(rpc, chain_fixture['pre'], 0)
Ejemplo n.º 8
0
    def fetch_swept_parameters(self, config, sweep_paths,
                               swept_parameter_paths):
        """
        Construct named tuples for all the parameters being swept.
        """
        # Filter sweep paths to all 3 sweep types
        disjoint_sweep_paths = list(
            filter(lambda s: 'disjoint' in last(s), sweep_paths))
        product_sweep_paths = list(
            filter(lambda s: 'product' in last(s), sweep_paths))
        default_sweep_paths = list(
            filter(lambda s: 'default' in last(s), sweep_paths))

        # Construct SweptParameter and Swept__Parameter namedtuples,
        # making it
        # consisting of the path to the parameter and its sweep configuration
        construct_swept_parameters = lambda subtype, paths, wrapper: list(
            map(lambda p: subtype(wrapper(p), tz.get_in(p, coll=config)), paths
                ))

        swept_parameters = construct_swept_parameters(SweptParameter,
                                                      swept_parameter_paths,
                                                      identity)
        swept_disjoint_parameters = construct_swept_parameters(
            SweptDisjointParameter, disjoint_sweep_paths,
            QuinSweep.get_parameter_path)
        swept_product_parameters = construct_swept_parameters(
            SweptProductParameter, product_sweep_paths,
            QuinSweep.get_parameter_path)
        swept_default_parameters = construct_swept_parameters(
            SweptDefaultParameter, default_sweep_paths,
            QuinSweep.get_parameter_path)

        return swept_parameters, swept_disjoint_parameters, swept_product_parameters, swept_default_parameters
Ejemplo n.º 9
0
    def __init__(self, schema=None, **kwargs):
        super(QuinineArgumentParser, self).__init__(**kwargs)

        # Add a default argument for the path to the YAML configuration file that will be passed in
        self.add_argument('--config',
                          type=str,
                          required=True,
                          help="YAML configuration file.")
        self.schema = schema

        if self.schema is not None:
            # Populate the argument parser with arguments from the schema
            paths_to_type = list(
                filter(lambda l: l[-1] == 'type',
                       get_all_leaf_paths(self.schema)))
            type_lookup = dict([(tuple(filter(lambda e: e != 'schema',
                                              e[:-1])), tz.get_in(e, schema))
                                for e in paths_to_type])

            valid_params = self.get_all_params(schema)
            for param in valid_params:
                self.add_argument(f'--{".".join(param)}',
                                  type=self.types[type_lookup[param]])

            self.schema = merge(self.schema, {
                'config': tstring,
                'inherit': tlist
            })
Ejemplo n.º 10
0
def extract_serialize(x):
    """ Pull out Serialize objects from message

    This also remove large bytestrings from the message into a second
    dictionary.

    Examples
    --------
    >>> from distributed.protocol import to_serialize
    >>> msg = {'op': 'update', 'data': to_serialize(123)}
    >>> extract_serialize(msg)
    ({'op': 'update'}, {('data',): <Serialize: 123>}, set())
    """
    ser = {}
    _extract_serialize(x, ser)
    if ser:
        x = container_copy(x)
        for path in ser:
            t = get_in(path[:-1], x)
            if isinstance(t, dict):
                del t[path[-1]]
            else:
                t[path[-1]] = None

    bytestrings = set()
    for k, v in ser.items():
        if type(v) is bytes:
            ser[k] = to_serialize(v)
            bytestrings.add(k)
    return x, ser, bytestrings
Ejemplo n.º 11
0
def cascading_lookup(paths, data):
    NOT_FOUND = object()
    for path in paths:
        result = get_in(path, data, default=NOT_FOUND)
        if result is not NOT_FOUND:
            return result
    else:
        return None
Ejemplo n.º 12
0
    def fetch_tape(self, path: List[str]) -> InteractionTape:
        """Fetch an InteractionTape.

        Args:
            path:

        Returns:
        """
        return tz.get_in(path, self.interactions)
def train(ctx, cfg):
    '''Train an xgboost model'''

    logger.info("training model")

    itrain, itest, dtrain, dtest = train_test_split(
        ctx['independent'],
        ctx['dependent'],
        test_size=get_in(['xgboost', 'test_size'], cfg))

    train_matrix = xgb.DMatrix(data=itrain, label=dtrain)
    test_matrix = xgb.DMatrix(data=itest, label=dtest)
    watch_list = watchlist(train_matrix, test_matrix)

    model = xgb.train(params=get_in(['xgboost', 'parameters'], cfg),
                      dtrain=train_matrix,
                      num_boost_round=get_in(['xgboost', 'num_round'], cfg),
                      evals=watch_list,
                      early_stopping_rounds=get_in(
                          ['xgboost', 'early_stopping_rounds'], cfg),
                      verbose_eval=get_in(['xgboost', 'verbose_eval'], cfg))

    ctx['independent'] = None
    ctx['dependent'] = None
    itrain = None
    itest = None
    dtrain = None
    dtest = None
    train_matrix = None
    test_matrix = None
    watch_list = None
    del ctx['independent']
    del ctx['dependent']
    del itrain
    del itest
    del dtrain
    del dtest
    del train_matrix
    del test_matrix
    del watch_list

    return assoc(ctx, 'model', model)
Ejemplo n.º 14
0
def test_get_in_doctest():
    # Original doctest:
    #     >>> get_in(['y'], {}, no_default=True)
    #     Traceback (most recent call last):
    #         ...
    #     KeyError: 'y'

    # cytoolz result:
    #     KeyError:

    raises(KeyError, lambda: cytoolz.get_in(['y'], {}, no_default=True))
Ejemplo n.º 15
0
    def parse_quinfig(self):
        # Parse all the arguments from the command line, overriding defaults in the argparse
        args = self.parse_args()

        cli_keys = []
        for cli_arg in _sys.argv[1:]:
            if cli_arg.startswith('--'):
                if str(cli_arg) == '--config':
                    continue
                cli_keys.append(cli_arg[2:].replace("-", "_"))
            elif cli_arg.startswith('-'):
                raise NotImplementedError(
                    "QuinineArgumentParser doesn't support abbreviated arguments."
                )
            else:
                continue

        # Get parameters which need to be overridden from command line
        override_args = project(args.__dict__, cli_keys)

        # Trick: first load the config without a schema as a base config
        # quinfig = Quinfig(config_path=args.config)

        # Override all the defaults using the yaml config
        # quinfig = rmerge(Quinfig(config=args.__dict__), quinfig)
        # print(quinfig)

        # Use all the defaults in args to populate a dictionary
        quinfig = {}
        for param, val in args.__dict__.items():
            param_path = param.split(".")
            quinfig = tz.assoc_in(quinfig, param_path, val)

        # Override all the defaults using the yaml config
        quinfig = rmerge(quinfig, Quinfig(config_path=args.config))

        # Replace all the arguments passed into command line
        if len(override_args) > 0:
            print(
                f"\n(quinine) Overriding parameters in {args.config} from command line (___ is unspecified)."
            )

        for param, val in override_args.items():
            param_path = param.split(".")
            old_val = tz.get_in(param_path, quinfig)

            if old_val != val:
                print(f"> ({param}): {old_val} --> {val}")
            else:
                print(f"> ({param}): ___ --> {val}")
            quinfig = tz.assoc_in(quinfig, param_path, val)

        # Load the config again
        return Quinfig(config=quinfig, schema=self.schema)
Ejemplo n.º 16
0
def test_get_in_doctest():
    # Original doctest:
    #     >>> get_in(['y'], {}, no_default=True)
    #     Traceback (most recent call last):
    #         ...
    #     KeyError: 'y'

    # cytoolz result:
    #     KeyError:

    raises(KeyError, lambda: cytoolz.get_in(['y'], {}, no_default=True))
Ejemplo n.º 17
0
def filter_markets(
    markets,
    desired_commodities,
    min_price=1,
    min_demand=1,
    max_update_seconds=24 * 3600,
):
    result = []
    if not markets:
        return []

    for market in markets:
        commodities1 = (get_in(["market", "commodities"], market, default=None)
                        or [])

        commodities = [
            c for c in commodities1 if c["name"] in desired_commodities
        ]

        price_demand_ok = any(
            (c["name"] in desired_commodities and c["sellPrice"] >= min_price
             and c["demand"] >= min_demand) for c in commodities)
        if not price_demand_ok:
            continue

        market_update = get_in(
            ["station", "updateTime", "market"],
            market,
            default=None,
        )
        if market_update is None:
            continue
        delta = time_since(market_update)
        update_ok = delta.total_seconds() < max_update_seconds
        if not update_ok:
            continue

        # otherwise
        result.append(market)

    return result
Ejemplo n.º 18
0
 def _filter_sell_commodity(market):
     commodities = get_in(
         ["market", "commodities"],
         market,
         default=None,
     )
     commodities = commodities or []
     return next(
         (c for c in commodities if c["name"].lower() == name.lower()
          and c["demand"] >= min_demand and c["sellPrice"] >= min_price),
         None,
     )
Ejemplo n.º 19
0
    def get_bygeom(self, geometry: GTYPE, geo_crs: str) -> gpd.GeoDataFrame:
        """Retrieve NID data within a geometry.

        Parameters
        ----------
        geometry : Polygon, MultiPolygon, or tuple of length 4
            Geometry or bounding box (west, south, east, north) for extracting the data.
        geo_crs : list of str
            The CRS of the input geometry, defaults to epsg:4326.

        Returns
        -------
        geopandas.GeoDataFrame
            GeoDataFrame of NID data

        Examples
        --------
        >>> from pygeohydro import NID
        >>> nid = NID()
        >>> dams = nid.get_bygeom((-69.77, 45.07, -69.31, 45.45), "epsg:4326")
        >>> print(dams.name.iloc[0])
        Little Moose
        """
        _geometry = geoutils.geo2polygon(geometry, geo_crs, DEF_CRS)
        wbd = ArcGISRESTful(
            ServiceURL().restful.wbd,
            4,
            outformat="json",
            outfields="huc8",
            expire_after=self.expire_after,
            disable_caching=self.disable_caching,
        )
        resp = wbd.get_features(wbd.oids_bygeom(_geometry), return_geom=False)
        huc_ids = [
            tlz.get_in(["attributes", "huc8"], i) for r in resp
            for i in tlz.get_in(["features"], r)
        ]

        dams = self.get_byfilter([{"huc8": huc_ids}])[0]
        return dams[dams.within(_geometry)].copy()
Ejemplo n.º 20
0
def combine(ctx):
    '''Combine segments with matching aux entry'''

    data = []

    for s in ctx['segments']:

        key = (s.cx, s.cy, s.px, s.py)
        a = get_in(['aux', key], ctx, None)

        if a is not None:
            data.append(merge(a, s._asdict()))

    return assoc(ctx, 'data', data)
Ejemplo n.º 21
0
def combine(ctx):
    '''Combine segments with matching aux entry'''

    data = []

    for s in ctx['segments']:

        key = (s['cx'], s['cy'], s['px'], s['py'])
        a = get_in(['aux', key], ctx, None)

        if a is not None:
            data.append(merge(a, s))

    return assoc(ctx, 'data', data)
Ejemplo n.º 22
0
    def network_id(self) -> int:
        if self._network_id is not None:
            return self._network_id

        try:
            return get_in(
                [self.chain_identifier, 'network_id'],
                CHAIN_CONFIG_DEFAULTS,
                no_default=True,
            )
        except KeyError:
            raise ValueError(
                "The network_id for the chain '{0}' was not explicitely set and "
                "is not for a known network.  Please specify a network_id")
Ejemplo n.º 23
0
def create(x, y, acquired, cfg):
    """Create a timeseries.

    Args:
        x (int): x coordinate
        y (int): y coordinate
        acquired (string): iso8601 date range
        cfg (dict): A Merlin configuration

    Returns:
        tuple - Results of format_fn applied to results of chips_fn
    """

    x, y = get_in(['chip', 'proj-pt'], cfg['snap_fn'](x=x, y=y))

    # get specs
    specmap = cfg['specs_fn'](specs=cfg['registry_fn']())

    # get function that will return chipmap.
    # Don't create state with a realized variable to preserve memory
    chipmap = partial(chips.mapped,
                      x=x,
                      y=y,
                      acquired=acquired,
                      specmap=specmap,
                      chips_fn=cfg['chips_fn'])

    # calculate locations chip.  There's another function
    # here to be split out and organized.

    grid = first(filter(lambda x: x['name'] == 'chip', cfg['grid_fn']()))

    cw, ch = specs.refspec(specmap).get('data_shape')

    locations = partial(chips.locations,
                        x=x,
                        y=y,
                        cw=cw,
                        ch=ch,
                        rx=grid.get('rx'),
                        ry=grid.get('ry'),
                        sx=grid.get('sx'),
                        sy=grid.get('sy'))

    return cfg['format_fn'](x=x,
                            y=y,
                            locations=locations(),
                            dates_fn=cfg['dates_fn'],
                            specmap=specmap,
                            chipmap=chipmap())
Ejemplo n.º 24
0
def test_pyccd():
    c = cfg.get('chipmunk-ard', env=test.env)

    x, y = get_in(['chip', 'proj-pt'], c['snap_fn'](x=test.x, y=test.y))

    # get specs
    specmap = c['specs_fn'](specs=c['registry_fn']())

    # get function that will return chipmap.
    # Don't create state with a realized variable to preserve memory
    chipmap = partial(chips.mapped,
                      x=test.x,
                      y=test.y,
                      acquired=test.acquired,
                      specmap=specmap,
                      chips_fn=c['chips_fn'])

    # calculate locations chip.  There's another function
    # here to be split out and organized.

    grid = first(filter(lambda x: x['name'] == 'chip', c['grid_fn']()))

    cw, ch = specs.refspec(specmap).get('data_shape')

    locations = chips.locations(x=x,
                                y=y,
                                cw=cw,
                                ch=ch,
                                rx=grid.get('rx'),
                                ry=grid.get('ry'),
                                sx=grid.get('sx'),
                                sy=grid.get('sy'))

    data = c['format_fn'](x=x,
                          y=y,
                          locations=locations,
                          dates_fn=c['dates_fn'],
                          specmap=specmap,
                          chipmap=chipmap())

    # we are only testing the structure of the response here.
    # Full data validation is being done in the test for merlin.create()
    assert type(data) is tuple
    assert len(data) == 10000
    assert type(first(data)) is tuple
    assert type(first(first(data))) is tuple
    assert type(second(first(data))) is dict
    assert type(second(second(first(data)))) is tuple or list
    assert len(second(second(first(data)))) > 0
Ejemplo n.º 25
0
def hypothetical_sale(commodities, market):
    mkt_commodities = get_in(["market", "commodities"], market, default=[])
    mkt = {c["name"]: c for c in mkt_commodities}
    matches = [{
        "name": name,
        "sellPrice": mkt[name]["sellPrice"],
        "revenue": mkt[name]["sellPrice"] * quantity,
    } for (name, quantity) in commodities.items() if name in mkt]
    return {
        "total":
        sum(m["revenue"] for m in matches),
        "matched":
        matches,
        "missing":
        [name for (name, quantity) in commodities.items() if name not in mkt],
    }
Ejemplo n.º 26
0
def loads(frames, deserialize=True):
    """ Transform bytestream back into Python value """
    frames = frames[::-1]  # reverse order to improve pop efficiency
    if not isinstance(frames, list):
        frames = list(frames)
    try:
        small_header = frames.pop()
        small_payload = frames.pop()
        msg = loads_msgpack(small_header, small_payload)
        if not frames:
            return msg

        header = frames.pop()
        header = msgpack.loads(header, encoding='utf8', use_list=False)
        keys = header['keys']
        headers = header['headers']
        bytestrings = set(header['bytestrings'])

        for key in keys:
            head = headers[key]
            count = head['count']
            if count:
                fs = frames[-count::][::-1]
                del frames[-count:]
            else:
                fs = []

            if deserialize or key in bytestrings:
                if 'compression' in head:
                    fs = decompress(head, fs)
                fs = merge_frames(head, fs)
                value = _deserialize(head, fs)
            else:
                value = Serialized(head, fs)

            get_in(key[:-1], msg)[key[-1]] = value

        return msg
    except Exception:
        logger.critical("Failed to deserialize", exc_info=True)
        raise
Ejemplo n.º 27
0
    def parse_quinfig(self):
        # Parse all the arguments
        args = self.parse_args()
        override_args = dict(
            select_values(lambda v: v is not None,
                          omit(args.__dict__, ['config'])))

        # Trick: first load the config without a schema
        quinfig = Quinfig(config_path=args.config)

        # Replace all the arguments passed into command line
        if len(override_args) > 0:
            print(f"Overriding arguments in {args.config} from command line.")
        for param, val in override_args.items():
            param_path = param.split(".")
            old_val = tz.get_in(param_path, quinfig)
            print(f"> ({param}): {old_val} --> {val}")
            quinfig = tz.assoc_in(quinfig, param_path, val)

        # Load the config again, this time with the schema
        return Quinfig(config=quinfig.__dict__, schema=self.schema)
Ejemplo n.º 28
0
def loads(frames, deserialize=True):
    """ Transform bytestream back into Python value """
    frames = frames[::-1]  # reverse order to improve pop efficiency
    if not isinstance(frames, list):
        frames = list(frames)
    try:
        small_header = frames.pop()
        small_payload = frames.pop()
        msg = loads_msgpack(small_header, small_payload)
        if not frames:
            return msg

        header = frames.pop()
        header = msgpack.loads(header, encoding='utf8', use_list=False)
        keys = header['keys']
        headers = header['headers']
        bytestrings = set(header['bytestrings'])

        for key in keys:
            head = headers[key]
            lengths = head['lengths']
            count = head['count']
            fs = frames[-count::][::-1]
            del frames[-count:]

            if deserialize or key in bytestrings:
                fs = decompress(head, fs)
                fs = merge_frames(head, fs)
                value = _deserialize(head, fs)
            else:
                value = Serialized(head, fs)

            get_in(key[:-1], msg)[key[-1]] = value

        return msg
    except Exception as e:
        logger.critical("Failed to deserialize", exc_info=True)
        raise
Ejemplo n.º 29
0
def difference(*colls):
    """
    Find the keys that have different values in an arbitrary number of (nested) collections. Any key
    that differs in at least 2 collections is considered to fit this criterion.
    """

    # Get all the leaf paths for each collection: make each path a tuple
    leaf_paths_by_coll = list(
        map(lambda c: list(map(tuple, get_all_leaf_paths(c))), colls))

    # Find the union of all leaf paths: merge all the paths and keep only the unique paths
    union_leaf_paths = list(distinct(concat(*leaf_paths_by_coll)))

    # Get the values corresponding to these leaf paths in every collection: if a leaf path doesn't exist, assumes None
    values_by_coll = list(
        map(lambda lp: list(map(lambda coll: tz.get_in(lp, coll), colls)),
            union_leaf_paths))

    # Filter out the leaf paths that have identical values across the collections
    keep_leaf_paths = list(
        map(
            0,
            filter(lambda t: not allequal(t[1]),
                   zip(union_leaf_paths, values_by_coll))))
    keep_values = list(
        map(
            1,
            filter(lambda t: not allequal(t[1]),
                   zip(union_leaf_paths, values_by_coll))))

    # Rearrange to construct a list of dictionaries -- one per original collection.
    # Each of these dictionaries maps a 'kept' leaf path to its corresponding
    # value in the collection
    differences = list(
        map(lambda vals: dict(zip(keep_leaf_paths, vals)),
            list(zip(*keep_values))))

    return differences
def booster(cfg, model_bytes):
    return segaux.booster_from_bytes(
        model_bytes,
        {'nthread': get_in(['xgboost', 'parameters', 'nthread'], cfg)})
Ejemplo n.º 31
0
    def replace_underscores(self, swept_parameter):
        """
        Replace all the underscore references in sweep of swept_parameter.
        """

        # Find all the references (i.e. dependencies) made by the swept_parameter
        references = []
        for token in QuinSweep.SWEEP_TOKENS[:
                                            -1]:  # omit default since it's value is never a dict
            if f"~{token}" in swept_parameter.sweep and is_mapping(
                    swept_parameter.sweep[f"~{token}"]):
                references.extend(
                    list(swept_parameter.sweep[f"~{token}"].keys()))

        # Find all the referred parameters
        parsed_references = list(map(QuinSweep.parse_ref_dotpath, references))
        dotpaths = list(cat(parsed_references))
        ref_dict = merge_with(compose(list, cat),
                              *list(map(lambda e: dict([e]), dotpaths)))

        # TODO: there's a bug here potentially
        assert all(map(lambda l: len(l) == len(set(l)), list(itervalues(ref_dict)))), \
            'All conditions must be distinct.'

        ref_dict_no_underscores = walk_values(
            compose(set,
                    autocurry(map)(int),
                    autocurry(filter)(lambda e: e != '_')), ref_dict)

        if not references:
            return swept_parameter

        def compute_possibilities(full_dotpath, reference):
            # Look up the parameter using the dotpath
            parameter = self.swept_parameters_dict[full_dotpath]

            # Use the reference to figure out how many possiblities exist for the underscore
            if len(reference) > 0:
                # Merge all the sweeps performed for this parameter
                merged_sweep = merge(
                    *list(filter(is_mapping, itervalues(parameter.sweep))))
                # Look up the reference
                return len(merged_sweep[reference])

            assert len(
                parameter.sweep
            ) == 1, 'If no reference, must be a single unconditional sweep.'
            # The number of possibilities is simply the number of values specified
            # in the (product/disjoint) unconditional sweep
            return len(list(parameter.sweep.values())[0])

        # Update the sweep by replacing underscores
        updated_sweep = swept_parameter.sweep

        # Loop over all the parsed references
        for parsed_ref in parsed_references:

            # Expand all the partial dotpaths
            # TODO: remove? expanding all the partial dotpaths in the beginning?
            parsed_ref = list(
                map(lambda t: (self.expand_partial_dotpath(t[0]), t[1]),
                    parsed_ref))

            # For each parsed reference, there will be multiple (dotpath, idx) pairs
            for i, (full_dotpath, ref_idx) in enumerate(parsed_ref):

                # If the reference index is not an underscore, continue
                if not ref_idx == '_':
                    continue

                # Compute the prefix reference
                prefix_reference = ".".join(list(cat(parsed_ref[:i])))

                # Compute the number of possible ways to replace the underscore
                n_possibilities = compute_possibilities(
                    full_dotpath, prefix_reference)
                replacements = set(range(
                    n_possibilities)) - ref_dict_no_underscores[full_dotpath]

                # Find the path to the underscore condition
                path_to_condition = get_only_paths(
                    updated_sweep,
                    lambda p: any(lambda e: '_' in e, p),
                    stop_at=full_dotpath)[0]

                # Find the value of the underscore condition
                value = tz.get_in(path_to_condition, updated_sweep)

                # Construct keys that are subtitutes for the underscore
                keys = list(map(lambda s: f'{full_dotpath}.{s}', replacements))
                keys = list(map(lambda k: path_to_condition[:-1] + [k], keys))

                # Update by adding those keys in
                for k in keys:
                    updated_sweep = tz.assoc_in(updated_sweep, k, value)

        # Create a new swept parameter with the updated sweep
        swept_parameter = SweptParameter(
            swept_parameter.path,
            walk_values(
                iffy(is_mapping,
                     autocurry(select_keys)(lambda k: '_' not in k)),
                updated_sweep))
        return swept_parameter
Ejemplo n.º 32
0
def block_num_from_jsonrpc_response(
        jsonrpc_response: SingleJsonRpcRequest = None) -> int:
    # pylint: disable=no-member
    block_id = cytoolz.get_in(['result', 'block_id'], jsonrpc_response)
    return block_num_from_id(block_id)
Ejemplo n.º 33
0
def format(cx, cy, px, py, dates, ccdresult):

    return [{
        'cx': int(cx),
        'cy': int(cy),
        'px': int(px),
        'py': int(py),
        'sday': date.fromordinal(get('start_day', cm, 1)).isoformat(),
        'eday': date.fromordinal(get('end_day', cm, 1)).isoformat(),
        'bday': date.fromordinal(get('break_day', cm, 1)).isoformat(),
        'chprob': get('change_probability', cm, 0.0),
        'curqa': get('curve_qa', cm, 0),
        'blmag': get_in(['blue', 'magnitude'], cm, 0.0),
        'grmag': get_in(['green', 'magnitude'], cm, 0.0),
        'remag': get_in(['red', 'magnitude'], cm, 0.0),
        'nimag': get_in(['nir', 'magnitude'], cm, 0.0),
        's1mag': get_in(['swir1', 'magnitude'], cm, 0.0),
        's2mag': get_in(['swir2', 'magnitude'], cm, 0.0),
        'thmag': get_in(['thermal', 'magnitude'], cm, 0.0),
        'blrmse': get_in(['blue', 'rmse'], cm, 0.0),
        'grrmse': get_in(['green', 'rmse'], cm, 0.0),
        'rermse': get_in(['red', 'rmse'], cm, 0.0),
        'nirmse': get_in(['nir', 'rmse'], cm, 0.0),
        's1rmse': get_in(['swir1', 'rmse'], cm, 0.0),
        's2rmse': get_in(['swir2', 'rmse'], cm, 0.0),
        'thrmse': get_in(['thermal', 'rmse'], cm, 0.0),
        'blcoef': coefficients(cm, 'blue'),
        'grcoef': coefficients(cm, 'green'),
        'recoef': coefficients(cm, 'red'),
        'nicoef': coefficients(cm, 'nir'),
        's1coef': coefficients(cm, 'swir1'),
        's2coef': coefficients(cm, 'swir2'),
        'thcoef': coefficients(cm, 'thermal'),
        'blint': get_in(['blue', 'intercept'], cm, 0.0),
        'grint': get_in(['green', 'intercept'], cm, 0.0),
        'reint': get_in(['red', 'intercept'], cm, 0.0),
        'niint': get_in(['nir', 'intercept'], cm, 0.0),
        's1int': get_in(['swir1', 'intercept'], cm, 0.0),
        's2int': get_in(['swir2', 'intercept'], cm, 0.0),
        'thint': get_in(['thermal', 'intercept'], cm, 0.0),
        'dates': [date.fromordinal(o).isoformat() for o in dates],
        'mask': get('processing_mask', ccdresult)
    } for cm in defaults(get('change_models', ccdresult, None))]
Ejemplo n.º 34
0
def coefficients(change_model, spectra):
    coefs = get_in([spectra, 'coefficients'], change_model)
    return list(coefs) if coefs else []