示例#1
0
文件: core.py 项目: umuzungu/zipline
    def most_recent_data(bundle_name, timestamp, environ=None):
        """Get the path to the most recent data after ``date``for the
        given bundle.

        Parameters
        ----------
        bundle_name : str
            The name of the bundle to lookup.
        timestamp : datetime
            The timestamp to begin searching on or before.
        environ : dict, optional
            An environment dict to forward to zipline_root.
        """
        if bundle_name not in bundles:
            raise UnknownBundle(bundle_name)

        try:
            candidates = os.listdir(pth.data_path([bundle_name], environ=environ))
            return pth.data_path(
                [bundle_name, max(filter(complement(pth.hidden), candidates), key=from_bundle_ingest_dirname)],
                environ=environ,
            )
        except (ValueError, OSError) as e:
            if getattr(e, "errno", ~errno.ENOENT) != errno.ENOENT:
                raise
            raise ValueError("no data for bundle %r on or before %s" % (bundle_name, timestamp))
示例#2
0
def _expect_element(collection):
    template = (
        "%(funcname)s() expected a value in {collection} "
        "for argument '%(argname)s', but got %(actual)s instead.").format(
            collection=collection)
    return make_check(
        ValueError,
        template,
        complement(op.contains(collection)),
        repr,
    )
示例#3
0
def _expect_element(collection):
    template = (
        "%(funcname)s() expected a value in {collection} "
        "for argument '%(argname)s', but got %(actual)s instead."
    ).format(collection=collection)
    return make_check(
        ValueError,
        template,
        complement(op.contains(collection)),
        repr,
    )
示例#4
0
def clean_dir_info(obj):
	"""
	If needing to display a dictionary of some class that doesnt have __dict__, the boltons.dir_dict
	will work, but this cleans it up for serializing

	"""
	try:
		d = dir_dict(obj)
	except Exception:
		d = dict()
	d1 = keyfilter(complement(_is_dunder), d)
	return dictfilter(d1, factory = AttributeDict)
示例#5
0
def read_results_frame(output_dir: str) -> DataFrame:
    """ Reads the results.json file into a pandas DataFrame.

        :param output_dir: The name of the output directory of the grid search.

        :returns: 
            The results as a pandas data frame. Excludes the cross validation 
            run lists (but keeps the cross validation mean values).

        :raises ValueError:
            If the output directory doesn't have a ``results.json`` file in it.
    """
    results = read_results(output_dir)
    return DataFrame(data=listmap(
        lambda r: keyfilter(complement(_frame_exclude_col), r), results))
示例#6
0
def compare_local_remote_versions(
    local_versions: Iterable[str],
    remotes: Iterable[Remote],
    worker_count: int,
) -> Tuple[Tuple[Remote, str]]:
    """Returns the list of remotes_with_new_versions with versions greater than the maximum local one"""
    max_version_local = max(set(
        filter(complement(_is_live_version), local_versions)),
                            default="")
    return tuple(
        filter(
            lambda remote_version: version_compare(remote_version[1],
                                                   max_version_local) > 0,
            process_remotes_list(remotes, worker_count=worker_count),
        ))
示例#7
0
    def _expect_element(collection):
        if isinstance(collection, (set, frozenset)):
            # Special case the error message for set and frozen set to make it
            # less verbose.
            collection_for_error_message = tuple(sorted(collection))
        else:
            collection_for_error_message = collection

        template = (
            "%(funcname)s() expected a value in {collection} "
            "for argument '%(argname)s', but got %(actual)s instead."
        ).format(collection=collection_for_error_message)
        return make_check(
            ValueError,
            template,
            complement(op.contains(collection)),
            repr,
        )
示例#8
0
    def _expect_element(collection):
        if isinstance(collection, (set, frozenset)):
            # Special case the error message for set and frozen set to make it
            # less verbose.
            collection_for_error_message = tuple(sorted(collection))
        else:
            collection_for_error_message = collection

        template = (
            "%(funcname)s() expected a value in {collection} "
            "for argument '%(argname)s', but got %(actual)s instead.").format(
                collection=collection_for_error_message)
        return make_check(
            ValueError,
            template,
            complement(op.contains(collection)),
            repr,
        )
示例#9
0
def make_while_loop_test_expr(loop_body_instrs):
    """
    Make an expression in the context of a while-loop test.

    Code of the form::

        while <expr>:
            <body>

    generates a POP_JUMP_IF_FALSE for the loop test, while code of the form::

        while not <expr>:
            <body>

    generates a POP_JUMP_IF_TRUE for the loop test.

    Code of the form::

        while True:
            <body>

    generates no jumps at all.
    """
    bottom_of_loop = loop_body_instrs[-1]
    is_jump_to_bottom = compose(op.is_(bottom_of_loop), op.attrgetter('arg'))

    # Consume instructions until we find a jump to the bottom of the loop.
    test_builders = deque(
        popwhile(complement(is_jump_to_bottom), loop_body_instrs, side='left')
    )
    # If we consumed the entire loop body without finding a jump, assume this
    # is a while True loop.  Return the rest of the instructions as the loop
    # body.
    if not loop_body_instrs:
        return ast.NameConstant(value=True), test_builders

    # Top of the body is either a POP_JUMP_IF_TRUE or POP_JUMP_IF_FALSE.
    jump = loop_body_instrs.popleft()
    expr = make_expr(test_builders)
    if isinstance(jump, instrs.POP_JUMP_IF_TRUE):
        return ast.UnaryOp(op=ast.Not(), operand=expr), loop_body_instrs
    else:
        return expr, loop_body_instrs
示例#10
0
def _dict_contains(small_dict: Dict[str, Any], big_dict: Dict[str,
                                                              Any]) -> bool:
    """ Returns true if ``small_dict`` is contained by ``big_dict``.

        :param small_dict: 
            The smaller dict that may or may not be contained by the bigger 
            dict.
        
        :param big_dict:
            The bigger dict that may or may not contain the smaller dict.

        :returns: True if the bigger dict contains the smaller dict.
    """
    # Remove the list values from big_dict because they are unhashable.
    # The implicit assumption is that small_dict doesn't have list elements.
    # Since this is called in the context of model hyperparameters I think
    # this is a safe assumption.
    return set(small_dict.items()) <= \
           set(valfilter(complement(listels), big_dict).items())
示例#11
0
    def most_recent_data(bundle_name, timestamp=None, environ=None):
        """Get the path to the most recent data after ``date``for the
        given bundle.

        Parameters
        ----------
        bundle_name : str
            The name of the bundle to lookup.
        timestamp : datetime
            The timestamp to begin searching on or before.
        environ : dict, optional
            An environment dict to forward to zipline_root.
        """
        if timestamp is None:
            timestamp = pd.Timestamp.utcnow()

        if bundle_name not in bundles:
            raise UnknownBundle(bundle_name)

        try:
            candidates = os.listdir(
                pth.data_path([bundle_name], environ=environ), )
            return pth.data_path(
                [
                    bundle_name,
                    max(
                        filter(complement(pth.hidden), candidates),
                        key=from_bundle_ingest_dirname,
                    )
                ],
                environ=environ,
            )
        except (ValueError, OSError) as e:
            if getattr(e, 'errno', errno.ENOENT) != errno.ENOENT:
                raise
            raise ValueError(
                'no data for bundle {bundle!r} on or before {timestamp}\n'
                'maybe you need to run: $ zipline ingest -b {bundle}'.format(
                    bundle=bundle_name,
                    timestamp=timestamp,
                ), )
示例#12
0
def dichotomize(predicate, iterable):
    """Take a predicate and an iterable and return the pair of iterables of
    elements which do and do not satisfy the predicate.

    Parameters
    ----------
    predicate : callable[any, bool]
        The predicate function to partition with.
    iterable : iterable[any]
        The elements to partition.

    Returns
    -------
    trues : iterable[any]
        The sequence of values where the predicate evaluated to True.
    falses : iterable[any]
        The sequence of values where the predicate evaluated to False.

    Notes
    -----
    This is a lazy version of:

    .. code-block:: Python

       def partition(predicate, sequence):
           sequence = list(sequence)
           return (
               filter(predicate, sequence),
               filter(complement(predicate), sequence),
           )
    """
    true_queue = deque()
    false_queue = deque()
    it = iter(iterable)

    return (
        _predicate_iter(it, predicate, true_queue, false_queue),
        _predicate_iter(it, complement(predicate), false_queue, true_queue),
    )
示例#13
0
    def most_recent_data(bundle_name, timestamp, environ=None):
        """Get the path to the most recent data after ``date``for the
        given bundle.

        Parameters
        ----------
        bundle_name : str
            The name of the bundle to lookup.
        timestamp : datetime
            The timestamp to begin searching on or before.
        environ : dict, optional
            An environment dict to forward to catalyst_root.
        """
        if bundle_name not in bundles:
            raise UnknownBundle(bundle_name)

        try:
            candidates = os.listdir(
                pth.data_path([bundle_name], environ=environ),
            )
            return pth.data_path(
                [bundle_name,
                 max(
                     filter(complement(pth.hidden), candidates),
                     key=from_bundle_ingest_dirname,
                 )],
                environ=environ,
            )
        except (ValueError, OSError) as e:
            if getattr(e, 'errno', errno.ENOENT) != errno.ENOENT:
                raise
            raise ValueError(
                'no data for bundle {bundle!r} on or before {timestamp}\n'
                'maybe you need to run: $ catalyst ingest -b {bundle}'.format(
                    bundle=bundle_name,
                    timestamp=timestamp,
                ),
            )
示例#14
0
def get_higher_versions_in_repology(
    package: Package, repology_cache: dict
) -> Optional[Iterable]:
    """Returns all versions from repology which are higher than the ones in package.

    If the package is not in repology cache - returns None"""
    repology_name = _get_repology_name_for_pkg(
        atomname=package.atomname, cache=repology_cache
    )
    if repology_name is None:
        return None

    my_versions = tuple(
        map(
            compose(Version, strip_revision),
            filter(complement(_is_live_version), package.versions),
        )
    )
    if my_versions:
        max_version = max(my_versions).value
        op = compose(lambda x: x < 0, partial(version_compare, max_version))
        return filter(op, _get_versions_from_repology_repos(repology_name))
    else:
        return _get_versions_from_repology_repos(repology_name)
示例#15
0
def get_dirs_and_files_in_path(path):
    # filter function
    def isdir(a): return os.path.isdir(a)
    # gives the opposite results as above
    not_isdir = toolz.complement(isdir)

    if not path and platform.system() == 'Windows':
        import win32api
        drives = win32api.GetLogicalDriveStrings()
        drives = [d for d in drives.split('\000') if d]
        return drives

    elif os.path.exists(path):
        r = os.listdir(path)
        # 2x acccess means I have to remove the generator
        f = [os.path.join(path, a) for a in r]
        dirs = filter(isdir, f)
        files = filter(not_isdir, f)

    else:
        try:
            head, tail = os.path.split(path)
            r = os.listdir(head)
            filtered_everything = filter(lambda a: a.startswith(tail), r)
            # because this was accesssed twice, I needed to remove the generator
            filtered_everything = [os.path.join(head, a) for a in filtered_everything]
            dirs = filter(isdir, filtered_everything)
            files = filter(not_isdir, filtered_everything)

        except Exception as e:
            print('{0} doesn\'t even exist you stupid'.format(head))
            return None

    result = (sorted(list(toolz.take(100, dirs))),
              sorted(list(toolz.take(100, files))))
    return result
示例#16
0
class HQRules(RuleBasedStateMachine):
    items_received = Bundle('received items')

    def __init__(self):
        super().__init__()
        self.items_added = []
        self.n_items_received = 0
        self.hq = None

    @property
    def n_items_added(self):
        return len(self.items_added)

    def has_hq(self):
        return self.hq is not None

    def backlog_is_empty(self):
        return self.n_items_received == self.n_items_added

    def backlog_is_too_full(self):
        return (
            self.has_hq() and
            self.hq.max_backlog and
            (self.n_items_added - self.n_items_received == self.hq.max_backlog)
        )

    def history_is_full(self):
        return self.has_hq() and self.hq.history_full()

    @precondition(complement(has_hq))
    @rule(
        history_size=st.one_of(st.just(None), st.integers(min_value=0, max_value=4)),
        max_backlog=st.integers(min_value=0, max_value=4),
    )
    def initialize(self, history_size, max_backlog):
        self.hq = HistoryQueue(history_size, max_backlog=max_backlog)

    @preconditions(has_hq, complement(backlog_is_too_full))
    @rule(item=anything())
    def put(self, item):
        self.hq.put_nowait(item)
        self.items_added.append(item)

    @precondition(backlog_is_too_full)
    @rule(item=anything())
    def attempt_to_put(self, item):
        assert self.hq.backlog_full()
        with pytest.raises(asyncio.QueueFull):
            self.hq.put_nowait(item)

    @preconditions(has_hq, backlog_is_empty)
    def attempt_to_get(self):
        assert self.hq.backlog_empty()
        with pytest.raises(asyncio.QueueEmpty):
            self.hq.get_nowait()

    @precondition(complement(backlog_is_empty))
    @rule(target=items_received)
    def get(self):
        item = ReceivedItem(self.n_items_received, self.hq.get_nowait())
        print(item)
        self.n_items_received += 1
        return item

    @precondition(history_is_full)
    def check_full(self):
        assert self.hq.history_full()

    @rule(item=items_received)
    def check_item(self, item):
        if self.hq.history_size is None or item[0] <= self.hq.history_size:
            self.check_item_not_filled_yet(item)
        else:
            self.check_item_filled(item)

    def check_item_filled(self, item_off_bundle):
        i, item = item_off_bundle
        assert len(item) == self.hq.history_size + 1
        for item_added, item_received in zip(self.items_added[i - self.hq.history_size:],
                                             reversed(item)):
            assert item_added is item_received

    def check_item_not_filled_yet(self, item_off_bundle):
        i, item = item_off_bundle
        assert len(item) == i + 1
        for item_added, item_received in zip(self.items_added, reversed(item)):
            assert item_added is item_received
示例#17
0
文件: core.py 项目: RoyHsiao/zipline
    def clean(name,
              before=None,
              after=None,
              keep_last=None,
              environ=os.environ):
        """Clean up data that was created with ``ingest`` or
        ``$ python -m zipline ingest``

        Parameters
        ----------
        name : str
            The name of the bundle to remove data for.
        before : datetime, optional
            Remove data ingested before this date.
            This argument is mutually exclusive with: keep_last
        after : datetime, optional
            Remove data ingested after this date.
            This argument is mutually exclusive with: keep_last
        keep_last : int, optional
            Remove all but the last ``keep_last`` ingestions.
            This argument is mutually exclusive with:
              before
              after
        environ : mapping, optional
            The environment variables. Defaults of os.environ.

        Returns
        -------
        cleaned : set[str]
            The names of the runs that were removed.

        Raises
        ------
        BadClean
            Raised when ``before`` and or ``after`` are passed with
            ``keep_last``. This is a subclass of ``ValueError``.
        """
        try:
            all_runs = sorted(
                filter(
                    complement(pth.hidden),
                    os.listdir(pth.data_path([name], environ=environ)),
                ),
                key=from_bundle_ingest_dirname,
            )
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
            raise UnknownBundle(name)
        if ((before is not None or after is not None) and
                keep_last is not None):
            raise BadClean(before, after, keep_last)

        if keep_last is None:
            def should_clean(name):
                dt = from_bundle_ingest_dirname(name)
                return (
                    (before is not None and dt < before) or
                    (after is not None and dt > after)
                )

        else:
            last_n_dts = set(all_runs[-keep_last:])

            def should_clean(name):
                return name not in last_n_dts

        cleaned = set()
        for run in all_runs:
            if should_clean(run):
                path = pth.data_path([name, run], environ=environ)
                shutil.rmtree(path)
                cleaned.add(path)

        return cleaned
示例#18
0
def not_correct_order(lines):
    valid = pipe(
        all_indices(lines),
        complement(monotonic),
    )
    return (valid, 'Start and end tags are not in the correct order.')
示例#19
0
文件: poser.py 项目: deathbeds/poser
 def complement(x, object=None):
     return λ[toolz.complement(x)] if object == None else x[toolz.complement(object)]
示例#20
0
    def ingest(name,
               environ=os.environ,
               timestamp=None,
               assets_versions=(),
               show_progress=False,
               writer="bcolz"):
        """Ingest data for a given bundle.

        Parameters
        ----------
        name : str
            The name of the bundle.
        environ : mapping, optional
            The environment variables. By default this is os.environ.
        timestamp : datetime, optional
            The timestamp to use for the load.
            By default this is the current time.
        assets_versions : Iterable[int], optional
            Versions of the assets db to which to downgrade.
        show_progress : bool, optional
            Tell the ingest function to display the progress where possible.
        incremental : bool, optional
            Tell the ingest function to incremental ingest
        """
        try:
            bundle = bundles[name]
        except KeyError:
            raise UnknownBundle(name)

        calendar = get_calendar(bundle.calendar_name)

        start_session = bundle.start_session
        end_session = bundle.end_session

        if start_session is None or start_session < calendar.first_session:
            start_session = calendar.first_session

        if end_session is None or end_session > calendar.last_session:
            end_session = calendar.last_session
        try:
            candidates = os.listdir(pth.data_path([name], environ=environ), )
            timestr = max(
                filter(complement(pth.hidden), candidates),
                key=from_bundle_ingest_dirname,
            )
        except Exception:
            if timestamp is None:
                timestamp = pd.Timestamp.utcnow()
            timestamp = timestamp.tz_convert('utc').tz_localize(None)

            timestr = to_bundle_ingest_dirname(timestamp)
        cachepath = cache_path(name, environ=environ)
        pth.ensure_directory(pth.data_path([name, timestr], environ=environ))
        pth.ensure_directory(cachepath)
        with dataframe_cache(cachepath, clean_on_failure=False) as cache, \
                ExitStack() as stack:
            # we use `cleanup_on_failure=False` so that we don't purge the
            # cache directory if the load fails in the middle
            if bundle.create_writers:
                wd = stack.enter_context(
                    working_dir(pth.data_path([], environ=environ)))
                daily_bars_path = daily_equity_path(name,
                                                    timestr,
                                                    environ=environ)
                pth.ensure_directory(daily_bars_path)
                daily_bar_writer = BcolzDailyBarWriter(
                    daily_bars_path,
                    calendar,
                    start_session,
                    end_session,
                )
                # Do an empty write to ensure that the daily ctables exist
                # when we create the SQLiteAdjustmentWriter below. The
                # SQLiteAdjustmentWriter needs to open the daily ctables so
                # that it can compute the adjustment ratios for the dividends.

                daily_bar_writer.write(())

                minute_bars_path = minute_equity_path(name,
                                                      timestr,
                                                      environ=environ)
                pth.ensure_directory(minute_bars_path)
                if writer == "rocksdb":
                    minute_bar_writer = RocksdbMinuteBarWriter(
                        minute_bars_path,
                        calendar,
                        start_session,
                        end_session,
                        minutes_per_day=bundle.minutes_per_day,
                    )
                else:
                    minute_bar_writer = BcolzMinuteBarWriter(
                        minute_bars_path,
                        calendar,
                        start_session,
                        end_session,
                        minutes_per_day=bundle.minutes_per_day,
                    )
                wd.ensure_dir(
                    name,
                    timestr,
                )
                asset_db_writer = AssetDBWriter(
                    wd.getpath(*asset_db_relative(
                        name,
                        timestr,
                        environ=environ,
                    )))

                fundamental_db_writer = FundamentalWriter(
                    wd.getpath(*fundamental_db_releative(
                        name, timestr, environ=environ)))

                adjustment_db_writer = stack.enter_context(
                    SQLiteAdjustmentWriter(
                        wd.getpath(*adjustment_db_relative(
                            name, timestr, environ=environ)),
                        BcolzDailyBarReader(daily_bars_path),
                        calendar.all_sessions,
                        overwrite=True,
                    ))
            else:
                daily_bar_writer = None
                minute_bar_writer = None
                asset_db_writer = None
                adjustment_db_writer = None
                fundamental_db_writer = None
                if assets_versions:
                    raise ValueError('Need to ingest a bundle that creates '
                                     'writers in order to downgrade the assets'
                                     ' db.')
            bundle.ingest(
                environ,
                asset_db_writer,
                minute_bar_writer,
                daily_bar_writer,
                adjustment_db_writer,
                fundamental_db_writer,
                calendar,
                start_session,
                end_session,
                cache,
                show_progress,
                pth.data_path([name, timestr], environ=environ),
            )

            for version in sorted(set(assets_versions), reverse=True):
                version_path = wd.getpath(*asset_db_relative(
                    name,
                    timestr,
                    environ=environ,
                    db_version=version,
                ))
                with working_file(version_path) as wf:
                    shutil.copy2(assets_db_path, wf.path)
                    downgrade(wf.path, version)
示例#21
0
def not_a(type_):
    """More curryable version of not isinstance."""
    return complement(is_a(type_))
示例#22
0
def clean_text(text: str) -> str:
    return filter_text(text, complement(is_control_char))
示例#23
0
def remove_control_chars(string):
    return "".join(list(filter(complement(is_control_char), string)))
示例#24
0
def default_if_true(predicate, default):
    """Implements the rule: default if v else v"""
    predicate = complement(predicate)
    return default_if_false(predicate, default)
示例#25
0
    def ingest(name,
               environ=os.environ,
               timestamp=None,
               assets_versions=(),
               show_progress=False,
               writer="bcolz"):
        """Ingest data for a given bundle.

        Parameters
        ----------
        name : str
            The name of the bundle.
        environ : mapping, optional
            The environment variables. By default this is os.environ.
        timestamp : datetime, optional
            The timestamp to use for the load.
            By default this is the current time.
        assets_versions : Iterable[int], optional
            Versions of the assets db to which to downgrade.
        show_progress : bool, optional
            Tell the ingest function to display the progress where possible.
        incremental : bool, optional
            Tell the ingest function to incremental ingest
        """
        try:
            bundle = bundles[name]
        except KeyError:
            raise UnknownBundle(name)

        calendar = get_calendar(bundle.calendar_name)

        start_session = bundle.start_session
        end_session = bundle.end_session

        if start_session is None or start_session < calendar.first_session:
            start_session = calendar.first_session

        if end_session is None or end_session > calendar.last_session:
            end_session = calendar.last_session
        try:
            candidates = os.listdir(
                pth.data_path([name], environ=environ),
            )
            timestr = max(
                filter(complement(pth.hidden), candidates),
                key=from_bundle_ingest_dirname,
            )
        except Exception:
            if timestamp is None:
                timestamp = pd.Timestamp.utcnow()
            timestamp = timestamp.tz_convert('utc').tz_localize(None)

            timestr = to_bundle_ingest_dirname(timestamp)
        cachepath = cache_path(name, environ=environ)
        pth.ensure_directory(pth.data_path([name, timestr], environ=environ))
        pth.ensure_directory(cachepath)
        with dataframe_cache(cachepath, clean_on_failure=False) as cache, \
                ExitStack() as stack:
            # we use `cleanup_on_failure=False` so that we don't purge the
            # cache directory if the load fails in the middle
            if bundle.create_writers:
                wd = stack.enter_context(working_dir(
                    pth.data_path([], environ=environ))
                )
                daily_bars_path = daily_equity_path(name, timestr, environ=environ)
                pth.ensure_directory(
                    daily_bars_path
                )
                daily_bar_writer = BcolzDailyBarWriter(
                    daily_bars_path,
                    calendar,
                    start_session,
                    end_session,
                )
                # Do an empty write to ensure that the daily ctables exist
                # when we create the SQLiteAdjustmentWriter below. The
                # SQLiteAdjustmentWriter needs to open the daily ctables so
                # that it can compute the adjustment ratios for the dividends.

                daily_bar_writer.write(())

                minute_bars_path = minute_equity_path(name, timestr, environ=environ)
                pth.ensure_directory(
                    minute_bars_path
                )
                if writer == "rocksdb":
                    minute_bar_writer = RocksdbMinuteBarWriter(
                        minute_bars_path,
                        calendar,
                        start_session,
                        end_session,
                        minutes_per_day=bundle.minutes_per_day,
                    )
                else:
                    minute_bar_writer = BcolzMinuteBarWriter(
                        minute_bars_path,
                        calendar,
                        start_session,
                        end_session,
                        minutes_per_day=bundle.minutes_per_day,
                    )
                wd.ensure_dir(
                    name, timestr,
                )
                asset_db_writer = AssetDBWriter(
                    wd.getpath(*asset_db_relative(
                        name, timestr, environ=environ,
                    ))
                )

                fundamental_db_writer = FundamentalWriter(
                    wd.getpath(*fundamental_db_releative(
                        name, timestr, environ=environ
                    ))
                )

                adjustment_db_writer = stack.enter_context(
                    SQLiteAdjustmentWriter(
                        wd.getpath(*adjustment_db_relative(
                            name, timestr, environ=environ)),
                        BcolzDailyBarReader(daily_bars_path),
                        calendar.all_sessions,
                        overwrite=True,
                    )
                )
            else:
                daily_bar_writer = None
                minute_bar_writer = None
                asset_db_writer = None
                adjustment_db_writer = None
                fundamental_db_writer = None
                if assets_versions:
                    raise ValueError('Need to ingest a bundle that creates '
                                     'writers in order to downgrade the assets'
                                     ' db.')
            bundle.ingest(
                environ,
                asset_db_writer,
                minute_bar_writer,
                daily_bar_writer,
                adjustment_db_writer,
                fundamental_db_writer,
                calendar,
                start_session,
                end_session,
                cache,
                show_progress,
                pth.data_path([name, timestr], environ=environ),
            )

            for version in sorted(set(assets_versions), reverse=True):
                version_path = wd.getpath(*asset_db_relative(
                    name, timestr, environ=environ, db_version=version,
                ))
                with working_file(version_path) as wf:
                    shutil.copy2(assets_db_path, wf.path)
                    downgrade(wf.path, version)
示例#26
0
# -*- encoding: utf-8 -*-
import toolz

print_list = lambda x: print(list(x))

l = list(range(25))


# 1. use toolz.curry instead functools.partial
@toolz.curry
def not_multiple_of(x, y):
    return y % x != 0


print_list(filter(not_multiple_of(3), l))
print_list(filter(not_multiple_of(7), l))
print('-' * 20, '\n')

# 2. use toolz.remove
print_list(toolz.remove(toolz.complement(not_multiple_of(3)), l))
print_list(toolz.remove(toolz.complement(not_multiple_of(7)), l))
示例#27
0
def check(condition, exception):
    return do_if(toolz.complement(condition), make_raise(exception))
示例#28
0
文件: core.py 项目: seanahmad/zipline
    def clean(name,
              before=None,
              after=None,
              keep_last=None,
              environ=os.environ):
        """Clean up data that was created with ``ingest`` or
        ``$ python -m zipline ingest``

        Parameters
        ----------
        name : str
            The name of the bundle to remove data for.
        before : datetime, optional
            Remove data ingested before this date.
            This argument is mutually exclusive with: keep_last
        after : datetime, optional
            Remove data ingested after this date.
            This argument is mutually exclusive with: keep_last
        keep_last : int, optional
            Remove all but the last ``keep_last`` ingestions.
            This argument is mutually exclusive with:
              before
              after
        environ : mapping, optional
            The environment variables. Defaults of os.environ.

        Returns
        -------
        cleaned : set[str]
            The names of the runs that were removed.

        Raises
        ------
        BadClean
            Raised when ``before`` and or ``after`` are passed with
            ``keep_last``. This is a subclass of ``ValueError``.
        """
        try:
            all_runs = sorted(
                filter(
                    complement(pth.hidden),
                    os.listdir(pth.data_path([name], environ=environ)),
                ),
                key=from_bundle_ingest_dirname,
            )
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
            raise UnknownBundle(name)
        if ((before is not None or after is not None)
                and keep_last is not None):
            raise BadClean(before, after, keep_last)

        if keep_last is None:

            def should_clean(name):
                dt = from_bundle_ingest_dirname(name)
                return ((before is not None and dt < before)
                        or (after is not None and dt > after))

        elif keep_last >= 0:
            last_n_dts = set(take(keep_last, reversed(all_runs)))

            def should_clean(name):
                return name not in last_n_dts
        else:
            raise BadClean(before, after, keep_last)

        cleaned = set()
        for run in all_runs:
            if should_clean(run):
                path = pth.data_path([name, run], environ=environ)
                shutil.rmtree(path)
                cleaned.add(path)

        return cleaned
示例#29
0
from zipline.utils.numpy_utils import repeat_last_axis


AD_FIELD_NAME = 'asof_date'
TS_FIELD_NAME = 'timestamp'
SID_FIELD_NAME = 'sid'
valid_deltas_node_types = (
    bz.expr.Field,
    bz.expr.ReLabel,
    bz.expr.Symbol,
)
traversable_nodes = (
    bz.expr.Field,
    bz.expr.Label,
)
is_invalid_deltas_node = complement(flip(isinstance, valid_deltas_node_types))
getname = op.attrgetter('__name__')


class _ExprRepr(object):
    """Box for repring expressions with the str of the expression.

    Parameters
    ----------
    expr : Expr
        The expression to box for repring.
    """
    __slots__ = 'expr',

    def __init__(self, expr):
        self.expr = expr
示例#30
0
def not_a(type_):
    """More curryable version of not isinstance."""
    return complement(is_a(type_))