예제 #1
0
def init_interactive(
    name: str,
    defaults: Dict[str, str],
    provided: Dict[str, str],
    validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None,
    live: bool = False,
    stream: Optional[TextIO] = None,
) -> Dict[str, str]:
    command = provided.pop("cmd", None)
    primary = lremove(provided.keys(), ["code", "data", "models", "params"])
    secondary = lremove(provided.keys(),
                        ["live"] if live else ["metrics", "plots"])
    prompts = primary + secondary

    workspace = {**defaults, **provided}
    if not live and "live" not in provided:
        workspace.pop("live", None)
    for key in ("plots", "metrics"):
        if live and key not in provided:
            workspace.pop(key, None)

    ret: Dict[str, str] = {}
    if command:
        ret["cmd"] = command

    if not prompts and command:
        return ret

    ui.error_write(
        f"This command will guide you to set up a [bright_blue]{name}[/]",
        "stage in [green]dvc.yaml[/].",
        f"\nSee [repr.url]{PIPELINE_FILE_LINK}[/].\n",
        styled=True,
    )

    if not command:
        ret.update(
            compact(_prompts(["cmd"], allow_omission=False, stream=stream)))
        if prompts:
            ui.error_write(styled=True)

    if not prompts:
        return ret

    ui.error_write(
        "Enter the paths for dependencies and outputs of the command.",
        styled=True,
    )
    if workspace:
        ui.error_write(build_workspace_tree(workspace), styled=True)
    ui.error_write(styled=True)
    ret.update(
        compact(_prompts(prompts, defaults, validator=validator,
                         stream=stream)))
    return ret
예제 #2
0
파일: init.py 프로젝트: pmrowla/dvc
def init_interactive(
    defaults: Dict[str, str],
    provided: Dict[str, str],
    validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None,
    stream: Optional[TextIO] = None,
) -> Dict[str, str]:
    command_prompts = lremove(provided.keys(), ["cmd"])
    dependencies_prompts = lremove(provided.keys(), ["code", "data", "params"])
    output_keys = ["models"]
    if "live" not in provided:
        output_keys.extend(["metrics", "plots"])
    outputs_prompts = lremove(provided.keys(), output_keys)

    ret: Dict[str, str] = {}
    if "cmd" in provided:
        ret["cmd"] = provided["cmd"]

    for heading, prompts, allow_omission in (
        ("", command_prompts, False),
        ("Enter experiment dependencies.", dependencies_prompts, True),
        ("Enter experiment outputs.", outputs_prompts, True),
    ):
        if prompts and heading:
            ui.error_write(heading, styled=True)
        response = _prompts(
            prompts,
            defaults=defaults,
            allow_omission=allow_omission,
            validator=validator,
            stream=stream,
        )
        ret.update(compact(response))
        if prompts:
            ui.error_write(styled=True)
    return ret
예제 #3
0
def normalize(string: str):
    words = compact(RE_SPLIT(string.lower()))
    for word in words:
        if RE_CYRILLIC(word):
            yield word.translate(NORM)
        else:
            yield word
예제 #4
0
파일: init.py 프로젝트: pared/dvc
    def run(self):
        from dvc.command.stage import parse_cmd

        cmd = parse_cmd(self.args.command)
        if not self.args.interactive and not cmd:
            raise InvalidArgumentError("command is not specified")

        from dvc.repo.experiments.init import init

        defaults = {}
        if not self.args.explicit:
            config = self.repo.config["exp"]
            defaults.update({**self.DEFAULTS, **config})

        cli_args = compact({
            "cmd": cmd,
            "code": self.args.code,
            "data": self.args.data,
            "models": self.args.models,
            "metrics": self.args.metrics,
            "params": self.args.params,
            "plots": self.args.plots,
            "live": self.args.live,
        })

        initialized_stage = init(
            self.repo,
            name=self.args.name,
            type=self.args.type,
            defaults=defaults,
            overrides=cli_args,
            interactive=self.args.interactive,
            force=self.args.force,
        )

        text = ui.rich_text.assemble(
            "\n" if self.args.interactive else "",
            "Created ",
            (self.args.name, "bright_blue"),
            " stage in ",
            ("dvc.yaml", "green"),
            ".",
        )
        if not self.args.run:
            text.append_text(
                ui.rich_text.assemble(
                    " To run, use ",
                    ('"dvc exp run"', "green"),
                    ".\nSee ",
                    (self.EXP_LINK, "repr.url"),
                    ".",
                ))

        ui.write(text, styled=True)
        if self.args.run:
            return self.repo.experiments.run(
                targets=[initialized_stage.addressing])

        return 0
예제 #5
0
async def expand_multiple_items(item_list, relations):
    for field, (type_, subfields) in relations.items():
        path = field.split('.')
        field = path[-1]
        items_with_field = _find_items_having_path(item_list, path)

        related_ids = list(compact(pluck(field, items_with_field)))
        if not related_ids:
            continue

        related_items = await fetch_multiple_items(type_, related_ids, subfields)
        related_by_id = {item['id']: item for item in compact(related_items)}

        for item in items_with_field:
            item[field] = related_by_id.get(item[field])

    return item_list
예제 #6
0
파일: lang.py 프로젝트: jacob414/micropy
 def __call__(self, *params: Any) -> Any:
     self.truthy = []
     super().__call__(*params)
     passed = funcy.compact(self.truthy)
     if len(passed) == self.conjunctions:
         return self.format(passed[-1])
     else:
         return False
예제 #7
0
 def _run_cur(self) -> ASTQuery:
     "Runs current Query."
     root = self.rep
     if len(self.ops) > 0:
         return funcy.compact(self.truthy)
     else:
         raise ValueError("INTERNAL: Snack._run_cur() called without query "
                          "terms specified")
예제 #8
0
async def expand_multiple_items(item_list, relations):
    for field, (type_, subfields) in relations.items():
        path = field.split('.')
        field = path[-1]
        items_with_field = _find_items_having_path(item_list, path)

        related_ids = list(compact(pluck(field, items_with_field)))
        if not related_ids:
            continue

        related_items = await fetch_multiple_items(type_, related_ids,
                                                   subfields)
        related_by_id = {item['id']: item for item in compact(related_items)}

        for item in items_with_field:
            item[field] = related_by_id.get(item[field])

    return item_list
예제 #9
0
    def __init__(self, host, port, name, user=None, passwd=None):
        # параметры аутетификации к БД
        self.auth = {"host": host, "port": port, "db": name, "user": user, "password": passwd}
        self.auth = funcy.compact(self.auth)
        service_log.put("Get params for authenticate with redis: %s" % self.auth)

        # параметры для работы внутри БД
        self.pool = None
        self.cursor = None
        self.cursor_now = None
예제 #10
0
def _pack_remotes(conf):
    # Drop empty sections
    result = compact(conf)

    # Transform remote.name -> 'remote "name"'
    for name, val in conf["remote"].items():
        result[f'remote "{name}"'] = val
    result.pop("remote", None)

    return result
예제 #11
0
def _pack_named(conf):
    # Drop empty sections
    result = compact(conf)

    # Transform remote.name -> 'remote "name"'
    for key in ("remote", "machine"):
        for name, val in conf[key].items():
            result[f'{key} "{name}"'] = val
        result.pop(key, None)

    return result
예제 #12
0
def thread_for_stack(start, end):
    process = lldb.debugger.GetSelectedTarget().GetProcess()
    for thread in process:
        if start <= thread.frame[0].sp and end >= thread.frame[0].sp:
            descriptions = [
                str(thread.GetIndexID()),
                thread.GetQueueName(),
                thread.GetName()
            ]
            return " thread %s" % " ".join(compact(descriptions))
    return ""
예제 #13
0
def upgrade():
    tags_regex = re.compile('^([\w\s]+):|#([\w-]+)', re.I | re.U)
    connection = op.get_bind()

    dashboards = connection.execute("SELECT id, name FROM dashboards")

    update_query = text("UPDATE dashboards SET tags = :tags WHERE id = :id")
    
    for dashboard in dashboards:
        tags = compact(flatten(tags_regex.findall(dashboard[1])))
        if tags:
            connection.execute(update_query, tags=tags, id=dashboard[0])
def upgrade():
    tags_regex = re.compile("^([\w\s]+):|#([\w-]+)", re.I | re.U)
    connection = op.get_bind()

    dashboards = connection.execute("SELECT id, name FROM dashboards")

    update_query = text("UPDATE dashboards SET tags = :tags WHERE id = :id")

    for dashboard in dashboards:
        tags = compact(flatten(tags_regex.findall(dashboard[1])))
        if tags:
            connection.execute(update_query, tags=tags, id=dashboard[0])
예제 #15
0
async def fetch(url, *, headers=None, proxy=None, timeout=sentinel):
    default_headers = settings.get('headers', {})
    headers = compact(merge(default_headers, headers or {}))

    session = SESSION.get()
    async with session.get(url, headers=headers, proxy=proxy, timeout=timeout) as response:
        body = await response.text()
        return Response(
            method=response.method, url=str(response.url), body=body,
            status=response.status, reason=response.reason,
            headers=dict(response.headers)
        )
예제 #16
0
    def add(self, config, strip_app_name=False, filter_by_app_name=False,
            key_normalisation_func=default_key_normalisation_func):
        """
        Add a dict of config data. Values from later dicts will take precedence
        over those added earlier, so the order data is added matters.

        Note: Double underscores can be used to indicate dict key name
        boundaries. i.e. if we have a dict like:

        {
            'logging': {
                'level': INFO
                ...
            }
        }

        we could pass an environment variable LOGGING__LEVEL=DEBUG to override
        the log level.

        Note: Key names will be normalised by recursively applying the
        key_normalisation_func function. By default this will:

            1) Convert keys to lowercase
            2) Replace hyphens with underscores
            3) Strip leading underscores

        This allows key names from different sources (e.g. CLI args, env vars,
        etc.) to be able to override each other.

        :param config dict: config data
        :param strip_app_name boolean: If True, the configured app_name will
        stripped from the start of top-level input keys if present.
        :param filter_by_app_name boolean: If True, keys that don't begin with
        the app name will be discarded.
        :return:
        """
        config = walk_recursive(key_normalisation_func, OrderedDict(config))

        if filter_by_app_name:
            config = funcy.compact(funcy.select_keys(
                lambda k: k.startswith(self._app_name), config))

        if strip_app_name:
            strip_app_name_regex = re.compile("^%s" % self._app_name)
            config = funcy.walk_keys(
                lambda k: re.sub(strip_app_name_regex, '', k), config)

        self._sources.append(config)

        return self             # enables a fluent interface
예제 #17
0
파일: diff.py 프로젝트: samlex20/dvc
 def run(self):
     try:
         msg = self.repo.diff(self.args.a_ref,
                              target=self.args.target,
                              b_ref=self.args.b_ref)
         self._show(msg)
     except DvcException:
         msg = "failed to get 'diff {}'"
         args = " ".join(
             compact([self.args.target, self.args.a_ref, self.args.b_ref]))
         msg = msg.format(args)
         logger.exception(msg)
         return 1
     return 0
예제 #18
0
def guessdesc(value: Any) -> str:
    "Guesstimate a plausible description of a value."
    denoms = funcy.compact([
        getattr(value, 'name', ''),
        getattr(value, 'desc', ''),
        getattr(value, 'id', '')
    ])
    named = denoms[0] if len(denoms) == 1 else ''
    spec = value.__class__.__name__
    if named:
        desc = f'{named}:{spec}'
        return desc
    else:
        desc = f'{value!r}:{spec}'
        return desc
예제 #19
0
    def run(self):
        from dvc.command.stage import parse_cmd

        cmd = parse_cmd(self.args.cmd)
        if not self.args.interactive and not cmd:
            raise InvalidArgumentError("command is not specified")

        from dvc.repo.experiments.init import init

        global_defaults = {
            "code": self.CODE,
            "data": self.DATA,
            "models": self.MODELS,
            "metrics": self.DEFAULT_METRICS,
            "params": self.DEFAULT_PARAMS,
            "plots": self.PLOTS,
            "live": self.DVCLIVE,
        }
        defaults = {}
        if not self.args.explicit:
            config = {}  # TODO
            defaults.update({**global_defaults, **config})

        cli_args = compact({
            "cmd": cmd,
            "code": self.args.code,
            "data": self.args.data,
            "models": self.args.models,
            "metrics": self.args.metrics,
            "params": self.args.params,
            "plots": self.args.plots,
            "live": self.args.live,
        })

        initialized_stage = init(
            self.repo,
            name=self.args.name,
            type=self.args.type,
            defaults=defaults,
            overrides=cli_args,
            interactive=self.args.interactive,
            force=self.args.force,
        )
        if self.args.run:
            return self.repo.experiments.run(
                targets=[initialized_stage.addressing])
        return 0
예제 #20
0
def init_interactive(
    name: str,
    defaults: Dict[str, str],
    provided: Dict[str, str],
    show_tree: bool = False,
    live: bool = False,
) -> Dict[str, str]:
    primary = lremove(provided.keys(),
                      ["cmd", "code", "data", "models", "params"])
    secondary = lremove(provided.keys(),
                        ["live"] if live else ["metrics", "plots"])

    if not (primary or secondary):
        return {}

    message = ui.rich_text.assemble(
        "This command will guide you to set up a ",
        (name, "bright_blue"),
        " stage in ",
        ("dvc.yaml", "green"),
        ".",
    )
    doc_link = ui.rich_text.assemble("See ", (PIPELINE_FILE_LINK, "repr.url"),
                                     ".")
    ui.error_write(message, doc_link, "", sep="\n", styled=True)

    if show_tree:
        from rich.tree import Tree

        tree = Tree(
            "DVC assumes the following workspace structure:",
            highlight=True,
        )
        workspace = {**defaults, **provided}
        workspace.pop("cmd", None)
        if not live and "live" not in provided:
            workspace.pop("live", None)
        for value in sorted(workspace.values()):
            tree.add(f"[green]{value}[/green]")
        ui.error_write(tree, styled=True)
        ui.error_write()

    return compact({
        **_prompts(primary, defaults),
        **_prompts(secondary, defaults),
    })
예제 #21
0
 def get_ContentItemDto(contentType, text=None, picture=None, dealContent=None, flag_compact=True):
     """ Элемент сообщения.
     В зависимости от тип содержимого должно быть заполнено один из др.параметров.
     :param contentType: тип контента, type(ContentType)
     :param text: текст, type(TextContentDto)
     :param picture: изоюражение, type(PictureContentDto)
     :param dealContent: сделка, type(DealContentDto)
     :param flag_compact: флаг определяет будут ли отсылаться параметры со значением None
     :return:
     """
     # todo: deprecated
     params = dict(contentType=contentType, text=text, picture=picture, dealContent=dealContent)
     if flag_compact is True:
         params = funcy.compact(params)
     content_item = ContentItemDto(**params)
     service_log.put("Created ContentItemDto: %s" % str(content_item))
     return content_item
예제 #22
0
 def __call__(self, *args, **kwargs):
     """ Подменяем вызов функции на её динамическое использование.
     :param args: аргументы передаваемые функции
     :param kwargs: аргументы передаваемые функции
     :return:
     """
     call_m = self.__iterable_exec(self.name_exc_item, self.conns)
     result = [index(*args, **kwargs) for index in call_m]
     result_c = funcy.compact(result)
     if len(result_c) >= 2:
         msg_error = "Found several response!"
         service_log.error(msg_error)
         raise AssertionError(msg_error)
     else:
         service_log.put("Get response from nutcracker.")
         if len(result_c) == 0:
             return result_c
         else:
             return result_c[0]
예제 #23
0
 def __call__(self, *args, **kwargs):
     """ Подменяем вызов функции на её динамическое использование.
     :param args: аргументы передаваемые функции
     :param kwargs: аргументы передаваемые функции
     :return:
     """
     call_m = self.__iterable_exec(self.name_exc_item, self.conns)
     result = [index(*args, **kwargs) for index in call_m]
     result_c = funcy.compact(result)
     if len(result_c) >= 2:
         msg_error = "Found several response!"
         service_log.error(msg_error)
         raise AssertionError(msg_error)
     else:
         service_log.put("Get response from nutcracker.")
         if len(result_c) == 0:
             return result_c
         else:
             return result_c[0]
예제 #24
0
 def get_ContentItemDto(cont_type, text=None, pict_id=None, user_id=None, deal_cont=None, ware_id=None,
                        flag_compact=True):
     """ Элемент сообщения.
     В зависимости от тип содержимого должно быть заполнено один из др.параметров.
     :param cont_type: тип контента, type(ContentType)
     :param text: текст, type(TextContentDto)
     :param pict_id: идентификатор изоюражения
     :param user_id: идентификатор пользователя
     :param deal_cont: сделка, type(DealContentDto)
     :param ware_id: идентификатор товара
     :param flag_compact: флаг определяет будут ли отсылаться параметры со значением None
     :return: type(ContentItemDto)
     """
     params = dict(contentType=cont_type, text=text, pictureId=pict_id, userId=user_id, dealContent=deal_cont,
                   wareId=ware_id)
     if flag_compact is True:
         params = funcy.compact(params)
     content_item = ContentItemDto(**params)
     service_log.put("Created ContentItemDto: %s" % str(content_item))
     return content_item
예제 #25
0
def init_interactive(
    defaults: Dict[str, str],
    provided: Iterable[str],
    show_heading: bool = False,
    live: bool = False,
) -> Dict[str, str]:
    primary = lremove(provided, ["cmd", "code", "data", "models", "params"])
    secondary = lremove(provided, ["live"] if live else ["metrics", "plots"])

    if not (primary or secondary):
        return {}

    message = ("This command will guide you to set up your first stage in "
               "[green]dvc.yaml[/green].\n")
    if show_heading:
        ui.error_write(message, styled=True)

    return compact({
        **_prompts(primary, defaults),
        **_prompts(secondary, defaults),
    })
예제 #26
0
파일: init.py 프로젝트: pmrowla/dvc
    def run(self):
        from dvc.commands.stage import parse_cmd

        cmd = parse_cmd(self.args.command)
        if not self.args.interactive and not cmd:
            raise InvalidArgumentError("command is not specified")

        from dvc.repo.experiments.init import init

        defaults = {}
        if not self.args.explicit:
            config = self.repo.config["exp"]
            defaults.update({**self.DEFAULTS, **config})

        cli_args = compact({
            "cmd": cmd,
            "code": self.args.code,
            "data": self.args.data,
            "models": self.args.models,
            "metrics": self.args.metrics,
            "params": self.args.params,
            "plots": self.args.plots,
            "live": self.args.live,
        })

        initialized_stage, initialized_deps, initialized_out_dirs = init(
            self.repo,
            name=self.args.name,
            type=self.args.type,
            defaults=defaults,
            overrides=cli_args,
            interactive=self.args.interactive,
            force=self.args.force,
        )
        self._post_init_display(initialized_stage, initialized_deps,
                                initialized_out_dirs)
        if self.args.run:
            self.repo.experiments.run(targets=[initialized_stage.addressing])
        return 0
예제 #27
0
def csvdictstream(filename, encoding="utf-8", fields=None, stripchars="\r\n"):
    """Stream every line in the given file interpreting each line as a dictionary of fields to items.

    :param filename: A ``str`` filename, A ``py._path.local.LocalPath`` instance or open ``file`` instnace.
    :type filename:  ``str``, ``py._path.local.LocalPath`` or ``file``.

    :param encoding: A ``str`` indicating the charset/encoding to use.
    :type encoding:  ``str``

    :param stripchars: An iterable of characters to strip from the surrounding line. ``line.strip(...)`` is used.
    :type stripchars: ``list``, ``tuple`` or ``str``

    This is a wrapper around ``csvstream`` where the stream is treated as dict of field(s) to item(s).
    """

    stream = csvstream(filename, encoding=encoding, stripchars=stripchars)

    if fields is None:
        fields = map(strip, next(stream))

    for values in stream:
        yield compact(zipdict(fields, values))
예제 #28
0
def init(
    repo: "Repo",
    name: str = None,
    type: str = "default",  # pylint: disable=redefined-builtin
    defaults: Dict[str, str] = None,
    overrides: Dict[str, str] = None,
    interactive: bool = False,
    force: bool = False,
) -> "Stage":
    from dvc.dvcfile import make_dvcfile

    dvcfile = make_dvcfile(repo, "dvc.yaml")
    name = name or type

    _check_stage_exists(dvcfile, name, force=force)

    defaults = defaults or {}
    overrides = overrides or {}

    with_live = type == "live"
    if interactive:
        defaults = init_interactive(
            defaults=defaults or {},
            show_heading=not dvcfile.exists(),
            live=with_live,
            provided=overrides.keys(),
        )
    else:
        if with_live:
            # suppress `metrics`/`params` if live is selected, unless
            # it is also provided via overrides/cli.
            # This makes output to be a checkpoint as well.
            defaults.pop("metrics")
            defaults.pop("params")
        else:
            defaults.pop("live")  # suppress live otherwise

    context: Dict[str, str] = {**defaults, **overrides}
    assert "cmd" in context

    params_kv = []
    if context.get("params"):
        from dvc.utils.serialize import LOADERS

        path = context["params"]
        assert isinstance(path, str)
        _, ext = os.path.splitext(path)
        params_kv = [{path: list(LOADERS[ext](path))}]

    checkpoint_out = bool(context.get("live"))
    models = context.get("models")
    return repo.stage.add(
        name=name,
        cmd=context["cmd"],
        deps=compact([context.get("code"),
                      context.get("data")]),
        params=params_kv,
        metrics_no_cache=compact([context.get("metrics")]),
        plots_no_cache=compact([context.get("plots")]),
        live=context.get("live"),
        force=force,
        **{"checkpoints" if checkpoint_out else "outs": compact([models])},
    )
예제 #29
0
def assert_valid_bst(mode, ixy_map,
                     ixy_arr, tree, n_inserted, n_node):
    ''' tree is bst '''
    key = prop(mode)
    # Num of leaves ixy ref = num of inserted ixys
    # Parent must be positive value except root.
    for i,node in enumerate(tree[1:n_inserted+1]):
        assert node.parent >= 0, (n_inserted, i, pyobj(node))
                               
    #   Get ixy idxes from tree structure
    ixy_idxes = all_ixy_idxes(
        #tup_tree(tree[:n_inserted+50]))
        tup_tree(tree[:n_node+100]))
    if DBG: print(f' after[{n_node}]',#tup_tree(tree[:n_node+10]))
                [f'{p} {l} {r}' for _,p,l,r in 
                tup_tree(tree[:n_node+10])])###########
    if DBG: print('iidxes', ixy_idxes)
    if DBG: print('n_node =',n_node)
    # Inserted number of ixys preserved?
    no0idxes = F.compact([abs(i) for i in ixy_idxes])
    assert n_inserted == len(no0idxes), \
        'ixy_idxes = {}, tup_tree = {}'.format(
            ixy_idxes, tup_tree(tree[:n_inserted+4]))
    # All ixy have unique index.
    assert len(set(no0idxes)) == n_inserted,\
        f'{len(set(no0idxes))} == {n_inserted}'
    # All leaves point ixy(neg idx), not inode.
    assert all(idx <= 0 for idx in ixy_idxes), \
        'ixy_idxes = {}, tree = {}'.format(
            ixy_idxes, tup_tree(tree[:n_inserted+4]))

    # Inserted ixys are sorted in ascending order.
    inserted_ixys = F.lmap(
        lambda i: ixy_arr[abs(i)], ixy_idxes)
    for ixy1, ixy2 in F.pairwise(inserted_ixys): 
        assert key(ixy1) <= key(ixy2), 'tree = {}' \
            .format(tup_tree(tree[:n_inserted+4]))

    # All leaves: l <= r
    leaves = F.lfilter(is_leaf, tree[:n_inserted+4])
    for leaf in leaves:
        l = leaf.left; r = leaf.right
        if l and r:
            l_val = key(ixy_map[abs(l)])
            r_val = key(ixy_map[abs(r)])
            assert l_val <= r_val  

    # All inodes must be sorted in ascending order.
    inodes = all_inodes(tup_tree(tree[:n_node+100]))
    for n1, n2 in F.pairwise(inodes):
        k1 = n1[0]; k2 = n2[0]
        assert k1 <= k2

    # Inserted ixys are sorted in ascending order.
    neg_idxeseq = F.mapcat(tup(
        lambda k,p,l,r: 
        ((l,) if l < 0 else ()) + ((r,) if r < 0 else ())),
        inodes)
    ixy_idxes = F.map(abs, neg_idxeseq)
    saved_ixys = F.map(lambda i: pyobj(ixy_arr[i]), ixy_idxes)
    keys = F.lmap(key, saved_ixys)
    for k1,k2 in F.pairwise(keys):
        assert k1 <= k2
예제 #30
0
 def __call__(self, *params, **opts):
     root = self.rep
     return funcy.compact(self.truthy)
예제 #31
0
파일: experiments.py 프로젝트: gcoter/dvc
    def run(self):
        from dvc.command.stage import parse_cmd

        cmd = parse_cmd(self.args.cmd)
        if not self.args.interactive and not cmd:
            raise InvalidArgumentError("command is not specified")

        from dvc.dvcfile import make_dvcfile

        global_defaults = {
            "code": self.CODE,
            "data": self.DATA,
            "models": self.MODELS,
            "metrics": self.DEFAULT_METRICS,
            "params": self.DEFAULT_PARAMS,
            "plots": self.PLOTS,
            "live": self.DVCLIVE,
        }

        dvcfile = make_dvcfile(self.repo, "dvc.yaml")
        name = self.args.name or self.args.type

        dvcfile_exists = dvcfile.exists()
        if not self.args.force and dvcfile_exists and name in dvcfile.stages:
            from dvc.stage.exceptions import DuplicateStageName

            hint = "Use '--force' to overwrite."
            raise DuplicateStageName(
                f"Stage '{name}' already exists in 'dvc.yaml'. {hint}"
            )

        context = ChainMap()
        if not self.args.explicit:
            config = {}  # TODO
            context.maps.extend([config, global_defaults])

        with_live = self.args.type == "live"
        if self.args.interactive:
            try:
                context = self.init_interactive(
                    defaults=context,
                    show_heading=not dvcfile_exists,
                    live=with_live,
                )
            except (KeyboardInterrupt, EOFError):
                ui.error_write()
                raise
        elif with_live:
            # suppress `metrics`/`params` if live is selected, unless
            # also provided via cli, also make output to be a checkpoint.
            context = context.new_child({"metrics": None, "params": None})
        else:
            # suppress live otherwise
            context = context.new_child({"live": None})

        if not self.args.interactive:
            d = compact(
                {
                    "cmd": cmd,
                    "code": self.args.code,
                    "data": self.args.data,
                    "models": self.args.models,
                    "metrics": self.args.metrics,
                    "params": self.args.params,
                    "plots": self.args.plots,
                    "live": self.args.live,
                }
            )
            context = context.new_child(d)

        assert "cmd" in context
        command = context["cmd"]
        code = context.get("code")
        data = context.get("data")
        models = context.get("models")
        metrics = context.get("metrics")
        plots = context.get("plots")
        live = context.get("live")

        params_kv = []
        if context.get("params"):
            from dvc.utils.serialize import LOADERS

            path = context["params"]
            _, ext = os.path.splitext(path)
            params_kv = [{path: list(LOADERS[ext](path))}]

        checkpoint_out = bool(context.get("live"))
        stage = self.repo.stage.add(
            name=name,
            cmd=command,
            deps=compact([code, data]),
            params=params_kv,
            metrics_no_cache=compact([metrics]),
            plots_no_cache=compact([plots]),
            live=live,
            force=self.args.force,
            **{"checkpoints" if checkpoint_out else "outs": compact([models])},
        )

        if self.args.run:
            return self.repo.experiments.run(targets=[stage.addressing])
        return 0
예제 #32
0
def init(
    repo: "Repo",
    name: str = None,
    type: str = "default",  # pylint: disable=redefined-builtin
    defaults: Dict[str, str] = None,
    overrides: Dict[str, str] = None,
    interactive: bool = False,
    force: bool = False,
    stream: Optional[TextIO] = None,
) -> "Stage":
    from dvc.dvcfile import make_dvcfile

    dvcfile = make_dvcfile(repo, "dvc.yaml")
    name = name or type

    _check_stage_exists(dvcfile, name, force=force)

    defaults = defaults.copy() if defaults else {}
    overrides = overrides.copy() if overrides else {}

    with_live = type == "live"

    if interactive:
        defaults = init_interactive(
            name,
            validator=validate_prompts,
            defaults=defaults,
            live=with_live,
            provided=overrides,
            stream=stream,
        )
    else:
        if with_live:
            # suppress `metrics`/`plots` if live is selected, unless
            # it is also provided via overrides/cli.
            # This makes output to be a checkpoint as well.
            defaults.pop("metrics", None)
            defaults.pop("plots", None)
        else:
            defaults.pop("live", None)  # suppress live otherwise

    context: Dict[str, str] = {**defaults, **overrides}
    assert "cmd" in context

    params_kv = []
    params = context.get("params")
    if params:
        params_kv.append(loadd_params(params))

    checkpoint_out = bool(context.get("live"))
    models = context.get("models")
    stage = repo.stage.create(
        name=name,
        cmd=context["cmd"],
        deps=compact([context.get("code"),
                      context.get("data")]),
        params=params_kv,
        metrics_no_cache=compact([context.get("metrics")]),
        plots_no_cache=compact([context.get("plots")]),
        live=context.get("live"),
        force=force,
        **{"checkpoints" if checkpoint_out else "outs": compact([models])},
    )

    if interactive:
        ui.error_write(Rule(style="green"), styled=True)
        _yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage)))
        syn = Syntax(_yaml, "yaml", theme="ansi_dark")
        ui.error_write(syn, styled=True)

    from dvc.ui.prompt import Confirm

    if not interactive or Confirm.ask(
            "Do you want to add the above contents to dvc.yaml?",
            console=ui.error_console,
            default=True,
            stream=stream,
    ):
        with _disable_logging(), repo.scm_context(autostage=True, quiet=True):
            stage.dump(update_lock=False)
            stage.ignore_outs()
            if params:
                repo.scm_context.track_file(params)
    else:
        raise DvcException("Aborting ...")
    return stage
예제 #33
0
def mama(target, options, args, result):
    ret = []
    needle = None
    if len(args):
        needle = exp(" ".join(args))
    arch = trgt()
    expr_options = lldb.SBExpressionOptions()
    expr_options.SetIgnoreBreakpoints(True)
    expr_options.SetFetchDynamicValue(lldb.eNoDynamicValues)
    expr_options.SetTimeoutInMicroSeconds(30 * 1000 *
                                          1000)  # 30 second timeout
    expr_options.SetTryAllThreads(False)
    max_results = 0x1000
    frame = lldb.debugger.GetSelectedTarget().GetProcess().GetSelectedThread(
    ).GetSelectedFrame()
    expr_sbvalue = frame.EvaluateExpression(
        mach_vm_region_expr() % (arch['extra'](), max_results), expr_options)
    if expr_sbvalue.error.Success():
        result_value = lldb.value(expr_sbvalue)
        starts = []
        malloc_tags = [1, 2, 3, 4, 6, 7, 8, 9, 11, 53, 61]
        for i in range(max_results):
            region_lol = result_value[i]
            start = int(region_lol.addr)
            size = int(region_lol.size)
            user_tag_int = int(region_lol.info.user_tag)
            if user_tag_int in malloc_tags and (not needle or in_region(
                    needle, start, size)):
                starts.append(start)
        _zones = {}
        if len(starts):
            _zones = zone(starts)

        reached_the_end = False
        for i in range(max_results):
            region_lol = result_value[i]
            size = int(region_lol.size)
            start = int(region_lol.addr)
            if start >= arch['shared_region_base']:
                reached_the_end = True
            end = start + size
            info = region_lol.info
            prot = int(info.protection)
            user_tag_int = int(info.user_tag)
            # from /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.8.sdk/System/Library/Frameworks/Kernel.framework/Headers/mach/vm_statistics.h
            # I only put in the ones I ran into
            user_tagz = {
                0: "unknown",
                1: "malloc",
                2: "malloc small",
                3: "malloc large",
                4: "malloc huge",
                # 5: "sbrk",
                6: "realloc",
                7: "malloc tiny",
                8: "malloc large reusable",
                9: "malloc large reused",
                10: "memory analysis tool",
                11: "malloc nano",
                20: "mach msg",
                21: "iokit",
                30: "stack",
                31: "guard",
                32: "shared pmap",
                33: "dylib",
                34: "objc dispatchers",
                35: "unshared pmap",
                40: "appkit",
                41: "foundation",
                42: "core graphics",
                43: "core services",
                44: "java",
                45: "coredata",
                46: "coredata objectids",
                50: "ats",
                51: "layerkit",
                52: "cgimage",
                53: "tcmalloc",
                54: "coregraphics data",
                55: "coregraphics shared",
                56: "coregraphics framebuffers",
                57: "coregraphics backingstores",
                60: "dyld",
                61: "dyld mallco",
                62: "sqlite",
                63: "javascript core heap",
                64: "javascript JIT executable allocator",
                65: "javascript JIT register file",
                66: "GLSL",
                67: "OpenCL",
                68: "core image",
                69: "webcore purgeable buffers",
                70: "imageio",
                71: "coreprofile",
                72: "assetsd / MobileSlideShow",
                73: "kernel alloc once",
                74: "libdispatch",
                75: "accelerate",
                76: "coreui",
                242: "application specific 242",
                251: "application specific 251"
            }
            user_tag = user_tagz.get(user_tag_int)
            if user_tag_int == 30:
                user_tag = "".join([user_tag, thread_for_stack(start, end)])
            if not user_tag:
                print "USER TAG NOT FOUDN: %s" % user_tag_int
            ref_count = int(info.ref_count)
            object_name = int(region_lol.object_name)
            if object_name != 0:
                print "YEHAHA %#x" % object_name
            share_mode = int(info.share_mode)
            share_modez = [
                '????', 'COW', 'PRV', 'NUL', 'ALI', 'SHM', 'ZER', 'S/A',
                'large page'
            ]
            share_mode = share_modez[share_mode]
            if start < arch['shared_region_base']:
                lookup_output = run_command(
                    "target modules lookup -v -a %#x" % start, True)
            elif needle:
                lookup_output = run_command(
                    "target modules lookup -v -a %#x" % needle, True)
            filename = ""
            section = ""
            if lookup_output:
                if user_tag_int != 0 and user_tag_int != 33 and user_tag_int != 35 and user_tag_int != 32:
                    print "oh shit! %d" % user_tag_int
                    break
                filename = re_find(r"file = \"([^\"]*)", lookup_output)
                filename = re.sub(r".*/SDKs", "...", filename)
                filename = re.sub(r".*/iOS DeviceSupport", "...", filename)
                section = re_find(r"__[\w]*", lookup_output)
                if section and filename and user_tag_int == 0:
                    user_tag = ""
            else:
                if user_tag_int == 33:
                    print "noh shit! %#x" % user_tag_int
                    break
            maybe_guard = "guard" if prot == 0 and user_tag_int == 30 else ""
            maybe_unallocated = "" if ref_count else "(zero refcount)"
            maybes = [maybe_guard, maybe_unallocated][not maybe_guard]
            if size == 0xe800000 and user_tag_int == 32 and ref_count == 0:
                reached_the_end = True
            else:
                if not needle or in_region(needle, start, size):
                    _zone = _zones[start] if user_tag_int in malloc_tags else ""
                    ret.append(
                        (start, size,
                         "{:#x} - {:#x} - {:<5s} - {:} SM={:} {:}".format(
                             start, end, human(size), prot_string(prot),
                             share_mode, " ".join(
                                 compact([
                                     section, user_tag, maybes, filename, _zone
                                 ])))))
            if share_mode == "NUL" and ref_count == 0 and prot == 5:
                break
        if not reached_the_end:
            print "didn't reach the end?"
        return ret
    else:
        print "failure, %s" % expr_sbvalue.error
예제 #34
0
파일: init.py 프로젝트: pmrowla/dvc
def init(
    repo: "Repo",
    name: str = "train",
    type: str = "default",  # pylint: disable=redefined-builtin
    defaults: Dict[str, str] = None,
    overrides: Dict[str, str] = None,
    interactive: bool = False,
    force: bool = False,
    stream: Optional[TextIO] = None,
) -> Tuple[PipelineStage, List["Dependency"], List[str]]:
    from dvc.dvcfile import make_dvcfile

    dvcfile = make_dvcfile(repo, "dvc.yaml")
    _check_stage_exists(dvcfile, name, force=force)

    defaults = defaults.copy() if defaults else {}
    overrides = overrides.copy() if overrides else {}

    if interactive:
        defaults = init_interactive(
            validator=partial(validate_prompts, repo),
            defaults=defaults,
            provided=overrides,
            stream=stream,
        )
    else:
        if "live" in overrides:
            # suppress `metrics`/`plots` if live is selected.
            defaults.pop("metrics", None)
            defaults.pop("plots", None)
        else:
            defaults.pop("live", None)  # suppress live otherwise

    context: Dict[str, str] = {**defaults, **overrides}
    assert "cmd" in context

    params = context.get("params")
    if params:
        from dvc.dependency.param import (
            MissingParamsFile,
            ParamsDependency,
            ParamsIsADirectoryError,
        )

        try:
            ParamsDependency(None, params, repo=repo).validate_filepath()
        except ParamsIsADirectoryError as exc:
            raise DvcException(f"{exc}.")  # swallow cause for display
        except MissingParamsFile:
            pass

    models = context.get("models")
    live_path = context.pop("live", None)
    live_metrics = f"{live_path}.json" if live_path else None
    live_plots = os.path.join(live_path, "scalars") if live_path else None

    stage = repo.stage.create(
        name=name,
        cmd=context["cmd"],
        deps=compact([context.get("code"),
                      context.get("data")]),
        params=[{
            params: None
        }] if params else None,
        metrics_no_cache=compact([context.get("metrics"), live_metrics]),
        plots_no_cache=compact([context.get("plots"), live_plots]),
        force=force,
        **{
            "checkpoints" if type == "checkpoint" else "outs":
            compact([models])
        },
    )

    with _disable_logging(), repo.scm_context(autostage=True, quiet=True):
        stage.dump(update_lock=False)
        initialized_out_dirs = init_out_dirs(stage)
        stage.ignore_outs()
        initialized_deps = init_deps(stage)
        if params:
            repo.scm_context.track_file(params)

    assert isinstance(stage, PipelineStage)
    return stage, initialized_deps, initialized_out_dirs
예제 #35
0
파일: json_ds.py 프로젝트: zxct/redash
def _sort_columns_with_fields(columns, fields):
    if fields:
        columns = compact(
            [_get_column_by_name(columns, field) for field in fields])

    return columns
    data_dict.get('fpid', None),
    data_dict.get('user_id', None)
] if item), None)

# Method 2
# Notice that this has Python recursive limitation


def until(terminate, iterator, default):
    if isinstance(iterator, list):
        iterator = iter(iterator)
    try:
        i = next(iterator)
        if terminate(i):
            return i
        return until(terminate, iterator, default)
    except StopIteration:
        return default


# candidates = [None, None, None, None]
candidates = [None, None, '25', None]
id = until(lambda x: x is not None, iter(candidates), '1')

# Method 3
# Introducing FP(Functional Programming)

from funcy import first, compact
candidates = [None, None, '25', None]
id = first(compact(candidates))  # 25
예제 #37
0
def init(
    repo: "Repo",
    name: str = None,
    type: str = "default",  # pylint: disable=redefined-builtin
    defaults: Dict[str, str] = None,
    overrides: Dict[str, str] = None,
    interactive: bool = False,
    force: bool = False,
) -> "Stage":
    from dvc.dvcfile import make_dvcfile

    dvcfile = make_dvcfile(repo, "dvc.yaml")
    name = name or type

    _check_stage_exists(dvcfile, name, force=force)

    defaults = defaults or {}
    overrides = overrides or {}

    with_live = type == "live"
    if interactive:
        defaults = init_interactive(
            name,
            defaults=defaults,
            live=with_live,
            provided=overrides,
            show_tree=True,
        )
    else:
        if with_live:
            # suppress `metrics`/`params` if live is selected, unless
            # it is also provided via overrides/cli.
            # This makes output to be a checkpoint as well.
            defaults.pop("metrics")
            defaults.pop("params")
        else:
            defaults.pop("live")  # suppress live otherwise

    context: Dict[str, str] = {**defaults, **overrides}
    assert "cmd" in context

    params_kv = []
    if context.get("params"):
        from dvc.utils.serialize import LOADERS

        path = context["params"]
        assert isinstance(path, str)
        _, ext = os.path.splitext(path)
        params_kv = [{path: list(LOADERS[ext](path))}]

    checkpoint_out = bool(context.get("live"))
    models = context.get("models")
    stage = repo.stage.create(
        name=name,
        cmd=context["cmd"],
        deps=compact([context.get("code"),
                      context.get("data")]),
        params=params_kv,
        metrics_no_cache=compact([context.get("metrics")]),
        plots_no_cache=compact([context.get("plots")]),
        live=context.get("live"),
        force=force,
        **{"checkpoints" if checkpoint_out else "outs": compact([models])},
    )

    if interactive:
        ui.write(Rule(style="green"), styled=True)
        _yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage)))
        syn = Syntax(_yaml, "yaml", theme="ansi_dark")
        ui.error_write(syn, styled=True)

    if not interactive or ui.confirm(
            "Do you want to add the above contents to dvc.yaml?"):
        with _disable_logging():
            stage.dump(update_lock=False)
        stage.ignore_outs()
    else:
        raise DvcException("Aborting ...")
    return stage