def init_interactive(
name: str,
defaults: Dict[str, str],
provided: Dict[str, str],
validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None,
live: bool = False,
stream: Optional[TextIO] = None,
) -> Dict[str, str]:
command = provided.pop("cmd", None)
primary = lremove(provided.keys(), ["code", "data", "models", "params"])
secondary = lremove(provided.keys(),
["live"] if live else ["metrics", "plots"])
prompts = primary + secondary
workspace = {**defaults, **provided}
if not live and "live" not in provided:
workspace.pop("live", None)
for key in ("plots", "metrics"):
if live and key not in provided:
workspace.pop(key, None)
ret: Dict[str, str] = {}
if command:
ret["cmd"] = command
if not prompts and command:
return ret
ui.error_write(
f"This command will guide you to set up a [bright_blue]{name}[/]",
"stage in [green]dvc.yaml[/].",
f"\nSee [repr.url]{PIPELINE_FILE_LINK}[/].\n",
styled=True,
)
if not command:
ret.update(
compact(_prompts(["cmd"], allow_omission=False, stream=stream)))
if prompts:
ui.error_write(styled=True)
if not prompts:
return ret
ui.error_write(
"Enter the paths for dependencies and outputs of the command.",
styled=True,
)
if workspace:
ui.error_write(build_workspace_tree(workspace), styled=True)
ui.error_write(styled=True)
ret.update(
compact(_prompts(prompts, defaults, validator=validator,
stream=stream)))
return ret
def init_interactive(
defaults: Dict[str, str],
provided: Dict[str, str],
validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None,
stream: Optional[TextIO] = None,
) -> Dict[str, str]:
command_prompts = lremove(provided.keys(), ["cmd"])
dependencies_prompts = lremove(provided.keys(), ["code", "data", "params"])
output_keys = ["models"]
if "live" not in provided:
output_keys.extend(["metrics", "plots"])
outputs_prompts = lremove(provided.keys(), output_keys)
ret: Dict[str, str] = {}
if "cmd" in provided:
ret["cmd"] = provided["cmd"]
for heading, prompts, allow_omission in (
("", command_prompts, False),
("Enter experiment dependencies.", dependencies_prompts, True),
("Enter experiment outputs.", outputs_prompts, True),
):
if prompts and heading:
ui.error_write(heading, styled=True)
response = _prompts(
prompts,
defaults=defaults,
allow_omission=allow_omission,
validator=validator,
stream=stream,
)
ret.update(compact(response))
if prompts:
ui.error_write(styled=True)
return ret
def normalize(string: str):
words = compact(RE_SPLIT(string.lower()))
for word in words:
if RE_CYRILLIC(word):
yield word.translate(NORM)
else:
yield word
def run(self):
from dvc.command.stage import parse_cmd
cmd = parse_cmd(self.args.command)
if not self.args.interactive and not cmd:
raise InvalidArgumentError("command is not specified")
from dvc.repo.experiments.init import init
defaults = {}
if not self.args.explicit:
config = self.repo.config["exp"]
defaults.update({**self.DEFAULTS, **config})
cli_args = compact({
"cmd": cmd,
"code": self.args.code,
"data": self.args.data,
"models": self.args.models,
"metrics": self.args.metrics,
"params": self.args.params,
"plots": self.args.plots,
"live": self.args.live,
})
initialized_stage = init(
self.repo,
name=self.args.name,
type=self.args.type,
defaults=defaults,
overrides=cli_args,
interactive=self.args.interactive,
force=self.args.force,
)
text = ui.rich_text.assemble(
"\n" if self.args.interactive else "",
"Created ",
(self.args.name, "bright_blue"),
" stage in ",
("dvc.yaml", "green"),
".",
)
if not self.args.run:
text.append_text(
ui.rich_text.assemble(
" To run, use ",
('"dvc exp run"', "green"),
".\nSee ",
(self.EXP_LINK, "repr.url"),
".",
))
ui.write(text, styled=True)
if self.args.run:
return self.repo.experiments.run(
targets=[initialized_stage.addressing])
return 0
async def expand_multiple_items(item_list, relations):
for field, (type_, subfields) in relations.items():
path = field.split('.')
field = path[-1]
items_with_field = _find_items_having_path(item_list, path)
related_ids = list(compact(pluck(field, items_with_field)))
if not related_ids:
continue
related_items = await fetch_multiple_items(type_, related_ids, subfields)
related_by_id = {item['id']: item for item in compact(related_items)}
for item in items_with_field:
item[field] = related_by_id.get(item[field])
return item_list
def __call__(self, *params: Any) -> Any:
self.truthy = []
super().__call__(*params)
passed = funcy.compact(self.truthy)
if len(passed) == self.conjunctions:
return self.format(passed[-1])
else:
return False
def _run_cur(self) -> ASTQuery:
"Runs current Query."
root = self.rep
if len(self.ops) > 0:
return funcy.compact(self.truthy)
else:
raise ValueError("INTERNAL: Snack._run_cur() called without query "
"terms specified")
async def expand_multiple_items(item_list, relations):
for field, (type_, subfields) in relations.items():
path = field.split('.')
field = path[-1]
items_with_field = _find_items_having_path(item_list, path)
related_ids = list(compact(pluck(field, items_with_field)))
if not related_ids:
continue
related_items = await fetch_multiple_items(type_, related_ids,
subfields)
related_by_id = {item['id']: item for item in compact(related_items)}
for item in items_with_field:
item[field] = related_by_id.get(item[field])
return item_list
def __init__(self, host, port, name, user=None, passwd=None):
# параметры аутетификации к БД
self.auth = {"host": host, "port": port, "db": name, "user": user, "password": passwd}
self.auth = funcy.compact(self.auth)
service_log.put("Get params for authenticate with redis: %s" % self.auth)
# параметры для работы внутри БД
self.pool = None
self.cursor = None
self.cursor_now = None
def _pack_remotes(conf):
# Drop empty sections
result = compact(conf)
# Transform remote.name -> 'remote "name"'
for name, val in conf["remote"].items():
result[f'remote "{name}"'] = val
result.pop("remote", None)
return result
def _pack_named(conf):
# Drop empty sections
result = compact(conf)
# Transform remote.name -> 'remote "name"'
for key in ("remote", "machine"):
for name, val in conf[key].items():
result[f'{key} "{name}"'] = val
result.pop(key, None)
return result
def thread_for_stack(start, end):
process = lldb.debugger.GetSelectedTarget().GetProcess()
for thread in process:
if start <= thread.frame[0].sp and end >= thread.frame[0].sp:
descriptions = [
str(thread.GetIndexID()),
thread.GetQueueName(),
thread.GetName()
]
return " thread %s" % " ".join(compact(descriptions))
return ""
def upgrade():
tags_regex = re.compile('^([\w\s]+):|#([\w-]+)', re.I | re.U)
connection = op.get_bind()
dashboards = connection.execute("SELECT id, name FROM dashboards")
update_query = text("UPDATE dashboards SET tags = :tags WHERE id = :id")
for dashboard in dashboards:
tags = compact(flatten(tags_regex.findall(dashboard[1])))
if tags:
connection.execute(update_query, tags=tags, id=dashboard[0])
def upgrade():
tags_regex = re.compile("^([\w\s]+):|#([\w-]+)", re.I | re.U)
connection = op.get_bind()
dashboards = connection.execute("SELECT id, name FROM dashboards")
update_query = text("UPDATE dashboards SET tags = :tags WHERE id = :id")
for dashboard in dashboards:
tags = compact(flatten(tags_regex.findall(dashboard[1])))
if tags:
connection.execute(update_query, tags=tags, id=dashboard[0])
async def fetch(url, *, headers=None, proxy=None, timeout=sentinel):
default_headers = settings.get('headers', {})
headers = compact(merge(default_headers, headers or {}))
session = SESSION.get()
async with session.get(url, headers=headers, proxy=proxy, timeout=timeout) as response:
body = await response.text()
return Response(
method=response.method, url=str(response.url), body=body,
status=response.status, reason=response.reason,
headers=dict(response.headers)
)
def add(self, config, strip_app_name=False, filter_by_app_name=False,
key_normalisation_func=default_key_normalisation_func):
"""
Add a dict of config data. Values from later dicts will take precedence
over those added earlier, so the order data is added matters.
Note: Double underscores can be used to indicate dict key name
boundaries. i.e. if we have a dict like:
{
'logging': {
'level': INFO
...
}
}
we could pass an environment variable LOGGING__LEVEL=DEBUG to override
the log level.
Note: Key names will be normalised by recursively applying the
key_normalisation_func function. By default this will:
1) Convert keys to lowercase
2) Replace hyphens with underscores
3) Strip leading underscores
This allows key names from different sources (e.g. CLI args, env vars,
etc.) to be able to override each other.
:param config dict: config data
:param strip_app_name boolean: If True, the configured app_name will
stripped from the start of top-level input keys if present.
:param filter_by_app_name boolean: If True, keys that don't begin with
the app name will be discarded.
:return:
"""
config = walk_recursive(key_normalisation_func, OrderedDict(config))
if filter_by_app_name:
config = funcy.compact(funcy.select_keys(
lambda k: k.startswith(self._app_name), config))
if strip_app_name:
strip_app_name_regex = re.compile("^%s" % self._app_name)
config = funcy.walk_keys(
lambda k: re.sub(strip_app_name_regex, '', k), config)
self._sources.append(config)
return self # enables a fluent interface
def run(self):
try:
msg = self.repo.diff(self.args.a_ref,
target=self.args.target,
b_ref=self.args.b_ref)
self._show(msg)
except DvcException:
msg = "failed to get 'diff {}'"
args = " ".join(
compact([self.args.target, self.args.a_ref, self.args.b_ref]))
msg = msg.format(args)
logger.exception(msg)
return 1
return 0
def guessdesc(value: Any) -> str:
"Guesstimate a plausible description of a value."
denoms = funcy.compact([
getattr(value, 'name', ''),
getattr(value, 'desc', ''),
getattr(value, 'id', '')
])
named = denoms[0] if len(denoms) == 1 else ''
spec = value.__class__.__name__
if named:
desc = f'{named}:{spec}'
return desc
else:
desc = f'{value!r}:{spec}'
return desc
def run(self):
from dvc.command.stage import parse_cmd
cmd = parse_cmd(self.args.cmd)
if not self.args.interactive and not cmd:
raise InvalidArgumentError("command is not specified")
from dvc.repo.experiments.init import init
global_defaults = {
"code": self.CODE,
"data": self.DATA,
"models": self.MODELS,
"metrics": self.DEFAULT_METRICS,
"params": self.DEFAULT_PARAMS,
"plots": self.PLOTS,
"live": self.DVCLIVE,
}
defaults = {}
if not self.args.explicit:
config = {} # TODO
defaults.update({**global_defaults, **config})
cli_args = compact({
"cmd": cmd,
"code": self.args.code,
"data": self.args.data,
"models": self.args.models,
"metrics": self.args.metrics,
"params": self.args.params,
"plots": self.args.plots,
"live": self.args.live,
})
initialized_stage = init(
self.repo,
name=self.args.name,
type=self.args.type,
defaults=defaults,
overrides=cli_args,
interactive=self.args.interactive,
force=self.args.force,
)
if self.args.run:
return self.repo.experiments.run(
targets=[initialized_stage.addressing])
return 0
def init_interactive(
name: str,
defaults: Dict[str, str],
provided: Dict[str, str],
show_tree: bool = False,
live: bool = False,
) -> Dict[str, str]:
primary = lremove(provided.keys(),
["cmd", "code", "data", "models", "params"])
secondary = lremove(provided.keys(),
["live"] if live else ["metrics", "plots"])
if not (primary or secondary):
return {}
message = ui.rich_text.assemble(
"This command will guide you to set up a ",
(name, "bright_blue"),
" stage in ",
("dvc.yaml", "green"),
".",
)
doc_link = ui.rich_text.assemble("See ", (PIPELINE_FILE_LINK, "repr.url"),
".")
ui.error_write(message, doc_link, "", sep="\n", styled=True)
if show_tree:
from rich.tree import Tree
tree = Tree(
"DVC assumes the following workspace structure:",
highlight=True,
)
workspace = {**defaults, **provided}
workspace.pop("cmd", None)
if not live and "live" not in provided:
workspace.pop("live", None)
for value in sorted(workspace.values()):
tree.add(f"[green]{value}[/green]")
ui.error_write(tree, styled=True)
ui.error_write()
return compact({
**_prompts(primary, defaults),
**_prompts(secondary, defaults),
})
def get_ContentItemDto(contentType, text=None, picture=None, dealContent=None, flag_compact=True):
""" Элемент сообщения.
В зависимости от тип содержимого должно быть заполнено один из др.параметров.
:param contentType: тип контента, type(ContentType)
:param text: текст, type(TextContentDto)
:param picture: изоюражение, type(PictureContentDto)
:param dealContent: сделка, type(DealContentDto)
:param flag_compact: флаг определяет будут ли отсылаться параметры со значением None
:return:
"""
# todo: deprecated
params = dict(contentType=contentType, text=text, picture=picture, dealContent=dealContent)
if flag_compact is True:
params = funcy.compact(params)
content_item = ContentItemDto(**params)
service_log.put("Created ContentItemDto: %s" % str(content_item))
return content_item
def __call__(self, *args, **kwargs):
""" Подменяем вызов функции на её динамическое использование.
:param args: аргументы передаваемые функции
:param kwargs: аргументы передаваемые функции
:return:
"""
call_m = self.__iterable_exec(self.name_exc_item, self.conns)
result = [index(*args, **kwargs) for index in call_m]
result_c = funcy.compact(result)
if len(result_c) >= 2:
msg_error = "Found several response!"
service_log.error(msg_error)
raise AssertionError(msg_error)
else:
service_log.put("Get response from nutcracker.")
if len(result_c) == 0:
return result_c
else:
return result_c[0]
def __call__(self, *args, **kwargs):
""" Подменяем вызов функции на её динамическое использование.
:param args: аргументы передаваемые функции
:param kwargs: аргументы передаваемые функции
:return:
"""
call_m = self.__iterable_exec(self.name_exc_item, self.conns)
result = [index(*args, **kwargs) for index in call_m]
result_c = funcy.compact(result)
if len(result_c) >= 2:
msg_error = "Found several response!"
service_log.error(msg_error)
raise AssertionError(msg_error)
else:
service_log.put("Get response from nutcracker.")
if len(result_c) == 0:
return result_c
else:
return result_c[0]
def get_ContentItemDto(cont_type, text=None, pict_id=None, user_id=None, deal_cont=None, ware_id=None,
flag_compact=True):
""" Элемент сообщения.
В зависимости от тип содержимого должно быть заполнено один из др.параметров.
:param cont_type: тип контента, type(ContentType)
:param text: текст, type(TextContentDto)
:param pict_id: идентификатор изоюражения
:param user_id: идентификатор пользователя
:param deal_cont: сделка, type(DealContentDto)
:param ware_id: идентификатор товара
:param flag_compact: флаг определяет будут ли отсылаться параметры со значением None
:return: type(ContentItemDto)
"""
params = dict(contentType=cont_type, text=text, pictureId=pict_id, userId=user_id, dealContent=deal_cont,
wareId=ware_id)
if flag_compact is True:
params = funcy.compact(params)
content_item = ContentItemDto(**params)
service_log.put("Created ContentItemDto: %s" % str(content_item))
return content_item
def init_interactive(
defaults: Dict[str, str],
provided: Iterable[str],
show_heading: bool = False,
live: bool = False,
) -> Dict[str, str]:
primary = lremove(provided, ["cmd", "code", "data", "models", "params"])
secondary = lremove(provided, ["live"] if live else ["metrics", "plots"])
if not (primary or secondary):
return {}
message = ("This command will guide you to set up your first stage in "
"[green]dvc.yaml[/green].\n")
if show_heading:
ui.error_write(message, styled=True)
return compact({
**_prompts(primary, defaults),
**_prompts(secondary, defaults),
})
def run(self):
from dvc.commands.stage import parse_cmd
cmd = parse_cmd(self.args.command)
if not self.args.interactive and not cmd:
raise InvalidArgumentError("command is not specified")
from dvc.repo.experiments.init import init
defaults = {}
if not self.args.explicit:
config = self.repo.config["exp"]
defaults.update({**self.DEFAULTS, **config})
cli_args = compact({
"cmd": cmd,
"code": self.args.code,
"data": self.args.data,
"models": self.args.models,
"metrics": self.args.metrics,
"params": self.args.params,
"plots": self.args.plots,
"live": self.args.live,
})
initialized_stage, initialized_deps, initialized_out_dirs = init(
self.repo,
name=self.args.name,
type=self.args.type,
defaults=defaults,
overrides=cli_args,
interactive=self.args.interactive,
force=self.args.force,
)
self._post_init_display(initialized_stage, initialized_deps,
initialized_out_dirs)
if self.args.run:
self.repo.experiments.run(targets=[initialized_stage.addressing])
return 0
def csvdictstream(filename, encoding="utf-8", fields=None, stripchars="\r\n"):
"""Stream every line in the given file interpreting each line as a dictionary of fields to items.
:param filename: A ``str`` filename, A ``py._path.local.LocalPath`` instance or open ``file`` instnace.
:type filename: ``str``, ``py._path.local.LocalPath`` or ``file``.
:param encoding: A ``str`` indicating the charset/encoding to use.
:type encoding: ``str``
:param stripchars: An iterable of characters to strip from the surrounding line. ``line.strip(...)`` is used.
:type stripchars: ``list``, ``tuple`` or ``str``
This is a wrapper around ``csvstream`` where the stream is treated as dict of field(s) to item(s).
"""
stream = csvstream(filename, encoding=encoding, stripchars=stripchars)
if fields is None:
fields = map(strip, next(stream))
for values in stream:
yield compact(zipdict(fields, values))
def init(
repo: "Repo",
name: str = None,
type: str = "default", # pylint: disable=redefined-builtin
defaults: Dict[str, str] = None,
overrides: Dict[str, str] = None,
interactive: bool = False,
force: bool = False,
) -> "Stage":
from dvc.dvcfile import make_dvcfile
dvcfile = make_dvcfile(repo, "dvc.yaml")
name = name or type
_check_stage_exists(dvcfile, name, force=force)
defaults = defaults or {}
overrides = overrides or {}
with_live = type == "live"
if interactive:
defaults = init_interactive(
defaults=defaults or {},
show_heading=not dvcfile.exists(),
live=with_live,
provided=overrides.keys(),
)
else:
if with_live:
# suppress `metrics`/`params` if live is selected, unless
# it is also provided via overrides/cli.
# This makes output to be a checkpoint as well.
defaults.pop("metrics")
defaults.pop("params")
else:
defaults.pop("live") # suppress live otherwise
context: Dict[str, str] = {**defaults, **overrides}
assert "cmd" in context
params_kv = []
if context.get("params"):
from dvc.utils.serialize import LOADERS
path = context["params"]
assert isinstance(path, str)
_, ext = os.path.splitext(path)
params_kv = [{path: list(LOADERS[ext](path))}]
checkpoint_out = bool(context.get("live"))
models = context.get("models")
return repo.stage.add(
name=name,
cmd=context["cmd"],
deps=compact([context.get("code"),
context.get("data")]),
params=params_kv,
metrics_no_cache=compact([context.get("metrics")]),
plots_no_cache=compact([context.get("plots")]),
live=context.get("live"),
force=force,
**{"checkpoints" if checkpoint_out else "outs": compact([models])},
)
def assert_valid_bst(mode, ixy_map,
ixy_arr, tree, n_inserted, n_node):
''' tree is bst '''
key = prop(mode)
# Num of leaves ixy ref = num of inserted ixys
# Parent must be positive value except root.
for i,node in enumerate(tree[1:n_inserted+1]):
assert node.parent >= 0, (n_inserted, i, pyobj(node))
# Get ixy idxes from tree structure
ixy_idxes = all_ixy_idxes(
#tup_tree(tree[:n_inserted+50]))
tup_tree(tree[:n_node+100]))
if DBG: print(f' after[{n_node}]',#tup_tree(tree[:n_node+10]))
[f'{p} {l} {r}' for _,p,l,r in
tup_tree(tree[:n_node+10])])###########
if DBG: print('iidxes', ixy_idxes)
if DBG: print('n_node =',n_node)
# Inserted number of ixys preserved?
no0idxes = F.compact([abs(i) for i in ixy_idxes])
assert n_inserted == len(no0idxes), \
'ixy_idxes = {}, tup_tree = {}'.format(
ixy_idxes, tup_tree(tree[:n_inserted+4]))
# All ixy have unique index.
assert len(set(no0idxes)) == n_inserted,\
f'{len(set(no0idxes))} == {n_inserted}'
# All leaves point ixy(neg idx), not inode.
assert all(idx <= 0 for idx in ixy_idxes), \
'ixy_idxes = {}, tree = {}'.format(
ixy_idxes, tup_tree(tree[:n_inserted+4]))
# Inserted ixys are sorted in ascending order.
inserted_ixys = F.lmap(
lambda i: ixy_arr[abs(i)], ixy_idxes)
for ixy1, ixy2 in F.pairwise(inserted_ixys):
assert key(ixy1) <= key(ixy2), 'tree = {}' \
.format(tup_tree(tree[:n_inserted+4]))
# All leaves: l <= r
leaves = F.lfilter(is_leaf, tree[:n_inserted+4])
for leaf in leaves:
l = leaf.left; r = leaf.right
if l and r:
l_val = key(ixy_map[abs(l)])
r_val = key(ixy_map[abs(r)])
assert l_val <= r_val
# All inodes must be sorted in ascending order.
inodes = all_inodes(tup_tree(tree[:n_node+100]))
for n1, n2 in F.pairwise(inodes):
k1 = n1[0]; k2 = n2[0]
assert k1 <= k2
# Inserted ixys are sorted in ascending order.
neg_idxeseq = F.mapcat(tup(
lambda k,p,l,r:
((l,) if l < 0 else ()) + ((r,) if r < 0 else ())),
inodes)
ixy_idxes = F.map(abs, neg_idxeseq)
saved_ixys = F.map(lambda i: pyobj(ixy_arr[i]), ixy_idxes)
keys = F.lmap(key, saved_ixys)
for k1,k2 in F.pairwise(keys):
assert k1 <= k2
def __call__(self, *params, **opts):
root = self.rep
return funcy.compact(self.truthy)
def run(self):
from dvc.command.stage import parse_cmd
cmd = parse_cmd(self.args.cmd)
if not self.args.interactive and not cmd:
raise InvalidArgumentError("command is not specified")
from dvc.dvcfile import make_dvcfile
global_defaults = {
"code": self.CODE,
"data": self.DATA,
"models": self.MODELS,
"metrics": self.DEFAULT_METRICS,
"params": self.DEFAULT_PARAMS,
"plots": self.PLOTS,
"live": self.DVCLIVE,
}
dvcfile = make_dvcfile(self.repo, "dvc.yaml")
name = self.args.name or self.args.type
dvcfile_exists = dvcfile.exists()
if not self.args.force and dvcfile_exists and name in dvcfile.stages:
from dvc.stage.exceptions import DuplicateStageName
hint = "Use '--force' to overwrite."
raise DuplicateStageName(
f"Stage '{name}' already exists in 'dvc.yaml'. {hint}"
)
context = ChainMap()
if not self.args.explicit:
config = {} # TODO
context.maps.extend([config, global_defaults])
with_live = self.args.type == "live"
if self.args.interactive:
try:
context = self.init_interactive(
defaults=context,
show_heading=not dvcfile_exists,
live=with_live,
)
except (KeyboardInterrupt, EOFError):
ui.error_write()
raise
elif with_live:
# suppress `metrics`/`params` if live is selected, unless
# also provided via cli, also make output to be a checkpoint.
context = context.new_child({"metrics": None, "params": None})
else:
# suppress live otherwise
context = context.new_child({"live": None})
if not self.args.interactive:
d = compact(
{
"cmd": cmd,
"code": self.args.code,
"data": self.args.data,
"models": self.args.models,
"metrics": self.args.metrics,
"params": self.args.params,
"plots": self.args.plots,
"live": self.args.live,
}
)
context = context.new_child(d)
assert "cmd" in context
command = context["cmd"]
code = context.get("code")
data = context.get("data")
models = context.get("models")
metrics = context.get("metrics")
plots = context.get("plots")
live = context.get("live")
params_kv = []
if context.get("params"):
from dvc.utils.serialize import LOADERS
path = context["params"]
_, ext = os.path.splitext(path)
params_kv = [{path: list(LOADERS[ext](path))}]
checkpoint_out = bool(context.get("live"))
stage = self.repo.stage.add(
name=name,
cmd=command,
deps=compact([code, data]),
params=params_kv,
metrics_no_cache=compact([metrics]),
plots_no_cache=compact([plots]),
live=live,
force=self.args.force,
**{"checkpoints" if checkpoint_out else "outs": compact([models])},
)
if self.args.run:
return self.repo.experiments.run(targets=[stage.addressing])
return 0
def init(
repo: "Repo",
name: str = None,
type: str = "default", # pylint: disable=redefined-builtin
defaults: Dict[str, str] = None,
overrides: Dict[str, str] = None,
interactive: bool = False,
force: bool = False,
stream: Optional[TextIO] = None,
) -> "Stage":
from dvc.dvcfile import make_dvcfile
dvcfile = make_dvcfile(repo, "dvc.yaml")
name = name or type
_check_stage_exists(dvcfile, name, force=force)
defaults = defaults.copy() if defaults else {}
overrides = overrides.copy() if overrides else {}
with_live = type == "live"
if interactive:
defaults = init_interactive(
name,
validator=validate_prompts,
defaults=defaults,
live=with_live,
provided=overrides,
stream=stream,
)
else:
if with_live:
# suppress `metrics`/`plots` if live is selected, unless
# it is also provided via overrides/cli.
# This makes output to be a checkpoint as well.
defaults.pop("metrics", None)
defaults.pop("plots", None)
else:
defaults.pop("live", None) # suppress live otherwise
context: Dict[str, str] = {**defaults, **overrides}
assert "cmd" in context
params_kv = []
params = context.get("params")
if params:
params_kv.append(loadd_params(params))
checkpoint_out = bool(context.get("live"))
models = context.get("models")
stage = repo.stage.create(
name=name,
cmd=context["cmd"],
deps=compact([context.get("code"),
context.get("data")]),
params=params_kv,
metrics_no_cache=compact([context.get("metrics")]),
plots_no_cache=compact([context.get("plots")]),
live=context.get("live"),
force=force,
**{"checkpoints" if checkpoint_out else "outs": compact([models])},
)
if interactive:
ui.error_write(Rule(style="green"), styled=True)
_yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage)))
syn = Syntax(_yaml, "yaml", theme="ansi_dark")
ui.error_write(syn, styled=True)
from dvc.ui.prompt import Confirm
if not interactive or Confirm.ask(
"Do you want to add the above contents to dvc.yaml?",
console=ui.error_console,
default=True,
stream=stream,
):
with _disable_logging(), repo.scm_context(autostage=True, quiet=True):
stage.dump(update_lock=False)
stage.ignore_outs()
if params:
repo.scm_context.track_file(params)
else:
raise DvcException("Aborting ...")
return stage
def mama(target, options, args, result):
ret = []
needle = None
if len(args):
needle = exp(" ".join(args))
arch = trgt()
expr_options = lldb.SBExpressionOptions()
expr_options.SetIgnoreBreakpoints(True)
expr_options.SetFetchDynamicValue(lldb.eNoDynamicValues)
expr_options.SetTimeoutInMicroSeconds(30 * 1000 *
1000) # 30 second timeout
expr_options.SetTryAllThreads(False)
max_results = 0x1000
frame = lldb.debugger.GetSelectedTarget().GetProcess().GetSelectedThread(
).GetSelectedFrame()
expr_sbvalue = frame.EvaluateExpression(
mach_vm_region_expr() % (arch['extra'](), max_results), expr_options)
if expr_sbvalue.error.Success():
result_value = lldb.value(expr_sbvalue)
starts = []
malloc_tags = [1, 2, 3, 4, 6, 7, 8, 9, 11, 53, 61]
for i in range(max_results):
region_lol = result_value[i]
start = int(region_lol.addr)
size = int(region_lol.size)
user_tag_int = int(region_lol.info.user_tag)
if user_tag_int in malloc_tags and (not needle or in_region(
needle, start, size)):
starts.append(start)
_zones = {}
if len(starts):
_zones = zone(starts)
reached_the_end = False
for i in range(max_results):
region_lol = result_value[i]
size = int(region_lol.size)
start = int(region_lol.addr)
if start >= arch['shared_region_base']:
reached_the_end = True
end = start + size
info = region_lol.info
prot = int(info.protection)
user_tag_int = int(info.user_tag)
# from /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.8.sdk/System/Library/Frameworks/Kernel.framework/Headers/mach/vm_statistics.h
# I only put in the ones I ran into
user_tagz = {
0: "unknown",
1: "malloc",
2: "malloc small",
3: "malloc large",
4: "malloc huge",
# 5: "sbrk",
6: "realloc",
7: "malloc tiny",
8: "malloc large reusable",
9: "malloc large reused",
10: "memory analysis tool",
11: "malloc nano",
20: "mach msg",
21: "iokit",
30: "stack",
31: "guard",
32: "shared pmap",
33: "dylib",
34: "objc dispatchers",
35: "unshared pmap",
40: "appkit",
41: "foundation",
42: "core graphics",
43: "core services",
44: "java",
45: "coredata",
46: "coredata objectids",
50: "ats",
51: "layerkit",
52: "cgimage",
53: "tcmalloc",
54: "coregraphics data",
55: "coregraphics shared",
56: "coregraphics framebuffers",
57: "coregraphics backingstores",
60: "dyld",
61: "dyld mallco",
62: "sqlite",
63: "javascript core heap",
64: "javascript JIT executable allocator",
65: "javascript JIT register file",
66: "GLSL",
67: "OpenCL",
68: "core image",
69: "webcore purgeable buffers",
70: "imageio",
71: "coreprofile",
72: "assetsd / MobileSlideShow",
73: "kernel alloc once",
74: "libdispatch",
75: "accelerate",
76: "coreui",
242: "application specific 242",
251: "application specific 251"
}
user_tag = user_tagz.get(user_tag_int)
if user_tag_int == 30:
user_tag = "".join([user_tag, thread_for_stack(start, end)])
if not user_tag:
print "USER TAG NOT FOUDN: %s" % user_tag_int
ref_count = int(info.ref_count)
object_name = int(region_lol.object_name)
if object_name != 0:
print "YEHAHA %#x" % object_name
share_mode = int(info.share_mode)
share_modez = [
'????', 'COW', 'PRV', 'NUL', 'ALI', 'SHM', 'ZER', 'S/A',
'large page'
]
share_mode = share_modez[share_mode]
if start < arch['shared_region_base']:
lookup_output = run_command(
"target modules lookup -v -a %#x" % start, True)
elif needle:
lookup_output = run_command(
"target modules lookup -v -a %#x" % needle, True)
filename = ""
section = ""
if lookup_output:
if user_tag_int != 0 and user_tag_int != 33 and user_tag_int != 35 and user_tag_int != 32:
print "oh shit! %d" % user_tag_int
break
filename = re_find(r"file = \"([^\"]*)", lookup_output)
filename = re.sub(r".*/SDKs", "...", filename)
filename = re.sub(r".*/iOS DeviceSupport", "...", filename)
section = re_find(r"__[\w]*", lookup_output)
if section and filename and user_tag_int == 0:
user_tag = ""
else:
if user_tag_int == 33:
print "noh shit! %#x" % user_tag_int
break
maybe_guard = "guard" if prot == 0 and user_tag_int == 30 else ""
maybe_unallocated = "" if ref_count else "(zero refcount)"
maybes = [maybe_guard, maybe_unallocated][not maybe_guard]
if size == 0xe800000 and user_tag_int == 32 and ref_count == 0:
reached_the_end = True
else:
if not needle or in_region(needle, start, size):
_zone = _zones[start] if user_tag_int in malloc_tags else ""
ret.append(
(start, size,
"{:#x} - {:#x} - {:<5s} - {:} SM={:} {:}".format(
start, end, human(size), prot_string(prot),
share_mode, " ".join(
compact([
section, user_tag, maybes, filename, _zone
])))))
if share_mode == "NUL" and ref_count == 0 and prot == 5:
break
if not reached_the_end:
print "didn't reach the end?"
return ret
else:
print "failure, %s" % expr_sbvalue.error
def init(
repo: "Repo",
name: str = "train",
type: str = "default", # pylint: disable=redefined-builtin
defaults: Dict[str, str] = None,
overrides: Dict[str, str] = None,
interactive: bool = False,
force: bool = False,
stream: Optional[TextIO] = None,
) -> Tuple[PipelineStage, List["Dependency"], List[str]]:
from dvc.dvcfile import make_dvcfile
dvcfile = make_dvcfile(repo, "dvc.yaml")
_check_stage_exists(dvcfile, name, force=force)
defaults = defaults.copy() if defaults else {}
overrides = overrides.copy() if overrides else {}
if interactive:
defaults = init_interactive(
validator=partial(validate_prompts, repo),
defaults=defaults,
provided=overrides,
stream=stream,
)
else:
if "live" in overrides:
# suppress `metrics`/`plots` if live is selected.
defaults.pop("metrics", None)
defaults.pop("plots", None)
else:
defaults.pop("live", None) # suppress live otherwise
context: Dict[str, str] = {**defaults, **overrides}
assert "cmd" in context
params = context.get("params")
if params:
from dvc.dependency.param import (
MissingParamsFile,
ParamsDependency,
ParamsIsADirectoryError,
)
try:
ParamsDependency(None, params, repo=repo).validate_filepath()
except ParamsIsADirectoryError as exc:
raise DvcException(f"{exc}.") # swallow cause for display
except MissingParamsFile:
pass
models = context.get("models")
live_path = context.pop("live", None)
live_metrics = f"{live_path}.json" if live_path else None
live_plots = os.path.join(live_path, "scalars") if live_path else None
stage = repo.stage.create(
name=name,
cmd=context["cmd"],
deps=compact([context.get("code"),
context.get("data")]),
params=[{
params: None
}] if params else None,
metrics_no_cache=compact([context.get("metrics"), live_metrics]),
plots_no_cache=compact([context.get("plots"), live_plots]),
force=force,
**{
"checkpoints" if type == "checkpoint" else "outs":
compact([models])
},
)
with _disable_logging(), repo.scm_context(autostage=True, quiet=True):
stage.dump(update_lock=False)
initialized_out_dirs = init_out_dirs(stage)
stage.ignore_outs()
initialized_deps = init_deps(stage)
if params:
repo.scm_context.track_file(params)
assert isinstance(stage, PipelineStage)
return stage, initialized_deps, initialized_out_dirs
def _sort_columns_with_fields(columns, fields):
if fields:
columns = compact(
[_get_column_by_name(columns, field) for field in fields])
return columns
data_dict.get('fpid', None),
data_dict.get('user_id', None)
] if item), None)
# Method 2
# Notice that this has Python recursive limitation
def until(terminate, iterator, default):
if isinstance(iterator, list):
iterator = iter(iterator)
try:
i = next(iterator)
if terminate(i):
return i
return until(terminate, iterator, default)
except StopIteration:
return default
# candidates = [None, None, None, None]
candidates = [None, None, '25', None]
id = until(lambda x: x is not None, iter(candidates), '1')
# Method 3
# Introducing FP(Functional Programming)
from funcy import first, compact
candidates = [None, None, '25', None]
id = first(compact(candidates)) # 25
def init(
repo: "Repo",
name: str = None,
type: str = "default", # pylint: disable=redefined-builtin
defaults: Dict[str, str] = None,
overrides: Dict[str, str] = None,
interactive: bool = False,
force: bool = False,
) -> "Stage":
from dvc.dvcfile import make_dvcfile
dvcfile = make_dvcfile(repo, "dvc.yaml")
name = name or type
_check_stage_exists(dvcfile, name, force=force)
defaults = defaults or {}
overrides = overrides or {}
with_live = type == "live"
if interactive:
defaults = init_interactive(
name,
defaults=defaults,
live=with_live,
provided=overrides,
show_tree=True,
)
else:
if with_live:
# suppress `metrics`/`params` if live is selected, unless
# it is also provided via overrides/cli.
# This makes output to be a checkpoint as well.
defaults.pop("metrics")
defaults.pop("params")
else:
defaults.pop("live") # suppress live otherwise
context: Dict[str, str] = {**defaults, **overrides}
assert "cmd" in context
params_kv = []
if context.get("params"):
from dvc.utils.serialize import LOADERS
path = context["params"]
assert isinstance(path, str)
_, ext = os.path.splitext(path)
params_kv = [{path: list(LOADERS[ext](path))}]
checkpoint_out = bool(context.get("live"))
models = context.get("models")
stage = repo.stage.create(
name=name,
cmd=context["cmd"],
deps=compact([context.get("code"),
context.get("data")]),
params=params_kv,
metrics_no_cache=compact([context.get("metrics")]),
plots_no_cache=compact([context.get("plots")]),
live=context.get("live"),
force=force,
**{"checkpoints" if checkpoint_out else "outs": compact([models])},
)
if interactive:
ui.write(Rule(style="green"), styled=True)
_yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage)))
syn = Syntax(_yaml, "yaml", theme="ansi_dark")
ui.error_write(syn, styled=True)
if not interactive or ui.confirm(
"Do you want to add the above contents to dvc.yaml?"):
with _disable_logging():
stage.dump(update_lock=False)
stage.ignore_outs()
else:
raise DvcException("Aborting ...")
return stage
