def main(args: argparse.Namespace) -> None:
workspace = tsrc.cli.get_workspace(args)
cmd_runner = CmdRunner(workspace.root_path,
args.cmd,
args.cmd_as_str,
shell=args.shell)
manifest = workspace.local_manifest.get_manifest()
workspace_config = workspace.config
groups_from_config = workspace_config.repo_groups
all_remote_repos = manifest.get_repos(all_=True)
cloned_repos = [
x for x in all_remote_repos if (workspace.root_path / x.src).exists()
]
if args.groups_from_config:
requested_repos = manifest.get_repos(groups=groups_from_config)
elif args.groups:
requested_repos = manifest.get_repos(groups=args.groups)
else:
requested_repos = cloned_repos
found = [x for x in requested_repos if x in cloned_repos]
missing = [x for x in requested_repos if x not in cloned_repos]
tsrc.run_sequence(found, cmd_runner)
if missing:
ui.warning(
"The following repos were requested but missing from the workspace:"
)
for repo in missing:
ui.info("*", repo.src, fileobj=sys.stderr)
raise MissingRepos(missing)
else:
ui.info("OK", ui.check)
def info(self, *args: Any, **kwargs: Any) -> None:
"""Same as cli_ui.info(), except this is a no-op if the
task is run in parallel with other tasks.
"""
if not self.parallel:
ui.info(*args, **kwargs)
def repos_from_config(
manifest: Manifest, workspace_config: WorkspaceConfig
) -> List[tsrc.Repo]:
"""
Given a workspace config, returns a list of repos.
"""
clone_all_repos = workspace_config.clone_all_repos
repo_groups = workspace_config.repo_groups
if clone_all_repos:
# workspace config contains clone_all_repos: true,
# return everything
return manifest.get_repos(all_=True)
if repo_groups:
# workspace config contains some groups, use that,
# fmt: off
ui.info(
ui.green, "*", ui.reset, "Using groups from workspace config:",
", ".join(repo_groups),
)
# fmt: on
return manifest.get_repos(groups=repo_groups)
else:
# workspace config does not specify clone_all_repos nor
# a list of groups, ask the manifest for the list of default
# repos
return manifest.get_repos(groups=None)
def run_git(
working_path: Path,
*cmd: str,
check: bool = True,
show_output: bool = True,
show_cmd: bool = True,
) -> None:
"""Run git `cmd` in given `working_path`.
Raise GitCommandError if return code is non-zero and `check` is True.
"""
assert_working_path(working_path)
git_cmd = list(cmd)
git_cmd.insert(0, "git")
if show_cmd:
ui.info(ui.blue, "$", ui.reset, *git_cmd)
if show_output:
process = subprocess.run(git_cmd,
cwd=working_path,
universal_newlines=True)
else:
process = subprocess.run(
git_cmd,
cwd=working_path,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
if process.returncode != 0 and check:
raise GitCommandError(working_path, cmd, output=process.stdout)
def run(args: argparse.Namespace) -> None:
# Note:
# we want to support both:
# $ tsrc foreach -c 'shell command'
# and
# $ tsrc foreach -- some-cmd --some-opts
#
# Due to argparse limitations, `cmd` will always be a list,
# but we need a *string* when using 'shell=True'.
#
# So transform use the value from `cmd` and `shell` so that:
# * action.command is suitable as argument to pass to subprocess.run()
# * action.description is suitable for display purposes
command: Command = []
if args.shell:
if len(args.cmd) != 1:
die("foreach -c must be followed by exactly one argument")
command = args.cmd[0]
description = args.cmd[0]
else:
if not args.cmd:
die("needs a command to run")
command = args.cmd
description = " ".join(args.cmd)
shell = args.shell
command = command
description = description
workspace = get_workspace_with_repos(args)
cmd_runner = CmdRunner(workspace.root_path, command, description, shell=shell)
tsrc.run_sequence(workspace.repos, cmd_runner)
ui.info("OK", ui.check)
def process_item(self, index: int, count: int, item: T) -> Outcome:
# We want to keep all output when processing items it parallel on just
# one line (like ninja-build)
#
# To do that, we need a lock on stdout. We also need task.process() to
# be silent, which should be the case if it is implemented correctly
tokens = self.task.describe_process_start(item)
if tokens:
with self.lock:
erase_last_line()
ui.info_count(index, count, *tokens, end="\r")
result = self.task.process(index, count, item)
# Note: we don't know if tasks will be finished in the same order
# they were started, so to keep the output relevant, we need a
# done_count here.
self.done_count += 1
tokens = self.task.describe_process_end(item)
if tokens:
with self.lock:
erase_last_line()
ui.info_count(self.done_count - 1, count, *tokens, end="\r")
if self.done_count == count:
ui.info()
return result
def test_parallel_happy() -> None:
task = FakeTask()
ui.info("Frobnicating 4 items with two workers")
actual = process_items_parallel(["foo", "bar", "baz", "quux"], task, num_jobs=2)
ui.info("Done")
for outcome in actual.values():
assert outcome.success()
def status(self):
if not (os.path.exists(self.restore_log_path) and os.path.isfile(self.restore_log_path)):
cli_ui.warning('No restore log found!')
sys.exit(0) # FIXME: Don't exit, rather throw an exception
restore_log = self._get_restore_log()
if not self.rubrik:
creds = self._read_credentials(ignore_stored=True, presets={'address': restore_log['cluster']['ip']})
self._connect(creds)
statuses = []
for job in restore_log['jobs']:
klass = config_class(job['configType'])
status = klass(self.path, self.rubrik, logging.getLogger()).status(job)
if status:
statuses.append(status)
status_rows = list(map(
lambda s: [
(status_color(s[0]), s[0]),
(cli_ui.lightgray, s[1]),
(cli_ui.lightgray, s[2]),
(cli_ui.lightgray, s[3]),
(cli_ui.bold, s[4])],
statuses
))
cli_ui.info('\nBackup Id:', cli_ui.turquoise, restore_log['backupId'], end='\n\n')
cli_ui.info_table(status_rows, headers=['Status', 'Start time', 'End time', 'Type', 'Name'])
def test(self) -> None:
self.build()
if not self._is_host_target:
ui.info(self.profile, "is a cross-compiled target, skipping tests")
return
tankerci.run("cargo", "fmt", "--", "--check", cwd=self.src_path)
tankerci.run(
"cargo",
"clippy",
"--all-targets",
"--",
"--deny",
"warnings",
"--allow",
"unknown-lints",
cwd=self.src_path,
)
if self._is_windows_target:
shutil.copy(
Path("native") / self.target_triplet / "ctanker.dll",
Path("target") / "debug/deps",
)
self._cargo("test")
def post_push(self) -> None:
self.pull_request = self.ensure_pull_request()
assert self.pull_request
if self.args.close:
ui.info_2("Closing merge request #%s" % self.pull_request.number)
self.pull_request.close()
return
params = dict()
if self.requested_target_branch:
params["base"] = self.requested_target_branch
if self.requested_title:
params["title"] = self.requested_title
self.pull_request.update(**params)
if self.requested_reviewers:
message = [
"Requesting review from", ", ".join(self.requested_reviewers)
]
ui.info_2(*message)
tsrc.github.request_reviewers(self.repository,
self.pull_request.number,
self.requested_reviewers)
if self.requested_assignee:
ui.info_2("Assigning to", self.requested_assignee)
self.assign_pull_request()
if self.args.merge:
self.merge_pull_request()
ui.info(ui.green, "::", ui.reset, "See pull request at",
self.pull_request.html_url)
def download():
cli_ui.info_2("Downloading the newest version of PortCMS")
if (not checker.checkGitStatus()):
cli_ui.error(
"Please download and install git to continue: https://git-scm.com/downloads"
)
os._exit(1)
isUpdate = False
try:
os.mkdir("PortCMS")
except:
what = cli_ui.ask_choice(
"PortCMS is already downloaded. What do You want to do?",
choices=["Update", "Check database config"])
isUpdate = what == "Update"
if (not isUpdate):
os.chdir("PortCMS")
dbconf()
os._exit(0)
if (isUpdate):
os.chdir("PortCMS")
os.system("git pull")
else:
os.system(
"git clone https://github.com/PetrusTryb/portfolio.git PortCMS")
os.chdir("PortCMS")
cli_ui.info(cli_ui.check, "Download complete")
def print_self(self) -> None:
# fmt: off
ui.info(
ui.red,
"- ",
ui.reset,
ui.bold,
self.src,
":",
ui.reset,
ui.darkgray,
self.lineno + 1,
ui.reset,
" ",
ui.red,
self.old_line.strip(),
sep="",
)
ui.info(
ui.green,
"+ ",
ui.reset,
ui.bold,
self.src,
":",
ui.reset,
ui.darkgray,
self.lineno + 1,
ui.reset,
" ",
ui.green,
self.new_line.strip(),
sep="",
)
def dump_logcat_for_failed_tests() -> None:
try:
dump_path = "tanker-bindings/build/reports/androidTests/connected/flavors/releaseAndroidTest/logcat.txt" # noqa: E501
tankerci.android.dump_logcat(dump_path)
ui.info("Tests have failed, logcat dumped to", dump_path)
except Exception as e:
ui.error("Failed to dump logcat:", e)
def post_push(self) -> None:
merge_request = self.ensure_merge_request()
assert self.gitlab_api
if self.args.close:
ui.info_2("Closing merge request #%s" % merge_request.iid)
merge_request.state_event = "close"
merge_request.save()
return
assignee = None
if self.requested_assignee:
assignee = self.handle_assignee()
if assignee:
ui.info_2("Assigning to", assignee.username)
title = self.handle_title(merge_request)
merge_request.title = title
merge_request.remove_source_branch = True
if self.requested_target_branch:
merge_request.target_branch = self.requested_target_branch
if assignee:
merge_request.assignee_id = assignee.id
approvers = self.handle_approvers()
merge_request.approvals.set_approvers([x.id for x in approvers])
merge_request.save()
if self.args.accept:
merge_request.merge(merge_when_pipeline_succeeds=True)
ui.info(ui.green, "::",
ui.reset, "See merge request at", merge_request.web_url)
def foreach(workspace: tsrc.Workspace, *args: Any, **kwargs: Any) -> None:
cmd: List[str] = args # type: ignore
shell: bool = kwargs["shell"]
# Note:
# we want to support both:
# $ tsrc foreach -c 'shell command'
# and
# $ tsrc foreach -- some-cmd --some-opts
#
# Due to argparse limitations, `cmd` will always be a list,
# but we need a *string* when using 'shell=True'.
#
# So transform use the value from `cmd` and `shell` to build:
# * `subprocess_cmd`, suitable as argument to pass to subprocess.run()
# * `cmd_as_str`, suitable for display purposes
command: Command = []
if shell:
if len(cmd) != 1:
die("foreach -c must be followed by exactly one argument")
command = cmd[0]
description = cmd[0]
else:
if not cmd:
die("needs a command to run")
command = cmd
description = " ".join(cmd)
cmd_runner = CmdRunner(workspace.root_path,
command,
description,
shell=shell)
tsrc.run_sequence(workspace.repos, cmd_runner)
ui.info("OK", ui.check)
def main(args: argparse.Namespace) -> None:
tsrc_distribution = pkg_resources.get_distribution("tsrc")
version = tsrc_distribution.version
message = "tsrc version %s" % version
location = Path(tsrc_distribution.location)
message += get_details(location)
ui.info(message)
def display_statuses(statuses: List[Tuple[str, tsrc.git.Status]]) -> None:
if not statuses:
return
max_src = max((len(x[0]) for x in statuses))
for src, status in statuses:
message = [ui.green, "*", ui.reset, src.ljust(max_src)]
message += describe(status)
ui.info(*message)
def handle_errors(self) -> None:
self.task.on_failure(num_errors=len(self.errors))
for item, error in self.errors:
item_desc = self.task.display_item(item)
message = [ui.green, "*", " ", ui.reset, ui.bold, item_desc]
if error.message:
message.extend([ui.reset, ": ", error.message])
ui.info(*message, sep="", fileobj=sys.stderr)
raise ExecutorFailed()
def generate_test_config(src_path: Path, *, config_name: str) -> None:
filepath = ci.tanker_configs.get_path()
to_write = textwrap.dedent(f"""\
#define TANKER_CONFIG_FILEPATH @"{filepath}"
#define TANKER_CONFIG_NAME @"{config_name}"
""")
config_header = src_path / "TKRTestConfig.h"
config_header.write_text(to_write)
ui.info("Config written to", config_header)
def test_info_characters(smart_tty: SmartTTY) -> None:
cli_ui.info("Doing stuff",
cli_ui.ellipsis,
"sucess",
cli_ui.check,
fileobj=smart_tty)
actual = smart_tty.getvalue()
expected = f"Doing stuff {RESET_ALL}{RESET_ALL}… {RESET_ALL}sucess {RESET_ALL}{GREEN}✓ {RESET_ALL}\n{RESET_ALL}"
assert actual == expected
def get_branch_name() -> Optional[str]:
branch = os.environ.get("CI_COMMIT_BRANCH", None)
if not branch:
branch = os.environ.get("CI_COMMIT_REF_NAME", None)
if not branch:
branch = tankerci.git.get_current_branch(Path.cwd())
if not branch:
return None
ui.info(f"Running on branch {branch}")
return branch
def test_info_characters(smart_tty: SmartTTY) -> None:
cli_ui.info("Doing stuff",
cli_ui.ellipsis,
"sucess",
cli_ui.check,
fileobj=smart_tty)
actual = smart_tty.getvalue()
expected = ("Doing stuff " + colorama.Style.RESET_ALL + "…" + " sucess " +
colorama.Fore.GREEN + "✓")
assert_equal_strings(actual, expected)
def find_files(working_path: Path, current_version: str) -> List[str]:
ui.info_2("Looking for files matching", ui.bold, current_version)
cmd = ["grep", "--fixed-strings", "--files-with-matches", current_version]
_, out = tbump.git.run_git_captured(working_path, *cmd, check=True)
res = [] # type: List[str]
ui.info("Found following matching files")
for file in out.splitlines():
ui.info(" * ", file)
res.append(file)
return res
def main(args: argparse.Namespace) -> None:
workspace_path = args.workspace_path or os.getcwd()
workspace = tsrc.Workspace(Path(workspace_path))
ui.info_1("Configuring workspace in", ui.bold, workspace_path)
manifest_config = tsrc.workspace.ManifestConfig.from_args(args)
workspace.configure_manifest(manifest_config)
workspace.load_manifest()
workspace.clone_missing()
workspace.set_remotes()
workspace.copy_files()
ui.info("Done", ui.check)
def on_success(self) -> None:
erase_last_line()
if not self.statuses:
ui.info_2("Workspace is empty")
return
ui.info_2("Workspace status:")
max_dest = max(len(x) for x in self.statuses.keys())
for dest, status in self.statuses.items():
message = [ui.green, "*", ui.reset, dest.ljust(max_dest)]
message += describe_status(status)
ui.info(*message)
def process(self, repo: tsrc.Repo) -> None:
# fmt: off
ui.info(repo.src, "\n",
ui.lightgray, "$ ",
ui.reset, ui.bold, self.cmd_as_str,
sep="")
# fmt: on
full_path = self.workspace_path / repo.src
rc = subprocess.call(self.cmd, cwd=full_path, shell=self.shell)
if rc != 0:
raise CommandFailed()
def test_update_title(always_color: None, smart_tty: SmartTTY) -> None:
# fmt: off
cli_ui.info("Something",
cli_ui.bold,
"bold",
fileobj=smart_tty,
update_title=True)
expected = ("\x1b]0;Something bold\n\x07"
f"Something {BRIGHT}bold\n{RESET_ALL}")
# fmt: on
actual = smart_tty.getvalue()
assert actual == expected
def test_info_stdout_is_not_a_tty(dumb_tty: DumbTTY) -> None:
# fmt: off
cli_ui.info(cli_ui.red,
"this is red",
cli_ui.reset,
cli_ui.green,
"this is green",
fileobj=dumb_tty)
# fmt: on
expected = "this is red this is green\n"
actual = dumb_tty.getvalue()
assert_equal_strings(actual, expected)
def test_info_stdout_no_colors(dumb_tty: DumbTTY) -> None:
# fmt: off
cli_ui.info(cli_ui.red,
"this is red",
cli_ui.reset,
cli_ui.green,
"this is green",
fileobj=dumb_tty)
# fmt: on
expected = "this is red this is green\n"
actual = dumb_tty.getvalue()
assert actual == expected
def run(args: argparse.Namespace) -> None:
# Note:
# we want to support both:
# $ tsrc foreach -c 'shell command'
# and
# $ tsrc foreach -- some-cmd --some-opts
#
# Due to argparse limitations, `cmd` will always be a list,
# but we need a *string* when using 'shell=True'.
#
# So transform use the value from `cmd` and `shell` so that:
# * action.command is suitable as argument to pass to subprocess.run()
# * action.description is suitable for display purposes
command: Command = []
if args.shell:
if len(args.cmd) != 1:
die("foreach -c must be followed by exactly one argument")
command = args.cmd[0]
description = args.cmd[0]
else:
if not args.cmd:
die("needs a command to run")
command = args.cmd
description = " ".join(args.cmd)
shell = args.shell
command = command
description = description
num_jobs = get_num_jobs(args)
workspace = get_workspace_with_repos(args)
cmd_runner = CmdRunner(workspace.root_path,
command,
description,
shell=shell)
repos = workspace.repos
ui.info_1(f"Running `{description}` on {len(repos)} repos")
collection = process_items(repos, cmd_runner, num_jobs=num_jobs)
errors = collection.errors
if errors:
ui.error(f"Command failed for {len(errors)} repo(s)")
if cmd_runner.parallel:
# Print output of failed commands that were hidden
for (item, error) in errors.items():
ui.info(item)
ui.info("-" * len(item))
ui.info(error)
else:
# Just print the repos
for item in errors:
ui.info(ui.green, "*", ui.reset, item)
raise ForeachError()
else:
ui.info("OK", ui.check)
