def et(ctx: click.core.Context, verbose: bool): """ Primary top-level group command. Calling directly with no parameters will display help. """ ctx.obj = {} ctx.obj['verbose'] = verbose
def main(ctx: click.core.Context, config_file: str): path = Path(config_file) data = {} if path.exists() and path.is_file(): with open(config_file, 'rb') as fd: data = yaml.safe_load(fd) ctx.obj = {'config': data, 'config_file': path, 'config_dir': path.parent}
def main(ctx: click.core.Context, username: str, password: str, stdin_password: bool) -> None: if not (password or stdin_password): raise click.UsageError( "Must supply one of `password` or `stdin_password`") if stdin_password: password = input() ctx.obj = AirtableCredentials(username=username, password=password)
def cli(ctx: click.core.Context, config: str, endpoint: str, username: str, password: str, skip_verify: bool) -> None: """Nessie cli tool. Interact with Nessie branches and tables via the command line """ if config: os.environ["NESSIE_CLIENTDIR"] = config config = build_config({"endpoint": endpoint, "username": username, "password": password, "verify": not skip_verify}) nessie = NessieClient(config) ctx.obj = dict() ctx.obj["nessie"] = nessie
def main(ctx: click.core.Context, host: str, user: str, password: typing.Optional[str], schema: str) -> None: """Main entrypoint for fuzzy-tribble. Type --help after any subcommand for additional help.""" ctx.obj = {} creds = tribble.database.Creds(host, user, password, schema) engine = tribble.database.connect_db(creds) contract.Session.configure(bind=engine) ctx.obj['creds'] = creds ctx.obj['engine'] = engine
def cli(ctx: click.core.Context, json: bool, verbose: bool, endpoint: str) -> None: """Nessie cli tool. Interact with Nessie branches and tables via the command line """ try: config = build_config({"endpoint": endpoint} if endpoint else None) nessie = NessieClient(config) ctx.obj = ContextObject(nessie, verbose, json) except confuse.exceptions.ConfigTypeError as e: raise click.ClickException(str(e))
def drclip(ctx: click.core.Context, config: TextIOWrapper, registry: str): """Runs commands against docker registries""" ctx.obj = CmdContext(RegistryV2API(registry, DockerCredentials(config))) err = None try: ctx.obj.api.head() # Simple version check / connectivity check except CredentialsNotFound: err = f'Error: Credentials for {registry} could not be located (you may need to run docker login ... )' except CredentialsException as e: err = e if err: click.echo(err, err=True) sys.exit(1)
def main( ctx: click.core.Context, username: str, password: str, token: str, locale: str, stdin_password: bool, ) -> None: if not (password or stdin_password): raise click.UsageError("Must supply one of `password` or `stdin_password`") if stdin_password: password = input() ctx.obj = CrunchyrollApi(username, password, token)
def cli( ctx: click.core.Context, host: str, username: str, password: str, device_type: str, ) -> None: """Interact with the device API.""" api_generator: Callable[[str, str, str], Union[GogoGate2Api, ISmartGateApi]] if device_type == DeviceType.GOGOGATE2.value: api_generator = GogoGate2Api else: api_generator = ISmartGateApi if password == "-": # nosec password = default_password() ctx.obj = {API: api_generator(host, username, password)}
def cli( ctx: click.core.Context, debug: bool, verbose: bool, url: str, username: str, password: str, print_headers: bool, ) -> None: log_level = Context.LOG_WARNING if debug: log_level = Context.LOG_DEBUG elif verbose: log_level = Context.LOG_INFO ctx.obj = Context(url, username, password, log_level, print_headers)
def packaging_integration(ctx: click.core.Context, url: str, token: str): """ Allow you to perform actions on packaging integration.\n Alias for the command is pi. """ ctx.obj = PackagingIntegrationClient(url, token)
def toolchain_integration(ctx: click.core.Context, url: str, token: str): """ Allow you to perform actions on toolchain integration.\n Alias for the command is ti. """ ctx.obj = ToolchainIntegrationClient(url, token)
def bulk(ctx: click.core.Context, url: str, token: str): """ Bulk operations on Odahuflow resources """ ctx.obj = RemoteAPIClient(url, token)
def cli(ctx: click.core.Context) -> None: """ Mackerel is a minimal static site generator written in typed Python 3.6+. """ ctx.obj = {}
def tests( context: click.core.Context, base_path: str, session: Optional[str], build_name: Optional[str], post_chunk: int, subsetting_id: str, flavor, no_base_path_inference, report_paths, ): logger = Logger() org, workspace = ensure_org_workspace() client = LaunchableClient(test_runner=context.invoked_subcommand, dry_run=context.obj.dry_run) file_path_normalizer = FilePathNormalizer( base_path, no_base_path_inference=no_base_path_inference) try: if subsetting_id: result = get_session_and_record_start_at_from_subsetting_id( subsetting_id, client) session_id = result["session"] record_start_at = result["start_at"] else: session_id = find_or_create_session(context, session, build_name, flavor) build_name = read_build() record_start_at = get_record_start_at(session_id, client) build_name, test_session_id = parse_session(session_id) except Exception as e: if os.getenv(REPORT_ERROR_KEY): raise e else: traceback.print_exc() return # TODO: placed here to minimize invasion in this PR to reduce the likelihood of # PR merge hell. This should be moved to a top-level class class RecordTests: # The most generic form of parsing, where a path to a test report # is turned into a generator by using CaseEvent.create() ParseFunc = Callable[[str], Generator[CaseEventType, None, None]] # A common mechanism to build ParseFunc by building JUnit XML report in-memory (or build it the usual way # and patch it to fix things up). This is handy as some libraries produce invalid / broken JUnit reports JUnitXmlParseFunc = Callable[[str], ET.Element] @property def path_builder(self) -> CaseEvent.TestPathBuilder: """ This function, if supplied, is used to build a test path that uniquely identifies a test case """ return self._path_builder @path_builder.setter def path_builder(self, v: CaseEvent.TestPathBuilder): self._path_builder = v @property def parse_func(self) -> ParseFunc: return self._parse_func @parse_func.setter def parse_func(self, f: ParseFunc): self._parse_func = f # setter only property that sits on top of the parse_func property def set_junitxml_parse_func(self, f: JUnitXmlParseFunc): """ Parse XML report file with the JUnit report file, possibly with the custom parser function 'f' that can be used to build JUnit ET.Element tree from scratch or do some patch up. If f=None, the default parse code from JUnitParser module is used. """ def parse(report: str) -> Generator[CaseEventType, None, None]: # To understand JUnit XML format, https://llg.cubic.org/docs/junit/ is helpful # TODO: robustness: what's the best way to deal with broken XML file, if any? try: xml = JUnitXml.fromfile(report, f) except Exception as e: click.echo(click.style( "Warning: error reading JUnitXml file {filename}: {error}" .format(filename=report, error=e), fg="yellow"), err=True) # `JUnitXml.fromfile()` will raise `JUnitXmlError` and other lxml related errors if the file has wrong format. # https://github.com/weiwei/junitparser/blob/master/junitparser/junitparser.py#L321 return if isinstance(xml, JUnitXml): testsuites = [suite for suite in xml] elif isinstance(xml, TestSuite): testsuites = [xml] else: raise InvalidJUnitXMLException(filename=report) for suite in testsuites: for case in suite: yield CaseEvent.from_case_and_suite( self.path_builder, case, suite, report) self.parse_func = parse junitxml_parse_func = property(None, set_junitxml_parse_func) @property def check_timestamp(self): return self._check_timestamp @check_timestamp.setter def check_timestamp(self, enable: bool): self._check_timestamp = enable def __init__(self, dry_run=False): self.reports = [] self.skipped_reports = [] self.path_builder = CaseEvent.default_path_builder( file_path_normalizer) self.junitxml_parse_func = None self.check_timestamp = True self.base_path = base_path self.dry_run = dry_run self.no_base_path_inference = no_base_path_inference def make_file_path_component(self, filepath) -> TestPathComponent: """Create a single TestPathComponent from the given file path""" if base_path: filepath = os.path.relpath(filepath, start=base_path) return {"type": "file", "name": filepath} def report(self, junit_report_file: str): ctime = datetime.datetime.fromtimestamp( os.path.getctime(junit_report_file)) if self.check_timestamp and ctime.timestamp( ) < record_start_at.timestamp(): format = "%Y-%m-%d %H:%M:%S" logger.warning( "skip: {} is too old to report. start_record_at: {} file_created_at: {}" .format(junit_report_file, record_start_at.strftime(format), ctime.strftime(format))) self.skipped_reports.append(junit_report_file) return self.reports.append(junit_report_file) def scan(self, base: str, pattern: str): """ Starting at the 'base' path, recursively add everything that matches the given GLOB pattern scan('build/test-reports', '**/*.xml') """ for t in glob.iglob(os.path.join(base, pattern), recursive=True): self.report(t) def run(self): count = 0 # count number of test cases sent def testcases( reports: List[str] ) -> Generator[CaseEventType, None, None]: exceptions = [] for report in reports: try: yield from self.parse_func(report) except Exception as e: exceptions.append( Exception( "Failed to process a report file: {}".format( report), e)) if len(exceptions) > 0: # defer XML parsing exceptions so that we can send what we can send before we bail out raise Exception(exceptions) # generator that creates the payload incrementally def payload( cases: Generator[TestCase, None, None] ) -> Tuple[Dict[str, List], List[Exception]]: nonlocal count cs = [] exs = [] while True: try: cs.append(next(cases)) except StopIteration: break except Exception as ex: exs.append(ex) count += len(cs) return {"events": cs}, exs def send(payload: Dict[str, List]) -> None: res = client.request("post", "{}/events".format(session_id), payload=payload, compress=True) if res.status_code == HTTPStatus.NOT_FOUND: if session: build, _ = parse_session(session) click.echo(click.style( "Session {} was not found. Make sure to run `launchable record session --build {}` before `launchable record tests`" .format(session, build), 'yellow'), err=True) elif build_name: click.echo(click.style( "Build {} was not found. Make sure to run `launchable record build --name {}` before `launchable record tests`" .format(build_name, build_name), 'yellow'), err=True) res.raise_for_status() def recorded_result() -> Tuple[int, int, int, float]: test_count = 0 success_count = 0 fail_count = 0 duration = float(0) for tc in testcases(self.reports): test_count += 1 status = tc.get("status") if status == 0: fail_count += 1 elif status == 1: success_count += 1 duration += float(tc.get("duration") or 0) # sec return test_count, success_count, fail_count, duration / 60 # sec to min try: tc = testcases(self.reports) if report_paths: # diagnostics mode to just report test paths for t in tc: print(unparse_test_path(t['testPath'])) return exceptions = [] for chunk in ichunked(tc, post_chunk): p, es = payload(chunk) send(p) exceptions.extend(es) res = client.request("patch", "{}/close".format(session_id)) res.raise_for_status() if len(exceptions) > 0: raise Exception(exceptions) except Exception as e: if os.getenv(REPORT_ERROR_KEY): raise e else: traceback.print_exc() return if count == 0: if len(self.skipped_reports) != 0: click.echo( click.style( "{} test reports were skipped because they were created before `launchable record build` was run.\nMake sure to run tests after running `launchable record build`." .format(len(self.skipped_reports)), 'yellow')) return else: click.echo( click.style( "Looks like tests didn't run? If not, make sure the right files/directories are passed", 'yellow')) return file_count = len(self.reports) test_count, success_count, fail_count, duration = recorded_result() click.echo( "Launchable recorded tests for build {} (test session {}) to workspace {}/{} from {} files:\n" .format(build_name, test_session_id, org, workspace, file_count)) header = [ "Files found", "Tests found", "Tests passed", "Tests failed", "Total duration (min)" ] rows = [[ file_count, test_count, success_count, fail_count, "{:0.4f}".format(duration) ]] click.echo(tabulate(rows, header, tablefmt="github")) click.echo( "\nVisit https://app.launchableinc.com/organizations/{organization}/workspaces/{workspace}/test-sessions/{test_session_id} to view uploaded test results (or run `launchable inspect tests --test-session-id {test_session_id}`)" .format( organization=org, workspace=workspace, test_session_id=test_session_id, )) context.obj = RecordTests(dry_run=context.obj.dry_run)
def main(ctx: click.core.Context, quiet: bool, status_path: Path) -> None: ctx.obj = Status(status_path) if quiet: # pylint: disable=consider-using-with sys.stdout = sys.stderr = open(os.devnull, "w", encoding="utf-8")
def split_subset(context: click.core.Context, subset_id: str, bin, rest: str, base_path: str): TestPathWriter.base_path = base_path class SplitSubset(TestPathWriter): def __init__(self, dry_run: bool = False): super(SplitSubset, self).__init__(dry_run=dry_run) def run(self): index = bin[0] count = bin[1] if (index == 0 or count == 0): click.echo(click.style( 'Error: invalid bin value. Make sure to set over 0 like `--bin 1/2` but set `--bin {}`' .format(bin), 'yellow'), err=True) return if count < index: click.echo(click.style( 'Error: invalid bin value. Make sure to set below 1 like `--bin 1/2`, `--bin 2/2` but set `--bin {}`' .format(bin), 'yellow'), err=True) return output = [] rests = [] try: client = LaunchableClient( test_runner=context.invoked_subcommand, dry_run=context.obj.dry_run) payload = { "sliceCount": count, "sliceIndex": index, } res = client.request("POST", "{}/slice".format(subset_id), payload=payload) res.raise_for_status() output = res.json()["testPaths"] rests = res.json()["rest"] except Exception as e: if os.getenv(REPORT_ERROR_KEY): raise e else: click.echo(e, err=True) click.echo(click.style( "Warning: the service failed to split subset. Falling back to running all tests", fg='yellow'), err=True) return if len(output) == 0: click.echo(click.style( "Error: no tests found in this subset id.", 'yellow'), err=True) return if rest: if len(rests) == 0: rests.append(output[0]) self.write_file(rest, rests) self.print(output) context.obj = SplitSubset(dry_run=context.obj.dry_run)
def cli(ctx: click.core.Context) -> None: ctx.obj = create_app_context()
def training(ctx: click.core.Context, url: str, token: str): """ Allow you to perform actions on trainings.\n Alias for the command is train. """ ctx.obj = ModelTrainingClient(url, token)
def deployment(ctx: click.core.Context, url: str, token: str): """ Allow you to perform actions on deployments.\n Alias for the command is dep. """ ctx.obj = ModelDeploymentClient(url, token)
def main(ctx: click.core.Context, connection: str, batch_size: typing.Optional[str]) -> None: ctx.obj = {"connection_string": connection, "batch_size": batch_size}
def main(ctx: click.core.Context, connection: str) -> None: ctx.obj = {"connection_string": connection}
def connection(ctx: click.core.Context, url: str, token: str): """ Allow you to perform actions on connections.\n Alias for the command is conn. """ ctx.obj = ConnectionClient(url, token)
def cli(ctx: click.core.Context, verbose: bool, endpoint: str): ctx.obj = { 'verbose': verbose, 'endpoint': endpoint, 'url': lambda s: f'{endpoint}{s}' }
def subset( context: click.core.Context, target: Optional[PercentageType], session: Optional[str], base_path: Optional[str], build_name: Optional[str], rest: str, duration: Optional[DurationType], flavor: KeyValueType, confidence: Optional[PercentageType], split: bool, no_base_path_inference: bool, ): session_id = find_or_create_session(context, session, build_name, flavor) file_path_normalizer = FilePathNormalizer( base_path, no_base_path_inference=no_base_path_inference) # TODO: placed here to minimize invasion in this PR to reduce the likelihood of # PR merge hell. This should be moved to a top-level class TestPathWriter.base_path = base_path class Optimize(TestPathWriter): # test_paths: List[TestPath] # doesn't work with Python 3.5 # Where we take TestPath, we also accept a path name as a string. TestPathLike = Union[str, TestPath] # output_handler: Callable[[ # List[TestPathLike], List[TestPathLike]], None] def __init__(self, dry_run=False): self.test_paths = [] self.output_handler = self._default_output_handler super(Optimize, self).__init__(dry_run=dry_run) def _default_output_handler(self, output, rests): # regardless of whether we managed to talk to the service we produce # test names if rest: if len(rests) == 0: rests.append(output[0]) self.write_file(rest, rests) self.print(output) def test_path(self, path: TestPathLike): def rel_base_path(path): if isinstance(path, str): return pathlib.Path( file_path_normalizer.relativize(path)).as_posix() else: return path if isinstance(path, str) and any(s in path for s in ('*', "?")): for i in glob.iglob(path, recursive=True): if os.path.isfile(i): self.test_paths.append( self.to_test_path(rel_base_path(i))) return """register one test""" self.test_paths.append(self.to_test_path(rel_base_path(path))) def stdin(self): """ Returns sys.stdin, but after ensuring that it's connected to something reasonable. This prevents a typical problem where users think CLI is hanging because they didn't feed anything from stdin """ if sys.stdin.isatty(): click.echo(click.style( "Warning: this command reads from stdin but it doesn't appear to be connected to anything. Did you forget to pipe from another command?", fg='yellow'), err=True) return sys.stdin @staticmethod def to_test_path(x: TestPathLike) -> TestPath: """Convert input to a TestPath""" if isinstance(x, str): # default representation for a file return [{'type': 'file', 'name': x}] else: return x def scan(self, base: str, pattern: str, path_builder: Optional[Callable[[str], Union[TestPath, str, None]]] = None): """ Starting at the 'base' path, recursively add everything that matches the given GLOB pattern scan('src/test/java', '**/*.java') 'path_builder' is a function used to map file name into a custom test path. It takes a single string argument that represents the portion matched to the glob pattern, and its return value controls what happens to that file: - skip a file by returning a False-like object - if a str is returned, that's interpreted as a path name and converted to the default test path representation. Typically, `os.path.join(base,file_name) - if a TestPath is returned, that's added as is """ if path_builder == None: # default implementation of path_builder creates a file name relative to `source` so as not # to be affected by the path def default_path_builder(file_name): return pathlib.Path( file_path_normalizer.relativize(join( base, file_name))).as_posix() path_builder = default_path_builder for b in glob.iglob(base): for t in glob.iglob(join(b, pattern), recursive=True): if path_builder: path = path_builder(os.path.relpath(t, b)) if path: self.test_paths.append(self.to_test_path(path)) def get_payload(self, session_id, target, duration): payload = { "testPaths": self.test_paths, "session": { # expecting just the last component, not the whole path "id": os.path.basename(session_id) } } if target is not None: payload["target"] = target elif duration is not None: payload["goal"] = { "type": "subset-by-absolute-time", "duration": duration, } elif confidence is not None: payload["goal"] = { "type": "subset-by-confidence", "percentage": confidence } return payload def run(self): """called after tests are scanned to compute the optimized order""" # When Error occurs, return the test name as it is passed. output = self.test_paths rests = [] summary = {} subset_id = "" is_brainless = False if not session_id: # Session ID in --session is missing. It might be caused by Launchable API errors. pass else: try: client = LaunchableClient( test_runner=context.invoked_subcommand, dry_run=context.obj.dry_run) # temporarily extend the timeout because subset API response has become slow # TODO: remove this line when API response return respose within 60 sec timeout = (5, 180) payload = self.get_payload(session_id, target, duration) res = client.request("post", "subset", timeout=timeout, payload=payload, compress=True) res.raise_for_status() output = res.json()["testPaths"] rests = res.json()["rest"] subset_id = res.json()["subsettingId"] summary = res.json()["summary"] is_brainless = res.json()["isBrainless"] except Exception as e: if os.getenv(REPORT_ERROR_KEY): raise e else: click.echo(e, err=True) click.echo(click.style( "Warning: the service failed to subset. Falling back to running all tests", fg='yellow'), err=True) if len(output) == 0: click.echo(click.style( "Error: no tests found matching the path.", 'yellow'), err=True) return if split: click.echo("subset/{}".format(subset_id)) else: self.output_handler(output, rests) # When Launchable returns an error, the cli skips showing summary report if "subset" not in summary.keys() or "rest" not in summary.keys(): return build_name, test_session_id = parse_session(session_id) org, workspace = get_org_workspace() header = [ "", "Candidates", "Estimated duration (%)", "Estimated duration (min)" ] rows = [ [ "Subset", len(output), summary["subset"].get("rate", 0.0), summary["subset"].get("duration", 0.0), ], [ "Remainder", len(rests), summary["rest"].get("rate", 0.0), summary["rest"].get("duration", 0.0), ], [], [ "Total", len(output) + len(rests), summary["subset"].get("rate", 0.0) + summary["rest"].get("rate", 0.0), summary["subset"].get("rate", 0.0) + summary["rest"].get("duration", 0.0), ], ] if is_brainless: click.echo("Your model is currently in training", err=True) click.echo( "Launchable created subset {} for build {} (test session {}) in workspace {}/{}\n" .format(subset_id, build_name, test_session_id, org, workspace), err=True) click.echo(tabulate(rows, header, tablefmt="github"), err=True) click.echo( "\nRun `launchable inspect subset --subset-id {}` to view full subset details" .format(subset_id), err=True) context.obj = Optimize(dry_run=context.obj.dry_run)
def route(ctx: click.core.Context, url: str, token: str): """ Allow you to perform actions on routes """ ctx.obj = ModelRouteClient(url, token)
def training(ctx: click.core.Context, url: str, token: str): """ Local training process.\n Alias for the command is train. """ ctx.obj = ModelTrainingClient(url, token)
def packaging_group(ctx: click.core.Context, url: str, token: str): """ Local packaging process.\n Alias for the command is pack. """ ctx.obj = ModelPackagingClient(url, token)