def list_images(author: str = 'azure', latest: bool = False) -> None: try: author = f'private-{author}' if author != 'azure' else author repo = f'azurecli-test-{author}' output = check_output(shlex.split( f'az acr repository show-tags --repository {repo} ' f'-n {DROID_CONTAINER_REGISTRY} -ojson', posix=not IS_WINDOWS), shell=IS_WINDOWS) # assume all on same python version and platform image_list = [ _DroidImage(f'{DROID_CONTAINER_REGISTRY}.azurecr.io', repo, tag) for tag in json.loads(output) ] image_list = sorted(image_list, key=lambda img: img.build_number, reverse=True) if latest: print(image_list[0].__repr__()) else: print( tabulate.tabulate([{ 'image': img.__repr__() } for img in image_list])) except CalledProcessError: get_logger(__name__).exception(f'Fail to list images for {repo}')
def login(endpoint: str, service_principal: bool = False, username: str = None, password: str = None) -> None: logger = get_logger('login') try: requests.get(f'https://{endpoint}/api/healthy').raise_for_status() except (requests.HTTPError, requests.ConnectionError): logger.error(f'Cannot reach endpoint https://{endpoint}/api/healthy') sys.exit(1) auth = AuthSettings() if service_principal: if not username or not password: logger.error('Username or password is missing.') sys.exit(1) if not auth.login_service_principal(username, password): logger.error(f'Fail to login using service principal {username}.') sys.exit(1) else: if not auth.login(): sys.exit(1) config = A01Config() config.endpoint = endpoint if not config.save(): logger.error(f'Cannot read or write to file {CONFIG_FILE}') sys.exit(1) sys.exit(0)
class RunCollection(object): logger = get_logger('RunCollection') def __init__(self, runs: List[Run]) -> None: self.runs = runs def get_table_view(self) -> Generator[List, None, None]: for run in self.runs: time = (run.creation - datetime.timedelta(hours=8)).strftime('%Y-%m-%d %H:%M PST') remark = run.details.get('remark', None) or run.settings.get( 'a01.reserved.remark', '') owner = run.owner or run.details.get( 'creator', None) or run.details.get('a01.reserved.creator', '') status = run.status row = [run.id, run.name, time, status, remark, owner] if remark and remark.lower() == 'official': for i, column in enumerate(row): row[i] = colorama.Style.BRIGHT + str( column) + colorama.Style.RESET_ALL yield row @staticmethod def get_table_header() -> Tuple: return 'Id', 'Name', 'Creation', 'Status', 'Remark', 'Owner' @classmethod def get(cls, **kwargs) -> 'RunCollection': try: url = f'{cls.endpoint_uri()}/runs' query = {} for key, value in kwargs.items(): if value is not None: query[key] = value if query: url = f'{url}?{urllib.parse.urlencode(query)}' resp = session.get(url) resp.raise_for_status() runs = [Run.from_dict(each) for each in resp.json()] runs = sorted(runs, key=lambda r: r.id, reverse=True) return RunCollection(runs) except HTTPError: cls.logger.debug('HttpError', exc_info=True) raise ValueError('Fail to get runs.') except (KeyError, json.JSONDecodeError, TypeError): cls.logger.debug('JsonError', exc_info=True) raise ValueError('Fail to parse the runs data.') @staticmethod def endpoint_uri(): config = A01Config() config.ensure_config() return config.endpoint_uri
def __init__(self): self.logger = get_logger(__class__.__name__) self._token_raw = None try: with open(TOKEN_FILE, 'r') as token_file: self._token_raw = json.load(token_file) except IOError: self.logger.info(f'Token file {TOKEN_FILE} missing.') except (json.JSONDecodeError, TypeError): self.logger.exception(f'Fail to parse the file {TOKEN_FILE}.')
def __init__(self, image_name: str) -> None: self.logger = get_logger(DroidImage.__class__.__name__) self.image_name = image_name try: self.repo, self.tag = image_name.rsplit(':', 1) except ValueError: raise ValueError('Incorrect image name. The tag is missing.') self.client = docker.from_env() self._image = None
def __init__(self): self.logger = get_logger(__class__.__name__) self.auth = AuthSettings()
from a01.common import get_logger, A01Config from a01.cli import cmd, arg from a01.communication import session # pylint: disable=too-many-arguments, invalid-name logger = get_logger(__name__) @cmd('restart run', desc='Restart a run. This command is used when the Kubernetes Job behaves abnormally. It will ' 'create new group of controller job and test job with the same settings.') @arg('run_id', help='Then run to restart', positional=True) def restart_run(run_id: str): config = A01Config() resp = session.post(f'{config.endpoint_uri}/run/{run_id}/restart') resp.raise_for_status() @cmd('delete run', desc='Delete a run as well as the tasks associate with it.') @arg('run_id', help='Ids of the run to be deleted.', positional=True) def delete_run(run_id: str) -> None: config = A01Config() resp = session.delete(f'{config.endpoint_uri}/run/{run_id}') resp.raise_for_status()
class TaskCollection(object): logger = get_logger('TaskCollection') def __init__(self, tasks: List[Task], run_id: str) -> None: self.tasks = sorted(tasks, key=lambda t: t.identifier) self.run_name = run_id def get_failed_tasks(self) -> Generator[Task, None, None]: for task in self.tasks: if task.result == 'Passed' or task.status == 'initialized': continue yield task def post(self) -> None: try: payload = [t.to_dict() for t in self.tasks] session.post(f'{self.endpoint_uri()}/run/{self.run_name}/tasks', json=payload).raise_for_status() except HTTPError: self.logger.debug('HttpError', exc_info=True) raise ValueError('Failed to create tasks in the task store.') @classmethod def get(cls, run_id: str) -> 'TaskCollection': try: resp = session.get(f'{cls.endpoint_uri()}/run/{run_id}/tasks') resp.raise_for_status() tasks = [Task.from_dict(each) for each in resp.json()] return TaskCollection(tasks=tasks, run_id=run_id) except HTTPError as error: cls.logger.debug('HttpError', exc_info=True) if error.response.status_code == 404: raise ValueError(f'Run {run_id} is not found.') raise ValueError('Fail to get runs.') except (KeyError, json.JSONDecodeError, TypeError): cls.logger.debug('JsonError', exc_info=True) raise ValueError('Fail to parse the runs data.') @staticmethod def endpoint_uri(): config = A01Config() config.ensure_config() return config.endpoint_uri def get_summary(self) -> Tuple[Tuple[str, str], Tuple[str, str]]: statuses = defaultdict(lambda: 0) results = defaultdict(lambda: 0) for task in self.tasks: statuses[task.status] = statuses[task.status] + 1 results[task.result] = results[task.result] + 1 status_summary = ' | '.join([ f'{status_name}: {count}' for status_name, count in statuses.items() ]) result_summary = f'{colorama.Fore.GREEN}Pass: {results["Passed"]}{colorama.Fore.RESET} | ' \ f'{colorama.Fore.RED}Fail: {results["Failed"]}{colorama.Fore.RESET} | ' \ f'Error: {results["Error"]}' return ('Task', status_summary), ('Result', result_summary) def get_table_view( self, failed: bool = True) -> Generator[Tuple[str, ...], None, None]: for task in self.get_failed_tasks() if failed else self.tasks: yield task.get_table_view() @staticmethod def get_table_header() -> Tuple[str, ...]: return Task.get_table_header()
class Task(object): # pylint: disable=too-many-instance-attributes logger = get_logger('Task') def __init__(self, name: str, annotation: str, settings: dict) -> None: self.name = name self.annotation = annotation self.settings = settings self.id = None # pylint: disable=invalid-name self.status = None self.result_details = {} # dict self.result = None self.duration = None # int self.run_id = None @property def identifier(self) -> str: return self.settings['classifier']['identifier'] @property def command(self) -> str: return self.settings['execution']['command'] @classmethod def get(cls, task_id: str) -> 'Task': try: resp = session.get(f'{cls.endpoint_uri()}/task/{task_id}') resp.raise_for_status() return Task.from_dict(resp.json()) except requests.HTTPError as error: cls.logger.debug('HttpError', exc_info=True) if error.response.status_code == 404: raise ValueError(f'Task {task_id} is not found.') raise ValueError('Fail to get runs.') except (KeyError, json.JSONDecodeError, TypeError): cls.logger.debug('JsonError', exc_info=True) raise ValueError('Fail to parse the runs data.') def to_dict(self) -> dict: result = { 'name': self.name, 'annotation': self.annotation, 'settings': self.settings, } return result @staticmethod def from_dict(data: dict) -> 'Task': result = Task(name=data['name'], annotation=data['annotation'], settings=data['settings']) result.id = str(data['id']) result.status = data['status'] result.result_details = data['result_details'] or {} result.result = data['result'] result.duration = data['duration'] result.run_id = str(data['run_id']) return result def get_log_content(self) -> Generator[str, None, None]: log_path = self.result_details.get('a01.reserved.tasklogpath', None) if not log_path: return resp = requests.get(log_path) if resp.status_code == 404: yield '>', 'Log not found (task might still be running, or storage was not setup for this run)\n' for index, line in enumerate(resp.content.decode('utf-8').split('\n')): yield '>', f' {index}\t{line}' def get_table_view(self) -> Tuple[str, ...]: return self.id, self.name, self.status, self.result, self.result_details.get('agent', None), self.duration @staticmethod def get_table_header() -> Tuple[str, ...]: return 'Id', 'Name', 'Status', 'Result', 'Agent', 'Duration(ms)' def download_recording(self, az_mode: bool) -> None: recording_path = self.result_details.get('a01.reserved.taskrecordpath', None) if not recording_path: return resp = requests.get(recording_path) if resp.status_code != 200: return path_paths = self.settings['classifier']['identifier'].split('.') if az_mode: module_name = path_paths[3] method_name = path_paths[-1] profile_name = path_paths[-4] recording_path = os.path.join('recording', f'azure-cli-{module_name}', 'azure', 'cli', 'command_modules', module_name, 'tests', profile_name, 'recordings', f'{method_name}.yaml') else: path_paths[-1] = path_paths[-1] + '.yaml' path_paths.insert(0, 'recording') recording_path = os.path.join(*path_paths) os.makedirs(os.path.dirname(recording_path), exist_ok=True) with open(recording_path, 'wb') as recording_file: recording_file.write(resp.content) @staticmethod def endpoint_uri(): config = A01Config() config.ensure_config() return config.endpoint_uri
class Run(object): logger = get_logger('Run') def __init__( self, name: str, settings: dict, details: dict, owner: str, # pylint: disable=too-many-arguments status: str) -> None: self.name = name self.settings = settings self.details = details self.owner = owner self.status = status self.id = None # pylint: disable=invalid-name self.creation = None # prune to_delete = [k for k, v in self.settings.items() if not v] for k in to_delete: del self.settings[k] to_delete = [k for k, v in self.details.items() if not v] for k in to_delete: del self.details[k] @property def image(self) -> str: return self.settings['a01.reserved.imagename'] @property def product(self) -> str: return self.details['a01.reserved.product'] def to_dict(self) -> dict: result = { 'name': self.name, 'settings': self.settings, 'details': self.details, 'owner': self.owner, 'status': self.status } return result @classmethod def get(cls, run_id: str) -> 'Run': try: resp = session.get(f'{cls.endpoint_uri()}/run/{run_id}') resp.raise_for_status() return Run.from_dict(resp.json()) except HTTPError: cls.logger.debug('HttpError', exc_info=True) raise ValueError('Failed to find the run in the task store.') except (json.JSONDecodeError, TypeError): cls.logger.debug('JsonError', exc_info=True) raise ValueError('Failed to deserialize the response content.') def post(self) -> 'Run': try: resp = session.post(f'{self.endpoint_uri()}/run', json=self.to_dict()) return Run.from_dict(resp.json()) except HTTPError: self.logger.debug('HttpError', exc_info=True) raise ValueError('Failed to create run in the task store.') except (json.JSONDecodeError, TypeError): self.logger.debug('JsonError', exc_info=True) raise ValueError('Failed to deserialize the response content.') @staticmethod def from_dict(data: dict) -> 'Run': result = Run(name=data['name'], settings=data['settings'], details=data['details'], owner=data.get('owner', None), status=data.get('status', 'N/A')) result.id = data['id'] result.creation = datetime.datetime.strptime(data['creation'], '%Y-%m-%dT%H:%M:%SZ') return result @staticmethod def endpoint_uri(): config = A01Config() config.ensure_config() return config.endpoint_uri
class Task(object): # pylint: disable=too-many-instance-attributes logger = get_logger('Task') def __init__(self, name: str, annotation: str, settings: dict) -> None: self.name = name self.annotation = annotation self.settings = settings self.id = None # pylint: disable=invalid-name self.status = None self.result_details = {} # dict self.result = None self.duration = None # int self.run_id = None @property def identifier(self) -> str: return self.settings['classifier']['identifier'] @property def command(self) -> str: return self.settings['execution']['command'] @property def log_resource_uri(self): return self.result_details.get('a01.reserved.tasklogpath', None) @property def record_resource_uri(self): return self.result_details.get('a01.reserved.taskrecordpath', None) def to_dict(self) -> dict: result = { 'name': self.name, 'annotation': self.annotation, 'settings': self.settings, } return result @staticmethod def from_dict(data: dict) -> 'Task': result = Task(name=data['name'], annotation=data['annotation'], settings=data['settings']) result.id = str(data['id']) result.status = data['status'] result.result_details = data['result_details'] or {} result.result = data['result'] result.duration = data['duration'] result.run_id = str(data['run_id']) return result def get_table_view(self) -> Tuple[str, ...]: return self.id, self.name, self.status, self.result, self.result_details.get( 'agent', None), self.duration @staticmethod def get_table_header() -> Tuple[str, ...]: return 'Id', 'Name', 'Status', 'Result', 'Agent', 'Duration(ms)'
import argparse import inspect from typing import Callable, Collection from a01.common import get_logger logger = get_logger(__name__) # pylint: disable=invalid-name COMMAND_TABLE = dict() def cmd(name: str, desc: str = None): logger.info(f'register command [{name}]') def _decorator(func): if hasattr(func, '__command_definition'): raise SyntaxError(f'Duplicate @cmd decorator on {func}') command_definition = CommandDefinition(name, func, desc) COMMAND_TABLE[name] = command_definition argument_definitions = getattr(func, '__argument_definitions', None) if argument_definitions: for each in argument_definitions: command_definition.add_argument(each) delattr(func, '__argument_definitions') setattr(func, '__command_definition', COMMAND_TABLE[name]) return func