def list_images(author: str = 'azure', latest: bool = False) -> None:
try:
author = f'private-{author}' if author != 'azure' else author
repo = f'azurecli-test-{author}'
output = check_output(shlex.split(
f'az acr repository show-tags --repository {repo} '
f'-n {DROID_CONTAINER_REGISTRY} -ojson',
posix=not IS_WINDOWS),
shell=IS_WINDOWS)
# assume all on same python version and platform
image_list = [
_DroidImage(f'{DROID_CONTAINER_REGISTRY}.azurecr.io', repo, tag)
for tag in json.loads(output)
]
image_list = sorted(image_list,
key=lambda img: img.build_number,
reverse=True)
if latest:
print(image_list[0].__repr__())
else:
print(
tabulate.tabulate([{
'image': img.__repr__()
} for img in image_list]))
except CalledProcessError:
get_logger(__name__).exception(f'Fail to list images for {repo}')
def login(endpoint: str,
service_principal: bool = False,
username: str = None,
password: str = None) -> None:
logger = get_logger('login')
try:
requests.get(f'https://{endpoint}/api/healthy').raise_for_status()
except (requests.HTTPError, requests.ConnectionError):
logger.error(f'Cannot reach endpoint https://{endpoint}/api/healthy')
sys.exit(1)
auth = AuthSettings()
if service_principal:
if not username or not password:
logger.error('Username or password is missing.')
sys.exit(1)
if not auth.login_service_principal(username, password):
logger.error(f'Fail to login using service principal {username}.')
sys.exit(1)
else:
if not auth.login():
sys.exit(1)
config = A01Config()
config.endpoint = endpoint
if not config.save():
logger.error(f'Cannot read or write to file {CONFIG_FILE}')
sys.exit(1)
sys.exit(0)
class RunCollection(object):
logger = get_logger('RunCollection')
def __init__(self, runs: List[Run]) -> None:
self.runs = runs
def get_table_view(self) -> Generator[List, None, None]:
for run in self.runs:
time = (run.creation -
datetime.timedelta(hours=8)).strftime('%Y-%m-%d %H:%M PST')
remark = run.details.get('remark', None) or run.settings.get(
'a01.reserved.remark', '')
owner = run.owner or run.details.get(
'creator', None) or run.details.get('a01.reserved.creator', '')
status = run.status
row = [run.id, run.name, time, status, remark, owner]
if remark and remark.lower() == 'official':
for i, column in enumerate(row):
row[i] = colorama.Style.BRIGHT + str(
column) + colorama.Style.RESET_ALL
yield row
@staticmethod
def get_table_header() -> Tuple:
return 'Id', 'Name', 'Creation', 'Status', 'Remark', 'Owner'
@classmethod
def get(cls, **kwargs) -> 'RunCollection':
try:
url = f'{cls.endpoint_uri()}/runs'
query = {}
for key, value in kwargs.items():
if value is not None:
query[key] = value
if query:
url = f'{url}?{urllib.parse.urlencode(query)}'
resp = session.get(url)
resp.raise_for_status()
runs = [Run.from_dict(each) for each in resp.json()]
runs = sorted(runs, key=lambda r: r.id, reverse=True)
return RunCollection(runs)
except HTTPError:
cls.logger.debug('HttpError', exc_info=True)
raise ValueError('Fail to get runs.')
except (KeyError, json.JSONDecodeError, TypeError):
cls.logger.debug('JsonError', exc_info=True)
raise ValueError('Fail to parse the runs data.')
@staticmethod
def endpoint_uri():
config = A01Config()
config.ensure_config()
return config.endpoint_uri
def __init__(self):
self.logger = get_logger(__class__.__name__)
self._token_raw = None
try:
with open(TOKEN_FILE, 'r') as token_file:
self._token_raw = json.load(token_file)
except IOError:
self.logger.info(f'Token file {TOKEN_FILE} missing.')
except (json.JSONDecodeError, TypeError):
self.logger.exception(f'Fail to parse the file {TOKEN_FILE}.')
def __init__(self, image_name: str) -> None:
self.logger = get_logger(DroidImage.__class__.__name__)
self.image_name = image_name
try:
self.repo, self.tag = image_name.rsplit(':', 1)
except ValueError:
raise ValueError('Incorrect image name. The tag is missing.')
self.client = docker.from_env()
self._image = None
def __init__(self):
self.logger = get_logger(__class__.__name__)
self.auth = AuthSettings()
from a01.common import get_logger, A01Config
from a01.cli import cmd, arg
from a01.communication import session
# pylint: disable=too-many-arguments, invalid-name
logger = get_logger(__name__)
@cmd('restart run', desc='Restart a run. This command is used when the Kubernetes Job behaves abnormally. It will '
'create new group of controller job and test job with the same settings.')
@arg('run_id', help='Then run to restart', positional=True)
def restart_run(run_id: str):
config = A01Config()
resp = session.post(f'{config.endpoint_uri}/run/{run_id}/restart')
resp.raise_for_status()
@cmd('delete run', desc='Delete a run as well as the tasks associate with it.')
@arg('run_id', help='Ids of the run to be deleted.', positional=True)
def delete_run(run_id: str) -> None:
config = A01Config()
resp = session.delete(f'{config.endpoint_uri}/run/{run_id}')
resp.raise_for_status()
class TaskCollection(object):
logger = get_logger('TaskCollection')
def __init__(self, tasks: List[Task], run_id: str) -> None:
self.tasks = sorted(tasks, key=lambda t: t.identifier)
self.run_name = run_id
def get_failed_tasks(self) -> Generator[Task, None, None]:
for task in self.tasks:
if task.result == 'Passed' or task.status == 'initialized':
continue
yield task
def post(self) -> None:
try:
payload = [t.to_dict() for t in self.tasks]
session.post(f'{self.endpoint_uri()}/run/{self.run_name}/tasks',
json=payload).raise_for_status()
except HTTPError:
self.logger.debug('HttpError', exc_info=True)
raise ValueError('Failed to create tasks in the task store.')
@classmethod
def get(cls, run_id: str) -> 'TaskCollection':
try:
resp = session.get(f'{cls.endpoint_uri()}/run/{run_id}/tasks')
resp.raise_for_status()
tasks = [Task.from_dict(each) for each in resp.json()]
return TaskCollection(tasks=tasks, run_id=run_id)
except HTTPError as error:
cls.logger.debug('HttpError', exc_info=True)
if error.response.status_code == 404:
raise ValueError(f'Run {run_id} is not found.')
raise ValueError('Fail to get runs.')
except (KeyError, json.JSONDecodeError, TypeError):
cls.logger.debug('JsonError', exc_info=True)
raise ValueError('Fail to parse the runs data.')
@staticmethod
def endpoint_uri():
config = A01Config()
config.ensure_config()
return config.endpoint_uri
def get_summary(self) -> Tuple[Tuple[str, str], Tuple[str, str]]:
statuses = defaultdict(lambda: 0)
results = defaultdict(lambda: 0)
for task in self.tasks:
statuses[task.status] = statuses[task.status] + 1
results[task.result] = results[task.result] + 1
status_summary = ' | '.join([
f'{status_name}: {count}'
for status_name, count in statuses.items()
])
result_summary = f'{colorama.Fore.GREEN}Pass: {results["Passed"]}{colorama.Fore.RESET} | ' \
f'{colorama.Fore.RED}Fail: {results["Failed"]}{colorama.Fore.RESET} | ' \
f'Error: {results["Error"]}'
return ('Task', status_summary), ('Result', result_summary)
def get_table_view(
self,
failed: bool = True) -> Generator[Tuple[str, ...], None, None]:
for task in self.get_failed_tasks() if failed else self.tasks:
yield task.get_table_view()
@staticmethod
def get_table_header() -> Tuple[str, ...]:
return Task.get_table_header()
class Task(object): # pylint: disable=too-many-instance-attributes
logger = get_logger('Task')
def __init__(self, name: str, annotation: str, settings: dict) -> None:
self.name = name
self.annotation = annotation
self.settings = settings
self.id = None # pylint: disable=invalid-name
self.status = None
self.result_details = {} # dict
self.result = None
self.duration = None # int
self.run_id = None
@property
def identifier(self) -> str:
return self.settings['classifier']['identifier']
@property
def command(self) -> str:
return self.settings['execution']['command']
@classmethod
def get(cls, task_id: str) -> 'Task':
try:
resp = session.get(f'{cls.endpoint_uri()}/task/{task_id}')
resp.raise_for_status()
return Task.from_dict(resp.json())
except requests.HTTPError as error:
cls.logger.debug('HttpError', exc_info=True)
if error.response.status_code == 404:
raise ValueError(f'Task {task_id} is not found.')
raise ValueError('Fail to get runs.')
except (KeyError, json.JSONDecodeError, TypeError):
cls.logger.debug('JsonError', exc_info=True)
raise ValueError('Fail to parse the runs data.')
def to_dict(self) -> dict:
result = {
'name': self.name,
'annotation': self.annotation,
'settings': self.settings,
}
return result
@staticmethod
def from_dict(data: dict) -> 'Task':
result = Task(name=data['name'], annotation=data['annotation'], settings=data['settings'])
result.id = str(data['id'])
result.status = data['status']
result.result_details = data['result_details'] or {}
result.result = data['result']
result.duration = data['duration']
result.run_id = str(data['run_id'])
return result
def get_log_content(self) -> Generator[str, None, None]:
log_path = self.result_details.get('a01.reserved.tasklogpath', None)
if not log_path:
return
resp = requests.get(log_path)
if resp.status_code == 404:
yield '>', 'Log not found (task might still be running, or storage was not setup for this run)\n'
for index, line in enumerate(resp.content.decode('utf-8').split('\n')):
yield '>', f' {index}\t{line}'
def get_table_view(self) -> Tuple[str, ...]:
return self.id, self.name, self.status, self.result, self.result_details.get('agent', None), self.duration
@staticmethod
def get_table_header() -> Tuple[str, ...]:
return 'Id', 'Name', 'Status', 'Result', 'Agent', 'Duration(ms)'
def download_recording(self, az_mode: bool) -> None:
recording_path = self.result_details.get('a01.reserved.taskrecordpath', None)
if not recording_path:
return
resp = requests.get(recording_path)
if resp.status_code != 200:
return
path_paths = self.settings['classifier']['identifier'].split('.')
if az_mode:
module_name = path_paths[3]
method_name = path_paths[-1]
profile_name = path_paths[-4]
recording_path = os.path.join('recording', f'azure-cli-{module_name}', 'azure', 'cli', 'command_modules',
module_name, 'tests', profile_name, 'recordings', f'{method_name}.yaml')
else:
path_paths[-1] = path_paths[-1] + '.yaml'
path_paths.insert(0, 'recording')
recording_path = os.path.join(*path_paths)
os.makedirs(os.path.dirname(recording_path), exist_ok=True)
with open(recording_path, 'wb') as recording_file:
recording_file.write(resp.content)
@staticmethod
def endpoint_uri():
config = A01Config()
config.ensure_config()
return config.endpoint_uri
class Run(object):
logger = get_logger('Run')
def __init__(
self,
name: str,
settings: dict,
details: dict,
owner: str, # pylint: disable=too-many-arguments
status: str) -> None:
self.name = name
self.settings = settings
self.details = details
self.owner = owner
self.status = status
self.id = None # pylint: disable=invalid-name
self.creation = None
# prune
to_delete = [k for k, v in self.settings.items() if not v]
for k in to_delete:
del self.settings[k]
to_delete = [k for k, v in self.details.items() if not v]
for k in to_delete:
del self.details[k]
@property
def image(self) -> str:
return self.settings['a01.reserved.imagename']
@property
def product(self) -> str:
return self.details['a01.reserved.product']
def to_dict(self) -> dict:
result = {
'name': self.name,
'settings': self.settings,
'details': self.details,
'owner': self.owner,
'status': self.status
}
return result
@classmethod
def get(cls, run_id: str) -> 'Run':
try:
resp = session.get(f'{cls.endpoint_uri()}/run/{run_id}')
resp.raise_for_status()
return Run.from_dict(resp.json())
except HTTPError:
cls.logger.debug('HttpError', exc_info=True)
raise ValueError('Failed to find the run in the task store.')
except (json.JSONDecodeError, TypeError):
cls.logger.debug('JsonError', exc_info=True)
raise ValueError('Failed to deserialize the response content.')
def post(self) -> 'Run':
try:
resp = session.post(f'{self.endpoint_uri()}/run',
json=self.to_dict())
return Run.from_dict(resp.json())
except HTTPError:
self.logger.debug('HttpError', exc_info=True)
raise ValueError('Failed to create run in the task store.')
except (json.JSONDecodeError, TypeError):
self.logger.debug('JsonError', exc_info=True)
raise ValueError('Failed to deserialize the response content.')
@staticmethod
def from_dict(data: dict) -> 'Run':
result = Run(name=data['name'],
settings=data['settings'],
details=data['details'],
owner=data.get('owner', None),
status=data.get('status', 'N/A'))
result.id = data['id']
result.creation = datetime.datetime.strptime(data['creation'],
'%Y-%m-%dT%H:%M:%SZ')
return result
@staticmethod
def endpoint_uri():
config = A01Config()
config.ensure_config()
return config.endpoint_uri
class Task(object): # pylint: disable=too-many-instance-attributes
logger = get_logger('Task')
def __init__(self, name: str, annotation: str, settings: dict) -> None:
self.name = name
self.annotation = annotation
self.settings = settings
self.id = None # pylint: disable=invalid-name
self.status = None
self.result_details = {} # dict
self.result = None
self.duration = None # int
self.run_id = None
@property
def identifier(self) -> str:
return self.settings['classifier']['identifier']
@property
def command(self) -> str:
return self.settings['execution']['command']
@property
def log_resource_uri(self):
return self.result_details.get('a01.reserved.tasklogpath', None)
@property
def record_resource_uri(self):
return self.result_details.get('a01.reserved.taskrecordpath', None)
def to_dict(self) -> dict:
result = {
'name': self.name,
'annotation': self.annotation,
'settings': self.settings,
}
return result
@staticmethod
def from_dict(data: dict) -> 'Task':
result = Task(name=data['name'],
annotation=data['annotation'],
settings=data['settings'])
result.id = str(data['id'])
result.status = data['status']
result.result_details = data['result_details'] or {}
result.result = data['result']
result.duration = data['duration']
result.run_id = str(data['run_id'])
return result
def get_table_view(self) -> Tuple[str, ...]:
return self.id, self.name, self.status, self.result, self.result_details.get(
'agent', None), self.duration
@staticmethod
def get_table_header() -> Tuple[str, ...]:
return 'Id', 'Name', 'Status', 'Result', 'Agent', 'Duration(ms)'
import argparse
import inspect
from typing import Callable, Collection
from a01.common import get_logger
logger = get_logger(__name__) # pylint: disable=invalid-name
COMMAND_TABLE = dict()
def cmd(name: str, desc: str = None):
logger.info(f'register command [{name}]')
def _decorator(func):
if hasattr(func, '__command_definition'):
raise SyntaxError(f'Duplicate @cmd decorator on {func}')
command_definition = CommandDefinition(name, func, desc)
COMMAND_TABLE[name] = command_definition
argument_definitions = getattr(func, '__argument_definitions', None)
if argument_definitions:
for each in argument_definitions:
command_definition.add_argument(each)
delattr(func, '__argument_definitions')
setattr(func, '__command_definition', COMMAND_TABLE[name])
return func
