def test_search_app(self):
m = Manifest(bucket='bucket',
repo_name='repo',
storage=StorageMock(self.SEARCH_DATA))
found = m.search(app_name='spark')
expected = [
{
'app': 'spark',
'branch': 'dev',
'commit': '111111',
'built_at': '2018-11-01T05:01:01.000001+00:00',
'url': 'gs://test_file.cfg'
},
{
'app': 'spark',
'branch': 'master',
'commit': '222222',
'built_at': '2019-12-01T05:01:01.000001+00:00',
'url': 'gs://app.jar'
},
{
'app': 'spark',
'branch': 'master',
'commit': '222222',
'built_at': '2019-12-01T05:01:01.000001+00:00',
'url': 'gs://app.cfg'
},
]
self.assertEqual(expected, found)
def test_search_not_found(self):
m = Manifest(bucket='bucket',
repo_name='repo',
storage=StorageMock(self.SEARCH_DATA))
found = m.search(app_name='no_wired_no_world')
expected = []
self.assertEqual(expected, found)
def list(ctx, bucket, repo, app, branch, include_fields):
"""
Listing for all latest build binaries (sorted by: branch, app name, time).
[ mfutil builds latest ls ] will prints all last success built binaries for all branches.
[ mfutil builds latest ls --branch <branch-name> ] will prints all binaries for target branch
[ mfutil builds latest ls --branch <branch-name> --app <app-name> ] will prints all binaries for target branch and target app
"""
ctx.ensure_object(dict)
project = ctx.obj[PROJECT_OPT]
if project is None and (bucket is None and repo is None):
click.echo(
f'Config file not found in [{ctx.obj["root_dir"]}] and --bucket not specifies.\n'
'Please specify --bucket and --repo parameters or --config file path',
err=True)
return 1
manifest = Manifest(project.bucket, project.repository)
binaries_list = manifest.search(branch_name=branch, app_name=app)
if len(binaries_list) == 0:
click.echo('no builds found...')
if include_fields:
keys = set(str(include_fields).split(','))
fields_filter = lambda d: dict(
filter(lambda kv: kv[0] in keys, d.items()))
else:
fields_filter = lambda d: d
format_ = ctx.obj[FORMAT_OPT]
if format_ == 'json':
for d in binaries_list:
click.echo(json.dumps(fields_filter(d)))
elif format_ == 'csv':
w = csv.DictWriter(sys.stdout,
fieldnames=fields_filter(binaries_list[0]).keys()
if len(binaries_list) > 0 else set())
w.writeheader()
for d in binaries_list:
w.writerow(fields_filter(d))
elif format_ == 'text':
w = csv.DictWriter(sys.stdout,
delimiter='\t',
fieldnames=fields_filter(binaries_list[0]).keys()
if len(binaries_list) > 0 else set())
for d in binaries_list:
w.writerow(fields_filter(d))
def test_generate_manifest(self):
branch_name = 'dev'
m = Manifest(bucket='BUCKET', repo_name='ARepo', storage=StorageMock())
b = BuildInfo(git_sha='431refrqewr',
git_branch=branch_name,
build_id='aaaa-bbb-ccc',
date=datetime(2018, 11, 1, 5, 1, 1, 1))
p = Project({
'bucket': 'BUCKET',
'repository': 'ARepo',
'components': {
'spark': {
'type': 'some',
'assets': [{
'glob': './**/test_dir/test_file.cfg'
}]
}
}
})
content = m.update(b, p, False)
expected = \
{
'@ns': {
'dev': {
'@last_success': {
'@build_id': 'aaaa-bbb-ccc',
'@built_at': '2018-11-01T05:01:01.000001+00:00',
'@rev': '431refrqewr',
'@include': {
'spark': {
'@binaries': [
{
'@md5': '1B2M2Y8AsgTpgAmY7PhCfg==',
'@ref': 'gs://BUCKET/ARepo/dev/431refrqewr/spark/test_file.cfg'
}
],
'@metadata': {},
'@type': 'some'
}
}
}
}
}
}
self.assertEqual(expected, content)
def test_search_app_and_branch(self):
m = Manifest(bucket='bucket',
repo_name='repo',
storage=StorageMock(self.SEARCH_DATA))
found = m.search(app_name='pyspark', branch_name='master')
expected = [
{
'app': 'pyspark',
'branch': 'master',
'commit': '222222',
'built_at': '2019-12-01T05:01:01.000001+00:00',
'url': 'gs://main.py'
},
]
self.assertEqual(expected, found)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--git_sha",
help="A git revision checksum",
required=True)
parser.add_argument("--git_branch",
help="Current git branch",
required=True)
parser.add_argument("--build_id", help="Current GCB id", required=True)
parser.add_argument("--upload",
help="Do not upload file",
default=False,
action='store_true')
parser.add_argument("--mf_file", help="Path to config file, that describes a repository. " \
"By default search for {} in the root directory for current script"
.format(DEFAULT_CONFIG_FILE_NAME), required=False)
args = parser.parse_args()
try:
root_dir = Path('.').absolute()
LOGGER.info(f'Set current project root dir ({root_dir})')
conf_file = config(root_dir, args.mf_file)
build_info = BuildInfo(git_branch=args.git_branch,
git_sha=args.git_sha,
date=datetime.datetime.utcnow(),
build_id=args.build_id)
actual_manifest = Manifest(conf_file.bucket, conf_file.repository)
new_content = actual_manifest.update(build_info,
conf_file,
upload=args.upload)
if not args.upload:
LOGGER.info("skipping upload...")
print(json.dumps(new_content, indent=4))
return 0
except Exception as ex:
LOGGER.error("error: %s", ex)
return 1
def get(ctx, bucket, repo, app, branch, destination):
"""
Download all found binaries.
"""
ctx.ensure_object(dict)
project = ctx.obj[PROJECT_OPT]
if project is None and (bucket is None and repo is None):
click.echo(
f'Config file not found in [{ctx.obj["root_dir"]}] and --bucket not specifies.\n'
'Please specify --bucket and --repo parameters or --config file path',
err=True)
return 1
manifest = Manifest(project.bucket, project.repository)
binaries_list = manifest.search(branch_name=branch, app_name=app)
for bin in binaries_list:
manifest.download(bin, dest=destination)
LOGGER.info("Downloading... %s", bin['url'])
pass
def put(ctx, git_branch, git_commit, build_id, no_upload):
"""
Scan current folder for .mf.json file that contains description of current repository.
Based on configuration upload all found binaries into gcs and update manifest.json with information about success build.
"""
ctx.ensure_object(dict)
root_dir = ctx.obj['root_dir']
project = ctx.obj[PROJECT_OPT]
assert git_branch and len(
str(git_branch)) > 0, '--git_branch have to be non empty string'
assert git_commit and len(
str(git_commit)) > 0, '--git_commit have to be non empty string'
assert build_id and len(
str(build_id)) > 0, '--build_id have to be non empty string'
if project is None:
click.echo(f'config file not found in {root_dir}', err=True)
return 1
LOGGER.debug("Current project %s", project)
if no_upload:
click.echo('Content wont be uploaded...')
build_info = BuildInfo(git_branch=git_branch,
git_sha=git_commit,
build_id=build_id,
date=datetime.datetime.utcnow())
actual_manifest = Manifest(project.bucket, project.repository)
new = actual_manifest.update(build_info, project, upload=not no_upload)
if no_upload:
click.echo(json.dumps(new, indent=4))