def build_project(project, zip, url, alias, replace, create):
"""Build project."""
if zip:
project.build(zip, overwrite=replace)
stdout.write(
'Project successfully built and saved as %r (size: %s).\n'
% (zip, human_readable(getsize(zip)))
)
else:
with temppath() as zip:
project.build(zip)
session = Session(url, alias)
while True:
try:
res = session.upload_project(project.name, zip)
except AzkabanError as err:
if create:
session.create_project(project.name, project.name)
else:
raise err
else:
break
stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(getsize(zip)),
res['version'],
session.url,
project,
)
)
def build_project(project, _zip, _url, _alias, _replace, _create, _option):
"""Build project."""
if _option:
project.properties = flatten(project.properties)
# to make sure we properly override nested options, we flatten first
project.properties.update(_parse_option(_option))
if _zip:
if osp.isdir(_zip):
_zip = osp.join(_zip, '%s.zip' % (project.versioned_name, ))
project.build(_zip, overwrite=_replace)
sys.stdout.write(
'Project %s successfully built and saved as %r (size: %s).\n'
% (project, _zip, human_readable(osp.getsize(_zip)))
)
else:
with temppath() as _zip:
project.build(_zip)
archive_name = '%s.zip' % (project.versioned_name, )
session = _get_session(_url, _alias)
res = _upload_zip(session, project.name, _zip, _create, archive_name)
sys.stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, upload: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project,
)
)
def build_project(project, _zip, _url, _alias, _replace, _create, _option):
"""Build project."""
if _option:
project.properties = flatten(project.properties)
# to make sure we properly override nested options, we flatten first
project.properties.update(_parse_option(_option))
if _zip:
if osp.isdir(_zip):
_zip = osp.join(_zip, '%s.zip' % (project.versioned_name, ))
project.build(_zip, overwrite=_replace)
sys.stdout.write(
'Project %s successfully built and saved as %r (size: %s).\n'
% (project, _zip, human_readable(osp.getsize(_zip)))
)
else:
with temppath() as _zip:
project.build(_zip)
archive_name = '%s.zip' % (project.versioned_name, )
session = _get_session(_url, _alias)
res = _upload_zip(session, project.name, _zip, _create, archive_name)
sys.stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, upload: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project,
)
)
def upload_project(project_name, _zip, _url, _alias, _create):
"""Upload project."""
session = Session(_url, _alias)
while True:
try:
res = session.upload_project(
name=project_name,
path=_zip,
callback=_upload_callback
)
except AzkabanError as err:
if _create:
session.create_project(project_name, project_name)
else:
raise err
else:
break
sys.stdout.write(
'Project %s successfully uploaded (id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project_name,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project_name,
)
)
def upload_project(project_name, _zip, _url, _alias, _create):
"""Upload project."""
session = Session(_url, _alias)
while True:
try:
res = session.upload_project(
name=project_name,
path=_zip,
callback=_upload_callback
)
except AzkabanError as err:
if _create:
session.create_project(project_name, project_name)
else:
raise err
else:
break
sys.stdout.write(
'Project %s successfully uploaded (id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project_name,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project_name,
)
)
def upload_project(project_name, zip, url, alias, create):
"""Upload project."""
session = Session(url, alias)
while True:
try:
res = session.upload_project(project_name, zip)
except AzkabanError as err:
if create:
session.create_project(project_name, project_name)
else:
raise err
else:
break
stdout.write(
'Project %s successfully uploaded (id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project_name,
res['projectId'],
human_readable(getsize(zip)),
res['version'],
session.url,
project_name,
)
)
def build_project(project, _zip, _url, _alias, _replace, _create, _option):
"""Build project."""
if _option:
project.properties = flatten(project.properties)
# to make sure we properly override nested options, we flatten first
project.properties.update(_parse_option(_option))
if _zip:
if osp.isdir(_zip):
_zip = osp.join(_zip, '%s.zip' % (project.versioned_name, ))
project.build(_zip, overwrite=_replace)
sys.stdout.write(
'Project %s successfully built and saved as %r (size: %s).\n'
% (project, _zip, human_readable(osp.getsize(_zip)))
)
else:
with temppath() as _zip:
project.build(_zip)
archive_name = '%s.zip' % (project.versioned_name, )
session = Session(_url, _alias)
while True:
try:
res = session.upload_project(
name=project.name,
path=_zip,
archive_name=archive_name,
callback=_upload_callback
)
except AzkabanError as err:
if _create and str(err).endswith("doesn't exist."):
session.create_project(project.name, project.name)
else:
raise err
else:
break
sys.stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, upload: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project,
)
)
def build_project(project, _zip, _url, _alias, _replace, _create, _option):
"""Build project."""
if _option:
project.properties = flatten(project.properties)
# to make sure we properly override nested options, we flatten first
project.properties.update(_parse_option(_option))
if _zip:
if osp.isdir(_zip):
_zip = osp.join(_zip, '%s.zip' % (project.versioned_name, ))
project.build(_zip, overwrite=_replace)
sys.stdout.write(
'Project %s successfully built and saved as %r (size: %s).\n'
% (project, _zip, human_readable(osp.getsize(_zip)))
)
else:
with temppath() as _zip:
project.build(_zip)
archive_name = '%s.zip' % (project.versioned_name, )
session = Session(_url, _alias)
while True:
try:
res = session.upload_project(
name=project.name,
path=_zip,
archive_name=archive_name,
callback=_upload_callback
)
except AzkabanError as err:
if _create and str(err).endswith("doesn't exist."):
session.create_project(project.name, project.name)
else:
raise err
else:
break
sys.stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, upload: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project,
)
)
def upload_project(project_name, _zip, _url, _alias, _create):
"""Upload project."""
session = _get_session(_url, _alias)
res = _upload_zip(session, project_name, _zip, _create)
sys.stdout.write(
'Project %s successfully uploaded (id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n' % (
project_name,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project_name,
))
def upload_project(project_name, _zip, _url, _alias, _create):
"""Upload project."""
session = _get_session(_url, _alias)
res = _upload_zip(session, project_name, _zip, _create)
sys.stdout.write(
'Project %s successfully uploaded (id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project_name,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project_name,
)
)
def main(project=None):
"""Command line argument parser.
:param project: `EmptyProject` or `Project` instance
"""
args = docopt(__doc__, version=__version__)
name = args['--project']
try:
if args['run']:
flow = args['FLOW']
jobs = args['JOB']
if not project:
if name:
project = EmptyProject(name)
else:
project = Project.load_from_script(args['--script'])
session = project.get_session(url=args['--url'], alias=args['--alias'])
res = project.run(
flow=flow,
url=session['url'],
session_id=session['session_id'],
jobs=jobs,
block=args['--block'],
)
exec_id = res['execid']
job_names = ', jobs: %s' % (', '.join(jobs), ) if jobs else ''
stdout.write(
'Flow %s successfully submitted (execution id: %s%s).\n'
'Details at %s/executor?execid=%s\n'
% (flow, exec_id, job_names, session['url'], exec_id)
)
elif args['create']:
name = raw_input('Project name: ').strip()
project = EmptyProject(name)
session = project.get_session(url=args['--url'], alias=args['--alias'])
description = raw_input('Project description [%s]: ' % (name, )) or name
project.create(
description=description,
url=session['url'],
session_id=session['session_id'],
)
stdout.write(
'Project %s successfully created.\n'
'Details at %s/manager?project=%s\n'
% (project.name, session['url'], project.name)
)
elif args['delete']:
name = raw_input('Project name: ')
project = project or EmptyProject(name)
session = project.get_session(url=args['--url'], alias=args['--alias'])
project.delete(
url=session['url'],
session_id=session['session_id'],
)
stdout.write(
'Project %s successfully deleted.\n'
% (name, )
)
elif args['build']:
project = project or Project.load_from_script(args['--script'], name)
size = project.build(
args['--zip'] or '%s.zip' % (project.name, ),
overwrite=args['--overwrite'],
)
stdout.write(
'Project successfully built (size: %s).\n'
% (human_readable(size), )
)
elif args['upload']:
path = args['--zip']
if not project:
if path:
project = EmptyProject(name)
else:
project = Project.load_from_script(args['--script'], name)
session = project.get_session(url=args['--url'], alias=args['--alias'])
with temppath() as tpath:
if path:
size = getsize(path)
else:
path = tpath
size = project.build(path)
try:
res = project.upload(
path,
url=session['url'],
session_id=session['session_id'],
)
except AzkabanError as err:
if args['--create']:
project.create(
description=project.name,
url=session['url'],
session_id=session['session_id'],
)
res = project.upload(
path,
url=session['url'],
session_id=session['session_id'],
)
else:
raise err
stdout.write(
'Project %s successfully uploaded (id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project.name,
res['projectId'],
human_readable(size),
res['version'],
session['url'],
project.name
)
)
elif args['view']:
project = project or Project.load_from_script(args['--script'], name)
job_name = args['JOB'][0]
if job_name in project.jobs:
pretty_print(project.jobs[job_name])
else:
raise AzkabanError('missing job %r' % (job_name, ))
elif args['list']:
project = project or Project.load_from_script(args['--script'], name)
if args['--files']:
for path in project._files:
stdout.write('%s\n' % (relpath(path), ))
else:
for name in project.jobs:
stdout.write('%s\n' % (name, ))
except AzkabanError as err:
stderr.write('%s\n' % (err, ))
exit(1)
