def build_project(project, zip, url, alias, replace, create):
"""Build project."""
if zip:
project.build(zip, overwrite=replace)
stdout.write(
'Project successfully built and saved as %r (size: %s).\n'
% (zip, human_readable(getsize(zip)))
)
else:
with temppath() as zip:
project.build(zip)
session = Session(url, alias)
while True:
try:
res = session.upload_project(project.name, zip)
except AzkabanError as err:
if create:
session.create_project(project.name, project.name)
else:
raise err
else:
break
stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(getsize(zip)),
res['version'],
session.url,
project,
)
)
def upload_project(project_name, zip, url, alias, create):
"""Upload project."""
session = Session(url, alias)
while True:
try:
res = session.upload_project(project_name, zip)
except AzkabanError as err:
if create:
session.create_project(project_name, project_name)
else:
raise err
else:
break
stdout.write(
'Project %s successfully uploaded (id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project_name,
res['projectId'],
human_readable(getsize(zip)),
res['version'],
session.url,
project_name,
)
)
def upload_project(project_name, _zip, _url, _alias, _create):
"""Upload project."""
session = Session(_url, _alias)
while True:
try:
res = session.upload_project(
name=project_name,
path=_zip,
callback=_upload_callback
)
except AzkabanError as err:
if _create:
session.create_project(project_name, project_name)
else:
raise err
else:
break
sys.stdout.write(
'Project %s successfully uploaded (id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project_name,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project_name,
)
)
def upload_project(project_name, _zip, _url, _alias, _create):
"""Upload project."""
session = Session(_url, _alias)
while True:
try:
res = session.upload_project(
name=project_name,
path=_zip,
callback=_upload_callback
)
except AzkabanError as err:
if _create:
session.create_project(project_name, project_name)
else:
raise err
else:
break
sys.stdout.write(
'Project %s successfully uploaded (id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project_name,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project_name,
)
)
def run_flow(project_name, flow, job, url, alias, skip):
"""Run workflow."""
session = Session(url, alias)
res = session.run_workflow(project_name, flow, job, skip)
exec_id = res['execid']
job_names = ', jobs: %s' % (', '.join(job), ) if job else ''
stdout.write(
'Flow %s successfully submitted (execution id: %s%s).\n'
'Details at %s/executor?execid=%s\n'
% (flow, exec_id, job_names, session.url, exec_id)
)
def create_project(url, alias):
"""Create new project on remote Azkaban server."""
session = Session(url, alias)
name = raw_input('Project name: ').strip()
description = raw_input('Project description [%s]: ' % (name, ))
session.create_project(name, description.strip() or name)
stdout.write(
'Project %s successfully created.\n'
'Details at %s/manager?project=%s\n'
% (name, session.url, name)
)
def _get_session(url, alias):
"""Get appropriate session.
:param url: URL (has precedence over alias).
:param alias: Alias name.
"""
config = Config()
if url:
return Session(url=url, config=config)
else:
alias = alias or config.get_option('azkaban', 'default.alias')
return Session.from_alias(alias=alias, config=config)
def build_project(project, _zip, _url, _alias, _replace, _create, _option):
"""Build project."""
if _option:
project.properties = flatten(project.properties)
# to make sure we properly override nested options, we flatten first
project.properties.update(_parse_option(_option))
if _zip:
if osp.isdir(_zip):
_zip = osp.join(_zip, '%s.zip' % (project.versioned_name, ))
project.build(_zip, overwrite=_replace)
sys.stdout.write(
'Project %s successfully built and saved as %r (size: %s).\n'
% (project, _zip, human_readable(osp.getsize(_zip)))
)
else:
with temppath() as _zip:
project.build(_zip)
archive_name = '%s.zip' % (project.versioned_name, )
session = Session(_url, _alias)
while True:
try:
res = session.upload_project(
name=project.name,
path=_zip,
archive_name=archive_name,
callback=_upload_callback
)
except AzkabanError as err:
if _create and str(err).endswith("doesn't exist."):
session.create_project(project.name, project.name)
else:
raise err
else:
break
sys.stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, upload: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project,
)
)
def build_project(project, _zip, _url, _alias, _replace, _create, _option):
"""Build project."""
if _option:
project.properties = flatten(project.properties)
# to make sure we properly override nested options, we flatten first
project.properties.update(_parse_option(_option))
if _zip:
if osp.isdir(_zip):
_zip = osp.join(_zip, '%s.zip' % (project.versioned_name, ))
project.build(_zip, overwrite=_replace)
sys.stdout.write(
'Project %s successfully built and saved as %r (size: %s).\n'
% (project, _zip, human_readable(osp.getsize(_zip)))
)
else:
with temppath() as _zip:
project.build(_zip)
archive_name = '%s.zip' % (project.versioned_name, )
session = Session(_url, _alias)
while True:
try:
res = session.upload_project(
name=project.name,
path=_zip,
archive_name=archive_name,
callback=_upload_callback
)
except AzkabanError as err:
if _create and str(err).endswith("doesn't exist."):
session.create_project(project.name, project.name)
else:
raise err
else:
break
sys.stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, upload: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project,
)
)
def setup_class(cls):
# don't create project automatically (goal of these tests...)
if not cls.session:
config = Config()
try:
alias = config.parser.get('azkaban', 'test.alias')
except (NoOptionError, NoSectionError):
pass
else:
cls.session = Session.from_alias(alias)
def setup_class(cls):
# don't create project automatically (goal of these tests...)
if not cls.session:
config = Config()
try:
alias = config.parser.get('azkaban', 'test.alias')
except (NoOptionError, NoSectionError):
pass
else:
cls.session = Session.from_alias(alias)
def schedule_workflow(project_name, _date, _time, _span, _flow, _job, _url,
_alias, _bounce, _kill, _email, _option):
"""Schedule workflow."""
session = Session(_url, _alias)
session.schedule_workflow(
name=project_name,
flow=_flow,
date=_date,
time=_time,
period=_span,
jobs=_job,
concurrent=not _bounce,
on_failure='cancel' if _kill else 'finish',
emails=_email,
properties=_parse_option(_option),
)
sys.stdout.write(
'Flow %s scheduled successfully.\n' % (_flow, )
)
def schedule_workflow(project_name, _date, _time, _span, _flow, _job, _url,
_alias, _bounce, _kill, _email, _option):
"""Schedule workflow."""
session = Session(_url, _alias)
session.schedule_workflow(
name=project_name,
flow=_flow,
date=_date,
time=_time,
period=_span,
jobs=_job,
concurrent=not _bounce,
on_failure='cancel' if _kill else 'finish',
emails=_email,
properties=_parse_option(_option),
)
sys.stdout.write(
'Flow %s scheduled successfully.\n' % (_flow, )
)
def run_workflow(project_name, _flow, _job, _url, _alias, _bounce, _kill,
_email, _option):
"""Run workflow."""
session = Session(_url, _alias)
res = session.run_workflow(
name=project_name,
flow=_flow,
jobs=_job,
concurrent=not _bounce,
on_failure='cancel' if _kill else 'finish',
emails=_email,
properties=_parse_option(_option),
)
exec_id = res['execid']
job_names = ', jobs: %s' % (', '.join(_job), ) if _job else ''
sys.stdout.write(
'Flow %s successfully submitted (execution id: %s%s).\n'
'Details at %s/executor?execid=%s\n'
% (_flow, exec_id, job_names, session.url, exec_id)
)
def run_workflow(project_name, _flow, _job, _url, _alias, _bounce, _kill,
_email, _option):
"""Run workflow."""
session = Session(_url, _alias)
res = session.run_workflow(
name=project_name,
flow=_flow,
jobs=_job,
concurrent=not _bounce,
on_failure='cancel' if _kill else 'finish',
emails=_email,
properties=_parse_option(_option),
)
exec_id = res['execid']
job_names = ', jobs: %s' % (', '.join(_job), ) if _job else ''
sys.stdout.write(
'Flow %s successfully submitted (execution id: %s%s).\n'
'Details at %s/executor?execid=%s\n'
% (_flow, exec_id, job_names, session.url, exec_id)
)
def _get_session(url, alias):
"""Get appropriate session.
:param url: URL (has precedence over alias).
:param alias: Alias name.
"""
config = Config()
if url:
return Session(url=url, config=config)
else:
alias = alias or config.get_option('azkaban', 'default.alias')
return Session.from_alias(alias=alias, config=config)
def setup_class(cls):
if not cls.session:
config = Config()
try:
alias = config.parser.get('azkaban', 'test.alias')
except (NoOptionError, NoSectionError):
pass
else:
cls.session = Session.from_alias(alias)
try:
cls.session.create_project(cls.project_name, 'Testing project.')
except AzkabanError:
pass # project already exists somehow
def setup_class(cls):
if not cls.session:
config = Config()
try:
alias = config.parser.get('azkaban', 'test.alias')
except (NoOptionError, NoSectionError):
pass
else:
cls.session = Session(alias=alias)
try:
cls.session.create_project(cls.project_name, 'Testing project.')
except AzkabanError:
pass # project already exists somehow
def view_log(_execution, _job, _url, _alias):
"""View workflow or job execution logs."""
session = Session(_url, _alias)
exc = Execution(session, _execution)
logs = exc.job_logs(_job[0]) if _job else exc.logs()
try:
for line in logs:
sys.stdout.write('%s\n' % (line.encode('utf-8'), ))
except HTTPError:
# Azkaban responds with 500 if the execution or job isn't found
if _job:
raise AzkabanError(
'Execution %s and/or job %s not found.', _execution, _job
)
else:
raise AzkabanError('Execution %s not found.', _execution)
def session(self):
if self.parsed.azkaban_url is not None:
return Session(url=self.parsed.azkaban_url, verify=True)
if self.parsed.azkaban_alias is not None:
return Session.from_alias(self.parsed.azkaban_alias)
def delete_project(url, alias):
"""Delete a project on remote Azkaban server."""
session = Session(url, alias)
project = raw_input('Project name: ')
session.delete_project(project)
stdout.write('Project %s successfully deleted.\n' % (project, ))