def main():
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.add_argument("--external_id", required=True)
args = arg_parser.parse_args()
external_id = load(args.external_id)
session = DrmaaSessionFactory().get()
external_id = session.kill(external_id)
def main():
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.add_argument("--external_id", required=True)
args = arg_parser.parse_args()
external_id = load(args.external_id)
session = DrmaaSessionFactory().get()
external_id = session.kill(external_id)
def main():
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.add_argument("--job_attributes", required=True)
args = arg_parser.parse_args()
job_attributes = load(open(args.job_attributes, "r"))
session = DrmaaSessionFactory().get()
external_id = session.run_job(**job_attributes)
print(external_id)
def run_executor(argv=None):
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.parse_args(argv)
ensure_mesos_libs()
log.info("Starting Pulsar executor")
driver = MesosExecutorDriver(PulsarExecutor())
exit_code = 0
if not driver.run() == mesos_pb2.DRIVER_STOPPED:
exit_code = 1
return exit_code
def main(argv=None):
arg_parser = ArgumentParser(description=DESCRIPTION, )
arg_parser.add_argument("--master", default=None, required=True)
PulsarManagerConfigBuilder.populate_options(arg_parser)
args = arg_parser.parse_args(argv)
ensure_mesos_libs()
config_builder = PulsarManagerConfigBuilder(args)
config_builder.setup_logging()
config = config_builder.load()
run(master=args.master,
manager_options=config_builder.to_dict(),
config=config)
def add_argument(self, *args, **kwds):
if "skip_on_windows" in kwds:
skip_on_windows = kwds["skip_on_windows"]
if skip_on_windows and IS_WINDOWS:
return
del kwds["skip_on_windows"]
return ArgumentParser.add_argument(self, *args, **kwds)
def add_argument(self, *args, **kwds):
if "skip_on_windows" in kwds:
skip_on_windows = kwds["skip_on_windows"]
if skip_on_windows and IS_WINDOWS:
return
del kwds["skip_on_windows"]
return ArgumentParser.add_argument(self, *args, **kwds)
def main():
ensure_mesos_libs()
arg_parser = ArgumentParser(
description=DESCRIPTION,
)
arg_parser.add_argument("--master", default=None, required=True)
PulsarManagerConfigBuilder.populate_options(arg_parser)
args = arg_parser.parse_args()
config_builder = PulsarManagerConfigBuilder(args)
config_builder.setup_logging()
config = config_builder.load()
run(
master=args.master,
manager_options=config_builder.to_dict(),
config=config
)
def main():
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.add_argument("--user", required=True)
arg_parser.add_argument("--job_id")
arg_parser.add_argument("--job_directory")
args = arg_parser.parse_args()
user = args.user
job_id = args.job_id
if args.job_id:
staging_directory = PulsarConfigBuilder().load().get(
'staging_directory', DEFAULT_STAGING_DIRECTORY)
job_directory = abspath(join(staging_directory, job_id))
assert job_directory.startswith(staging_directory)
elif FORCE_PRODUCTION:
raise Exception(
"In production mode, must specify a job_id instead of a working directory."
)
else:
job_directory = abspath(args.job_directory)
assert job_directory
command = "chown -R '%s' '%s'" % (user, job_directory)
system(command)
def main(argv=None):
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.add_argument("--user", required=True)
arg_parser.add_argument("--job_id")
arg_parser.add_argument("--job_directory")
args = arg_parser.parse_args(argv)
user = args.user
job_id = args.job_id
if args.job_id:
staging_directory = PulsarConfigBuilder().load().get('staging_directory', DEFAULT_STAGING_DIRECTORY)
job_directory = abspath(join(staging_directory, job_id))
assert job_directory.startswith(staging_directory)
elif FORCE_PRODUCTION:
raise Exception("In production mode, must specify a job_id instead of a working directory.")
else:
job_directory = abspath(args.job_directory)
assert job_directory
command = "chown -R '%s' '%s'" % (user, job_directory)
system(command)
def main(args=None):
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.add_argument("--file", default=None)
arg_parser.add_argument("--base64", default=None)
PulsarManagerConfigBuilder.populate_options(arg_parser)
args = arg_parser.parse_args(args)
config_builder = PulsarManagerConfigBuilder(args)
manager, app = manager_from_args(config_builder)
try:
job_config = __load_job_config(args)
submit_job(manager, job_config)
wait_for_job(manager, job_config)
except BaseException:
log.exception("Failure submitting or waiting on job.")
finally:
app.shutdown()
def main(args=None):
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.add_argument("--file", default=None)
arg_parser.add_argument("--base64", default=None)
PulsarManagerConfigBuilder.populate_options(arg_parser)
args = arg_parser.parse_args(args)
config_builder = PulsarManagerConfigBuilder(args)
manager, app = manager_from_args(config_builder)
try:
job_config = __load_job_config(args)
submit_job(manager, job_config)
wait_for_job(manager, job_config)
except BaseException:
log.exception("Failure submitting or waiting on job.")
finally:
app.shutdown()
def main(argv=None):
mod_docstring = sys.modules[__name__].__doc__
arg_parser = ArgumentParser(description=mod_docstring)
add_common_submit_args(arg_parser)
arg_parser.add_argument('--url', default=DEFAULT_CLIENT_URL, help=HELP_URL)
arg_parser.add_argument('--amqp_url', default=DEFAULT_CLIENT_URL, help=HELP_AMQP_URL)
arg_parser.add_argument('--private_token', default=None, help=HELP_PRIVATE_TOKEN)
# TODO: choices...
arg_parser.add_argument('--default_file_action', default="none")
arg_parser.add_argument('--file_action_config', default=None)
arg_parser.add_argument('--transport', default=None, choices=["urllib", "curl"], help=HELP_TRANSPORT) # set to curl to use pycurl
arg_parser.add_argument('--suppress_output', default=False, action="store_true", help=HELP_SUPPRESS_OUTPUT)
arg_parser.add_argument('--disable_cleanup', dest="cleanup", default=True, action="store_false", help=HELP_DISABLE_CLEANUP)
arg_parser.add_argument('--server', default=False, action="store_true", help=HELP_SERVER)
arg_parser.add_argument('--job_id', default=None, help=HELP_JOB_ID)
arg_parser.add_argument('--command', help=HELP_COMMAND)
arg_parser.add_argument('--working_directory', default=".", help=HELP_WORKING_DIRECTORY)
arg_parser.add_argument('--result_json', default=None)
arg_parser.add_argument('--output', default=[], action="append", help=HELP_OUTPUT)
arg_parser.add_argument('--output_pattern', default=[], action="append", help=HELP_OUTPUT_PATTERN)
args = arg_parser.parse_args(argv)
if args.server:
return run_server_for_job(args)
else:
failed = _run_client_for_job(args)
if failed:
return 1
else:
return 0
def main(args=None):
mod_docstring = sys.modules[__name__].__doc__
arg_parser = ArgumentParser(description=mod_docstring)
add_common_submit_args(arg_parser)
args = arg_parser.parse_args(args)
run_server_for_job(args)
def main(argv=None):
if argv is None:
argv = sys.argv
dependencies = []
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.add_argument("--directory",
default=".",
help=HELP_DIRECTORY)
arg_parser.add_argument("--mq",
action="store_true",
default=False,
help=HELP_MQ)
arg_parser.add_argument("--no_logging",
dest="logging",
action="store_false",
default=True,
help=HELP_MQ)
arg_parser.add_argument("--supervisor",
action="store_true",
default=False,
help=HELP_SUPERVISOR)
arg_parser.add_argument("--wsgi_server",
choices=["paste", "uwsgi"],
default=None,
help=HELP_WSGI_SERVER)
arg_parser.add_argument("--libdrmaa_path",
help=HELP_LIBDRMAA)
arg_parser.add_argument("--host",
default="localhost",
help=HELP_HOST)
arg_parser.add_argument("--port",
default="8913",
help=HELP_PORT)
arg_parser.add_argument("--install",
help=HELP_INSTALL)
arg_parser.add_argument("--force",
action="store_true",
default=False,
help=HELP_FORCE)
arg_parser.add_argument("--pip_install_args",
default="pulsar-app",
help=HELP_PIP_INSTALL_ARGS_HELP)
args = arg_parser.parse_args(argv)
directory = args.directory
directory = os.path.abspath(directory)
mode = _determine_mode(args)
if mode == "uwsgi":
dependencies.append("uwsgi")
if not os.path.exists(directory):
os.makedirs(directory)
_handle_app_yaml(args, directory)
_handle_server_ini(args, directory)
_handle_local_env(args, directory, dependencies)
_handle_supervisor(args, mode, directory, dependencies)
_handle_install(args, dependencies)