def __new__(cls):
if not cls._instance:
cls._instance = object.__new__(cls)
cls._instance.root_path = os.path.join(
"/litemanager_%s:%s" %
(CONFIG["http_host"], CONFIG["http_port"]), "data")
cls._instance.ldfs = LiteDFSClient(CONFIG["ldfs_http_host"],
CONFIG["ldfs_http_port"])
return cls._instance
LOG.debug("input_data: %s", input_data)
task_name = input_data["action_info"]["task_name"]
action_name = input_data["action_info"]["action_name"]
task_id = input_data["task_id"]
ldfs_host = input_data[
"ldfs_host"] if "ldfs_host" in input_data and input_data[
"ldfs_host"] else None
ldfs_port = input_data[
"ldfs_port"] if "ldfs_port" in input_data and input_data[
"ldfs_port"] else None
data = {}
if ldfs_host and ldfs_port:
ldfs = LiteDFSClient(ldfs_host, ldfs_port)
source_app_path = input_data["source"]
target_app_path = input_data["target"]
target_app_log_path = target_app_path + ".txt"
source_format = "tar.gz"
if "zip" in os.path.split(source_app_path)[-1]:
source_format = "zip"
target_format = "tar.gz"
if "zip" in os.path.split(target_app_path)[-1]:
target_format = "zip"
local_tmp_path = os.path.join(workspace, "app.%s" % source_format)
if os.path.exists(local_tmp_path):
os.remove(local_tmp_path)
LOG.info("remove old: %s", local_tmp_path)
def __init__(self, host, port):
self.client = LiteDFSClient(host, port)
task_name = input_data["action_info"]["task_name"]
action_name = input_data["action_info"]["action_name"]
task_id = input_data["task_id"]
data = {"messages": []}
for i in range(10):
now = datetime.datetime.now()
message = "%s: hello world, tornado(%03d): %s, numpy: %s" % (
now, i, tornado.version, numpy.__version__)
data["messages"].append(message)
LOG.debug(message)
time.sleep(1)
ldfs_host = input_data[
"ldfs_host"] if "ldfs_host" in input_data and input_data[
"ldfs_host"] else None
ldfs_port = input_data[
"ldfs_port"] if "ldfs_port" in input_data and input_data[
"ldfs_port"] else None
if ldfs_host and ldfs_port:
ldfs = LiteDFSClient(ldfs_host, ldfs_port)
data_file_path = os.path.join(workspace, "data.json")
data_ldfs_path = os.path.join("/litepipeline_test", task_id,
"%s.json" % action_name)
with open(data_file_path, "w") as fp:
fp.write(json.dumps(data, indent=4))
ldfs.create_file(data_file_path, data_ldfs_path, replica=1)
Action.set_output(data={"ldfs_data_path": data_ldfs_path})
LOG.debug("test end")
def main():
logger.config_logging(log_level="DEBUG", console=True, filesystem=False)
try:
lpl_address = args.litepipeline
lpl_host, lpl_port = lpl_address.split(":")
lpl = LitePipelineClient(lpl_host, lpl_port)
manager_info = lpl.cluster_info(include=["manager"])
ldfs_host = manager_info["info"]["manager"]["ldfs_http_host"]
ldfs_port = manager_info["info"]["manager"]["ldfs_http_port"]
ldfs = LiteDFSClient(ldfs_host, ldfs_port)
pack_app_id = args.pack_app_id
operate = args.operate
app_path = args.input
app_format = args.format
app_id = args.app_id
app_name = args.name
app_description = args.description
workspace = "."
now = datetime.datetime.now()
LOG.info("workspace directory: %s, now: %s", workspace, str(now))
if operate in ["pack", "create", "update"]:
if os.path.exists(app_path) and os.path.isdir(app_path):
app_dir_name = os.path.split(app_path.strip("/"))[-1]
tar_name = "%s.tar.gz" % app_dir_name
target_name = "%s.%s" % (app_dir_name, app_format)
tmp_tar_path = os.path.join(workspace, tar_name)
if os.path.exists(tmp_tar_path):
os.remove(tmp_tar_path)
LOG.warning("remove old file [%s]", tmp_tar_path)
LOG.info("compress [%s] start", tmp_tar_path)
with tarfile.open(tmp_tar_path, "w:gz") as tar:
tar.add(app_path, arcname=app_dir_name)
LOG.info("compress [%s] end", tmp_tar_path)
remote_source_path = os.path.join("/pack_application/source",
str(now), tar_name)
ldfs.delete_file(remote_source_path)
success = ldfs.create_file(tmp_tar_path,
remote_source_path,
replica=1)
if success:
LOG.info("upload [%s] to remote [%s] success",
tmp_tar_path, remote_source_path)
remote_target_path = os.path.join(
"/pack_application/target", str(now), target_name)
task_name = "pack %s" % app_dir_name
task_input = {
"source": remote_source_path,
"target": remote_target_path,
}
r = lpl.task_create(task_name, pack_app_id, task_input)
if r:
task_id = r["task_id"]
LOG.info("create pack task [task_id: %s] success",
task_id)
stage = "running"
task_info = None
while stage != "finished":
task_info = lpl.task_info(task_id)
if task_info:
stage = task_info["task_info"]["stage"]
update_at = task_info["task_info"]["update_at"]
LOG.info(
"task[task_id: %s], stage: %s, update_at: %s",
task_id, stage, update_at)
time.sleep(5)
ldfs.delete_directory(
os.path.split(remote_source_path)[0])
if task_info:
if task_info["task_info"]["status"] == "success":
task_result = task_info["task_info"]["result"][
"pack application"]["result"]["data"]
remote_pack_log_path = task_result[
"app_log_path"]
remote_pack_path = task_result["app_path"]
LOG.info(
"pack task success, remote pack log: %s, remote packed application: %s",
remote_pack_log_path, remote_pack_path)
local_pack_log_path = os.path.join(
workspace, "%s.pack.log" % app_dir_name)
local_pack_path = os.path.join(
workspace,
"%s.pack.%s" % (app_dir_name, app_format))
if os.path.exists(local_pack_log_path
) and os.path.isfile(
local_pack_log_path):
os.remove(local_pack_log_path)
if os.path.exists(
local_pack_path) and os.path.isfile(
local_pack_path):
os.remove(local_pack_path)
success = ldfs.download_file(
remote_pack_log_path, local_pack_log_path)
if success:
LOG.info(
"download pack log as [%s] success",
local_pack_log_path)
else:
LOG.error("download pack log failed")
success = ldfs.download_file(
remote_pack_path, local_pack_path)
if success:
ldfs.delete_directory(
os.path.split(remote_pack_path)[0])
LOG.info(
"download packed application as [%s] success",
local_pack_path)
else:
LOG.error(
"download packed application failed")
if operate == "create":
if app_name:
r = lpl.application_create(
local_pack_path,
app_name,
description=app_description)
if r:
LOG.info(
"create application[app_id: %s] success",
r["app_id"])
else:
LOG.error(
"create application failed")
else:
LOG.error("need -n/--name parameter")
elif operate == "update":
if app_id:
if app_name and app_description:
r = lpl.application_update(
app_id,
file_path=local_pack_path,
name=app_name,
description=app_description)
elif app_name:
r = lpl.application_update(
app_id,
file_path=local_pack_path,
name=app_name)
elif app_description:
r = lpl.application_update(
app_id,
file_path=local_pack_path,
description=app_description)
else:
r = lpl.application_update(
app_id,
file_path=local_pack_path)
if r:
LOG.info(
"update application[app_id: %s] success",
app_id)
else:
LOG.error(
"update application failed")
else:
LOG.error("need -a/--app_id parameter")
else:
LOG.error("pack task failed\ntask_info: %s",
json.dumps(task_info, indent=4))
else:
LOG.error("pack task failed")
else:
LOG.error("create pack task failed")
else:
LOG.error("upload [%s] to remote [%s] failed",
tmp_tar_path, remote_source_path)
else:
LOG.error("input [%s] directory not exists", app_path)
else:
LOG.error("unknown operate: %s", operate)
except Exception as e:
LOG.exception(e)
console=False)
LOG.debug("test start")
LOG.debug("input_data: %s", input_data)
task_name = input_data["action_info"]["task_name"]
action_name = input_data["action_info"]["action_name"]
task_id = input_data["task_id"]
ldfs_host = input_data[
"ldfs_host"] if "ldfs_host" in input_data and input_data[
"ldfs_host"] else None
ldfs_port = input_data[
"ldfs_port"] if "ldfs_port" in input_data and input_data[
"ldfs_port"] else None
if ldfs_host and ldfs_port:
ldfs = LiteDFSClient(ldfs_host, ldfs_port)
first_data_path = input_data["first action"]["ldfs_data_path"]
second_data_path = input_data["second action"]["ldfs_data_path"]
first_data = json.loads(
ldfs.open_remote_file(first_data_path).read().decode("utf-8"))
second_data = json.loads(
ldfs.open_remote_file(second_data_path).read().decode("utf-8"))
data = {"messages": []}
for message in first_data["messages"]:
data["messages"].append(message)
for message in second_data["messages"]:
data["messages"].append(message)
for i in range(20, 30):