def update(self, app_id, name, description, source_path):
result = True
try:
data = {}
need_update = False
if name:
data["name"] = name
if description:
data["description"] = description
if os.path.exists(source_path) and os.path.isfile(source_path):
sha1 = file_sha1sum(source_path)
data["sha1"] = sha1
LOG.debug("sha1: %s, %s", sha1, type(sha1))
app_path = self.make_app_version_path(app_id, sha1)
self.ldfs.delete_directory(app_path)
self.ldfs.create_file(source_path, os.path.join(app_path, "app.zip"), replica = 1)
os.remove(source_path)
need_update = True
if data or need_update:
success = Applications.instance().update(app_id, data)
if success:
if "sha1" in data:
description = "" if "description" not in data else data["description"]
ApplicationHistory.instance().add(app_id, data["sha1"], description = description)
else:
result = False
except Exception as e:
LOG.exception(e)
return result
def create(self, name, description, source_path):
sha1 = file_sha1sum(source_path)
LOG.debug("sha1: %s, %s", sha1, type(sha1))
app_id = Applications.instance().add(name, sha1, description = description)
app_path = self.make_app_version_path(app_id, sha1)
self.ldfs.delete_directory(app_path)
self.ldfs.create_file(source_path, os.path.join(app_path, "app.zip"), replica = 1)
os.remove(source_path)
ApplicationHistory.instance().add(app_id, sha1, description = description)
return app_id
def create(self, name, description, source_path):
sha1 = file_sha1sum(source_path)
LOG.debug("sha1: %s, %s", sha1, type(sha1))
app_id = Applications.instance().add(name, sha1, description = description)
app_path = self.make_app_version_path(app_id, sha1)
if os.path.exists(app_path):
shutil.rmtree(app_path)
os.makedirs(app_path)
shutil.copy2(source_path, os.path.join(app_path, "app.zip"))
os.remove(source_path)
ApplicationHistory.instance().add(app_id, sha1, description = description)
return app_id
def delete(self, app_id):
result = False
try:
success = Applications.instance().delete(app_id)
if success:
ApplicationHistory.instance().delete_by_app_id(app_id)
app_path = self.make_app_path(app_id)
self.ldfs.delete_directory(app_path)
LOG.debug("remove ldfs directory: %s", app_path)
result = True
except Exception as e:
LOG.exception(e)
return result
def delete(self, app_id):
result = False
try:
success = Applications.instance().delete(app_id)
if success:
ApplicationHistory.instance().delete_by_app_id(app_id)
app_path = self.make_app_path(app_id)
if os.path.exists(app_path):
shutil.rmtree(app_path)
LOG.debug("remove directory: %s", app_path)
result = True
except Exception as e:
LOG.exception(e)
return result
def delete_history(self, history_id, app_id):
result = False
try:
history = ApplicationHistory.instance().delete_by_history_id_app_id(history_id, app_id)
if history and history is not None:
filters = ApplicationHistory.instance().parse_filters({"app_id": history["application_id"], "sha1": history["sha1"]})
num = ApplicationHistory.instance().count(filters)
if num == 0:
app_path = self.make_app_version_path(history["application_id"], history["sha1"])
self.ldfs.delete_directory(app_path)
LOG.debug("remove ldfs directory: %s", app_path)
result = True
except Exception as e:
LOG.exception(e)
return result
def create(self, name, description, source_path):
sha1 = file_sha1sum(source_path)
LOG.debug("sha1: %s, %s", sha1, type(sha1))
app_id = Applications.instance().add(name, sha1, description = description)
app_path = self.make_app_version_path(app_id, sha1)
if os.path.exists(app_path):
shutil.rmtree(app_path)
os.makedirs(app_path)
shutil.copy2(source_path, os.path.join(app_path, "app.tar.gz"))
os.remove(source_path)
if os.path.exists(os.path.join(app_path, "app")):
shutil.rmtree(os.path.join(app_path, "app"))
t = tarfile.open(os.path.join(app_path, "app.tar.gz"), "r")
t.extractall(app_path)
path_parts = splitall(t.getnames()[0])
tar_root_name = path_parts[1] if path_parts[0] == "." else path_parts[0]
t.close()
os.rename(os.path.join(app_path, tar_root_name), os.path.join(app_path, "app"))
ApplicationHistory.instance().add(app_id, sha1, description = description)
return app_id
def update(self, app_id, name, description, source_path):
result = True
try:
data = {}
need_update = False
if name:
data["name"] = name
if description:
data["description"] = description
if os.path.exists(source_path) and os.path.isfile(source_path):
sha1 = file_sha1sum(source_path)
data["sha1"] = sha1
LOG.debug("sha1: %s, %s", sha1, type(sha1))
app_path = self.make_app_version_path(app_id, sha1)
if os.path.exists(app_path):
shutil.rmtree(app_path)
os.makedirs(app_path)
shutil.copy2(source_path, os.path.join(app_path, "app.tar.gz"))
os.remove(source_path)
if os.path.exists(os.path.join(app_path, "app")):
shutil.rmtree(os.path.join(app_path, "app"))
t = tarfile.open(os.path.join(app_path, "app.tar.gz"), "r")
t.extractall(app_path)
tar_root_name = splitall(t.getnames()[0])[0]
os.rename(os.path.join(app_path, tar_root_name), os.path.join(app_path, "app"))
need_update = True
if data or need_update:
success = Applications.instance().update(app_id, data)
if success:
if "sha1" in data:
description = "" if "description" not in data else data["description"]
ApplicationHistory.instance().add(app_id, data["sha1"], description = description)
else:
result = False
except Exception as e:
LOG.exception(e)
return result
def info_history(self, history_id, app_id = ""):
return ApplicationHistory.instance().get(history_id, app_id = app_id)
def list_history(self, offset, limit, filters = {}):
return ApplicationHistory.instance().list(offset = offset, limit = limit, filters = filters)
def main():
parser = argparse.ArgumentParser(prog='litemanager')
parser.add_argument("-c",
"--config",
required=True,
help="configuration file path")
parser.add_argument("-v",
"--version",
action='version',
version='%(prog)s ' + __version__)
args = parser.parse_args()
if args.config:
success = load_config(args.config)
if success:
common.init_storage()
logger.config_logging(file_name="manager.log",
log_level=CONFIG["log_level"],
dir_name=CONFIG["log_path"],
day_rotate=False,
when="D",
interval=1,
max_size=20,
backup_count=5,
console=True)
LOG.info("service start")
try:
if CONFIG["ldfs_http_host"] and CONFIG["ldfs_http_port"]:
LDFS = LiteDFS(CONFIG["ldfs_http_host"],
CONFIG["ldfs_http_port"])
venvs_db = Venvs()
venv_history_db = VenvHistory()
tasks_db = Tasks()
applications_db = Applications()
application_history_db = ApplicationHistory()
workflows_db = Workflows()
works_db = Works()
schedules_db = Schedules()
services_db = Services()
venv_manager = VenvManager()
app_manager = AppManager()
task_scheduler = Scheduler(CONFIG["scheduler_interval"])
http_server = tornado.httpserver.HTTPServer(
Application(),
max_buffer_size=CONFIG["max_buffer_size"],
chunk_size=10 * 1024 * 1024)
http_server.listen(CONFIG["http_port"],
address=CONFIG["http_host"])
# http_server.bind(CONFIG["http_port"], address = CONFIG["http_host"])
listener = DiscoveryListener(Connection, task_scheduler)
listener.listen(CONFIG["tcp_port"], CONFIG["tcp_host"])
stop_service.Servers.HTTP_SERVER = http_server
stop_service.Servers.SERVERS.append(task_scheduler)
stop_service.Servers.SERVERS.append(venvs_db)
stop_service.Servers.SERVERS.append(venv_history_db)
stop_service.Servers.SERVERS.append(applications_db)
stop_service.Servers.SERVERS.append(application_history_db)
stop_service.Servers.SERVERS.append(tasks_db)
stop_service.Servers.SERVERS.append(workflows_db)
stop_service.Servers.SERVERS.append(works_db)
stop_service.Servers.SERVERS.append(schedules_db)
stop_service.Servers.SERVERS.append(services_db)
stop_service.Servers.SERVERS.append(venv_manager)
stop_service.Servers.SERVERS.append(app_manager)
signal.signal(signal.SIGTERM, stop_service.sig_handler)
signal.signal(signal.SIGINT, stop_service.sig_handler)
tornado.ioloop.IOLoop.instance().start()
except Exception as e:
LOG.exception(e)
LOG.info("service end")
else:
print("failed to load configuration: %s" % args.config)
else:
parser.print_help()
logger.config_logging(file_name="test_sqlite_application_history.log",
log_level="DEBUG",
dir_name=os.path.join(cwd, "logs"),
day_rotate=False,
when="D",
interval=1,
max_size=20,
backup_count=5,
console=True)
LOG.debug("test start")
try:
CONFIG["data_path"] = cwd
db = ApplicationHistory()
LOG.debug("add: %s", db.add("id_1", "sha1_1", "description_1"))
LOG.debug("add: %s", db.add("id_1", "sha1_2", "description_2"))
LOG.debug("add: %s", db.add("id_1", "sha1_3", "description_3"))
LOG.debug("add: %s", db.add("id_2", "sha1_4", "description_4"))
LOG.debug("add: %s", db.add("id_2", "sha1_5", "description_5"))
LOG.debug("add: %s", db.add("id_3", "sha1_6", "description_6"))
LOG.debug("list: %s", json.dumps(db.list(), indent=4))
LOG.debug("list id_1: %s",
json.dumps(db.list(filters={"id": "id_1"}), indent=4))
LOG.debug("latest id_1: %s", json.dumps(db.get_latest("id_1"),
indent=4))
LOG.debug("delete id_1: %s", db.delete_by_app_id("id_1"))