source_app_path = input_data["source"] target_app_path = input_data["target"] target_app_log_path = target_app_path + ".txt" source_format = "tar.gz" if "zip" in os.path.split(source_app_path)[-1]: source_format = "zip" target_format = "tar.gz" if "zip" in os.path.split(target_app_path)[-1]: target_format = "zip" local_tmp_path = os.path.join(workspace, "app.%s" % source_format) if os.path.exists(local_tmp_path): os.remove(local_tmp_path) LOG.info("remove old: %s", local_tmp_path) success = ldfs.download_file(source_app_path, local_tmp_path) if success: if source_format == "zip": z = zipfile.ZipFile(local_tmp_path, "r") path_parts = splitall(z.namelist()[0]) app_root_name = path_parts[1] if path_parts[ 0] == "." else path_parts[0] local_tmp_app_path = os.path.join(workspace, app_root_name) if os.path.exists(local_tmp_app_path): shutil.rmtree(local_tmp_app_path) LOG.info("remove old: %s", local_tmp_app_path) z.extractall(workspace) z.close()
def main(): logger.config_logging(log_level="DEBUG", console=True, filesystem=False) try: lpl_address = args.litepipeline lpl_host, lpl_port = lpl_address.split(":") lpl = LitePipelineClient(lpl_host, lpl_port) manager_info = lpl.cluster_info(include=["manager"]) ldfs_host = manager_info["info"]["manager"]["ldfs_http_host"] ldfs_port = manager_info["info"]["manager"]["ldfs_http_port"] ldfs = LiteDFSClient(ldfs_host, ldfs_port) pack_app_id = args.pack_app_id operate = args.operate app_path = args.input app_format = args.format app_id = args.app_id app_name = args.name app_description = args.description workspace = "." now = datetime.datetime.now() LOG.info("workspace directory: %s, now: %s", workspace, str(now)) if operate in ["pack", "create", "update"]: if os.path.exists(app_path) and os.path.isdir(app_path): app_dir_name = os.path.split(app_path.strip("/"))[-1] tar_name = "%s.tar.gz" % app_dir_name target_name = "%s.%s" % (app_dir_name, app_format) tmp_tar_path = os.path.join(workspace, tar_name) if os.path.exists(tmp_tar_path): os.remove(tmp_tar_path) LOG.warning("remove old file [%s]", tmp_tar_path) LOG.info("compress [%s] start", tmp_tar_path) with tarfile.open(tmp_tar_path, "w:gz") as tar: tar.add(app_path, arcname=app_dir_name) LOG.info("compress [%s] end", tmp_tar_path) remote_source_path = os.path.join("/pack_application/source", str(now), tar_name) ldfs.delete_file(remote_source_path) success = ldfs.create_file(tmp_tar_path, remote_source_path, replica=1) if success: LOG.info("upload [%s] to remote [%s] success", tmp_tar_path, remote_source_path) remote_target_path = os.path.join( "/pack_application/target", str(now), target_name) task_name = "pack %s" % app_dir_name task_input = { "source": remote_source_path, "target": remote_target_path, } r = lpl.task_create(task_name, pack_app_id, task_input) if r: task_id = r["task_id"] LOG.info("create pack task [task_id: %s] success", task_id) stage = "running" task_info = None while stage != "finished": task_info = lpl.task_info(task_id) if task_info: stage = task_info["task_info"]["stage"] update_at = task_info["task_info"]["update_at"] LOG.info( "task[task_id: %s], stage: %s, update_at: %s", task_id, stage, update_at) time.sleep(5) ldfs.delete_directory( os.path.split(remote_source_path)[0]) if task_info: if task_info["task_info"]["status"] == "success": task_result = task_info["task_info"]["result"][ "pack application"]["result"]["data"] remote_pack_log_path = task_result[ "app_log_path"] remote_pack_path = task_result["app_path"] LOG.info( "pack task success, remote pack log: %s, remote packed application: %s", remote_pack_log_path, remote_pack_path) local_pack_log_path = os.path.join( workspace, "%s.pack.log" % app_dir_name) local_pack_path = os.path.join( workspace, "%s.pack.%s" % (app_dir_name, app_format)) if os.path.exists(local_pack_log_path ) and os.path.isfile( local_pack_log_path): os.remove(local_pack_log_path) if os.path.exists( local_pack_path) and os.path.isfile( local_pack_path): os.remove(local_pack_path) success = ldfs.download_file( remote_pack_log_path, local_pack_log_path) if success: LOG.info( "download pack log as [%s] success", local_pack_log_path) else: LOG.error("download pack log failed") success = ldfs.download_file( remote_pack_path, local_pack_path) if success: ldfs.delete_directory( os.path.split(remote_pack_path)[0]) LOG.info( "download packed application as [%s] success", local_pack_path) else: LOG.error( "download packed application failed") if operate == "create": if app_name: r = lpl.application_create( local_pack_path, app_name, description=app_description) if r: LOG.info( "create application[app_id: %s] success", r["app_id"]) else: LOG.error( "create application failed") else: LOG.error("need -n/--name parameter") elif operate == "update": if app_id: if app_name and app_description: r = lpl.application_update( app_id, file_path=local_pack_path, name=app_name, description=app_description) elif app_name: r = lpl.application_update( app_id, file_path=local_pack_path, name=app_name) elif app_description: r = lpl.application_update( app_id, file_path=local_pack_path, description=app_description) else: r = lpl.application_update( app_id, file_path=local_pack_path) if r: LOG.info( "update application[app_id: %s] success", app_id) else: LOG.error( "update application failed") else: LOG.error("need -a/--app_id parameter") else: LOG.error("pack task failed\ntask_info: %s", json.dumps(task_info, indent=4)) else: LOG.error("pack task failed") else: LOG.error("create pack task failed") else: LOG.error("upload [%s] to remote [%s] failed", tmp_tar_path, remote_source_path) else: LOG.error("input [%s] directory not exists", app_path) else: LOG.error("unknown operate: %s", operate) except Exception as e: LOG.exception(e)