def installPluginFromUrl(self, add): if not os.path.exists(self.plugin_cache_path): os.mkdir(self.plugin_cache_path) plugin_url = add if (not plugin_url.endswith(".jar")) and not (":" in plugin_url): raise Exception( "The parameter in --add should like be [plugin_name]:[version] e.g. app_runtime_with_db:1.0.0" ) if (not plugin_url.endswith(".jar")) and (":" in plugin_url): plugin_name, version = plugin_url.split(":") plugin_jar_cache_path = self.downloadPlugin( self.plugin_cache_path, plugin_name, version) plugin_url = plugin_jar_cache_path elif plugin_url.startswith("http://") or plugin_url.startswith( "https://"): plugin_url = self._downloadPlugin(plugin_url, {}) else: run_cmd(["cp", "-r", plugin_url, self.plugin_cache_path]) plugin_url = os.path.join(self.plugin_cache_path, plugin_url.split(os.path.pathsep)[-1]) self.installPluginFromCache(plugin_url)
def release_mlsql_plugin(actionManager, mvn, deploy, module_name, user, password): group = [] with open("./{}/desc.plugin".format(module_name), "r") as f: config = {} for line in f.readlines(): if line and line.strip(): clean_line = line.strip() if clean_line == "__SPLITTER__": group.append(config) config = {} else: (k, v) = clean_line.split("=", 1) config[k] = v group.append(config) for config in group: print(config) plugin_name = config.get("moduleName") or module_name version = config["version"] scala_version = config["scala_version"] spark_version = "spark_version" in config and config["spark_version"] if not mvn: mvn = "mvn" spark_params = [] if spark_version: spark_params = ["-Pspark-{}".format(spark_version)] command = [ mvn, "-DskipTests", "clean", "package", "-Pshade", ] + spark_params + ["-Pscala-{}".format(scala_version) ] + ["-pl", module_name] run_cmd(command) if deploy == "store": deploy = "http://store.mlsql.tech/run" full_path = pathlib.Path().absolute() print("uploading {}.....".format(plugin_name)) jar_name = module_name if spark_version: jar_name = module_name + "-" + spark_version ab_file_path = "{}/{}/target/{}".format( full_path, module_name, "{}_{}-{}.jar".format(jar_name, scala_version, version)) print("upload file: {}...".format(ab_file_path)) actionManager.uploadPlugin( deploy, ab_file_path, { "userName": user, "password": password, "pluginType": "MLSQL_PLUGIN", "pluginName": plugin_name, **config })
def cache_app_jar(app_runtime_jar): cache_path = get_cache_path() cache_dir = pathlib.Path(cache_path).parent if not cache_dir.exists(): cache_dir.mkdir(parents=True) def is_http_loc(): return (app_runtime_jar.startswith("http://") or \ app_runtime_jar.startswith("https://")) if not os.path.exists(cache_path) and is_http_loc(): run_cmd(["wget", app_runtime_jar, "-O", cache_path]) if not os.path.exists(cache_path) and not is_http_loc(): command = ["cp", "-r", app_runtime_jar, cache_path] print(" ".join(command)) run_cmd(command)
def compile(dev, mvn, pl): if not mvn: mvn = "./build/mvn" project_name = get_project_name() # run_cmd("mvn", "-DskipTests", "clean", "install") if not pl: pl = "{}-lib".format(project_name) if pl: os.chdir(os.path.join(".", pl)) mvn = mvn.replace(".", "..") if dev: print("=======================") print("incremental compile... ") print("Using Ctrl+C to stop") print("=======================") run_cmd([mvn, "clean", "scala:cc", "-Dfsc=false"]) else: run_cmd([mvn, "clean", "compile"])
def create(name, empty, include_ui, ui_command): scala_prefix = "2.11" pm.clone_project(name) pm.change_project_name(name) pm.generate_admin_token_in_yml(name) if empty: pm.clean_files_for_empty_project(name) return pm.change_all_poms(name, scala_prefix) pm.create_cache_dir(name) pm.create_project_file(name) shutil.rmtree(os.path.join(".", name, ".git")) pm.change_plugin_db_scala_file(name) if include_ui: command = "create-react-app web_console" if ui_command: command = ui_command cwd = os.getcwd() os.chdir(os.path.join(name)) run_cmd(command) pm.change_package_json() # run_cmd(["npm", "i", "@allwefantasy/web-platform-ui", "--save"]) run_cmd("cp -r web-ui-template/* web_console/src/", shell=True) run_cmd("rm -rf web-ui-template", shell=True) os.chdir(cwd) print("done")
def change_project_name(name): run_cmd(["mv", "baseweb", name]) run_cmd([ "mv", os.path.join(name, "baseweb-bin"), os.path.join(name, "{}-bin".format(name)) ]) run_cmd([ "mv", os.path.join(name, "baseweb-lib"), os.path.join(name, "{}-lib".format(name)) ])
def release(mvn, install, deploy, user, password, skip_ui, mlsql_store, module): actionManager = ActionManager(None, None, None) if mlsql_store: release_mlsql_plugin(actionManager, mvn, deploy, module, user, password) return project_name = get_project_name() if not mvn: mvn = "./build/mvn" bin_project = "{}-bin".format(project_name) if not skip_ui and os.path.exists("web_console"): resource_dir = os.path.join( bin_project, "src", "main", "resources", project_name, project_name) if os.path.exists(resource_dir): shutil.rmtree(resource_dir) if not os.path.exists(resource_dir): run_cmd(["mkdir", "-p", resource_dir]) cwd = os.getcwd() os.chdir("./web_console") run_cmd(["npm", "run", "build"]) os.chdir(cwd) run_cmd(["rm", "-rf", os.path.join(resource_dir, "*")]) run_cmd("cp -r {} {}".format(os.path.join("web_console", "build", "*"), resource_dir), shell=True) if install: install_module = "{}-{}".format(project_name, install) command = [mvn, "-DskipTests", "clean", "install", "-pl", install_module, "-am"] run_cmd(command) print("execute: {}".format(" ".join(command))) return command = [mvn, "-DskipTests", "clean", "package", "-Pshade", "-pl", bin_project, "-am"] print("execute: {}".format(" ".join(command))) run_cmd(command) if os.path.exists("release"): shutil.rmtree("release") os.mkdir("release") files = [file for file in os.listdir(os.path.join(bin_project, "target")) if file.endswith(".jar") and not file.endswith("-sources.jar") and not file.startswith("original-") and not file.endswith("-javadoc.jar")] for file in files: run_cmd(["cp", "-r", "{}/target/{}".format(bin_project, file), "release"]) full_path = pathlib.Path().absolute() print("{}/release/{}".format(full_path, file)) if deploy: if deploy == "store": deploy = "http://store.mlsql.tech/run" actionManager.uploadPlugin(deploy, "{}/release/{}".format(full_path, file), {"userName": user, "password": password, "pluginName": project_name})
def run(runtime, plugin_name, dev, mvn, debug_port): project_name = get_project_name() if not mvn: mvn = "./build/mvn" app_runtime_jar = jarmanager.get_app_jar_path(runtime) cache_path = jarmanager.get_cache_path() jarmanager.cache_app_jar(app_runtime_jar) if os.path.exists(cache_path): app_runtime_jar = cache_path path_manager = PathManager(project_name) lib_build_class_path = path_manager.lib_classes() bin_build_class_path = path_manager.bin_classes() # dependencies_output_path = os.path.join(".", "release", "libs") if dev: run_cmd([mvn, "-DskipTests", "install", "-pl", "{}-lib".format(project_name), "-am"]) run_cmd([mvn, "-DskipTests", "compile", "-pl", "{}-bin".format(project_name)]) plugins = [app_runtime_jar] # run_cmd(["mvn", "dependency:copy-dependencies", "-DincludeScope=runtime", # "-DoutputDirectory={}".format(dependencies_output_path)], "-fn") class_path_str_file = os.path.join(".sfcli", ".classpath") run_cmd([mvn, "dependency:build-classpath", "-Dmdep.outputFile={}".format(class_path_str_file)]) with open(class_path_str_file, "r") as f: class_path_str = f.readlines()[0].strip("\n") app_runtime_jar = app_runtime_jar + ":" + \ class_path_str + ":" + lib_build_class_path else: plugins = ["./release/{}".format(jarName) for jarName in os.listdir("release") if jarName.endswith(".jar")] try: os.setpgrp() except OSError as e: eprint("setpgrp failed, processes may not be " "cleaned up properly: {}.".format(e)) # Don't start the reaper in this case as it could result in killing # other user processes. return None main_class = appruntime.get_app_runtime_main_class() if plugin_name is None: args = appruntime.get_plugin_main_class() else: args = plugin_name pluginPaths = ",".join(plugins) pluginNames = args debug_args = "" if dev: debug_args = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address={}".format( str(debug_port)) command = ["java", "-Xms2g", debug_args, "-cp", ".:{}".format(app_runtime_jar), main_class, "-pluginPaths {} -pluginNames {}".format(bin_build_class_path, pluginNames)] print("start:{}".format(" ".join(command))) def block_sigint(): import signal signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT}) modified_env = os.environ.copy() cwd = None # stdout_file, stderr_file = new_log_files("sfcli-logs") pipe_stdin = False def start_process(): wow = subprocess.Popen( command, env=modified_env, cwd=cwd, stdout=sys.stdout, stderr=sys.stderr, stdin=subprocess.PIPE if pipe_stdin else None, preexec_fn=block_sigint) return wow ProcessInfo.inner_process = start_process() def handler(event): ProcessInfo.event_buffer.append(event) monitor_dir(lib_build_class_path, handler) while True: time.sleep(1) if len(ProcessInfo.event_buffer) > 0: print("Restarting....") time.sleep(3) ProcessInfo.event_buffer.clear() ProcessInfo.kill() count = 10 while ProcessInfo.inner_process is None and count > 0: count -= 1 try: ProcessInfo.inner_process = start_process() except Exception as e: time.sleep(1) ProcessInfo.kill()
def clean_files_for_empty_project(name): for item in [".git", "{}-bin".format(name), "{}-lib".format(name), "src"]: shutil.rmtree(os.path.join(name, item)) for item in ["README.md", "pom.xml"]: run_cmd(["rm", os.path.join(name, item)])
def clone_project(name): command = ["git", "clone", "https://github.com/allwefantasy/baseweb"] run_cmd(command)