def installPluginFromUrl(self, add):
if not os.path.exists(self.plugin_cache_path):
os.mkdir(self.plugin_cache_path)
plugin_url = add
if (not plugin_url.endswith(".jar")) and not (":" in plugin_url):
raise Exception(
"The parameter in --add should like be [plugin_name]:[version] e.g. app_runtime_with_db:1.0.0"
)
if (not plugin_url.endswith(".jar")) and (":" in plugin_url):
plugin_name, version = plugin_url.split(":")
plugin_jar_cache_path = self.downloadPlugin(
self.plugin_cache_path, plugin_name, version)
plugin_url = plugin_jar_cache_path
elif plugin_url.startswith("http://") or plugin_url.startswith(
"https://"):
plugin_url = self._downloadPlugin(plugin_url, {})
else:
run_cmd(["cp", "-r", plugin_url, self.plugin_cache_path])
plugin_url = os.path.join(self.plugin_cache_path,
plugin_url.split(os.path.pathsep)[-1])
self.installPluginFromCache(plugin_url)
def release_mlsql_plugin(actionManager, mvn, deploy, module_name, user,
password):
group = []
with open("./{}/desc.plugin".format(module_name), "r") as f:
config = {}
for line in f.readlines():
if line and line.strip():
clean_line = line.strip()
if clean_line == "__SPLITTER__":
group.append(config)
config = {}
else:
(k, v) = clean_line.split("=", 1)
config[k] = v
group.append(config)
for config in group:
print(config)
plugin_name = config.get("moduleName") or module_name
version = config["version"]
scala_version = config["scala_version"]
spark_version = "spark_version" in config and config["spark_version"]
if not mvn:
mvn = "mvn"
spark_params = []
if spark_version:
spark_params = ["-Pspark-{}".format(spark_version)]
command = [
mvn,
"-DskipTests",
"clean",
"package",
"-Pshade",
] + spark_params + ["-Pscala-{}".format(scala_version)
] + ["-pl", module_name]
run_cmd(command)
if deploy == "store":
deploy = "http://store.mlsql.tech/run"
full_path = pathlib.Path().absolute()
print("uploading {}.....".format(plugin_name))
jar_name = module_name
if spark_version:
jar_name = module_name + "-" + spark_version
ab_file_path = "{}/{}/target/{}".format(
full_path, module_name,
"{}_{}-{}.jar".format(jar_name, scala_version, version))
print("upload file： {}...".format(ab_file_path))
actionManager.uploadPlugin(
deploy, ab_file_path, {
"userName": user,
"password": password,
"pluginType": "MLSQL_PLUGIN",
"pluginName": plugin_name,
**config
})
def cache_app_jar(app_runtime_jar):
cache_path = get_cache_path()
cache_dir = pathlib.Path(cache_path).parent
if not cache_dir.exists():
cache_dir.mkdir(parents=True)
def is_http_loc():
return (app_runtime_jar.startswith("http://") or \
app_runtime_jar.startswith("https://"))
if not os.path.exists(cache_path) and is_http_loc():
run_cmd(["wget", app_runtime_jar, "-O", cache_path])
if not os.path.exists(cache_path) and not is_http_loc():
command = ["cp", "-r", app_runtime_jar, cache_path]
print(" ".join(command))
run_cmd(command)
def compile(dev, mvn, pl):
if not mvn:
mvn = "./build/mvn"
project_name = get_project_name()
# run_cmd("mvn", "-DskipTests", "clean", "install")
if not pl:
pl = "{}-lib".format(project_name)
if pl:
os.chdir(os.path.join(".", pl))
mvn = mvn.replace(".", "..")
if dev:
print("=======================")
print("incremental compile... ")
print("Using Ctrl+C to stop")
print("=======================")
run_cmd([mvn, "clean", "scala:cc", "-Dfsc=false"])
else:
run_cmd([mvn, "clean", "compile"])
def create(name, empty, include_ui, ui_command):
scala_prefix = "2.11"
pm.clone_project(name)
pm.change_project_name(name)
pm.generate_admin_token_in_yml(name)
if empty:
pm.clean_files_for_empty_project(name)
return
pm.change_all_poms(name, scala_prefix)
pm.create_cache_dir(name)
pm.create_project_file(name)
shutil.rmtree(os.path.join(".", name, ".git"))
pm.change_plugin_db_scala_file(name)
if include_ui:
command = "create-react-app web_console"
if ui_command:
command = ui_command
cwd = os.getcwd()
os.chdir(os.path.join(name))
run_cmd(command)
pm.change_package_json()
# run_cmd(["npm", "i", "@allwefantasy/web-platform-ui", "--save"])
run_cmd("cp -r web-ui-template/* web_console/src/", shell=True)
run_cmd("rm -rf web-ui-template", shell=True)
os.chdir(cwd)
print("done")
def change_project_name(name):
run_cmd(["mv", "baseweb", name])
run_cmd([
"mv",
os.path.join(name, "baseweb-bin"),
os.path.join(name, "{}-bin".format(name))
])
run_cmd([
"mv",
os.path.join(name, "baseweb-lib"),
os.path.join(name, "{}-lib".format(name))
])
def release(mvn, install, deploy, user, password, skip_ui, mlsql_store, module):
actionManager = ActionManager(None, None, None)
if mlsql_store:
release_mlsql_plugin(actionManager, mvn, deploy,
module, user, password)
return
project_name = get_project_name()
if not mvn:
mvn = "./build/mvn"
bin_project = "{}-bin".format(project_name)
if not skip_ui and os.path.exists("web_console"):
resource_dir = os.path.join(
bin_project, "src", "main", "resources", project_name, project_name)
if os.path.exists(resource_dir):
shutil.rmtree(resource_dir)
if not os.path.exists(resource_dir):
run_cmd(["mkdir", "-p", resource_dir])
cwd = os.getcwd()
os.chdir("./web_console")
run_cmd(["npm", "run", "build"])
os.chdir(cwd)
run_cmd(["rm", "-rf", os.path.join(resource_dir, "*")])
run_cmd("cp -r {} {}".format(os.path.join("web_console",
"build", "*"), resource_dir), shell=True)
if install:
install_module = "{}-{}".format(project_name, install)
command = [mvn, "-DskipTests", "clean",
"install", "-pl", install_module, "-am"]
run_cmd(command)
print("execute: {}".format(" ".join(command)))
return
command = [mvn, "-DskipTests", "clean", "package",
"-Pshade", "-pl", bin_project, "-am"]
print("execute: {}".format(" ".join(command)))
run_cmd(command)
if os.path.exists("release"):
shutil.rmtree("release")
os.mkdir("release")
files = [file for file in os.listdir(os.path.join(bin_project, "target")) if
file.endswith(".jar")
and not file.endswith("-sources.jar")
and not file.startswith("original-")
and not file.endswith("-javadoc.jar")]
for file in files:
run_cmd(["cp", "-r", "{}/target/{}".format(bin_project, file), "release"])
full_path = pathlib.Path().absolute()
print("{}/release/{}".format(full_path, file))
if deploy:
if deploy == "store":
deploy = "http://store.mlsql.tech/run"
actionManager.uploadPlugin(deploy, "{}/release/{}".format(full_path, file),
{"userName": user, "password": password, "pluginName": project_name})
def run(runtime, plugin_name, dev, mvn, debug_port):
project_name = get_project_name()
if not mvn:
mvn = "./build/mvn"
app_runtime_jar = jarmanager.get_app_jar_path(runtime)
cache_path = jarmanager.get_cache_path()
jarmanager.cache_app_jar(app_runtime_jar)
if os.path.exists(cache_path):
app_runtime_jar = cache_path
path_manager = PathManager(project_name)
lib_build_class_path = path_manager.lib_classes()
bin_build_class_path = path_manager.bin_classes()
# dependencies_output_path = os.path.join(".", "release", "libs")
if dev:
run_cmd([mvn, "-DskipTests", "install", "-pl",
"{}-lib".format(project_name), "-am"])
run_cmd([mvn, "-DskipTests", "compile",
"-pl", "{}-bin".format(project_name)])
plugins = [app_runtime_jar]
# run_cmd(["mvn", "dependency:copy-dependencies", "-DincludeScope=runtime",
# "-DoutputDirectory={}".format(dependencies_output_path)], "-fn")
class_path_str_file = os.path.join(".sfcli", ".classpath")
run_cmd([mvn, "dependency:build-classpath",
"-Dmdep.outputFile={}".format(class_path_str_file)])
with open(class_path_str_file, "r") as f:
class_path_str = f.readlines()[0].strip("\n")
app_runtime_jar = app_runtime_jar + ":" + \
class_path_str + ":" + lib_build_class_path
else:
plugins = ["./release/{}".format(jarName)
for jarName in os.listdir("release") if jarName.endswith(".jar")]
try:
os.setpgrp()
except OSError as e:
eprint("setpgrp failed, processes may not be "
"cleaned up properly: {}.".format(e))
# Don't start the reaper in this case as it could result in killing
# other user processes.
return None
main_class = appruntime.get_app_runtime_main_class()
if plugin_name is None:
args = appruntime.get_plugin_main_class()
else:
args = plugin_name
pluginPaths = ",".join(plugins)
pluginNames = args
debug_args = ""
if dev:
debug_args = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address={}".format(
str(debug_port))
command = ["java", "-Xms2g", debug_args, "-cp", ".:{}".format(app_runtime_jar), main_class,
"-pluginPaths {} -pluginNames {}".format(bin_build_class_path, pluginNames)]
print("start:{}".format(" ".join(command)))
def block_sigint():
import signal
signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT})
modified_env = os.environ.copy()
cwd = None
# stdout_file, stderr_file = new_log_files("sfcli-logs")
pipe_stdin = False
def start_process():
wow = subprocess.Popen(
command,
env=modified_env,
cwd=cwd,
stdout=sys.stdout,
stderr=sys.stderr,
stdin=subprocess.PIPE if pipe_stdin else None,
preexec_fn=block_sigint)
return wow
ProcessInfo.inner_process = start_process()
def handler(event):
ProcessInfo.event_buffer.append(event)
monitor_dir(lib_build_class_path, handler)
while True:
time.sleep(1)
if len(ProcessInfo.event_buffer) > 0:
print("Restarting....")
time.sleep(3)
ProcessInfo.event_buffer.clear()
ProcessInfo.kill()
count = 10
while ProcessInfo.inner_process is None and count > 0:
count -= 1
try:
ProcessInfo.inner_process = start_process()
except Exception as e:
time.sleep(1)
ProcessInfo.kill()
def clean_files_for_empty_project(name):
for item in [".git", "{}-bin".format(name), "{}-lib".format(name), "src"]:
shutil.rmtree(os.path.join(name, item))
for item in ["README.md", "pom.xml"]:
run_cmd(["rm", os.path.join(name, item)])
def clone_project(name):
command = ["git", "clone", "https://github.com/allwefantasy/baseweb"]
run_cmd(command)