def test_apps(p_changed_apps_files_map):
assert isinstance(p_changed_apps_files_map, dict)
print(
"\n\n TEST APPS ----------------------------------------------------- \n\n"
)
build_meta_map = gf_meta.get()['build_info_map']
apps_changes_deps_map = gf_meta.get()['apps_changes_deps_map']
apps_gf_packages_map = apps_changes_deps_map["apps_gf_packages_map"]
# AWS_CREDS
aws_creds_map = gf_os_aws_creds.get_from_env_vars()
assert isinstance(aws_creds_map, dict)
# nothing changed
if len(p_changed_apps_files_map.keys()) == 0:
return
else:
#------------------------
# GO
print("\nGO--------\n")
for app_name_str, v in p_changed_apps_files_map["go"].items():
test_name_str = "all"
# IMPORTANT!! - get all packages that are dependencies of this app, so that
# tests for all these packages can be run. dont just run the tests of the app
# but of all the packages that are its dependencies as well.
app_gf_packages_lst = apps_gf_packages_map[app_name_str]
# RUN_TESTS_FOR_ALL_APP_PACKAGES
for app_gf_package_name_str in app_gf_packages_lst:
print("about to run tests for - %s" %
(app_gf_package_name_str))
if build_meta_map.has_key(app_gf_package_name_str):
gf_package_meta_map = build_meta_map[
app_gf_package_name_str]
gf_tests.run(
app_gf_package_name_str,
test_name_str,
gf_package_meta_map,
aws_creds_map,
# IMPORTANT!! - in case the tests that gf_test.run() executes fail,
# run() should call exit() and force this whole process to exit,
# so that CI marks the build as failed.
p_exit_on_fail_bool=True)
def build_rust():
print("building RUST...")
apps_names_lst = [
# "gf_data_viz",
"gf_images_jobs"
]
build_meta_map = gf_meta.get()["build_info_map"]
for app_name_str in apps_names_lst:
print("\n\nRUST-------- %s\n\n" % (app_name_str))
app_meta_map = build_meta_map[app_name_str]
assert app_meta_map["type_str"] == "lib_rust"
for cargo_crate_map in app_meta_map["cargo_crate_specs_lst"]:
cargo_crate_dir_path_str = cargo_crate_map["dir_path_str"]
assert os.path.dirname(cargo_crate_dir_path_str)
# IMPORTANT!! - it can be specified on a per-crate basis if it should
# be compiled statically or not.
# for gf_images_jobs_py for example we specifically want to
# build a dynamic lib (even in Alpine) for importing into the Py VM.
static_bool = cargo_crate_map.get("static_bool", False)
# RUN
gf_build_rust.run(cargo_crate_dir_path_str,
p_static_bool=static_bool)
# PREPARE_LIBS
gf_build_rust.prepare_libs(app_name_str, cargo_crate_dir_path_str,
app_meta_map["type_str"])
def publish_apps_containers(p_changed_apps_files_map,
p_gf_dockerhub_user_str,
p_gf_dockerhub_pass_str,
p_git_commit_hash_str=None,
p_docker_sudo_bool=True):
assert isinstance(p_gf_dockerhub_user_str, str)
assert isinstance(p_gf_dockerhub_pass_str, str)
build_meta_map = gf_meta.get()["build_info_map"]
# "all" - this key holds a map with all the apps that had either their Go or Web code changed
apps_names_lst = p_changed_apps_files_map["all"].keys()
for app_name_str in apps_names_lst:
assert build_meta_map.has_key(app_name_str)
app_build_meta_map = build_meta_map[app_name_str]
# PUBLISH
gf_containers.publish(app_name_str,
app_build_meta_map,
p_gf_dockerhub_user_str,
p_gf_dockerhub_pass_str,
gf_log.log_fun,
p_git_commit_hash_str=p_git_commit_hash_str,
p_exit_on_fail_bool=True,
p_docker_sudo_bool=p_docker_sudo_bool)
def cont__build(p_app_name_str, p_dockerhub_user_name_str, p_log_fun, p_docker_sudo_bool = False): assert isinstance(p_dockerhub_user_name_str, str) build_meta_map = gf_meta.get()["build_info_map"] assert p_app_name_str in build_meta_map.keys() gf_builder_meta_map = build_meta_map[p_app_name_str] cont_image_name_str = gf_builder_meta_map["cont_image_name_str"] cont_image_version_str = gf_builder_meta_map["version_str"] cont_image_dockerfile_path_str = os.path.abspath(gf_builder_meta_map["dockerfile_path_str"]) assert os.path.isfile(cont_image_dockerfile_path_str) # DOCKER_BUILD image_full_name_str = "%s/%s:%s"%(p_dockerhub_user_name_str, cont_image_name_str, cont_image_version_str) gf_os_docker.build_image([image_full_name_str], cont_image_dockerfile_path_str, p_log_fun, p_exit_on_fail_bool = True, p_docker_sudo_bool = p_docker_sudo_bool)
def build_apps(p_changed_apps_files_map,
p_build_web_bool=True,
p_build_go_bool=True,
p_static_bool=True):
assert isinstance(p_changed_apps_files_map, dict)
print(
"\n\n BUILD APPS ----------------------------------------------------- \n\n"
)
build_meta_map = gf_meta.get()["build_info_map"]
# nothing changed
if len(p_changed_apps_files_map.keys()) == 0:
return
else:
# IMPORTANT!! - for each app that has any of its code changed rebuild both the Go and Web code,
# since the containers has to be fully rebuilt.
# "all" - this key holds a map with all the apps that had either their Go or Web code changed
apps_names_lst = list(p_changed_apps_files_map["all"].keys())
#------------------------
# WEB
if p_build_web_bool:
print("\n\nWEB--------\n\n")
web_meta_map = gf_web_meta.get()
gf_web__build.build(apps_names_lst, web_meta_map, gf_log.log_fun)
#------------------------
# GO
if p_build_go_bool:
for app_name_str in apps_names_lst:
app_meta_map = build_meta_map[app_name_str]
print("\n\nGO--------\n\n")
app_go_path_str = app_meta_map["go_path_str"]
app_go_output_path_str = app_meta_map["go_output_path_str"]
gf_build_go.run(
app_name_str,
app_go_path_str,
app_go_output_path_str,
# IMPORTANT!! - binaries are packaged in Alpine Linux, which uses a different standard library then stdlib,
# so all binary dependencies are to be statically linked into the output binary
# without depending on standard dynamic linking.
p_static_bool=p_static_bool,
# gf_build.run_go() should exit if the "go build" CLI run returns with a non-zero exit code.
# gf_builder.py is meant to run in CI environments, and so we want the stage in which it runs
# to be marked as failed because of the non-zero exit code.
p_exit_on_fail_bool=True)
def get_changed_apps():
print("DIFF")
apps_changes_deps_map = gf_meta.get()["apps_changes_deps_map"]
# LIST_CHANGED_APPS - determine how which apps/services changed
changed_apps_files_map = gf_build_changes.list_changed_apps(
apps_changes_deps_map, p_commits_lookback_int=1, p_mark_all_bool=True)
# VIEW
gf_build_changes.view_changed_apps(changed_apps_files_map, "go")
gf_build_changes.view_changed_apps(changed_apps_files_map, "web")
return changed_apps_files_map
def main():
print('')
print(' ------------- %sOPS%s %sGLOFLOW%s -------------'%(bg('dark_orange_3a'), attr(0), bg('cyan'), attr(0)))
print('')
b_meta_map = gf_meta.get()['build_info_map']
args_map = parse_args()
run_str = args_map['run']
aws_creds_file_path_str = args_map['aws_creds']
aws_creds_map = gf_aws_s3.parse_creds(aws_creds_file_path_str)
assert isinstance(aws_creds_map,dict)
#-------------
if run_str == 's3_data_info':
aws_access_key_id_str = aws_creds_map['GF_AWS_ACCESS_KEY_ID']
aws_secret_access_key_str = aws_creds_map['GF_AWS_SECRET_ACCESS_KEY']
gf_s3_data_info.stats__image_buckets_general(aws_access_key_id_str, aws_secret_access_key_str)
def build_apps_containers(p_changed_apps_files_map,
p_gf_dockerhub_user_str,
p_git_commit_hash_str=None,
p_docker_sudo_bool=True):
assert isinstance(p_changed_apps_files_map, dict)
build_meta_map = gf_meta.get()["build_info_map"]
web_meta_map = gf_web_meta.get()
# IMPORTANT!! - for each app that has any of its code changed rebuild both the Go and Web code,
# since the containers has to be fully rebuilt.
# "all" - this key holds a map with all the apps that had either their Go or Web code changed
apps_names_lst = p_changed_apps_files_map["all"].keys()
for app_name_str in apps_names_lst:
assert build_meta_map.has_key(app_name_str)
app_build_meta_map = build_meta_map[app_name_str]
assert web_meta_map.has_key(app_name_str)
app_web_meta_map = web_meta_map[app_name_str]
gf_containers.build(
app_name_str,
app_build_meta_map,
app_web_meta_map,
gf_log.log_fun,
p_git_commit_hash_str=p_git_commit_hash_str,
# DOCKERHUB_USER
p_user_name_str=p_gf_dockerhub_user_str,
# gf_containers.build() should exit if the docker build CLI run returns with a non-zero exit code.
# gf_builder.py is meant to run in CI environments, and so we want the stage in which it runs
# to be marked as failed because of the non-zero exit code.
p_exit_on_fail_bool=True,
p_docker_sudo_bool=p_docker_sudo_bool)
def main():
print('')
print(' %sBUILD GLOFLOW%s' %
(fg('green'), attr(0)))
print('')
#--------------------------------------------------
def log_fun(g, m):
if g == "ERROR":
print('%s%s%s:%s%s%s' %
(bg('red'), g, attr(0), fg('red'), m, attr(0)))
else:
print('%s%s%s:%s%s%s' %
(fg('yellow'), g, attr(0), fg('green'), m, attr(0)))
#--------------------------------------------------
b_meta_map = gf_meta.get()['build_info_map']
args_map = parse_args()
run_str = args_map['run']
app_name_str = args_map['app']
assert b_meta_map.has_key(app_name_str)
#--------------------------------------------------
def go_build(p_static_bool):
app_meta_map = b_meta_map[app_name_str]
if not app_meta_map.has_key('go_output_path_str'):
print("not a main package")
exit()
gf_build.run_go(app_name_str,
app_meta_map['go_path_str'],
app_meta_map['go_output_path_str'],
p_static_bool=p_static_bool)
#--------------------------------------------------
#-------------
#BUILD
if run_str == 'build':
#build using dynamic linking, its quicker while in dev.
go_build(False)
#-------------
#BUILD_WEB
elif run_str == 'build_web':
apps_names_lst = [app_name_str]
apps_meta_map = gf_web_meta.get()
gf_web__build.build(apps_names_lst, apps_meta_map, log_fun)
#TEMPORARY!! - copies built web code to the bin/ dir where the Go binary is placed,
# so if the binary is run directly it can serve the web code.
web_build_dir_str = apps_meta_map[app_name_str]['pages_map'][
'gf_trader']['build_dir_str']
gf_cli_utils.run_cmd('cp -r %s/* %s/../bin/gf_trader/static' %
(web_build_dir_str, cwd_str))
#-------------
#BUILD_CONTAINERS
elif run_str == 'build_containers':
#build using static linking, containers are based on Alpine linux,
#which has a minimal stdlib and other libraries, so we want to compile
#everything needed by this Go package into a single binary.
go_build(True)
gf_containers.build(app_name_str, log_fun)
#-------------
#TEST
elif run_str == 'test':
test_name_str = args_map['test_name']
gf_tests.run(app_name_str, test_name_str, app_meta_map)
#-------------
else:
print("unknown run command - %s" % (run_str))
exit()
def main():
print("")
print(" %sGLOFLOW BUILD TOOL%s"%(fg("green"), attr(0)))
print("")
build_meta_map = gf_meta.get()["build_info_map"]
apps_changes_deps_map = gf_meta.get()["apps_changes_deps_map"]
args_map = parse_args()
run_str = args_map["run"]
app_name_str = args_map["app"]
assert app_name_str in build_meta_map.keys()
#--------------------------------------------------
def go_build_app_in_cont(p_static_bool):
app_meta_map = build_meta_map[app_name_str]
if not "go_output_path_str" in app_meta_map.keys():
print("not a main package")
exit()
gf_build_go.run_in_cont(app_name_str,
app_meta_map["go_path_str"],
app_meta_map["go_output_path_str"])
#--------------------------------------------------
def go_build_app(p_static_bool):
app_meta_map = build_meta_map[app_name_str]
if not "go_output_path_str" in app_meta_map.keys():
print("not a main package")
exit()
fetch_deps_bool = args_map["fetch_deps"]
gf_build_go.run(app_name_str,
app_meta_map["go_path_str"],
app_meta_map["go_output_path_str"],
p_static_bool = p_static_bool,
p_go_get_bool = fetch_deps_bool)
#--------------------------------------------------
def rust_build_apps_in_cont():
print("RUST BUILD IN CONTAINER...")
gf_build_rust.run_in_cont()
#--------------------------------------------------
def rust_build_apps():
print(app_name_str)
assert app_name_str == "gf_images_jobs"
# APP_META
app_meta_map = build_meta_map[app_name_str]
assert "type_str" in app_meta_map.keys()
assert app_meta_map["type_str"] == "lib_rust"
# CARGO_CRATE_DIR_PATHS
assert "cargo_crate_specs_lst" in app_meta_map.keys()
cargo_crate_specs_lst = app_meta_map["cargo_crate_specs_lst"]
assert isinstance(cargo_crate_specs_lst, list)
for s_map in cargo_crate_specs_lst:
assert "dir_path_str" in s_map.keys()
assert os.path.isdir(s_map["dir_path_str"])
for s_map in cargo_crate_specs_lst:
dir_path_str = s_map["dir_path_str"]
print("")
print("------------------------------------------------------------")
print(" BUILD CARGO CRATE - %s"%(dir_path_str))
print("")
# STATIC
static_bool = False
if "static_bool" in s_map.keys():
static_bool = s_map["static_bool"]
# BUILD
gf_build_rust.run(dir_path_str,
p_static_bool = static_bool)
# PREPARE_LIBS
gf_build_rust.prepare_libs(app_name_str,
dir_path_str,
app_meta_map["type_str"])
#--------------------------------------------------
# AWS_CREDS
def aws_creds_get():
aws_creds_file_path_str = args_map["aws_creds"]
aws_creds_file_path_abs_str = os.path.abspath(aws_creds_file_path_str)
print(aws_creds_file_path_abs_str)
assert os.path.isfile(aws_creds_file_path_abs_str)
aws_creds_map = gf_aws_s3.parse_creds(aws_creds_file_path_str)
return aws_creds_map
#-------------
# BUILD_GO
if run_str == "build" or run_str == "build_go":
build_outof_cont_bool = args_map["build_outof_cont"]
if build_outof_cont_bool:
go_build_app(False)
else:
go_build_app_in_cont(False)
#-------------
# BUILD_RUST
elif run_str == "build_rust":
build_outof_cont_bool = args_map["build_outof_cont"]
if build_outof_cont_bool:
rust_build_apps()
else:
rust_build_apps_in_cont()
#-------------
# BUILD_WEB
elif run_str == "build_web":
apps_names_lst = [app_name_str]
web_meta_map = gf_web_meta.get()
page_name_str = args_map["page_name"]
build_outof_cont_bool = args_map["build_outof_cont"]
if build_outof_cont_bool:
gf_web__build.build(apps_names_lst, web_meta_map,
gf_log.log_fun,
p_page_name_str=page_name_str)
else:
gf_web__build.run_in_cont(app_name_str,
p_page_name_str=page_name_str)
#-------------
# BUILD_CONTAINERS
elif run_str == "build_containers":
assert app_name_str in build_meta_map.keys()
app_build_meta_map = build_meta_map[app_name_str]
# if app_build_meta_map["type_str"] == "main_go":
#
# # STATIC_LINKING
# # build using static linking, containers are based on Alpine linux,
# # which has a minimal stdlib and other libraries, so we want to compile
# # everything needed by this Go package into a single binary.
# go_build_app_in_cont(True)
dockerhub_user_str = args_map["dockerhub_user"]
docker_sudo_bool = args_map["docker_sudo"]
# GF_BUILDER
if app_name_str.startswith("gf_builder"):
gf_builder_ops.cont__build(app_name_str,
dockerhub_user_str,
gf_log.log_fun,
p_docker_sudo_bool = docker_sudo_bool)
else:
# GIT_COMMIT_HASH
git_commit_hash_str = None
if "DRONE_COMMIT" in os.environ.keys():
git_commit_hash_str = os.environ["DRONE_COMMIT"]
app_web_meta_map = None
web_meta_map = gf_web_meta.get()
if app_name_str in web_meta_map.keys():
app_web_meta_map = web_meta_map[app_name_str]
gf_containers.build(app_name_str,
app_build_meta_map,
gf_log.log_fun,
p_app_web_meta_map = app_web_meta_map,
p_user_name_str = dockerhub_user_str,
p_git_commit_hash_str = git_commit_hash_str,
p_docker_sudo_bool = docker_sudo_bool)
#-------------
# PUBLISH_CONTAINERS
elif run_str == "publish_containers":
dockerhub_user_str = args_map["dockerhub_user"]
dockerhub_pass_str = args_map["dockerhub_pass"]
docker_sudo_bool = args_map["docker_sudo"]
assert app_name_str in build_meta_map.keys()
app_build_meta_map = build_meta_map[app_name_str]
# GIT_COMMIT_HASH
git_commit_hash_str = None
if "DRONE_COMMIT" in os.environ.keys():
git_commit_hash_str = os.environ["DRONE_COMMIT"]
# # GF_BUILDER
# if app_name_str.startswith("gf_builder"):
#
# gf_builder_ops.cont__publish(app_name_str,
# app_build_meta_map,
# dockerhub_user_str,
# dockerhub_pass_str,
# gf_log.log_fun,
# p_docker_sudo_bool = docker_sudo_bool)
# else:
gf_builder_ops.cont__publish(app_name_str,
app_build_meta_map,
dockerhub_user_str,
dockerhub_pass_str,
gf_log.log_fun,
p_git_commit_hash_str = git_commit_hash_str,
p_docker_sudo_bool = docker_sudo_bool)
#-------------
# TEST
elif run_str == "test":
app_meta_map = build_meta_map[app_name_str]
test_name_str = args_map["test_name"]
aws_creds_map = aws_creds_get()
gf_tests.run(app_name_str,
test_name_str,
app_meta_map,
aws_creds_map)
#-------------
# LIST_CHANGED_APPS
elif run_str == "list_changed_apps":
changed_apps_map = gf_build_changes.list_changed_apps(apps_changes_deps_map)
gf_build_changes.view_changed_apps(changed_apps_map, "go")
gf_build_changes.view_changed_apps(changed_apps_map, "web")
#-------------
# # GF_BUILDER__CONTAINER_BUILD
# elif run_str == "gf_builder__cont_build":
# dockerhub_user_str = args_map["dockerhub_user"]
# docker_sudo_bool = args_map["docker_sudo"]
#
# gf_builder_ops.cont__build(dockerhub_user_str,
# gf_log.log_fun,
# p_docker_sudo_bool = docker_sudo_bool)
#-------------
else:
print("unknown run command - %s"%(run_str))
exit()