class Config:
"""
General configuration for CI/CD steps
"""
SERVER_ADDRESS = os.getenv("SERVER_ADDRESS", "localhost:8080")
EXTENSION_TO_SDK = {
"java": SDK_JAVA,
"go": SDK_GO,
"py": SDK_PYTHON,
"scala": SDK_SCIO
}
SUPPORTED_SDK = (Sdk.Name(SDK_JAVA), Sdk.Name(SDK_GO),
Sdk.Name(SDK_PYTHON), Sdk.Name(SDK_SCIO))
BUCKET_NAME = "playground-precompiled-objects"
TEMP_FOLDER = "temp"
SDK_TO_EXTENSION = {
SDK_JAVA: "java",
SDK_GO: "go",
SDK_PYTHON: "py",
SDK_SCIO: "scala"
}
NO_STORE = "no-store"
ERROR_STATUSES = [
STATUS_VALIDATION_ERROR, STATUS_ERROR, STATUS_PREPARATION_ERROR,
STATUS_COMPILE_ERROR, STATUS_RUN_TIMEOUT, STATUS_RUN_ERROR
]
BEAM_PLAYGROUND_TITLE = "beam-playground:\n"
BEAM_PLAYGROUND = "beam-playground"
PAUSE_DELAY = 10
CI_STEP_NAME = "CI"
CD_STEP_NAME = "CD"
CI_CD_LITERAL = Literal["CI", "CD"]
LINK_PREFIX = "https://github.com/apache/beam/blob/master"
def _write_to_local_fs(self, example: Example):
"""
Write code of an example, output and meta info
to the filesystem (in temp folder)
Args:
example: example object
Returns: dict {path_at_the_bucket:path_at_the_os}
"""
path_to_object_folder = os.path.join(Config.TEMP_FOLDER,
example.pipeline_id,
Sdk.Name(example.sdk),
example.tag.name)
Path(path_to_object_folder).mkdir(parents=True, exist_ok=True)
file_names = {}
code_path = self._get_gcs_object_name(
sdk=example.sdk,
base_folder_name=example.tag.name,
file_name=example.tag.name)
output_path = self._get_gcs_object_name(
sdk=example.sdk,
base_folder_name=example.tag.name,
file_name=example.tag.name,
extension=PrecompiledExample.OUTPUT_EXTENSION)
log_path = self._get_gcs_object_name(
sdk=example.sdk,
base_folder_name=example.tag.name,
file_name=example.tag.name,
extension=PrecompiledExample.LOG_EXTENSION)
graph_path = self._get_gcs_object_name(
sdk=example.sdk,
base_folder_name=example.tag.name,
file_name=example.tag.name,
extension=PrecompiledExample.GRAPH_EXTENSION)
meta_path = self._get_gcs_object_name(
sdk=example.sdk,
base_folder_name=example.tag.name,
file_name=PrecompiledExample.META_NAME,
extension=PrecompiledExample.META_EXTENSION)
file_names[code_path] = example.code
file_names[output_path] = example.output
meta = example.tag._asdict()
meta["type"] = example.type
meta["link"] = example.link
file_names[meta_path] = json.dumps(meta)
file_names[log_path] = example.logs
if example.sdk == SDK_PYTHON or example.sdk == SDK_JAVA:
file_names[graph_path] = example.graph
for file_name, file_content in file_names.items():
local_file_path = os.path.join(Config.TEMP_FOLDER,
example.pipeline_id, file_name)
with open(local_file_path, "w", encoding="utf-8") as file:
file.write(file_content)
# don't need content anymore, instead save the local path
file_names[file_name] = local_file_path
return file_names
def _get_gcs_object_name(self,
sdk: Sdk,
base_folder_name: str,
file_name: str,
extension: str = None):
"""
Get the path where file will be stored at the bucket.
Args:
sdk: sdk of the example
file_name: name of the example
base_folder_name: name of the folder where example is stored
(eq. to example name)
extension: extension of the file
Returns: file name
"""
if extension is None:
extension = Config.EXTENSIONS[Sdk.Name(sdk)]
return os.path.join(Sdk.Name(sdk), base_folder_name,
f"{file_name}.{extension}")
def test__write_default_example_path_to_local_fs(delete_temp_folder):
"""
Test writing default example link of sdk to
the filesystem (in temp folder)
Args:
delete_temp_folder: python fixture to clean up temp folder
after method execution
"""
sdk = Sdk.Name(SDK_GO)
default_example_path = "SDK_GO/PRECOMPILED_OBJECT_TYPE_EXAMPLE/WordCount"
expected_result = str(pathlib.Path(sdk, Config.DEFAULT_PRECOMPILED_OBJECT))
cloud_path = CDHelper()._write_default_example_path_to_local_fs(
default_example_path)
assert cloud_path == expected_result
assert os.path.exists(os.path.join("temp", cloud_path))
raise KeyError(
"BEAM_ROOT_DIR environment variable should be specified in os")
if categories_file is None:
raise KeyError(
"BEAM_EXAMPLE_CATEGORIES environment variable should be specified in os"
)
def _run_ci_cd(step: config.Config.CI_CD_LITERAL, sdk: Sdk):
supported_categories = get_supported_categories(categories_file)
logging.info("Start of searching Playground examples ...")
examples = find_examples(root_dir, supported_categories, sdk)
logging.info("Finish of searching Playground examples")
logging.info("Number of found Playground examples: %s", len(examples))
if step == config.Config.CI_STEP_NAME:
logging.info("Start of verification Playground examples ...")
_ci_step(examples=examples)
logging.info("Finish of verification Playground examples")
if step == config.Config.CD_STEP_NAME:
logging.info("Start of storing Playground examples ...")
_cd_step(examples=examples)
logging.info("Finish of storing Playground examples")
if __name__ == "__main__":
parser = parser.parse_args()
_check_envs()
setup_logger()
_run_ci_cd(parser.step, Sdk.Value(parser.sdk))
