Exemplo n.º 1
0
def get_test_info():
    http = HttpResponse()
    io_utils = IOUtils()
    variables = "testinfo.json"
    file = EnvConstants.VARIABLES_PATH + "/" + variables

    try:
        file_path = Path(file)
        if not file_path.is_file():
            io_utils.write_to_file_dict(file, test_info_init)
        test_env_vars = json.loads(io_utils.read_file(file))
        test_env_vars["processes"] = [p.info for p in psutil.process_iter(attrs=['pid', 'name', 'username', 'status'])]

    except Exception as e:
        exception = "Exception({0})".format(e.__str__())
        return Response(json.dumps(http.failure(ApiCodeConstants.GET_CONTAINER_TEST_INFO_FAILURE,
                                                ErrorCodes.HTTP_CODE.get(
                                                    ApiCodeConstants.GET_CONTAINER_TEST_INFO_FAILURE),
                                                exception,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")
    return Response(
        json.dumps(
            http.success(ApiCodeConstants.SUCCESS, ErrorCodes.HTTP_CODE.get(ApiCodeConstants.SUCCESS), test_env_vars)),
        200,
        mimetype="application/json")
Exemplo n.º 2
0
    def delete_deployment_id(self, depl_id):
        depl_id = depl_id.strip()
        docker_utils = DockerUtils()
        depl_folder = f"{EnvInit.init.get(EnvConstants.DEPLOY_PATH)}/{depl_id}"
        file = f"{depl_folder}/docker-compose.yml"
        try:
            status = docker_utils.down(file)
            if "Cannot connect to the Docker daemon".lower() in status.get(
                    'err').lower():
                raise Exception(status.get('err'))
            app.logger.debug({"msg": status})
            status = docker_utils.ps(depl_id)
            result = status.get('out').split("\n")[1:]
            DeploymentMetadataSingleton.get_instance(
            ).delete_metadata_for_deployment(depl_id)
            IOUtils.remove_directory(f"{depl_folder}")
        except Exception as e:
            raise ApiExceptionDocker(
                ApiCode.DEPLOY_STOP_FAILURE.value,
                ErrorMessage.HTTP_CODE.get(ApiCode.DEPLOY_STOP_FAILURE.value),
                e)

        return Response(json.dumps(HttpResponse().response(
            ApiCode.SUCCESS.value,
            ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value), result)),
                        200,
                        mimetype="application/json")
Exemplo n.º 3
0
def get_results_folder():
    io_utils = IOUtils()
    http = HttpResponse()
    archive_name = "results"
    header_key = 'Folder-Path'

    folder_path = request.headers.get(f"{header_key}")
    if not folder_path:
        return Response(json.dumps(http.failure(ApiCodeConstants.HTTP_HEADER_NOT_PROVIDED,
                                                ErrorCodes.HTTP_CODE.get(
                                                    ApiCodeConstants.HTTP_HEADER_NOT_PROVIDED) % header_key,
                                                ErrorCodes.HTTP_CODE.get(
                                                    ApiCodeConstants.HTTP_HEADER_NOT_PROVIDED) % header_key,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")

    try:
        io_utils.zip_file(archive_name, folder_path)
    except FileNotFoundError as e:
        result = "Exception({0})".format(e.__str__())
        return Response(json.dumps(http.failure(ApiCodeConstants.GET_FILE_FAILURE,
                                                ErrorCodes.HTTP_CODE.get(
                                                    ApiCodeConstants.GET_FILE_FAILURE),
                                                result,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")
    except:
        result = "Exception({0})".format(sys.exc_info()[0])
        return Response(json.dumps(http.failure(ApiCodeConstants.FOLDER_ZIP_FAILURE,
                                                ErrorCodes.HTTP_CODE.get(
                                                    ApiCodeConstants.FOLDER_ZIP_FAILURE) % folder_path,
                                                result,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")
    return flask.send_file(
        f"/tmp/{archive_name}.zip",
        mimetype='application/zip',
        as_attachment=True), 200
Exemplo n.º 4
0
def test_stop():
    io_utils = IOUtils()
    process_utils = ProcessUtils(logger)
    http = HttpResponse()
    variables = "testinfo.json"
    id = json.loads(io_utils.read_file(os.environ.get('VARS_DIR') + f"/{variables}"))["id"]

    try:
        response = get_test_info()
        pid = json.loads(response.get_data()).get('message').get('pid')
        if not isinstance(pid, str):
            if psutil.pid_exists(int(pid)):
                parent = psutil.Process()

                children = parent.children()
                for p in children:
                    p.terminate()
                _, alive = psutil.wait_procs(children, timeout=3, callback=process_utils.on_terminate)
                for p in alive:
                    p.kill()
    except:
        exception = "Exception({0})".format(sys.exc_info()[0])
        return Response(json.dumps(http.failure(ApiCodeConstants.TEST_STOP_FAILURE,
                                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.TEST_STOP_FAILURE) % id,
                                                exception,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")

    return Response(
        json.dumps(http.success(ApiCodeConstants.SUCCESS, ErrorCodes.HTTP_CODE.get(ApiCodeConstants.SUCCESS), id)), 200,
        mimetype="application/json")
Exemplo n.º 5
0
    def deploy_start(self):
        http = HttpResponse()
        kubectl_utils = KubectlUtils()
        fluentd_tag = "deploy_start"
        token = token_hex(8)
        deploy_dir = f"{EnvInit.init.get(EnvConstants.DEPLOY_PATH)}/{token}"
        file = f"{deploy_dir}/k8s-deployment.yml"

        try:
            IOUtils.create_dir(deploy_dir)
            input_data = request.data.decode('utf-8')
            IOUtils.write_to_file(file, input_data)
            status = kubectl_utils.up(f"{file}")
            self.fluentd.emit(tag=fluentd_tag, msg={"msg": status})
            if status.get('err'):
                raise Exception(status.get('err'))
        except Exception as e:
            raise ApiExceptionKubectl(
                ApiCode.DEPLOY_START_FAILURE.value,
                ErrorMessage.HTTP_CODE.get(ApiCode.DEPLOY_START_FAILURE.value),
                e)

        return Response(json.dumps(
            http.response(ApiCode.SUCCESS.value,
                          ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value),
                          token)),
                        200,
                        mimetype="application/json")
Exemplo n.º 6
0
    def env_clean_up(fluentd_utils, path=EnvInit.init.get(EnvConstants.DEPLOY_PATH), env_expire_in=1440):  # 1 day
        fluentd_tag = 'docker_env_clean_up'
        message_dumper = MessageDumper()
        active_deployments = [item.get('id') for item in DockerUtils.get_active_deployments()]

        for item in active_deployments:
            directory = f"{path}/{item}"
            if (datetime.datetime.now() - datetime.datetime.fromtimestamp(
                    os.path.getmtime(directory))) > datetime.timedelta(minutes=env_expire_in):
                result = DockerUtils.down(f"{directory}/docker-compose.yml")
                IOUtils.remove_directory(directory)
                fluentd_utils.emit(tag=fluentd_tag,
                                   msg=message_dumper.dump_message(
                                       {"action": f"{fluentd_tag}", "out": result.get('out'),
                                        "err": result.get('err')}))
Exemplo n.º 7
0
def test_start(test_id):
    test_id = test_id.strip()
    variables = "testinfo.json"
    start_py_path = str(Path(".").absolute()) + "/start.py"
    os.environ['TEMPLATE'] = "start.py"
    os.environ['VARIABLES'] = variables
    io_utils = IOUtils()
    cmd_utils = CmdUtils()
    http = HttpResponse()
    input_data = request.data.decode("UTF-8", "replace").strip()

    if not input_data:
        return Response(json.dumps(http.failure(ApiCodeConstants.EMPTY_REQUEST_BODY_PROVIDED,
                                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.EMPTY_REQUEST_BODY_PROVIDED),
                                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.EMPTY_REQUEST_BODY_PROVIDED),
                                                str(traceback.format_exc()))), 404, mimetype="application/json")
    try:
        input_data_list = io_utils.get_filtered_list_regex(input_data.split("\n"),
                                                           re.compile(r'(\s+|[^a-z]|^)rm\s+.*$'))
        input_data_list = list(map(lambda x: x.strip(), input_data_list))
        test_info_init["id"] = test_id
        io_utils.write_to_file_dict(EnvConstants.TEST_INFO_PATH, test_info_init)
    except Exception as e:
        exception = "Exception({0})".format(e.__str__())
        return Response(json.dumps(http.failure(ApiCodeConstants.TEST_START_FAILURE,
                                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.TEST_START_FAILURE) % test_id,
                                                exception,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")

    try:
        os.chmod(start_py_path, stat.S_IRWXU)
        input_data_list.insert(0, test_id)
        input_data_list.insert(0, start_py_path)
        # input_data_list.insert(0, "python")
        cmd_utils.run_cmd_detached(input_data_list)
    except Exception as e:
        result = "Exception({0})".format(e.__str__())
        return Response(json.dumps(http.failure(ApiCodeConstants.TEST_START_FAILURE,
                                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.TEST_START_FAILURE) % test_id,
                                                result,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")

    return Response(
        json.dumps(http.success(ApiCodeConstants.SUCCESS, ErrorCodes.HTTP_CODE.get(ApiCodeConstants.SUCCESS), test_id)),
        200,
        mimetype="application/json")
Exemplo n.º 8
0
    def send_request(self, app, request_object):
        resp = self.send_http_request(app, request_object=request_object)

        try:
            response = resp.json()
            response["homePageUrl"] = app.get('homePageUrl')
            response["ip_port"] = f"{app.get('ipAddr')}:{app.get('port')}"

            self.response_list.append(response)
        except:
            file_name = re.findall(
                ".*filename=(.+)", resp.headers["Content-Disposition"]
            )[0] if resp.headers.get("Content-Disposition") else ""

            self.__is_response_zip = True
            IOUtils.create_dir(
                f"{self.download_folder}/{self.__source_zip_folder}")
            IOUtils.write_to_file(
                f"{self.download_folder}/{self.__source_zip_folder}/{app.get('ipAddr')}_{app.get('port')}_{file_name}",
                resp.content)
Exemplo n.º 9
0
    def get_active_deployments():
        active_deployments = []
        env_list = [folder.lower() for folder in IOUtils.get_list_dir(f"{EnvInit.init.get(EnvConstants.DEPLOY_PATH)}")]
        for item in env_list:
            container_list = DockerUtils.ps(item).get('out').split("\n")[1:]
            for container in container_list:
                if item in container:
                    active_deployments.append(ActiveDeployment.docker_deployment(item.strip(), container_list))
                    break

        return active_deployments
Exemplo n.º 10
0
def execute_command():
    test_id = "none"
    variables = "commandinfo.json"
    os.environ['TEMPLATE'] = "start.py"
    os.environ['VARIABLES'] = variables
    io_utils = IOUtils()
    http = HttpResponse()
    input_data = request.data.decode("UTF-8", "replace").strip()

    if not input_data:
        return Response(json.dumps(http.failure(ApiCodeConstants.EMPTY_REQUEST_BODY_PROVIDED,
                                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.EMPTY_REQUEST_BODY_PROVIDED),
                                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.EMPTY_REQUEST_BODY_PROVIDED),
                                                str(traceback.format_exc()))), 404, mimetype="application/json")
    try:
        input_data_list = io_utils.get_filtered_list_regex(input_data.split("\n"),
                                                           re.compile(r'(\s+|[^a-z]|^)rm\s+.*$'))
        input_data_list = list(map(lambda x: x.strip(), input_data_list))
    except Exception as e:
        exception = "Exception({0})".format(e.__str__())
        return Response(json.dumps(http.failure(ApiCodeConstants.TEST_START_FAILURE,
                                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.TEST_START_FAILURE) % test_id,
                                                exception,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")

    try:
        testrunner = TestRunnerInMemory()
        response = testrunner.run_commands(input_data_list)
    except Exception as e:
        exception = "Exception({0})".format(e.__str__())
        return Response(json.dumps(http.failure(ApiCodeConstants.COMMAND_EXEC_FAILURE,
                                                ErrorCodes.HTTP_CODE.get(
                                                    ApiCodeConstants.COMMAND_EXEC_FAILURE),
                                                exception,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")

    return Response(
        json.dumps(
            http.success(ApiCodeConstants.SUCCESS, ErrorCodes.HTTP_CODE.get(ApiCodeConstants.SUCCESS), response)),
        200,
        mimetype="application/json")
Exemplo n.º 11
0
    def folder_clean_up(path=EnvInit.init.get(EnvConstants.DEPLOY_PATH), delete_period=60):
        deleted_folders = []
        active_deployments = [item.get('id') for item in DockerUtils.get_active_deployments()]

        full_deployments_list = map(lambda x: x.rstrip(), IOUtils.get_list_dir(f"{path}"))
        for item in full_deployments_list:
            if item not in active_deployments and (datetime.datetime.now() - datetime.datetime.fromtimestamp(
                    os.path.getmtime(f"{path}/{item}"))) > datetime.timedelta(minutes=delete_period):
                shutil.rmtree(f"{path}/{item}")
                deleted_folders.append(item)

        return deleted_folders
Exemplo n.º 12
0
def upload_file():
    io_utils = IOUtils()
    http = HttpResponse()
    header_key = 'File-Path'
    try:
        file_content = request.get_data()
        file_path = request.headers.get(f"{header_key}")
        if not file_path:
            return Response(json.dumps(http.failure(ApiCodeConstants.HTTP_HEADER_NOT_PROVIDED,
                                                    ErrorCodes.HTTP_CODE.get(
                                                        ApiCodeConstants.HTTP_HEADER_NOT_PROVIDED) % header_key,
                                                    ErrorCodes.HTTP_CODE.get(
                                                        ApiCodeConstants.HTTP_HEADER_NOT_PROVIDED) % header_key,
                                                    str(traceback.format_exc()))), 404, mimetype="application/json")
        if not file_content:
            return Response(json.dumps(http.failure(ApiCodeConstants.EMPTY_REQUEST_BODY_PROVIDED,
                                                    ErrorCodes.HTTP_CODE.get(
                                                        ApiCodeConstants.EMPTY_REQUEST_BODY_PROVIDED),
                                                    ErrorCodes.HTTP_CODE.get(
                                                        ApiCodeConstants.EMPTY_REQUEST_BODY_PROVIDED),
                                                    str(traceback.format_exc()))), 404, mimetype="application/json")
    except Exception as e:
        exception = "Exception({0})".format(e.__str__())
        return Response(json.dumps(http.failure(ApiCodeConstants.UPLOAD_TEST_CONFIG_FAILURE,
                                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.UPLOAD_TEST_CONFIG_FAILURE),
                                                exception,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")
    try:
        io_utils.write_to_file_binary(file_path, file_content)
    except Exception as e:
        exception = "Exception({0})".format(e.__str__())
        return Response(json.dumps(http.failure(ApiCodeConstants.UPLOAD_TEST_CONFIG_FAILURE,
                                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.UPLOAD_TEST_CONFIG_FAILURE),
                                                exception,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")

    return Response(
        json.dumps(http.success(ApiCodeConstants.SUCCESS, ErrorCodes.HTTP_CODE.get(ApiCodeConstants.SUCCESS),
                                ErrorCodes.HTTP_CODE.get(ApiCodeConstants.SUCCESS))), 200,
        mimetype="application/json")
Exemplo n.º 13
0
    def upload_file(self):
        http = HttpResponse()
        io_utils = IOUtils()
        header_key = 'File-Path'
        file_content = request.get_data()
        file_path = request.headers.get(f"{header_key}")

        if not file_path:
            raise ApiExceptionDocker(
                ApiCode.HTTP_HEADER_NOT_PROVIDED.value,
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.HTTP_HEADER_NOT_PROVIDED.value) % header_key,
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.HTTP_HEADER_NOT_PROVIDED.value) % header_key)
        if not file_content:
            raise ApiExceptionDocker(
                ApiCode.EMPTY_REQUEST_BODY_PROVIDED.value,
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.EMPTY_REQUEST_BODY_PROVIDED.value),
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.EMPTY_REQUEST_BODY_PROVIDED.value))

        try:
            io_utils.write_to_file_binary(file_path, file_content)
        except Exception as e:
            raise ApiExceptionDocker(
                ApiCode.UPLOAD_FILE_FAILURE.value,
                ErrorMessage.HTTP_CODE.get(ApiCode.UPLOAD_FILE_FAILURE.value),
                e)

        return Response(json.dumps(
            http.response(ApiCode.SUCCESS.value,
                          ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value),
                          ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value))),
                        200,
                        mimetype="application/json")
Exemplo n.º 14
0
def get_file():
    io_utils = IOUtils()
    http = HttpResponse()
    header_key = 'File-Path'

    file_path = request.headers.get(f"{header_key}")
    if not file_path:
        return Response(json.dumps(http.failure(ApiCodeConstants.HTTP_HEADER_NOT_PROVIDED,
                                                ErrorCodes.HTTP_CODE.get(
                                                    ApiCodeConstants.HTTP_HEADER_NOT_PROVIDED) % header_key,
                                                ErrorCodes.HTTP_CODE.get(
                                                    ApiCodeConstants.HTTP_HEADER_NOT_PROVIDED) % header_key,
                                                str(traceback.format_exc()))), 404, mimetype="application/json")

    try:
        response = io_utils.read_file_byte_array(file_path), 200
    except Exception as e:
        exception = "Exception({0})".format(e.__str__())
        response = Response(json.dumps(http.failure(ApiCodeConstants.GET_FILE_FAILURE,
                                                    ErrorCodes.HTTP_CODE.get(
                                                        ApiCodeConstants.GET_FILE_FAILURE),
                                                    exception,
                                                    str(traceback.format_exc()))), 404, mimetype="application/json")
    return response
Exemplo n.º 15
0
    def get_file(self):
        header_key = 'File-Path'

        file_path = request.headers.get(f"{header_key}")
        if not file_path:
            raise ApiExceptionKubectl(
                ApiCode.HTTP_HEADER_NOT_PROVIDED.value,
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.HTTP_HEADER_NOT_PROVIDED.value) % header_key,
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.HTTP_HEADER_NOT_PROVIDED.value) % header_key)
        try:
            file_content = IOUtils.read_file(file_path)
        except Exception as e:
            raise ApiExceptionKubectl(
                ApiCode.GET_FILE_FAILURE.value,
                ErrorMessage.HTTP_CODE.get(ApiCode.GET_FILE_FAILURE.value), e)
        return Response(file_content, 200, mimetype="text/plain")
Exemplo n.º 16
0
    def receive_prepared_deployment_and_unpack(self):
        token = token_hex(8)
        io_utils = IOUtils()
        deployment_id = request.headers.get("Deployment-Id").lower(
        ) if request.headers.get("Deployment-Id") else token
        deploy_dir = f"{EnvInit.init.get(EnvConstants.DEPLOY_PATH)}/{deployment_id}"
        file_path = f"{deploy_dir}/archive.zip"
        file_content = request.get_data()
        # send here the complete env. The deployment template can be overridden at deploy start
        if not file_content:
            raise ApiExceptionDocker(
                ApiCode.EMPTY_REQUEST_BODY_PROVIDED.value,
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.EMPTY_REQUEST_BODY_PROVIDED.value),
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.EMPTY_REQUEST_BODY_PROVIDED.value))

        try:
            io_utils.create_dir(deploy_dir)
            io_utils.write_to_file_binary(file_path, file_content)
        except Exception as e:
            raise ApiExceptionDocker(
                ApiCode.UPLOAD_FILE_FAILURE.value,
                ErrorMessage.HTTP_CODE.get(ApiCode.UPLOAD_FILE_FAILURE.value),
                e)
        try:
            shutil.unpack_archive(file_path, deploy_dir)
            io_utils.remove_file(file_path)
        except Exception as e:
            raise ApiExceptionDocker(
                ApiCode.FOLDER_UNZIP_FAILURE.value,
                ErrorMessage.HTTP_CODE.get(ApiCode.FOLDER_UNZIP_FAILURE.value),
                e)

        return Response(json.dumps(HttpResponse().response(
            ApiCode.SUCCESS.value,
            ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value), deployment_id)),
                        200,
                        mimetype="application/json")
    def set_metadata_for_deployment_from_file(self, deployment, file):
        """ labels is dict. """
        metadata = {}
        if self.metadata.get(
                deployment) is not None:  # jump over, already in memory
            return False

        try:
            metadata = json.loads(IOUtils.read_file(file=file))
        except:
            pass  # don't care. metadata is optional

        if not isinstance(metadata, dict):
            return False
        if len(self.metadata
               ) < self.METADATA_SPACE_MAX_SIZE and deployment != "":
            self.metadata[deployment] = metadata
            return True

        return False
class TestRunner:

    def __init__(self):
        self.__cmd_utils = CmdUtils()
        self.__io_utils = IOUtils()

    def run_commands(self, json_file, test_id, commands):
        start_total = datetime.datetime.now()

        status_finished = "finished"
        status_in_progress = "in progress"
        command_dict = test_info_init

        command_dict['id'] = test_id
        command_dict['pid'] = os.getpid()
        input_data_dict = dict.fromkeys(commands, {"status": "scheduled", "details": {}})
        command_dict["started"] = "true"
        command_dict["commands"] = input_data_dict
        command_dict["startedat"] = str(datetime.datetime.now())

        details = {}
        for command in commands:
            start = datetime.datetime.now()
            command_dict['commands'][command.strip()] = {"status": "scheduled", "details": {}}
            command_dict['commands'][command.strip()]['status'] = status_in_progress
            command_dict['commands'][command.strip()]['startedat'] = str(start)
            self.__io_utils.write_to_file_dict(json_file, command_dict)
            try:
                if platform.system() == "Windows":
                    details[command.strip()] = self.__cmd_utils.run_cmd_shell_true(command.split())
                else:
                    details[command.strip()] = self.__cmd_utils.run_cmd_shell_true([command.strip()])
            except Exception as e:
                details[command.strip()] = "Exception({0})".format(e.__str__())
            command_dict['commands'][command.strip()]['status'] = status_finished
            end = datetime.datetime.now()
            command_dict['commands'][command.strip()]['finishedat'] = str(end)
            command_dict['commands'][command.strip()]['duration'] = round((end - start).total_seconds())
            command_dict['commands'][command.strip()]['details'] = details[command.strip()]
            self.__io_utils.write_to_file_dict(json_file, command_dict)

        command_dict['finished'] = "true"
        command_dict['started'] = "false"
        end_total = datetime.datetime.now()
        command_dict['finishedat'] = str(end_total)
        command_dict['duration'] = round((end_total - start_total).total_seconds())
        self.__io_utils.write_to_file_dict(json_file, command_dict)

        return command_dict
Exemplo n.º 19
0
    def start_deployment(self):
        docker_utils = DockerUtils()
        token = token_hex(8)
        deployment_id = request.headers.get("Deployment-Id").lower(
        ) if request.headers.get("Deployment-Id") else token
        deploy_dir = f"{EnvInit.init.get(EnvConstants.DEPLOY_PATH)}/{deployment_id}"
        file = f"{deploy_dir}/docker-compose.yml"
        header_key = 'Eureka-Server'
        eureka_server_header = request.headers.get(f"{header_key}")
        config_env_vars = EnvStartupSingleton.get_instance(
        ).get_config_env_vars()
        input_data = request.data.decode('UTF-8').strip()

        status = CmdUtils.run_cmd_shell_false(["docker", "ps"])
        if "Cannot connect to the Docker daemon".lower() in status.get(
                'err').lower():
            raise ApiExceptionDocker(
                ApiCode.DOCKER_DAEMON_NOT_RUNNING.value,
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.DOCKER_DAEMON_NOT_RUNNING.value),
                status.get('err'))

        active_deployments = docker_utils.get_active_deployments()
        if len(active_deployments) >= EnvInit.init.get(
                EnvConstants.MAX_DEPLOYMENTS):
            raise ApiExceptionDocker(
                ApiCode.MAX_DEPLOYMENTS_REACHED.value,
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.MAX_DEPLOYMENTS_REACHED.value) %
                str(EnvInit.init.get(EnvConstants.MAX_DEPLOYMENTS)),
                active_deployments)
        try:
            template_file_name = f"deployment_{deployment_id}.yml"
            template_file_path = f"{EnvInit.init.get(EnvConstants.TEMPLATES_DIR)}/{template_file_name}"
            app.logger.debug({
                "msg": {
                    "file": template_file_path,
                    "file_content": f"{input_data}"
                }
            })
            IOUtils.write_to_file(template_file_path, input_data)

            IOUtils.create_dir(deploy_dir)
            EnvironmentSingleton.get_instance().set_env_var(
                EnvConstants.TEMPLATE, template_file_name)
            env_vars = EnvironmentSingleton.get_instance(
            ).get_env_and_virtual_env()
            render = Render(env_vars.get(EnvConstants.TEMPLATE),
                            env_vars.get(EnvConstants.VARIABLES))
            if config_env_vars.get(
                    EnvConstants.EUREKA_SERVER) and config_env_vars.get(
                        EnvConstants.APP_IP_PORT):
                # if {{app_ip_port}} and {{eureka_server}} then register that instance too
                if '{{app_ip_port}}' in input_data and '{{eureka_server}}' in input_data:
                    eureka_server = config_env_vars.get(
                        EnvConstants.EUREKA_SERVER)
                    # header value overwrite the eureka server
                    if eureka_server_header:
                        eureka_server = eureka_server_header
                    input_data = render.get_jinja2env().get_template(
                        env_vars.get(EnvConstants.TEMPLATE)).render({
                            "deployment_id":
                            f"{deployment_id}",
                            "eureka_server":
                            eureka_server,
                            "app_ip_port":
                            config_env_vars.get(
                                EnvConstants.APP_IP_PORT).split("/")[0]
                        })
            os.remove(template_file_path) if os.path.exists(
                template_file_path) else None
            app.logger.debug(
                {"msg": {
                    "file": file,
                    "file_content": f"{input_data}"
                }})
            IOUtils.write_to_file(file, input_data) if input_data else None
            CmdUtils.run_cmd_detached(
                rf'''docker-compose -f {file} pull && docker-compose -f {file} up -d'''
            )
        except Exception as e:
            app.logger.debug({"msg": docker_utils.down(file)})
            raise ApiExceptionDocker(
                ApiCode.DEPLOY_START_FAILURE.value,
                ErrorMessage.HTTP_CODE.get(ApiCode.DEPLOY_START_FAILURE.value),
                e)

        DeploymentMetadataSingleton.get_instance() \
            .delete_metadata_for_inactive_deployments(DockerUtils.get_active_deployments())
        metadata = DeploymentReader.get_metadata_for_deployment(
            IOUtils.read_file(file=file))
        IOUtils.write_to_file_dict(f"{deploy_dir}/metadata.json", metadata)
        DeploymentMetadataSingleton.get_instance().set_metadata_for_deployment(
            deployment_id, metadata)

        return Response(json.dumps(HttpResponse().response(
            ApiCode.SUCCESS.value,
            ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value), deployment_id)),
                        200,
                        mimetype="application/json")
Exemplo n.º 20
0
    def start_deployment_with_template(self, template, variables):
        http = HttpResponse()
        kubectl_utils = KubectlUtils()
        fluentd_tag = "deploy_start"
        try:
            input_json = request.get_json(force=True)
            for key, value in input_json.items():
                if key not in EnvironmentSingleton.get_instance().get_env():
                    EnvironmentSingleton.get_instance().set_env_var(
                        str(key), str(value))
        except Exception as e:
            app.logger.debug(f"Exception: {e.__str__()}")

        EnvironmentSingleton.get_instance().set_env_var(
            EnvConstants.TEMPLATE, template.strip())
        EnvironmentSingleton.get_instance().set_env_var(
            EnvConstants.VARIABLES, variables.strip())
        app.logger.debug({
            "msg": {
                "template_file":
                EnvironmentSingleton.get_instance().get_env_and_virtual_env().
                get(EnvConstants.TEMPLATE)
            }
        })
        app.logger.debug({
            "msg": {
                "variables_file":
                EnvironmentSingleton.get_instance().get_env_and_virtual_env().
                get(EnvConstants.VARIABLES)
            }
        })
        token = token_hex(8)
        deploy_dir = f"{EnvInit.init.get(EnvConstants.DEPLOY_PATH)}/{token}"
        file = f"{deploy_dir}/k8s-deployment.yml"

        try:
            r = Render(
                EnvironmentSingleton.get_instance().get_env_and_virtual_env(
                ).get(EnvConstants.TEMPLATE),
                EnvironmentSingleton.get_instance().get_env_and_virtual_env().
                get(EnvConstants.VARIABLES))
            IOUtils.create_dir(deploy_dir)
            IOUtils.write_to_file(file)
            IOUtils.write_to_file(file, r.rend_template())
            status = kubectl_utils.up(f"{file}")
            self.fluentd.emit(tag=fluentd_tag, msg={"msg": status})
            if status.get('err'):
                raise Exception(status.get('error'))
            result = str(token)
        except Exception as e:
            raise ApiExceptionKubectl(
                ApiCode.DEPLOY_START_FAILURE.value,
                ErrorMessage.HTTP_CODE.get(ApiCode.DEPLOY_START_FAILURE.value),
                e)

        return Response(json.dumps(
            http.response(ApiCode.SUCCESS.value,
                          ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value),
                          result)),
                        200,
                        mimetype="application/json")
Exemplo n.º 21
0
from about import properties
from rest.api.constants.env_constants import EnvConstants
from rest.api.definitions import test_info_init
from rest.api.logginghelpers.message_dumper import MessageDumper
from rest.api.routes import app
from rest.api.routes import fluentd_utils
from rest.utils.io_utils import IOUtils

if __name__ == "__main__":
    port = properties["port"]
    host = '0.0.0.0'
    fluentd_tag = "startup"
    variables = "testinfo.json"

    message_dumper = MessageDumper()
    io_utils = IOUtils()

    if os.environ.get('PORT'):
        port = int(os.environ.get("PORT"))  # override port  if set from env

    io_utils.create_dir(Path(EnvConstants.TEMPLATES_PATH))
    io_utils.create_dir(Path(EnvConstants.VARIABLES_PATH))

    file = EnvConstants.VARIABLES_PATH + "/" + variables

    try:
        test_info_init["pid"] = os.getpid()
        io_utils.write_to_file_dict(Path(file), test_info_init)
    except Exception as e:
        raise e
Exemplo n.º 22
0
class TestRunnerParallel:
    def __init__(self):
        self.__cmd_utils = CmdUtils()
        self.__io_utils = IOUtils()

    def run_command(self, manager_dict, dictionary, command):
        print("Input json is: " + json.dumps(dictionary) + "\n")
        status_finished = "finished"
        status_in_progress = "in progress"
        dictionary['commands'][command.strip()]['status'] = status_in_progress
        start = datetime.datetime.now()
        dictionary['commands'][command.strip()]['startedat'] = str(start)
        if platform.system() == "Windows":
            details = self.__cmd_utils.run_cmd_shell_true(command.split())
        else:
            details = self.__cmd_utils.run_cmd_shell_true([command.strip()])
        dictionary['commands'][command.strip()]['status'] = status_finished
        end = datetime.datetime.now()
        dictionary['commands'][command.strip()]['finishedat'] = str(end)
        dictionary['commands'][command.strip()]['duration'] = round(
            (end - start).total_seconds())
        dictionary['commands'][command.strip()]['details'] = details

        manager_dict[command.strip()] = {}
        manager_dict[command.strip()] = dictionary['commands'][command.strip()]

    def run_commands(self, json_file, commands):
        with Manager() as manager:
            try:
                manager_dict = manager.dict()
                command_dict = self.__io_utils.read_dict_from_file(json_file)
                command_dict['start_pid'] = os.getpid()
                start_total = datetime.datetime.now()

                procs = [
                    Process(target=self.run_command,
                            args=(
                                manager_dict,
                                command_dict,
                                command.strip(),
                            )) for command in commands
                ]

                # start processes
                for p in procs:
                    p.start()
                    p.join()

                command_dict['commands'] = dict(manager_dict)

                self.__io_utils.write_to_file_dict(json_file, command_dict)
                command_dict['finished'] = "true"
                command_dict['started'] = "false"
                end_total = datetime.datetime.now()
                command_dict['finishedat'] = str(end_total)
                command_dict['duration'] = round(
                    (end_total - start_total).total_seconds())
                self.__io_utils.write_to_file_dict(json_file, command_dict)

            except Exception as e:
                raise e
Exemplo n.º 23
0
 def __init__(self):
     self.__cmd_utils = CmdUtils()
     self.__io_utils = IOUtils()
Exemplo n.º 24
0
DockerView.register(app=app)
KubectlView.register(app=app)

app.register_error_handler(ApiExceptionDocker, DockerView.handle_api_error)
app.register_error_handler(ApiExceptionKubectl, KubectlView.handle_api_error)

if __name__ == "__main__":
    cli = sys.modules['flask.cli']
    cli.show_server_banner = lambda *x: None

    fluentd_tag = "startup"
    host = '0.0.0.0'
    port = EnvStartupSingleton.get_instance().get_config_env_vars().get(EnvConstants.PORT)
    message_dumper = MessageDumper()
    io_utils = IOUtils()

    if EnvStartupSingleton.get_instance().get_config_env_vars().get(EnvConstants.EUREKA_SERVER):
        Eureka(EnvStartupSingleton.get_instance().get_config_env_vars().get(EnvConstants.EUREKA_SERVER)).register_app(
            EnvStartupSingleton.get_instance().get_config_env_vars().get(EnvConstants.APP_IP_PORT),
            EnvStartupSingleton.get_instance().get_config_env_vars().get(EnvConstants.APP_APPEND_LABEL))

    io_utils.create_dirs([Path(EnvInit.init.get(EnvConstants.DEPLOY_PATH)),
                          Path(EnvInit.init.get(EnvConstants.TEMPLATES_DIR)),
                          Path(EnvInit.init.get(EnvConstants.VARS_DIR))])

    DockerEnvExpireScheduler(fluentd_utils=DockerView.fluentd,
                             poll_interval=EnvStartupSingleton.get_instance().get_config_env_vars().get(
                                 EnvConstants.SCHEDULER_POLL_INTERVAL),  # seconds
                             env_expire_in=EnvStartupSingleton.get_instance().get_config_env_vars().get(
                                 EnvConstants.ENV_EXPIRE_IN)).start()  # minutes
Exemplo n.º 25
0
def agents_request(text):
    path = text.strip()
    eureka = Eureka(
        EnvStartupSingleton.get_instance().get_config_env_vars().get(
            EnvConstants.EUREKA_SERVER))
    ip_addr_port_header_key = 'IpAddr-Port'  # target specific agent
    home_page_url_header_key = 'HomePageUrl'  # target specific agent
    application = "agent"
    try:
        input_data = request.get_data()
    except:
        input_data = ""

    try:
        request_object = {
            "uri": path.lstrip("/"),
            "method": request.method,
            "headers": request.headers,
            "data": input_data
        }
        app.logger.debug({"msg": f"{request_object}"})
        agent_apps = eureka.get_type_eureka_apps(application)

        if request.headers.get(f"{ip_addr_port_header_key}"):  # not mandatory
            ip_port_list = request.headers.get(
                f"{ip_addr_port_header_key}").split(",")
            agent_apps = [
                app for app in agent_apps
                if f"{app.get('ipAddr')}:{app.get('port')}" in ip_port_list
            ]

        if request.headers.get(f"{home_page_url_header_key}"):  # not mandatory
            home_page_url_list = request.headers.get(
                f"{home_page_url_header_key}").split(",")
            agent_apps = [
                app for app in agent_apps
                if f"{app.get('homePageUrl')}" in home_page_url_list
            ]

        th_utils = ThreadUtils(apps=agent_apps, headers={})
        th_utils.spawn_threads_send_request(request_object)

    except Exception as e:
        raise ApiException(
            ApiCode.DISCOVERY_ERROR.value,
            ErrorMessage.HTTP_CODE.get(ApiCode.DISCOVERY_ERROR.value), e)

    threads_response = th_utils.get_threads_response()

    if th_utils.is_response_zip():
        folder_path = f"{th_utils.download_folder}/{th_utils.get_source_zip_folder()}"
        archive_path = f"{th_utils.download_folder}/{th_utils.get_source_zip_folder()}"
        try:
            IOUtils.zip_file(archive_path, folder_path)
        except Exception as e:
            raise ApiException(
                ApiCode.FOLDER_ZIP_FAILURE.value,
                ErrorMessage.HTTP_CODE.get(ApiCode.FOLDER_ZIP_FAILURE.value) %
                folder_path, e)
        IOUtils.delete_file(folder_path)

        return flask.send_file(f"{archive_path}.zip",
                               mimetype='application/zip',
                               as_attachment=True), 200

    return Response(json.dumps(HttpResponse().response(
        ApiCode.SUCCESS.value,
        ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value), threads_response)),
                    200,
                    mimetype="application/json")
Exemplo n.º 26
0
#!/usr/bin/env python3
import json
import os
import sys

from rest.utils.io_utils import IOUtils
from rest.utils.testrunner import TestRunner

if __name__ == '__main__':
    io_utils = IOUtils()
    WORKSPACE = os.environ.get('WORKSPACE') if os.environ.get('WORKSPACE') else "tmp"
    VARIABLES_PATH = WORKSPACE + "/variables"
    COMMAND_LOGGER_PATH = WORKSPACE + "/commandlogger.txt"
    file_path = VARIABLES_PATH + "/testinfo.json"

    io_utils.append_to_file(COMMAND_LOGGER_PATH, " ".join(sys.argv))

    min_args = 3
    if len(sys.argv) < min_args:
        raise Exception(
            "Error: Expecting at least {} args. Got {}, args={}".format(min_args, len(sys.argv), sys.argv))

    test_id = sys.argv[1]
    commands_list = sys.argv[2:]

    test_runner = TestRunner()
    dictionary = test_runner.run_commands(file_path, test_id, commands_list)
    dictionary = io_utils.read_dict_from_file(file_path)

    print(json.dumps(dictionary) + "\n")
Exemplo n.º 27
0
    def start_deployment_with_templates(self, template, variables):
        http = HttpResponse()
        docker_utils = DockerUtils()
        token = token_hex(8)
        deployment_id = request.headers.get("Deployment-Id").lower(
        ) if request.headers.get("Deployment-Id") else token
        deploy_dir = f"{EnvInit.init.get(EnvConstants.DEPLOY_PATH)}/{deployment_id}"
        file = f"{deploy_dir}/docker-compose.yml"

        try:
            input_json = request.get_json(force=True)
            for key, value in input_json.items():
                if key not in EnvironmentSingleton.get_instance().get_env():
                    EnvironmentSingleton.get_instance().set_env_var(
                        str(key), str(value))
        except Exception as e:
            app.logger.debug(
                f"Could not parse the input from the request as JSON: {e.__str__()}"
            )

        EnvironmentSingleton.get_instance().set_env_var(
            EnvConstants.TEMPLATE, template.strip())
        EnvironmentSingleton.get_instance().set_env_var(
            EnvConstants.VARIABLES, variables.strip())
        env_vars = EnvironmentSingleton.get_instance().get_env_and_virtual_env(
        )
        app.logger.debug(
            {"msg": {
                "template_file": env_vars.get(EnvConstants.TEMPLATE)
            }})
        app.logger.debug(
            {"msg": {
                "variables_file": env_vars.get(EnvConstants.VARIABLES)
            }})

        status = CmdUtils.run_cmd_shell_false(["docker", "ps"])
        if "Cannot connect to the Docker daemon".lower() in status.get(
                'err').lower():
            raise ApiExceptionDocker(
                ApiCode.DOCKER_DAEMON_NOT_RUNNING.value,
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.DOCKER_DAEMON_NOT_RUNNING.value),
                status.get('err'))

        active_deployments = docker_utils.get_active_deployments()
        if len(active_deployments) >= EnvInit.init.get(
                EnvConstants.MAX_DEPLOYMENTS):
            raise ApiExceptionDocker(
                ApiCode.MAX_DEPLOYMENTS_REACHED.value,
                ErrorMessage.HTTP_CODE.get(
                    ApiCode.MAX_DEPLOYMENTS_REACHED.value) %
                str(EnvInit.init.get(EnvConstants.MAX_DEPLOYMENTS)),
                active_deployments)
        try:
            r = Render(env_vars.get(EnvConstants.TEMPLATE),
                       env_vars.get(EnvConstants.VARIABLES))
            IOUtils.create_dir(deploy_dir)
            IOUtils.write_to_file(file, r.rend_template())
            CmdUtils.run_cmd_detached(
                rf'''docker-compose -f {file} pull && docker-compose -f {file} up -d'''
            )
        except Exception as e:
            raise ApiExceptionDocker(
                ApiCode.DEPLOY_START_FAILURE.value,
                ErrorMessage.HTTP_CODE.get(ApiCode.DEPLOY_START_FAILURE.value),
                e)

        DeploymentMetadataSingleton.get_instance() \
            .delete_metadata_for_inactive_deployments(DockerUtils.get_active_deployments())
        metadata = DeploymentReader.get_metadata_for_deployment(
            IOUtils.read_file(file=file))
        IOUtils.write_to_file_dict(f"{deploy_dir}/metadata.json", metadata)
        DeploymentMetadataSingleton.get_instance().set_metadata_for_deployment(
            deployment_id, metadata)

        return Response(json.dumps(
            http.response(ApiCode.SUCCESS.value,
                          ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value),
                          deployment_id)),
                        200,
                        mimetype="application/json")
Exemplo n.º 28
0
import platform

import psutil

from rest.utils.io_utils import IOUtils

properties = {
    "name": "Estuary-Discovery",
    "version": "4.2.4",
    "description":
    "Discover Estuary Apps registered with Eureka. Execute commands and tests with Estuary Agents.",
    "author": "Catalin Dinuta",
    "platforms": ["Linux", "Mac", "Windows"],
    "license": "Apache-2.0"
}

about_system = {
    "system": platform.system(),
    "platform": platform.platform(),
    "release": platform.release(),
    "version": platform.version(),
    "architecture": platform.architecture()[0],
    "machine": platform.machine(),
    "layer": "Docker" if IOUtils.does_file_exist("/.dockerenv") else "Machine",
    "hostname": platform.uname().node,
    "cpu": platform.processor(),
    "ram": str(round(psutil.virtual_memory().total / (1024.0**3))) + " GB",
    "python": platform.python_version()
}