def _setup_check_background_services(logger): use_docker_services = False logger(bold("Checking for existing background services...")) healthcheck_passed, healthcheck_results = DMServices.services_healthcheck(threading.Event(), check_once=True) first_result = next(iter(healthcheck_results.values())) # Used to ensure all results are identical if not healthcheck_passed and not all(map(lambda x: x is first_result, healthcheck_results.values())): services_up = [x[0].title() for x in list(filter(lambda y: y[1] is True, healthcheck_results.items()))] services_down = [x.title() for x in set(healthcheck_results.keys()) - set([x.lower() for x in services_up])] logger( red( "* You have some services running locally (Up: {}. Down: {}).".format( ", ".join(services_up), ", ".join(services_down) ) ) ) logger( red( "* You can either manage all background services yourself or allow DMRunner to manage them for you " "- but not a bit of both." ) ) return EXITCODE_BAD_SERVICES, False elif healthcheck_passed: logger(green("* Discovered full suite of existing background services.")) else: logger(green("* None found. Background services will be managed for you.")) use_docker_services = True return 0, use_docker_services
def _setup_bootstrap_repositories(logger: Callable, config: dict, settings: dict): exitcode = 0 logger(bold("Bootstrapping repositories ...")) try: nested_repositories = group_by_key(settings["repositories"], "run-order", include_missing=True) for repo_name in itertools.chain.from_iterable(nested_repositories): if "bootstrap" in settings["repositories"][repo_name]: app_info = get_app_info(repo_name, config, settings, {}) logger(green("* Starting bootstrap of") + " " + app_info["name"] + " ...", log_name="setup") bootstrap_command = settings["repositories"][repo_name]["bootstrap"] exitcode = DMProcess(app=app_info, logger=logger, app_command=bootstrap_command).wait() if exitcode: logger( red("* Bootstrap failed for ") + app_info["name"] + red(" with exit code {}").format(exitcode) ) exitcode = EXITCODE_BOOTSTRAP_FAILED break else: logger(green("* Bootstrap completed for") + " " + app_info["name"] + " ", log_name="setup") except KeyboardInterrupt: exitcode = EXITCODE_SETUP_ABORT return exitcode
def _setup_check_node_version(logger): exitcode = 0 logger(bold("Checking Node version ...")) try: node_version = LooseVersion( subprocess.check_output(["node", "-v"], universal_newlines=True).strip()) except Exception: logger( red("* Unable to verify Node version. Please check that you have Node installed and in your path." )) exitcode = EXITCODE_NODE_NOT_IN_PATH else: try: assert node_version == SPECIFIC_NODE_VERSION logger( green( "* You are using a suitable version of Node ({}).".format( node_version))) except AssertionError: logger( red("* You have Node {} installed; you should use {}".format( node_version, SPECIFIC_NODE_VERSION))) exitcode = EXITCODE_NODE_VERSION_NOT_SUITABLE return exitcode
def _setup_check_docker_available(logger): logger(bold("Verifying Docker is available ...")) try: docker_client = docker.from_env() except requests.exceptions.ConnectionError: logger( red( "* You do not appear to have Docker installed and/or running. Please install Docker and " "ensure it is running in the background." ) ) return EXITCODE_DOCKER_NOT_AVAILABLE except docker.errors.APIError as e: logger( red( "* An error occurred connecting to the Docker API. Please make sure it has finished starting up and " "is running properly: {}".format(e) ) ) return EXITCODE_DOCKER_NOT_AVAILABLE except Exception as e: logger( red( "* Unknown error connecting to Docker. Please make sure it has finished starting up and is running " "properly: {}".format(e) ) ) return EXITCODE_DOCKER_NOT_AVAILABLE try: docker_version = LooseVersion(docker_client.version()["Version"]) assert docker_version >= MINIMUM_DOCKER_VERSION except AssertionError: logger( yellow( "* WARNING - You are running Docker version {}. If you are on macOS, you need " "Docker for Mac version {} or higher.".format(docker_version, MINIMUM_DOCKER_VERSION) ) ) else: logger( green("* Docker is available and a suitable version appears to be installed ({}).".format(docker_version)) ) return 0
def _setup_buckets(logger: Callable, config: dict, settings: dict): exitcode = 0 if not os.getenv("DM_S3_ENDPOINT_PORT"): logger("* Skipping s3 setup as envvar DM_S3_ENDPOINT_PORT is empty") return exitcode logger(bold("Bootstrapping localstack s3 buckets...")) s3_region = "eu-west-1" s3_endpoint_url = f"http://localhost:{os.environ['DM_S3_ENDPOINT_PORT']}" s3 = boto3.resource("s3", region_name=s3_region, endpoint_url=s3_endpoint_url) try: s3.create_bucket( Bucket="digitalmarketplace-dev-uploads", CreateBucketConfiguration={"LocationConstraint": s3_region}) except s3.meta.client.exceptions.BucketAlreadyExists: pass except Exception as e: logger( red(f"* Could not create bucket digitalmarketplace-dev-uploads: {e}" )) exitcode = 1 return exitcode
def _setup_download_repos(logger, config, settings): exitcode = 0 logger(bold("Checking authentication with GitHub ...")) try: retcode = subprocess.call(["ssh", "-T", "*****@*****.**"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if retcode != 1: logger(red(*"Authentication failed - check that your local SSH keys have been uploaded to GitHub.")) return EXITCODE_GIT_AUTH_FAILED else: logger(green("* Authentication to Github succeeded.")) code_directory = os.path.realpath(os.path.join(".", config["code"]["directory"])) logger(bold(f"Ensuring you have local copies of Digital Marketplace code in {code_directory} ...")) os.makedirs(code_directory, exist_ok=True) nested_repositories = group_by_key(settings["repositories"], "run-order", include_missing=True) for repo_name in itertools.chain.from_iterable(nested_repositories): repo_path = os.path.join(code_directory, repo_name) if os.path.isdir(repo_path): continue logger(green("* Downloading") + " " + settings["repositories"][repo_name].get("name", repo_name) + " ") process = subprocess.run( ["git", "clone", os.path.join(settings["base-git-url"], repo_name)], cwd=code_directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, ) if process.returncode != 0: logger(red(process.stdout)) return process.returncode if not exitcode: logger(green("* Your Digital Marketplace code is all present and accounted for.")) except KeyboardInterrupt: exitcode = EXITCODE_SETUP_ABORT return exitcode
def _setup_check_yarn_version(logger): exitcode = 0 logger(bold("Checking Yarn version ...")) try: yarn_version = subprocess.check_output(["yarn", "-v"], universal_newlines=True).strip() except Exception: logger(red("* Unable to verify Yarn version. Please check that you have Node installed and in your path.")) exitcode = EXITCODE_YARN_NOT_IN_PATH else: try: assert LooseVersion(yarn_version) >= LooseVersion(SPECIFIC_YARN_VERSION) logger(green("* You are using a suitable version of Yarn ({}).".format(yarn_version))) except AssertionError: logger(red("* You have Yarn {} installed; you should use >={}".format(yarn_version, SPECIFIC_YARN_VERSION))) exitcode = EXITCODE_YARN_VERSION_NOT_SUITABLE return exitcode
def _setup_check_git_available(logger): logger(bold("Verifying Git is available ...")) try: subprocess.check_call(["git", "--version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) logger(green("* Git is available. Obviously.")) except Exception: # noqa logger(red("* You do not appear to have Git installed and/or in your path. Please install it.")) return EXITCODE_GIT_NOT_AVAILABLE return 0
def _setup_check_node_version(logger): exitcode = 0 logger(bold("Checking Node version ...")) try: node_version = LooseVersion( subprocess.check_output(["node", "-v"], universal_newlines=True).strip()) node_major_version = node_version.version[1] node_release_schedule = requests.get( "https://raw.githubusercontent.com/nodejs/Release/main/schedule.json" ).json() codename = node_release_schedule.get(f"v{node_major_version}").get( "codename") except Exception as e: logger( red("* Unable to verify Node version. Please check that you have Node installed and in your path." )) logger(red(e)) exitcode = EXITCODE_NODE_NOT_IN_PATH else: try: assert codename.lower() == SPECIFIC_NODE_VERSION.replace( "lts/", "") logger( green( "* You are using a suitable version of Node ({}).".format( node_version))) except AssertionError: logger( red("* You have Node {} installed; you should use {}".format( node_version, SPECIFIC_NODE_VERSION))) exitcode = EXITCODE_NODE_VERSION_NOT_SUITABLE return exitcode
def _setup_config_modifications(logger, config, config_path): exitcode, interim_config = load_config(config_path) if not exitcode: default_code_directory = os.path.realpath( interim_config["code"]["directory"]) logger( "If you are an existing developer, enter the directory where your current Digital Marketplace code is " "checked out.") logger( "If you do not have code currently checked out, enter the directory you would like " "code to be downloaded to.") logger("[current value: {}]:".format(yellow(default_code_directory)), end="") requested_code_directory = os.path.realpath( input(" ").strip() or default_code_directory) os.makedirs(requested_code_directory, exist_ok=True) logger("Code directory set to " + yellow(requested_code_directory)) interim_config["code"]["directory"] = requested_code_directory current_decryption = interim_config["credentials"]["sops"] logger("") logger( "Do you want to decrypt credentials automatically (requires security clearance)?" ) logger("Y/N [current value: {}]:".format( yellow("Y" if current_decryption is True else "N")), end="") cleaned_input = input(" ").strip().lower() decrypt_credentials = current_decryption if not cleaned_input else True if cleaned_input == "y" else False logger("Credentials " + (green("will") if decrypt_credentials else red("will not")) + " be decrypted automatically.") interim_config["credentials"]["sops"] = decrypt_credentials save_config(interim_config, config_path) # Patch the runner config with our new/modified configuration. config.update(interim_config) return 0
def setup_and_check_requirements(logger: Callable, config: dict, config_path: str, settings: Dict, command: str): """This runs some basic checks to ensure that the User has everything required for DMRunner to function correctly, eg their own config file, Docker (and possibly Nix in the future), docker images, checked-out code. """ exitcode = 0 use_docker_services = False only_check_services = True if command == RUNNER_COMMAND_RUN else False only_setup_data = True if command == RUNNER_COMMAND_DATA else False if only_check_services: exitcode, interim_config = load_config(config_path, must_exist=True) config.update(interim_config) logger(bold("Starting service check ...")) if not exitcode: exitcode, use_docker_services = _setup_check_background_services( logger) elif only_setup_data: exitcode, interim_config = load_config(config_path, must_exist=True) config.update(interim_config) if only_setup_data: logger(bold("Starting data setup ...")) logger( red("WARNING: ") + "This will delete " + bold("ALL") + " of your existing database and elasticsearch" " data, then re-populate it.") if get_yes_no_input(logger, "Are you sure you want to proceed?", default="n") != "y": exitcode = EXITCODE_SETUP_ABORT else: exitcode, use_docker_services = ( _setup_check_background_services(logger) if not exitcode else (exitcode, False)) if not use_docker_services: logger( bold( "Cannot run a data setup if you are managing your own backing services. Sorry!" )) exitcode = EXITCODE_SETUP_ABORT exitcode = exitcode or _setup_check_postgres_data_if_required( logger, settings, use_docker_services, prompt_delete_existing=True) with (background_services( logger, docker_compose_folder=settings["docker-compose-path"], clean=True) if use_docker_services and not exitcode else blank_context()): exitcode = exitcode or _setup_indices( logger, config, settings) exitcode = exitcode or _setup_buckets( logger, config, settings) else: logger(bold("Starting setup ...")) try: exitcode = _setup_config_modifications(logger, config, config_path) exitcode = exitcode or _setup_logging_directory(config) exitcode = exitcode or _setup_check_git_available(logger) exitcode = exitcode or _setup_check_docker_available(logger) exitcode = exitcode or _setup_check_node_version(logger) exitcode = exitcode or _setup_download_repos( logger, config, settings) exitcode, use_docker_services = ( _setup_check_background_services(logger) if not exitcode else (exitcode, False)) exitcode = exitcode or _setup_check_postgres_data_if_required( logger, settings, use_docker_services) with (background_services( logger, docker_compose_folder=settings["docker-compose-path"]) if use_docker_services and not exitcode else blank_context()): exitcode = exitcode or _setup_bootstrap_repositories( logger, config, settings) exitcode = exitcode or _setup_indices(logger, config, settings) exitcode = exitcode or _setup_buckets(logger, config, settings) except BaseException: exitcode = EXITCODE_SETUP_ABORT if exitcode: if only_check_services: if exitcode == EXITCODE_CONFIG_NO_EXIST: logger( red("Configuration file not found. Run `invoke setup` to generate." )) else: logger(red( "Startup failed with exitcode: {}".format(exitcode))) else: logger(red("Aborting setup ...")) elif not only_check_services: logger(bold("Setup completed successfully.")) return exitcode, use_docker_services, config
def _setup_indices(logger: Callable, config: dict, settings: dict): exitcode = 0 manager = multiprocessing.Manager() logger(bold("Bootstrapping search indices ...")) dependencies = [] for dependency in settings["index"]["dependencies"]: dependency_app_info = get_app_info(dependency, config, settings, manager.dict()) dependencies.append( (DMProcess(app=dependency_app_info, logger=nologger, app_command=APP_COMMAND_RESTART), dependency_app_info)) time.sleep(10) for index in settings["index"]["indices"]: index_name = index["keyword"]["index"] app_info = get_app_info(settings["index"]["repository"], config, settings, manager.dict()) try: assert requests.get(settings["index"]["test"].format( index=index_name)).status_code == 200 except Exception: index_command = "{command} {keyword} {positional}".format( command=settings["index"]["command"], keyword=" ".join([ "--{k}={v}".format(k=k, v=v) for k, v in index["keyword"].items() ]), positional=" ".join(index["positional"]), ) logger("* Creating index '{}' ...".format(index_name)) exitcode = DMProcess(app=app_info, logger=logger, app_command=index_command).wait() if exitcode: logger( red("* Something went wrong when creating the '{}' index: exitcode " "{}".format(index_name, exitcode))) exitcode = EXITCODE_BOOTSTRAP_FAILED break else: logger( green("* Index '{}' created successfully.".format( index_name))) else: logger(green("* Index '{}' already exists.".format(index_name))) for dependency, dependency_app_info in dependencies: try: p = psutil.Process(dependency_app_info["process"]) for child in p.children(recursive=True): child.kill() p.kill() except Exception as e: logger(str(e)) exitcode = EXITCODE_BOOTSTRAP_FAILED return exitcode
def _setup_check_postgres_data_if_required(logger, settings, use_docker_services, prompt_delete_existing=False): exitcode = 0 logger( bold( "Checking that you have data available to populate your Postgres database." )) if use_docker_services: data_path = os.path.join(os.path.realpath("."), settings["sql-data-path"]) os.makedirs(data_path, exist_ok=True) if prompt_delete_existing: prompt = "Do you need want to delete any existing Postgres data dumps in order to download a newer one?" if get_yes_no_input(logger, prompt, default="n") == "y": sql_files = glob.glob(os.path.join( data_path, "*.sql")) + glob.glob( os.path.join(data_path, "*.sql.gz")) for sql_file in sql_files: logger(f"Removing file `{sql_file}` ...") os.remove(sql_file) def data_available(): return glob.glob(os.path.join(data_path, "*.sql")) or glob.glob( os.path.join(data_path, "*.sql.gz")) while not data_available(): logger( red("* No data is available.") + " When you press ENTER, a link will be opened for you. Please " "download the file to `{data_path}` then press ENTER " "again.".format(data_path=data_path), end="", ) input(" ") webbrowser.open(settings["data-dump-url"]) logger("* ") logger( "* Press ENTER, after saving the file to `{data_path}`, to continue, or type anything to " "abort.".format(data_path=data_path), end="", ) user_input = input(" ").strip() if user_input: raise KeyboardInterrupt if not exitcode: gzip_sql_files = glob.glob(os.path.join(data_path, "*.sql.gz")) for gzip_sql_file in gzip_sql_files: target_sql_file = gzip_sql_file[:-3] # Remove '.gz' suffix if not os.path.isfile(target_sql_file): logger("* Extracting {} ...".format(gzip_sql_file)) try: with open(target_sql_file, "wb") as outfile, gzip.open( gzip_sql_file, "rb") as infile: before_read = -1 while before_read < infile.tell(): before_read = infile.tell() # Read and write in chunks to avoid macs failing on writes > 2GB outfile.write(infile.read(2**30)) outfile.flush() except KeyboardInterrupt: os.remove(target_sql_file) exitcode = EXITCODE_SETUP_ABORT else: os.remove(gzip_sql_file) logger("* Extracted.") if not exitcode: logger( green( "* You have data available to populate your Postgres database." )) return exitcode