import datetime import hashlib import json import time import os import docker from logless import get_logger, logged, logged_block import runnow import uio MAX_ECS_WAIT = 12 * 60 * 60 # max 12 hours wait docker_client = docker.from_env() logging = get_logger("dock-r") def build(dockerfile_path, tag_as, addl_args=None): """ Build an image. 'tag_as' can be a string or list of strings """ folder_path = os.path.dirname(dockerfile_path) addl_args = addl_args or "" tag_as = _to_list(tag_as) if tag_as: tags = " ".join([f"-t {t}" for t in tag_as]) cmd = f"docker build {addl_args} {tags} {folder_path} -f {dockerfile_path}" else: cmd = f"docker build {addl_args} {folder_path} -f {dockerfile_path}" runnow.run(cmd)
import configparser as _configparser import functools as _functools import json as _json import os as _os from pathlib import Path as _Path import platform import shutil as _shutil import tempfile as _tempfile import fire as _fire import logless as _logs import runnow as _jobs _LOGGER = _logs.get_logger("uio") def _warn_failed_import(library_name, install_hint, ex): _LOGGER.debug(ex) _LOGGER.warning( f"Could not load '{library_name}' library. Some functionality may be disabled. " f"Please confirm '{library_name}' is installed or install via '{install_hint}'." ) # try: # import pandas as pd # except Exception as ex: # pd = None # _warn_failed_import("pandas", "pip install pandas") try:
import os import platform import subprocess import sys import time from logless import logged, get_logger, flush_buffers logging = get_logger("runnow") def _grep(full_text, match_with, insensitive=True, fn=any): lines = full_text.splitlines() if isinstance(match_with, str): match_with = [match_with] if insensitive: return "\n".join([ l for l in lines if fn([m.lower() in l.lower() for m in match_with]) ]) else: return "\n".join( [l for l in lines if fn([m in l for m in match_with])]) @logged("running command: {'(hidden)' if hide else cmd}") def run( cmd: str, working_dir=None, echo=True, raise_error=True,
"""tapdance.docker - Module for containerization and image build functions.""" from typing import List, Optional, Tuple # import dock_r from logless import get_logger import runnow import uio import yaml from tapdance.config import SINGER_PLUGINS_INDEX logging = get_logger("tapdance") try: import dock_r except Exception as ex: dock_r = None # type: ignore logging.warning(f"Docker libraries were not able to be loaded ({ex}).") BASE_DOCKER_REPO = "dataopstk/tapdance" # Deprecated in favor of partial dockerization: # def rerun_dockerized( # tap_alias: str, # target_alias: str = None, # *, # tap_exe: str, # target_exe: str, # args: List[str] = None, # ) -> bool:
""" slalom.dataops.pandasutils module """ import os from logless import ( get_logger, logged, logged_block, ) import uio USE_SCRATCH_DIR = False logging = get_logger("slalom.dataops.sparkutils") try: import pandas as pd except Exception as ex: pd = None logging.warning( f"Could not load pandas library. Try 'pip install pandas'. {ex}") def _raise_if_missing_pandas(as_warning=False, ex=None): if not pd: msg = f"Could not load pandas library. Try 'pip install pandas'. {ex or ''}" if as_warning: logging.warning(msg) else: raise RuntimeError(msg)
from pathlib import Path from typing import Dict, List import fire from logless import get_logger, logged, logged_block import runnow from tqdm import tqdm import uio code_file = os.path.realpath(__file__) repo_dir = os.path.dirname(os.path.dirname(os.path.dirname(code_file))) sys.path.append(os.path.join(repo_dir, "src")) sys.path.append(os.path.dirname(code_file)) DEBUG = False logging = get_logger("slalom.dataops.infra", debug=DEBUG) SPECIAL_CASE_WORDS = [ "AWS", "ECR", "ECS", "IAM", "VPC", "DBT", "EC2", "RDS", "MySQL" ] def _proper(str: str, title_case=True, special_case_words=None): """ Return the same string in proper case, respected override rules for acronyms and special-cased words. """ special_case_words = special_case_words or SPECIAL_CASE_WORDS word_lookup = {w.lower(): w for w in special_case_words} if title_case: str = str.title()
import platform import sys import fire from logless import get_logger, logged, logged_block import runnow import uio if os.name == "nt": import ctypes else: ctypes = None CACHED_INSTALL_LIST: str = None DEBUG = False logging = get_logger("slalom.dataops.env", debug=DEBUG) CHOCO_INSTALL_CMD = """ @"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" && SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" """.strip() WINDOWS_INSTALL_LIST = { "chocolatey": CHOCO_INSTALL_CMD, "chocolateygui": None, "sudo": None, "git": 'choco install -y git.install --params "/GitOnlyOnPath /SChannel /NoAutoCrlf /WindowsTerminal"', "docker": "docker-desktop", "python": "python3", "terraform": None, "vscode": None, }
#!/bin/python3 import json from pathlib import Path import os from typing import Dict, List from logless import get_logger, logged_block import runnow import uio from templates import DOCS_HEADER, DOCS_FOOTER, CATALOG_TEMPLATE logging = get_logger("infra-catalog.build") SPECIAL_CASE_WORDS = [ "AWS", "ECR", "ECS", "IAM", "VPC", "DBT", "EC2", "RDS", "MySQL", "ML", "MLOps", "SFTP", ]