import datetime
import hashlib
import json
import time
import os
import docker
from logless import get_logger, logged, logged_block
import runnow
import uio
MAX_ECS_WAIT = 12 * 60 * 60 # max 12 hours wait
docker_client = docker.from_env()
logging = get_logger("dock-r")
def build(dockerfile_path, tag_as, addl_args=None):
""" Build an image. 'tag_as' can be a string or list of strings """
folder_path = os.path.dirname(dockerfile_path)
addl_args = addl_args or ""
tag_as = _to_list(tag_as)
if tag_as:
tags = " ".join([f"-t {t}" for t in tag_as])
cmd = f"docker build {addl_args} {tags} {folder_path} -f {dockerfile_path}"
else:
cmd = f"docker build {addl_args} {folder_path} -f {dockerfile_path}"
runnow.run(cmd)
import configparser as _configparser
import functools as _functools
import json as _json
import os as _os
from pathlib import Path as _Path
import platform
import shutil as _shutil
import tempfile as _tempfile
import fire as _fire
import logless as _logs
import runnow as _jobs
_LOGGER = _logs.get_logger("uio")
def _warn_failed_import(library_name, install_hint, ex):
_LOGGER.debug(ex)
_LOGGER.warning(
f"Could not load '{library_name}' library. Some functionality may be disabled. "
f"Please confirm '{library_name}' is installed or install via '{install_hint}'."
)
# try:
# import pandas as pd
# except Exception as ex:
# pd = None
# _warn_failed_import("pandas", "pip install pandas")
try:
import os
import platform
import subprocess
import sys
import time
from logless import logged, get_logger, flush_buffers
logging = get_logger("runnow")
def _grep(full_text, match_with, insensitive=True, fn=any):
lines = full_text.splitlines()
if isinstance(match_with, str):
match_with = [match_with]
if insensitive:
return "\n".join([
l for l in lines
if fn([m.lower() in l.lower() for m in match_with])
])
else:
return "\n".join(
[l for l in lines if fn([m in l for m in match_with])])
@logged("running command: {'(hidden)' if hide else cmd}")
def run(
cmd: str,
working_dir=None,
echo=True,
raise_error=True,
"""tapdance.docker - Module for containerization and image build functions."""
from typing import List, Optional, Tuple
# import dock_r
from logless import get_logger
import runnow
import uio
import yaml
from tapdance.config import SINGER_PLUGINS_INDEX
logging = get_logger("tapdance")
try:
import dock_r
except Exception as ex:
dock_r = None # type: ignore
logging.warning(f"Docker libraries were not able to be loaded ({ex}).")
BASE_DOCKER_REPO = "dataopstk/tapdance"
# Deprecated in favor of partial dockerization:
# def rerun_dockerized(
# tap_alias: str,
# target_alias: str = None,
# *,
# tap_exe: str,
# target_exe: str,
# args: List[str] = None,
# ) -> bool:
""" slalom.dataops.pandasutils module """
import os
from logless import (
get_logger,
logged,
logged_block,
)
import uio
USE_SCRATCH_DIR = False
logging = get_logger("slalom.dataops.sparkutils")
try:
import pandas as pd
except Exception as ex:
pd = None
logging.warning(
f"Could not load pandas library. Try 'pip install pandas'. {ex}")
def _raise_if_missing_pandas(as_warning=False, ex=None):
if not pd:
msg = f"Could not load pandas library. Try 'pip install pandas'. {ex or ''}"
if as_warning:
logging.warning(msg)
else:
raise RuntimeError(msg)
from pathlib import Path
from typing import Dict, List
import fire
from logless import get_logger, logged, logged_block
import runnow
from tqdm import tqdm
import uio
code_file = os.path.realpath(__file__)
repo_dir = os.path.dirname(os.path.dirname(os.path.dirname(code_file)))
sys.path.append(os.path.join(repo_dir, "src"))
sys.path.append(os.path.dirname(code_file))
DEBUG = False
logging = get_logger("slalom.dataops.infra", debug=DEBUG)
SPECIAL_CASE_WORDS = [
"AWS", "ECR", "ECS", "IAM", "VPC", "DBT", "EC2", "RDS", "MySQL"
]
def _proper(str: str, title_case=True, special_case_words=None):
"""
Return the same string in proper case, respected override rules for
acronyms and special-cased words.
"""
special_case_words = special_case_words or SPECIAL_CASE_WORDS
word_lookup = {w.lower(): w for w in special_case_words}
if title_case:
str = str.title()
import platform
import sys
import fire
from logless import get_logger, logged, logged_block
import runnow
import uio
if os.name == "nt":
import ctypes
else:
ctypes = None
CACHED_INSTALL_LIST: str = None
DEBUG = False
logging = get_logger("slalom.dataops.env", debug=DEBUG)
CHOCO_INSTALL_CMD = """
@"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" && SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
""".strip()
WINDOWS_INSTALL_LIST = {
"chocolatey": CHOCO_INSTALL_CMD,
"chocolateygui": None,
"sudo": None,
"git":
'choco install -y git.install --params "/GitOnlyOnPath /SChannel /NoAutoCrlf /WindowsTerminal"',
"docker": "docker-desktop",
"python": "python3",
"terraform": None,
"vscode": None,
}
#!/bin/python3
import json
from pathlib import Path
import os
from typing import Dict, List
from logless import get_logger, logged_block
import runnow
import uio
from templates import DOCS_HEADER, DOCS_FOOTER, CATALOG_TEMPLATE
logging = get_logger("infra-catalog.build")
SPECIAL_CASE_WORDS = [
"AWS",
"ECR",
"ECS",
"IAM",
"VPC",
"DBT",
"EC2",
"RDS",
"MySQL",
"ML",
"MLOps",
"SFTP",
]
