def test_git_repo_errors(repo):
with repo.as_cwd():
# If the path doesn't exist, we still remain
# in the original cwd
with pytest.raises(FileNotFoundError):
git.git_repo_root("/foo/bar/baz")
assert os.getcwd() == repo
# If the path isn't part of a git repo
# we still remain in the original cwd
with pytest.raises(subprocess.CalledProcessError):
git.git_repo_root("/tmp")
assert os.getcwd() == repo
def execute_build_image(python_version, base_image_version):
check.str_param(python_version, 'python_version')
check.str_param(base_image_version, 'base_image_version')
os.chdir(os.path.dirname(os.path.abspath(__file__)))
builder_image = local_unit_snapshot_builder_image(python_version,
base_image_version)
docker_args = {'BASE_IMAGE': unit_base_image(python_version)}
execute_docker_build(image=builder_image, docker_args=docker_args)
root = git_repo_root()
target_reqs_path = os.path.join(
root,
'.buildkite',
'images',
'unit',
'snapshot-reqs-{python_version}.txt'.format(
python_version=python_version),
)
# e.g.
# docker run snapshot-builder pip freeze --exclude-editable
check.invariant(
os.system(
'docker run {builder_image} pip freeze --exclude-editable > {reqs}'
.format(builder_image=builder_image, reqs=target_reqs_path)) == 0)
def helm(helm_repo, ver, dry_run):
"""Publish the Dagster Helm chart
See: https://medium.com/containerum/how-to-make-and-share-your-own-helm-package-50ae40f6c221
for more info on this process
"""
helm_path = os.path.join(git_repo_root(), "python_modules", "libraries", "dagster-k8s", "helm")
check.invariant(
get_git_repo_branch(helm_repo) == "gh-pages", "helm repo must be on gh-pages branch"
)
cmd = [
"helm",
"package",
"dagster",
"-d",
helm_repo,
"--app-version",
ver,
"--version",
ver,
]
click.echo(click.style("Running command: " + " ".join(cmd) + "\n", fg="green"))
click.echo(subprocess.check_output(cmd, cwd=helm_path))
cmd = ["helm", "repo", "index", ".", "--url", "https://dagster-io.github.io/helm/"]
click.echo(click.style("Running command: " + " ".join(cmd) + "\n", fg="green"))
click.echo(subprocess.check_output(cmd, cwd=helm_repo))
# Commit and push Helm updates for this Dagster release
msg = "Helm release for Dagster release {}".format(ver)
git_commit_updates(helm_repo, msg)
git_push(cwd=helm_repo, dry_run=dry_run)
def __init__(self):
# Removed hard-coded list in favor of automatic scan of libraries folder
# List of subdirectories in directory: https://stackoverflow.com/a/973488
self.library_module_names = set(
next(
os.walk(
os.path.join(git_repo_root(), 'python_modules',
'libraries')))[1])
# Construct list of core modules
self.core_modules = []
for name in CORE_MODULE_NAMES:
additional_steps = ['./build_js.sh'] if name == 'dagit' else None
self.core_modules.append(
DagsterModule(name,
is_library=False,
additional_steps=additional_steps))
# Construct list of library modules, some of which may not be published
self.library_modules = [
DagsterModule(name, is_library=True)
for name in self.library_module_names
]
self.all_modules = self.core_modules + self.library_modules
self.all_publishable_modules = [
m for m in self.all_modules if m.should_publish
]
def check_api_docs(library_name):
api_docs_root = os.path.join(git_repo_root(), 'docs', 'sections', 'api',
'apidocs', 'libraries')
underscore_name = library_name.replace('-', '_')
if not os.path.exists(
os.path.join(api_docs_root, '%s.rst' % underscore_name)):
print('API docs not found for library %s!' % library_name)
sys.exit(1)
def bad_core_module():
try:
mod = os.path.join(git_repo_root(), "python_modules",
"bad_core_module")
os.mkdir(mod)
yield
finally:
shutil.rmtree(mod, ignore_errors=True)
def check_for_cruft(self, autoclean):
'''We need to ensure directories don't have spurious files in them before publishing to
PyPI.
Args:
autoclean (bool): Should the script automatically clean up (remove) extraneous files?
Raises:
Exception: Raised when we aren't able to resolve cruft issues
'''
found_cruft = list(
itertools.chain.from_iterable(module.find_cruft()
for module in self.all_modules))
if found_cruft:
if autoclean:
wipeout = '!'
else:
wipeout = input(
'Found potentially crufty directories:\n'
' {found_cruft}\n'
'***We strongly recommend releasing from a fresh git clone!***\n'
'Automatically remove these directories and continue? (N/!)'
.format(found_cruft='\n '.join(found_cruft)))
if wipeout == '!':
for cruft_dir in found_cruft:
subprocess.check_output(['rm', '-rfv', cruft_dir])
else:
raise Exception(
'Bailing: Cowardly refusing to publish with potentially crufty directories '
'present! We strongly recommend releasing from a fresh git clone.'
)
found_pyc_files = []
for root, _, files in os.walk(git_repo_root()):
for file_ in files:
if file_.endswith('.pyc'):
found_pyc_files.append(os.path.join(root, file_))
if found_pyc_files:
if autoclean:
wipeout = '!'
else:
wipeout = input(
'Found {n_files} .pyc files.\n'
'We strongly recommend releasing from a fresh git clone!\n'
'Automatically remove these files and continue? (N/!)'.
format(n_files=len(found_pyc_files)))
if wipeout == '!':
for file_ in found_pyc_files:
os.unlink(file_)
else:
raise Exception(
'Bailing: Cowardly refusing to publish with .pyc files present! '
'We strongly recommend releasing from a fresh git clone.')
def k8s_example_cm(cwd):
example_project_dir = os.path.join(git_repo_root(), 'examples',
'deploy_k8s', 'example_project')
try:
print('Syncing {} to build dir {}...'.format(example_project_dir, cwd))
shutil.copytree(example_project_dir,
os.path.join(cwd, 'example_project'))
yield
finally:
shutil.rmtree(os.path.join(cwd, 'example_project'))
def get_core_module_directories():
'''List core module directories (not including libraries) under python_modules.
Returns:
List(os.DirEntry): List of core module directories
'''
core_module_root_dir = os.path.join(git_repo_root(), 'python_modules')
module_directories = [
dir_ for dir_ in os.scandir(core_module_root_dir)
if dir_.is_dir() and not dir_.name.startswith('.')
]
return module_directories
def get_library_module_directories():
'''List library module directories under python_modules/libraries.
Returns:
List(os.DirEntry): List of core module directories
'''
library_module_root_dir = os.path.join(git_repo_root(), 'python_modules',
'libraries')
library_directories = [
dir_.name for dir_ in os.scandir(library_module_root_dir)
if dir_.is_dir() and not dir_.name.startswith('.')
]
return library_directories
def validate_library_readmes():
dirs = get_library_module_directories()
for library_name in dirs:
library_root = os.path.join(git_repo_root(), 'python_modules', 'libraries')
readme_file = os.path.join(library_root, library_name, 'README.md')
check_readme_exists(readme_file, library_name)
check_readme_contents(readme_file, library_name)
check_api_docs(library_name)
print(':white_check_mark: All README.md contents exist and content validated!')
print(':white_check_mark: All API docs exist!')
def helm(chart_path, helm_repo, ver, dry_run):
"""Publish the Dagster Helm chart
See: https://medium.com/containerum/how-to-make-and-share-your-own-helm-package-50ae40f6c221
for more info on this process
"""
helm_path = os.path.join(git_repo_root(), "helm")
check.invariant(
get_git_repo_branch(helm_repo) == "gh-pages", "helm repo must be on gh-pages branch"
)
cmd = [
"helm",
"package",
chart_path,
"--dependency-update",
"--destination",
helm_repo,
"--app-version",
ver,
"--version",
ver,
]
click.echo(click.style("Running command: " + " ".join(cmd) + "\n", fg="green"))
click.echo(subprocess.check_output(cmd, cwd=helm_path))
cmd = [
"helm",
"repo",
"index",
".",
"--merge",
"index.yaml",
"--url",
"https://dagster-io.github.io/helm/",
]
click.echo(click.style("Running command: " + " ".join(cmd) + "\n", fg="green"))
click.echo(subprocess.check_output(cmd, cwd=helm_repo))
# Commit and push Helm updates for this Dagster release
chart_name = os.path.basename(os.path.normpath(chart_path))
msg = f"Helm release for {chart_name} release {ver}"
git_commit_updates(helm_repo, msg)
git_push(cwd=helm_repo, dry_run=dry_run)
def execute_image_build(python_version, image_version):
root = git_repo_root()
# always set cwd to the directory where the file lives
os.chdir(os.path.dirname(os.path.abspath(__file__)))
scala_modules_dir = os.path.join(root, 'scala_modules')
try:
rsync_args = ((
"rsync -av --exclude='*target*' --exclude='*.idea*' --exclude='*.class' "
"{scala_modules_dir} .").format(
scala_modules_dir=scala_modules_dir).split(' '))
check_output(rsync_args)
execute_build_integration_image(python_version, image_version)
finally:
shutil.rmtree('./scala_modules')
def helm(helm_repo, ver, dry_run):
'''Publish the Dagster Helm chart
See: https://medium.com/containerum/how-to-make-and-share-your-own-helm-package-50ae40f6c221
for more info on this process
'''
helm_path = os.path.join(git_repo_root(), 'python_modules', 'libraries',
'dagster-k8s', 'helm')
check.invariant(
get_git_repo_branch(helm_repo) == 'gh-pages',
'helm repo must be on gh-pages branch')
cmd = [
'helm',
'package',
'dagster',
'-d',
helm_repo,
'--app-version',
ver,
'--version',
ver,
]
click.echo(
click.style('Running command: ' + ' '.join(cmd) + '\n', fg='green'))
click.echo(subprocess.check_output(cmd, cwd=helm_path))
cmd = [
'helm', 'repo', 'index', '.', '--url',
'https://dagster-io.github.io/helm/'
]
click.echo(
click.style('Running command: ' + ' '.join(cmd) + '\n', fg='green'))
click.echo(subprocess.check_output(cmd, cwd=helm_repo))
# Commit and push Helm updates for this Dagster release
msg = 'Helm release for Dagster release {}'.format(ver)
git_commit_updates(helm_repo, msg)
git_push(cwd=helm_repo, dry_run=dry_run)
def buildkite_integration_cm(cwd):
'''For the buildkite integration base image, we first copy over scala_modules into the image
build directory.
'''
scala_modules_dir = os.path.join(git_repo_root(), 'scala_modules')
try:
cmd = [
'rsync',
'-av',
"--exclude='*target*'",
"--exclude='*.idea*'",
"--exclude='*.class'",
scala_modules_dir,
'.',
]
print('Syncing scala_modules to build dir...')
print(cmd)
subprocess.call(cmd, cwd=cwd)
yield
finally:
shutil.rmtree(os.path.join(cwd, 'scala_modules'))
def copy_directories(paths, cwd):
check.invariant(os.path.exists(cwd), 'Image directory does not exist')
build_cache_dir = os.path.join(cwd, 'build_cache')
paths_to_copy = []
for path in paths:
src_path = os.path.join(git_repo_root(), path)
check.invariant(os.path.exists(src_path),
'Path for copying to image build does not exist')
_, dest_name = os.path.split(path)
dest_path = os.path.join(build_cache_dir, dest_name)
paths_to_copy.append((src_path, dest_path))
try:
for src_path, dest_path in paths_to_copy:
print('Syncing {} to build dir {}...'.format(src_path, dest_path))
shutil.copytree(src_path, dest_path)
yield
finally:
shutil.rmtree(build_cache_dir)