def restore(identity):
config = load_config()
s3 = boto3.resource('s3')
bucket = s3.Bucket(config['s3']['bucket'])
prefix = compute_top_prefix(config, identity)
i = get_sstables_to_download(config, bucket, prefix)
sstables = list(compute_cf_dirs(LocalPath('mirrored'), i))
create_metadata_directories(sstables)
instructions = get_download_instructions(identity, bucket, prefix,
sstables)
# It is assumed we'll be disk-bound, so I've chosen a typical disk queue depth.
with futures.ThreadPoolExecutor(max_workers=32) as executor:
fs = [
executor.submit(download_sstable_to_path, config, *i)
for i in instructions
]
for f in futures.as_completed(fs):
try:
# Raise any exceptions from the executor
f.result()
except:
# If there is an exception, cancel the inflight futures
# No point throwing good work after bad
for inflight in fs:
inflight.cancel()
raise
with MovingTemporaryDirectory(LocalPath('data')) as d:
copy_back(LocalPath('mirrored'), d.path)
d.finalize()
def get_file_from_cache_or_s3(bucket, fn, dst, cache=True):
hostname, _, _ = socket.gethostname().partition('.')
look_dirs = get_property('s3_cache', hostname)
if cache:
cache_dir = look_dirs[-1:]
else:
cache_dir = []
for d in look_dirs:
path = (LocalPath(d) / bucket / fn)
if path.exists():
mkdir_p(dst)
path.copy(dst)
break
else:
b = s3.lookup(bucket)
key = b.lookup(fn)
if key is None:
raise NoFileOnS3(
"Key missing from bucket {bucket}: {key_name}".format(
bucket=bucket, key_name=fn))
mkdir_p(dst)
with open(dst, 'w') as out_f:
out_f.write(key.read())
for d in cache_dir:
path = (LocalPath(d) / bucket / fn)
if not path.exists():
mkdir_p(path)
LocalPath(dst).copy(path)
def setup_temp_dir():
tempdir = LocalPath(mkdtemp(dir='.', prefix='test_run.'))
(tempdir / 'logs').mkdir()
confdir = (tempdir / 'conf')
confdir.mkdir()
LocalPath('../../conf/dynomite.pem').symlink(confdir / 'dynomite.pem')
return tempdir
class Config:
rel_ini_path = LocalPath('dct.ini')
def __init__(self):
config = ConfigParser()
config.read(self.rel_ini_path)
dct = config['dct']
if not all(dct.values()):
log.error('please complete the settings in %s', self.rel_ini_path)
exit(1)
self.package = dct['package']
self.devpi_user = dct['devpi_user']
self.devpi_index = dct['devpi_index']
@classmethod
def necessary(cls, func):
def _necessary(*args, **kwargs):
if not cls.rel_ini_path.exists():
log.error("no %s in '%s'" % (cls.rel_ini_path, local.cwd))
exit(1)
return func(*args, **kwargs)
return _necessary
def do_backup():
logging.basicConfig(stream=sys.stderr)
logger.setLevel(logging.DEBUG)
config = load_config()
data_dir = LocalPath(config.get('data_dir', '/var/lib/cassandra'))
state_dir = config.get('state_dir')
state_dir = (
LocalPath(state_dir) if state_dir is not None
else data_dir / 'mirroring'
)
state_dir.mkdir()
locs = Locations(data_dir, data_dir / 'data', state_dir, state_dir / 'links')
backup_all_sstables(config, locs)
mark_obsoleted(locs)
cleanup_obsoleted(locs, 0)
def _render_files(self, version):
"""only render the files for the trigger."""
for src in LocalPath('tpl').list():
self.__render(src,
PACKAGE=config.package,
VERSION=version,
DEVPI_USER=config.devpi_user,
DEVPI_INDEX=config.devpi_index,
TIMESTAMP=datetime.now().ctime())
def test_local_glob_path(tmpdir):
p = tmpdir.mkdir("a*b?c")
p2 = tmpdir.mkdir("aanythingbxc")
p2.join("something.txt").write("content")
p.join("hello.txt").write("content")
p.join("other.txt").write("content")
pp = LocalPath(str(p))
assert len(pp // "*.txt") == 2
def warn_shebangs(template_file: LocalPath):
for line in template_file.read().splitlines():
if '#!/' in line:
warn(
f"Be warned! Template includes '{line}', "
"which includes what pyratemp considers a template comment, "
"to be omitted from the rendered output. "
"\nTo fix, replace '#!' with '#@!!@!'."
)
def save_config() -> None:
"""Persist VaRA config to a yaml file."""
if vara_cfg()["config_file"].value is None:
config_file = ".varats.yaml"
else:
config_file = str(vara_cfg()["config_file"])
vara_cfg()["config_file"] = path.abspath(config_file)
create_missing_folders()
vara_cfg().store(LocalPath(config_file))
def create(self, package, devpi_user='', devpi_index=''):
parent = local.cwd / ('devpi-cloud-test-' + package)
if parent.exists():
log.error("%s already exists - aborting", parent)
exit(1)
log.info("create new cloud test in %s", parent)
try:
blueprint_path = LocalPath(__file__).dirname / 'blueprint'
blueprint_path.copy(parent)
with local.cwd(parent):
self.__render(Config.rel_ini_path,
PACKAGE=package,
DEVPI_USER=devpi_user,
DEVPI_INDEX=devpi_index)
self._init_repository(parent)
except Exception:
log.exception("creation failed")
parent.delete()
exit(1)
def get_file_remote(key, session=None):
cache = LocalPath(CACHE)
cache.mkdir()
local_path = cache / (key + '.tar.gz')
if not local_path.exists():
if not session:
with SshMachine(FS) as rem:
path = rem.path(DIR) / (key + '.tar.gz')
assert path.exists()
plumbum.path.utils.copy(path, local_path)
assert local_path.exists()
else:
rem = session
path = rem.path(DIR) / (key + '.tar.gz')
assert path.exists()
plumbum.path.utils.copy(path, local_path)
assert local_path.exists()
fileobj = open(str(local_path))
return tarfile.open(fileobj=fileobj)
def s3_files_for(r_from, r_to, projects=[], versions=[]):
for project, version in iter_versions(projects, versions):
key = 'results:test-methods:{project}:{version}:dev'.format(
project=project, version=version
)
test_methods = r_to.lrange(key, 0, -1)
print "{project}:{version} has {tms} methods".format(
project=project, version=version,
tms=len(test_methods)
)
for test_method in test_methods:
c_name, _, m_name = test_method.partition('::')
is_class_empty = []
for tool in ["codecover", "cobertura", "jmockit"]:
class_key = mk_key('test-classes-cvg', [tool, project, version])
class_r = json.loads(r_from.hget(class_key, c_name))
is_class_empty.append(is_empty(tool, class_r))
if all(is_class_empty):
continue
is_method_nonempty = []
for tool in ["codecover", "cobertura", "jmockit"]:
method_key = mk_key('test-methods-run-cvg', [tool, project, version])
try:
method_r = json.loads(r_from.hget(method_key, test_method))
if not is_empty(tool, method_r):
key = ':'.join([tool, project, str(version), test_method])
DIR = '/scratch/darioush/files'
p = LocalPath(DIR) / '{key}.tar.gz'.format(key=key)
assert p.exists()
is_method_nonempty.append(True)
with open(str(p)) as f:
s3_up = put_into_s3('cvg-files', [tool, project, version, 'dev'], test_method, f)
else:
is_method_nonempty.append(False)
except TypeError:
print "--> Missing result {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version)
except AssertionError:
print "--> Missing file {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version)
if any(is_method_nonempty):
mut_key = mk_key('test-methods-run-cvg', ['major', project, version])
try:
mut_r = json.loads(r_from.hget(mut_key, test_method))
if not is_empty('major', mut_r):
key = ':'.join(['major', project, str(version), test_method])
DIR = '/scratch/darioush/files'
p = LocalPath(DIR) / '{key}.tar.gz'.format(key=key)
assert p.exists()
with open(str(p)) as f:
s3_up = put_into_s3('cvg-files', ['major', project, version, 'dev'], test_method, f)
except TypeError:
tool = 'major'
print "--> Missing result {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version)
except AssertionError:
print "--> Missing file {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version)
def download_to_path(marker_path, s3_object, destination, provider_args,
encryption_context):
context = serialize_context(encryption_context)
logger.debug("Invoking keypipe with context %s", context)
keypipe_partial = functools.partial(
keypipe.unseal,
provider_args,
context,
)
gof3r_cmd = s3gof3r['get', '--no-md5', '-b', s3_object.bucket_name, '-k',
s3_object.key, ]
prefix = marker_path.name + '.'
with TemporaryDirectory(prefix=prefix, dir=destination.up()) as d:
temp_destination = LocalPath(d)
start_time = time.time()
(gof3r_cmd / keypipe_partial | lz4['-d']
| tar['-C', temp_destination, '-x']) & FG
finish_time = time.time()
elapsed = finish_time - start_time
max_mtime = max(f.stat().st_mtime_ns for f in temp_destination)
size = sum(f.stat().st_size for f in temp_destination)
speed = size / elapsed / 1024
logger.info(
'Downloaded from %s. %s bytes in %.3f seconds: %s KB/s',
s3_object.key,
size,
elapsed,
"{:,.2f}".format(speed),
)
timed_touch(marker_path, max_mtime)
"""Invariant: marker path exists before files are moved into the
final data_dir.
This prevents files that we just downloaded from being re-uploaded.
Note that, unlike when we upload, the presence of this marker file does
not inhibit downloads.
"""
for i in temp_destination:
i.link(destination / i.name)
def do_restore():
parser = argparse.ArgumentParser(
description=
'Restore a Cassandra backup into the current working directory.')
parser.add_argument(
'source_identity',
help=
'The identity (typically UUID) of the node whose backup should be restored'
)
args = parser.parse_args()
logging.basicConfig(stream=sys.stderr)
logger.setLevel(logging.INFO)
if LocalPath('data').exists():
logger.info(
'Skipping restoration because a data directory already exists')
return
restore(args.source_identity)
def main(project, version):
tools = ['cobertura', 'codecover', 'jmockit']
key = "%s:%d" % (project, version)
r = StrictRedis.from_url(get_property('redis_url'))
rkey = ':'.join(['results', 'test-methods-agree-cvg-nonempty', key])
tms = r.lrange(rkey, 0, -1)
for tool in tools:
missings = [
tm for (fn, tm) in [('/scratch/darioush/files/%s:%s:%s.tar.gz' %
(tool, key, tm), tm)
for tm in tms] if not LocalPath(fn).exists()
]
command = {
'redo': True,
'cvg_tool': tool,
'test_methods': missings,
'project': project,
'version': version
}
if missings:
print "python main.py q cvgmeasure.cvg.test_cvg_methods -j '%s' -t 1800 -q localH --commit" % json.dumps(
command)
def test_basename(self):
name = LocalPath("/some/long/path/to/file.txt").basename
self.assertTrue(isinstance(name, six.string_types))
self.assertEqual("file.txt", str(name))
def construct_path(self, toc_entry):
parts = str(self.path).split('-')[:-1]
parts.append(toc_entry)
return LocalPath('-'.join(parts))
def test_empty(self):
with pytest.raises(TypeError):
LocalPath()
assert local.path() == local.path('.')
class LocalPathTest(unittest.TestCase):
def setUp(self):
self.longpath = LocalPath("/some/long/path/to/file.txt")
def test_name(self):
name = self.longpath.name
self.assertTrue(isinstance(name, six.string_types))
self.assertEqual("file.txt", str(name))
def test_dirname(self):
name = self.longpath.dirname
self.assertTrue(isinstance(name, LocalPath))
self.assertEqual("/some/long/path/to", str(name).replace("\\", "/"))
def test_indexing(self):
p = LocalPath("/some/long/path/to/dir")
self.assertEqual(p[:-1], LocalPath("/some/long/path/to"))
self.assertEqual(p[1:-1], RelativePath("long/path/to".split("/")))
self.assertEqual(p[1], "long")
self.assertEqual(p[::2], "some path dir".split())
self.assertEqual(p['file.txt'], LocalPath("/some/long/path/to/dir/file.txt"))
self.assertEqual(p['subdir/file.txt'], LocalPath("/some/long/path/to/dir/subdir/file.txt"))
self.assertEqual(p['/root/file.txt'], LocalPath("/root/file.txt"))
def test_uri(self):
self.assertEqual("file:///some/long/path/to/file.txt", self.longpath.as_uri())
@skip_without_chown
def test_chown(self):
with local.tempdir() as dir:
p = dir / "foo.txt"
p.write(six.b("hello"))
self.assertEqual(p.uid, os.getuid())
self.assertEqual(p.gid, os.getgid())
p.chown(p.uid.name)
self.assertEqual(p.uid, os.getuid())
def test_split(self):
p = local.path("/var/log/messages")
self.assertEqual(p.split(), ["var", "log", "messages"])
def test_suffix(self):
p1 = self.longpath
p2 = local.path("file.tar.gz")
self.assertEqual(p1.suffix, ".txt")
self.assertEqual(p1.suffixes, [".txt"])
self.assertEqual(p2.suffix, ".gz")
self.assertEqual(p2.suffixes, [".tar",".gz"])
self.assertEqual(p1.with_suffix(".tar.gz"), local.path("/some/long/path/to/file.tar.gz"))
self.assertEqual(p2.with_suffix(".other"), local.path("file.tar.other"))
self.assertEqual(p2.with_suffix(".other", 2), local.path("file.other"))
self.assertEqual(p2.with_suffix(".other", 0), local.path("file.tar.gz.other"))
self.assertEqual(p2.with_suffix(".other", None), local.path("file.other"))
def test_newname(self):
p1 = self.longpath
p2 = local.path("file.tar.gz")
self.assertEqual(p1.with_name("something.tar"), local.path("/some/long/path/to/something.tar"))
self.assertEqual(p2.with_name("something.tar"), local.path("something.tar"))
def test_relative_to(self):
p = local.path("/var/log/messages")
self.assertEqual(p.relative_to("/var/log/messages"), RelativePath([]))
self.assertEqual(p.relative_to("/var/"), RelativePath(["log", "messages"]))
self.assertEqual(p.relative_to("/"), RelativePath(["var", "log", "messages"]))
self.assertEqual(p.relative_to("/var/tmp"), RelativePath(["..", "log", "messages"]))
self.assertEqual(p.relative_to("/opt"), RelativePath(["..", "var", "log", "messages"]))
self.assertEqual(p.relative_to("/opt/lib"), RelativePath(["..", "..", "var", "log", "messages"]))
for src in [local.path("/var/log/messages"), local.path("/var"), local.path("/opt/lib")]:
delta = p.relative_to(src)
self.assertEqual(src + delta, p)
def test_read_write(self):
with local.tempdir() as dir:
f = dir / "test.txt"
text = six.b('hello world\xd7\xa9\xd7\x9c\xd7\x95\xd7\x9d').decode("utf8")
f.write(text, "utf8")
text2 = f.read("utf8")
self.assertEqual(text, text2)
def test_parts(self):
parts = self.longpath.parts
self.assertEqual(parts, ('/', 'some', 'long', 'path', 'to', 'file.txt'))
def test_stem(self):
self.assertEqual(self.longpath.stem, "file")
p = local.path("/some/directory")
self.assertEqual(p.stem, "directory")
@skipIf(pathlib is None, "This test requires pathlib")
def test_root_drive(self):
pl_path = pathlib.Path("/some/long/path/to/file.txt").absolute()
self.assertEqual(self.longpath.root, pl_path.root)
self.assertEqual(self.longpath.drive, pl_path.drive)
p_path = local.cwd / "somefile.txt"
pl_path = pathlib.Path("somefile.txt").absolute()
self.assertEqual(p_path.root, pl_path.root)
self.assertEqual(p_path.drive, pl_path.drive)
@skipIf(pathlib is None, "This test requires pathlib")
def test_compare_pathlib(self):
def filename_compare(name):
p = local.path(str(name))
pl = pathlib.Path(str(name)).absolute()
self.assertEqual(str(p), str(pl))
self.assertEqual(p.parts, pl.parts)
self.assertEqual(p.exists(), pl.exists())
self.assertEqual(p.as_uri(), pl.as_uri())
self.assertEqual(str(p.with_suffix('.this')), str(pl.with_suffix('.this')))
self.assertEqual(p.name, pl.name)
filename_compare("/some/long/path/to/file.txt")
filename_compare(local.cwd / "somefile.txt")
filename_compare("/some/long/path/")
filename_compare("/some/long/path")
filename_compare(__file__)
def test_suffix_expected(self):
self.assertEqual(self.longpath.preferred_suffix('.tar'), self.longpath)
self.assertEqual((local.cwd / 'this').preferred_suffix('.txt'), local.cwd / 'this.txt')
def setUp(self):
self.longpath = LocalPath("/some/long/path/to/file.txt")
from plumbum import LocalPath PROJECT_PATH = LocalPath(__file__).dirname.up() """:type: LocalPath""" OUTPUT_PATH = PROJECT_PATH / '_output' """:type: LocalPath"""
import os
from plumbum import LocalPath
ABS_PATH = LocalPath(os.path.abspath(__file__)).dirname
JAR_PATH = str(LocalPath(ABS_PATH) / 'tgLister.jar')
LINES = ['line:' + T for T in ('cobertura', 'codecover', 'jmockit', 'major')]
BRANCHES = ['branch:' + T for T in ('cobertura', 'codecover', 'jmockit')]
TERMS = ['term:' + T for T in ('cobertura', 'codecover', 'jmockit')]
STATEMENTS = ['statement:' + T for T in ('codecover', )]
LOOPS = ['loop:' + T for T in ('codecover', )]
DATA = ['data:' + T for T in ('jmockit', )]
PATHS = ['path:' + T for T in ('jmockit', )]
MUTANTS = ['mutant:' + T for T in ('major', )]
MUTCVGS = ['mutcvg:' + T for T in ('major', )]
ALL_TGS = LINES + BRANCHES + STATEMENTS + LOOPS + DATA + PATHS + MUTANTS + MUTCVGS # + TERMS
class TestBase(ABC):
"""
Base class for tests. Tests are executed as:
- init
- setup, consisting of the sub steps
- setup_prepare
- setup_start
- _run
- teardown
A test can override any of these methods.
The `_run` method must be defined by each test.
The commandline subcommands for `setup`, `run` and `teardown` allow to run the steps separately.
The `init` function is always called.
Tests should write all their artifacts to the directory `self.artifacts`, which is created
during setup.
"""
@cli.switch("executable",
NameExecutable,
list=True,
help="Paths for executables, format name:path")
def _set_executables(self, executables):
self.executables = {
name: executable
for (name, executable) in executables
}
container_loaders = cli.SwitchAttr(
"container-loader",
ContainerLoader,
list=True,
help="Container loader, format tag#path")
artifacts = cli.SwitchAttr(
"artifacts-dir",
LocalPath,
envname="TEST_UNDECLARED_OUTPUTS_DIR",
default=LocalPath("/tmp/artifacts-scion"),
help="Directory for test artifacts. " +
"Environment variable TEST_UNDECLARED_OUTPUTS_DIR")
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._setup_prepare_failed = False
def init(self):
""" init is called first. The Test object can be initialized here.
The cli parameters have already been parsed when this is called (contrasting to __init__).
"""
pass
def setup(self):
try:
self._setup_prepare_failed = False
self.setup_prepare()
except: # noqa E722, we really want to handle any exception
self._setup_prepare_failed = True
raise
self.setup_start()
@abstractmethod
def _run(self):
"""Run the actual test. Must be implemented by concrete test.
Note: underscored name because this clashes with plumbum.cli.Application
"""
pass
def teardown(self):
pass
def setup_prepare(self):
"""Unpacks loads local docker images and generates the topology.
"""
docker.assert_no_networks()
self._setup_artifacts()
self._setup_container_loaders()
# Define where coredumps will be stored.
print(
cmd.docker("run", "--rm", "--privileged", "alpine", "sysctl", "-w",
"kernel.core_pattern=/share/coredump"))
def setup_start(self):
pass
def _setup_artifacts(self):
# Delete old artifacts, if any.
cmd.rm("-rf", self.artifacts)
cmd.mkdir(self.artifacts)
print("artifacts dir: %s" % self.artifacts)
def _setup_container_loaders(self):
for tag, script in self.container_loaders:
o = local[script]()
idx = o.index("as ")
if idx < 0:
logger.error("extracting tag from loader script %s" % tag)
continue
bazel_tag = o[idx + len("as "):].strip()
logger.info("docker tag %s %s" % (bazel_tag, tag))
cmd.docker("tag", bazel_tag, tag)
def get_executable(self, name: str):
"""Resolve the executable by name.
If the executable is not in the executables mapping, the return value
is './bin/<name>'
"""
return self.executables.get(name, None) or local["./bin/" + name]
def test_chown(self):
path = LocalPath("/tmp/delme.txt")
path.delete()
path.write('test')
self.assertTrue('nobody' != path.owner)
self.assertTrue('nogroup' != path.group)
# chown group
path.chown(group='nogroup')
self.assertEqual('nogroup', path.group)
self.assertTrue('nobody' != path.owner)
# chown owner
path.chown('nobody')
self.assertEqual('nobody', path.owner)
# chown both / numerical ids
path.chown(uid=0, gid=0)
self.assertEqual('root', path.owner)
self.assertEqual('root', path.group)
# recursive
path.chown('root', recursive=True)
# set properties
path.owner = 'nobody'
self.assertEqual('nobody', path.owner)
path.group = 'nogroup'
self.assertEqual('nogroup', path.group)
path.delete()
def __render(path, **kwargs):
tpl = Template(path.read(encoding='utf-8'))
result = tpl.safe_substitute(**kwargs)
dst = LocalPath(path.name)
log.info("## rendered %s ##\n%s\n## -> %s ##\n", path, result, dst)
dst.write(result, encoding='utf-8')
def mkdir_p(dst):
LocalPath(LocalPath(dst).dirname).mkdir()
def test_dirname(self):
name = LocalPath("/some/long/path/to/file.txt").dirname
self.assertTrue(isinstance(name, LocalPath))
self.assertEqual("/some/long/path/to", str(name).replace("\\", "/"))
def test_issue_139(self):
LocalPath(local.cwd)
class Dct:
"""Normal usage:
dct trigger <version>
Advanced usage see:
https://github.com/google/python-fire/blob/master/doc/using-cli.md
"""
_tpl_path = LocalPath('tpl')
def trigger(self, version):
"""Trigger tests by rendering file and pushing changes"""
self._render_files(version)
self._git_push(version)
def create(self, package, devpi_user='', devpi_index=''):
parent = local.cwd / ('devpi-cloud-test-' + package)
if parent.exists():
log.error("%s already exists - aborting", parent)
exit(1)
log.info("create new cloud test in %s", parent)
try:
blueprint_path = LocalPath(__file__).dirname / 'blueprint'
blueprint_path.copy(parent)
with local.cwd(parent):
self.__render(Config.rel_ini_path,
PACKAGE=package,
DEVPI_USER=devpi_user,
DEVPI_INDEX=devpi_index)
self._init_repository(parent)
except Exception:
log.exception("creation failed")
parent.delete()
exit(1)
@Config.necessary
def _render_files(self, version):
"""only render the files for the trigger."""
for src in LocalPath('tpl').list():
self.__render(src,
PACKAGE=config.package,
VERSION=version,
DEVPI_USER=config.devpi_user,
DEVPI_INDEX=config.devpi_index,
TIMESTAMP=datetime.now().ctime())
@staticmethod
def __render(path, **kwargs):
tpl = Template(path.read(encoding='utf-8'))
result = tpl.safe_substitute(**kwargs)
dst = LocalPath(path.name)
log.info("## rendered %s ##\n%s\n## -> %s ##\n", path, result, dst)
dst.write(result, encoding='utf-8')
@Config.necessary
def _git_push(self, version):
"""only push the changes to origin"""
self._git_commit('%s==%s' % (config.package, version))
git('push', 'origin', 'master')
log.info("triggered test by pushing %s", local.cwd)
def _init_repository(self, parent):
with local.cwd(parent):
git('init')
self._git_commit('init')
log.info("initialized. You can add your remote and push.")
def _git_commit(self, msg):
git('add', '.')
git('commit', '-m', msg)
