def test_create_tags(test_dir, monkeypatch, tags, expected):
"""Test setting tags.
"""
monkeypatch.chdir(test_dir)
archive_path = Path(archive_name(tags=["tags"], counter="create_tags"))
Archive().create(archive_path, "", [Path("base")], tags=tags)
with Archive().open(archive_path) as archive:
assert archive.manifest.tags == expected
def test_create_fileinfos_generator(test_dir, monkeypatch):
"""Create the archive from FileInfo.iterpaths() which returns a generator.
"""
monkeypatch.chdir(test_dir)
fileinfos = FileInfo.iterpaths([Path("base")], set())
archive_path = Path("archive-fi-generator.tar")
Archive().create(archive_path, "", fileinfos=fileinfos)
with Archive().open(archive_path) as archive:
check_manifest(archive.manifest, testdata)
archive.verify()
def test_create_fileinfos_list(test_dir, monkeypatch):
"""Create the archive from a list of FileInfo objects.
"""
monkeypatch.chdir(test_dir)
fileinfos = list(FileInfo.iterpaths([Path("base")], set()))
archive_path = Path("archive-fi-list.tar")
Archive().create(archive_path, "", fileinfos=fileinfos)
with Archive().open(archive_path) as archive:
check_manifest(archive.manifest, testdata)
archive.verify()
def test_create_add_symlink(test_dir, monkeypatch):
"""Check adding explicitly adding a symbolic link. (Issue #37)
"""
monkeypatch.chdir(test_dir)
archive_path = Path("archive-symlink.tar")
paths = [Path("base", "data", "misc"), Path("base", "data", "s.dat")]
data = [ testdata[i] for i in (1,3,4) ]
Archive().create(archive_path, "", paths)
with Archive().open(archive_path) as archive:
check_manifest(archive.manifest, data)
archive.verify()
def test_create_default_basedir_rel(test_dir, monkeypatch):
"""Check the default basedir with relative paths. (Issue #8)
"""
monkeypatch.chdir(test_dir)
archive_path = Path("archive-rel.tar")
p = Path("base", "data")
Archive().create(archive_path, "", [p])
with Archive().open(archive_path) as archive:
assert archive.basedir == Path("base")
check_manifest(archive.manifest, testdata)
archive.verify()
def test_dir(tmpdir):
with tmp_chdir(tmpdir):
rel_paths = archive_paths(Path(""), False)
abs_paths = archive_paths(Path(""), True)
for i, data in enumerate(testdata):
base = next(filter(lambda e: e.type == 'd', data)).path
setup_testdata(tmpdir, data)
Archive().create(rel_paths[i], "bz2", [base])
Archive().create(abs_paths[i], "bz2", [tmpdir / base])
shutil.rmtree(base)
return tmpdir
def test_create_default_basedir_abs(test_dir, monkeypatch):
"""Check the default basedir with absolute paths. (Issue #8)
"""
monkeypatch.chdir(test_dir)
archive_path = Path("archive-abs.tar")
p = test_dir / Path("base", "data")
Archive().create(archive_path, "", [p])
with Archive().open(archive_path) as archive:
assert archive.basedir == Path("archive-abs")
check_manifest(archive.manifest, testdata, prefix_dir=test_dir)
archive.verify()
def test_create_fileinfos_manifest(test_dir, monkeypatch):
"""Create the archive from a Manifest.
A Manifest is an iterable of FileInfo objects.
"""
monkeypatch.chdir(test_dir)
manifest = Manifest(paths=[Path("base")])
archive_path = Path("archive-fi-manifest.tar")
Archive().create(archive_path, "", fileinfos=manifest)
with Archive().open(archive_path) as archive:
check_manifest(archive.manifest, testdata)
archive.verify()
def test_create_exclude_file(test_dir, testname, monkeypatch):
"""Exclude one single file.
"""
monkeypatch.chdir(str(test_dir))
name = archive_name(tags=[testname])
paths = [Path("base")]
excludes = [Path("base", "msg.txt")]
data = sub_testdata(testdata, excludes[0])
Archive().create(Path(name), "", paths, excludes=excludes)
with Archive().open(Path(name)) as archive:
check_manifest(archive.manifest, data)
archive.verify()
def test_create_exclude_subdir(test_dir, testname, monkeypatch):
"""Exclude a subdirectory.
"""
monkeypatch.chdir(test_dir)
name = archive_name(tags=[testname])
paths = [Path("base")]
excludes = [Path("base", "data")]
data = sub_testdata(testdata, excludes[0])
Archive().create(Path(name), "", paths, excludes=excludes)
with Archive().open(Path(name)) as archive:
check_manifest(archive.manifest, data)
archive.verify()
def test_create_long_directory_name(tmpdir, monkeypatch):
"""An archive containing a directory with a long path name.
Verification fails if the archive is created in the GNU tar format.
"""
setup_testdata(tmpdir, testdata_long_dir)
monkeypatch.chdir(tmpdir)
archive_path = Path("archive-longdir.tar")
Archive().create(archive_path, "", [long_dir_path])
with Archive().open(archive_path) as archive:
check_manifest(archive.manifest, testdata_long_dir)
archive.verify()
def test_create_exclude_samelevel(test_dir, testname, monkeypatch):
"""Exclude a directory explictely named in paths.
"""
monkeypatch.chdir(test_dir)
name = archive_name(tags=[testname])
paths = [Path("base", "data"), Path("base", "empty")]
excludes = [paths[1]]
data = sub_testdata(testdata, Path("base"), paths[0])
Archive().create(Path(name), "", paths, excludes=excludes)
with Archive().open(Path(name)) as archive:
check_manifest(archive.manifest, data)
archive.verify()
def test_create_exclude_explicit_include(test_dir, testname, monkeypatch):
"""Exclude a directory, but explicitely include an item in that
directory.
"""
monkeypatch.chdir(test_dir)
name = archive_name(tags=[testname])
paths = [Path("base"), Path("base", "data", "rnd1.dat")]
excludes = [Path("base", "data")]
data = sub_testdata(testdata, excludes[0], paths[1])
Archive().create(Path(name), "", paths, excludes=excludes)
with Archive().open(Path(name)) as archive:
check_manifest(archive.manifest, data)
archive.verify()
def test_create_invalid_file_fifo(test_dir, testname, monkeypatch):
"""Create an archive from a directory containing a FIFO.
"""
monkeypatch.chdir(test_dir)
archive_path = Path(archive_name(tags=[testname]))
p = Path("base")
fp = p / "fifo"
with tmp_fifo(fp):
with pytest.warns(ArchiveWarning, match="%s: FIFO ignored" % fp):
Archive().create(archive_path, "", [p])
with Archive().open(archive_path) as archive:
assert archive.basedir == Path("base")
check_manifest(archive.manifest, testdata)
archive.verify()
def test_create_invalid_file_socket(test_dir, testname, monkeypatch):
"""Create an archive from a directory containing a socket.
"""
monkeypatch.chdir(str(test_dir))
name = archive_name(tags=[testname])
p = Path("base")
fp = p / "socket"
with tmp_socket(fp):
with pytest.warns(ArchiveWarning, match="%s: socket ignored" % fp):
Archive().create(name, "", [p])
with Archive().open(name) as archive:
assert archive.basedir == Path("base")
check_manifest(archive.manifest, testdata)
archive.verify()
def test_create_fileinfos_subset(test_dir, monkeypatch):
"""Do not include the content of a directory.
This test verifies that creating an archive from fileinfos does
not implicitly descend subdirectories.
"""
monkeypatch.chdir(test_dir)
excludes = [Path("base", "data", "rnd.dat")]
fileinfos = FileInfo.iterpaths([Path("base")], set(excludes))
data = sub_testdata(testdata, excludes[0])
archive_path = Path("archive-fi-subset.tar")
Archive().create(archive_path, "", fileinfos=fileinfos)
with Archive().open(archive_path) as archive:
check_manifest(archive.manifest, data)
archive.verify()
def test_create_workdir(test_dir, monkeypatch, abs_wd):
"""Pass an absolute or relative workdir to Archive.create().
(Issue #53)
"""
monkeypatch.chdir(test_dir)
if abs_wd:
workdir = test_dir / "work"
else:
workdir = Path("work")
archive_path = Path(archive_name(tags=[absflag(abs_wd)]))
Archive().create(archive_path, "", [Path("base")], workdir=workdir)
with Archive().open(workdir / archive_path) as archive:
assert archive.basedir == Path("base")
check_manifest(archive.manifest, testdata)
archive.verify()
def test_diff_metadata(test_data, testname, monkeypatch, abspath):
"""Diff two archives having one file's file system metadata modified.
This difference should be ignored by default.
"""
monkeypatch.chdir(str(test_data))
if abspath:
archive_ref_path = Path("archive-abs.tar")
base_dir = test_data / "base"
else:
archive_ref_path = Path("archive-rel.tar")
base_dir = Path("base")
p = base_dir / "rnd.dat"
p.chmod(0o0444)
flag = absflag(abspath)
archive_path = Path(archive_name(ext="bz2", tags=[testname, flag]))
Archive().create(archive_path, "bz2", [base_dir])
with TemporaryFile(mode="w+t", dir=str(test_data)) as f:
args = ["diff", str(archive_ref_path), str(archive_path)]
callscript("archive-tool.py", args, stdout=f)
f.seek(0)
assert list(get_output(f)) == []
with TemporaryFile(mode="w+t", dir=str(test_data)) as f:
args = [
"diff", "--report-meta",
str(archive_ref_path),
str(archive_path)
]
callscript("archive-tool.py", args, returncode=100, stdout=f)
f.seek(0)
out = list(get_output(f))
assert len(out) == 1
assert out[0] == ("File system metadata for %s:%s and %s:%s differ" %
(archive_ref_path, p, archive_path, p))
def test_RevCIAddB_revise_ignoring(self):
met1 = mnm_repr.Metabolite('met1')
met2 = mnm_repr.Metabolite('met2')
comp1 = mnm_repr.Medium()
cond_subst_1 = mnm_repr.PresentEntity(met1, comp1)
cond_subst_2 = mnm_repr.PresentEntity(met2, comp1)
a1 = mnm_repr.Reaction('act1', [], [cond_subst_1, cond_subst_2])
a1.remove_cost = None
a1.reversibility = False
a2 = mnm_repr.Reaction('act1', [], [cond_subst_1])
a2.reversibility = False
# model to be revised
mod = mnm_repr.Model('m_0', [], [a1], [])
# results
des1 = exp_repr.ExperimentDescription(exp_repr.DetectionEntity('met1'),
[])
des2 = exp_repr.ExperimentDescription(exp_repr.DetectionEntity('met2'),
[])
res1 = exp_repr.Result('res_0', des1, 'false')
res2 = exp_repr.Result('res_1', des2, 'true')
exp1 = exp_repr.Experiment('exp_0', [res1])
exp2 = exp_repr.Experiment('exp_1', [res2])
# archive with results and parts for revision
arch = Archive()
arch.known_results = [exp1, exp2]
arch.mnm_activities = [a1, a2]
arch.mnm_entities = [met1, met2]
arch.mnm_compartments = [comp1]
rev = RevCIAddB(arch)
out = rev.revise(mod)
self.assertEqual(out[0], [])
self.assertEqual(out[1], True)
def prepare_forsale():
archive = Archive()
df = archive.read_forsale_baseline()
df = df[df['alias'].isin(['V','R'])]
df = add_days_ago(df, 'created_date', 'days')
df = df[df['days'] < 30]
df['maps_url'] = df.apply(lambda x: f'https://www.google.com/maps?q={x.lat},{x.lon}', axis=1)
df['boliga_url'] = df.apply(lambda x: f'https://www.boliga.dk/bolig/{x.estate_id}', axis=1)
df['city'] = df.apply(lambda x: 'Lyngby' if x.city == 'Kongens Lyngby' else x.city, axis=1)
df['list_price'] = df.apply(lambda x: fix_pricing(x.list_price), axis=1)
df['sqm_price'] = df.apply(lambda x: fix_pricing(x.sqm_price), axis=1)
cols = [
'city',
'address',
'alias',
'rooms',
'living_area',
'lot_area',
'energy_class',
'build_year',
'list_price',
'sqm_price',
'boliga_url',
'estate_url',
'maps_url',
'days'
]
df = df[cols]
df = df.sort_values(by=['city', 'days']).reset_index(drop=True)
df = df.rename(columns={'alias':'type', 'energy_class':'energy', 'build_year':'built'})
return df
def test_create_empty(test_dir, testname, monkeypatch):
"""Creating an empty archive will be refused.
"""
monkeypatch.chdir(test_dir)
name = archive_name(tags=[testname])
with pytest.raises(ArchiveCreateError):
Archive().create(Path(name), "", [], basedir=Path("base"))
def main(args):
confs = CameraConfig(args.config_file)
ncams = confs.n_cameras()
cams = CameraList()
archive = Archive(confs)
framenos = [0] * ncams
location = confs.data['location']
sleep = confs.data['cameras'][0]['sleep']
start_video = confs.data['cameras'][0]['start_video']
stop_video = confs.data['cameras'][0]['stop_video']
for c in confs.data['cameras']:
camid = c['id']
addr = c['url']
cam = Camera(addr, camid)
cams.append(cam)
framenos[cam.camid] = int(archive.next_prefix(cam.camid))
while True:
if is_daylight(location) or args.debug:
for i, cam in enumerate(cams):
im = cam.grab()
timestamp = datetime.now()
if im is None:
break
framenos[i] += 1
archive.save_image(cam.camid, timestamp, framenos[i], im)
time.sleep(sleep)
def urlsToImages(urls):
cdir = Archive().clearArchive()
imageCount = 0
images = []
for url in urls:
Debug().p("url:%s" % url)
imageTypes = ['full', 'thumb']
raw_img = None
for t in imageTypes:
try:
Debug().p("open image type: %s image: %s" % (t, url[t]))
response = requests.get(url[t], timeout=20)
raw_img = response.content
break
except Exception as e:
traceback.print_exc()
print "return from exception for type %s url %s: %s" % (
t, url[t], e)
continue
if raw_img != None:
fname = "%s/urlimage%d.jpg" % (cdir, imageCount)
imageCount += 1
f = open(fname, "wb")
f.write(raw_img)
f.close()
images.append(fname)
return images
def setUp(self):
self.archive = Archive()
exd = ExperimentDescription(DetectionEntity(None))
res1 = Result('res_0', exd, None)
res2 = Result('res_1', exd, None)
res3 = Result('res_2', exd, None)
res4 = Result('res_3', exd, None)
exp1 = Experiment('exp_0', [res1])
exp2 = Experiment('exp_1', [res2])
exp3 = Experiment('exp_2', [res3])
exp4 = Experiment('exp_3', [res4])
r1 = NewResults(exp1)
r2 = NewResults(exp2)
r3 = NewResults(exp3)
r4 = NewResults(exp4)
self.mod1 = Model('m_0', [1], [], [])
self.mod1.ignored_results = frozenset([res3])
self.mod1.results_covered = frozenset([res1, res4])
self.mod2 = Model('m_1', [2], [], [])
self.mod2.ignored_results = frozenset([res3])
self.mod2.results_covered = frozenset([res1, res2, res4])
m = AdditionalModels([self.mod1, self.mod2])
self.archive.record(r1)
self.archive.record(m)
self.archive.record(r2)
self.archive.record(r3)
self.archive.record(r4)
def test_diff_symlink_target(test_data, testname, monkeypatch, abspath):
"""Diff two archives having one symlink's target modified.
"""
monkeypatch.chdir(str(test_data))
if abspath:
archive_ref_path = Path("archive-abs.tar")
base_dir = test_data / "base"
else:
archive_ref_path = Path("archive-rel.tar")
base_dir = Path("base")
p = base_dir / "s.dat"
p.unlink()
p.symlink_to(Path("msg.txt"))
flag = absflag(abspath)
archive_path = Path(archive_name(ext="bz2", tags=[testname, flag]))
Archive().create(archive_path, "bz2", [base_dir])
with TemporaryFile(mode="w+t", dir=str(test_data)) as f:
args = ["diff", str(archive_ref_path), str(archive_path)]
callscript("archive-tool.py", args, returncode=101, stdout=f)
f.seek(0)
out = list(get_output(f))
assert len(out) == 1
assert out[0] == (
"Symbol links %s:%s and %s:%s have different target" %
(archive_ref_path, p, archive_path, p))
def test_diff_mult(test_data, testname, monkeypatch, abspath):
"""Diff two archives having multiple differences.
"""
monkeypatch.chdir(str(test_data))
if abspath:
archive_ref_path = Path("archive-abs.tar")
base_dir = test_data / "base"
else:
archive_ref_path = Path("archive-rel.tar")
base_dir = Path("base")
pm = base_dir / "data" / "rnd.dat"
shutil.copy(str(gettestdata("rnd2.dat")), str(pm))
p1 = base_dir / "msg.txt"
p2 = base_dir / "o.txt"
p1.rename(p2)
flag = absflag(abspath)
archive_path = Path(archive_name(ext="bz2", tags=[testname, flag]))
Archive().create(archive_path, "bz2", [base_dir])
with TemporaryFile(mode="w+t", dir=str(test_data)) as f:
args = ["diff", str(archive_ref_path), str(archive_path)]
callscript("archive-tool.py", args, returncode=102, stdout=f)
f.seek(0)
out = list(get_output(f))
assert len(out) == 3
assert out[0] == ("Files %s:%s and %s:%s differ" %
(archive_ref_path, pm, archive_path, pm))
assert out[1] == "Only in %s: %s" % (archive_ref_path, p1)
assert out[2] == "Only in %s: %s" % (archive_path, p2)
def test_1_0_check_manifest(legacy_1_0_archive):
with Archive().open(legacy_1_0_archive) as archive:
assert archive.manifest.version == "1.0"
assert isinstance(archive.manifest.date, datetime.datetime)
assert len(archive.manifest.checksums) > 0
manifest_path = archive.basedir / ".manifest.yaml"
assert archive.manifest.metadata == (str(manifest_path), )
def test_diff_basedir_mod_file(test_data, testname, monkeypatch):
"""Diff two archives with different base directories having one file's
content modified.
This test makes only sense for archives with relative path names.
"""
monkeypatch.chdir(str(test_data))
base = Path("base")
newbase = Path("newbase")
shutil.rmtree(str(newbase), ignore_errors=True)
base.rename(newbase)
archive_ref_path = Path("archive-rel.tar")
p = base / "rnd.dat"
pn = newbase / "rnd.dat"
shutil.copy(str(gettestdata("rnd2.dat")), str(pn))
archive_path = Path(archive_name(ext="bz2", tags=[testname, "rel"]))
Archive().create(archive_path, "bz2", [newbase])
with TemporaryFile(mode="w+t", dir=str(test_data)) as f:
args = ["diff", str(archive_ref_path), str(archive_path)]
callscript("archive-tool.py", args, returncode=101, stdout=f)
f.seek(0)
out = list(get_output(f))
assert len(out) == 1
assert out[0] == ("Files %s:%s and %s:%s differ" %
(archive_ref_path, p, archive_path, pn))
def test_check_content(test_dir, dep_testcase, inclmeta):
compression, abspath = dep_testcase
flag = absflag(abspath)
archive_path = test_dir / archive_name(ext=compression, tags=[flag])
outdir = test_dir / "out"
shutil.rmtree(outdir, ignore_errors=True)
outdir.mkdir()
if abspath:
cwd = outdir / "archive" / test_dir.relative_to(test_dir.anchor)
else:
cwd = outdir
with Archive().open(archive_path) as archive:
archive.extract(outdir, inclmeta=inclmeta)
metadata = archive.manifest.metadata
for f in metadata:
assert (outdir / f).is_file() == inclmeta
try:
sha256 = subprocess.Popen([sha256sum, "--check"],
cwd=cwd,
stdin=subprocess.PIPE)
except FileNotFoundError:
pytest.skip("%s program not found" % sha256sum)
for f in testdata:
if f.type == 'f':
l = "%s %s\n" % (f.checksum, f.path)
sha256.stdin.write(l.encode('ascii'))
sha256.stdin.close()
sha256.wait()
assert sha256.returncode == 0
def test_diff_dircontent(test_data, testname, monkeypatch, abspath):
"""Diff two archives with one subdirectory missing.
"""
monkeypatch.chdir(str(test_data))
if abspath:
archive_ref_path = Path("archive-abs.tar")
base_dir = test_data / "base"
else:
archive_ref_path = Path("archive-rel.tar")
base_dir = Path("base")
pd = base_dir / "data"
shutil.rmtree(str(pd))
flag = absflag(abspath)
archive_path = Path(archive_name(ext="bz2", tags=[testname, flag]))
Archive().create(archive_path, "bz2", [base_dir])
with TemporaryFile(mode="w+t", dir=str(test_data)) as f:
args = ["diff", str(archive_ref_path), str(archive_path)]
callscript("archive-tool.py", args, returncode=102, stdout=f)
f.seek(0)
out = list(get_output(f))
assert len(out) == 2
assert out[0] == "Only in %s: %s" % (archive_ref_path, pd)
assert out[1] == "Only in %s: %s" % (archive_ref_path, pd / "rnd.dat")
with TemporaryFile(mode="w+t", dir=str(test_data)) as f:
args = [
"diff", "--skip-dir-content",
str(archive_ref_path),
str(archive_path)
]
callscript("archive-tool.py", args, returncode=102, stdout=f)
f.seek(0)
out = list(get_output(f))
assert len(out) == 1
assert out[0] == "Only in %s: %s" % (archive_ref_path, pd)
