def run_backup(self):
"""
Make a backup
"""
ball = "%s.tar.bz2" % self.output
with local.cwd(self.outputdir):
chain = ((self.tar["cf", "-", "results", "log"] | self.bzip2) >
ball)
if self.verbose:
self.print_to_stdout("\t\tbackup command:\n")
self.print_to_stdout("\t\t\t%s " % chain)
self.print_to_stdout("\n")
chain()
if os.path.exists(os.path.join(self.resultdir, ball)):
now = datetime.datetime.now()
backup = "%s.%s-%s-%s.tar.bz2" % (self.output, now.year, now.month,
now.day)
move(os.path.join(self.resultdir, ball),
os.path.join(self.backupdir, backup))
if self.verbose:
self.print_to_stdout("\t\tbackup previous run:\n")
self.print_to_stdout("\t\t\t%s " % backup)
self.print_to_stdout("\n")
move(os.path.join(self.outputdir, ball),
os.path.join(self.resultdir, ball))
def on_success(self):
super().on_success()
move(
"{}/{}/index".format(self.data_folder, self.collection_name),
"{}/{}/index-sorted".format(self.data_folder,
self.collection_name))
move(
"{}/{}/index-prunning".format(self.data_folder,
self.collection_name),
"{}/{}/index".format(self.data_folder, self.collection_name))
def on_success(self):
super().on_success()
move(
"{}/outputs_{}/index".format(self.data_folder,
self.collection_name),
"{}/outputs_{}/index-orig".format(self.data_folder,
self.collection_name))
move(
"{}/outputs_{}/index-sorted".format(self.data_folder,
self.collection_name),
"{}/outputs_{}/index".format(self.data_folder,
self.collection_name))
def do_reset(self):
"""
start with a fresh output
"""
if (self.output == self.name):
if self.verbose:
self.print_to_stdout("\t\tReset output: %s\n" % self.outputdir)
delete(self.tempdir)
move(self.outputdir, os.path.join(self.tempdir, "output"))
self.make_dir(self.outputdir)
self.make_dir(self.logsdir)
def test_copy_move_delete(self):
from plumbum.cmd import touch
with local.tempdir() as dir:
(dir / "orog").mkdir()
(dir / "orog" / "rec").mkdir()
for i in range(20):
touch(dir / "orog" / ("f%d.txt" % (i, )))
for i in range(20, 40):
touch(dir / "orog" / "rec" / ("f%d.txt" % (i, )))
move(dir / "orog", dir / "orig")
s1 = sorted(f.basename for f in (dir / "orig").walk())
copy(dir / "orig", dir / "dup")
s2 = sorted(f.basename for f in (dir / "dup").walk())
self.assertEqual(s1, s2)
with SshMachine("localhost") as rem:
with rem.tempdir() as dir2:
copy(dir / "orig", dir2)
s3 = sorted(f.basename for f in (dir2 / "orig").walk())
self.assertEqual(s1, s3)
copy(dir2 / "orig", dir2 / "dup")
s4 = sorted(f.basename for f in (dir2 / "dup").walk())
self.assertEqual(s1, s4)
copy(dir2 / "dup", dir / "dup2")
s5 = sorted(f.basename for f in (dir / "dup2").walk())
self.assertEqual(s1, s5)
with SshMachine("localhost") as rem2:
with rem2.tempdir() as dir3:
copy(dir2 / "dup", dir3)
s6 = sorted(f.basename
for f in (dir3 / "dup").walk())
self.assertEqual(s1, s6)
move(dir3 / "dup", dir / "superdup")
self.assertFalse((dir3 / "dup").exists())
s7 = sorted(f.basename
for f in (dir / "superdup").walk())
self.assertEqual(s1, s7)
# test rm
delete(dir)
def test_copy_move_delete(self):
from plumbum.cmd import touch
with local.tempdir() as dir:
(dir / "orog").mkdir()
(dir / "orog" / "rec").mkdir()
for i in range(20):
touch(dir / "orog" / ("f%d.txt" % (i, )))
for i in range(20, 40):
touch(dir / "orog" / "rec" / ("f%d.txt" % (i, )))
move(dir / "orog", dir / "orig")
s1 = sorted(f.name for f in (dir / "orig").walk())
copy(dir / "orig", dir / "dup")
s2 = sorted(f.name for f in (dir / "dup").walk())
assert s1 == s2
with SshMachine("localhost") as rem:
with rem.tempdir() as dir2:
copy(dir / "orig", dir2)
s3 = sorted(f.name for f in (dir2 / "orig").walk())
assert s1 == s3
copy(dir2 / "orig", dir2 / "dup")
s4 = sorted(f.name for f in (dir2 / "dup").walk())
assert s1 == s4
copy(dir2 / "dup", dir / "dup2")
s5 = sorted(f.name for f in (dir / "dup2").walk())
assert s1 == s5
with SshMachine("localhost") as rem2:
with rem2.tempdir() as dir3:
copy(dir2 / "dup", dir3)
s6 = sorted(f.name for f in (dir3 / "dup").walk())
assert s1 == s6
move(dir3 / "dup", dir / "superdup")
assert not (dir3 / "dup").exists()
s7 = sorted(f.name
for f in (dir / "superdup").walk())
assert s1 == s7
# test rm
delete(dir)
def test_copy_move_delete(self):
from plumbum.cmd import touch
with local.tempdir() as dir:
(dir / "orog").mkdir()
(dir / "orog" / "rec").mkdir()
for i in range(20):
touch(dir / "orog" / ("f%d.txt" % (i,)))
for i in range(20,40):
touch(dir / "orog" / "rec" / ("f%d.txt" % (i,)))
move(dir / "orog", dir / "orig")
s1 = sorted(f.name for f in (dir / "orig").walk())
copy(dir / "orig", dir / "dup")
s2 = sorted(f.name for f in (dir / "dup").walk())
assert s1 == s2
with SshMachine("localhost") as rem:
with rem.tempdir() as dir2:
copy(dir / "orig", dir2)
s3 = sorted(f.name for f in (dir2 / "orig").walk())
assert s1 == s3
copy(dir2 / "orig", dir2 / "dup")
s4 = sorted(f.name for f in (dir2 / "dup").walk())
assert s1 == s4
copy(dir2 / "dup", dir / "dup2")
s5 = sorted(f.name for f in (dir / "dup2").walk())
assert s1 == s5
with SshMachine("localhost") as rem2:
with rem2.tempdir() as dir3:
copy(dir2 / "dup", dir3)
s6 = sorted(f.name for f in (dir3 / "dup").walk())
assert s1 == s6
move(dir3 / "dup", dir / "superdup")
assert not (dir3 / "dup").exists()
s7 = sorted(f.name for f in (dir / "superdup").walk())
assert s1 == s7
# test rm
delete(dir)
def test_copy_move_delete(self):
from plumbum.cmd import touch
with local.tempdir() as dir:
(dir / "orog").mkdir()
(dir / "orog" / "rec").mkdir()
for i in range(20):
touch(dir / "orog" / ("f%d.txt" % (i,)))
for i in range(20,40):
touch(dir / "orog" / "rec" / ("f%d.txt" % (i,)))
move(dir / "orog", dir / "orig")
s1 = sorted(f.basename for f in (dir / "orig").walk())
copy(dir / "orig", dir / "dup")
s2 = sorted(f.basename for f in (dir / "dup").walk())
self.assertEqual(s1, s2)
with SshMachine("localhost") as rem:
with rem.tempdir() as dir2:
copy(dir / "orig", dir2)
s3 = sorted(f.basename for f in (dir2 / "orig").walk())
self.assertEqual(s1, s3)
copy(dir2 / "orig", dir2 / "dup")
s4 = sorted(f.basename for f in (dir2 / "dup").walk())
self.assertEqual(s1, s4)
copy(dir2 / "dup", dir / "dup2")
s5 = sorted(f.basename for f in (dir / "dup2").walk())
self.assertEqual(s1, s5)
with SshMachine("localhost") as rem2:
with rem2.tempdir() as dir3:
copy(dir2 / "dup", dir3)
s6 = sorted(f.basename for f in (dir3 / "dup").walk())
self.assertEqual(s1, s6)
move(dir3 / "dup", dir / "superdup")
self.assertFalse((dir3 / "dup").exists())
s7 = sorted(f.basename for f in (dir / "superdup").walk())
self.assertEqual(s1, s7)
# test rm
delete(dir)
def run(self):
self.print_to_stdout("Harvest run started, output=%s, name=%s.\n" %
(self.output, self.name))
self.print_to_stdout("\tInitializing.\n")
self.initialize()
self.print_to_stdout("\tDone\n")
self.print_to_stdout("\tRunning harvester.\n")
self.run_harvest()
self.print_to_stdout("\tDone\n")
self.print_to_stdout("\tExpand map.\n")
self.expand_map()
self.print_to_stdout("\tDone\n")
self.print_to_stdout("\tReset output.\n")
self.do_reset()
self.print_to_stdout("\tDone\n")
self.print_to_stdout("\tMerging harvest result to output.\n")
self.merge("results/cmdi")
self.merge("results/cmdi-1_1")
self.merge("oai-pmh")
# TODO merge map.csv
move(os.path.join(self.workdir, "map.csv"),
os.path.join(self.outputdir, "results", "map.csv"))
self.merge_logs()
self.print_to_stdout("\tDone\n")
self.print_to_stdout("\tBackup harvest result.\n")
self.run_backup()
self.print_to_stdout("\tDone\n")
move(os.path.join(self.outputdir, "results", "map.csv"),
os.path.join(self.workdir, "map.csv"))
self.print_to_stdout(
"\tGenerate update for the harvest view database.\n")
self.run_viewer()
self.print_to_stdout("\tDone\n")
if self.pg:
self.print_to_stdout("\tUpdating harvest view database.\n")
self.run_psql()
self.print_to_stdout("\tDone\n")
move(os.path.join(self.workdir, "map.csv"),
os.path.join(self.outputdir, "results", "map.csv"))
self.print_to_stdout("\tCleanup.\n")
self.do_cleanup()
self.print_to_stdout("\tDone\n")
self.print_to_stdout("Done\n")
def merge_logs(self):
"""
save the logs
"""
for log in local.path(self.logdir).list():
move(log, os.path.join(self.logsdir, log.name))
