def test_get_local_filelist_no_result(self):
uniq_id = str(uuid.uuid4())
tmpdir = "/tmp/testcase-{}".format(uniq_id)
os.mkdir(tmpdir)
helper = ReuploadHelper()
files = helper.get_local_filelist(tmpdir)
os.rmdir(tmpdir)
def test_get_pdc_filelist(self):
self.scripts = mockprocess.MockProc()
helper = ReuploadHelper()
self.scripts.append("dsmc",
returncode=0,
script="""#!/bin/bash
cat tests/resources/dsmc_output/dsmc_pdc_filelist.txt
""")
with self.scripts:
archive_path = "/data/mm-xart002/runfolders/johanhe_test_0809_001-AG2UJ_archive"
filelist = helper.get_pdc_filelist(
archive_path,
"e374bd6b-ab36-4f41-94d3-f4eaea9f30d4",
dsmc_log_dir="")
with open("tests/resources/dsmc_output/dsmc_pdc_converted_filelist.txt"
) as f:
nr_of_files = 0
for line in f:
size, name = line.split()
self.assertEqual(int(filelist[name]), int(size))
nr_of_files += 1
self.assertEqual(len(filelist.keys()), nr_of_files)
def test_get_pdc_descr_failing_proc(self):
self.scripts = mockprocess.MockProc()
helper = ReuploadHelper()
self.scripts.append("dsmc", returncode=10)
with self.scripts:
archive_path = "/foo"
descr = helper.get_pdc_descr(archive_path, dsmc_log_dir="")
def test_get_pdc_filelist_failing_proc(self):
self.scripts = mockprocess.MockProc()
helper = ReuploadHelper()
self.scripts.append("dsmc", returncode=13)
with self.scripts:
archive_path = "foo"
filelist = helper.get_pdc_filelist(archive_path, "foo-bar", dsmc_log_dir="")
def test_get_pdc_filelist_no_result(self):
self.scripts = mockprocess.MockProc()
helper = ReuploadHelper()
self.scripts.append("dsmc", returncode=0, script="""#!/bin/bash
echo uggla
""")
with self.scripts:
archive_path = "foo"
filelist = helper.get_pdc_filelist(archive_path, "foo-bar", dsmc_log_dir="")
def test_get_pdc_descr_no_results(self):
self.scripts = mockprocess.MockProc()
helper = ReuploadHelper()
self.scripts.append("dsmc", returncode=0, script="""#!/bin/bash
echo apa
""")
with self.scripts:
archive_path = "foobar"
descr = helper.get_pdc_descr(archive_path, dsmc_log_dir="")
def test_reupload(self):
helper = ReuploadHelper()
uniq_id = "test"
dsmc_log_dir = "/tmp/foo"
dsmc_log_file = "foolog"
descr = "foodescr"
run_dir = "foodir"
uniq_id = str(uuid.uuid4())
reupload_file = "/tmp/test_reupload-{}".format(uniq_id)
exp_id = 72
class MyRunner(object):
def start(self,
cmd,
nbr_of_cores,
run_dir,
stdout=dsmc_log_file,
stderr=dsmc_log_file):
self.components = cmd.split("=")
self.cmd = cmd
return exp_id
runsrv = MyRunner()
local_files = {"foo": 23, "bar": 46, "uggla": 72}
uploaded_files = {"foo": 23, "uggla": 15}
exp_upload = ['"bar"\n', '"uggla"\n']
reupload_files = helper.get_files_to_reupload(local_files,
uploaded_files)
def my_tmp_file(ReuploadHelper, component):
return reupload_file
with mock.patch \
("archive_upload.handlers.dsmc_handlers.ReuploadHelper._tmp_file",\
autospec=True) as mock__tmp_file:
mock__tmp_file.side_effect = my_tmp_file
res_id = helper.reupload(reupload_files, descr, dsmc_log_dir,
runsrv)
self.assertEqual(res_id, exp_id)
self.assertEqual(runsrv.components[-1], descr)
with open(reupload_file) as f:
uploaded = f.readlines()
import sets
uploaded = sets.Set(uploaded)
exp_upload = sets.Set(exp_upload)
self.assertEqual(len(uploaded.symmetric_difference(exp_upload)), 0)
def test_get_pdc_descr(self):
self.scripts = mockprocess.MockProc()
helper = ReuploadHelper()
# FIXME: this dsmc-output can not be removed, or the test will fail
self.scripts.append("dsmc", returncode=0,
script="""#!/bin/bash
cat tests/resources/dsmc_output/dsmc_descr.txt
""")
with self.scripts:
archive_path = "/data/mm-xart002/runfolders/johanhe_test_0809_001-AG2UJ_archive"
descr = helper.get_pdc_descr(archive_path, dsmc_log_dir="")
self.assertEqual(descr, "e374bd6b-ab36-4f41-94d3-f4eaea9f30d4")
def test_get_files_to_reupload(self):
helper = ReuploadHelper()
local_files = {"foo": 23, "bar": 46}
uploaded_files = {"foo": 23}
expected = ["bar"]
result = helper.get_files_to_reupload(local_files, uploaded_files)
self.assertItemsEqual(expected, result)
uploaded_files = {"foo": 44}
expected = ["foo", "bar"]
result = helper.get_files_to_reupload(local_files, uploaded_files)
self.assertItemsEqual(expected, result)
local_files = {"foo": 44}
uploaded_files = {"foo": 44}
expected = []
result = helper.get_files_to_reupload(local_files, uploaded_files)
self.assertItemsEqual(expected, result)
def test_get_local_filelist(self):
helper = ReuploadHelper()
path = "tests/resources/archives/archive_from_pdc"
cmd = "find {} -type f -exec du -b {{}} \;".format(path)
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
du_out, du_err = p.communicate()
du_out = du_out.splitlines()
files = helper.get_local_filelist(path)
for line in du_out:
size, filename = line.split()
path = os.path.join(path, filename)
self.assertEqual(files[filename], int(size))
self.assertEqual(len(files.keys()), len(du_out))