def quarantine(dest, logger, *files):
"""Quarantine problematic tarballs.
Errors here are fatal but we log an error message to help diagnose
problems.
"""
try:
os.mkdir(dest)
except FileExistsError:
# directory already exists, ignore
pass
except Exception:
logger.exception('quarantine {} {!r}: "mkdir -p {}/" failed', dest,
files, dest)
sys.exit(101)
for afile in files:
if not os.path.exists(afile) and not os.path.islink(afile):
continue
try:
# If the file we're moving is a tarball, update the dataset
# state. (If it's the associated MD5 file, skip that.)
if str(afile).endswith(".tar.xz"):
try:
Dataset.attach(path=afile, state=States.QUARANTINED)
except DatasetNotFound:
logger.exception("quarantine dataset {} not found", afile)
shutil.move(afile, os.path.join(dest, os.path.basename(afile)))
except Exception:
logger.exception('quarantine {} {!r}: "mv {} {}/" failed', dest,
files, afile, dest)
sys.exit(102)
def test_attach_none(self):
""" Test expected failure when we try to attach to a dataset that
does not exist.
"""
with pytest.raises(DatasetNotFound):
Dataset.attach(controller="frodo",
name="venus",
state=States.UPLOADING)
def process(self, link: str, state: States) -> int:
"""
process Create Dataset records for pre-existing server tarballs that
are in a specified filesystem "state" (the link directory in the
archive tree), in a specified Dataset state.
Each tarball for which a Dataset record already exists is IGNORED,
and we don't attempt to advance the state.
Args:
:link (str): Filesystem "state" link directory
(e.g., TO-INDEX)
:state (States): A state enum value representing desired Dataset
state.
Returns:
int: Status (0 success, 1 failure)
"""
logger = self.logger
done = 0
fail = 0
ignore = 0
args = {}
owner = User.validate_user(self.options.user)
for tarball in self._collect_tb(link):
if self.options.verify:
print(f"Processing {tarball} from {link} -> state {state}")
try:
args["path"] = tarball
try:
dataset = Dataset.attach(**args)
if self.options.verify:
print(f"Found existing {dataset}: {dataset.state}")
ignore = ignore + 1
except DatasetNotFound:
a = args.copy()
a["md5"] = open(f"{tarball}.md5").read().split()[0]
# NOTE: including "state" on attach above would attempt to
# advance the dataset's state, which we don't want for
# import, so we add it only here. "owner" would be ignored
# by attach, but we add it here anyway for clarity.
a["state"] = state
a["owner"] = owner
dataset = Dataset.create(**a)
print(f"Imported {dataset}: {state}")
done = done + 1
except Exception as e:
# Stringify any exception and report it; then fail
logger.exception("Import of dataset {} failed", tarball)
print(f"{_NAME_}: dataset {tarball} failed with {e}", file=sys.stderr)
fail = fail + 1
print(
f"Imported {done} datasets from {link} with {fail} errors and {ignore} ignored"
)
return 1 if fail > 0 else 0
def test_construct(self):
""" Test dataset contructor
"""
ds = Dataset(owner="drb", controller="frodo", name="fio")
ds.add()
assert ds.owner == "drb"
assert ds.controller == "frodo"
assert ds.name == "fio"
assert ds.state == States.UPLOADING
assert ds.md5 is None
assert ds.created <= ds.transition
assert ds.id is not None
assert "drb|frodo|fio" == str(ds)
def test_attach_filename(self):
""" Test that we can create a dataset using the full tarball
file path.
"""
ds1 = Dataset(owner="webb",
path="/foo/bilbo/rover.tar.xz",
state=States.QUARANTINED)
ds1.add()
ds2 = Dataset.attach(controller="bilbo", name="rover")
assert ds2.owner == ds1.owner
assert ds2.controller == ds1.controller
assert ds2.name == ds1.name
assert ds2.state == States.QUARANTINED
assert ds2.md5 is ds1.md5
assert ds2.id is ds1.id
def test_upload(client, pytestconfig, caplog, server_config):
filename = "log.tar.xz"
datafile = Path("./lib/pbench/test/unit/server/fixtures/upload/", filename)
controller = socket.gethostname()
with open(f"{datafile}.md5") as md5sum_check:
md5sum = md5sum_check.read()
with open(datafile, "rb") as data_fp:
response = client.put(
f"{server_config.rest_uri}/upload/ctrl/{controller}",
data=data_fp,
headers={"filename": filename, "Content-MD5": md5sum},
)
assert response.status_code == 201, repr(response)
sfilename = secure_filename(filename)
tmp_d = pytestconfig.cache.get("TMP", None)
receive_dir = os.path.join(
tmp_d, "srv", "pbench", "pbench-move-results-receive", "fs-version-002"
)
assert os.path.exists(receive_dir), (
f"receive_dir = '{receive_dir}', filename = '{filename}',"
f" sfilename = '{sfilename}'"
)
dataset = Dataset.attach(controller=controller, path=filename)
assert dataset is not None
assert dataset.md5 == md5sum
assert dataset.controller == controller
assert dataset.name == "log"
assert dataset.state == States.UPLOADED
for record in caplog.records:
assert record.levelname not in ("WARNING", "ERROR", "CRITICAL")
def test_construct_bad_state(self):
"""Test with a non-States state value
"""
with pytest.raises(DatasetBadParameterType):
Dataset(owner="me",
controller="potter",
name="harry",
state="notStates")
def test_attach_exists(self):
""" Test that we can attach to a dataset
"""
ds1 = Dataset(owner="drb",
controller="frodo",
name="fido",
state=States.INDEXING)
ds1.add()
ds2 = Dataset.attach(controller="frodo",
name="fido",
state=States.INDEXED)
assert ds2.owner == ds1.owner
assert ds2.controller == ds1.controller
assert ds2.name == ds1.name
assert ds2.state == States.INDEXED
assert ds2.md5 is ds1.md5
assert ds2.id is ds1.id
def test_attach_controller_path(self):
""" Test that we can attach using controller and name to a
dataset created by file path.
"""
ds1 = Dataset(
owner="pete",
path="/foo/frodo/rover.tar.xz",
state=States.INDEXING,
)
ds1.add()
ds2 = Dataset.attach(controller="frodo", name="rover")
assert ds2.owner == ds1.owner
assert ds2.controller == ds1.controller
assert ds2.name == ds1.name
assert ds2.state == States.INDEXING
assert ds2.md5 is ds1.md5
assert ds2.id is ds1.id
def test_advanced_bad_state(self):
"""Test with a non-States state value
"""
ds = Dataset(owner="test", controller="me", name="too")
ds.add()
with pytest.raises(DatasetBadParameterType):
ds.advance("notStates")
def test_advanced_illegal(self):
""" Test that we can't advance to a state that's not a
successor to the initial state.
"""
ds = Dataset(owner="drb", controller="sam", name="fio")
ds.add()
with pytest.raises(DatasetBadStateTransition):
ds.advance(States.EXPIRED)
def test_advanced_good(self):
""" Test advancing the state of a dataset
"""
ds = Dataset(owner="drb", controller="frodo", name="precious")
ds.add()
ds.advance(States.UPLOADED)
assert ds.state == States.UPLOADED
assert ds.created <= ds.transition
def test_advanced_terminal(self):
""" Test that we can't advance from a terminal state
"""
ds = Dataset(owner="drb",
controller="gimli",
name="fio",
state=States.EXPIRED)
ds.add()
with pytest.raises(DatasetTerminalStateViolation):
ds.advance(States.UPLOADING)
def test_metadata_remove(self):
""" Test that we can remove a Metadata key
"""
ds = Dataset.create(owner="user1",
controller="controller",
name="other")
assert ds.metadatas == []
m = Metadata(key=Metadata.ARCHIVED, value="TRUE")
m.add(ds)
assert ds.metadatas == [m]
Metadata.remove(ds, Metadata.ARCHIVED)
assert ds.metadatas == []
with pytest.raises(MetadataNotFound) as exc:
Metadata.get(ds, Metadata.ARCHIVED)
assert exc.value.dataset == ds
assert exc.value.key == Metadata.ARCHIVED
Metadata.remove(ds, Metadata.REINDEX)
assert ds.metadatas == []
def test_lifecycle(self):
""" Advance a dataset through the entire lifecycle using the state
transition dict.
"""
ds = Dataset(owner="dave", controller="bilbo", name="Fred")
ds.add()
assert ds.state == States.UPLOADING
beenthere = [ds.state]
while ds.state in Dataset.transitions:
advances = Dataset.transitions[ds.state]
for n in advances:
if n not in beenthere:
next = n
break
else:
break # avoid infinite reindex loop!
beenthere.append(next)
ds.advance(next)
assert ds.state == next
lifecycle = ",".join([s.name for s in beenthere])
assert (
lifecycle ==
"UPLOADING,UPLOADED,UNPACKING,UNPACKED,INDEXING,INDEXED,EXPIRING,EXPIRED"
)
def process_tb(self, tarballs):
"""Process Tarballs For Indexing and creates report
"tarballs" - List of tarball, it is the second value of
the tuple returned by collect_tb() """
res = 0
idxctx = self.idxctx
error_code = self.error_code
tb_deque = deque(sorted(tarballs))
# At this point, tarballs contains a list of tar balls sorted by size
# that were available as symlinks in the various 'linksrc' directories.
idxctx.logger.debug("Preparing to index {:d} tar balls", len(tb_deque))
try:
# Now that we are ready to begin the actual indexing step, ensure we
# have the proper index templates in place.
idxctx.logger.debug("update_templates [start]")
idxctx.templates.update_templates(idxctx.es)
except TemplateError as e:
res = self.emit_error(idxctx.logger.error,
"TEMPLATE_CREATION_ERROR", e)
except SigTermException:
# Re-raise a SIGTERM to avoid it being lumped in with general
# exception handling below.
raise
except Exception:
idxctx.logger.exception(
"update_templates [end]: Unexpected template"
" processing error")
res = error_code["GENERIC_ERROR"]
else:
idxctx.logger.debug("update_templates [end]")
res = error_code["OK"]
if not res.success:
# Exit early if we encounter any errors.
return res.value
report = Report(
idxctx.config,
self.name,
es=idxctx.es,
pid=idxctx.getpid(),
group_id=idxctx.getgid(),
user_id=idxctx.getuid(),
hostname=idxctx.gethostname(),
version=VERSION,
templates=idxctx.templates,
)
# We use the "start" report ID as the tracking ID for all indexed
# documents.
try:
tracking_id = report.post_status(tstos(idxctx.time()), "start")
except SigTermException:
# Re-raise a SIGTERM to avoid it being lumped in with general
# exception handling below.
raise
except Exception:
idxctx.logger.error("Failed to post initial report status")
return error_code["GENERIC_ERROR"].value
else:
idxctx.set_tracking_id(tracking_id)
with tempfile.TemporaryDirectory(prefix=f"{self.name}.",
dir=idxctx.config.TMP) as tmpdir:
idxctx.logger.debug("start processing list of tar balls")
tb_list = Path(tmpdir, f"{self.name}.{idxctx.TS}.list")
try:
with tb_list.open(mode="w") as lfp:
# Write out all the tar balls we are processing so external
# viewers can follow along from home.
for size, controller, tb in tarballs:
print(f"{size:20d} {controller} {tb}", file=lfp)
indexed = Path(tmpdir, f"{self.name}.{idxctx.TS}.indexed")
erred = Path(tmpdir, f"{self.name}.{idxctx.TS}.erred")
skipped = Path(tmpdir, f"{self.name}.{idxctx.TS}.skipped")
ie_filepath = Path(
tmpdir, f"{self.name}.{idxctx.TS}.indexing-errors.json")
# We use a list object here so that when we close over this
# variable in the handler, the list object will be closed over,
# but not its contents.
sigquit_interrupt = [False]
def sigquit_handler(*args):
sigquit_interrupt[0] = True
sighup_interrupt = [False]
def sighup_handler(*args):
sighup_interrupt[0] = True
signal.signal(signal.SIGQUIT, sigquit_handler)
signal.signal(signal.SIGHUP, sighup_handler)
count_processed_tb = 0
try:
while len(tb_deque) > 0:
size, controller, tb = tb_deque.popleft()
# Sanity check source tar ball path
linksrc_dir = Path(tb).parent
linksrc_dirname = linksrc_dir.name
count_processed_tb += 1
assert linksrc_dirname == self.linksrc, (
f"Logic bomb! tar ball "
f"path {tb} does not contain {self.linksrc}")
idxctx.logger.info("Starting {} (size {:d})", tb, size)
dataset = None
ptb = None
username = None
try:
path = os.path.realpath(tb)
try:
dataset = Dataset.attach(
path=path,
state=States.INDEXING,
)
except DatasetNotFound:
idxctx.logger.warn(
"Unable to locate Dataset {}",
path,
)
except DatasetTransitionError as e:
# TODO: This means the Dataset is known, but not in a
# state where we'd expect to be indexing it. So what do
# we do with it? (Note: this is where an audit log will
# be handy; i.e., how did we get here?) For now, just
# let it go.
idxctx.logger.warn(
"Unable to advance dataset state: {}",
str(e))
else:
username = dataset.owner
# "Open" the tar ball represented by the tar ball object
idxctx.logger.debug("open tar ball")
ptb = PbenchTarBall(
idxctx,
username,
path,
tmpdir,
Path(self.incoming, controller),
)
# Construct the generator for emitting all actions. The
# `idxctx` dictionary is passed along to each generator so
# that it can add its context for error handling to the
# list.
idxctx.logger.debug("generator setup")
if self.options.index_tool_data:
actions = ptb.mk_tool_data_actions()
else:
actions = ptb.make_all_actions()
# File name for containing all indexing errors that
# can't/won't be retried.
with ie_filepath.open(mode="w") as fp:
idxctx.logger.debug("begin indexing")
try:
signal.signal(signal.SIGINT,
sigint_handler)
es_res = es_index(
idxctx.es,
actions,
fp,
idxctx.logger,
idxctx._dbg,
)
except SigIntException:
idxctx.logger.exception(
"Indexing interrupted by SIGINT, continuing to next tarball"
)
continue
finally:
# Turn off the SIGINT handler when not indexing.
signal.signal(signal.SIGINT,
signal.SIG_IGN)
except UnsupportedTarballFormat as e:
tb_res = self.emit_error(idxctx.logger.warning,
"TB_META_ABSENT", e)
except BadDate as e:
tb_res = self.emit_error(idxctx.logger.warning,
"BAD_DATE", e)
except FileNotFoundError as e:
tb_res = self.emit_error(idxctx.logger.warning,
"FILE_NOT_FOUND_ERROR", e)
except BadMDLogFormat as e:
tb_res = self.emit_error(idxctx.logger.warning,
"BAD_METADATA", e)
except SigTermException:
idxctx.logger.exception(
"Indexing interrupted by SIGTERM, terminating")
break
except Exception as e:
tb_res = self.emit_error(idxctx.logger.exception,
"GENERIC_ERROR", e)
else:
beg, end, successes, duplicates, failures, retries = es_res
idxctx.logger.info(
"done indexing (start ts: {}, end ts: {}, duration:"
" {:.2f}s, successes: {:d}, duplicates: {:d},"
" failures: {:d}, retries: {:d})",
tstos(beg),
tstos(end),
end - beg,
successes,
duplicates,
failures,
retries,
)
tb_res = error_code[
"OP_ERROR" if failures > 0 else "OK"]
finally:
if dataset:
try:
dataset.advance(
States.INDEXED if tb_res.
success else States.QUARANTINED)
Metadata.remove(dataset, Metadata.REINDEX)
except DatasetTransitionError:
idxctx.logger.exception(
"Dataset state error")
try:
ie_len = ie_filepath.stat().st_size
except FileNotFoundError:
# Above operation never made it to actual indexing, ignore.
pass
except SigTermException:
# Re-raise a SIGTERM to avoid it being lumped in with
# general exception handling below.
raise
except Exception:
idxctx.logger.exception(
"Unexpected error handling"
" indexing errors file: {}",
ie_filepath,
)
else:
# Success fetching indexing error file size.
if ie_len > len(tb) + 1:
try:
report.post_status(tstos(end), "errors",
ie_filepath)
except Exception:
idxctx.logger.exception(
"Unexpected error issuing"
" report status with errors: {}",
ie_filepath,
)
finally:
# Unconditionally remove the indexing errors file.
try:
os.remove(ie_filepath)
except SigTermException:
# Re-raise a SIGTERM to avoid it being lumped in with
# general exception handling below.
raise
except Exception:
pass
# Distinguish failure cases, so we can retry the indexing
# easily if possible. Different `linkerrdest` directories for
# different failures; the rest are going to end up in
# `linkerrdest` for later retry.
controller_path = linksrc_dir.parent
if tb_res is error_code["OK"]:
idxctx.logger.info(
"{}: {}/{}: success",
idxctx.TS,
controller_path.name,
os.path.basename(tb),
)
# Success
with indexed.open(mode="a") as fp:
print(tb, file=fp)
rename_tb_link(
tb, Path(controller_path, self.linkdest),
idxctx.logger)
elif tb_res is error_code["OP_ERROR"]:
idxctx.logger.warning(
"{}: index failures encountered on {}",
idxctx.TS, tb)
with erred.open(mode="a") as fp:
print(tb, file=fp)
rename_tb_link(
tb,
Path(controller_path, f"{self.linkerrdest}.1"),
idxctx.logger,
)
elif tb_res in (error_code["CFG_ERROR"],
error_code["BAD_CFG"]):
assert False, (
f"Logic Bomb! Unexpected tar ball handling "
f"result status {tb_res.value:d} for tar ball {tb}"
)
elif tb_res.tarball_error:
# # Quietly skip these errors
with skipped.open(mode="a") as fp:
print(tb, file=fp)
rename_tb_link(
tb,
Path(
controller_path,
f"{self.linkerrdest}.{tb_res.value:d}",
),
idxctx.logger,
)
else:
idxctx.logger.error(
"{}: index error {:d} encountered on {}",
idxctx.TS,
tb_res.value,
tb,
)
with erred.open(mode="a") as fp:
print(tb, file=fp)
rename_tb_link(
tb,
Path(controller_path, self.linkerrdest),
idxctx.logger,
)
idxctx.logger.info(
"Finished{} {} (size {:d})",
"[SIGQUIT]" if sigquit_interrupt[0] else "",
tb,
size,
)
if sigquit_interrupt[0]:
break
if sighup_interrupt[0]:
status, new_tb = self.collect_tb()
if status == 0:
if not set(new_tb).issuperset(tb_deque):
idxctx.logger.info(
"Tarballs supposed to be in 'TO-INDEX' are no longer present",
set(tb_deque).difference(new_tb),
)
tb_deque = deque(sorted(new_tb))
idxctx.logger.info(
"SIGHUP status (Current tar ball indexed: ({}), Remaining: {}, Completed: {}, Errors_encountered: {}, Status: {})",
Path(tb).name,
len(tb_deque),
count_processed_tb,
_count_lines(erred),
tb_res,
)
sighup_interrupt[0] = False
continue
except SigTermException:
idxctx.logger.exception(
"Indexing interrupted by SIGQUIT, stop processing tarballs"
)
finally:
# Turn off the SIGQUIT and SIGHUP handler when not indexing.
signal.signal(signal.SIGQUIT, signal.SIG_IGN)
signal.signal(signal.SIGHUP, signal.SIG_IGN)
except SigTermException:
# Re-raise a SIGTERM to avoid it being lumped in with general
# exception handling below.
raise
except Exception:
idxctx.logger.exception(error_code["GENERIC_ERROR"].message)
res = error_code["GENERIC_ERROR"]
else:
# No exceptions while processing tar ball, success.
res = error_code["OK"]
finally:
if idxctx:
idxctx.dump_opctx()
idxctx.logger.debug("stopped processing list of tar balls")
idx = _count_lines(indexed)
skp = _count_lines(skipped)
err = _count_lines(erred)
idxctx.logger.info(
"{}.{}: indexed {:d} (skipped {:d}) results,"
" {:d} errors",
self.name,
idxctx.TS,
idx,
skp,
err,
)
if err > 0:
if skp > 0:
subj = (
f"{self.name}.{idxctx.TS} - Indexed {idx:d} results, skipped {skp:d}"
f" results, w/ {err:d} errors")
else:
subj = (
f"{self.name}.{idxctx.TS} - Indexed {idx:d} results, w/ {err:d}"
" errors")
else:
if skp > 0:
subj = f"{self.name}.{idxctx.TS} - Indexed {idx:d} results, skipped {skp:d} results"
else:
subj = f"{self.name}.{idxctx.TS} - Indexed {idx:d} results"
report_fname = Path(tmpdir, f"{self.name}.{idxctx.TS}.report")
with report_fname.open(mode="w") as fp:
print(subj, file=fp)
if idx > 0:
print("\nIndexed Results\n===============", file=fp)
with indexed.open() as ifp:
for line in sorted(ifp):
print(line.strip(), file=fp)
if err > 0:
print(
"\nResults producing errors"
"\n========================",
file=fp,
)
with erred.open() as efp:
for line in sorted(efp):
print(line.strip(), file=fp)
if skp > 0:
print("\nSkipped Results\n===============", file=fp)
with skipped.open() as sfp:
for line in sorted(sfp):
print(line.strip(), file=fp)
try:
report.post_status(tstos(idxctx.time()), "status",
report_fname)
except SigTermException:
# Re-raise a SIGTERM to avoid it being lumped in with general
# exception handling below.
raise
except Exception:
pass
return res.value
def process_tb(config, logger, receive_dir, qdir_md5, duplicates, errors):
# Check for results that are ready for processing: version 002 agents
# upload the MD5 file as xxx.md5.check and they rename it to xxx.md5
# after they are done with MD5 checking so that's what we look for.
list_check = glob.glob(os.path.join(receive_dir, "**", "*.tar.xz.md5"),
recursive=True)
archive = config.ARCHIVE
logger.info("{}", config.TS)
list_check.sort()
nstatus = ""
ntotal = ntbs = nerrs = nquarantined = ndups = 0
for tbmd5 in list_check:
ntotal += 1
# full pathname of tarball
tb = Path(tbmd5[0:-4])
tbmd5 = Path(tbmd5)
# directory
tbdir = tb.parent
# resultname: get the basename foo.tar.xz and then strip the .tar.xz
resultname = tb.name
controller = tbdir.name
dest = archive / controller
# Create a new dataset tracker in UPLOADING state, and add it to the
# database.
#
# NOTE: Technically, this particular workflow has no "UPLOADING" as
# the `pbench-server-prep-shim-002` command isn't invoked until the
# tarball and MD5 has been entirely uploaded by the agent via `ssh`;
# this method however can't be supported once we have authorized user
# ownership, and the model fits the server `PUT` method where an
# unexpected termination could leave a tarball in "Uploading" state.
#
# TODO: We have no way to identify an owner here, so assign it to
# the arbitrary "pbench" user. This will go away when we drop this
# component entirely in favor of PUT.
try:
dataset = Dataset.create(controller=controller,
path=resultname,
owner="pbench")
except DatasetError as e:
logger.error("Unable to create dataset {}>{}: {}", controller,
resultname, str(e))
# TODO: Should we quarantine over this? Note it's not quite
# straightforward, as quarantine() expects that the Dataset has
# been created, so we'll get a cascade failure. Since prep-shim's
# days are numbered, I'm inclined not to worry about it here.
dataset = None
if all([(dest / resultname).is_file(), (dest / tbmd5.name).is_file()]):
logger.error("{}: Duplicate: {} duplicate name", config.TS, tb)
quarantine((duplicates / controller), logger, tb, tbmd5)
ndups += 1
continue
archive_tar_hex_value, archive_md5_hex_value = md5_check(
tb, tbmd5, logger)
if any([
archive_tar_hex_value != archive_md5_hex_value,
archive_tar_hex_value is None,
archive_md5_hex_value is None,
]):
logger.error("{}: Quarantined: {} failed MD5 check", config.TS, tb)
logger.info("{}: FAILED", tb.name)
logger.info("md5sum: WARNING: 1 computed checksum did NOT match")
quarantine((qdir_md5 / controller), logger, tb, tbmd5)
nquarantined += 1
continue
if dataset:
try:
dataset.md5 = archive_md5_hex_value
dataset.update()
except DatasetError as e:
logger.warn("Unable to update dataset {} with md5: {}",
str(dataset), str(e))
# make the destination directory and its TODO subdir if necessary.
try:
os.makedirs(dest / "TODO")
except FileExistsError:
# directory already exists, ignore
pass
except Exception:
logger.error("{}: Error in creating TODO directory.", config.TS)
quarantine(os.path.join(errors, controller), logger, tb, tbmd5)
nerrs += 1
continue
# First, copy the small .md5 file to the destination. That way, if
# that operation fails it will fail quickly since the file is small.
try:
shutil.copy2(tbmd5, dest)
except Exception:
logger.error("{}: Error in copying .md5 file to Destination path.",
config.TS)
try:
os.remove(dest / tbmd5.name)
except FileNotFoundError:
logger.error(
"{}: Warning: cleanup of copy failure failed itself.",
config.TS)
quarantine((errors / controller), logger, tb, tbmd5)
nerrs += 1
continue
# Next, mv the "large" tar ball to the destination. If the destination
# is on the same device, the move should be quick. If the destination is
# on a different device, the move will be a copy and delete, and will
# take a bit longer. If it fails, the file will NOT be at the
# destination.
try:
shutil.move(str(tb), str(dest))
except Exception:
logger.error(
"{}: Error in moving tarball file to Destination path.",
config.TS)
try:
os.remove(dest / resultname)
except FileNotFoundError:
logger.error(
"{}: Warning: cleanup of copy failure failed itself.",
config.TS)
quarantine((errors / controller), logger, tb, tbmd5)
nerrs += 1
continue
# Restore the SELinux context properly
try:
selinux.restorecon(dest / tb.name)
selinux.restorecon(dest / tbmd5.name)
except Exception as e:
# log it but do not abort
logger.error("{}: Error: 'restorecon {}', {}", config.TS,
dest / tb.name, e)
# Now that we have successfully moved the tar ball and its .md5 to the
# destination, we can remove the original .md5 file.
try:
os.remove(tbmd5)
except Exception as exc:
logger.error(
"{}: Warning: cleanup of successful copy operation failed: '{}'",
config.TS,
exc,
)
try:
os.symlink((dest / resultname), (dest / "TODO" / resultname))
except Exception as exc:
logger.error("{}: Error in creation of symlink. '{}'", config.TS,
exc)
# if we fail to make the link, we quarantine the (already moved)
# tarball and .md5.
quarantine(
(errors / controller),
logger,
(dest / tb),
(dest / tbmd5),
)
nerrs += 1
continue
ntbs += 1
try:
if dataset:
dataset.advance(States.UPLOADED)
except Exception:
logger.exception("Unable to finalize {}", dataset)
nstatus = f"{nstatus}{config.TS}: processed {tb}\n"
logger.info(f"{tb.name}: OK")
return Results(
nstatus=nstatus,
ntotal=ntotal,
ntbs=ntbs,
nquarantined=nquarantined,
ndups=ndups,
nerrs=nerrs,
)
def put(self, controller):
# FIXME: This should be assigned from the decoded authorization token
username = "******"
if not request.headers.get("filename"):
self.logger.debug(
"Tarfile upload: Post operation failed due to missing filename header"
)
abort(
400,
message=
"Missing filename header, POST operation requires a filename header to name the uploaded file",
)
filename = secure_filename(request.headers.get("filename"))
if not request.headers.get("Content-MD5"):
self.logger.debug(
f"Tarfile upload: Post operation failed due to missing md5sum header for file {filename}"
)
abort(
400,
message=
"Missing md5sum header, POST operation requires md5sum of an uploaded file in header",
)
md5sum = request.headers.get("Content-MD5")
self.logger.debug("Receiving file: {}", filename)
if not self.allowed_file(filename):
self.logger.debug(
f"Tarfile upload: Bad file extension received for file {filename}"
)
abort(400, message="File extension not supported. Only .xz")
try:
content_length = int(request.headers.get("Content-Length"))
except ValueError:
self.logger.debug(
f"Tarfile upload: Invalid content-length header, not an integer for file {filename}"
)
abort(400, message="Invalid content-length header, not an integer")
except Exception:
self.logger.debug(
f"Tarfile upload: No Content-Length header value found for file {filename}"
)
abort(400, message="Missing required content-length header")
else:
if content_length > self.max_content_length:
self.logger.debug(
f"Tarfile upload: Content-Length exceeded maximum upload size allowed. File: {filename}"
)
abort(
400,
message=
f"Payload body too large, {content_length:d} bytes, maximum size should be less than "
f"or equal to {humanize.naturalsize(self.max_content_length)}",
)
elif content_length == 0:
self.logger.debug(
f"Tarfile upload: Content-Length header value is 0 for file {filename}"
)
abort(
400,
message=
"Upload failed, Content-Length received in header is 0",
)
path = self.upload_directory / controller
path.mkdir(exist_ok=True)
tar_full_path = Path(path, filename)
md5_full_path = Path(path, f"{filename}.md5")
bytes_received = 0
# TODO: Need real user from PUT!
# Create a tracking dataset object; it'll begin in UPLOADING state
try:
dataset = Dataset(owner=username,
controller=controller,
path=tar_full_path,
md5=md5sum)
dataset.add()
except Exception:
self.logger.exception("unable to create dataset for {}", filename)
abort(
HTTPStatus.INTERNAL_SERVER_ERROR,
message="INTERNAL ERROR",
)
self.logger.info("Uploading file {} to {}", filename, dataset)
with tempfile.NamedTemporaryFile(mode="wb", dir=path) as ofp:
chunk_size = 4096
self.logger.debug("Writing chunks")
hash_md5 = hashlib.md5()
try:
while True:
chunk = request.stream.read(chunk_size)
bytes_received += len(chunk)
if len(chunk) == 0 or bytes_received > content_length:
break
ofp.write(chunk)
hash_md5.update(chunk)
except Exception:
self.logger.exception(
"Tarfile upload: There was something wrong uploading {}",
filename)
abort(
500,
message=f"There was something wrong uploading {filename}")
if bytes_received != content_length:
self.logger.debug(
f"Tarfile upload: Bytes received does not match with content length header value for file {filename}"
)
message = (
f"Bytes received ({bytes_received}) does not match with content length header"
f" ({content_length}), upload failed")
abort(400, message=message)
elif hash_md5.hexdigest() != md5sum:
self.logger.debug(
f"Tarfile upload: md5sum check failed for file {filename}")
message = f"md5sum check failed for {filename}, upload failed"
abort(400, message=message)
# First write the .md5
try:
with md5_full_path.open("w") as md5fp:
md5fp.write(f"{md5sum} {filename}\n")
except Exception:
try:
os.remove(md5_full_path)
except FileNotFoundError:
pass
except Exception as exc:
self.logger.warning(
"Failed to remove .md5 %s when trying to clean up: %s",
md5_full_path,
exc,
)
self.logger.exception("Failed to write .md5 file, '%s'",
md5_full_path)
raise
# Then create the final filename link to the temporary file.
try:
os.link(ofp.name, tar_full_path)
except Exception:
try:
os.remove(md5_full_path)
except Exception as exc:
self.logger.warning(
"Failed to remove .md5 %s when trying to clean up: %s",
md5_full_path,
exc,
)
self.logger.exception(
"Failed to rename tar ball '%s' to '%s'",
ofp.name,
md5_full_path,
)
raise
try:
dataset.advance(States.UPLOADED)
except Exception:
self.logger.exception("Unable to finalize {}", dataset)
abort(HTTPStatus.INTERNAL_SERVER_ERROR, message="INTERNAL ERROR")
response = jsonify(dict(message="File successfully uploaded"))
response.status_code = 201
return response
def backup_data(lb_obj, s3_obj, config, logger):
qdir = config.QDIR
tarlist = glob.iglob(
os.path.join(config.ARCHIVE, "*", _linksrc, "*.tar.xz"))
ntotal = nbackup_success = nbackup_fail = ns3_success = ns3_fail = nquaran = 0
for tb in sorted(tarlist):
ntotal += 1
# resolve the link
try:
tar = Path(tb).resolve(strict=True)
except FileNotFoundError:
logger.error(
"Tarball link, '{}', does not resolve to a real location", tb)
logger.debug("Start backup of {}.", tar)
# check tarball exist and it is a regular file
if tar.exists() and tar.is_file():
pass
else:
# tarball does not exist or it is not a regular file
quarantine(qdir, logger, tb)
nquaran += 1
logger.error(
"Quarantine: {}, {} does not exist or it is not a regular file",
tb,
tar,
)
continue
archive_md5 = Path(f"{tar}.md5")
# check that the md5 file exists and it is a regular file
if archive_md5.exists() and archive_md5.is_file():
pass
else:
# md5 file does not exist or it is not a regular file
quarantine(qdir, logger, tb)
nquaran += 1
logger.error(
"Quarantine: {}, {} does not exist or it is not a regular file",
tb,
archive_md5,
)
continue
# read the md5sum from md5 file
try:
with archive_md5.open() as f:
archive_md5_hex_value = f.readline().split(" ")[0]
except Exception:
# Could not read file.
quarantine(qdir, logger, tb)
nquaran += 1
logger.exception("Quarantine: {}, Could not read {}", tb,
archive_md5)
continue
# match md5sum of the tarball to its md5 file
try:
archive_tar_hex_value = md5sum(tar)
except Exception:
# Could not read file.
quarantine(qdir, logger, tb)
nquaran += 1
logger.exception("Quarantine: {}, Could not read {}", tb, tar)
continue
if archive_tar_hex_value != archive_md5_hex_value:
quarantine(qdir, logger, tb)
nquaran += 1
logger.error(
"Quarantine: {}, md5sum of {} does not match with its md5 file {}",
tb,
tar,
archive_md5,
)
continue
resultname = tar.name
controller_path = tar.parent
controller = controller_path.name
try:
# This tool can't see a dataset until it's been prepared either
# by server PUT or by pbench-server-prep-shim-002.py; in either
# case, the Dataset object must already exist.
dataset = Dataset.attach(controller=controller, path=resultname)
except DatasetError as e:
logger.warning("Trouble tracking {}:{}: {}", controller,
resultname, str(e))
dataset = None
# This will handle all the local backup related
# operations and count the number of successes and failures.
local_backup_result = backup_to_local(
lb_obj,
logger,
controller_path,
controller,
tb,
tar,
resultname,
archive_md5,
archive_md5_hex_value,
)
if local_backup_result == Status.SUCCESS:
nbackup_success += 1
elif local_backup_result == Status.FAIL:
nbackup_fail += 1
else:
assert (
False
), f"Impossible situation, local_backup_result = {local_backup_result!r}"
# This will handle all the S3 bucket related operations
# and count the number of successes and failures.
s3_backup_result = backup_to_s3(
s3_obj,
logger,
controller_path,
controller,
tb,
tar,
resultname,
archive_md5_hex_value,
)
if s3_backup_result == Status.SUCCESS:
ns3_success += 1
elif s3_backup_result == Status.FAIL:
ns3_fail += 1
else:
assert (
False
), f"Impossible situation, s3_backup_result = {s3_backup_result!r}"
if local_backup_result == Status.SUCCESS and (
s3_obj is None or s3_backup_result == Status.SUCCESS):
# Move tar ball symlink to its final resting place
rename_tb_link(tb, Path(controller_path, _linkdest), logger)
else:
# Do nothing when the backup fails, allowing us to retry on a
# future pass.
pass
if dataset:
Metadata.create(dataset=dataset,
key=Metadata.ARCHIVED,
value="True")
logger.debug("End backup of {}.", tar)
return Results(
ntotal=ntotal,
nbackup_success=nbackup_success,
nbackup_fail=nbackup_fail,
ns3_success=ns3_success,
ns3_fail=ns3_fail,
nquaran=nquaran,
)
def reindex(controller_name, tb_name, archive_p, incoming_p, dry_run=False):
"""reindex - re-index the given tar ball name.
This method is responsible for finding the current symlink to the tar ball
and moving it to the TO-RE-INDEX directory, creating that directory if
it does not exist.
"""
assert tb_name.endswith(".tar.xz"), f"invalid tar ball name, '{tb_name}'"
if not (incoming_p / controller_name / tb_name[:-7]).exists():
# Can't re-index tar balls that are not unpacked
return (controller_name, tb_name, "not-unpacked", "")
# Construct the controller path object used throughout the rest of this
# method.
controller_p = archive_p / controller_name
# Construct the target path to which all tar ball symlinks will be moved.
newpath = controller_p.joinpath("TO-RE-INDEX", tb_name)
paths = []
_linkdirs = ("TO-INDEX-TOOL", "INDEXED")
for linkname_p in controller_p.glob(f"*/{tb_name}"):
# Consider all existing tar ball symlinks
if linkname_p.parent.name in ("TO-INDEX", "TO-RE-INDEX"):
msg = (
f"WARNING: {linkname_p.parent.name} link already exists for"
f" {controller_p / tb_name}"
)
# Indicate no action taken, and exit early.
return (controller_name, tb_name, "exists", msg)
elif linkname_p.parent.name in _linkdirs:
# One of the expected symlinks, add it for consideration below.
paths.append(linkname_p)
elif linkname_p.parent.name.startswith("WONT-INDEX"):
# One of the expected WONT-INDEX* symlinks, also added for
# consideration below.
paths.append(linkname_p)
# else:
# All other symlinks are not considered.
if not paths:
# No existing TO-INDEX or TO-RE-INDEX symlink, and no previous
# indexing symlinks, exit early.
return (controller_name, tb_name, "noop", "")
if len(paths) > 1:
# If we have more than one path then just flag this as a bad state
# and exit early.
return (controller_name, tb_name, "badstate", "")
# At this point we are guaranteed to have only one path.
assert len(paths) == 1, f"Logic bomb! len(paths) ({len(paths)}) != 1"
try:
if not dry_run:
paths[0].rename(newpath)
ds = Dataset.attach(controller=controller_name, path=tb_name)
Metadata.create(dataset=ds, key=Metadata.REINDEX, value="True")
except DatasetError as exc:
msg = f"WARNING: unable to set REINDEX metadata for {controller_name}:{tb_name}: {str(exc)}"
res = "error"
except Exception as exc:
msg = (
f"WARNING: failed to rename symlink '{paths[0]}' to"
f" '{newpath}', '{exc}'"
)
res = "error"
else:
msg = ""
res = "succ"
return (controller_name, tb_name, res, msg)
def test_metadata(self):
""" Various tests on Metadata keys
"""
# See if we can create a metadata row
ds = Dataset.create(owner="redhat",
controller="controller",
name="name")
assert ds.metadatas == []
m = Metadata.create(key=Metadata.REINDEX, value="TRUE", dataset=ds)
assert m is not None
assert ds.metadatas == [m]
# Try to get it back
m1 = Metadata.get(ds, Metadata.REINDEX)
assert m1.key == m.key
assert m1.value == m.value
assert m.id == m1.id
assert m.dataset_ref == m1.dataset_ref
# Check the str()
assert "redhat|controller|name>>REINDEX" == str(m)
# Try to get a metadata key that doesn't exist
with pytest.raises(MetadataNotFound) as exc:
Metadata.get(ds, Metadata.TARBALL_PATH)
assert exc.value.dataset == ds
assert exc.value.key == Metadata.TARBALL_PATH
# Try to remove a metadata key that doesn't exist (No-op)
Metadata.remove(ds, Metadata.TARBALL_PATH)
# Try to create a metadata with a bad key
badkey = "THISISNOTTHEKEYYOURELOOKINGFOR"
with pytest.raises(MetadataBadKey) as exc:
Metadata(key=badkey, value=None)
assert exc.value.key == badkey
# Try to create a key without a value
with pytest.raises(MetadataMissingKeyValue):
Metadata(key=Metadata.REINDEX)
# Try to add a duplicate metadata key
with pytest.raises(MetadataDuplicateKey) as exc:
m1 = Metadata(key=Metadata.REINDEX, value="IRRELEVANT")
m1.add(ds)
assert exc.value.key == Metadata.REINDEX
assert exc.value.dataset == ds
assert ds.metadatas == [m]
# Try to add a Metadata key to something that's not a dataset
with pytest.raises(DatasetBadParameterType) as exc:
m1 = Metadata(key=Metadata.TARBALL_PATH, value="DONTCARE")
m1.add("foobar")
assert exc.value.bad_value == "foobar"
assert exc.value.expected_type == Dataset
# Try to create a Metadata with a bad value for the dataset
with pytest.raises(DatasetBadParameterType) as exc:
m1 = Metadata.create(key=Metadata.REINDEX,
value="TRUE",
dataset=[ds])
assert exc.value.bad_value == [ds]
assert exc.value.expected_type == Dataset
# Try to update the metadata key
m.value = "False"
m.update()
m1 = Metadata.get(ds, Metadata.REINDEX)
assert m.id == m1.id
assert m.dataset_ref == m1.dataset_ref
assert m.key == m1.key
assert m.value == "False"
# Delete the key and make sure its gone
m.delete()
with pytest.raises(MetadataNotFound) as exc:
Metadata.get(ds, Metadata.REINDEX)
assert exc.value.dataset == ds
assert exc.value.key == Metadata.REINDEX
assert ds.metadatas == []