def _verify_entry(self, entry, tx_check_only):
""" Check that the entry (file/dir) has the proper meta data.
Args:
entry (list): Entry from the arhive file containing all info about
this particular file/directory.
tx_check_only (boolean): If True then for files only check their
existence, size and checksum values.
Raises:
CheckEntryException: if entry verification fails.
"""
self.logger.debug("Verify entry={0}".format(entry))
is_dir, path = (entry[0] == 'd'), entry[1]
__, dst = self.get_endpoints(path)
url = client.URL(dst.encode("utf-8"))
if self.d2t: # for PUT check entry size and checksum if possible
fs = self.get_fs(dst)
st, stat_info = fs.stat(url.path.encode("utf-8"))
if not st.ok:
err_msg = "Entry={0} failed stat".format(dst)
self.logger.error(err_msg)
raise CheckEntryException("failed stat")
if not is_dir: # check file size match
indx = self.header["file_meta"].index("size") + 2
orig_size = int(entry[indx])
if stat_info.size != orig_size:
err_msg = ("Verify entry={0}, expect_size={1}, size={2}"
"").format(dst, orig_size, stat_info.size)
self.logger.error(err_msg)
raise CheckEntryException("failed file size match")
# Check checksum only if it is adler32 - only one supported by CASTOR
indx = self.header["file_meta"].index("xstype") + 2
# !!!HACK!!! Check the checksum only if file size is not 0 since
# CASTOR does not store any checksum for 0 size files
if stat_info.size != 0 and entry[indx] == "adler":
indx = self.header["file_meta"].index("xs") + 2
xs = entry[indx]
st, xs_resp = fs.query(QueryCode.CHECKSUM, url.path)
if not st.ok:
err_msg = "Entry={0} failed xs query".format(dst)
self.logger.error(err_msg)
raise CheckEntryException("failed xs query")
# Result has an annoying \x00 character at the end and it
# contains the xs type (adler32) and the xs value
resp = xs_resp.split('\x00')[0].split()
# If checksum value is not 8 char long then we need padding
if len(resp[1]) != 8:
resp[1] = "{0:0>8}".format(resp[1])
if resp[0] == "adler32" and resp[1] != xs:
err_msg = (
"Entry={0} xs value missmatch xs_expected={1} "
"xs_got={2}").format(dst, xs, resp[1])
self.logger.error(err_msg)
raise CheckEntryException("xs value missmatch")
else: # for GET check all metadata
if is_dir:
tags = self.header['dir_meta']
else:
tags = self.header['file_meta']
try:
if self.header['twindow_type'] and self.header[
'twindow_val']:
dfile = dict(zip(tags, entry[2:]))
twindow_sec = int(self.header['twindow_val'])
tentry_sec = int(
float(dfile[self.header['twindow_type']]))
if tentry_sec < twindow_sec:
# No check for this entry
return
# This is a backup so don't check atomic version files
if is_atomic_version_file(entry[1]):
return
except KeyError as __:
# This is not a backup transfer but an archive one, carry on
pass
try:
meta_info = get_entry_info(url, path, tags, is_dir)
except (AttributeError, IOError, KeyError) as __:
self.logger.error(
"Failed getting metainfo entry={0}".format(dst))
raise CheckEntryException("failed getting metainfo")
# Check if we have any excluded xattrs
try:
excl_xattr = self.header['excl_xattr']
except KeyError as __:
excl_xattr = list()
if is_dir and excl_xattr:
# For directories and configurations containing excluded xattrs
# we refine the checks. If "*" in excl_xattr then no check is done.
if "*" not in excl_xattr:
ref_dict = dict(zip(tags, entry[2:]))
new_dict = dict(zip(tags, meta_info[2:]))
for key, val in ref_dict.iteritems():
if not isinstance(val, dict):
if new_dict[key] != val:
err_msg = (
"Verify failed for entry={0} expect={1} got={2}"
" at key={3}").format(
dst, entry, meta_info, key)
self.logger.error(err_msg)
raise CheckEntryException(
"failed metainfo match")
else:
for kxattr, vxattr in val.iteritems():
if kxattr not in excl_xattr:
if vxattr != new_dict[key][kxattr]:
err_msg = (
"Verify failed for entry={0} expect={1} got={2}"
" at xattr key={3}").format(
dst, entry, meta_info, kxattr)
self.logger.error(err_msg)
raise CheckEntryException(
"failed metainfo match")
else:
# For files with tx_check_only verification, we refine the checks
if tx_check_only and not is_dir:
idx_size = self.header["file_meta"].index("size") + 2
idx_xstype = self.header["file_meta"].index("xstype") + 2
idx_xsval = self.header["file_meta"].index("xs") + 2
if (meta_info[idx_size] != entry[idx_size]
or meta_info[idx_xstype] != entry[idx_xstype]
or meta_info[idx_xsval] != entry[idx_xsval]):
err_msg = (
"Partial verify failed for entry={0} expect={1} got={2}"
"").format(dst, entry, meta_info)
self.logger.error(err_msg)
raise CheckEntryException(
"failed metainfo partial match")
else:
if not meta_info == entry:
err_msg = (
"Verify failed for entry={0} expect={1} got={2}"
"").format(dst, entry, meta_info)
self.logger.error(err_msg)
raise CheckEntryException("failed metainfo match")
self.logger.info("Entry={0}, status={1}".format(dst, True))
def _verify_entry(self, entry, tx_check_only):
""" Check that the entry (file/dir) has the proper meta data.
Args:
entry (list): Entry from the arhive file containing all info about
this particular file/directory.
tx_check_only (boolean): If True then for files only check their
existence, size and checksum values.
Raises:
CheckEntryException: if entry verification fails.
"""
self.logger.debug("Verify entry={0}".format(entry))
is_dir, path = (entry[0] == 'd'), entry[1]
__, dst = self.get_endpoints(path)
url = client.URL(dst.encode("utf-8"))
if self.d2t: # for PUT check entry size and checksum if possible
fs = self.get_fs(dst)
st, stat_info = fs.stat(url.path.encode("utf-8"))
if not st.ok:
err_msg = "Entry={0} failed stat".format(dst)
self.logger.error(err_msg)
raise CheckEntryException("failed stat")
if not is_dir: # check file size match
indx = self.header["file_meta"].index("size") + 2
orig_size = int(entry[indx])
if stat_info.size != orig_size:
err_msg = ("Verify entry={0}, expect_size={1}, size={2}"
"").format(dst, orig_size, stat_info.size)
self.logger.error(err_msg)
raise CheckEntryException("failed file size match")
# Check checksum only if it is adler32 - only one supported by CASTOR
indx = self.header["file_meta"].index("xstype") + 2
# !!!HACK!!! Check the checksum only if file size is not 0 since
# CASTOR does not store any checksum for 0 size files
if stat_info.size != 0 and entry[indx] == "adler":
indx = self.header["file_meta"].index("xs") + 2
xs = entry[indx]
st, xs_resp = fs.query(QueryCode.CHECKSUM, url.path)
if not st.ok:
err_msg = "Entry={0} failed xs query".format(dst)
self.logger.error(err_msg)
raise CheckEntryException("failed xs query")
# Result has an annoying \x00 character at the end and it
# contains the xs type (adler32) and the xs value
resp = xs_resp.split('\x00')[0].split()
# If checksum value is not 8 char long then we need padding
if len(resp[1]) != 8:
resp[1] = "{0:0>8}".format(resp[1])
if resp[0] == "adler32" and resp[1] != xs:
err_msg = ("Entry={0} xs value missmatch xs_expected={1} "
"xs_got={2}").format(dst, xs, resp[1])
self.logger.error(err_msg)
raise CheckEntryException("xs value missmatch")
else: # for GET check all metadata
if is_dir:
tags = self.header['dir_meta']
else:
tags = self.header['file_meta']
try:
if self.header['twindow_type'] and self.header['twindow_val']:
dfile = dict(zip(tags, entry[2:]))
twindow_sec = int(self.header['twindow_val'])
tentry_sec = int(float(dfile[self.header['twindow_type']]))
if tentry_sec < twindow_sec:
# No check for this entry
return
# This is a backup so don't check atomic version files
if is_atomic_version_file(entry[1]):
return
except KeyError as __:
# This is not a backup transfer but an archive one, carry on
pass
try:
meta_info = get_entry_info(url, path, tags, is_dir)
except (AttributeError, IOError, KeyError) as __:
self.logger.error("Failed getting metainfo entry={0}".format(dst))
raise CheckEntryException("failed getting metainfo")
# Check if we have any excluded xattrs
try:
excl_xattr = self.header['excl_xattr']
except KeyError as __:
excl_xattr = list()
if is_dir and excl_xattr:
# For directories and configurations containing excluded xattrs
# we refine the checks. If "*" in excl_xattr then no check is done.
if "*" not in excl_xattr:
ref_dict = dict(zip(tags, entry[2:]))
new_dict = dict(zip(tags, meta_info[2:]))
for key, val in ref_dict.iteritems():
if not isinstance(val, dict):
if new_dict[key] != val:
err_msg = ("Verify failed for entry={0} expect={1} got={2}"
" at key={3}").format(dst, entry, meta_info, key)
self.logger.error(err_msg)
raise CheckEntryException("failed metainfo match")
else:
for kxattr, vxattr in val.iteritems():
if kxattr not in excl_xattr:
if vxattr != new_dict[key][kxattr]:
err_msg = ("Verify failed for entry={0} expect={1} got={2}"
" at xattr key={3}").format(dst, entry, meta_info, kxattr)
self.logger.error(err_msg)
raise CheckEntryException("failed metainfo match")
else:
# For files with tx_check_only verification, we refine the checks
if tx_check_only and not is_dir:
idx_size = self.header["file_meta"].index("size") + 2
idx_xstype = self.header["file_meta"].index("xstype") + 2
idx_xsval = self.header["file_meta"].index("xs") + 2
if (meta_info[idx_size] != entry[idx_size] or
meta_info[idx_xstype] != entry[idx_xstype] or
meta_info[idx_xsval] != entry[idx_xsval]):
err_msg = ("Partial verify failed for entry={0} expect={1} got={2}"
"").format(dst, entry, meta_info)
self.logger.error(err_msg)
raise CheckEntryException("failed metainfo partial match")
else:
if not meta_info == entry:
err_msg = ("Verify failed for entry={0} expect={1} got={2}"
"").format(dst, entry, meta_info)
self.logger.error(err_msg)
raise CheckEntryException("failed metainfo match")
self.logger.info("Entry={0}, status={1}".format(dst, True))