def _verify_entry(self, entry, tx_check_only): """ Check that the entry (file/dir) has the proper meta data. Args: entry (list): Entry from the arhive file containing all info about this particular file/directory. tx_check_only (boolean): If True then for files only check their existence, size and checksum values. Raises: CheckEntryException: if entry verification fails. """ self.logger.debug("Verify entry={0}".format(entry)) is_dir, path = (entry[0] == 'd'), entry[1] __, dst = self.get_endpoints(path) url = client.URL(dst.encode("utf-8")) if self.d2t: # for PUT check entry size and checksum if possible fs = self.get_fs(dst) st, stat_info = fs.stat(url.path.encode("utf-8")) if not st.ok: err_msg = "Entry={0} failed stat".format(dst) self.logger.error(err_msg) raise CheckEntryException("failed stat") if not is_dir: # check file size match indx = self.header["file_meta"].index("size") + 2 orig_size = int(entry[indx]) if stat_info.size != orig_size: err_msg = ("Verify entry={0}, expect_size={1}, size={2}" "").format(dst, orig_size, stat_info.size) self.logger.error(err_msg) raise CheckEntryException("failed file size match") # Check checksum only if it is adler32 - only one supported by CASTOR indx = self.header["file_meta"].index("xstype") + 2 # !!!HACK!!! Check the checksum only if file size is not 0 since # CASTOR does not store any checksum for 0 size files if stat_info.size != 0 and entry[indx] == "adler": indx = self.header["file_meta"].index("xs") + 2 xs = entry[indx] st, xs_resp = fs.query(QueryCode.CHECKSUM, url.path) if not st.ok: err_msg = "Entry={0} failed xs query".format(dst) self.logger.error(err_msg) raise CheckEntryException("failed xs query") # Result has an annoying \x00 character at the end and it # contains the xs type (adler32) and the xs value resp = xs_resp.split('\x00')[0].split() # If checksum value is not 8 char long then we need padding if len(resp[1]) != 8: resp[1] = "{0:0>8}".format(resp[1]) if resp[0] == "adler32" and resp[1] != xs: err_msg = ( "Entry={0} xs value missmatch xs_expected={1} " "xs_got={2}").format(dst, xs, resp[1]) self.logger.error(err_msg) raise CheckEntryException("xs value missmatch") else: # for GET check all metadata if is_dir: tags = self.header['dir_meta'] else: tags = self.header['file_meta'] try: if self.header['twindow_type'] and self.header[ 'twindow_val']: dfile = dict(zip(tags, entry[2:])) twindow_sec = int(self.header['twindow_val']) tentry_sec = int( float(dfile[self.header['twindow_type']])) if tentry_sec < twindow_sec: # No check for this entry return # This is a backup so don't check atomic version files if is_atomic_version_file(entry[1]): return except KeyError as __: # This is not a backup transfer but an archive one, carry on pass try: meta_info = get_entry_info(url, path, tags, is_dir) except (AttributeError, IOError, KeyError) as __: self.logger.error( "Failed getting metainfo entry={0}".format(dst)) raise CheckEntryException("failed getting metainfo") # Check if we have any excluded xattrs try: excl_xattr = self.header['excl_xattr'] except KeyError as __: excl_xattr = list() if is_dir and excl_xattr: # For directories and configurations containing excluded xattrs # we refine the checks. If "*" in excl_xattr then no check is done. if "*" not in excl_xattr: ref_dict = dict(zip(tags, entry[2:])) new_dict = dict(zip(tags, meta_info[2:])) for key, val in ref_dict.iteritems(): if not isinstance(val, dict): if new_dict[key] != val: err_msg = ( "Verify failed for entry={0} expect={1} got={2}" " at key={3}").format( dst, entry, meta_info, key) self.logger.error(err_msg) raise CheckEntryException( "failed metainfo match") else: for kxattr, vxattr in val.iteritems(): if kxattr not in excl_xattr: if vxattr != new_dict[key][kxattr]: err_msg = ( "Verify failed for entry={0} expect={1} got={2}" " at xattr key={3}").format( dst, entry, meta_info, kxattr) self.logger.error(err_msg) raise CheckEntryException( "failed metainfo match") else: # For files with tx_check_only verification, we refine the checks if tx_check_only and not is_dir: idx_size = self.header["file_meta"].index("size") + 2 idx_xstype = self.header["file_meta"].index("xstype") + 2 idx_xsval = self.header["file_meta"].index("xs") + 2 if (meta_info[idx_size] != entry[idx_size] or meta_info[idx_xstype] != entry[idx_xstype] or meta_info[idx_xsval] != entry[idx_xsval]): err_msg = ( "Partial verify failed for entry={0} expect={1} got={2}" "").format(dst, entry, meta_info) self.logger.error(err_msg) raise CheckEntryException( "failed metainfo partial match") else: if not meta_info == entry: err_msg = ( "Verify failed for entry={0} expect={1} got={2}" "").format(dst, entry, meta_info) self.logger.error(err_msg) raise CheckEntryException("failed metainfo match") self.logger.info("Entry={0}, status={1}".format(dst, True))
def _verify_entry(self, entry, tx_check_only): """ Check that the entry (file/dir) has the proper meta data. Args: entry (list): Entry from the arhive file containing all info about this particular file/directory. tx_check_only (boolean): If True then for files only check their existence, size and checksum values. Raises: CheckEntryException: if entry verification fails. """ self.logger.debug("Verify entry={0}".format(entry)) is_dir, path = (entry[0] == 'd'), entry[1] __, dst = self.get_endpoints(path) url = client.URL(dst.encode("utf-8")) if self.d2t: # for PUT check entry size and checksum if possible fs = self.get_fs(dst) st, stat_info = fs.stat(url.path.encode("utf-8")) if not st.ok: err_msg = "Entry={0} failed stat".format(dst) self.logger.error(err_msg) raise CheckEntryException("failed stat") if not is_dir: # check file size match indx = self.header["file_meta"].index("size") + 2 orig_size = int(entry[indx]) if stat_info.size != orig_size: err_msg = ("Verify entry={0}, expect_size={1}, size={2}" "").format(dst, orig_size, stat_info.size) self.logger.error(err_msg) raise CheckEntryException("failed file size match") # Check checksum only if it is adler32 - only one supported by CASTOR indx = self.header["file_meta"].index("xstype") + 2 # !!!HACK!!! Check the checksum only if file size is not 0 since # CASTOR does not store any checksum for 0 size files if stat_info.size != 0 and entry[indx] == "adler": indx = self.header["file_meta"].index("xs") + 2 xs = entry[indx] st, xs_resp = fs.query(QueryCode.CHECKSUM, url.path) if not st.ok: err_msg = "Entry={0} failed xs query".format(dst) self.logger.error(err_msg) raise CheckEntryException("failed xs query") # Result has an annoying \x00 character at the end and it # contains the xs type (adler32) and the xs value resp = xs_resp.split('\x00')[0].split() # If checksum value is not 8 char long then we need padding if len(resp[1]) != 8: resp[1] = "{0:0>8}".format(resp[1]) if resp[0] == "adler32" and resp[1] != xs: err_msg = ("Entry={0} xs value missmatch xs_expected={1} " "xs_got={2}").format(dst, xs, resp[1]) self.logger.error(err_msg) raise CheckEntryException("xs value missmatch") else: # for GET check all metadata if is_dir: tags = self.header['dir_meta'] else: tags = self.header['file_meta'] try: if self.header['twindow_type'] and self.header['twindow_val']: dfile = dict(zip(tags, entry[2:])) twindow_sec = int(self.header['twindow_val']) tentry_sec = int(float(dfile[self.header['twindow_type']])) if tentry_sec < twindow_sec: # No check for this entry return # This is a backup so don't check atomic version files if is_atomic_version_file(entry[1]): return except KeyError as __: # This is not a backup transfer but an archive one, carry on pass try: meta_info = get_entry_info(url, path, tags, is_dir) except (AttributeError, IOError, KeyError) as __: self.logger.error("Failed getting metainfo entry={0}".format(dst)) raise CheckEntryException("failed getting metainfo") # Check if we have any excluded xattrs try: excl_xattr = self.header['excl_xattr'] except KeyError as __: excl_xattr = list() if is_dir and excl_xattr: # For directories and configurations containing excluded xattrs # we refine the checks. If "*" in excl_xattr then no check is done. if "*" not in excl_xattr: ref_dict = dict(zip(tags, entry[2:])) new_dict = dict(zip(tags, meta_info[2:])) for key, val in ref_dict.iteritems(): if not isinstance(val, dict): if new_dict[key] != val: err_msg = ("Verify failed for entry={0} expect={1} got={2}" " at key={3}").format(dst, entry, meta_info, key) self.logger.error(err_msg) raise CheckEntryException("failed metainfo match") else: for kxattr, vxattr in val.iteritems(): if kxattr not in excl_xattr: if vxattr != new_dict[key][kxattr]: err_msg = ("Verify failed for entry={0} expect={1} got={2}" " at xattr key={3}").format(dst, entry, meta_info, kxattr) self.logger.error(err_msg) raise CheckEntryException("failed metainfo match") else: # For files with tx_check_only verification, we refine the checks if tx_check_only and not is_dir: idx_size = self.header["file_meta"].index("size") + 2 idx_xstype = self.header["file_meta"].index("xstype") + 2 idx_xsval = self.header["file_meta"].index("xs") + 2 if (meta_info[idx_size] != entry[idx_size] or meta_info[idx_xstype] != entry[idx_xstype] or meta_info[idx_xsval] != entry[idx_xsval]): err_msg = ("Partial verify failed for entry={0} expect={1} got={2}" "").format(dst, entry, meta_info) self.logger.error(err_msg) raise CheckEntryException("failed metainfo partial match") else: if not meta_info == entry: err_msg = ("Verify failed for entry={0} expect={1} got={2}" "").format(dst, entry, meta_info) self.logger.error(err_msg) raise CheckEntryException("failed metainfo match") self.logger.info("Entry={0}, status={1}".format(dst, True))