def check_eos_access(url): """ Check that the current user executing the programm is mapped as root in EOS otherwise he will not be able to set all the necessary attributes for the newly built archive. Make sure also that the root destination does not exist already. Args: url (XRootD.URL): EOS URL to the destination path Raises: EosAccessException """ fwhoami = ''.join( [url.protocol, "://", url.hostid, "//proc/user/?mgm.cmd=whoami"]) (status, out, __) = exec_cmd(fwhoami) if not status: msg = "Failed to execute EOS whoami command" raise EosAccessException(msg) # Extrach the uid and gid from the response out.strip("\0\n ") lst = out.split(' ') try: for token in lst: if token.startswith("uid="): uid = int(token[4:]) elif token.startswith("gid="): gid = int(token[4:]) except ValueError as __: msg = "Failed while parsing uid/gid response to EOS whoami command" raise EosAccessException(msg) if uid != 0 or gid != 0: msg = "User {0} does not have full rights in EOS - aborting".format( os.getuid()) raise EosAccessException(msg) # Check that root directory does not exist already fs = client.FileSystem(str(url)) st, __ = fs.stat(url.path) if st.ok: msg = "EOS root directory already exists" raise EosAccessException(msg) fmkdir = ''.join([ url.protocol, "://", url.hostid, "//proc/user/?mgm.cmd=mkdir&" "mgm.path=", url.path ]) (status, __, __) = exec_cmd(fmkdir) if not status: msg = "Failed to create EOS directory: {0}".format(url.path) raise EosAccessException(msg)
def check_eos_access(url): """ Check that the current user executing the programm is mapped as root in EOS otherwise he will not be able to set all the necessary attributes for the newly built archive. Make sure also that the root destination does not exist already. Args: url (XRootD.URL): EOS URL to the destination path Raises: EosAccessException """ fwhoami = ''.join([url.protocol, "://", url.hostid, "//proc/user/?mgm.cmd=whoami"]) (status, out, __) = exec_cmd(fwhoami) if not status: msg = "Failed to execute EOS whoami command" raise EosAccessException(msg) # Extrach the uid and gid from the response out.strip("\0\n ") lst = out.split(' ') try: for token in lst: if token.startswith("uid="): uid = int(token[4:]) elif token.startswith("gid="): gid = int(token[4:]) except ValueError as __: msg = "Failed while parsing uid/gid response to EOS whoami command" raise EosAccessException(msg) if uid != 0 or gid != 0: msg = "User {0} does not have full rights in EOS - aborting".format(os.getuid()) raise EosAccessException(msg) # Check that root directory does not exist already fs = client.FileSystem(str(url)) st, __ = fs.stat(url.path) if st.ok: msg = "EOS root directory already exists" raise EosAccessException(msg) fmkdir = ''.join([url.protocol, "://", url.hostid, "//proc/user/?mgm.cmd=mkdir&" "mgm.path=", url.path]) (status, __, __) = exec_cmd(fmkdir) if not status: msg = "Failed to create EOS directory: {0}".format(url.path) raise EosAccessException(msg)
def archive_prepare(self): """ Prepare requested archive operation. Raises: IOError: Failed to rename or transfer archive file. """ # Rename archive file in EOS efile_url = client.URL(self.efile_full.encode("utf-8")) eosf_rename = ''.join( [self.efile_root, self.config.ARCH_FN, ".", self.oper, ".err"]) rename_url = client.URL(eosf_rename.encode("utf-8")) frename = ''.join([ rename_url.protocol, "://", rename_url.hostid, "//proc/user/?mgm.cmd=file&mgm.subcmd=rename" "&mgm.path=", efile_url.path, "&mgm.file.source=", efile_url.path, "&mgm.file.target=", rename_url.path ]) (status, __, stderr) = exec_cmd(frename) if not status: err_msg = ("Failed to rename archive file {0} to {1}, msg={2}" "").format(self.efile_full, rename_url, stderr) self.logger.error(err_msg) raise IOError(err_msg) # Copy archive file from EOS to the local disk self.efile_full = eosf_rename eos_fs = client.FileSystem(self.efile_full.encode("utf-8")) st, _ = eos_fs.copy(self.efile_full + "?eos.ruid=0&eos.rgid=0", self.tx_file, True) if not st.ok: err_msg = ("Failed to copy archive file={0} to local disk at={1}" "").format(self.efile_full, self.tx_file) self.logger.error(err_msg) raise IOError(err_msg) # Create the ArchiveFile object d2t = (self.oper == self.config.PUT_OP) self.archive = ArchiveFile(self.tx_file, d2t)
def archive_prepare(self): """ Prepare requested archive operation. Raises: IOError: Failed to rename or transfer archive file. """ # Rename archive file in EOS efile_url = client.URL(self.efile_full.encode("utf-8")) eosf_rename = ''.join([self.efile_root, self.config.ARCH_FN, ".", self.oper, ".err"]) rename_url = client.URL(eosf_rename.encode("utf-8")) frename = ''.join([rename_url.protocol, "://", rename_url.hostid, "//proc/user/?mgm.cmd=file&mgm.subcmd=rename" "&mgm.path=", efile_url.path, "&mgm.file.source=", efile_url.path, "&mgm.file.target=", rename_url.path]) (status, __, stderr) = exec_cmd(frename) if not status: err_msg = ("Failed to rename archive file {0} to {1}, msg={2}" "").format(self.efile_full, rename_url, stderr) self.logger.error(err_msg) raise IOError(err_msg) # Copy archive file from EOS to the local disk self.efile_full = eosf_rename eos_fs = client.FileSystem(self.efile_full.encode("utf-8")) st, _ = eos_fs.copy(self.efile_full + "?eos.ruid=0&eos.rgid=0", self.tx_file, True) if not st.ok: err_msg = ("Failed to copy archive file={0} to local disk at={1}" "").format(self.efile_full, self.tx_file) self.logger.error(err_msg) raise IOError(err_msg) # Create the ArchiveFile object d2t = (self.oper == self.config.PUT_OP) self.archive = ArchiveFile(self.tx_file, d2t)
def archive_tx_clean(self, check_ok): """ Clean the transfer by renaming the archive file in EOS adding the following extensions: .done - the transfer was successful .err - there were errors during the transfer. These are logged in the file .archive.log in the same directory. Args: check_ok (bool): True if no error occured during transfer, otherwise false. """ # Rename arch file in EOS to reflect the status if not check_ok: eosf_rename = ''.join( [self.efile_root, self.config.ARCH_FN, ".", self.oper, ".err"]) else: eosf_rename = ''.join([ self.efile_root, self.config.ARCH_FN, ".", self.oper, ".done" ]) old_url = client.URL(self.efile_full.encode("utf-8")) new_url = client.URL(eosf_rename.encode("utf-8")) frename = ''.join([ old_url.protocol, "://", old_url.hostid, "//proc/user/?", "mgm.cmd=file&mgm.subcmd=rename&mgm.path=", old_url.path, "&mgm.file.source=", old_url.path, "&mgm.file.target=", new_url.path ]) (status, __, stderr) = exec_cmd(frename) if not status: err_msg = ("Failed to rename {0} to {1}, msg={2}" "").format(self.efile_full, eosf_rename, stderr) self.logger.error(err_msg) # TODO: raise IOError else: # For successful delete operations remove also the archive file if self.oper == self.config.DELETE_OP and check_ok: fs = client.FileSystem(self.efile_full.encode("utf-8")) st_rm, __ = fs.rm(new_url.path + "?eos.ruid=0&eos.rgid=0") if not st_rm.ok: warn_msg = "Failed to delete archive {0}".format( new_url.path) self.logger.warning(warn_msg) # Copy local log file back to EOS directory and set the ownership to the # identity of the client who triggered the archive dir_root = self.efile_root[self.efile_root.rfind('//') + 1:] eos_log = ''.join([ old_url.protocol, "://", old_url.hostid, "/", dir_root, self.config.ARCH_FN, ".log?eos.ruid=0&eos.rgid=0" ]) self.logger.debug("Copy log:{0} to {1}".format(self.config.LOG_FILE, eos_log)) self.config.handler.flush() cp_client = client.FileSystem(self.efile_full.encode("utf-8")) st, __ = cp_client.copy(self.config.LOG_FILE, eos_log, force=True) if not st.ok: self.logger.error(("Failed to copy log file {0} to EOS at {1}" "").format(self.config.LOG_FILE, eos_log)) else: # User triggering archive operation owns the log file eos_log_url = client.URL(eos_log) fs = client.FileSystem(eos_log.encode("utf-8")) arg = ''.join([ eos_log_url.path, "?eos.ruid=0&eos.rgid=0&mgm.pcmd=chown&uid=", self.uid, "&gid=", self.gid ]) xrd_st, __ = fs.query(QueryCode.OPAQUEFILE, arg.encode("utf-8")) if not xrd_st.ok: err_msg = ("Failed setting ownership of the log file in" " EOS: {0}").format(eos_log) self.logger.error(err_msg) raise IOError(err_msg) else: # Delete log if successfully copied to EOS and changed ownership try: os.remove(self.config.LOG_FILE) except OSError as __: pass # Delete all local files associated with this transfer try: os.remove(self.tx_file) except OSError as __: pass # Join async status thread self.thread_status.do_finish() self.thread_status.join()
def check_root_dir(self): """ Do the necessary checks for the destination directory depending on the type of the transfer. Raises: IOError: Root dir state inconsistent. """ root_str = self.header['dst' if self.d2t else 'src'] fs = self.get_fs(root_str) url = client.URL(root_str.encode("utf-8")) arg = url.path + "?eos.ruid=0&eos.rgid=0" st, __ = fs.stat(arg.encode("utf-8")) if self.d2t: if st.ok: # For PUT destination dir must NOT exist err_msg = "Root PUT dir={0} exists".format(root_str) self.logger.error(err_msg) raise IOError(err_msg) else: # Make sure the rest of the path exists as for the moment CASTOR # mkdir -p /path/to/file does not work properly pos = url.path.find('/', 1) while pos != -1: dpath = url.path[:pos] pos = url.path.find('/', pos + 1) st, __ = fs.stat(dpath.encode("utf-8")) if not st.ok: st, __ = fs.mkdir(dpath.encode("utf-8")) if not st.ok: err_msg = ("Dir={0} failed mkdir errmsg={1}" "").format(dpath, st.message.decode("utf-8")) self.logger.error(err_msg) raise IOError(err_msg) elif not self.d2t: # For GET destination must exist and contain just the archive file if not st.ok: err_msg = "Root GET dir={0} does NOT exist".format(root_str) self.logger.error(err_msg) raise IOError(err_msg) else: ffindcount = ''.join([ url.protocol, "://", url.hostid, "//proc/user/?mgm.cmd=find&mgm.path=", seal_path(url.path), "&mgm.option=Z" ]) (status, stdout, stderr) = exec_cmd(ffindcount) if status: for entry in stdout.split(): tag, num = entry.split('=') if ((tag == 'nfiles' and num not in ['1', '2']) or (tag == 'ndirectories' and num != '1')): err_msg = ( "Root GET dir={0} should contain at least " "one file and at most two - clean up and " "try again").format(root_str) self.logger.error(err_msg) raise IOError(err_msg) else: err_msg = ("Error doing find count on GET destination={0}" ", msg={1}").format(root_str, stderr) self.logger.error(err_msg) raise IOError(err_msg)
def make_mutable(self): """ Make the EOS sub-tree pointed by header['src'] mutable. Raises: IOError when operation fails. """ url = client.URL(self.header['src'].encode("utf-8")) for dentry in self.dirs(): dir_path = url.path + dentry[1] fgetattr = ''.join([ url.protocol, "://", url.hostid, "//proc/user/", "?mgm.cmd=attr&mgm.subcmd=get&mgm.attr.key=sys.acl", "&mgm.path=", seal_path(dir_path) ]) (status, stdout, __) = exec_cmd(fgetattr) if not status: warn_msg = "No xattr sys.acl found for dir={0}".format( dir_path) self.logger.warning(warn_msg) else: # Remove the 'z:i' rule from the acl list stdout = stdout.replace('"', '') acl_val = stdout[stdout.find('=') + 1:] rules = acl_val.split(',') new_rules = [] for rule in rules: if rule.startswith("z:"): tag, definition = rule.split(':') pos = definition.find('i') if pos != -1: definition = definition[:pos] + definition[pos + 1:] if definition: new_rules.append(':'.join([tag, definition])) continue new_rules.append(rule) acl_val = ','.join(new_rules) self.logger.error("new acl: {0}".format(acl_val)) if acl_val: # Set the new sys.acl xattr fmutable = ''.join([ url.protocol, "://", url.hostid, "//proc/user/?", "mgm.cmd=attr&mgm.subcmd=set&mgm.attr.key=sys.acl", "&mgm.attr.value=", acl_val, "&mgm.path=", dir_path ]) (status, __, stderr) = exec_cmd(fmutable) if not status: err_msg = "Error making dir={0} mutable, msg={1}".format( dir_path, stderr) self.logger.error(err_msg) raise IOError(err_msg) else: # sys.acl empty, remove it from the xattrs frmattr = ''.join([ url.protocol, "://", url.hostid, "//proc/user/?", "mgm.cmd=attr&mgm.subcmd=rm&mgm.attr.key=sys.acl", "&mgm.path=", dir_path ]) (status, __, stderr) = exec_cmd(frmattr) if not status: err_msg = ( "Error removing xattr=sys.acl for dir={0}, msg={1}" "").format(dir_path, stderr) self.logger.error(err_msg) raise IOError(err_msg)
def check_root_dir(self): """ Do the necessary checks for the destination directory depending on the type of the transfer. Raises: IOError: Root dir state inconsistent. """ root_str = self.header['dst' if self.d2t else 'src'] fs = self.get_fs(root_str) url = client.URL(root_str.encode("utf-8")) arg = url.path + "?eos.ruid=0&eos.rgid=0" st, __ = fs.stat(arg.encode("utf-8")) if self.d2t: if st.ok: # For PUT destination dir must NOT exist err_msg = "Root PUT dir={0} exists".format(root_str) self.logger.error(err_msg) raise IOError(err_msg) else: # Make sure the rest of the path exists as for the moment CASTOR # mkdir -p /path/to/file does not work properly pos = url.path.find('/', 1) while pos != -1: dpath = url.path[: pos] pos = url.path.find('/', pos + 1) st, __ = fs.stat(dpath.encode("utf-8")) if not st.ok: st, __ = fs.mkdir(dpath.encode("utf-8")) if not st.ok: err_msg = ("Dir={0} failed mkdir errmsg={1}" "").format(dpath, st.message.decode("utf-8")) self.logger.error(err_msg) raise IOError(err_msg) elif not self.d2t: # For GET destination must exist and contain just the archive file if not st.ok: err_msg = "Root GET dir={0} does NOT exist".format(root_str) self.logger.error(err_msg) raise IOError(err_msg) else: ffindcount = ''.join([url.protocol, "://", url.hostid, "//proc/user/?mgm.cmd=find&mgm.path=", seal_path(url.path), "&mgm.option=Z"]) (status, stdout, stderr) = exec_cmd(ffindcount) if status: for entry in stdout.split(): tag, num = entry.split('=') if ((tag == 'nfiles' and num not in ['1', '2']) or (tag == 'ndirectories' and num != '1')): err_msg = ("Root GET dir={0} should contain at least " "one file and at most two - clean up and " "try again").format(root_str) self.logger.error(err_msg) raise IOError(err_msg) else: err_msg = ("Error doing find count on GET destination={0}" ", msg={1}").format(root_str, stderr) self.logger.error(err_msg) raise IOError(err_msg)
def make_mutable(self): """ Make the EOS sub-tree pointed by header['src'] mutable. Raises: IOError when operation fails. """ url = client.URL(self.header['src'].encode("utf-8")) for dentry in self.dirs(): dir_path = url.path + dentry[1] fgetattr = ''.join([url.protocol, "://", url.hostid, "//proc/user/", "?mgm.cmd=attr&mgm.subcmd=get&mgm.attr.key=sys.acl", "&mgm.path=", seal_path(dir_path)]) (status, stdout, __) = exec_cmd(fgetattr) if not status: warn_msg = "No xattr sys.acl found for dir={0}".format(dir_path) self.logger.warning(warn_msg) else: # Remove the 'z:i' rule from the acl list stdout = stdout.replace('"', '') acl_val = stdout[stdout.find('=') + 1:] rules = acl_val.split(',') new_rules = [] for rule in rules: if rule.startswith("z:"): tag, definition = rule.split(':') pos = definition.find('i') if pos != -1: definition = definition[:pos] + definition[pos + 1:] if definition: new_rules.append(':'.join([tag, definition])) continue new_rules.append(rule) acl_val = ','.join(new_rules) self.logger.error("new acl: {0}".format(acl_val)) if acl_val: # Set the new sys.acl xattr fmutable = ''.join([url.protocol, "://", url.hostid, "//proc/user/?", "mgm.cmd=attr&mgm.subcmd=set&mgm.attr.key=sys.acl", "&mgm.attr.value=", acl_val, "&mgm.path=", dir_path]) (status, __, stderr) = exec_cmd(fmutable) if not status: err_msg = "Error making dir={0} mutable, msg={1}".format( dir_path, stderr) self.logger.error(err_msg) raise IOError(err_msg) else: # sys.acl empty, remove it from the xattrs frmattr = ''.join([url.protocol, "://", url.hostid, "//proc/user/?", "mgm.cmd=attr&mgm.subcmd=rm&mgm.attr.key=sys.acl", "&mgm.path=", dir_path]) (status, __, stderr) = exec_cmd(frmattr) if not status: err_msg = ("Error removing xattr=sys.acl for dir={0}, msg={1}" "").format(dir_path, stderr) self.logger.error(err_msg) raise IOError(err_msg)
def archive_tx_clean(self, check_ok): """ Clean the transfer by renaming the archive file in EOS adding the following extensions: .done - the transfer was successful .err - there were errors during the transfer. These are logged in the file .archive.log in the same directory. Args: check_ok (bool): True if no error occured during transfer, otherwise false. """ # Rename arch file in EOS to reflect the status if not check_ok: eosf_rename = ''.join([self.efile_root, self.config.ARCH_FN, ".", self.oper, ".err"]) else: eosf_rename = ''.join([self.efile_root, self.config.ARCH_FN, ".", self.oper, ".done"]) old_url = client.URL(self.efile_full.encode("utf-8")) new_url = client.URL(eosf_rename.encode("utf-8")) frename = ''.join([old_url.protocol, "://", old_url.hostid, "//proc/user/?", "mgm.cmd=file&mgm.subcmd=rename&mgm.path=", old_url.path, "&mgm.file.source=", old_url.path, "&mgm.file.target=", new_url.path]) (status, __, stderr) = exec_cmd(frename) if not status: err_msg = ("Failed to rename {0} to {1}, msg={2}" "").format(self.efile_full, eosf_rename, stderr) self.logger.error(err_msg) # TODO: raise IOError else: # For successful delete operations remove also the archive file if self.oper == self.config.DELETE_OP and check_ok: fs = client.FileSystem(self.efile_full.encode("utf-8")) st_rm, __ = fs.rm(new_url.path + "?eos.ruid=0&eos.rgid=0") if not st_rm.ok: warn_msg = "Failed to delete archive {0}".format(new_url.path) self.logger.warning(warn_msg) # Copy local log file back to EOS directory and set the ownership to the # identity of the client who triggered the archive dir_root = self.efile_root[self.efile_root.rfind('//') + 1:] eos_log = ''.join([old_url.protocol, "://", old_url.hostid, "/", dir_root, self.config.ARCH_FN, ".log?eos.ruid=0&eos.rgid=0"]) self.logger.debug("Copy log:{0} to {1}".format(self.config.LOG_FILE, eos_log)) self.config.handler.flush() cp_client = client.FileSystem(self.efile_full.encode("utf-8")) st, __ = cp_client.copy(self.config.LOG_FILE, eos_log, force=True) if not st.ok: self.logger.error(("Failed to copy log file {0} to EOS at {1}" "").format(self.config.LOG_FILE, eos_log)) else: # User triggering archive operation owns the log file eos_log_url = client.URL(eos_log) fs = client.FileSystem(eos_log.encode("utf-8")) arg = ''.join([eos_log_url.path, "?eos.ruid=0&eos.rgid=0&mgm.pcmd=chown&uid=", self.uid, "&gid=", self.gid]) xrd_st, __ = fs.query(QueryCode.OPAQUEFILE, arg.encode("utf-8")) if not xrd_st.ok: err_msg = ("Failed setting ownership of the log file in" " EOS: {0}").format(eos_log) self.logger.error(err_msg) raise IOError(err_msg) else: # Delete log if successfully copied to EOS and changed ownership try: os.remove(self.config.LOG_FILE) except OSError as __: pass # Delete all local files associated with this transfer try: os.remove(self.tx_file) except OSError as __: pass # Join async status thread self.thread_status.do_finish() self.thread_status.join()