def check_eos_access(url):
""" Check that the current user executing the programm is mapped as root in
EOS otherwise he will not be able to set all the necessary attributes for
the newly built archive. Make sure also that the root destination does not
exist already.
Args:
url (XRootD.URL): EOS URL to the destination path
Raises:
EosAccessException
"""
fwhoami = ''.join(
[url.protocol, "://", url.hostid, "//proc/user/?mgm.cmd=whoami"])
(status, out, __) = exec_cmd(fwhoami)
if not status:
msg = "Failed to execute EOS whoami command"
raise EosAccessException(msg)
# Extrach the uid and gid from the response
out.strip("\0\n ")
lst = out.split(' ')
try:
for token in lst:
if token.startswith("uid="):
uid = int(token[4:])
elif token.startswith("gid="):
gid = int(token[4:])
except ValueError as __:
msg = "Failed while parsing uid/gid response to EOS whoami command"
raise EosAccessException(msg)
if uid != 0 or gid != 0:
msg = "User {0} does not have full rights in EOS - aborting".format(
os.getuid())
raise EosAccessException(msg)
# Check that root directory does not exist already
fs = client.FileSystem(str(url))
st, __ = fs.stat(url.path)
if st.ok:
msg = "EOS root directory already exists"
raise EosAccessException(msg)
fmkdir = ''.join([
url.protocol, "://", url.hostid, "//proc/user/?mgm.cmd=mkdir&"
"mgm.path=", url.path
])
(status, __, __) = exec_cmd(fmkdir)
if not status:
msg = "Failed to create EOS directory: {0}".format(url.path)
raise EosAccessException(msg)
def check_eos_access(url):
""" Check that the current user executing the programm is mapped as root in
EOS otherwise he will not be able to set all the necessary attributes for
the newly built archive. Make sure also that the root destination does not
exist already.
Args:
url (XRootD.URL): EOS URL to the destination path
Raises:
EosAccessException
"""
fwhoami = ''.join([url.protocol, "://", url.hostid, "//proc/user/?mgm.cmd=whoami"])
(status, out, __) = exec_cmd(fwhoami)
if not status:
msg = "Failed to execute EOS whoami command"
raise EosAccessException(msg)
# Extrach the uid and gid from the response
out.strip("\0\n ")
lst = out.split(' ')
try:
for token in lst:
if token.startswith("uid="):
uid = int(token[4:])
elif token.startswith("gid="):
gid = int(token[4:])
except ValueError as __:
msg = "Failed while parsing uid/gid response to EOS whoami command"
raise EosAccessException(msg)
if uid != 0 or gid != 0:
msg = "User {0} does not have full rights in EOS - aborting".format(os.getuid())
raise EosAccessException(msg)
# Check that root directory does not exist already
fs = client.FileSystem(str(url))
st, __ = fs.stat(url.path)
if st.ok:
msg = "EOS root directory already exists"
raise EosAccessException(msg)
fmkdir = ''.join([url.protocol, "://", url.hostid, "//proc/user/?mgm.cmd=mkdir&"
"mgm.path=", url.path])
(status, __, __) = exec_cmd(fmkdir)
if not status:
msg = "Failed to create EOS directory: {0}".format(url.path)
raise EosAccessException(msg)
def archive_prepare(self):
""" Prepare requested archive operation.
Raises:
IOError: Failed to rename or transfer archive file.
"""
# Rename archive file in EOS
efile_url = client.URL(self.efile_full.encode("utf-8"))
eosf_rename = ''.join(
[self.efile_root, self.config.ARCH_FN, ".", self.oper, ".err"])
rename_url = client.URL(eosf_rename.encode("utf-8"))
frename = ''.join([
rename_url.protocol, "://", rename_url.hostid,
"//proc/user/?mgm.cmd=file&mgm.subcmd=rename"
"&mgm.path=", efile_url.path, "&mgm.file.source=", efile_url.path,
"&mgm.file.target=", rename_url.path
])
(status, __, stderr) = exec_cmd(frename)
if not status:
err_msg = ("Failed to rename archive file {0} to {1}, msg={2}"
"").format(self.efile_full, rename_url, stderr)
self.logger.error(err_msg)
raise IOError(err_msg)
# Copy archive file from EOS to the local disk
self.efile_full = eosf_rename
eos_fs = client.FileSystem(self.efile_full.encode("utf-8"))
st, _ = eos_fs.copy(self.efile_full + "?eos.ruid=0&eos.rgid=0",
self.tx_file, True)
if not st.ok:
err_msg = ("Failed to copy archive file={0} to local disk at={1}"
"").format(self.efile_full, self.tx_file)
self.logger.error(err_msg)
raise IOError(err_msg)
# Create the ArchiveFile object
d2t = (self.oper == self.config.PUT_OP)
self.archive = ArchiveFile(self.tx_file, d2t)
def archive_prepare(self):
""" Prepare requested archive operation.
Raises:
IOError: Failed to rename or transfer archive file.
"""
# Rename archive file in EOS
efile_url = client.URL(self.efile_full.encode("utf-8"))
eosf_rename = ''.join([self.efile_root, self.config.ARCH_FN, ".", self.oper, ".err"])
rename_url = client.URL(eosf_rename.encode("utf-8"))
frename = ''.join([rename_url.protocol, "://", rename_url.hostid,
"//proc/user/?mgm.cmd=file&mgm.subcmd=rename"
"&mgm.path=", efile_url.path,
"&mgm.file.source=", efile_url.path,
"&mgm.file.target=", rename_url.path])
(status, __, stderr) = exec_cmd(frename)
if not status:
err_msg = ("Failed to rename archive file {0} to {1}, msg={2}"
"").format(self.efile_full, rename_url, stderr)
self.logger.error(err_msg)
raise IOError(err_msg)
# Copy archive file from EOS to the local disk
self.efile_full = eosf_rename
eos_fs = client.FileSystem(self.efile_full.encode("utf-8"))
st, _ = eos_fs.copy(self.efile_full + "?eos.ruid=0&eos.rgid=0",
self.tx_file, True)
if not st.ok:
err_msg = ("Failed to copy archive file={0} to local disk at={1}"
"").format(self.efile_full, self.tx_file)
self.logger.error(err_msg)
raise IOError(err_msg)
# Create the ArchiveFile object
d2t = (self.oper == self.config.PUT_OP)
self.archive = ArchiveFile(self.tx_file, d2t)
def archive_tx_clean(self, check_ok):
""" Clean the transfer by renaming the archive file in EOS adding the
following extensions:
.done - the transfer was successful
.err - there were errors during the transfer. These are logged in the
file .archive.log in the same directory.
Args:
check_ok (bool): True if no error occured during transfer,
otherwise false.
"""
# Rename arch file in EOS to reflect the status
if not check_ok:
eosf_rename = ''.join(
[self.efile_root, self.config.ARCH_FN, ".", self.oper, ".err"])
else:
eosf_rename = ''.join([
self.efile_root, self.config.ARCH_FN, ".", self.oper, ".done"
])
old_url = client.URL(self.efile_full.encode("utf-8"))
new_url = client.URL(eosf_rename.encode("utf-8"))
frename = ''.join([
old_url.protocol, "://", old_url.hostid, "//proc/user/?",
"mgm.cmd=file&mgm.subcmd=rename&mgm.path=", old_url.path,
"&mgm.file.source=", old_url.path, "&mgm.file.target=",
new_url.path
])
(status, __, stderr) = exec_cmd(frename)
if not status:
err_msg = ("Failed to rename {0} to {1}, msg={2}"
"").format(self.efile_full, eosf_rename, stderr)
self.logger.error(err_msg)
# TODO: raise IOError
else:
# For successful delete operations remove also the archive file
if self.oper == self.config.DELETE_OP and check_ok:
fs = client.FileSystem(self.efile_full.encode("utf-8"))
st_rm, __ = fs.rm(new_url.path + "?eos.ruid=0&eos.rgid=0")
if not st_rm.ok:
warn_msg = "Failed to delete archive {0}".format(
new_url.path)
self.logger.warning(warn_msg)
# Copy local log file back to EOS directory and set the ownership to the
# identity of the client who triggered the archive
dir_root = self.efile_root[self.efile_root.rfind('//') + 1:]
eos_log = ''.join([
old_url.protocol, "://", old_url.hostid, "/", dir_root,
self.config.ARCH_FN, ".log?eos.ruid=0&eos.rgid=0"
])
self.logger.debug("Copy log:{0} to {1}".format(self.config.LOG_FILE,
eos_log))
self.config.handler.flush()
cp_client = client.FileSystem(self.efile_full.encode("utf-8"))
st, __ = cp_client.copy(self.config.LOG_FILE, eos_log, force=True)
if not st.ok:
self.logger.error(("Failed to copy log file {0} to EOS at {1}"
"").format(self.config.LOG_FILE, eos_log))
else:
# User triggering archive operation owns the log file
eos_log_url = client.URL(eos_log)
fs = client.FileSystem(eos_log.encode("utf-8"))
arg = ''.join([
eos_log_url.path, "?eos.ruid=0&eos.rgid=0&mgm.pcmd=chown&uid=",
self.uid, "&gid=", self.gid
])
xrd_st, __ = fs.query(QueryCode.OPAQUEFILE, arg.encode("utf-8"))
if not xrd_st.ok:
err_msg = ("Failed setting ownership of the log file in"
" EOS: {0}").format(eos_log)
self.logger.error(err_msg)
raise IOError(err_msg)
else:
# Delete log if successfully copied to EOS and changed ownership
try:
os.remove(self.config.LOG_FILE)
except OSError as __:
pass
# Delete all local files associated with this transfer
try:
os.remove(self.tx_file)
except OSError as __:
pass
# Join async status thread
self.thread_status.do_finish()
self.thread_status.join()
def check_root_dir(self):
""" Do the necessary checks for the destination directory depending on
the type of the transfer.
Raises:
IOError: Root dir state inconsistent.
"""
root_str = self.header['dst' if self.d2t else 'src']
fs = self.get_fs(root_str)
url = client.URL(root_str.encode("utf-8"))
arg = url.path + "?eos.ruid=0&eos.rgid=0"
st, __ = fs.stat(arg.encode("utf-8"))
if self.d2t:
if st.ok:
# For PUT destination dir must NOT exist
err_msg = "Root PUT dir={0} exists".format(root_str)
self.logger.error(err_msg)
raise IOError(err_msg)
else:
# Make sure the rest of the path exists as for the moment CASTOR
# mkdir -p /path/to/file does not work properly
pos = url.path.find('/', 1)
while pos != -1:
dpath = url.path[:pos]
pos = url.path.find('/', pos + 1)
st, __ = fs.stat(dpath.encode("utf-8"))
if not st.ok:
st, __ = fs.mkdir(dpath.encode("utf-8"))
if not st.ok:
err_msg = ("Dir={0} failed mkdir errmsg={1}"
"").format(dpath,
st.message.decode("utf-8"))
self.logger.error(err_msg)
raise IOError(err_msg)
elif not self.d2t:
# For GET destination must exist and contain just the archive file
if not st.ok:
err_msg = "Root GET dir={0} does NOT exist".format(root_str)
self.logger.error(err_msg)
raise IOError(err_msg)
else:
ffindcount = ''.join([
url.protocol, "://", url.hostid,
"//proc/user/?mgm.cmd=find&mgm.path=",
seal_path(url.path), "&mgm.option=Z"
])
(status, stdout, stderr) = exec_cmd(ffindcount)
if status:
for entry in stdout.split():
tag, num = entry.split('=')
if ((tag == 'nfiles' and num not in ['1', '2'])
or (tag == 'ndirectories' and num != '1')):
err_msg = (
"Root GET dir={0} should contain at least "
"one file and at most two - clean up and "
"try again").format(root_str)
self.logger.error(err_msg)
raise IOError(err_msg)
else:
err_msg = ("Error doing find count on GET destination={0}"
", msg={1}").format(root_str, stderr)
self.logger.error(err_msg)
raise IOError(err_msg)
def make_mutable(self):
""" Make the EOS sub-tree pointed by header['src'] mutable.
Raises:
IOError when operation fails.
"""
url = client.URL(self.header['src'].encode("utf-8"))
for dentry in self.dirs():
dir_path = url.path + dentry[1]
fgetattr = ''.join([
url.protocol, "://", url.hostid, "//proc/user/",
"?mgm.cmd=attr&mgm.subcmd=get&mgm.attr.key=sys.acl",
"&mgm.path=",
seal_path(dir_path)
])
(status, stdout, __) = exec_cmd(fgetattr)
if not status:
warn_msg = "No xattr sys.acl found for dir={0}".format(
dir_path)
self.logger.warning(warn_msg)
else:
# Remove the 'z:i' rule from the acl list
stdout = stdout.replace('"', '')
acl_val = stdout[stdout.find('=') + 1:]
rules = acl_val.split(',')
new_rules = []
for rule in rules:
if rule.startswith("z:"):
tag, definition = rule.split(':')
pos = definition.find('i')
if pos != -1:
definition = definition[:pos] + definition[pos +
1:]
if definition:
new_rules.append(':'.join([tag, definition]))
continue
new_rules.append(rule)
acl_val = ','.join(new_rules)
self.logger.error("new acl: {0}".format(acl_val))
if acl_val:
# Set the new sys.acl xattr
fmutable = ''.join([
url.protocol, "://", url.hostid, "//proc/user/?",
"mgm.cmd=attr&mgm.subcmd=set&mgm.attr.key=sys.acl",
"&mgm.attr.value=", acl_val, "&mgm.path=", dir_path
])
(status, __, stderr) = exec_cmd(fmutable)
if not status:
err_msg = "Error making dir={0} mutable, msg={1}".format(
dir_path, stderr)
self.logger.error(err_msg)
raise IOError(err_msg)
else:
# sys.acl empty, remove it from the xattrs
frmattr = ''.join([
url.protocol, "://", url.hostid, "//proc/user/?",
"mgm.cmd=attr&mgm.subcmd=rm&mgm.attr.key=sys.acl",
"&mgm.path=", dir_path
])
(status, __, stderr) = exec_cmd(frmattr)
if not status:
err_msg = (
"Error removing xattr=sys.acl for dir={0}, msg={1}"
"").format(dir_path, stderr)
self.logger.error(err_msg)
raise IOError(err_msg)
def check_root_dir(self):
""" Do the necessary checks for the destination directory depending on
the type of the transfer.
Raises:
IOError: Root dir state inconsistent.
"""
root_str = self.header['dst' if self.d2t else 'src']
fs = self.get_fs(root_str)
url = client.URL(root_str.encode("utf-8"))
arg = url.path + "?eos.ruid=0&eos.rgid=0"
st, __ = fs.stat(arg.encode("utf-8"))
if self.d2t:
if st.ok:
# For PUT destination dir must NOT exist
err_msg = "Root PUT dir={0} exists".format(root_str)
self.logger.error(err_msg)
raise IOError(err_msg)
else:
# Make sure the rest of the path exists as for the moment CASTOR
# mkdir -p /path/to/file does not work properly
pos = url.path.find('/', 1)
while pos != -1:
dpath = url.path[: pos]
pos = url.path.find('/', pos + 1)
st, __ = fs.stat(dpath.encode("utf-8"))
if not st.ok:
st, __ = fs.mkdir(dpath.encode("utf-8"))
if not st.ok:
err_msg = ("Dir={0} failed mkdir errmsg={1}"
"").format(dpath, st.message.decode("utf-8"))
self.logger.error(err_msg)
raise IOError(err_msg)
elif not self.d2t:
# For GET destination must exist and contain just the archive file
if not st.ok:
err_msg = "Root GET dir={0} does NOT exist".format(root_str)
self.logger.error(err_msg)
raise IOError(err_msg)
else:
ffindcount = ''.join([url.protocol, "://", url.hostid,
"//proc/user/?mgm.cmd=find&mgm.path=",
seal_path(url.path), "&mgm.option=Z"])
(status, stdout, stderr) = exec_cmd(ffindcount)
if status:
for entry in stdout.split():
tag, num = entry.split('=')
if ((tag == 'nfiles' and num not in ['1', '2']) or
(tag == 'ndirectories' and num != '1')):
err_msg = ("Root GET dir={0} should contain at least "
"one file and at most two - clean up and "
"try again").format(root_str)
self.logger.error(err_msg)
raise IOError(err_msg)
else:
err_msg = ("Error doing find count on GET destination={0}"
", msg={1}").format(root_str, stderr)
self.logger.error(err_msg)
raise IOError(err_msg)
def make_mutable(self):
""" Make the EOS sub-tree pointed by header['src'] mutable.
Raises:
IOError when operation fails.
"""
url = client.URL(self.header['src'].encode("utf-8"))
for dentry in self.dirs():
dir_path = url.path + dentry[1]
fgetattr = ''.join([url.protocol, "://", url.hostid, "//proc/user/",
"?mgm.cmd=attr&mgm.subcmd=get&mgm.attr.key=sys.acl",
"&mgm.path=", seal_path(dir_path)])
(status, stdout, __) = exec_cmd(fgetattr)
if not status:
warn_msg = "No xattr sys.acl found for dir={0}".format(dir_path)
self.logger.warning(warn_msg)
else:
# Remove the 'z:i' rule from the acl list
stdout = stdout.replace('"', '')
acl_val = stdout[stdout.find('=') + 1:]
rules = acl_val.split(',')
new_rules = []
for rule in rules:
if rule.startswith("z:"):
tag, definition = rule.split(':')
pos = definition.find('i')
if pos != -1:
definition = definition[:pos] + definition[pos + 1:]
if definition:
new_rules.append(':'.join([tag, definition]))
continue
new_rules.append(rule)
acl_val = ','.join(new_rules)
self.logger.error("new acl: {0}".format(acl_val))
if acl_val:
# Set the new sys.acl xattr
fmutable = ''.join([url.protocol, "://", url.hostid, "//proc/user/?",
"mgm.cmd=attr&mgm.subcmd=set&mgm.attr.key=sys.acl",
"&mgm.attr.value=", acl_val, "&mgm.path=", dir_path])
(status, __, stderr) = exec_cmd(fmutable)
if not status:
err_msg = "Error making dir={0} mutable, msg={1}".format(
dir_path, stderr)
self.logger.error(err_msg)
raise IOError(err_msg)
else:
# sys.acl empty, remove it from the xattrs
frmattr = ''.join([url.protocol, "://", url.hostid, "//proc/user/?",
"mgm.cmd=attr&mgm.subcmd=rm&mgm.attr.key=sys.acl",
"&mgm.path=", dir_path])
(status, __, stderr) = exec_cmd(frmattr)
if not status:
err_msg = ("Error removing xattr=sys.acl for dir={0}, msg={1}"
"").format(dir_path, stderr)
self.logger.error(err_msg)
raise IOError(err_msg)
def archive_tx_clean(self, check_ok):
""" Clean the transfer by renaming the archive file in EOS adding the
following extensions:
.done - the transfer was successful
.err - there were errors during the transfer. These are logged in the
file .archive.log in the same directory.
Args:
check_ok (bool): True if no error occured during transfer,
otherwise false.
"""
# Rename arch file in EOS to reflect the status
if not check_ok:
eosf_rename = ''.join([self.efile_root, self.config.ARCH_FN, ".", self.oper, ".err"])
else:
eosf_rename = ''.join([self.efile_root, self.config.ARCH_FN, ".", self.oper, ".done"])
old_url = client.URL(self.efile_full.encode("utf-8"))
new_url = client.URL(eosf_rename.encode("utf-8"))
frename = ''.join([old_url.protocol, "://", old_url.hostid, "//proc/user/?",
"mgm.cmd=file&mgm.subcmd=rename&mgm.path=", old_url.path,
"&mgm.file.source=", old_url.path,
"&mgm.file.target=", new_url.path])
(status, __, stderr) = exec_cmd(frename)
if not status:
err_msg = ("Failed to rename {0} to {1}, msg={2}"
"").format(self.efile_full, eosf_rename, stderr)
self.logger.error(err_msg)
# TODO: raise IOError
else:
# For successful delete operations remove also the archive file
if self.oper == self.config.DELETE_OP and check_ok:
fs = client.FileSystem(self.efile_full.encode("utf-8"))
st_rm, __ = fs.rm(new_url.path + "?eos.ruid=0&eos.rgid=0")
if not st_rm.ok:
warn_msg = "Failed to delete archive {0}".format(new_url.path)
self.logger.warning(warn_msg)
# Copy local log file back to EOS directory and set the ownership to the
# identity of the client who triggered the archive
dir_root = self.efile_root[self.efile_root.rfind('//') + 1:]
eos_log = ''.join([old_url.protocol, "://", old_url.hostid, "/",
dir_root, self.config.ARCH_FN, ".log?eos.ruid=0&eos.rgid=0"])
self.logger.debug("Copy log:{0} to {1}".format(self.config.LOG_FILE, eos_log))
self.config.handler.flush()
cp_client = client.FileSystem(self.efile_full.encode("utf-8"))
st, __ = cp_client.copy(self.config.LOG_FILE, eos_log, force=True)
if not st.ok:
self.logger.error(("Failed to copy log file {0} to EOS at {1}"
"").format(self.config.LOG_FILE, eos_log))
else:
# User triggering archive operation owns the log file
eos_log_url = client.URL(eos_log)
fs = client.FileSystem(eos_log.encode("utf-8"))
arg = ''.join([eos_log_url.path, "?eos.ruid=0&eos.rgid=0&mgm.pcmd=chown&uid=",
self.uid, "&gid=", self.gid])
xrd_st, __ = fs.query(QueryCode.OPAQUEFILE, arg.encode("utf-8"))
if not xrd_st.ok:
err_msg = ("Failed setting ownership of the log file in"
" EOS: {0}").format(eos_log)
self.logger.error(err_msg)
raise IOError(err_msg)
else:
# Delete log if successfully copied to EOS and changed ownership
try:
os.remove(self.config.LOG_FILE)
except OSError as __:
pass
# Delete all local files associated with this transfer
try:
os.remove(self.tx_file)
except OSError as __:
pass
# Join async status thread
self.thread_status.do_finish()
self.thread_status.join()
