def rename(src, dest):
if (src.startswith('/pnfs/') or
dest.startswith('/pnfs/')) and (prefer_grid or not pnfs_is_mounted):
if debug:
print('*** Larbatch_posix: Rename %s to %s using ifdh.' % (src, dest))
src_uri = larbatch_utilities.gridftp_uri(src)
dest_path = larbatch_utilities.dcache_path(dest)
cmd = ['uberftp', '-rename', src_uri, dest_path]
jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
q = queue.Queue()
thread = threading.Thread(target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
thread.start()
thread.join(timeout=60)
if thread.is_alive():
if debug:
print('*** Larbatch_posix: Terminating subprocess.')
jobinfo.terminate()
thread.join()
rc = q.get()
jobout = convert_str(q.get())
joberr = convert_str(q.get())
if rc != 0:
raise IFDHError(cmd, rc, jobout, joberr)
else:
if debug:
print('*** Larbatch_posix: Rename %s to %s using posix.' % (src, dest))
os.rename(src, dest)
def symlink(src, dest):
# Make sure we have a kerberos ticket.
if src.startswith('/pnfs/') and not pnfs_is_mounted:
if debug:
print('*** Larbatch_posix: Make symbolic link from %s to %s using nfs server.' % (src, dest))
larbatch_utilities.test_ticket()
cmd = ['ssh', larbatch_utilities.nfs_server(), 'ln', '-s', src, dest]
jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
q = queue.Queue()
thread = threading.Thread(target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
thread.start()
thread.join(timeout=60)
if thread.is_alive():
if debug:
print('*** Larbatch_posix: Terminating subprocess.')
jobinfo.terminate()
thread.join()
rc = q.get()
jobout = convert_str(q.get())
joberr = convert_str(q.get())
if rc != 0:
raise IFDHError(cmd, rc, jobout, joberr)
else:
if debug:
print('*** Larbatch_posix: Make symbolic link from %s to %s using posix.' % (src, dest))
os.symlink(src, dest)
def addLayerTwo(path, recreate=True):
# Don't do anything if this file is not located in dCache (/pnfs/...)
# or has nonzero size.
if larbatch_posix.exists(path) and path[
0:6] == '/pnfs/' and larbatch_posix.stat(path).st_size == 0:
if recreate:
print('Adding layer two for path %s.' % path)
else:
print('Deleting empty file %s.' % path)
# Now we got a zero size file in dCache, which kind of files may be
# missing layer two.
# Delete the file and recreate it using ifdh.
larbatch_posix.remove(path)
if not recreate:
return
test_proxy()
# Make sure environment variables X509_USER_CERT and X509_USER_KEY
# are not defined (they confuse ifdh).
save_vars = {}
for var in ('X509_USER_CERT', 'X509_USER_KEY'):
if var in os.environ:
save_vars[var] = os.environ[var]
del os.environ[var]
# Do ifdh cp.
command = ['ifdh', 'cp', '/dev/null', path]
jobinfo = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
q = queue.Queue()
thread = threading.Thread(target=wait_for_subprocess,
args=[jobinfo, q])
thread.start()
thread.join(timeout=60)
if thread.is_alive():
print('Terminating subprocess.')
jobinfo.terminate()
thread.join()
rc = q.get()
jobout = convert_str(q.get())
joberr = convert_str(q.get())
if rc != 0:
for var in list(save_vars.keys()):
os.environ[var] = save_vars[var]
raise IFDHError(command, rc, jobout, joberr)
# Restore environment variables.
for var in list(save_vars.keys()):
os.environ[var] = save_vars[var]
def posix_cp(source, destination):
cmd = ['cp', source, destination]
# Fork buffer process.
buffer_pid = os.fork()
if buffer_pid == 0:
# In child process.
# Launch cp subprocess.
jobinfo = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
q = Queue.Queue()
thread = threading.Thread(target=wait_for_subprocess,
args=[jobinfo, q])
thread.start()
thread.join(timeout=600)
if thread.is_alive():
# Subprocess did not finish (may be hanging and unkillable).
# Try to kill the subprocess and exit process.
# Unkillable process will become detached.
print 'Terminating subprocess.'
jobinfo.kill()
os._exit(1)
else:
# Subprocess finished normally.
rc = q.get()
jobout = q.get()
joberr = q.get()
os._exit(rc)
else:
# In parent process.
# Wait for buffer subprocess to finish.
buffer_result = os.waitpid(buffer_pid, 0)
rc = buffer_result[1] / 256
if rc != 0:
raise IFDHError(cmd, rc, '', '')
# Done.
return
def ifdh_rm(path):
# Get proxy.
test_proxy()
# Make sure environment variables X509_USER_CERT and X509_USER_KEY
# are not defined (they confuse ifdh, or rather the underlying tools).
save_vars = {}
for var in ('X509_USER_CERT', 'X509_USER_KEY'):
if os.environ.has_key(var):
save_vars[var] = os.environ[var]
del os.environ[var]
# Do delete.
cmd = ['ifdh', 'rm', path]
jobinfo = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
q = Queue.Queue()
thread = threading.Thread(target=wait_for_subprocess, args=[jobinfo, q])
thread.start()
thread.join(timeout=60)
if thread.is_alive():
print 'Terminating subprocess.'
jobinfo.terminate()
thread.join()
rc = q.get()
jobout = q.get()
joberr = q.get()
if rc != 0:
for var in save_vars.keys():
os.environ[var] = save_vars[var]
raise IFDHError(cmd, rc, jobout, joberr)
# Restore environment variables.
for var in save_vars.keys():
os.environ[var] = save_vars[var]
# Done.
return
def ifdh_mv(src, dest):
# Get proxy.
test_proxy()
# Make sure environment variables X509_USER_CERT and X509_USER_KEY
# are not defined (they confuse ifdh, or rather the underlying tools).
save_vars = {}
for var in ('X509_USER_CERT', 'X509_USER_KEY'):
if var in os.environ:
save_vars[var] = os.environ[var]
del os.environ[var]
# Do rename.
cmd = ['ifdh', 'mv', src, dest]
jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
q = queue.Queue()
thread = threading.Thread(target=wait_for_subprocess, args=[jobinfo, q])
thread.start()
thread.join(timeout=60)
if thread.is_alive():
print('Terminating subprocess.')
jobinfo.terminate()
thread.join()
rc = q.get()
jobout = convert_str(q.get())
joberr = convert_str(q.get())
if rc != 0:
for var in list(save_vars.keys()):
os.environ[var] = save_vars[var]
raise IFDHError(cmd, rc, jobout, joberr)
# Restore environment variables.
for var in list(save_vars.keys()):
os.environ[var] = save_vars[var]
# Done.
return
def readlink(path):
result = ''
# Make sure we have a kerberos ticket.
if path.startswith('/pnfs/') and not_pnfs_is_mounted:
if debug:
print(
'*** Larbatch_posix: Read symbolic link %s using nfs server.' %
path)
larbatch_utilities.test_ticket()
cmd = ['ssh', larbatch_utilities.nfs_server(), 'readlink', path]
jobinfo = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
q = queue.Queue()
thread = threading.Thread(
target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
thread.start()
thread.join(timeout=60)
if thread.is_alive():
if debug:
print('*** Larbatch_posix: Terminating subprocess.')
jobinfo.terminate()
thread.join()
rc = q.get()
jobout = convert_str(q.get())
joberr = convert_str(q.get())
if rc != 0:
raise IFDHError(cmd, rc, jobout, joberr)
result = jobout.strip()
else:
if debug:
print('*** Larbatch_posix: Read symbolic link %s using posix.' %
path)
result = os.readlink(path)
# Done.
return result