def writefile(filename, ruid, rgid, content, noversion=0):
'''Write a file via xroot on behalf of the given uid,gid. The entire content is written
and any pre-existing file is deleted (or moved to the previous version if supported).
If noversion=1, the write explicitly disables versioning: this is useful for lock files.'''
size = len(content)
log.debug('msg="Invoking writeFile" filename="%s" size="%d"' % (filename, size))
if not xrdfs:
raise ValueError
f = XrdClient.File()
tstart = time.clock()
rc, statInfo_unused = f.open(storageserver + '/' + homepath + filename + _eosargs(ruid, rgid, 1, size) + \
('&sys.versioning=0' if noversion else ''), OpenFlags.DELETE)
tend = time.clock()
log.info('msg="File open for write" filename="%s" elapsedTimems="%.1f"' % (filename, (tend-tstart)*1000))
if not rc.ok:
log.warning('msg="Error opening the file for write" filename="%s" error="%s"' % (filename, rc.message.strip('\n')))
raise IOError(rc.message.strip('\n'))
# write the file. In a future implementation, we should find a way to only update the required chunks...
rc, statInfo_unused = f.write(content, offset=0, size=size)
if not rc.ok:
log.warning('msg="Error writing the file" filename="%s" error="%s"' % (filename, rc.message.strip('\n')))
raise IOError(rc.message.strip('\n'))
rc, statInfo_unused = f.truncate(size)
if not rc.ok:
log.warning('msg="Error truncing the file" filename="%s" error="%s"' % (filename, rc.message.strip('\n')))
raise IOError(rc.message.strip('\n'))
rc, statInfo_unused = f.close()
if not rc.ok:
log.warning('msg="Error closing the file" filename="%s" error="%s"' % (filename, rc.message.strip('\n')))
raise IOError(rc.message.strip('\n'))
def test_readlines_small():
f = client.File()
f.open(smallfile, OpenFlags.DELETE, open_mode)
f.write(smallbuffer)
f.close()
pylines = open('/tmp/spam').readlines()
for i in range(1, 100):
f = client.File()
f.open(smallfile)
response = f.readlines(offset=0, chunksize=i)
assert len(response) == 4
for j, line in enumerate(response):
if pylines[j].endswith('\n'):
assert line.endswith('\n')
f.close()
def test_filesystem():
c = client.FileSystem(SERVER_URL)
funcspecs = [
(c.locate, ('/tmp', OpenFlags.REFRESH), True),
(c.deeplocate, ('/tmp', OpenFlags.REFRESH), True),
(c.query, (QueryCode.SPACE, '/tmp'), True),
(c.truncate, ('/tmp/spam', 1000), False),
(c.mv, ('/tmp/spam', '/tmp/ham'), False),
(c.chmod, ('/tmp/ham', AccessMode.UR | AccessMode.UW), False),
(c.rm, ('/tmp/ham', ), False),
(c.mkdir, ('/tmp/somedir', MkDirFlags.MAKEPATH), False),
(c.rmdir, ('/tmp/somedir', ), False),
(c.ping, (), False),
(c.stat, ('/tmp', ), True),
(c.statvfs, ('/tmp', ), True),
(c.protocol, (), True),
(c.dirlist, ('/tmp', DirListFlags.STAT), True),
(c.sendinfo, ('important info', ), False),
(c.prepare, (['/tmp/foo'], PrepareFlags.STAGE), True),
]
for func, args, hasReturnObject in funcspecs:
sync(func, args, hasReturnObject)
# Create new temp file
f = client.File()
status, response = f.open(smallfile, OpenFlags.NEW)
for func, args, hasReturnObject in funcspecs:
async (func, args, hasReturnObject)
def test_write_big_async():
f = client.File()
pytest.raises(ValueError, 'f.read()')
status, __ = f.open(bigfile, OpenFlags.DELETE, open_mode)
assert status.ok
rand_data = os.urandom(64 * 1024)
max_size = 512 * 1024 # 512 K
offset = 0
lst_handlers = []
while offset <= max_size:
status, __ = f.write(smallbuffer)
assert status.ok
handler = AsyncResponseHandler()
lst_handlers.append(handler)
status = f.write(rand_data, offset, callback=handler)
assert status.ok
offset = offset + len(smallbuffer) + len(rand_data)
# Wait for async write responses
for handler in lst_handlers:
status, __, __ = handler.wait()
assert status.ok
f.close()
def readfile(endpoint, filepath, userid):
'''Read a file via xroot on behalf of the given userid. Note that the function is a generator, managed by Flask.'''
log.debug('msg="Invoking readFile" filepath="%s"' % filepath)
with XrdClient.File() as f:
fileurl = _geturlfor(endpoint) + '/' + homepath + filepath + _eosargs(
userid)
tstart = time.time()
rc, statInfo_unused = f.open(fileurl, OpenFlags.READ)
tend = time.time()
if not rc.ok:
# the file could not be opened: check the case of ENOENT and log it as info to keep the logs cleaner
if 'No such file or directory' in rc.message:
log.info('msg="File not found on read" filepath="%s"' %
filepath)
yield IOError('No such file or directory')
else:
log.warning('msg="Error opening the file for read" filepath="%s" code="%d" error="%s"' % \
(filepath, rc.shellcode, rc.message.strip('\n')))
yield IOError(rc.message)
else:
log.info(
'msg="File open for read" filepath="%s" elapsedTimems="%.1f"' %
(filepath, (tend - tstart) * 1000))
chunksize = config.getint('io', 'chunksize')
rc, statInfo = f.stat()
chunksize = min(chunksize, statInfo.size)
# the actual read is buffered and managed by the Flask server
for chunk in f.readchunks(offset=0, chunksize=chunksize):
yield chunk
def readfile(filename, ruid, rgid):
'''Read a file via xroot on behalf of the given uid,gid. Note that the function is a generator, managed by Flask.'''
log.debug('msg="Invoking readFile" filename="%s"' % filename)
if not xrdfs:
raise ValueError
with XrdClient.File() as f:
fileurl = storageserver + '/' + homepath + filename + _eosargs(ruid, rgid)
tstart = time.clock()
rc, statInfo_unused = f.open(fileurl, OpenFlags.READ)
tend = time.clock()
log.info('msg="File open for read" filename="%s" elapsedTimems="%.1f"' % (filename, (tend-tstart)*1000))
if not rc.ok:
# the file could not be opened: check the case of ENOENT and log it as info to keep the logs cleaner
if 'No such file or directory' in rc.message:
log.info('msg="Error opening the file for read" filename="%s" error="No such file or directory"' % filename)
else:
log.warning('msg="Error opening the file for read" filename="%s" error="%s"' % (filename, rc.message.strip('\n')))
# as this is a generator, we yield the error string instead of the file's contents
yield rc.message
else:
chunksize = config.getint('io', 'chunksize')
rc, statInfo = f.stat()
chunksize = min(chunksize, statInfo.size-1)
# the actual read is buffered and managed by the Flask server
for chunk in f.readchunks(offset=0, chunksize=chunksize):
yield chunk
def _xrootcmd(endpoint, cmd, subcmd, userid, args):
'''Perform the <cmd>/<subcmd> action on the special /proc/user path on behalf of the given userid.
Note that this is entirely EOS-specific.'''
with XrdClient.File() as f:
url = _geturlfor(endpoint) + '//proc/user/' + _eosargs(userid) + '&mgm.cmd=' + cmd + \
('&mgm.subcmd=' + subcmd if subcmd else '') + '&' + args
tstart = time.time()
rc, statInfo_unused = f.open(url, OpenFlags.READ)
tend = time.time()
log.info(
'msg="Invoked _xrootcmd" cmd="%s%s" url="%s" elapsedTimems="%.1f"'
% (cmd, ('/' + subcmd if subcmd else ''), url,
(tend - tstart) * 1000))
res = f.readline().decode('utf-8').strip('\n').split('&')
if len(
res
) == 3: # we may only just get stdout: in that case, assume it's all OK
rc = res[2]
rc = rc[rc.find('=') + 1:]
if rc != '0':
# failure: get info from stderr, log and raise
msg = res[1][res[1].find('=') + 1:]
log.info('msg="Error with xroot command" cmd="%s" subcmd="%s" args="%s" error="%s" rc="%s"' % \
(cmd, subcmd, args, msg, rc.strip('\00')))
raise IOError(msg)
# all right, return everything that came in stdout
return res[0][res[0].find('stdout=') + 7:]
def checkURL(self, site):
if (self.verbose): print("check URL %s" % (site))
urls = list(self.info[site]['urls'])
if (site == "T3_KR_KNU"):
urls.append("root://cluster142.knu.ac.kr:1094//store/$1")
for url in urls:
test_PFN = url.replace("/store/$1", self.LFNPath)
### Global Redirector will be dropped for Accessing.
if "cmsxrootd.fnal.gov" in test_PFN: continue
if "xrootd-cms.infn.it" in test_PFN: continue
if "cms-xrd-global.cern.ch" in test_PFN: continue
if (self.verbose):
print("Trying to using " + test_PFN)
with client.File() as f:
result = f.open(test_PFN, OpenFlags.READ,
XRootD.client.flags.AccessMode.NONE, 5)[0]
if result.ok:
self.mapped[self.LFNPath]['pfnChecked'] = True
self.mapped[self.LFNPath]['accessChecked'] = True
self.mapped[self.LFNPath]['PFNPath'] = test_PFN
self.info['preferredURL'] = url
print("Successfully find the file at [%s]" % site)
return result.ok
def backup_write_status(self, lst_failed, check_ok):
""" Create backup status file which constains the list of failed files
to transfer.
Args:
lst_filed (list): List of failed file transfers
check_ok (boolean): True if verification successful, otherwise
false
"""
if not check_ok:
self.logger.error("Failed verification for {0} entries".format(
len(lst_failed)))
fn_status = ''.join([
self.efile_root, ".sys.b#.backup.err.",
str(len(lst_failed)), "?eos.ruid=0&eos.rgid=0"
])
else:
self.logger.info("Backup successful - no errors detected")
fn_status = ''.join(
[self.efile_root, ".sys.b#.backup.done?eos.ruid=0&eos.rgid=0"])
with client.File() as f:
f.open(fn_status.encode("utf-8"),
OpenFlags.UPDATE | OpenFlags.DELETE)
offset = 0
for entry in lst_failed:
buff = "Failed entry={0}\n".format(entry).encode("utf-8")
f.write(buff, offset, len(buff))
offset += len(buff)
def _xrootcmd(cmd, subcmd, ruid, rgid, args):
'''Perform the <cmd>/<subcmd> action on the special /proc/user path on behalf of the given uid,gid.
Note that this is entirely EOS-specific.'''
if not xrdfs:
raise ValueError
with XrdClient.File() as f:
url = storageserver + '//proc/user/' + _eosargs(ruid, rgid) + '&mgm.cmd=' + cmd + \
('&mgm.subcmd=' + subcmd if subcmd else '') + '&' + args
log.debug('msg="Invoking _xrootcmd" cmd="%s%s" url="%s"' %
(cmd, ('/' + subcmd if subcmd else ''), url))
rc, statInfo_unused = f.open(url, OpenFlags.READ)
res = f.readline().strip('\n').split('&')
if len(
res
) == 3: # we may only just get stdout: in that case, assume it's all OK
rc = res[2]
rc = rc[rc.find('=') + 1:]
if rc != '0':
# failure: get info from stderr, log and raise
msg = res[1][res[1].find('=') + 1:]
log.info('msg="Error with xroot command" cmd="%s" subcmd="%s" args="%s" error="%s" rc="%s"' % \
(cmd, subcmd, args, msg, rc.strip('\00')))
raise IOError(msg)
# all right, return everything that came in stdout
return res[0][res[0].find('stdout=') + 7:]
def writefile(filename, ruid, rgid, content):
'''Write a file via xroot on behalf of the given uid,gid. The entire content is written
and any pre-existing file is deleted.'''
log.debug('msg="Invoking writeFile" filename="%s"' % filename)
size = len(content)
log.debug('msg="Invoking writeFile" filename="%s" size="%d"' %
(filename, size))
if not xrdfs:
raise ValueError
f = XrdClient.File()
rc, statInfo_unused = f.open(
storageserver + '/' + homepath + filename +
_eosargs(ruid, rgid, 1, size), OpenFlags.DELETE)
if not rc.ok:
log.info(
'msg="Error opening the file for write" filename="%s" error="%s"' %
(filename, rc.message.strip('\n')))
raise IOError(rc.message.strip('\n'))
# write the file. In a future implementation, we should find a way to only update the required chunks...
rc, statInfo_unused = f.write(content, offset=0, size=size)
if not rc.ok:
log.warning('msg="Error writing the file" filename="%s" error="%s"' %
(filename, rc.message.strip('\n')))
raise IOError(rc.message.strip('\n'))
rc, statInfo_unused = f.truncate(size)
if not rc.ok:
log.warning('msg="Error truncing the file" filename="%s" error="%s"' %
(filename, rc.message.strip('\n')))
raise IOError(rc.message.strip('\n'))
rc, statInfo_unused = f.close()
if not rc.ok:
log.warning('msg="Error closing the file" filename="%s" error="%s"' %
(filename, rc.message.strip('\n')))
raise IOError(rc.message.strip('\n'))
def main():
# open the file as a normal text file and read it
with client.File() as f:
f.open(server + path + "/board_mapping.json")
status, jsonStr = f.read()
# pass the read string to getBoardMapping()
return getBoardMapping(jsonStr)
def test_sync_sync():
f = client.File()
pytest.raises(ValueError, 'f.sync()')
status, __ = f.open(bigfile)
assert status.ok
status, __ = f.sync()
assert status.ok
f.close()
def asynInfo(run):
server = os.environ['EOSSHIP']
with client.File() as f:
location = "/eos/experiment/sndlhc/raw_data/commissioning/TB_H8_october/run_logs/log_" + str(
run) + ".txt"
f.open(server + location)
status, L = f.read()
A = L.decode().split('\n')
f.close()
info = {}
for x in A:
X = x.replace('\n', '')
if X == '': continue
if X.find('2021') == 0:
tag = X
info[tag] = {}
else:
if X.find('disc') > 0: info[tag]['summary'] = eval(X)
else: info[tag]['detail'] = eval(X)
# correction procedure
A = {41: {}, 43: {}, 59: {}, 60: {}}
C = {}
prev = {41: None, 43: None, 59: None, 60: None}
errorCounter = 0
T = list(info.keys())
T.sort()
for tag in T:
test = info[tag]['summary']
lastT = test['last_timestamp']
if not 'desync_info' in test: continue
C[tag] = {41: {}, 43: {}, 59: {}, 60: {}}
for L in test['desync_info']:
board = L[0]
t = L[1][0]
c = L[1][1]
C[tag][board][c] = t
# check that desync is constant over the complete run
for c in C[tag][41]:
tref = C[tag][41][c]
break
for board in C[tag]:
if not c in C[tag][board]:
#print('different trigger counts, problematic',tag,C[tag])
errorCounter += 1
else:
if not prev[board] == None:
if not C[tag][board][c] - tref == prev[board]:
print('desync changed, problematic', tag, prev, C[tag])
prev[board] = C[tag][board][c] - tref
T = list(C.keys())
for b in C[T[0]]:
for c in C[T[0]][b]:
A[b] = C[T[0]][b][c]
return A
def test_truncate_sync():
f = client.File()
pytest.raises(ValueError, 'f.truncate(10000)')
status, __ = f.open(smallfile, OpenFlags.DELETE)
assert status.ok
status, __ = f.truncate(size=10000)
assert status.ok
f.close()
def test_io_limits():
f = client.File()
pytest.raises(ValueError, 'f.read()')
status, __ = f.open(smallfile, OpenFlags.UPDATE)
assert status.ok
status, __ = f.stat()
assert status.ok
# Test read limits
pytest.raises(TypeError, 'f.read(0, [1, 2])')
pytest.raises(TypeError, 'f.read([1, 2], 0)')
pytest.raises(TypeError, 'f.read(0, 10, [0, 1, 2])')
pytest.raises(OverflowError, 'f.read(0, -10)')
pytest.raises(OverflowError, 'f.read(-1, 1)')
pytest.raises(OverflowError, 'f.read(0, 1, -1)')
pytest.raises(OverflowError, 'f.read(0, 10**11)')
pytest.raises(OverflowError, 'f.read(0, 10, 10**6)')
# Test readline limits
pytest.raises(TypeError, 'f.readline([0, 1], 1)')
pytest.raises(TypeError, 'f.readline(0, [0, 1])')
pytest.raises(TypeError, 'f.readline(0, 10, [0, 1])')
pytest.raises(OverflowError, 'f.readline(-1, 1)')
pytest.raises(OverflowError, 'f.readline(0, -1)')
pytest.raises(OverflowError, 'f.readline(0, 1, -1)')
pytest.raises(OverflowError, 'f.readline(0, 10**11)')
pytest.raises(OverflowError, 'f.readline(0, 10, 10**11)')
# Test write limits
data = "data that will never get written"
pytest.raises(TypeError, 'f.write(data, 0, [1, 2])')
pytest.raises(TypeError, 'f.write(data, [1, 2], 0)')
pytest.raises(TypeError, 'f.write(data, 0, 10, [0, 1, 2])')
pytest.raises(OverflowError, 'f.write(data, 0, -10)')
pytest.raises(OverflowError, 'f.write(data, -1, 1)')
pytest.raises(OverflowError, 'f.write(data, 0, 1, -1)')
pytest.raises(OverflowError, 'f.write(data, 0, 10**11)')
pytest.raises(OverflowError, 'f.write(data, 0, 10, 10**6)')
# Test vector_read limits
pytest.raises(TypeError, 'f.vector_read(chunks=100)')
pytest.raises(TypeError, 'f.vector_read(chunks=[1,2,3])')
pytest.raises(TypeError, 'f.vector_read(chunks=[("lol", "cakes")])')
pytest.raises(TypeError, 'f.vector_read(chunks=[(1), (2)])')
pytest.raises(TypeError, 'f.vector_read(chunks=[(1, 2), (3)])')
pytest.raises(OverflowError,
'f.vector_read(chunks=[(-1, -100), (-100, -100)])')
pytest.raises(OverflowError, 'f.vector_read(chunks=[(0, 10**10*10)])')
# Test truncate limits
pytest.raises(TypeError, 'f.truncate(0, [1, 2])')
pytest.raises(TypeError, 'f.truncate([1, 2], 0)')
pytest.raises(OverflowError, 'f.truncate(-1)')
pytest.raises(OverflowError, 'f.truncate(100, -10)')
pytest.raises(OverflowError, 'f.truncate(0, 10**6)')
status, __ = f.close()
assert status.ok
def cat(self, path):
from XRootD import client
with client.File() as f:
f.open(path)
output = b''.join(f.readlines())
if sys.version_info < (3, ):
return output
else:
return output.decode()
def main():
parser = argparse.ArgumentParser(
description='Merge csv cutflow tables from parallel condor runs.')
parser.add_argument('files', metavar='File', nargs='*', help='File name')
parser.add_argument('-l',
'--list',
metavar='list_file',
dest='list',
nargs='?',
help='File with list of csv filenames')
args = parser.parse_args()
if args.list is not None:
with open(args.list) as f:
csv_list = f.read().splitlines()
elif len(args.files) > 1:
csv_list = args.files
else:
print(
"Error! Need at least two files to merge or a list of files. Exiting..."
)
exit()
total_df = pd.DataFrame()
for csv_name in csv_list:
with client.File() as f:
print("Opening file ", csv_name)
f.open(csv_name, OpenFlags.READ)
try:
status, data = f.read()
except:
print("Unexpected error when reading file with XRootD:",
sys.exc_info()[0])
print("Skipping file...")
continue
data_as_file = StringIO(data.decode('utf-8').replace(',', ''))
df = pd.read_csv(data_as_file, delimiter=' ')
df.set_index(['Cut#', 'Description'], inplace=True)
total_df = df.copy() if total_df.empty else total_df.add(
df, fill_value=0)
# Rearrange columns to bring 'Data' up front and signal columns to the back
columns = list(total_df.columns.values)
if 'Data' in columns:
columns.remove('Data')
columns.insert(0, 'Data')
for col in columns:
if 'sig' in col:
columns.remove(col)
columns.insert(-1, col)
total_df = total_df[columns]
print("Total df")
print(total_df)
total_df.to_csv('merged_cutflow.csv', sep=' ', float_format='%g')
def test_threads():
f = client.File()
# f.open(smallfile, OpenFlags.DELETE)
# assert f.is_open()
# f.write(smallbuffer)
for i in xrange(3):
tt = TestThread(f, i)
tt.start()
tt.join()
def _iter_xrootd(filepath, chunksize, gzipped):
with client.File() as f:
f.open(filepath, OpenFlags.READ)
if gzipped:
dec = zlib.decompressobj(32 + zlib.MAX_WBITS)
for chunk in f.readchunks(offset=0, chunksize=chunksize):
yield dec.decompress(chunk)
else:
for chunk in f.readchunks(offset=0, chunksize=chunksize):
yield chunk
def test_sync_async():
f = client.File()
status, response = f.open(bigfile)
assert status.ok
handler = AsyncResponseHandler()
status = f.sync(callback=handler)
status, __, __ = handler.wait()
assert status.ok
f.close()
def test_UPDATE(tmppath):
fname, fconts = tstfile_a(tmppath)
ffpath = join(tmppath, fname)
xf = xclient.File()
xf.open(mkurl(ffpath), OpenFlags.UPDATE)
assert xf
# Can we read?
statmsg, content = xf.read()
assert content == fconts.encode()
assert statmsg.ok
# Can we write?
statmsg, res = xf.write('chhhh-eck it')
# assert xf.read()[1] == content
# doesn't truncate file
print((statmsg, res))
assert statmsg.ok
assert not statmsg.error
# Can we truncate?
statmsg, res = xf.truncate(0)
print((statmsg, res))
assert statmsg.ok
assert not statmsg.error
value = xf.read()[1]
assert value == b""
# what if the file doesn't exist?
ffpath = join(tmppath, "newfile")
xf = xclient.File()
statmsg, res = xf.open(mkurl(ffpath), OpenFlags.UPDATE)
print((statmsg, res))
assert not statmsg.ok
assert statmsg.error
xf = xclient.File()
ffpath = join(tmppath, "newfile")
statmsg, res = xf.open(mkurl(ffpath), OpenFlags.NEW)
print((statmsg, res))
assert statmsg.ok
assert not statmsg.error
def exec_cmd(cmd):
""" Execute an EOS /proc/user/ command.
Args:
cmd (str): Command to execute.
Returns:
Tuple containing the following elements: (status, stdout, stderr). Status
is a boolean value while the rest are string. If data needs to be returned
then it's put in stdout and any error messages are in stderr.
"""
logger.debug("Execute: {0}".format(cmd))
status, retc, stdout, stderr = False, "0", "", ""
# Execute the command as root if role not already set
if cmd.find("eos.ruid=") == -1:
if cmd.find('?') == -1:
cmd += "?eos.ruid=0&eos.rgid=0"
else:
cmd += "&eos.ruid=0&eos.rgid=0"
with client.File() as f:
st, __ = f.open(cmd.encode("utf-8"), OpenFlags.READ)
if st.ok:
# Read the whole response
data = ""
off, sz = 0, 4096
st, chunk = f.read(off, sz)
if st.ok:
while st.ok and len(chunk):
off += len(chunk)
data += chunk.decode("utf-8")
st, chunk = f.read(off, sz)
lpairs = data.split('&')
for elem in lpairs:
if "mgm.proc.retc=" in elem:
retc = elem[(elem.index('=') + 1):].strip()
status = True if (retc == "0") else False
elif "mgm.proc.stdout=" in elem:
stdout = elem[(elem.index('=') + 1):].strip()
stdout = unseal_path(stdout)
elif "mgm.proc.stderr=" in elem:
stderr = elem[(elem.index('=') + 1):].strip()
stderr = unseal_path(stdout)
else:
stderr = "error reading response for command: {0}".format(cmd)
else:
stderr = "error sending command: {0}".format(cmd)
# logger.debug("Return command: {0}".format((status, stdout, stderr)))
return (status, stdout, stderr)
def test_copy_bigfile(): f = client.File() s, r = f.open(bigfile) assert s.ok size1 = f.stat(force=True)[1].size f.close() c = client.CopyProcess() c.add_job( source=bigfile, target=bigcopy, force=True ) s = c.prepare() assert s.ok s, __ = c.run() assert s.ok f = client.File() s, r = f.open(bigcopy, OpenFlags.READ) size2 = f.stat()[1].size assert size1 == size2 f.close()
def test_truncate_async():
f = client.File()
status, __ = f.open(smallfile, OpenFlags.DELETE)
assert status.ok
handler = AsyncResponseHandler()
status = f.truncate(size=10000, callback=handler)
assert status.ok
status, __, __ = handler.wait()
assert status.ok
f.close()
def create_event_file(self):
"""
Creates an event file in the xrootd filesystem (if it doesn't exist)
"""
self.log.debug("Creating initial event file... (did you remember to "
"start xrootd? ;)")
status, statinfo = self.filesystem.stat("/tmp/event.txt")
if statinfo is None:
with client.File() as f:
f.open("root://localhost//tmp/event.txt", OpenFlags.NEW)
def test_readchunks_big():
f = client.File()
f.open(bigfile, OpenFlags.READ)
size = f.stat()[1].size
total = 0
for chunk in f.readchunks(chunksize=1024 * 1024 * 2):
total += len(chunk)
assert total == size
f.close()
def test_read_sync():
f = client.File()
pytest.raises(ValueError, 'f.read()')
status, response = f.open(bigfile, OpenFlags.READ)
assert status.ok
status, response = f.stat()
size = response.size
status, response = f.read()
assert status.ok
assert len(response) == size
f.close()
def file_size_xrootd(path, *args, **kwargs):
"""File size."""
if current_app.config['XROOTD_ENABLED'] and \
current_app.config['VIDEOS_XROOTD_ENDPOINT'] in path:
from XRootD import client
f = client.File()
status, response = f.open(path)
assert status.ok
status, response = f.stat()
assert status.ok
return response.size
return os.path.getsize(path)
def test_readchunks_small():
f = client.File()
f.open(smallfile, OpenFlags.READ)
size = f.stat()[1].size
total = 0
chunks = ['gre', '\0en', '\neg', 'gs\n', 'and', '\nha', 'm\n']
for i, chunk in enumerate(f.readchunks(chunksize=3)):
assert chunk == chunks[i]
total += len(chunk)
assert total == size
f.close()
