def test_logging(rucio_good_file_listing):
r = rucio(executor=rucio_good_file_listing)
lines = []
files = r.get_file_listing(
"mc16_13TeV:mc16_13TeV.311313.MadGraphPythia8EvtGen_A14NNPDF31LO_HSS_LLP_mH125_mS35_lthigh.deriv.DAOD_EXOT15.e7270_e5984_s3234_r10724_r10726_p3795",
log_func=lambda l: lines.append(l))
assert len(lines) == 21
def test_download_logging(rucio_good_ds_download):
r = rucio(executor=rucio_good_ds_download)
lines = []
r.download_files(
"mc16_13TeV.311309.MadGraphPythia8EvtGen_A14NNPDF31LO_HSS_LLP_mH125_mS5_ltlow.deriv.DAOD_EXOT15.e7270_e5984_s3234_r10201_r10210_p3795",
'/data',
log_func=lambda l: lines.append(l))
assert len(lines) > 10
assert any(['Files already found locally' in l for l in lines])
def test_download_bad_cert(rucio_no_cert_download):
try:
r = rucio(executor=rucio_no_cert_download)
r.download_files(
"mc16_13TeV.311309.MadGraphPythia8EvtGen_A14NNPDF31LO_HSS_LLP_mH125_mS5_ltlow.deriv.DAOD_EXOT15.e7270_e5984_s3234_r10201_r10210_p3795",
'/data')
assert False
except RucioException:
return
assert False
def test_no_cert(rucio_no_cert):
try:
r = rucio(executor=rucio_no_cert)
_ = r.get_file_listing(
"mc16_13TeV:mc16_13TeV.311313.MadGraphPythia8EvtGen_A14NNPDF31LO_HSS_LLP_mH125_mS35_lthigh.deriv.DAOD_EXOT15.e7270_e5984_s3234_r10724_r10726_p3795"
)
except RucioException as e:
assert "Try again" in str(e)
return
assert False
def test_good_file_list(rucio_good_file_listing):
r = rucio(executor=rucio_good_file_listing)
files = r.get_file_listing(
"mc16_13TeV:mc16_13TeV.311313.MadGraphPythia8EvtGen_A14NNPDF31LO_HSS_LLP_mH125_mS35_lthigh.deriv.DAOD_EXOT15.e7270_e5984_s3234_r10724_r10726_p3795"
)
assert 13 == len(files)
f_dict = {info.filename: info for info in files}
assert 30000 == f_dict[
"mc16_13TeV:DAOD_EXOT15.17545540._000013.pool.root.1"].events
assert int(
5.956 * 1024 * 1024 * 1024
) == f_dict["mc16_13TeV:DAOD_EXOT15.17545540._000013.pool.root.1"].size
def test_download_status(rucio_good_ds_download_takes_time):
d_status = dummy_status()
r = rucio(executor=rucio_good_ds_download_takes_time, status=d_status)
# Run in a different thread so that things can get going.
from concurrent.futures import ThreadPoolExecutor
tp = ThreadPoolExecutor(max_workers=1)
res_future = tp.submit(
rucio.download_files, r,
"mc16_13TeV.311309.MadGraphPythia8EvtGen_A14NNPDF31LO_HSS_LLP_mH125_mS5_ltlow.deriv.DAOD_EXOT15.e7270_e5984_s3234_r10201_r10210_p3795dude",
'/data')
while not rucio_good_ds_download_takes_time.Running:
sleep(0.005)
assert d_status.got('rucio_download')
v = d_status.status_value('rucio_download', 'downloading')
assert len(v) == 1
res_future.result()
assert 0 == len(d_status.status_value('rucio_download', 'downloading'))
tp.shutdown()
def __init__(self, data_mgr:dataset_cache_mgr, rucio_mgr: Optional[rucio] = None, seconds_between_retries:float=60.0*5, logger:logging_mgr = None):
'''
Setup a dataset_mgr
Arguments:
rucio_mgr Interface to query rucio directly to get back dataset file results.
'''
# We want to query rucio one dataset at a time.
self._exe = ThreadPoolExecutor(max_workers=1)
self._downloader = ThreadPoolExecutor(max_workers=3)
self._rucio = rucio_mgr if rucio_mgr is not None else rucio()
self._cache_mgr = data_mgr
self._seconds_between_retries = seconds_between_retries
self._log = logger if logger is not None else logging_mgr()
# Look for pending downloads and directory listings to see if we
# should resume anything.
self._check_for_pending_downloads()
self._check_for_pending_queries()
def test_download_bad_ds(rucio_bad_ds_download):
r = rucio(executor=rucio_bad_ds_download)
res = r.download_files(
"mc16_13TeV.311309.MadGraphPythia8EvtGen_A14NNPDF31LO_HSS_LLP_mH125_mS5_ltlow.deriv.DAOD_EXOT15.e7270_e5984_s3234_r10201_r10210_p3795dude",
'/data')
assert None is res
def test_bad_ds_name(rucio_bad_ds_name):
r = rucio(executor=rucio_bad_ds_name)
assert None is r.get_file_listing(
"mc16_13TeV:mc16_13TeV.311313.MadGraphPythia8EvtGen_A14NNPDF31LO_HSS_LLP_mH125_mS35_lthigh.deriv.DAOD_EXOT15.bogus"
)
def test_rucio_ctor():
_ = rucio()