def test_dataset_download_restart(rucio_do_nothing, rucio_2file_dataset,
cache_empty, simple_dataset):
rucio_2file_dataset._cache_mgr = cache_empty
# Trigger the download on one.
_ = rucio_cache_interface(cache_empty, rucio_mgr=rucio_do_nothing)
# Next, create a second one with the same cache.
dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset)
status, _ = dm.download_ds(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
def test_dataset_download_log(rucio_2file_dataset, cache_empty,
simple_dataset):
'Queue a download and look for it to show up'
rucio_2file_dataset._cache_mgr = cache_empty
dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset)
ln = []
_ = dm.download_ds(simple_dataset.Name, log_func=lambda l: ln.append(l))
assert len(ln) > 0
def test_dataset_download_good_nodownload(rucio_2file_dataset, cache_empty,
simple_dataset):
'Queue a download and look for it to show up'
rucio_2file_dataset._cache_mgr = cache_empty
dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset)
status, files = dm.download_ds(simple_dataset.Name, do_download=False)
assert None is files
assert DatasetQueryStatus.does_not_exist == status
def test_dataset_query_resolved_logs(rucio_2file_dataset, cache_empty,
simple_dataset):
'Make sure the dataset is logged correctly'
dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset)
ln = []
_ = dm.get_ds_contents(simple_dataset.Name,
log_func=lambda l: ln.append(l))
assert len(ln) > 0
def test_dataset_download_no_exist(rucio_2file_dataset, cache_empty):
'Queue a download and look for it to show up'
rucio_2file_dataset._cache_mgr = cache_empty
dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset)
# Now, make sure that we get back what we want here.
status, files = dm.download_ds('bogus')
assert DatasetQueryStatus.does_not_exist == status
assert None is files
def test_two_downloads_fail(simple_dataset, cache_empty, rucio_do_nothing,
rucio_2file_dataset):
dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset)
# Pretend to lock the dataset
with cache_empty.get_dataset_downloading_lock(simple_dataset.Name):
try:
logging.debug("Starting second download")
dm.download_ds(simple_dataset.Name)
assert False
except RucioAlreadyBeingDownloaded:
return
def test_dataset_query_resolved(rucio_2file_dataset, cache_empty,
simple_dataset):
'Queue and look for a dataset query result'
dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset)
# Now, make sure that we get back what we want here.
status, files = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
assert len(simple_dataset.FileList) == len(files)
# Make sure we didn't re-query for this.
assert 1 == rucio_2file_dataset.CountCalled == 1
_ = cache_empty.get_listing(simple_dataset.Name)
def test_dataset_download_good(rucio_2file_dataset, cache_empty,
simple_dataset):
'Queue a download and look for it to show up'
rucio_2file_dataset._cache_mgr = cache_empty
dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset)
# Now, make sure that we get back what we want here.
status, files = dm.download_ds(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
assert files is not None
assert len(simple_dataset.FileList) == len(files)
# Make sure we didn't re-query for this.
assert 1 == rucio_2file_dataset.CountCalledDL
def test_good_dataset_maxAgeIfNotSeenNoEffect(rucio_2file_dataset, cache_empty,
simple_dataset):
'Do not requery for the dataset'
dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset)
_ = dm.get_ds_contents(simple_dataset.Name)
wait_some_time(lambda: rucio_2file_dataset.CountCalled == 0)
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
# Query, but demand a quick re-check
status, _ = dm.get_ds_contents(
simple_dataset.Name, maxAgeIfNotSeen=datetime.timedelta(seconds=0))
assert DatasetQueryStatus.results_valid == status
assert 1 == rucio_2file_dataset.CountCalled
def test_dataset_always_missing_noretry(rucio_2file_dataset_shows_up_later,
cache_empty, simple_dataset):
'Do not requery for the dataset'
dm = rucio_cache_interface(cache_empty,
rucio_mgr=rucio_2file_dataset_shows_up_later)
_ = dm.get_ds_contents(simple_dataset.Name)
wait_some_time(lambda: rucio_2file_dataset_shows_up_later.CountCalled == 0)
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.does_not_exist == status
# Query, but demand a quick re-check
status, _ = dm.get_ds_contents(simple_dataset.Name, maxAgeIfNotSeen=None)
assert DatasetQueryStatus.does_not_exist == status
assert 1 == rucio_2file_dataset_shows_up_later.CountCalled
def test_dataset_appears(rucio_2file_dataset_shows_up_later, cache_empty,
simple_dataset):
'After a bad dataset has aged, automatically queue a new query'
dm = rucio_cache_interface(cache_empty,
rucio_mgr=rucio_2file_dataset_shows_up_later)
_ = dm.get_ds_contents(simple_dataset.Name)
wait_some_time(lambda: rucio_2file_dataset_shows_up_later.CountCalled == 0)
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.does_not_exist == status
# Query, but demand a quick re-check
status, _ = dm.get_ds_contents(
simple_dataset.Name, maxAgeIfNotSeen=datetime.timedelta(seconds=0))
assert DatasetQueryStatus.results_valid == status
def test_query_for_bad_dataset(rucio_2file_dataset, cache_empty,
simple_dataset):
'Ask for a bad dataset, and get back a null'
dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset)
_ = dm.get_ds_contents('bogus_ds')
wait_some_time(lambda: rucio_2file_dataset.CountCalled == 0)
# Make sure it comes back as bad.
status, files = dm.get_ds_contents('bogus_ds')
assert DatasetQueryStatus.does_not_exist == status
assert None is files
# Make sure that a timeout of an hour has been set on the dataset.
info = cache_empty.get_listing('bogus_ds')
assert datetime.datetime.now() == info.Created
def test_look_for_good_dataset_that_fails_a_bunch(
rucio_2file_dataset_with_fails, cache_empty, simple_dataset):
'Queue and look for a good dataset that takes a few queries to show up with results'
dm = rucio_cache_interface(cache_empty,
rucio_mgr=rucio_2file_dataset_with_fails,
seconds_between_retries=0.01)
_ = dm.get_ds_contents(simple_dataset.Name)
# Wait for the dataset query to run
wait_some_time(lambda: rucio_2file_dataset_with_fails.CountCalled < 5)
# Now, make sure that we get back what we want and that the number of tries matches what we think
# it should have.
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
assert 5 == rucio_2file_dataset_with_fails.CountCalled
def test_two_queries_for_good_dataset(rucio_2file_dataset_take_time,
cache_empty, simple_dataset):
'Make sure second query does not trigger second web download'
# Query twice, make sure we don't forget as we are doing this!
dm = rucio_cache_interface(cache_empty,
rucio_mgr=rucio_2file_dataset_take_time)
_ = dm.get_ds_contents(simple_dataset.Name)
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
# Now, make sure that we get back what we want here.
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
# Make sure we didn't re-query for this, and the expiration date is not set.
# Make sure to wait long enough for other timing stuff above to fall apart.
assert 1 == rucio_2file_dataset_take_time.CountCalled
def test_dataset_download_with_failures(rucio_2file_dataset_with_fails,
cache_empty, simple_dataset):
'Queue a download, it fails, but then gets there'
rucio_2file_dataset_with_fails._cache_mgr = cache_empty
dm = rucio_cache_interface(cache_empty,
rucio_mgr=rucio_2file_dataset_with_fails,
seconds_between_retries=0.01)
_ = dm.download_ds(simple_dataset.Name)
# Wait for the dataset query to run
wait_some_time(lambda: rucio_2file_dataset_with_fails.CountCalledDL < 5)
# Now, make sure that we get back what we want here.
status, files = dm.download_ds(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
assert len(simple_dataset.FileList) == len(files)
# 2 failures, so make sure we re-try the right number of times
assert 5 == rucio_2file_dataset_with_fails.CountCalledDL