def test_good_dataset_content_restart(rucio_do_nothing, rucio_2file_dataset, cache_empty, simple_dataset):
dm0 = dataset_mgr(cache_empty, rucio_mgr=rucio_do_nothing)
_ = dm0.get_ds_contents(simple_dataset.Name)
wait_some_time(lambda: rucio_do_nothing.CountCalled == 0)
# Start up a new one that should pick up the ball where it was dropped.
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset)
wait_some_time(lambda: rucio_2file_dataset.CountCalled == 0)
# Now, make sure that we get back what we want here.
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
def test_dataset_download_restart_and_marked(rucio_do_nothing, rucio_2file_dataset_with_fails, cache_empty, simple_dataset):
rucio_2file_dataset_with_fails._cache_mgr = cache_empty
rucio_2file_dataset_with_fails.DLSleep = 0.05
# Trigger the download on one.
dm0 = dataset_mgr(cache_empty, rucio_mgr=rucio_do_nothing)
_ = dm0.download_ds(simple_dataset.Name)
wait_some_time(lambda: rucio_do_nothing.CountCalledDL == 0)
# Next, create a second one with the same cache.
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset_with_fails, seconds_between_retries=0.05)
wait_some_time(lambda: rucio_2file_dataset_with_fails.CountCalledDL < 2)
assert cache_empty.download_in_progress(simple_dataset.Name)
def test_dataset_download_restart(rucio_do_nothing, rucio_2file_dataset, cache_empty, simple_dataset):
rucio_2file_dataset._cache_mgr = cache_empty
# Trigger the download on one.
dm0 = dataset_mgr(cache_empty, rucio_mgr=rucio_do_nothing)
_ = dm0.download_ds(simple_dataset.Name)
wait_some_time(lambda: rucio_do_nothing.CountCalledDL == 0)
# Next, create a second one with the same cache.
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset)
wait_some_time(lambda: rucio_2file_dataset.CountCalledDL == 0)
status, _ = dm.download_ds(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
def test_dataset_download_good_nodownload(rucio_2file_dataset, cache_empty, simple_dataset):
'Queue a download and look for it to show up'
rucio_2file_dataset._cache_mgr = cache_empty
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset)
status, files = dm.download_ds(simple_dataset.Name, do_download=False)
assert None is files
assert DatasetQueryStatus.does_not_exist == status
def test_dataset_query_queued(rucio_2file_dataset, cache_empty):
'Queue a dataset'
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset)
status, files = dm.get_ds_contents('a_dataset')
# Should have queued the result since this was a new ds manager
assert status == DatasetQueryStatus.query_queued
assert None is files
def test_good_dataset_maxAgeIfNotSeenNoEffect(rucio_2file_dataset, cache_empty, simple_dataset):
'Do not requery for the dataset'
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset)
_ = dm.get_ds_contents(simple_dataset.Name)
wait_some_time(lambda: rucio_2file_dataset.CountCalled == 0)
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
# Query, but demand a quick re-check
status, _ = dm.get_ds_contents(simple_dataset.Name, maxAgeIfNotSeen=datetime.timedelta(seconds=0))
assert DatasetQueryStatus.results_valid == status
assert 1 == rucio_2file_dataset.CountCalled
def test_dataset_always_missing_longretry(rucio_2file_dataset_shows_up_later, cache_empty, simple_dataset):
'Do not requery for the dataset'
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset_shows_up_later)
_ = dm.get_ds_contents(simple_dataset.Name)
wait_some_time(lambda: rucio_2file_dataset_shows_up_later.CountCalled == 0)
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.does_not_exist == status
# Query, but demand a quick re-check
status, _ = dm.get_ds_contents(simple_dataset.Name, maxAgeIfNotSeen=datetime.timedelta(seconds=1000))
assert DatasetQueryStatus.does_not_exist == status
assert 1 == rucio_2file_dataset_shows_up_later.CountCalled
def test_dataset_download_logs(rucio_2file_dataset, cache_empty, simple_dataset):
'Queue a download and look for it to show up'
rucio_2file_dataset._cache_mgr = cache_empty
lg = dummy_logger()
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset, logger=lg)
_ = dm.download_ds(simple_dataset.Name)
# Wait for the dataset query to run
wait_some_time(lambda: rucio_2file_dataset.CountCalledDL == 0)
# Make sure some lines were sent to the logger
assert len(lg.lines) > 0
def test_dataset_download_no_exist(rucio_2file_dataset, cache_empty):
'Queue a download and look for it to show up'
rucio_2file_dataset._cache_mgr = cache_empty
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset)
_ = dm.download_ds('bogus')
# Wait for the dataset query to run
wait_some_time(lambda: rucio_2file_dataset.CountCalledDL == 0)
# Now, make sure that we get back what we want here.
status, files = dm.download_ds('bogus')
assert DatasetQueryStatus.does_not_exist == status
assert None is files
def test_look_for_good_dataset_that_fails_a_bunch(rucio_2file_dataset_with_fails, cache_empty, simple_dataset):
'Queue and look for a good dataset that takes a few queries to show up with results'
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset_with_fails, seconds_between_retries=0.01)
_ = dm.get_ds_contents(simple_dataset.Name)
# Wait for the dataset query to run
wait_some_time(lambda: rucio_2file_dataset_with_fails.CountCalled < 5)
# Now, make sure that we get back what we want and that the number of tries matches what we think
# it should have.
status, files = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
assert 5 == rucio_2file_dataset_with_fails.CountCalled
def test_dataset_appears(rucio_2file_dataset_shows_up_later, cache_empty, simple_dataset):
'After a bad dataset has aged, automatically queue a new query'
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset_shows_up_later)
_ = dm.get_ds_contents(simple_dataset.Name)
wait_some_time(lambda: rucio_2file_dataset_shows_up_later.CountCalled == 0)
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.does_not_exist == status
# Query, but demand a quick re-check
status, _ = dm.get_ds_contents(simple_dataset.Name, maxAgeIfNotSeen=datetime.timedelta(seconds=0))
assert DatasetQueryStatus.query_queued == status
wait_some_time(lambda: rucio_2file_dataset_shows_up_later.CountCalled == 1)
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
def test_query_for_bad_dataset(rucio_2file_dataset, cache_empty, simple_dataset):
'Ask for a bad dataset, and get back a null'
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset)
_ = dm.get_ds_contents('bogus_ds')
wait_some_time(lambda: rucio_2file_dataset.CountCalled == 0)
# Make sure it comes back as bad.
status, files = dm.get_ds_contents('bogus_ds')
assert DatasetQueryStatus.does_not_exist == status
assert None is files
# Make sure that a timeout of an hour has been set on the dataset.
info = cache_empty.get_listing('bogus_ds')
assert datetime.datetime.now() == info.Created
def test_dataset_download_with_failures(rucio_2file_dataset_with_fails, cache_empty, simple_dataset):
'Queue a download, it fails, but then gets there'
rucio_2file_dataset_with_fails._cache_mgr = cache_empty
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset_with_fails, seconds_between_retries=0.01)
_ = dm.download_ds(simple_dataset.Name)
# Wait for the dataset query to run
wait_some_time(lambda: rucio_2file_dataset_with_fails.CountCalledDL < 5)
# Now, make sure that we get back what we want here.
status, files = dm.download_ds(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
assert len(simple_dataset.FileList) == len(files)
# 2 failures, so make sure we re-try the right number of times
assert 5 == rucio_2file_dataset_with_fails.CountCalledDL
def test_dataset_query_resolved(rucio_2file_dataset, cache_empty, simple_dataset):
'Queue and look for a dataset query result'
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset)
_ = dm.get_ds_contents(simple_dataset.Name)
# Wait for the dataset query to run
wait_some_time(lambda: rucio_2file_dataset.CountCalled == 0)
# Now, make sure that we get back what we want here.
status, files = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
assert len(simple_dataset.FileList) == len(files)
# Make sure we didn't re-query for this.
assert 1 == rucio_2file_dataset.CountCalled == 1
_ = cache_empty.get_listing(simple_dataset.Name)
def test_dataset_download_good(rucio_2file_dataset, cache_empty, simple_dataset):
'Queue a download and look for it to show up'
rucio_2file_dataset._cache_mgr = cache_empty
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset)
_ = dm.download_ds(simple_dataset.Name)
# Wait for the dataset query to run
wait_some_time(lambda: rucio_2file_dataset.CountCalledDL == 0)
# Now, make sure that we get back what we want here.
status, files = dm.download_ds(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
assert len(simple_dataset.FileList) == len(files)
# Make sure we didn't re-query for this.
assert 1 == rucio_2file_dataset.CountCalledDL
def test_two_queries_for_good_dataset(rucio_2file_dataset_take_time, cache_empty, simple_dataset):
'Make sure second query does not trigger second web download'
# Query twice, make sure we don't forget as we are doing this!
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset_take_time)
_ = dm.get_ds_contents(simple_dataset.Name)
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.query_queued == status
# Wait for the dataset query to run
wait_some_time(lambda: rucio_2file_dataset_take_time.CountCalled == 0)
# Now, make sure that we get back what we want here.
status, _ = dm.get_ds_contents(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
# Make sure we didn't re-query for this, and the expiration date is not set.
# Make sure to wait long enough for other timing stuff above to fall apart.
sleep(0.02)
assert 1 == rucio_2file_dataset_take_time.CountCalled
def test_dataset_download_good_ask_twice(rucio_2file_dataset_take_time, cache_empty, simple_dataset):
'Be impatient about asking how things are going'
rucio_2file_dataset_take_time._cache_mgr = cache_empty
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset_take_time)
_ = dm.download_ds(simple_dataset.Name)
# Ask again.
status, _ = dm.download_ds(simple_dataset.Name)
assert DatasetQueryStatus.query_queued == status
# Wait for the dataset query to run
wait_some_time(lambda: rucio_2file_dataset_take_time.CountCalledDL == 0)
# Now, make sure that we get back what we want here.
status, _ = dm.download_ds(simple_dataset.Name)
assert DatasetQueryStatus.results_valid == status
# Make sure we didn't re-query for this.
sleep(0.02)
assert 1 == rucio_2file_dataset_take_time.CountCalledDL
# Some global things shared in the webserver.
from src.grid.datasets import dataset_mgr
from src.utils.dataset_cache_mgr import dataset_cache_mgr
# Init the dataset manager
dataset_cache = dataset_cache_mgr("/data")
datasets = dataset_mgr(dataset_cache)
# Get the cache prefix
import os
cache_prefix = None if 'CACHE_PREFIX' not in os.environ else os.environ['CACHE_PREFIX']
def test_dataset_with_prefix(rucio_do_nothing, cache_with_ds):
dm = dataset_mgr(cache_with_ds, rucio_mgr=rucio_do_nothing)
status, files = dm.download_ds('dataset1', prefix='file://cache/')
assert DatasetQueryStatus.results_valid == status
assert 2 == len(files)
assert 'file://cache/f1.root' == files[0]
def test_dataset_download_query(rucio_2file_dataset, cache_empty, simple_dataset):
'Queue a download and look for it to show up'
dm = dataset_mgr(cache_empty, rucio_mgr=rucio_2file_dataset)
status, files = dm.download_ds(simple_dataset.Name)
assert files is None
assert DatasetQueryStatus.query_queued == status