def test_dataset_download_restart(rucio_do_nothing, rucio_2file_dataset, cache_empty, simple_dataset): rucio_2file_dataset._cache_mgr = cache_empty # Trigger the download on one. _ = rucio_cache_interface(cache_empty, rucio_mgr=rucio_do_nothing) # Next, create a second one with the same cache. dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset) status, _ = dm.download_ds(simple_dataset.Name) assert DatasetQueryStatus.results_valid == status
def test_dataset_download_log(rucio_2file_dataset, cache_empty, simple_dataset): 'Queue a download and look for it to show up' rucio_2file_dataset._cache_mgr = cache_empty dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset) ln = [] _ = dm.download_ds(simple_dataset.Name, log_func=lambda l: ln.append(l)) assert len(ln) > 0
def test_dataset_download_good_nodownload(rucio_2file_dataset, cache_empty, simple_dataset): 'Queue a download and look for it to show up' rucio_2file_dataset._cache_mgr = cache_empty dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset) status, files = dm.download_ds(simple_dataset.Name, do_download=False) assert None is files assert DatasetQueryStatus.does_not_exist == status
def test_dataset_query_resolved_logs(rucio_2file_dataset, cache_empty, simple_dataset): 'Make sure the dataset is logged correctly' dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset) ln = [] _ = dm.get_ds_contents(simple_dataset.Name, log_func=lambda l: ln.append(l)) assert len(ln) > 0
def test_dataset_download_no_exist(rucio_2file_dataset, cache_empty): 'Queue a download and look for it to show up' rucio_2file_dataset._cache_mgr = cache_empty dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset) # Now, make sure that we get back what we want here. status, files = dm.download_ds('bogus') assert DatasetQueryStatus.does_not_exist == status assert None is files
def test_two_downloads_fail(simple_dataset, cache_empty, rucio_do_nothing, rucio_2file_dataset): dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset) # Pretend to lock the dataset with cache_empty.get_dataset_downloading_lock(simple_dataset.Name): try: logging.debug("Starting second download") dm.download_ds(simple_dataset.Name) assert False except RucioAlreadyBeingDownloaded: return
def test_dataset_query_resolved(rucio_2file_dataset, cache_empty, simple_dataset): 'Queue and look for a dataset query result' dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset) # Now, make sure that we get back what we want here. status, files = dm.get_ds_contents(simple_dataset.Name) assert DatasetQueryStatus.results_valid == status assert len(simple_dataset.FileList) == len(files) # Make sure we didn't re-query for this. assert 1 == rucio_2file_dataset.CountCalled == 1 _ = cache_empty.get_listing(simple_dataset.Name)
def test_dataset_download_good(rucio_2file_dataset, cache_empty, simple_dataset): 'Queue a download and look for it to show up' rucio_2file_dataset._cache_mgr = cache_empty dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset) # Now, make sure that we get back what we want here. status, files = dm.download_ds(simple_dataset.Name) assert DatasetQueryStatus.results_valid == status assert files is not None assert len(simple_dataset.FileList) == len(files) # Make sure we didn't re-query for this. assert 1 == rucio_2file_dataset.CountCalledDL
def test_good_dataset_maxAgeIfNotSeenNoEffect(rucio_2file_dataset, cache_empty, simple_dataset): 'Do not requery for the dataset' dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset) _ = dm.get_ds_contents(simple_dataset.Name) wait_some_time(lambda: rucio_2file_dataset.CountCalled == 0) status, _ = dm.get_ds_contents(simple_dataset.Name) assert DatasetQueryStatus.results_valid == status # Query, but demand a quick re-check status, _ = dm.get_ds_contents( simple_dataset.Name, maxAgeIfNotSeen=datetime.timedelta(seconds=0)) assert DatasetQueryStatus.results_valid == status assert 1 == rucio_2file_dataset.CountCalled
def test_dataset_always_missing_noretry(rucio_2file_dataset_shows_up_later, cache_empty, simple_dataset): 'Do not requery for the dataset' dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset_shows_up_later) _ = dm.get_ds_contents(simple_dataset.Name) wait_some_time(lambda: rucio_2file_dataset_shows_up_later.CountCalled == 0) status, _ = dm.get_ds_contents(simple_dataset.Name) assert DatasetQueryStatus.does_not_exist == status # Query, but demand a quick re-check status, _ = dm.get_ds_contents(simple_dataset.Name, maxAgeIfNotSeen=None) assert DatasetQueryStatus.does_not_exist == status assert 1 == rucio_2file_dataset_shows_up_later.CountCalled
def test_dataset_appears(rucio_2file_dataset_shows_up_later, cache_empty, simple_dataset): 'After a bad dataset has aged, automatically queue a new query' dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset_shows_up_later) _ = dm.get_ds_contents(simple_dataset.Name) wait_some_time(lambda: rucio_2file_dataset_shows_up_later.CountCalled == 0) status, _ = dm.get_ds_contents(simple_dataset.Name) assert DatasetQueryStatus.does_not_exist == status # Query, but demand a quick re-check status, _ = dm.get_ds_contents( simple_dataset.Name, maxAgeIfNotSeen=datetime.timedelta(seconds=0)) assert DatasetQueryStatus.results_valid == status
def test_query_for_bad_dataset(rucio_2file_dataset, cache_empty, simple_dataset): 'Ask for a bad dataset, and get back a null' dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset) _ = dm.get_ds_contents('bogus_ds') wait_some_time(lambda: rucio_2file_dataset.CountCalled == 0) # Make sure it comes back as bad. status, files = dm.get_ds_contents('bogus_ds') assert DatasetQueryStatus.does_not_exist == status assert None is files # Make sure that a timeout of an hour has been set on the dataset. info = cache_empty.get_listing('bogus_ds') assert datetime.datetime.now() == info.Created
def test_look_for_good_dataset_that_fails_a_bunch( rucio_2file_dataset_with_fails, cache_empty, simple_dataset): 'Queue and look for a good dataset that takes a few queries to show up with results' dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset_with_fails, seconds_between_retries=0.01) _ = dm.get_ds_contents(simple_dataset.Name) # Wait for the dataset query to run wait_some_time(lambda: rucio_2file_dataset_with_fails.CountCalled < 5) # Now, make sure that we get back what we want and that the number of tries matches what we think # it should have. status, _ = dm.get_ds_contents(simple_dataset.Name) assert DatasetQueryStatus.results_valid == status assert 5 == rucio_2file_dataset_with_fails.CountCalled
def test_two_queries_for_good_dataset(rucio_2file_dataset_take_time, cache_empty, simple_dataset): 'Make sure second query does not trigger second web download' # Query twice, make sure we don't forget as we are doing this! dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset_take_time) _ = dm.get_ds_contents(simple_dataset.Name) status, _ = dm.get_ds_contents(simple_dataset.Name) assert DatasetQueryStatus.results_valid == status # Now, make sure that we get back what we want here. status, _ = dm.get_ds_contents(simple_dataset.Name) assert DatasetQueryStatus.results_valid == status # Make sure we didn't re-query for this, and the expiration date is not set. # Make sure to wait long enough for other timing stuff above to fall apart. assert 1 == rucio_2file_dataset_take_time.CountCalled
def test_dataset_download_with_failures(rucio_2file_dataset_with_fails, cache_empty, simple_dataset): 'Queue a download, it fails, but then gets there' rucio_2file_dataset_with_fails._cache_mgr = cache_empty dm = rucio_cache_interface(cache_empty, rucio_mgr=rucio_2file_dataset_with_fails, seconds_between_retries=0.01) _ = dm.download_ds(simple_dataset.Name) # Wait for the dataset query to run wait_some_time(lambda: rucio_2file_dataset_with_fails.CountCalledDL < 5) # Now, make sure that we get back what we want here. status, files = dm.download_ds(simple_dataset.Name) assert DatasetQueryStatus.results_valid == status assert len(simple_dataset.FileList) == len(files) # 2 failures, so make sure we re-try the right number of times assert 5 == rucio_2file_dataset_with_fails.CountCalledDL