def test_file_obtain(self):
# {{{
self.interface.register_dataset(
*self.create_file_dataset(
'test_file_obtain', 5, 3))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_file_obtain', 1)[0]
parameters = {'mask': {'lumis': None}}
TaskHandler(id, label, files, lumis, None, True).adjust(
parameters, [], [], se.StorageConfiguration({}))
assert parameters['mask']['lumis'] is None
(jr, jd, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""",
(label,)).fetchone()
assert jr == 3
assert jd == 0
assert er in (0, None)
assert ew in (0, None)
def query(self, url):
if not isinstance(url, list):
url = [url]
s = se.StorageConfiguration(output=[], input=url)
s.activate()
with util.PartiallyMutable.unlock():
with fs.alternative():
ds = dataset.Dataset(files='spam/')
info = ds.get_info()
assert len(info.files) == 10
def setup_class(cls):
os.environ['LOCALRT'] = ''
cls.workdir = tempfile.mkdtemp()
cls.interface = UnitStore(
Config(
label='test',
workdir=cls.workdir,
storage=se.StorageConfiguration(output=['file://' + cls.workdir]),
workflows=[],
advanced=AdvancedOptions(proxy=False, dashboard=False, osg_version="3.3")
)
)
def test_basics(self):
with util.PartiallyMutable.unlock():
s = se.StorageConfiguration(output=[],
input=['file://' + self.workdir])
s.activate()
with fs.alternative():
info = Dataset(files='eggs').get_info()
assert len(info.files) == 10
info = Dataset(files=['eggs', 'ham']).get_info()
assert len(info.files) == 15
info = Dataset(files='eggs/1.txt').get_info()
assert len(info.files) == 1
def permissions(self, url):
if not isinstance(url, list):
url = [url]
url = [os.path.join(u, 'bacon') for u in url]
s = se.StorageConfiguration(output=url)
s.activate()
assert not fs.exists('ham')
fs.makedirs('ham/eggs')
parent = os.stat(self.workdir).st_mode
child = os.stat(os.path.join(self.workdir, 'bacon/ham/eggs')).st_mode
assert parent == child
def test_flatten(self):
with util.PartiallyMutable.unlock():
s = se.StorageConfiguration(output=[],
input=['file://' + self.workdir])
s.activate()
with fs.alternative():
info = Dataset(files=['spam']).get_info()
assert len(info.files) == 8
info = Dataset(files=['spam'], patterns=['*.txt']).get_info()
assert len(info.files) == 5
info = Dataset(files=['spam'],
patterns=['[12].txt']).get_info()
assert len(info.files) == 2