def fail_task(self, filename, task, entry):
"""Record that a task failed during execution.
Parameters
----------
filename : str
Path of wrapper file relative to GUP directory.
task : str
Task being updated.
entry : str
Entry of NeXus file being updated.
"""
with NXLock(self.database):
f = self.get_file(filename)
for t in reversed(f.tasks):
if t.name == task and t.entry == entry:
break
else:
# No task recorded
return
t.status = FAILED
t.queue_time = None
t.start_time = None
t.end_time = None
self.update_status(f, task)
def sync_db(self, sample_dir):
""" Populate the database based on local files.
Parameters
----------
sample_dir : str
Directory containing the NeXus wrapper files.
"""
# Get a list of all the .nxs wrapper files
wrapper_files = [
os.path.join(sample_dir, filename)
for filename in os.listdir(sample_dir)
if filename.endswith('.nxs') and all(x not in filename
for x in ('parent', 'mask'))
]
with NXLock(self.database):
for wrapper_file in wrapper_files:
self.sync_file(self.get_file(wrapper_file))
tracked_files = list(self.session.query(File).all())
for f in tracked_files:
if f.filename not in [
self.get_filename(w) for w in wrapper_files
]:
self.session.delete(f)
self.session.commit()
def __init__(self, directory, autosave=True):
self.directory = directory
tempdir = os.path.join(directory, 'tempdir')
self.lockfile = os.path.join(directory, 'filequeue')
if not os.path.exists(tempdir):
os.makedirs(tempdir)
with NXLock(self.lockfile):
super().__init__(directory,
serializer=json,
autosave=autosave,
tempdir=tempdir)
self.fix_access()
def update_file(self, filename):
"""Update the File object for the specified file.
This is just a wrapper for 'sync_file' that includes database file
locking.
Parameters
----------
filename : str
Path of wrapper file relative to GUP directory.
"""
with NXLock(self.database):
self.sync_file(filename)
def __init__(self, db_file, echo=False):
"""Connect to the database, creating tables if necessary.
Parameters
----------
db_file : str
Path to the database file
echo : bool, optional
True if SQL statements are echoed to `stdout`, by default False.
"""
with NXLock(db_file):
connection = 'sqlite:///' + db_file
self.engine = create_engine(connection, echo=echo)
Base.metadata.create_all(self.engine)
self.database = os.path.realpath(self.engine.url.database)
self._session = None
def queue_task(self, filename, task, entry, queue_time=None):
"""Update a file to 'queued' status and create a matching task.
Parameters
----------
filename : str
Path of wrapper file relative to GUP directory.
task : str
Task being updated.
entry : str
Entry of NeXus file being updated.
"""
with NXLock(self.database):
f = self.get_file(filename)
t = self.get_task(f, task, entry)
t.status = QUEUED
if queue_time:
t.queue_time = queue_time
else:
t.queue_time = datetime.datetime.now()
t.start_time = t.end_time = None
self.update_status(f, task)
def is_file_locked(filename, wait=5, expiry=None):
_lock = NXLock(filename)
try:
if expiry is None:
expiry = nxgetlockexpiry()
if _lock.is_stale(expiry=expiry):
return False
else:
_lock.wait(wait)
return False
except NXLockException:
lock_time = modification_time(_lock.lock_file)
if confirm_action("File locked. Do you want to clear the lock?",
f"{filename}\nCreated: {lock_time}",
answer="no"):
_lock.clear()
return False
else:
return True
else:
return False
def start_task(self, filename, task, entry, start_time=None):
"""Record that a task has begun execution.
Parameters
----------
filename : str
Path of wrapper file relative to GUP directory.
task : str
Task being updated.
entry : str
Entry of NeXus file being updated.
"""
with NXLock(self.database):
f = self.get_file(filename)
t = self.get_task(f, task, entry)
t.status = IN_PROGRESS
if start_time:
t.start_time = start_time
else:
t.start_time = datetime.datetime.now()
t.pid = os.getpid()
t.end_time = None
self.update_status(f, task)
def end_task(self, filename, task, entry, end_time=None):
"""Record that a task finished execution.
Update the task's database entry, and set the matching column in
files to DONE if it's the last task to finish
Parameters
----------
filename : str
Path of wrapper file relative to GUP directory.
task : str
Task being updated.
entry : str
Entry of NeXus file being updated.
"""
with NXLock(self.database):
f = self.get_file(filename)
t = self.get_task(f, task, entry)
t.status = DONE
if end_time:
t.end_time = end_time
else:
t.end_time = datetime.datetime.now()
self.update_status(f, task)
def task_status(self, filename, task, entry=None):
"""Return the status of the task.
Parameters
----------
filename : str
Path of wrapper file.
task : str
Task being checked.
entry : str
Entry of NeXus file being checked.
"""
with NXLock(self.database):
f = self.get_file(filename)
if entry:
for t in reversed(f.tasks):
if t.name == task and t.entry == entry:
status = t.status
break
else:
status = NOT_STARTED
else:
status = getattr(f, task)
return status
def queued_items(self):
with NXLock(self.lockfile):
items = []
while self.qsize() > 0:
items.append(super().get(timeout=0))
return items
def get(self, block=True, timeout=None):
with NXLock(self.lockfile):
item = super().get(block=block, timeout=timeout)
self.fix_access()
return item
def put(self, item, block=True, timeout=None):
with NXLock(self.lockfile):
super().put(item, block=block, timeout=timeout)
self.fix_access()