def run(self, data, store, signal, context, **kwargs):
""" The main run method of the walk task.
Args:
data (MultiTaskData): The data object that has been passed from the
predecessor task.
store (DataStoreDocument): The persistent data store object that allows the
task to store data for access across the current
workflow run.
signal (TaskSignal): The signal object for tasks. It wraps the construction
and sending of signals into easy to use methods.
context (TaskContext): The context in which the tasks runs.
Raises:
LightflowFilesystemPathError: If the specified path is not absolute.
Returns:
Action: An Action object containing the data that should be passed on
to the next task and optionally a list of successor tasks that
should be executed.
"""
params = self.params.eval(data, store)
if not isabs(params.path):
raise LightflowFilesystemPathError(
'The specified path is not an absolute path')
for entry in self._scantree(params.path, params.recursive):
if self._callback is not None:
self._callback(entry, data, store, signal, context)
return Action(data)
def run(self, data, store, signal, context, **kwargs):
""" The main run method of the MoveTask task.
Args:
data (MultiTaskData): The data object that has been passed from the
predecessor task.
store (DataStoreDocument): The persistent data store object that allows the
task to store data for access across the current
workflow run.
signal (TaskSignal): The signal object for tasks. It wraps the construction
and sending of signals into easy to use methods.
context (TaskContext): The context in which the tasks runs.
Raises:
LightflowFilesystemPathError: If the source is a directory
but the target is not.
LightflowFilesystemMoveError: If the move process failed.
Returns:
Action: An Action object containing the data that should be passed on
to the next task and optionally a list of successor tasks that
should be executed.
"""
params = self.params.eval(data, store)
sources = [params.sources] if isinstance(params.sources,
str) else params.sources
for source in sources:
logger.info('Move {} to {}'.format(source, params.destination))
if not os.path.isabs(source):
raise LightflowFilesystemPathError(
'The source path is not an absolute path')
if not os.path.isabs(params.destination):
raise LightflowFilesystemPathError(
'The destination path is not an absolute path')
if os.path.isdir(source) and not os.path.isdir(params.destination):
raise LightflowFilesystemPathError(
'The destination is not a valid directory')
try:
shutil.move(source, params.destination)
except OSError as e:
raise LightflowFilesystemMoveError(e)
return Action(data)
def run(self, data, store, signal, context, **kwargs):
""" The main run method of the PvTriggerTask task.
Args:
data (MultiTaskData): The data object that has been passed from the
predecessor task.
store (DataStoreDocument): The persistent data store object that allows the
task to store data for access across the current
workflow run.
signal (TaskSignal): The signal object for tasks. It wraps the construction
and sending of signals into easy to use methods.
context (TaskContext): The context in which the tasks runs.
"""
params = self.params.eval(data, store)
skipped_initial = False if params.skip_initial_callback else True
polling_event_number = 0
queue = deque()
# set up the internal callback
pv = PV(params.pv_name,
callback=partial(self._pv_callback, queue=queue))
while True:
if params.event_trigger_time is not None:
time.sleep(params.event_trigger_time)
# check every stop_polling_rate events the stop signal
polling_event_number += 1
if polling_event_number > params.stop_polling_rate:
polling_event_number = 0
if signal.is_stopped:
break
# get all the events from the queue and call the callback function
while len(queue) > 0:
event = queue.pop()
if skipped_initial:
if self._callback is not None:
self._callback(data, store, signal, context, **event)
else:
skipped_initial = True
pv.clear_callbacks()
return Action(data)
def run(self, data, store, signal, context, **kwargs):
""" The main run method of the Python task.
Args:
data (:class:`.MultiTaskData`): The data object that has been passed from the
predecessor task.
store (:class:`.DataStoreDocument`): The persistent data store object that allows the
task to store data for access across the current workflow run.
signal (TaskSignal): The signal object for tasks. It wraps the construction
and sending of signals into easy to use methods.
context (TaskContext): The context in which the tasks runs.
Returns:
Action: An Action object containing the data that should be passed on
to the next task and optionally a list of successor tasks that
should be executed.
"""
if self._callback is not None:
result = self._callback(data, store, signal, context, **kwargs)
return result if result is not None else Action(data)
def run(self, data, store, signal, context, **kwargs):
""" The main run method of the MakeDir task.
Args:
data (MultiTaskData): The data object that has been passed from the
predecessor task.
store (DataStoreDocument): The persistent data store object that allows the
task to store data for access across the current
workflow run.
signal (TaskSignal): The signal object for tasks. It wraps the construction
and sending of signals into easy to use methods.
context (TaskContext): The context in which the tasks runs.
Raises:
AbsolutePathError: If the specified directories are not absolute paths.
Returns:
Action: An Action object containing the data that should be passed on
to the next task and optionally a list of successor tasks that
should be executed.
"""
params = self.params.eval(data, store)
paths = [params.paths] if isinstance(params.paths, str) else params.paths
#
for path in paths:
if not os.path.isabs(path):
raise LightflowFilesystemPathError(
'The specified path is not an absolute path')
if not os.path.exists(path):
try:
os.makedirs(path)
except OSError as e:
raise LightflowFilesystemMkdirError(e)
else:
logger.info('Directory {} already exists. Skip creation.'.format(path))
return Action(data)
def run(self, data, store, signal, context, **kwargs):
""" The main run method of the NotifyTriggerTask task.
Args:
data (MultiTaskData): The data object that has been passed from the
predecessor task.
store (DataStoreDocument): The persistent data store object that allows the
task to store data for access across the current
workflow run.
signal (TaskSignal): The signal object for tasks. It wraps the construction
and sending of signals into easy to use methods.
context (TaskContext): The context in which the tasks runs.
Raises:
LightflowFilesystemPathError: If the specified path is not absolute.
"""
params = self.params.eval(data, store)
if not os.path.isabs(params.path):
raise LightflowFilesystemPathError(
'The specified path is not an absolute path')
# if requested, pre-fill the file list with existing lines
lines = []
num_read_lines = 0
if params.use_existing:
with open(params.path, 'r') as file:
lines = file.readlines()
num_read_lines = len(lines)
if params.flush_existing and num_read_lines > 0:
if self._callback is not None:
self._callback(lines, data, store, signal, context)
del lines[:]
polling_event_number = 0
def watch_file(file_pointer, task_signal):
while True:
if task_signal.is_stopped:
break
new = file_pointer.readline()
if new:
yield new
else:
time.sleep(params.event_trigger_time)
file = open(params.path, 'r')
try:
if params.use_existing:
for i in range(num_read_lines):
file.readline()
else:
file.seek(0, 2)
for line in watch_file(file, signal):
lines.append(line)
# check every stop_polling_rate events the stop signal
polling_event_number += 1
if polling_event_number > params.stop_polling_rate:
polling_event_number = 0
if signal.is_stopped:
break
# as soon as enough lines have been aggregated call the callback function
if len(lines) >= params.aggregate:
chunks = len(lines) // params.aggregate
for i in range(0, chunks):
if self._callback is not None:
self._callback(lines[0:params.aggregate], data,
store, signal, context)
del lines[0:params.aggregate]
finally:
file.close()
return Action(data)
def decide_on_successor(data, store, signal, context):
data['number'] = random()
if data['number'] < 0.5:
return Action(data, limit=[small_number_task])
else:
return Action(data, limit=[large_number_task])
def branch_with_limit(data, store, signal, context):
return Action(data, limit=[lane1_print_task, 'lane2_print_task'])
def run(self, data, store, signal, context, **kwargs):
""" The main run method of the Python task.
Args:
data (:class:`.MultiTaskData`): The data object that has been passed from the
predecessor task.
store (:class:`.DataStoreDocument`): The persistent data store object that allows the
task to store data for access across the current workflow run.
signal (TaskSignal): The signal object for tasks. It wraps the construction
and sending of signals into easy to use methods.
context (TaskContext): The context in which the tasks runs.
Returns:
Action (Action): An Action object containing the data that should be passed on
to the next task and optionally a list of successor tasks that
should be executed.
"""
params = self.params.eval(data, store, exclude=['command'])
capture_stdout = self._callback_stdout is not None or params.capture_stdout
capture_stderr = self._callback_stderr is not None or params.capture_stderr
stdout_file = TemporaryFile() if params.capture_stdout else None
stderr_file = TemporaryFile() if params.capture_stderr else None
stdout = PIPE if capture_stdout else None
stderr = PIPE if capture_stderr else None
# change the user or group under which the process should run
if params.user is not None or params.group is not None:
pre_exec = self._run_as(params.user, params.group)
else:
pre_exec = None
# call the command
proc = Popen(self.params.eval_single('command', data, store),
cwd=params.cwd,
shell=True,
env=params.env,
preexec_fn=pre_exec,
stdout=stdout,
stderr=stderr,
stdin=PIPE if params.stdin is not None else None)
# if input is available, send it to the process
if params.stdin is not None:
proc.stdin.write(params.stdin.encode(sys.getfilesystemencoding()))
# send a notification that the process has been started
try:
if self._callback_process is not None:
self._callback_process(proc.pid, data, store, signal, context)
except (StopTask, AbortWorkflow):
proc.terminate()
raise
# send the output handling to a thread
if capture_stdout or capture_stderr:
output_reader = BashTaskOutputReader(proc, stdout_file,
stderr_file,
self._callback_stdout,
self._callback_stderr,
params.refresh_time, data,
store, signal, context)
output_reader.start()
else:
output_reader = None
# wait for the process to complete and watch for a stop signal
while proc.poll() is None or\
(output_reader is not None and output_reader.is_alive()):
sleep(params.refresh_time)
if signal.is_stopped:
proc.terminate()
if output_reader is not None:
output_reader.join()
data = output_reader.data
# if a stop or abort exception was raised, stop the bash process and re-raise
if output_reader.exc_obj is not None:
if proc.poll() is None:
proc.terminate()
raise output_reader.exc_obj
# send a notification that the process has completed
if self._callback_end is not None:
if stdout_file is not None:
stdout_file.seek(0)
if stderr_file is not None:
stderr_file.seek(0)
self._callback_end(proc.returncode, stdout_file, stderr_file, data,
store, signal, context)
if stdout_file is not None:
stdout_file.close()
if stderr_file is not None:
stderr_file.close()
return Action(data)
def run(self, data, store, signal, context, **kwargs):
""" The main run method of the NotifyTriggerTask task.
Args:
data (MultiTaskData): The data object that has been passed from the
predecessor task.
store (DataStoreDocument): The persistent data store object that allows the
task to store data for access across the current
workflow run.
signal (TaskSignal): The signal object for tasks. It wraps the construction
and sending of signals into easy to use methods.
context (TaskContext): The context in which the tasks runs.
Raises:
LightflowFilesystemPathError: If the specified path is not absolute.
"""
params = self.params.eval(data, store)
# build notification mask
on_file_create = constants.IN_CREATE if params.on_file_create else 0x00000000
on_file_close = constants.IN_CLOSE_WRITE if params.on_file_close else 0x00000000
on_file_delete = constants.IN_DELETE if params.on_file_delete else 0x00000000
on_file_move = constants.IN_MOVE if params.on_file_move else 0x00000000
mask = (on_file_create | on_file_close | on_file_delete | on_file_move)
if not os.path.isabs(params.path):
raise LightflowFilesystemPathError(
'The specified path is not an absolute path')
if params.recursive:
notify = adapters.InotifyTree(params.path.encode('utf-8'))
else:
notify = adapters.Inotify()
notify.add_watch(params.path.encode('utf-8'))
# setup regex
if isinstance(params.exclude_mask, str):
regex = re.compile(params.exclude_mask)
else:
regex = None
# if requested, pre-fill the file list with existing files
files = []
if params.use_existing:
for (dir_path, dir_names, filenames) in os.walk(params.path):
files.extend([os.path.join(dir_path, filename) for filename in filenames])
if not params.recursive:
break
if regex is not None:
files = [file for file in files if regex.search(file) is None]
if params.flush_existing and len(files) > 0:
if self._callback is not None:
self._callback(files, data, store, signal, context)
del files[:]
polling_event_number = 0
try:
for event in notify.event_gen():
if params.event_trigger_time is not None:
time.sleep(params.event_trigger_time)
# check every stop_polling_rate events the stop signal
polling_event_number += 1
if polling_event_number > params.stop_polling_rate:
polling_event_number = 0
if signal.is_stopped:
break
# in case of an event check whether it matches the mask and call a dag
if event is not None:
(header, type_names, watch_path, filename) = event
if (not header.mask & constants.IN_ISDIR) and\
(header.mask & mask):
new_file = os.path.join(watch_path.decode('utf-8'),
filename.decode('utf-8'))
add_file = not params.skip_duplicate or \
(params.skip_duplicate and new_file not in files)
if add_file and regex is not None:
add_file = regex.search(new_file) is None
if add_file:
files.append(new_file)
# as soon as enough files have been aggregated call the sub dag
if len(files) >= params.aggregate:
chunks = len(files) // params.aggregate
for i in range(0, chunks):
if self._callback is not None:
self._callback(files[0:params.aggregate], data,
store, signal, context)
del files[0:params.aggregate]
finally:
if not params.recursive:
notify.remove_watch(params.path.encode('utf-8'))
return Action(data)