def on_change(ignored, path, mask):
"""
@param ignored: DO NOT USE. Read: http://twistedmatrix.com/documents/current/api/twisted.internet.inotify.html
@param path: FilePath on which the event happened.
@param mask: inotify event as hexadecimal masks
"""
global proc
events = set(inotify.humanReadableMask(mask))
print(events)
if debug:
click.echo(
'==> reloopd, DEBUG : inotify event(s) - [{}], path: {}'.format(
', '.join(inotify.humanReadableMask(mask)), path))
if ('create' or 'delete' or 'modify') in events:
if proc is not None and proc.poll() is None:
click.echo('==> reloopd, INFO : terminating previous process')
proc.kill()
proc.wait()
if before_command:
subprocess.call(shlex.split(before_command))
proc = subprocess.Popen(shlex.split(command))
def test_humanReadableMask(self):
"""
L{inotify.humaReadableMask} translates all the possible event
masks to a human readable string.
"""
for mask, value in inotify._FLAG_TO_HUMAN:
self.assertEqual(inotify.humanReadableMask(mask)[0], value)
checkMask = inotify.IN_CLOSE_WRITE | inotify.IN_ACCESS | inotify.IN_OPEN
self.assertEqual(set(inotify.humanReadableMask(checkMask)), set(["close_write", "access", "open"]))
def test_humanReadableMask(self):
"""
L{inotify.humaReadableMask} translates all the possible event
masks to a human readable string.
"""
for mask, value in inotify._FLAG_TO_HUMAN:
self.assertEqual(inotify.humanReadableMask(mask)[0], value)
checkMask = (inotify.IN_CLOSE_WRITE | inotify.IN_ACCESS
| inotify.IN_OPEN)
self.assertEqual(set(inotify.humanReadableMask(checkMask)),
set(['close_write', 'access', 'open']))
def notify(self, ignore, filepath, mask):
# self.input = False
print "event %s on %s" % (', '.join(inotify.humanReadableMask(mask)), filepath)
action = inotify.humanReadableMask(mask)[0]
# This might break.... depending on the queue of the
path = os.path.relpath(filepath.path)
# Check for swp or temp file
check = path.endswith('swp') or path.endswith('~') or path.endswith('swx') or '.goutputstream' in path
# update userfilemap
if action == 'attrib' or action == "moved_to":
if not check:
self.sendData(['upload-request',path])
if action == 'delete' or action == 'moved_from':
if not check:
self.sendData(['remove',path])
def inotify(self, ignored, filepath, mask):
"""
Callback for the INotify. It should call the sse resource with the
changed layouts in the layout file if there are changes in the
layout file.
It calls the inotify_callback method first which must be overwritten
by its children.
"""
hmask = humanReadableMask(mask)
# Some editors move the file triggering several events in inotify. All
# of them change some attribute of the file, so if that event happens,
# see if there are changes and alert the sse resource in that case.
if 'attrib' in hmask or 'modify' in hmask:
self.fp.close()
self.fp = open(self.fp.name, 'r')
self.fp.seek(0)
jdata = {}
self.inotify_callback(jdata)
changes = 0
for _, l in jdata.items():
changes += len(l)
if changes > 0:
for callback in self.inotify_callbacks:
callback(jdata)
log.msg("Change in %s" % self.fp.name)
# Some editors move the file and inotify lose track of the file, so the
# notifier must be restarted when some attribute changed is received.
if 'attrib' in hmask:
self.notifier.stopReading()
self.setup_inotify()
def notify(self, notifier: INotify, filepath: FilePath, mask: int) -> None:
try:
maskNames = humanReadableMask(mask)
if maskNames[0] == 'delete_self':
if not filepath.exists():
log.info("%s delete_self", filepath)
self.fileGone()
return
else:
log.warn(
"%s delete_self event but file is here. "
"probably a new version moved in", filepath)
# we could filter these out in the watch() call, but I want
# the debugging
if maskNames[0] in ['open', 'access', 'close_nowrite', 'attrib']:
log.debug("%s %s event, ignoring" % (filepath, maskNames))
return
try:
if filepath.getModificationTime() == self.lastWriteTimestamp:
log.debug("%s changed, but we did this write", filepath)
return
except OSError as e:
log.error("%s: %r" % (filepath, e))
# getting OSError no such file, followed by no future reads
reactor.callLater(.5, self.addWatch) # ?
return
log.info("reread %s because of %s event", filepath, maskNames)
self.reread()
except Exception:
traceback.print_exc()
def _notify(ignored, filepath, mask, file=file): if filepath.isfile() and file.endswith(filepath.basename()): log.msg(self, 'change', filepath, humanReadableMask(mask)) # Reload file self.read(filepath.path)
def notify(self, notifier, filepath, mask):
try:
maskNames = humanReadableMask(mask)
if maskNames[0] == 'delete_self':
if not filepath.exists():
log.info("%s delete_self", filepath)
self.fileGone()
return
else:
log.warn("%s delete_self event but file is here. "
"probably a new version moved in",
filepath)
# we could filter these out in the watch() call, but I want
# the debugging
if maskNames[0] in ['open', 'access', 'close_nowrite', 'attrib']:
log.debug("%s %s event, ignoring" % (filepath, maskNames))
return
try:
if filepath.getModificationTime() == self.lastWriteTimestamp:
log.debug("%s changed, but we did this write", filepath)
return
except OSError as e:
log.error("%s: %r" % (filepath, e))
# getting OSError no such file, followed by no future reads
reactor.callLater(.5, self.addWatch) # ?
return
log.info("reread %s because of %s event", filepath, maskNames)
self.reread()
except Exception:
traceback.print_exc()
def onChange(self, watch, fpath, mask):
index = fpath.path.find('onedir')
path = fpath.path[index:]
match = re.search('CS3240/(.*)/onedir', fpath.path)
if match:
user = match.group(1)
cmd = ' '.join(inotify.humanReadableMask(mask))
self.dispatch(path, cmd, user)
def onChange(self, watch, fpath, mask):
index = fpath.path.find('onedir')
path = fpath.path[index:]
match = re.search('CS3240/(.*)/onedir', fpath.path)
if match:
user = match.group(1)
cmd = ' '.join(inotify.humanReadableMask(mask))
self.dispatch(path, cmd, user)
def notify(self, filepath, mask):
filepath = filepath.realpath().path[len(watch_dir) + 1:]
if not filepath.endswith('___jb_bak___') and not filepath.endswith('___jb_old___'):
print "%s> %s" % (', '.join(inotify.humanReadableMask(mask)), filepath)
new_cmd = raw_command + [os.path.join(dst_dir, filepath), os.path.join(src_ref, filepath)]
process = subprocess.Popen(new_cmd, env=env)
process.wait()
def notify(self, unknown, filepath, mask):
logger.debug("event %s on %s" % (", ".join(inotify.humanReadableMask(mask)), filepath))
basename = filepath.basename()
if not basename.endswith(config.GROWING_FILE_SUFFIX):
return
path = os.path.join(filepath.dirname(), basename)
if mask == inotify.IN_CREATE:
self.handle_in_create(path)
elif mask == inotify.IN_DELETE:
self.handle_in_delete(path)
def notify(ignored, filepath, mask):
"""
For historical reasons, an opaque handle is passed as first
parameter. This object should never be used.
@param filepath: FilePath on which the event happened.
@param mask: inotify event as hexadecimal masks
"""
print mask
print "event %s on %s" % (
', '.join(inotify.humanReadableMask(mask)), filepath)
def handle_change(self, stuff, path, mask):
mask_str = inotify.humanReadableMask(mask)
log.noise('Event: {} ({})'.format(path, mask_str))
## Filtering
path_real = path.realpath()
if not path_real.isfile():
log.debug( 'Ignoring event for'
' non-regular file: {} (realpath: {})'.format(path, path_real) )
return
dir_key = path_real.parent().realpath()
if dir_key not in self.paths_watch:
log.warn( 'Ignoring event for file outside of watched'
' set of paths: {} (realpath: {})'.format(path, path_real) )
return
for pat in self.paths_watch[dir_key]:
if fnmatch(bytes(path.basename()), pat): break
else:
log.noise( 'Non-matched path in one of'
' the watched dirs: {} (realpath: {})'.format(path, path_real) )
return
## Get last position
if self.paths_pos.get(path_real) is not None:
pos, size, data = self.paths_pos[path_real]
if self.file_end_check(path_real, pos, size=size, data=data):
log.debug(( 'Event (mask: {}) for unchanged'
' path, ignoring: {}' ).format(mask_str, path))
return
if path_real.getsize() < pos:
log.debug( 'Detected truncation'
' of a path, rewinding: {}'.format(path) )
pos = None
else: pos = None
## Actual processing
line = self.paths_buff.setdefault(path_real, '')
with path_real.open('rb') as src:
if pos:
src.seek(pos)
pos = None
while True:
buff = src.readline()
if not buff: # eof
self.paths_pos[path_real] = self.file_end_mark(path_real, data=line)
line += buff
if line.endswith('\n'):
log.noise('New line (source: {}): {!r}'.format(path, line))
reactor.callLater(0, self.handle_line, line)
line = self.paths_buff[path_real] = ''
else:
line, self.paths_buff[path_real] = None, line
break
def notify(self, unknown, filepath, mask):
logger.debug('event %s on %s' %
(', '.join(inotify.humanReadableMask(mask)), filepath))
basename = filepath.basename()
if not basename.endswith(config.GROWING_FILE_SUFFIX):
return
path = os.path.join(filepath.dirname(), basename)
if mask == inotify.IN_CREATE:
self.handle_in_create(path)
elif mask == inotify.IN_DELETE:
self.handle_in_delete(path)
def notify(self, ignored, filepath, mask):
"""
For historical reasons, an opaque handle is passed as first
parameter. This object should never be used.
@param filepath: FilePath on which the event happened.
@param mask: inotify event as hexadecimal masks
"""
if self.name_regex and not self.name_regex.match(filepath.basename):
return None
str_event = inotify.humanReadableMask(mask)[0]
if str_event in self.events:
self.callbacks.get(str_event)(filepath)
if 'all' in self.events:
self.callbacks['all'](str_event, filepath)
def notifyCallback(self, ignored, filepath, mask):
LOGGER.debug("Notify event %s on %s" % (humanReadableMask(mask), filepath.basename()))
if mask & IN_CREATE and filepath == self.lastCaptureLink:
capture = filepath.realpath().basename()
LOGGER.info("New capture detected: %s" % capture)
try:
self.lastCaptureTime = self.extractDateTimeFromCaptureName(capture)
self.lastCaptureName = capture
except ValueError:
self.errbackDefers(Failure())
if self.defers:
defers = self.defers
self.defers = []
for d in defers:
if not d.called:
d.callback(capture)
def status_file_changed(self, ignored, filepath, mask):
""" This is a callback for the twisted INotify module to inform about
file changes on status files. This methods searches for the
associated instances and schedules a loadInstances with a timeout
of one second to handle multiple file changes only once."""
instance = None
for one_instance in self.config.instances.values():
if one_instance.status_file == filepath.path:
instance = one_instance
break
if instance is None:
print('unknown status file: {0}'.format(filepath.path))
return
from twisted.internet import reactor
instance.version += 1
deferLater(reactor, 1, self.status_file_change_done,
instance, instance.version,
','.join(inotify.humanReadableMask(mask)))
def load_config(self, *args):
if args:
masks = inotify.humanReadableMask(args[2])
#print masks
if not set(masks).intersection(set(('delete_self', ))):
return
self.config = ConfigParser()
self.config.read(CONFIG)
self.commands = self.config.items('commands')
if args:
print "config reloaded"
notifier = inotify.INotify()
notifier.startReading()
#print "adding watcher"
notifier.watch(filepath.FilePath(CONFIG), callbacks=[self.load_config])
print "config watcher started"
reactor.callLater(.5, self.send, commands=self.commands)
def notify(self, ignore, path, mask, parameter=None):
self.info(
'Event %s on %s - parameter %r',
', '.join(humanReadableMask(mask)),
path.path,
parameter,
)
if mask & IN_CHANGED:
# FIXME react maybe on access right changes, loss of read rights?
# print(f'{path} was changed, parent {parameter:d} ({iwp.path})')
pass
if mask & IN_DELETE or mask & IN_MOVED_FROM:
self.info(f'{path.path} was deleted, '
f'parent {parameter!r} ({path.parent().path})')
id = self.get_id_by_name(parameter, path.path)
if id is not None:
self.remove(id)
if mask & IN_CREATE or mask & IN_MOVED_TO:
if mask & IN_ISDIR:
self.info(f'directory {path.path} was created, '
f'parent {parameter!r} ({path.parent().path})')
else:
self.info(f'file {path.path} was created, '
f'parent {parameter!r} ({path.parent().path})')
if self.get_id_by_name(parameter, path.path) is None:
if path.isdir():
self.walk(
path.path,
self.get_by_id(parameter),
self.ignore_file_pattern,
)
else:
if self.ignore_file_pattern.match(parameter) is None:
self.append(str(path.path),
str(self.get_by_id(parameter)))
def on_file_changed(self, ignored, filepath, mask):
handler = {self.EVENT_MODIFIED: self.read_more, self.EVENT_RENAMED: self.rebind_events}.get(mask)
if not handler:
print "Ignoring event %s (%s)" % (mask, inotify.humanReadableMask(mask))
return
return handler()
def inotify_twisted(self, ignored, path, mask):
alpha_event = inotify.humanReadableMask(mask)
event = {time.time(): {'mask':mask, 'event':alpha_event[0], 'path':self.path}}
self.signal_event(event)
def onChange(self, watch, fpath, mask):
cursor.execute("SELECT auto_sync FROM account WHERE user_id = %s", (self._user,))
if cursor.fetchone()[0] == 1:
path = adjustPath(fpath.path)
cmd = ' '.join(inotify.humanReadableMask(mask))
self.dispatch(path, cmd)
def handle_change(self, stuff, path, mask):
mask_str = inotify.humanReadableMask(mask)
log.noise('Event: {} ({})'.format(path, mask_str))
## Filtering
path_real = path.realpath()
if not path_real.isfile():
log.debug( 'Ignoring event for'
' non-regular file: {} (realpath: {})'.format(path, path_real) )
return
dir_key = path_real.parent().realpath()
if dir_key not in self.paths_watch:
log.warn( 'Ignoring event for file outside of watched'
' set of paths: {} (realpath: {})'.format(path, path_real) )
return
for pat in self.paths_watch[dir_key]:
if fnmatch(bytes(path.basename()), pat): break
else:
log.noise( 'Non-matched path in one of'
' the watched dirs: {} (realpath: {})'.format(path, path_real) )
return
for pat in self.exclude:
if pat.search(path.path):
log.noise( 'Matched path by exclude-pattern'
' ({}): {} (realpath: {})'.format(pat, path, path_real) )
return
## Get last position
pos = self.paths_pos.get(path_real)
if not pos: # try restoring it from xattr
try: pos = pickle.loads(xattr(path_real.path)[self.conf.xattr_name])
except KeyError:
log.debug('Failed to restore last log position from xattr for path: {}'.format(path))
else:
log.noise(
'Restored pos from xattr ({}) for path {}: {!r}'\
.format(self.conf.xattr_name, path_real, pos) )
if pos:
pos, size, data_hash = pos
if self.file_end_check(path_real, pos, size=size, data_hash=data_hash):
log.noise(( 'Event (mask: {}) for unchanged'
' path, ignoring: {}' ).format(mask_str, path))
return
if path_real.getsize() < pos:
log.debug( 'Detected truncation'
' of a path, rewinding: {}'.format(path) )
pos = None
## Actual processing
buff_agg = self.paths_buff.setdefault(path_real, '')
with path_real.open() as src:
if pos:
src.seek(pos)
pos = None
while True:
pos = src.tell()
try: buff, pos = self.read(src), src.tell()
except StopIteration:
buff_agg = ''
src.seek(pos) # revert back to starting position
buff, pos = self.read(src), src.tell()
if not buff: # eof, try to mark the position
if not buff_agg: # clean eof at the end of the chunk - mark it
pos = self.file_end_mark(path_real, pos=pos, data=buff_agg)
self.paths_pos[path_real] = pos
xattr(path_real.path)[self.conf.xattr_name] = pickle.dumps(pos)
log.noise( 'Updated xattr ({}) for path {} to: {!r}'\
.format(self.conf.xattr_name, path_real, pos) )
break
buff_agg = self.paths_buff[path_real] = self.process(buff_agg + buff, path)
def notify(self, ignored, path, mask):
print 'event {} on {}'.format(
inotify.humanReadableMask(mask),
path
)
def _debug_notify(self, notifier, path, mask):
from twisted.internet.inotify import humanReadableMask
print path, humanReadableMask(mask)
def notify(self, filepath, mask):
mstr = ', '.join(inotify.humanReadableMask(mask))
print "event %s on %s" % (mstr, filepath)
self._proto.sendLine('%s' %mstr)
def eventReceived(self, _Watch, filepath, mask):
print "event %s on %s" % (
', '.join(inotify.humanReadableMask(mask)), filepath)
for line in self.f.readlines():
self.lineReceived(line)
def _debug_notify(self, notifier, path, mask):
from twisted.internet.inotify import humanReadableMask
print path, humanReadableMask(mask)
def eventReceived(self, _Watch, filepath, mask):
print "event %s on %s" % (', '.join(
inotify.humanReadableMask(mask)), filepath)
for line in self.f.readlines():
self.lineReceived(line)
def notify(self, _, path: filepath.FilePath, mask: int) -> None:
if path.path.endswith("{}_config.json".format(
self.operatingsystem).encode('utf8')):
log.msg("event {} on {}".format(
', '.join(inotify.humanReadableMask(mask)), path.path))
asyncio.ensure_future(self.read_data())
def _invoke_callback(self, ignored, filepath, mask):
mask_h = inotify.humanReadableMask(mask)
# meta updating file via vim shows up as UPDATE, and triggers
# sometimes only once, and sometimes TWICE :/ (See inotify docs
# on coalesced events)
# meta updating file via `echo ' ' >> FILE` shows up as UPDATE
# meta updating file via rsync shows up as ATTRIB, then DELETE_SELF
if mask == self.UPDATE:
# When file is updated via linux `cp`,
# Sometimes UPDATE happens twice about 2 milliseconds apart.
# (And the file is still empty when the first one hits)
# Other times those two events are coalesced into one
# (See inotify man page regarding coalesced events)
# THEREFORE we trigger on only the first within a time window,
# BUT we also insert a small delay to give the file
# time to actually be populated.
now = datetime.now()
delta = now - self._recent_update
self._recent_update = now
delta_seconds = delta.seconds + delta.microseconds / 1e6
action = (
f'Callback skipped because duplicate. delta_seconds: {delta_seconds}'
)
# Rate limit this event by only allowing the first of events in a window
if delta > timedelta(
milliseconds=self.DUPLICATE_WINDOW_MILLISECONDS):
action = f'Callback will be invoked in {self.FILE_POPULATION_DELAY_SECONDS}s.'
# pylint: disable=redefined-outer-name,import-outside-toplevel
from twisted.internet import reactor
# pylint: disable=no-member
# Call it enough in the future that the file is actually there
reactor.callLater(self.FILE_POPULATION_DELAY_SECONDS,
self.callback)
if mask == self.ATTRIB:
# When file is updated (remotely) via rsync , we end up here
msg = self._archive_file()
action = f'{msg}Callback invoked now.'
self.callback()
if mask == self.DELETE_SELF:
# For some reason when updating the file via rsync (remotely)
# DELETE_SELF is triggered, which causes the file to no longer be watched
# Tell it to delete the file descriptor (connectionLost).
# Otherwise we eventually run out of file descriptors
# Note we do this in the future, because if we call it inline (now),
# our process stops watching the file.
action = 'restarted file watcher'
# pylint: disable=import-outside-toplevel
from twisted.internet import reactor
# pylint: disable=no-member
reactor.callLater(
self.CONNECTION_LOST_DELAY_SECONDS,
self._notifier.connectionLost,
'rsync issued DELETE_SELF',
)
# Start Over
self.watch()
print(f'MASK: {mask_h}, action: {action}')