def process_IN_CLOSE_WRITE(self, event):
self.check_delete_file(event)
filename = event.pathname
if not check_deny_file(filename):
if not os.path.exists(filename):
return
node = content.get_file_info(filename)
if not node:
return
# Modify existing record
ret = content.exist(node['uid'], node['size'], node['date'])
if ret == 'MODIFIED':
clog.debug('[WRITE] MODIFIED: %s' % filename)
content.modify_file(filename)
self.update_event_dir_list(filename)
self.up_to_date = False
# Insert New Record
elif ret == 'NEW_FILE':
clog.debug('[WRITE] CREATED: %s' % filename)
content.add_file(filename, True, node)
self.update_event_dir_list(filename)
self.up_to_date = False
# Already in the DB.(For Samba event)
elif ret == 'SAME_FILE':
pass
def process_IN_DELETE(self, event):
self.check_delete_file(event)
filename = event.pathname
if not check_deny_file(filename):
clog.debug('[DELETE] filename: %s' % filename)
content.delete_file(filename)
self.update_event_dir_list(filename)
self.up_to_date = False
def process_IN_MOVED_FROM(self, event):
self.check_delete_file(event)
filename = event.pathname
clog.debug('UID existence check..')
if not content.uid(filename):
return
if self._moved_from_cookie_prev == 0:
self._moved_from_cookie_prev = event.cookie
self._moved_from_file = event.pathname
clog.debug('[MV_FROM] filename: %s' % event.pathname)
self.update_event_dir_list(filename)
self.up_to_date = False
def __init__(self, paths):
self.wm = pyinotify.WatchManager()
mask = pyinotify.IN_CREATE | pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO | pyinotify.IN_DELETE | pyinotify.IN_CLOSE_WRITE # watched events
self.notifier = pyinotify.ThreadedNotifier(self.wm, default_proc_fun=EventHandler())
#self.notifier = pyinotify.ThreadedNotifier(self.wm)
self.running = False
self.paths = paths
self.timer = Timer(INITIAL_TIMER_INTERVAL, self.partial_scan_for_recent_events)
# Add watches with 'auto-add' option enabled..
for path in paths:
self.wm.add_watch(path, mask, rec=False, auto_add=True)
for base, dirs, files in os.walk(path):
for dir in dirs:
if not dir.startswith('.'):
self.wm.add_watch(os.path.join(base, dir), mask, rec=False, auto_add=True)
# each wdd is of dict format (path: wd)
for path in paths:
clog.debug('Monitored Paths: %s', path)
def check_delete_file(self, event):
# No previous MOVED_FROM event.
if self._moved_from_cookie_prev == 0:
return
# One of the other events except MOVED_FROM came in.
elif event.mask != pyinotify.IN_MOVED_TO and event.mask != pyinotify.IN_MOVED_FROM:
clog.debug('[1] No MOVED_* Event. Removing previous MOVED_FROM file.')
content.delete_file(self._moved_from_file)
self.up_to_date = False
self._moved_from_cookie_prev = 0
self._moved_from_file = ''
# Another MOVED_FROM event came in. Delete previous MOVED_FROM file.
elif event.mask == pyinotify.IN_MOVED_FROM:
clog.debug('[2] Another MOVED_FROM event. Removing previous MOVED_FROM file.')
content.delete_file(self._moved_from_file)
self.up_to_date = False
self._moved_from_cookie_prev = event.cookie
self._moved_from_file = event.pathname
else:
log.exception('[ERROR] Unexpected case!')
def partial_scan_for_recent_events(self):
status = self.get_scan_status()
# Monitor is idle.
if status == True:
# Some events happened after previous scanning.
if not self.notifier._default_proc_fun.up_to_date:
clog.debug('Some events occurred. Start scanning the below directories...')
for dir in self.notifier._default_proc_fun.event_dir_list:
clog.debug('| %s |' % dir)
clog.debug('=============== End of Event Dirs ===============')
for dir in self.notifier._default_proc_fun.event_dir_list:
content.scan(dir)
self.notifier._default_proc_fun.event_dir_list = []
self.notifier._default_proc_fun.up_to_date = True
clog.debug('Finished Partial Scanning...')
self.timer = Timer(TIMER_INTERVAL, self.partial_scan_for_recent_events)
self.timer.start()
def verify(self, topdir):
result = {'add': 0, 'move': 0, 'delete': 0}
cursor = self.conn.cursor()
todo = [topdir]
while len(todo) > 0:
dir = todo.pop(0)
id = self._get_uid(dir)
cursor.execute("SELECT name, uid FROM tree WHERE pid=%(uid)s",
{'uid': id})
db_list = cursor.fetchall()
db_list.sort()
miss = []
flist = os.listdir(dir)
flist.sort()
for f in flist:
fpath = os.path.join(dir, f)
if not check_deny_file(fpath):
uid = self._get_uid(fpath)
missed = True
for db_name, db_uid in db_list:
if db_name == f and db_uid == uid:
clog.debug("[1-1] PASS: "******"[2] Remaining item: " + str(only_in_db))
for only_in_fs in miss:
only_in_fs = os.path.join(dir, only_in_fs)
file_node = self.get_file_info(only_in_fs)
db_node = self.get_tree(file_node['uid'])
if db_node == None:
self.add_file(only_in_fs, verify=False, info=file_node)
result['add'] += 1
clog.debug("[3-1] Missing item: " + str(only_in_fs))
else:
db_node.update(self.get_data(file_node['uid']))
if db_node['type'] == file_node['type'] and db_node[
'size'] == file_node['size'] and db_node[
'date'] == file_node['date']:
self.mod_tree(
file_node['uid'], {
'name': os.path.basename(only_in_fs),
'pid': file_node['pid']
})
result['move'] += 1
clog.debug("[3-3] Missing item :" + str(only_in_fs))
else:
self.del_tree(file_node['uid'])
self.del_data(file_node['uid'])
result['delete'] += 1
self.add_file(only_in_fs, verify=False, info=file_node)
result['add'] += 1
clog.debug("[3-2] Missing item: " + str(only_in_fs))
cursor.close()
return result
def process_IN_MOVED_TO(self, event):
dst_path = event.pathname
node = content.get_file_info(dst_path)
# New Contents came in.
if event.src_exist == False:
if not check_deny_file(dst_path):
clog.debug('[MOVED_TO] Created: %s' % dst_path)
content.add_file(dst_path, True, node)
self.update_event_dir_list(dst_path)
self.up_to_date = False
else:
clog.debug('UID existence check..')
uid = content.uid(event.src_pathname)
# New contents came in.(This should be a dir creation on Samba.)
if not uid:
log.debug('UID doesn\'t exists in DB. Adding file...')
if not check_deny_file(dst_path):
clog.debug('[MOVED_TO] Created: %s' % dst_path)
content.add_file(dst_path, True, node)
self.update_event_dir_list(dst_path)
self.up_to_date = False
# Moved inside the content directory
else:
clog.debug('========= Before updating watches=========')
for w in event.wm.watches.values():
clog.debug('wd: %d, path: %s' % (w.wd, w.path))
clog.debug('==========================================')
# Update Watch
if (event.mask & pyinotify.IN_ISDIR) :
src_path = event.src_pathname
for w in event.wm.watches.values():
if w.path == src_path:
w.path = dst_path
clog.debug('========== After updating watches =============')
for w in event.wm.watches.values():
clog.debug('wd: %d, path: %s' % (w.wd, w.path))
clog.debug('===============================================')
src_path_len = len(src_path)
sep_len = len(os.path.sep)
for w in event.wm.watches.values():
if w.path.startswith(src_path):
# Note that dest_path is a normalized path.
w.path = os.path.join(dst_path, w.path[src_path_len + sep_len:])
#log.debug('[MOVED_TO] updated watch path: %s' % w.path)
clog.debug('======== After updating watch dependencies ==========')
for w in event.wm.watches.values():
clog.debug('wd: %d, path: %s' % (w.wd, w.path))
clog.debug('=====================================================')
clog.debug('[MOVED_TO] Moved: %s' % dst_path)
content.move_file(event.src_pathname, dst_path)
self.update_event_dir_list(dst_path)
self.up_to_date = False
self._moved_from_cookie_prev = 0
self._moved_from_file = ''
def process_IN_CREATE(self, event):
self.check_delete_file(event)
filename = event.pathname
if not check_deny_file(filename):
clog.debug('[CREATE] filename: %s' % filename)
def verify(self, topdir):
result = {'add':0, 'move':0, 'delete':0}
cursor = self.conn.cursor()
todo = [topdir]
while len(todo) > 0:
dir = todo.pop(0)
id = self._get_uid(dir)
cursor.execute("SELECT name, uid FROM tree WHERE pid=%(uid)s", {'uid':id})
db_list = cursor.fetchall()
db_list.sort()
miss = []
flist = os.listdir(dir)
flist.sort()
for f in flist:
fpath = os.path.join(dir, f)
if not check_deny_file(fpath):
uid = self._get_uid(fpath)
missed = True
for db_name, db_uid in db_list:
if db_name == f and db_uid == uid:
clog.debug("[1-1] PASS: "******"[2] Remaining item: " + str(only_in_db))
for only_in_fs in miss:
only_in_fs = os.path.join(dir, only_in_fs)
file_node = self.get_file_info(only_in_fs)
db_node = self.get_tree(file_node['uid'])
if db_node == None:
self.add_file(only_in_fs, verify=False, info=file_node)
result['add'] += 1
clog.debug("[3-1] Missing item: " + str(only_in_fs))
else:
db_node.update(self.get_data(file_node['uid']))
if db_node['type'] == file_node['type'] and db_node['size'] == file_node['size'] and db_node['date'] == file_node['date']:
self.mod_tree(file_node['uid'],
{'name': os.path.basename(only_in_fs), 'pid': file_node['pid']})
result['move'] += 1
clog.debug("[3-3] Missing item :" + str(only_in_fs))
else:
self.del_tree(file_node['uid'])
self.del_data(file_node['uid'])
result['delete'] += 1
self.add_file(only_in_fs, verify=False, info=file_node)
result['add'] += 1
clog.debug("[3-2] Missing item: " + str(only_in_fs))
cursor.close()
return result