class AWSDBSync(threading.Thread):
"""
Synchronize to AWS and act on any new events.
Fill in any local event DB entries from AWS that we don't have in our local DB. This can happen either by polling
or as the result of an AWS SQS message that another node has published.
"""
def __init__(self, app_data_folder, poll_period_sec, use_aws_local, event_filter):
super().__init__()
self.app_data_folder = app_data_folder
self.event_filter = event_filter
self.exit_event = threading.Event()
pref = preferences.Preferences(self.app_data_folder)
self.active_timer = activity_timer.ActivityTimer(pref.get_node_id() + '_' + self.get_type())
self.aws_event_table = TableEvents(use_aws_local)
self.node_table = TableNodes(use_aws_local)
self.poll_period_sec = poll_period_sec
self.s3 = latus.aws.aws_access.LatusS3(pref, use_aws_local)
latus.logger.log.info('poll period : %f sec' % float(self.poll_period_sec))
def run(self):
pref = preferences.Preferences(self.app_data_folder)
while not self.exit_event.is_set():
latus.logger.log.info('starting poll')
self._sync_with_aws(pref)
latus.logger.log.info('poll complete')
self.exit_event.wait(timeout=self.poll_period_sec)
self.event_filter.request_exit()
self.event_filter.join(latus.const.LONG_TIME_OUT)
if self.event_filter.isAlive():
logger.log.warn('g_event_filter is still alive')
logger.log.info('exiting node "%s"' % pref.get_node_id())
def _pull_down_new_db_entries(self, pref):
"""
Get new file system event DB entries from AWS. Keep a list of events that we got new entries for,
so that we can perform these events locally. In a sense this is a 'command' to perform this action,
and we will only do it once since it is only inserted into the local DB once.
:return: a list of events to take action on
"""
logger.log.debug('pulling down new entries')
this_node_id = pref.get_node_id()
node_db = nodedb.NodeDB(self.app_data_folder, this_node_id)
aws_event_table_resource = self.aws_event_table.get_table_resource()
updated_events = []
for node_id in self.node_table.get_all_nodes():
if node_id != this_node_id:
most_recent_local = node_db.get_most_recent_entry(node_id)
if most_recent_local:
most_recent_local_mivui = most_recent_local['mivui']
else:
most_recent_local_mivui = 0
logger.log.debug('most_recent_local_mivui for : %s : %d' % (node_id, most_recent_local_mivui))
query_response = aws_event_table_resource.query(
KeyConditionExpression=conditions.Key('originator').eq(node_id) & conditions.Key('mivui').gt(most_recent_local_mivui)
)
for q in query_response['Items']:
logger.log.info('query_response : %s' % str(q))
mtime = q['mtime']
if mtime:
mtime = maya.parse(mtime).datetime()
size = q['size']
if size:
size = int(size)
logger.log.info('new_db_entry : %s' % str(q))
node_db.insert(int(q['mivui']), q['originator'], int(q['event_type']), int(q['detection']), q['file_path'],
q['src_path'], size, q['file_hash'], mtime, False)
updated_events.append(q)
if len(updated_events) > 0:
latus.logger.log.info('updated events : %s' % str(updated_events))
return updated_events
@activity_and_lock
def _sync_with_aws(self, pref):
latus.logger.log.info('entering _sync')
updated_events = self._pull_down_new_db_entries(pref)
for fs_event in updated_events:
event_type = fs_event['event_type']
hash_value = fs_event['file_hash']
local_file_path = os.path.join(pref.get_latus_folder(), fs_event['file_path'])
if os.path.exists(local_file_path):
local_file_hash, _ = latus.hash.calc_sha512(local_file_path, pref.get_crypto_key())
else:
local_file_hash = None
if event_type == LatusFileSystemEvent.created or event_type == LatusFileSystemEvent.modified:
if hash_value != local_file_hash:
self.event_filter.add_event(local_file_path, event_type)
crypto_key = pref.get_crypto_key()
if crypto_key is None:
latus.logger.log.warning('no crypto_key yet')
return
crypto = latus.crypto.Crypto(crypto_key, pref.get_node_id())
if hash_value:
cache_fernet_file = os.path.join(pref.get_cache_folder(), hash_value + ENCRYPTION_EXTENSION)
self.s3.download_file(cache_fernet_file, hash_value)
latus.logger.log.info(
'originator=%s, event_type=%s, detection=%s, file_path="%s" - propagating to "%s" (file_hash=%s)' %
(fs_event['originator'], fs_event['event_type'], fs_event['detection'],
fs_event['file_path'], local_file_path, fs_event['file_hash']))
encrypt, shared, cloud = True, False, True # todo: get this from pref
if encrypt:
expand_ok = crypto.decrypt_file(cache_fernet_file, local_file_path)
if expand_ok:
mtime = MayaDT.from_iso8601(fs_event['mtime']).epoch
os.utime(local_file_path, (mtime, mtime))
else:
# todo: something more elegant than just calling fatal here
latus.logger.log.fatal('Unable to decrypt (possible latus key error) : %s : %s' % (
cache_fernet_file, local_file_path))
else:
cloud_file = os.path.join(pref.get_cache_folder(),
fs_event['file_hash'] + UNENCRYPTED_EXTENSION)
shutil.copy2(cloud_file, local_file_path)
else:
latus.logger.log.warning('%s : hash is None for %s' % (pref.get_node_id(), local_file_path))
elif event_type == LatusFileSystemEvent.deleted:
try:
if os.path.exists(local_file_path):
latus.logger.log.info('%s : %s : %s deleted %s' % (
pref.get_node_id(), fs_event['detection'], fs_event['originator'], fs_event['file_path']))
self.event_filter.add_event(local_file_path, event_type)
send2trash.send2trash(local_file_path)
except OSError:
# fallback
latus.logger.log.warn('%s : send2trash failed on %s' % (pref.get_node_id(), local_file_path))
elif event_type == LatusFileSystemEvent.moved:
# todo: make a specific 'moved' filter event - this one just uses the dest
latus_path = pref.get_latus_folder()
latus.logger.log.info('%s : %s : %s moved %s to %s' % (
pref.get_node_id(), fs_event['detection'], fs_event['originator'], fs_event['src_path'],
fs_event['file_path']))
dest_abs_path = os.path.join(latus_path, fs_event['file_path'])
src_abs_path = os.path.join(latus_path, fs_event['src_path'])
if not os.path.exists(src_abs_path):
logger.log.info('%s : most recent is move of %s to %s but source does not exist - nothing to do' % (
pref.get_node_id(), src_abs_path, dest_abs_path))
return
os.makedirs(os.path.dirname(dest_abs_path), exist_ok=True)
# we'll get events for both src and dest
self.event_filter.add_event(src_abs_path, event_type)
self.event_filter.add_event(dest_abs_path, event_type)
try:
shutil.move(src_abs_path, dest_abs_path)
except IOError as e:
latus.logger.log.error('%s : %s' % (pref.get_node_id(), str(e)))
if os.path.exists(dest_abs_path):
latus.logger.log.error(
'%s : attempting move but %s already exists' % (pref.get_node_id(), dest_abs_path))
if not os.path.exists(src_abs_path):
latus.logger.log.error(
'%s : attempting move but %s not found' % (pref.get_node_id(), src_abs_path))
else:
latus.logger.log.error('not yet implemented : %s' % str(event_type))
latus.logger.log.info('exiting _sync')
def get_type(self):
return 'aws_db_sync'
def request_exit(self):
latus.logger.log.info('request_exit')
self.active_timer.reset()
self.exit_event.set()
class AWSDBSync(threading.Thread):
"""
Fill in any local event DB entries from AWS that we don't have in our local DB. This can happen either by polling
or as the result of an AWS SQS message that another node has published.
"""
def __init__(self, app_data_folder, poll_period_sec):
super().__init__()
self.app_data_folder = app_data_folder
self.exit_event = threading.Event()
self.event_table = TableEvents()
self.node_table = TableNodes()
self.poll_period_sec = poll_period_sec
pref = preferences.Preferences(self.app_data_folder)
self.s3 = latus.aws.LatusS3(pref)
latus.logger.log.info('poll period : %f sec' % float(self.poll_period_sec))
def run(self):
pref = preferences.Preferences(self.app_data_folder)
while not self.exit_event.is_set():
self._pull_down_new_db_entries(pref)
self._sync(pref)
self.exit_event.wait(timeout=self.poll_period_sec)
def _pull_down_new_db_entries(self, pref):
logger.log.info('pulling down new entries')
this_node_id = pref.get_node_id()
node_db = nodedb.NodeDB(self.app_data_folder, this_node_id)
event_table_resource = self.event_table.get_table_resource()
for node_id in self.node_table.get_all_nodes():
if node_id != this_node_id:
most_recent_local = node_db.get_most_recent_entry(node_id)
if most_recent_local:
most_recent_local_mivui = most_recent_local['mivui']
else:
most_recent_local_mivui = 0
logger.log.info('most_recent_local_mivui for : %s : %d' % (node_id, most_recent_local_mivui))
query_response = event_table_resource.query(
KeyConditionExpression=conditions.Key('originator').eq(node_id) & conditions.Key('mivui').gt(most_recent_local_mivui)
)
for q in query_response['Items']:
logger.log.info('query_response : %s' % str(q))
mtime = q['mtime']
if mtime:
mtime = maya.parse(mtime).datetime()
size = q['size']
if size:
size = int(size)
logger.log.info('new_db_entry : %s' % str(q))
node_db.update(int(q['mivui']), q['originator'], int(q['event_type']), int(q['detection']), q['file_path'],
q['src_path'], size, q['file_hash'], mtime, False)
def _sync(self, pref):
this_node_id = pref.get_node_id()
node_db = nodedb.NodeDB(self.app_data_folder, this_node_id)
for path in node_db.get_paths():
self._one_sync(path, pref)
def _one_sync(self, path, pref):
logger.log.info('start _one_sync')
this_node_id = pref.get_node_id()
node_db = nodedb.NodeDB(self.app_data_folder, this_node_id)
most_recent = node_db.get_most_recent_entry_for_path(path)
if most_recent['originator'] == this_node_id:
# this node created the most recent state, so nothing to do
return
logger.log.info('most_recent : %s' % str(most_recent))
event = most_recent['event_type']
hash_value = most_recent['file_hash']
local_file_path = os.path.join(pref.get_latus_folder(), most_recent['file_path'])
if os.path.exists(local_file_path):
local_file_hash, _ = latus.hash.calc_sha512(local_file_path, pref.get_crypto_key())
else:
local_file_hash = None
if event == LatusFileSystemEvent.created or event == LatusFileSystemEvent.modified:
if hash_value != local_file_hash:
g_event_filter.add_event(local_file_path, event)
crypto_key = pref.get_crypto_key()
if crypto_key is None:
latus.logger.log.warning('no crypto_key yet')
return
crypto = latus.crypto.Crypto(crypto_key, pref.get_node_id())
if hash_value:
cache_fernet_file = os.path.join(pref.get_cache_folder(), hash_value + ENCRYPTION_EXTENSION)
self.s3.download_file(cache_fernet_file, hash_value)
latus.logger.log.info('originator=%s, event_type=%s, detection=%s, file_path="%s" - propagating to "%s" (file_hash=%s)' %
(most_recent['originator'], most_recent['event_type'], most_recent['detection'],
most_recent['file_path'], local_file_path, most_recent['file_hash']))
encrypt, shared, cloud = True, False, True # todo: get this from pref
if encrypt:
expand_ok = crypto.decrypt_file(cache_fernet_file, local_file_path)
if expand_ok:
mtime = (most_recent['mtime'] - datetime.datetime.utcfromtimestamp(0)).total_seconds()
os.utime(local_file_path, (mtime, mtime))
else:
# todo: something more elegant than just calling fatal here
latus.logger.log.fatal('Unable to decrypt (possible latus key error) : %s : %s' % (cache_fernet_file, local_file_path))
else:
cloud_file = os.path.join(pref.get_cache_folder(), most_recent['file_hash'] + UNENCRYPTED_EXTENSION)
shutil.copy2(cloud_file, local_file_path)
else:
latus.logger.log.warning('%s : hash is None for %s' % (pref.get_node_id(), local_file_path))
elif event == LatusFileSystemEvent.deleted:
latus.logger.log.info('%s : %s : %s deleted %s' % (pref.get_node_id(), most_recent['detection'], most_recent['originator'], most_recent['file_path']))
try:
if os.path.exists(local_file_path):
g_event_filter.add_event(local_file_path, event)
send2trash.send2trash(local_file_path)
except OSError:
# fallback
latus.logger.log.warn('%s : send2trash failed on %s' % (pref.get_node_id(), local_file_path))
elif event == LatusFileSystemEvent.moved:
# todo: make a specific 'moved' filter event - this one just uses the dest
latus_path = pref.get_latus_folder()
latus.logger.log.info('%s : %s : %s moved %s to %s' % (pref.get_node_id(), most_recent['detection'], most_recent['originator'], most_recent['src_path'], most_recent['file_path']))
dest_abs_path = os.path.join(latus_path, most_recent['file_path'])
src_abs_path = os.path.join(latus_path, most_recent['src_path'])
if not os.path.exists(src_abs_path):
logger.log.info('%s : most recent is move of %s to %s but source does not exist - nothing to do' % (pref.get_node_id(), src_abs_path, dest_abs_path))
return
os.makedirs(os.path.dirname(dest_abs_path), exist_ok=True)
# we'll get events for both src and dest
g_event_filter.add_event(src_abs_path, event)
g_event_filter.add_event(dest_abs_path, event)
try:
shutil.move(src_abs_path, dest_abs_path)
except IOError as e:
latus.logger.log.error('%s : %s' % (pref.get_node_id(), str(e)))
if os.path.exists(dest_abs_path):
latus.logger.log.error('%s : attempting move but %s already exists' % (pref.get_node_id(), dest_abs_path))
if not os.path.exists(src_abs_path):
latus.logger.log.error('%s : attempting move but %s not found' % (pref.get_node_id(), src_abs_path))
else:
latus.logger.log.error('not yet implemented : %s' % str(event))
def request_exit(self):
g_event_filter.request_exit()
self.exit_event.set()
g_event_filter.join(TIME_OUT)
return self.exit_event.wait(TIME_OUT) and not g_event_filter.is_alive()