def __init__(self, settings):
self.settings = settings
self.SIGNATURE = "LocalfsFileClient"
self.ROOTPATH = settings.local_root_path
self.CACHEPATH = settings.cache_path
self.syncer_dbtuple = settings.syncer_dbtuple
client_dbname = self.SIGNATURE + '.db'
self.client_dbtuple = common.DBTuple(dbtype=database.ClientDB,
dbname=utils.join_path(
settings.instance_path,
client_dbname))
database.initialize(self.client_dbtuple)
self.probe_candidates = utils.ThreadSafeDict()
self.check_enabled()
def __init__(self, settings):
self.settings = settings
self.SIGNATURE = "PithosFileClient"
self.auth_url = settings.auth_url
self.auth_token = settings.auth_token
self.container = settings.container
self.syncer_dbtuple = settings.syncer_dbtuple
client_dbname = self.SIGNATURE + '.db'
self.client_dbtuple = common.DBTuple(dbtype=database.ClientDB,
dbname=utils.join_path(
settings.instance_path,
client_dbname))
database.initialize(self.client_dbtuple)
self.endpoint = settings.endpoint
self.last_modification = "0000-00-00"
self.probe_candidates = utils.ThreadSafeDict()
self.check_enabled()
def __init__(self, settings, master, slave):
self.settings = settings
self.master = master
self.slave = slave
self.DECISION = 'DECISION'
self.SYNC = 'SYNC'
self.MASTER = master.SIGNATURE
self.SLAVE = slave.SIGNATURE
self.syncer_dbtuple = settings.syncer_dbtuple
self.clients = {self.MASTER: master, self.SLAVE: slave}
self.notifiers = {}
self.decide_thread = None
self.sync_threads = []
self.failed_serials = utils.ThreadSafeDict()
self.sync_queue = Queue.Queue()
self.messager = settings.messager
self.heartbeat = self.settings.heartbeat
class WebSocketProtocol(WebSocket):
"""Helper-side WebSocket protocol for communication with GUI:
-- INTERNAL HANDSAKE --
GUI: {"method": "post", "ui_id": <GUI ID>}
HELPER: {"ACCEPTED": 202, "action": "post ui_id"}" or
"{"REJECTED": 401, "action": "post ui_id"}
-- ERRORS WITH SIGNIFICANCE --
-- SHUT DOWN --
GUI: {"method": "post", "path": "shutdown"}
-- PAUSE --
GUI: {"method": "post", "path": "pause"}
HELPER: {"OK": 200, "action": "post pause"} or error
-- START --
GUI: {"method": "post", "path": "start"}
HELPER: {"OK": 200, "action": "post start"} or error
-- FORCE START --
GUI: {"method": "post", "path": "force"}
HELPER: {"OK": 200, "action": "post force"} or error
-- GET SETTINGS --
GUI: {"method": "get", "path": "settings"}
HELPER:
{
"action": "get settings",
"token": <user token>,
"url": <auth url>,
"container": <container>,
"directory": <local directory>,
"exclude": <file path>,
"language": <en|el>,
"ask_to_sync": <true|false>
} or {<ERROR>: <ERROR CODE>}
-- PUT SETTINGS --
GUI: {
"method": "put", "path": "settings",
"token": <user token>,
"url": <auth url>,
"container": <container>,
"directory": <local directory>,
"exclude": <file path>,
"language": <en|el>,
"ask_to_sync": <true|false>
}
HELPER: {"CREATED": 201, "action": "put settings",} or
{<ERROR>: <ERROR CODE>, "action": "get settings",}
-- GET STATUS --
GUI: {"method": "get", "path": "status"}
HELPER: {"code": <int>,
"synced": <int>, "unsynced": <int>, "failed": <int>,
"action": "get status"
} or {<ERROR>: <ERROR CODE>, "action": "get status"}
"""
status = utils.ThreadSafeDict()
with status.lock() as d:
d.update(code=STATUS['UNINITIALIZED'], synced=0, unsynced=0, failed=0)
ui_id = None
session_db = None
accepted = False
settings = dict(token=None,
url=None,
container=None,
directory=None,
exclude=None,
ask_to_sync=True,
language="en")
cnf = AgkyraConfig()
essentials = ('url', 'token', 'container', 'directory')
def get_status(self, key=None):
""":return: updated status dict or value of specified key"""
if self.syncer and self.can_sync():
self._consume_messages()
with self.status.lock() as d:
LOGGER.debug('Status is now %s' % d['code'])
return d.get(key, None) if key else dict(d)
def set_status(self, **kwargs):
with self.status.lock() as d:
LOGGER.debug('Set status to %s' % kwargs)
d.update(kwargs)
@property
def syncer(self):
""":returns: the first syncer object or None"""
with SYNCERS.lock() as d:
for sync_key, sync_obj in d.items():
return sync_obj
return None
def clean_db(self):
"""Clean DB from current session trace"""
LOGGER.debug('Remove current session trace')
with database.TransactedConnection(self.session_db) as db:
db.unregister_heartbeat(self.ui_id)
def shutdown_syncer(self, syncer_key=0, timeout=None):
"""Shutdown the syncer backend object"""
LOGGER.debug('Shutdown syncer')
with SYNCERS.lock() as d:
syncer = d.pop(syncer_key, None)
if syncer and self.can_sync():
remaining = syncer.stop_all_daemons(timeout=timeout)
LOGGER.debug('Wait open syncs to complete')
syncer.wait_sync_threads(timeout=remaining)
def _get_default_sync(self):
"""Get global.default_sync or pick the first sync as default
If there are no syncs, create a 'default' sync.
"""
sync = self.cnf.get('global', 'default_sync')
if not sync:
for sync in self.cnf.keys('sync'):
break
self.cnf.set('global', 'default_sync', sync or 'default')
return sync or 'default'
def _get_sync_cloud(self, sync):
"""Get the <sync>.cloud or pick the first cloud and use it
In case of cloud picking, set the cloud as the <sync>.cloud for future
sessions.
If no clouds are found, create a 'default' cloud, with an empty url.
"""
try:
cloud = self.cnf.get_sync(sync, 'cloud')
except KeyError:
cloud = None
if not cloud:
for cloud in self.cnf.keys('cloud'):
break
self.cnf.set_sync(sync, 'cloud', cloud or 'default')
return cloud or 'default'
def _load_settings(self):
LOGGER.debug('Start loading settings')
sync = self._get_default_sync()
cloud = self._get_sync_cloud(sync)
for option in ('url', 'token'):
try:
value = self.cnf.get_cloud(cloud, option)
if not value:
raise Exception()
self.settings[option] = value
except Exception:
self.settings[option] = None
self.set_status(code=STATUS['SETTINGS MISSING'])
self.settings['ask_to_sync'] = (self.cnf.get('global',
'ask_to_sync') == 'on')
self.settings['language'] = self.cnf.get('global', 'language')
# for option in ('container', 'directory', 'exclude'):
for option in ('container', 'directory'):
try:
value = self.cnf.get_sync(sync, option)
if not value:
raise KeyError()
self.settings[option] = value
except KeyError:
LOGGER.debug('No %s is set' % option)
self.set_status(code=STATUS['SETTINGS MISSING'])
LOGGER.debug('Finished loading settings')
def _dump_settings(self):
LOGGER.debug('Saving settings')
sync = self._get_default_sync()
cloud = self._get_sync_cloud(sync)
new_url = self.settings.get('url') or ''
new_token = self.settings.get('token') or ''
try:
old_url = self.cnf.get_cloud(cloud, 'url') or ''
except KeyError:
old_url = new_url
while old_url and old_url != new_url:
cloud = '%s_%s' % (cloud, sync)
try:
self.cnf.get_cloud(cloud, 'url')
except KeyError:
break
LOGGER.debug('Cloud name is %s' % cloud)
self.cnf.set_cloud(cloud, 'url', new_url)
self.cnf.set_cloud(cloud, 'token', new_token)
self.cnf.set_sync(sync, 'cloud', cloud)
LOGGER.debug('Save sync settings, name is %s' % sync)
# for option in ('directory', 'container', 'exclude'):
for option in ('directory', 'container'):
self.cnf.set_sync(sync, option, self.settings.get(option) or '')
self.cnf.set('global', 'language', self.settings.get('language', 'en'))
ask_to_sync = self.settings.get('ask_to_sync', True)
self.cnf.set('global', 'ask_to_sync', 'on' if ask_to_sync else 'off')
self.cnf.write()
LOGGER.debug('Settings saved')
def _essentials_changed(self, new_settings):
"""Check if essential settings have changed in new_settings"""
return any(
[self.settings[e] != new_settings[e] for e in self.essentials])
def _consume_messages(self, max_consumption=10):
"""Update status by consuming and understanding syncer messages"""
if self.can_sync():
msg = self.syncer.get_next_message()
# if not msg:
# with self.status.lock() as d:
# if d['unsynced'] == d['synced'] + d['failed']:
# d.update(unsynced=0, synced=0, failed=0)
while msg:
if isinstance(msg, messaging.SyncMessage):
LOGGER.debug('UNSYNCED +1 %s' %
getattr(msg, 'objname', ''))
self.set_status(unsynced=self.get_status('unsynced') + 1)
elif isinstance(msg, messaging.AckSyncMessage):
LOGGER.debug('SYNCED +1 %s' % getattr(msg, 'objname', ''))
self.set_status(synced=self.get_status('synced') + 1)
elif isinstance(msg, messaging.SyncErrorMessage):
LOGGER.debug('FAILED +1 %s' % getattr(msg, 'objname', ''))
self.set_status(failed=self.get_status('failed') + 1)
elif isinstance(msg, messaging.LocalfsSyncDisabled):
LOGGER.debug('STOP BACKEND, %s' %
getattr(msg, 'objname', ''))
LOGGER.debug('CHANGE STATUS TO: %s' %
STATUS['DIRECTORY ERROR'])
self.set_status(code=STATUS['DIRECTORY ERROR'])
self.syncer.stop_all_daemons()
elif isinstance(msg, messaging.PithosSyncDisabled):
LOGGER.debug('STOP BACKEND, %s' %
getattr(msg, 'objname', ''))
self.set_status(code=STATUS['CONTAINER ERROR'])
self.syncer.stop_all_daemons()
elif isinstance(msg, messaging.PithosAuthTokenError):
LOGGER.debug('STOP BACKEND, %s' %
getattr(msg, 'objname', ''))
self.set_status(code=STATUS['TOKEN ERROR'])
self.syncer.stop_all_daemons()
elif isinstance(msg, messaging.PithosGenericError):
LOGGER.debug('STOP BACKEND, %s' %
getattr(msg, 'objname', ''))
self.set_status(code=STATUS['CRITICAL ERROR'])
self.syncer.stop_all_daemons()
LOGGER.debug('Backend message: %s %s' % (msg.name, type(msg)))
# Limit the amount of messages consumed each time
max_consumption -= 1
if max_consumption:
msg = self.syncer.get_next_message()
else:
break
def can_sync(self):
"""Check if settings are enough to setup a syncing proccess"""
return all([self.settings[e] for e in self.essentials])
def init_sync(self, leave_paused=False):
"""Initialize syncer"""
self.set_status(code=STATUS['INITIALIZING'])
sync = self._get_default_sync()
kwargs = dict(agkyra_path=AGKYRA_DIR)
# Get SSL settings
cloud = self._get_sync_cloud(sync)
try:
ignore_ssl = self.cnf.get_cloud(cloud, 'ignore_ssl') in ('on', )
kwargs['ignore_ssl'] = ignore_ssl
except KeyError:
ignore_ssl = None
if not ignore_ssl:
try:
kwargs['ca_certs'] = self.cnf.get_cloud(cloud, 'ca_certs')
except KeyError:
pass
syncer_ = None
try:
syncer_settings = setup.SyncerSettings(self.settings['url'],
self.settings['token'],
self.settings['container'],
self.settings['directory'],
**kwargs)
master = pithos_client.PithosFileClient(syncer_settings)
slave = localfs_client.LocalfsFileClient(syncer_settings)
syncer_ = syncer.FileSyncer(syncer_settings, master, slave)
# Check if syncer is ready, by consuming messages
local_ok, remote_ok = False, False
for i in range(2):
LOGGER.debug('Get message %s' % (i + 1))
msg = syncer_.get_next_message(block=True)
LOGGER.debug('Got message: %s' % msg)
if isinstance(msg, messaging.LocalfsSyncDisabled):
self.set_status(code=STATUS['DIRECTORY ERROR'])
local_ok = False
break
elif isinstance(msg, messaging.PithosSyncDisabled):
self.set_status(code=STATUS['CONTAINER ERROR'])
remote_ok = False
break
elif isinstance(msg, messaging.LocalfsSyncEnabled):
local_ok = True
elif isinstance(msg, messaging.PithosSyncEnabled):
remote_ok = True
else:
LOGGER.error('Unexpected message %s' % msg)
self.set_status(code=STATUS['CRITICAL ERROR'])
break
if local_ok and remote_ok:
syncer_.initiate_probe()
new_status = 'PAUSED' if leave_paused else 'READY'
self.set_status(code=STATUS[new_status])
except pithos_client.ClientError as ce:
LOGGER.debug('backend init failed: %s %s' % (ce, ce.status))
try:
code = {
400: STATUS['AUTH URL ERROR'],
401: STATUS['TOKEN ERROR'],
}[ce.status]
except KeyError:
code = STATUS['UNINITIALIZED']
self.set_status(code=code)
finally:
self.set_status(synced=0, unsynced=0)
with SYNCERS.lock() as d:
d[0] = syncer_
# Syncer-related methods
def get_settings(self):
return self.settings
def set_settings(self, new_settings):
"""Set the settings and dump them to permanent storage if needed"""
# Prepare setting save
old_status = self.get_status('code')
ok_not_syncing = [STATUS['READY'], STATUS['PAUSING'], STATUS['PAUSED']]
active = ok_not_syncing + [STATUS['SYNCING']]
must_reset_syncing = self._essentials_changed(new_settings)
if must_reset_syncing and old_status in active:
LOGGER.debug('Temporary backend shutdown to save settings')
self.shutdown_syncer()
# save settings
self.settings.update(new_settings)
self._dump_settings()
# Restart
LOGGER.debug('Reload settings')
self._load_settings()
can_sync = must_reset_syncing and self.can_sync()
if can_sync:
leave_paused = old_status in ok_not_syncing
LOGGER.debug('Restart backend')
self.init_sync(leave_paused=leave_paused)
def _pause_syncer(self):
syncer_ = self.syncer
syncer_.stop_decide()
LOGGER.debug('Wait open syncs to complete')
syncer_.wait_sync_threads()
def pause_sync(self):
"""Pause syncing (assuming it is up and running)"""
if self.syncer:
self.set_status(code=STATUS['PAUSING'])
self.syncer.stop_decide()
self.set_status(code=STATUS['PAUSED'])
def start_sync(self):
"""Start syncing"""
self.syncer.start_decide()
self.set_status(code=STATUS['SYNCING'])
def force_sync(self):
"""Force syncing, assuming there is a directory or container problem"""
self.set_status(code=STATUS['INITIALIZING'])
self.syncer.settings.purge_db_archives_and_enable()
self.syncer.initiate_probe()
self.set_status(code=STATUS['READY'])
def send_json(self, msg):
LOGGER.debug('send: %s' % msg)
self.send(json.dumps(msg))
# Protocol handling methods
def _post(self, r):
"""Handle POST requests"""
if self.accepted:
action = r['path']
if action == 'shutdown':
# Clean db to cause syncer backend to shut down
self.set_status(code=STATUS['SHUTTING DOWN'])
self.shutdown_syncer(timeout=5)
self.clean_db()
return
{
'init': self.init_sync,
'start': self.start_sync,
'pause': self.pause_sync,
'force': self.force_sync
}[action]()
self.send_json({'OK': 200, 'action': 'post %s' % action})
elif r['ui_id'] == self.ui_id:
self.accepted = True
self.send_json({'ACCEPTED': 202, 'action': 'post ui_id'})
self._load_settings()
status = self.get_status('code')
if self.can_sync() and status == STATUS['UNINITIALIZED']:
self.set_status(code=STATUS['SETTINGS READY'])
else:
action = r.get('path', 'ui_id')
self.send_json({'REJECTED': 401, 'action': 'post %s' % action})
self.terminate()
def _put(self, r):
"""Handle PUT requests"""
if self.accepted:
LOGGER.debug('put %s' % r)
action = r.pop('path')
self.set_settings(r)
r.update({'CREATED': 201, 'action': 'put %s' % action})
self.send_json(r)
else:
action = r['path']
self.send_json({
'UNAUTHORIZED UI': 401,
'action': 'put %s' % action
})
self.terminate()
def _get(self, r):
"""Handle GET requests"""
action = r.pop('path')
if not self.accepted:
self.send_json({
'UNAUTHORIZED UI': 401,
'action': 'get %s' % action
})
self.terminate()
else:
data = {
'settings': self.get_settings,
'status': self.get_status,
}[action]()
data['action'] = 'get %s' % action
self.send_json(data)
def received_message(self, message):
"""Route requests to corresponding handling methods"""
try:
r = json.loads('%s' % message)
except ValueError as ve:
self.send_json({'BAD REQUEST': 400})
LOGGER.error('JSON ERROR: %s' % ve)
return
try:
method = r.pop('method')
{'post': self._post, 'put': self._put, 'get': self._get}[method](r)
except KeyError as ke:
action = method + ' ' + r.get('path', '')
self.send_json({'BAD REQUEST': 400, 'action': action})
LOGGER.error('KEY ERROR: %s' % ke)
except setup.ClientError as ce:
action = '%s %s' % (method,
r.get('path', 'ui_id' if 'ui_id' in r else ''))
self.send_json({'%s' % ce: ce.status, 'action': action})
return
except Exception as e:
self.send_json({'INTERNAL ERROR': 500})
reason = '%s %s' % (method or '', r)
LOGGER.error('EXCEPTION (%s): %s' % (reason, e))
self.terminate()
messaging, utils, database, common)
from agkyra.config import AgkyraConfig, AGKYRA_DIR
if getattr(sys, 'frozen', False):
# we are running in a |PyInstaller| bundle
BASEDIR = sys._MEIPASS
ISFROZEN = True
else:
# we are running in a normal Python environment
BASEDIR = os.path.dirname(os.path.realpath(__file__))
ISFROZEN = False
RESOURCES = os.path.join(BASEDIR, 'resources')
LOGGER = logging.getLogger(__name__)
SYNCERS = utils.ThreadSafeDict()
with open(os.path.join(RESOURCES, 'ui_data/common_en.json')) as f:
COMMON = json.load(f)
with open(os.path.join(RESOURCES, 'main.json')) as f:
MAINSETTINGS = json.load(f)
STATUS = MAINSETTINGS['STATUS']
class SessionDB(database.DB):
def init(self):
db = self.db
db.execute('CREATE TABLE IF NOT EXISTS heart ('
'ui_id VARCHAR(256), address text, beat VARCHAR(32)'