class MockManager(object):
def __init__(self):
self.hosts = {}
self.rows = {}
self.model = Model()
self.overlay = OverlayModel(self.model)
self.overlay.add_callback(self.on_commit)
self.placement = Placement(self)
def update_placement(self, hosts, name, pkey):
row = (name, pkey)
for host in self.rows.get(row, set()).difference(set(hosts)):
host = self.hosts.setdefault(host, MockHost())
host.desired.setdefault(name, set()).discard(pkey)
if not host.desired[name]:
del host.desired[name]
self.rows[row] = hosts
if not hosts:
del self.rows[row]
for host in hosts:
host = self.hosts.setdefault(host, MockHost())
host.desired.setdefault(name, set()).add(pkey)
def on_commit(self, rows):
damaged = set()
hosts = set()
for old, new in rows:
for row in self.placement.damage(old):
damaged.add((row.table.name, row.pkey))
for row in self.placement.damage(new):
damaged.add((row.table.name, row.pkey))
yield
for name, pkey in damaged:
hosts = set(self.placement.repair(Row(self.model[name], pkey)))
self.update_placement(hosts, name, pkey)
def __init__(self):
self.hosts = {}
self.rows = {}
self.model = Model()
self.overlay = OverlayModel(self.model)
self.overlay.add_callback(self.on_commit)
self.placement = Placement(self)
def __init__(self, router, db, notifier, apikey):
"""
Stores the event sinks for later use.
"""
self.db = db
self.router = router
# API secret key.
self.apikey = apikey
# This is where we keep the configuration and status data.
self.model = Model()
# This is an overlay where we stage changes for notifications
# and placement purposes.
self.overlay = OverlayModel(self.model)
# Create listener for applying changes in database.
self.listener = DatabaseListener(self.db.engine.url)
self.listener.add_callback(self.apply_changes)
# This is how we notify users via websockets
self.notifier = notifier
# Map of hosts by their uuids so that we can maintain some
# state information about our communication with them.
self.hosts = {}
# Mapping of `(name, pkey)` pairs to hosts the rows were placed on.
# Very relevant to the placement algorithm.
self.rows = {}
# Hook up the placement implementation with manager.
self.placement = Placement(self)
# Watch for model updates to be forwarded to placement and
# individual hosts.
self.overlay.add_callback(self.on_commit)
class Manager(object):
"""
The main application logic of Sparkle.
"""
def __init__(self, router, db, notifier, apikey):
"""
Stores the event sinks for later use.
"""
self.db = db
self.router = router
# API secret key.
self.apikey = apikey
# This is where we keep the configuration and status data.
self.model = Model()
# This is an overlay where we stage changes for notifications
# and placement purposes.
self.overlay = OverlayModel(self.model)
# Create listener for applying changes in database.
self.listener = DatabaseListener(self.db.engine.url)
self.listener.add_callback(self.apply_changes)
# This is how we notify users via websockets
self.notifier = notifier
# Map of hosts by their uuids so that we can maintain some
# state information about our communication with them.
self.hosts = {}
# Mapping of `(name, pkey)` pairs to hosts the rows were placed on.
# Very relevant to the placement algorithm.
self.rows = {}
# Hook up the placement implementation with manager.
self.placement = Placement(self)
# Watch for model updates to be forwarded to placement and
# individual hosts.
self.overlay.add_callback(self.on_commit)
def start(self):
"""
Launches startup-triggered asynchronous operations.
"""
print 'starting manager'
# We need to load data from database on startup.
self.schedule_load()
def schedule_load(self):
"""
Schedules an asynchronous attempt to load DB data.
If the load fails, it is automatically retried every 15
seconds until it succeeds. Call only once.
"""
print 'scheduling data load'
def load():
# Drop any cached changes.
self.db.rollback()
# Attempt to connect database change listener so that we
# don't miss out on any notifications.
self.listener.connect()
# Stash for the extracted data.
data = []
for name, table in schema.tables.iteritems():
if not table.virtual:
for row in getattr(self.db, name).all():
part = {c.name: getattr(row, c.name) for c in row.c}
pkey = table.primary_key(part)
data.append((name, pkey, 'desired', part))
# Return complete data set to be loaded into the model.
return data
# Attempt the load the data.
d = deferToThread(load)
# Load failure handler traps just the OperationalError from
# database, other exceptions need to be propagated so that we
# don't break debugging.
def failure(fail):
print 'database connection failed, retrying in 15 seconds'
reactor.callLater(15, self.schedule_load)
# In case of success
def success(data):
print 'data successfully loaded'
self.overlay.load(data)
self.overlay.commit()
# Start send out the database changes to the clients.
self.notifier.set_model(self.overlay)
self.notifier.start()
# Start processing database changes.
self.listener.start()
# Configure where to go from there.
d.addCallbacks(success, failure)
def apply_changes(self, changes):
"""Incorporate changes from database into the model."""
self.overlay.load(changes)
self.overlay.commit()
def receive(self, message, sender):
if message['uuid'] not in self.hosts:
self.hosts[message['uuid']] = Twilight(self, message['uuid'])
self.hosts[message['uuid']].receive(message, sender)
def on_commit(self, rows):
"""
Feed changed rows to the placement algorithm and adjust
placement for individual hosts.
"""
# Rows that need their placement repaired.
damaged = set()
# Affected hosts to be flushed after commit.
hosts = set()
# Damage all old rows and all rows that depend on them.
for old, new in rows:
for row in self.placement.damage(old):
damaged.add((row.table.name, row.pkey))
for row in self.placement.damage(new):
damaged.add((row.table.name, row.pkey))
# Let the commit proceed.
yield
# Create set of changed rows for faster lookup below when we
# decide whether to send them out or not.
modified = set(tuple(row[:2]) for row in rows)
# Repair placement for all those rows using their new version in
# the model.
for name, pkey in damaged:
row = Row(self.model[name], pkey)
row_hosts = set(self.placement.repair(row))
self.update_placement(row_hosts, name, pkey)
hosts.update(row_hosts)
# Notify hosts about actually modified rows only.
if (name, pkey) in modified:
for host in row_hosts:
self.hosts[host].on_row_changed(name, pkey)
# Flush the affected hosts.
for host in hosts:
self.hosts[host].send_pending_changes()
def update_placement(self, hosts, name, pkey):
"""
Update placement of specified row.
"""
old_hosts = self.rows.get((name, pkey), set())
for host in old_hosts.difference(hosts):
self.withdraw(host, name, pkey)
for host in hosts:
self.bestow(host, name, pkey)
for host in old_hosts.symmetric_difference(hosts):
self.hosts[host].on_row_changed(name, pkey)
def bestow(self, host, name, pkey):
if host not in self.hosts:
self.hosts[host] = Twilight(self, host)
host = self.hosts[host]
host.desired.setdefault(name, set()).add(pkey)
hosts = self.rows.setdefault((name, pkey), set())
hosts.add(host.uuid)
def withdraw(self, host, name, pkey):
if host not in self.hosts:
return
row = (name, pkey)
host = self.hosts[host]
host.desired.setdefault(name, set()).discard(row)
hosts = self.rows.setdefault(row, set())
hosts.discard(host.uuid)
if not hosts:
del self.rows[row]
if not host.desired[name]:
del host.desired[name]
def list_collection(self, path, keys):
"""
Called from API to obtain list of collection items.
"""
# Verify that collection parent exists.
self.model.path_row(path[:-1], keys)
# Find endpoint for the collection itself.
endpoint = schema.resolve_path(path)
if endpoint.parent.table is None:
# Top-level collections do not have any parents.
# Filter using the endpoint filter
filter = dict(endpoint.filter)
rows = self.model[endpoint.table.name].list(**filter)
else:
# Filter using the endpoint filter and parent relationship.
pname = endpoint.parent.table.name
filter = dict(endpoint.filter)
filter.update({pname: keys[pname]})
rows = self.model[endpoint.table.name].list(**filter)
if isinstance(schema.tables[endpoint.table.name].pkey, basestring):
# Return rows keyed by the primary key.
return {row.pkey: row.to_dict() for row in rows}
else:
# Return rows keyed by the missing portion of composite
# primary key.
for key in schema.tables[endpoint.table.name].pkey:
if key not in keys:
return {row.get(key): row.to_dict() for row in rows}
def list_collection_join(self, path, keys):
"""
Called from API to obtain list of collection items details.
"""
# Verify that collection parent exists.
self.model.path_row(path[:-1], keys)
# Find endpoint for the collection itself.
endpoint = schema.resolve_path(path)
if endpoint.parent.table is None:
# Top-level collections do not have any parents.
rows = self.model[endpoint.table.name].list()
else:
# Filter using the endpoint filter and parent relationship.
pname = endpoint.parent.table.name
filter = dict(endpoint.filter)
filter.update({pname: keys[pname]})
rows = self.model[endpoint.table.name].list(**filter)
join = schema.tables[endpoint.table.name].join
if not join:
return {}
else:
items = {}
# Return rows keyed by primary key of joined table.
to_expand = path[-1]
if to_expand in join:
for key in join:
if key not in keys:
for row in rows:
items[row.get(key)] = self.model[to_expand][row.get(key)].to_dict()
# Add data of the joining object
items[row.get(key)]['joined'] = {row.pkey: row.to_dict()}
return items
def get_entity(self, path, keys):
"""Called from API to obtain entity description."""
return self.model.path_row(path, keys).to_dict()