def update(self):
""" start the update process """
self.timer = Timer()
print '------[node cache update]-------------------------------'
clusters = Cluster.objects.all()
deferreds = [self.get_cluster_info(cluster) for cluster in clusters]
deferred_list = DeferredList(deferreds)
deferred_list.addCallback(self.complete)
return deferred_list
def update(self):
"""
Updates the cache for all all Jobs in all clusters. This method
processes the data in bulk, where possible, to reduce runtime. Generally
this should be faster than refreshing individual VirtualMachines.
"""
self.timer = Timer()
print '------[cache update]-------------------------------'
clusters = Cluster.objects.all()
deferreds = [self.get_cluster_info(cluster) for cluster in clusters]
deferred_list = DeferredList(deferreds)
deferred_list.addCallback(self.complete)
return deferred_list
def update(self):
"""
Updates the cache for all all Clusters. This method must query clusters
individually. This is faster than lazy cache only because it only
happens once, whereas it may happen multiple times using the lazy
mechanism
"""
self.timer = Timer()
print '------[cluster cache update]-------------------------------'
clusters = Cluster.objects.all().values('id','hostname','mtime','port')
deferreds = [self.get_cluster_info(data) for data in clusters]
deferred_list = DeferredList(deferreds)
deferred_list.addCallback(self.complete)
return deferred_list
class NodeCacheUpdater(object):
"""
Updates the cache for all all Nodes in all clusters. This method
processes the data in bulk, where possible, to reduce runtime. Generally
this should be faster than refreshing individual Nodes.
"""
def update(self):
""" start the update process """
self.timer = Timer()
print '------[node cache update]-------------------------------'
clusters = Cluster.objects.all()
deferreds = [self.get_cluster_info(cluster) for cluster in clusters]
deferred_list = DeferredList(deferreds)
deferred_list.addCallback(self.complete)
return deferred_list
def get_cluster_info(self, cluster):
"""
fetch cluster info from ganeti
"""
deferred = Deferred()
url = str(NODES_URL % (cluster.hostname, cluster.port))
d = client.getPage(url)
d.addCallback(self.process_cluster_info, cluster, deferred.callback)
# XXX even when the get fails we want to send the callback so the loop
# does not stop because a single cluster had an error
def error(*args, **kwargs):
print 'ERROR retrieving: %s ' % url
deferred.callback(None)
d.addErrback(error)
return deferred
def process_cluster_info(self, info, cluster, callback):
"""
process data received from ganeti.
"""
print '%s:' % cluster.hostname
infos = json.loads(info)
self.timer.tick('info fetched from ganeti ')
updated = Counter()
base = cluster.nodes.all()
mtimes = base.values_list('hostname', 'id', 'mtime')
data = {}
for hostname, id, mtime in mtimes:
data[hostname] = (id, float(mtime) if mtime else None)
self.timer.tick('mtimes fetched from db ')
deferreds = [self.update_node(cluster, info, data, updated) for info in infos]
deferred_list = DeferredList(deferreds)
# batch update the cache updated time for all Nodes in this cluster. This
# will set the last updated time for both Nodes that were modified and for
# those that weren't. even if it wasn't modified we want the last
# updated time to be up to date.
#
# XXX don't bother checking to see whether this query needs to run. With
# normal usage it will almost always need to
def update_timestamps(result):
print ' updated: %s out of %s' % (updated, len(infos))
base.update(cached=datetime.now())
self.timer.tick('records or timestamps updated')
deferred_list.addCallback(update_timestamps)
deferred_list.addCallback(callback)
return deferred_list
def update_node(self, cluster, info, data, updated):
"""
updates an individual node: this just sets up the work in a deferred
by using callLater. Actual work is done in _update_node().
@param cluster - cluster this node is on
@param info - info from ganeti
@param data - data from database
@param updated - counter object
@return Deferred chained to _update_node() call
"""
deferred = Deferred()
args = (cluster, info, data, updated, deferred.callback)
reactor.callLater(0, self._update_node, *args)
return deferred
def _update_node(self, cluster, info, data, updated, callback):
"""
updates an individual node, this is the actual work function
@param cluster - cluster this node is on
@param info - info from ganeti
@param data - data from database
@param updated - counter object
@param callback - callback fired when method is complete.
"""
hostname = info['name']
if hostname in data:
id, mtime = data[hostname]
if not mtime or mtime < info['mtime']:
print ' Node (updated) : %s' % hostname
#print ' %s :: %s' % (mtime, datetime.fromtimestamp(info['mtime']))
# only update the whole object if it is new or modified.
parsed = Node.parse_persistent_info(info)
Node.objects.filter(pk=id) \
.update(serialized_info=cPickle.dumps(info), **parsed)
updated += 1
else:
# new node
node = Node(cluster=cluster, hostname=info['name'])
node.info = info
node.save()
id = node.pk
updated += 1
# Updates relationships between a Node and its Primary and Secondary
# VirtualMachines. This always runs even when there are no updates but
# it should execute quickly since it runs against an indexed column
#
# XXX this blocks so it may be worthwhile to spin this off into a
# deferred just to break up this method.
VirtualMachine.objects \
.filter(hostname__in=info['pinst_list']) \
.update(primary_node=id)
VirtualMachine.objects \
.filter(hostname__in=info['sinst_list']) \
.update(secondary_node=id)
callback(id)
def complete(self, result):
""" callback fired when everything is complete """
self.timer.stop()
class VirtualMachineCacheUpdater(object):
def update(self):
"""
Updates the cache for all all VirtualMachines in all clusters. This method
processes the data in bulk, where possible, to reduce runtime. Generally
this should be faster than refreshing individual VirtualMachines.
"""
self.timer = Timer()
print '------[vm cache update]-------------------------------'
clusters = Cluster.objects.all()
deferreds = [self.get_cluster_info(cluster) for cluster in clusters]
deferred_list = DeferredList(deferreds)
deferred_list.addCallback(self.complete)
return deferred_list
def get_cluster_info(self, cluster):
"""
fetch cluster info from ganeti
"""
deferred = Deferred()
url = str(VMS_URL % (cluster.hostname, cluster.port))
d = client.getPage(url)
d.addCallback(self.process_cluster_info, cluster, deferred.callback)
# XXX even when the get fails we want to send the callback so the loop
# does not stop because a single cluster had an error
def error(*args, **kwargs):
print 'ERROR retrieving: %s ' % url
deferred.callback(None)
d.addErrback(error)
return deferred
def process_cluster_info(self, info, cluster, callback):
"""
process data received from ganeti.
"""
print '%s:' % cluster.hostname
infos = json.loads(info)
self.timer.tick('info fetched from ganeti ')
updated = Counter()
base = cluster.virtual_machines.all()
mtimes = base.values_list('hostname', 'id', 'mtime', 'status')
data = {}
for name, id, mtime, status in mtimes:
data[name] = (id, float(mtime) if mtime else None, status)
self.timer.tick('mtimes fetched from db ')
deferreds = [self.update_vm(cluster, info, data, updated) for info in infos]
deferred_list = DeferredList(deferreds)
# batch update the cache updated time for all VMs in this cluster. This
# will set the last updated time for both VMs that were modified and for
# those that weren't. even if it wasn't modified we want the last
# updated time to be up to date.
#
# XXX don't bother checking to see whether this query needs to run. It
# normal usage it will almost always need to
def update_timestamps(result):
print ' updated: %s out of %s' % (updated, len(infos))
base.update(cached=datetime.now())
self.timer.tick('records or timestamps updated')
deferred_list.addCallback(update_timestamps)
# XXX it would be nice if the deferred list could be returned and this
# callback hooked up outside of the method, but that doesn't seem
# possible
deferred_list.addCallback(callback)
def update_vm(self, cluster, info, data, updated):
"""
updates an individual VirtualMachine: this just sets up the work in a
deferred by using callLater. Actual work is done in _update_vm().
@param cluster - cluster this node is on
@param info - info from ganeti
@param data - data from database
@param updated - counter object
@return Deferred chained to _update_node() call
"""
deferred = Deferred()
args = (cluster, info, data, updated, deferred.callback)
reactor.callLater(0, self._update_vm, *args)
return deferred
def _update_vm(self, cluster, info, data, updated, callback):
"""
updates an individual VirtualMachine, this is the actual work function
@param cluster - cluster this node is on
@param info - info from ganeti
@param data - data from database
@param updated - counter object
@param callback - callback fired when method is complete.
"""
name = info['name']
if name in data:
id, mtime, status = data[name]
if not mtime or mtime < info['mtime'] \
or status != info['status']:
print ' Virtual Machine (updated) : %s' % name
#print ' %s :: %s' % (mtime, datetime.fromtimestamp(info['mtime']))
# only update the whole object if it is new or modified.
#
# XXX status changes will not always be reflected in mtime
# explicitly check status to see if it has changed. failing
# to check this would result in state changes being lost
parsed = VirtualMachine.parse_persistent_info(info)
if 'delete' in parsed:
VirtualMachine.objects.filter(pk=id).delete()
else:
VirtualMachine.objects.filter(pk=id) \
.update(serialized_info=cPickle.dumps(info), **parsed)
updated += 1
else:
# new vm
vm = VirtualMachine(cluster=cluster, hostname=info['name'])
vm.info = info
vm.save()
id = vm.id
updated += 1
callback(id)
def complete(self, result):
""" callback fired when everything is complete """
self.timer.stop()
class JobCacheUpdater(object):
def update_sync(self):
"""
TODO: make this into an async method. this method is just a test to see
if an efficient update algorithm is possible
1) query job ids from cluster
Running Jobs:
2) create base query of jobs with those ids
3) filter jobs based on those not yet finished
4) query running jobs individually, updating the job and related object as needed
5) remove id from list
New Jobs:
6) any job leftover in the list is not yet in the database
"""
#for cluster in Cluster.objects.all()
pass
def update(self):
"""
Updates the cache for all all Jobs in all clusters. This method
processes the data in bulk, where possible, to reduce runtime. Generally
this should be faster than refreshing individual VirtualMachines.
"""
self.timer = Timer()
print '------[cache update]-------------------------------'
clusters = Cluster.objects.all()
deferreds = [self.get_cluster_info(cluster) for cluster in clusters]
deferred_list = DeferredList(deferreds)
deferred_list.addCallback(self.complete)
return deferred_list
def get_cluster_info(self, cluster):
"""
fetch cluster info from ganeti
"""
deferred = Deferred()
d = client.getPage(str(JOBS_URL % (cluster.hostname, cluster.port)))
d.addCallback(self.process_cluster_info, cluster, deferred.callback)
return deferred
def process_cluster_info(self, info, cluster, callback):
"""
process data received from ganeti.
"""
print '%s:' % cluster.hostname
# parse json and repackage ids as a list
ids = set((int(d['id']) for d in json.loads(info)))
print ids
self.timer.tick('info fetched from ganeti ')
updated = Counter()
# fetch list of jobs in the cluster that are not yet finished. if the
# job is already finished then we don't need to update it
db_ids = set(cluster.jobs \
.exclude(status__in=COMPLETE_STATUS) \
.values_list('job_id', flat=True))
print 'running: ', db_ids
# update all running jobs and archive any that aren't found
# XXX this could be a choke point if there are many running jobs. each
# job will be a separate ganeti query
current = db_ids & ids
archived = db_ids - ids
deferreds = [self.update_job(cluster, id, updated) for id in current]
ids -= current
print ids, current, archived
# get list of jobs that are finished. use this to filter the list of
# ids further
# XXX this could be a joke point if there are a lot of IDs that are have
# completed but have not yet been archived by ganeti.
db_ids = cluster.jobs \
.filter(job_id__in=ids, status__in=COMPLETE_STATUS) \
.values_list('job_id', flat=True)
print 'completed: ', db_ids
ids -= set(db_ids)
print 'new: ', ids
# any job id still left in the list is a new job. Create the job and
# associate it with the object it relates to
for id in ids:
deferreds.append(self.import_job(cluster, id, updated))
# archive any jobs that we do not yet have a complete status for but
# were not found in list of jobs returned by ganeti
if archived:
self.archive_jobs(cluster, archived, updated)
# XXX it would be nice if the deferred list could be returned and this
# callback hooked up outside of the method, but that doesn't seem
# possible
deferred_list = DeferredList(deferreds)
deferred_list.addCallback(callback)
def archive_jobs(self, cluster, archived, updated):
"""
updates a job that has been archived
"""
print 'archiving!'
updated += len(archived)
# XXX clear all archived jobs
Job.objects.filter(cluster=cluster, job_id__in=archived) \
.update(ignore_cache=True, status='unknown')
# XXX load all related objects to trigger their specific cleanup code.
len(VirtualMachine.objects.filter(cluster=cluster, last_job_id__in=archived))
len(Node.objects.filter(cluster=cluster, last_job_id__in=archived))
len(Cluster.objects.filter(cluster=cluster, last_job_id__in=archived))
def update_job(self, cluster, id, updated):
"""
updates an individual Job
@param cluster - cluster this node is on
@param updated - counter object
@return Deferred chained to _update_job() call
"""
deferred = Deferred()
d = client.getPage(str(JOB_URL % (cluster.hostname, cluster.port, id)))
d.addCallback(self._update_job, cluster, id, updated, deferred.callback)
d.addErrback(self._update_error, deferred.callback)
return deferred
def _update_error(self, error, callback):
print 'Error updating: %s' % error
callback(-1)
def _update_job(self, info, cluster, id, updated, callback):
"""
updates an individual Job, this is the actual work function. Jobs that
have a complete state (success or error) are updated. All other jobs
are ignored since job state is never cached.
@param info - info from ganeti
@param cluster - cluster this job is on
@param id - job_id for job
@param updated - counter object
@param callback - callback fired when method is complete.
"""
if info['status'] in COMPLETE_STATUS:
parsed = Job.parse_persistent_info(info)
Job.objects.filter(job_id=id).update(
serialized_info=cPickle.dumps(info), **parsed)
# get related model and query the object. Loading it will trigger
# any logic in check_job_status
op = info['ops'][0]
model, hostname_key = IMPORTABLE_JOBS[op['OP_ID']]
hostname = op[hostname_key]
if not isinstance(model, Cluster):
base = model.objects.filter(cluster=cluster)
else:
base = model.objects.all()
obj = base.get(hostname=hostname)
updated += 1
callback(id)
def import_job(self, cluster, id, updated):
"""
import an individual Job
@param cluster - cluster this job is on
@param id - job_id of job
@param updated - counter object
@return Deferred chained to _import_job() call
"""
print 'importing : ', cluster, id
deferred = Deferred()
d = client.getPage(str(JOB_URL % (cluster.hostname, cluster.port, id)))
d.addCallback(self._import_job, cluster, id, updated, deferred.callback)
d.addErrback(self._update_error, deferred.callback)
return deferred
def _import_job(self, info, cluster, id, updated, callback):
"""
import an individual Job, this is the actual work function. Jobs that
have a complete state (success or error) are updated. All other jobs
are ignored since job state is never cached.
@param info - info from ganeti
@param cluster - cluster this job is on
@param id - job_id for job
@param updated - counter object
@param callback - callback fired when method is complete.
"""
print 'importing >>> : ', info, cluster, id
info = json.loads(info)
if any((op['OP_ID'] in IMPORTABLE_JOBS for op in info['ops'])):
# get related mode and object
op = info['ops'][0]
model, hostname_key = IMPORTABLE_JOBS[op['OP_ID']]
hostname = op[hostname_key]
base = model.objects.filter(hostname=hostname)
if not isinstance(base, Cluster):
base = base.filter(cluster=cluster)
(obj_id,) = base.values_list('pk', flat=True)
# create job
job = Job(cluster=cluster, job_id=id, obj_type=model, obj_id=obj_id)
job.cleared = info['status'] in COMPLETE_STATUS
job.info = info
job.save()
updated += 1
callback(id)
def complete(self, result):
""" callback fired when everything is complete """
self.timer.stop()
class ClusterCacheUpdater(object):
def update(self):
"""
Updates the cache for all all Clusters. This method must query clusters
individually. This is faster than lazy cache only because it only
happens once, whereas it may happen multiple times using the lazy
mechanism
"""
self.timer = Timer()
print '------[cluster cache update]-------------------------------'
clusters = Cluster.objects.all().values('id','hostname','mtime','port')
deferreds = [self.get_cluster_info(data) for data in clusters]
deferred_list = DeferredList(deferreds)
deferred_list.addCallback(self.complete)
return deferred_list
def get_cluster_info(self, data):
"""
fetch cluster info from ganeti
"""
deferred = Deferred()
url = str(CLUSTERS_URL % data)
d = client.getPage(url)
d.addCallback(self.process_cluster_info, data, deferred.callback)
# XXX even when the get fails we want to send the callback so the loop
# does not stop because a single cluster had an error
def error(*args, **kwargs):
print 'ERROR retrieving: %s ' % url
deferred.callback(None)
d.addErrback(error)
return deferred
def process_cluster_info(self, info, data, callback):
"""
process data received from ganeti.
@param info - info from ganeti
@param data - data from database
@param callback - callback fired when method is complete.
"""
print '%s:' % data['hostname']
info = json.loads(info)
self.timer.tick('info fetched from ganeti ')
deferred = Deferred()
args = (info, data, deferred.callback)
reactor.callLater(0, self.update_cluster, *args)
# XXX it would be nice if the deferred could be returned and this
# callback hooked up outside of the method, but that doesn't seem
# possible
deferred.addCallback(callback)
def update_cluster(self, info, data, callback):
"""
updates an individual Cluster, this is the actual work function
@param info - info from ganeti
@param data - data from database
@param callback - callback fired when method is complete.
"""
mtime = data['mtime']
if not mtime or mtime < info['mtime']:
print ' Cluster (updated) : %(hostname)s' % data
#print ' %s :: %s' % (mtime, datetime.fromtimestamp(info['mtime']))
# only update the whole object if it is new or modified.
#
parsed = Cluster.parse_persistent_info(info)
Cluster.objects.filter(pk=data['id']) \
.update(serialized_info=cPickle.dumps(info), **parsed)
callback(data['id'])
def complete(self, result):
""" callback fired when everything is complete """
# batch update the cache updated time for all VMs in this cluster. This
# will set the last updated time for both VMs that were modified and for
# those that weren't. even if it wasn't modified we want the last
# updated time to be up to date.
#
# XXX don't bother checking to see whether this query needs to run. It
# normal usage it will almost always need to
Cluster.objects.update(cached=datetime.now())
self.timer.tick('records or timestamps updated')
self.timer.stop()
