def _load_hist_agg(self):
self._debug("Loading data for historical_usage table")
# necessary because it's entirely possible for a last_update query to
# return no data
if len(self.hist_agg_data) > 0 or self.dry_run:
self._load_many(self.hist_agg_query, self.hist_agg_data)
DB.local().commit()
# note that we never /use/ this to determine whether to update or
# not, this is for informational purposes only
self.set_last_update(table="historical_usage")
def _load(self):
self._info("Loading data for " + self.table + " table")
cursor = DB.local_cursor()
# necessary because it's entirely possible for a last_update query to
# return no data
if len(self.data) > 0:
cursor.executemany(self.queries['update'],
self.data)
DB.local().commit()
self._debug("Rows updated: %d" % (cursor.rowcount))
self.set_last_update()
def set_last_update(self, table=None):
"""
Set the last_update field to the current time for the given table
"""
if not table:
table = self.table
if self.dry_run:
self._debug("Setting last update on table " + table)
return
cursor = DB.local_cursor(dictionary=False)
cursor.execute(self.metadata_update, (table, ))
DB.local().commit()
def _load_many(self, query, data):
if self.dry_run:
self._debug("Special query: " + query)
else:
cursor = DB.local_cursor()
cursor.executemany(query, data)
self._debug("Rows updated: %d" % (cursor.rowcount))
def _extract_all_last_update(self):
self._info("Extracting data for " + self.table
+ " table (last_update)")
cursor = DB.remote_cursor()
cursor.execute(self.queries['query_last_update'],
(self.last_update, self.last_update))
self.db_data = cursor.fetchall()
self._debug("Rows returned: %d" % (cursor.rowcount))
def extract(self):
start = datetime.now()
self._extract_no_last_update()
cursor = DB.remote_cursor()
cursor.execute(self.tenant_owner_query)
self.tenant_owner_data = cursor.fetchall()
cursor.execute(self.tenant_member_query)
self.tenant_member_data = cursor.fetchall()
self.extract_time = datetime.now() - start
def load(self):
start = datetime.now()
# the aggregate table is simple to deal with.
self._load_simple()
# we need to be a little careful with the aggregate_host table, because
# it's a real pain to know if hosts have been removed (we capture all
# the additions, of course, but not the removals). So we need to delete
# everything and start afresh with each update. To avoid people seeing
# things in an odd state we need to wrap this in a transaction.
if not self.dry_run:
DB.local().start_transaction()
cursor = DB.local_cursor()
self._run_sql_cursor(cursor, self.aggregate_host_cleanup)
self._load_many(self.aggregate_host_query, self.agg_host_data)
DB.local().commit()
self.load_time = datetime.now() - start
def _get_last_update(cls, table):
"""
Get the time that the data was updated most recently, so that we can
process only the updated data.
"""
cursor = DB.local_cursor(dictionary=False)
cursor.execute(cls.metadata_query, (table, ))
row = cursor.fetchone()
res = None
if row:
res = row[0]
return res
def _load_has_instance_data(self):
self._debug("Updating project table with has instance data")
if len(self.has_instance_data) > 0 or self.dry_run:
self._load_many(self.has_instance_update_query,
self.has_instance_data)
DB.local().commit()
def _extract_all(self):
self._info("Extracting data for table " + self.table)
cursor = DB.remote_cursor()
cursor.execute(self.queries['query'])
self.db_data = cursor.fetchall()
self._debug("Rows returned: %d" % (cursor.rowcount))
