def _load_hist_agg(self): self._debug("Loading data for historical_usage table") # necessary because it's entirely possible for a last_update query to # return no data if len(self.hist_agg_data) > 0 or self.dry_run: self._load_many(self.hist_agg_query, self.hist_agg_data) DB.local().commit() # note that we never /use/ this to determine whether to update or # not, this is for informational purposes only self.set_last_update(table="historical_usage")
def _load(self): self._info("Loading data for " + self.table + " table") cursor = DB.local_cursor() # necessary because it's entirely possible for a last_update query to # return no data if len(self.data) > 0: cursor.executemany(self.queries['update'], self.data) DB.local().commit() self._debug("Rows updated: %d" % (cursor.rowcount)) self.set_last_update()
def set_last_update(self, table=None): """ Set the last_update field to the current time for the given table """ if not table: table = self.table if self.dry_run: self._debug("Setting last update on table " + table) return cursor = DB.local_cursor(dictionary=False) cursor.execute(self.metadata_update, (table, )) DB.local().commit()
def _load_many(self, query, data): if self.dry_run: self._debug("Special query: " + query) else: cursor = DB.local_cursor() cursor.executemany(query, data) self._debug("Rows updated: %d" % (cursor.rowcount))
def _extract_all_last_update(self): self._info("Extracting data for " + self.table + " table (last_update)") cursor = DB.remote_cursor() cursor.execute(self.queries['query_last_update'], (self.last_update, self.last_update)) self.db_data = cursor.fetchall() self._debug("Rows returned: %d" % (cursor.rowcount))
def extract(self): start = datetime.now() self._extract_no_last_update() cursor = DB.remote_cursor() cursor.execute(self.tenant_owner_query) self.tenant_owner_data = cursor.fetchall() cursor.execute(self.tenant_member_query) self.tenant_member_data = cursor.fetchall() self.extract_time = datetime.now() - start
def load(self): start = datetime.now() # the aggregate table is simple to deal with. self._load_simple() # we need to be a little careful with the aggregate_host table, because # it's a real pain to know if hosts have been removed (we capture all # the additions, of course, but not the removals). So we need to delete # everything and start afresh with each update. To avoid people seeing # things in an odd state we need to wrap this in a transaction. if not self.dry_run: DB.local().start_transaction() cursor = DB.local_cursor() self._run_sql_cursor(cursor, self.aggregate_host_cleanup) self._load_many(self.aggregate_host_query, self.agg_host_data) DB.local().commit() self.load_time = datetime.now() - start
def _get_last_update(cls, table): """ Get the time that the data was updated most recently, so that we can process only the updated data. """ cursor = DB.local_cursor(dictionary=False) cursor.execute(cls.metadata_query, (table, )) row = cursor.fetchone() res = None if row: res = row[0] return res
def _load_has_instance_data(self): self._debug("Updating project table with has instance data") if len(self.has_instance_data) > 0 or self.dry_run: self._load_many(self.has_instance_update_query, self.has_instance_data) DB.local().commit()
def _extract_all(self): self._info("Extracting data for table " + self.table) cursor = DB.remote_cursor() cursor.execute(self.queries['query']) self.db_data = cursor.fetchall() self._debug("Rows returned: %d" % (cursor.rowcount))