def diff (self, other): diffs = [] types = (set(self.objects) | set(other.objects)) - {Column, Constraint} for t in progress_log(types, self.log, lambda cur_t: "Compared {{}} of schema {}.{}".format( self.name.schema, " Comparing {}...".format( _plural_type(cur_t)) if cur_t else '')): rename = t not in {Sequence, Synonym} diffs.extend(self.diff_subobjects(other, lambda o: o.objects.get(t, {}), rename=rename)) return diffs
def drop_invalid_objects (self, invalid_objs): self.log.info("Invalidating {}...".format(pluralize(len(invalid_objs), 'out of date object'))) for obj in progress_log(invalid_objs, self.log, "Purged {} of invalidated objects."): if obj.name.schema == self.name.schema: referenced_by = {ref.from_ for ref in obj._referenced_by} if referenced_by.difference(invalid_objs): if self.log.isEnabledFor(logging.DEBUG): self.log.debug("{} will revert to deferred. Referenced by [{}]" .format(obj.pretty_name, ", ".join(ref_obj.pretty_name for ref_obj in referenced_by))) self.make_deferred(obj) else: self.log.debug("{} can be discarded.".format(obj.pretty_name)) # no refences go outside the invalidated set obj._clear_dependencies() self.drop(obj)
def order_diffs (diffs): log = logging.getLogger('precog.diff.order_diffs') merged_diffs = {} for diff in progress_log(diffs, log, "Merging {} complete."): sql = tuple(diff.sql) if sql in merged_diffs: merged_diffs[sql].add_dependencies(diff._dependencies) else: merged_diffs[sql] = diff diffs = merged_diffs.values() # filter diffs to remove object drops that are autodropped by other diffs applicable_diffs = set(diffs) dropping = {diff.dropping: diff for diff in progress_log(diffs, log, "Gathering DROPs {} complete.") if diff.dropping} applicable_diffs = set() for diff in progress_log(diffs, log, "Filtering DROPs {} complete."): if diff.dropping: autodroppers = [obj for obj in diff.dropping.dependencies_with( Reference.AUTODROP) if obj in dropping] if autodroppers: if log.isEnabledFor(logging.DEBUG): log.debug( "Filtering {}: (Depends on {}) autodropped when dropping {}." .format(diff.pretty_name, diff.dropping.dependencies_with(Reference.AUTODROP), [a.pretty_name for a in autodroppers])) for d in autodroppers: # The other diff "produces" this diff dropping[d].produces.add(diff) continue applicable_diffs.add(diff) # Filter diffs to remove duplicate creates when a more encompassing statement # is creating the same thing as an individual one. creates = {} unnecessary_creates = set() for diff in progress_log(applicable_diffs, log, "Filtering CREATEs {} complete."): if diff.priority == Diff.CREATE and diff.produces: for product in diff.produces: if product in creates: other_diff = creates[product] if other_diff.produces != diff.produces: if other_diff.produces.issuperset(diff.produces): if log.isEnabledFor(logging.DEBUG): log.debug("Filtering {}, covered by {}" .format(diff, other_diff)) unnecessary_creates.add(diff) break; elif other_diff.produces.issubset(diff.produces): if log.isEnabledFor(logging.DEBUG): log.debug("Filtering {}, covered by {}" .format(other_diff, diff)) unnecessary_creates.add(other_diff) else: raise DuplicateCreationError(diff, other_diff) creates[product] = diff applicable_diffs.difference_update(unnecessary_creates) if log.isEnabledFor(logging.DEBUG): log.debug("All diffs:\n{}".format(pprint.pformat( {diff.pretty_name: {'sql': diff.sql, 'dependencies': {dep.pretty_name for dep in diff._dependencies}, 'produces': {product.pretty_name for product in diff.produces}, 'dropping': diff.dropping and diff.dropping.pretty_name, 'autodrop chain': diff.dropping and {dep.pretty_name for dep in diff.dropping .dependencies_with(Reference.AUTODROP)}, 'created': diff.created } for diff in diffs}))) sort_by = (lambda x: x.priority + (10 if isinstance(x, ErrorCheckingDiff) else 0) if isinstance(x, Diff) else 0) # edges is dict of obj: [dependencies, ...] edges = {} # Produced objects of diffs to be sorted S = sorted(applicable_diffs, key=sort_by) # create edge list def add_edge (from_, to): if from_ not in edges: edges[from_] = set() try: edges[from_].update(to) except TypeError: edges[from_].add(to) for diff in progress_log(diffs, log, "Edge list {} complete."): add_edge(diff, diff.dependencies) for product in diff.produces: add_edge(product, diff) for k,v in edges.items(): edges[k] = sorted(v, key=sort_by) if log.isEnabledFor(logging.DEBUG): log.debug("Edge list:\n{}".format(_edge_list(edges))) # list of sorted diffs L = [] visited = set() indent = '' def visit (node, this_visit=()): nonlocal indent cycle = False if node in this_visit: cycle = True this_visit += (node,) if cycle: raise DiffCycleError(edges, this_visit) if not node in visited: visited.add(node) # A diff may have been filtered out of diffs but remains for applicable = isinstance(node, Diff) and node in applicable_diffs if log.isEnabledFor(logging.DEBUG): if applicable: debugstr = "{}Can I apply {}?" else: debugstr = "{}Visiting {}" log.debug(debugstr.format(indent, node.pretty_name)) if node in edges: for dependent in edges[node]: indent += ' ' visit(dependent, this_visit) indent = indent[:-2] if applicable: if log.isEnabledFor(logging.DEBUG): log.debug("{}Apply {}".format(indent, node.pretty_name)) L.append(node) for node in progress_log(S, log, "Ordering {} complete."): visit(node) return L
def from_db (self): owner = self.name.schema self.log.info("Fetching schema {}...".format(owner)) schema = { 'objects': db.query_all( """ SELECT object_name , object_type , last_ddl_time FROM dba_objects WHERE owner = :o AND subobject_name IS NULL AND object_type IN ( 'FUNCTION' , 'INDEX' , 'PACKAGE' , 'PACKAGE BODY' , 'PROCEDURE' , 'SEQUENCE' , 'SYNONYM' , 'TABLE' , 'TRIGGER' , 'TYPE' , 'TYPE BODY' -- , 'VIEW' ) UNION ALL SELECT constraint_name , 'CONSTRAINT' , last_change FROM dba_constraints WHERE owner = :o -- Ignore constraints on tables in the recyclebin AND NOT (LENGTH(table_name) = 30 AND table_name LIKE 'BIN$%') """, o=owner, oracle_names=['object_name']), 'columns': db.query_all( """ SELECT table_name , COUNT(*) AS num_columns FROM dba_tab_cols WHERE owner = :o -- Ignore columns on tables in the recyclebin AND NOT (LENGTH(table_name) = 30 AND table_name LIKE 'BIN$%') GROUP BY table_name """, o=owner, oracle_names=['table_name']), 'grants': 0, # db.query_one( # """ SELECT COUNT(*) # FROM (SELECT DISTINCT owner, table_name # FROM dba_tab_privs # WHERE grantee = :o) # """, o=owner), } self.log.debug("Query complete.") total_objects = (len(schema['objects']) + sum(table['num_columns'] for table in schema['columns']) + schema['grants']) modified_times = {} for object in schema['objects']: object_name = OracleFQN(owner, object['object_name']) object_type = _to_type(object['object_type'], object_name) if issubclass(object_type, PlsqlCode): object_name = _mangle_plsql_name(object_type, object_name) object_type = PlsqlCode if object_type not in modified_times: modified_times[object_type] = {} modified_times[object_type][object_name] = object['last_ddl_time'] self.log.info("Schema {} has {}.".format(owner, pluralize(total_objects, 'object'))) to_refresh = self.read_cache(modified_times) if schema['grants']: # Refresh all grants, but only if there are actually any grants out there to_refresh[Grant] = None change_count = 0 for obj_type, names in to_refresh.items(): if obj_type is Column: for table in schema['columns']: if names is None or table['table_name'] in names: change_count += table['num_columns'] elif names is None: if obj_type in modified_times: change_count += len(modified_times[obj_type]) elif obj_type is Grant: change_count += schema['grants'] else: change_count += len(names) if to_refresh: def progress_message (o): return "Fetched {{}} of schema {}.{}".format(owner, " Currently fetching {}...".format(_plural_type(o)) if o else '') actual = 0 for obj in progress_log((obj for obj_type, names in to_refresh.items() for obj in obj_type.from_db( self.name.schema, self.database, names)), self.log, progress_message, count=change_count): actual += 1 self.add(obj) self.log.info("Fetching schema {} complete.".format(owner)) self.cache(modified_times) else: self.log.info('Using cached schema.')