def __init__(self, schema_name, table_name, caller=None, validation_rules=None): self.schema_name = schema_name self.table_name = table_name self.caller = caller self._column_counter = ( ColumnValidationCounter.ColumnValidationCounter()) self.record_validators = {} self.statistics_validators = {} self.sql_validators = {} self.descriptions = {} assert validation_rules is None or isinstance(validation_rules, list) num_rules = 0 if validation_rules: for r in validation_rules: log.trace("DbProfilerValidator: " + str(r)) assert len(r) == 9 if self.add_rule(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8]): num_rules += 1 log.debug( u"DbProfilerValidator: initialized with %d validation rules" % num_rules)
def get_validation_rule(self, id): """ Args: id(integer): Returns: tuple: (id,database_name,schema_name,table_name,column_name, description,rule,param,param2) or None """ query = (u"SELECT id,database_name,schema_name,table_name,column_name," u"description,rule,param,param2 FROM validation_rule " u"WHERE id = %d" % id) log.trace("get_validation_rule: %s" % query.replace('\n', '')) tup = None try: cursor = self._conn.cursor() cursor.execute(query) r = cursor.fetchone() if r: tup = tuple(r) except Exception as e: raise InternalError(_("Could not get validation rule: "), query=query, source=e) return tup
def get_table_list(self, database_name=None, schema_name=None, table_name=None): table_list = [] cond = [] if database_name: cond.append("database_name = '%s'" % database_name) if schema_name: cond.append("schema_name = '%s'" % schema_name) if table_name: cond.append("table_name = '%s'" % table_name) where = "WHERE (%s)" % " AND ".join(cond) if cond else '' query = """ SELECT DISTINCT database_name, schema_name, table_name FROM repo {0} ORDER BY database_name, schema_name, table_name """.format(where) log.trace("get_table_list: query = %s" % query) try: cursor = self._conn.cursor() for r in cursor.execute(query): table_list.append([r[0], r[1], r[2]]) except Exception as e: log.error(_("Could not get data."), detail=unicode(e)) return None return table_list
def validate_table(self, table_data): validated_count = 0 failed_count = 0 # Run statistics validators. for label in self.statistics_validators: validator = self.statistics_validators[label] log.info(_("Validating column statistics: %s") % '; '.join(validator.rule)) validated_count += 1 try: res = validator.validate(table_data) except ValidationError as e: log.error(u'%s' % e.value) res = False if res is False: log.trace("VALIDATION FAILED: %s %s %s" % (validator.label, unicode(validator.rule), validator.column_names)) self._column_counter.incr(validator.rule[0], validator.label) failed_count += 1 else: log.trace("VALIDATION OK: %s %s %s" % (validator.label, unicode(validator.rule), validator.column_names)) return (validated_count, failed_count)
def run(self): trace() n = 0 s = 0 _max = 0 _min = None for msg in self.iter: trace() assert issubclass( type(msg), WireMessage), "unexpected message type: %s" % str(type(msg)) n += 1 s += len(msg) if len(msg) > _max: _max = len(msg) if not _min or _min > len(msg): _min = len(msg) self.file.write(msg) self.file.close() show("%d messages read" % n) show("%d bytes read" % s) show("%d = average message size" % int(s / n)) show("%d = minimum message size" % _min) show("%d = maximum message size" % _max)
def run(self): n = 0 peers = {} for msg in self.iter: assert issubclass( type(msg), BGPMessage), "unexpected message type: %s" % str(type(msg)) n += 1 bgp_msg = None bgp_msg_type = -1 bmp_msg_type = msg.msg_type #trace(str( msg.peer )) peer = msg.peer['hash'] if not peer in peers: show("adding new BMP peer context") peers[peer] = bgplib.bgpcontext.BGP_context(msg.peer) #peers[peer] = bgplib.bgpcontext.new_headless_context() if not msg.msg is None: bgp_msg = BGP_message(msg.msg) peers[peer].consume(msg.msg) bgp_msg_type = bgp_msg.bgp_type trace("message %d type %d/%d" % (n, bmp_msg_type, bgp_msg_type)) show("%d messages read" % n) for peer in peers.values(): info(str(peer))
def run(self): show("run starts") n = 0 for msg in self.next: show("message received, type %s" % str(type(msg))) n += 1 trace("process message %d" % n) show("run ends - %d cycles" % n)
def run(self): """ Thread run routine """ logger.trace('starting configuration') # Start the local telescope state application _start_slave('lts', slave_map['lts']) logger.trace('configure done') self._mc.post_event(['configure done'])
def filter_term2popover(html, pos, term, content, orig=None): log.trace("filter_term2popover: %s" % term) title = orig if orig else term popover = (u'<a tabindex="0" data-toggle="popover" data-trigger="focus" ' u'data-html="true" title="{1}" data-content="{2}" ' u'class="glossary-term">{0}</a>'.format( term, title, content)) html_new = html[0:pos] + popover + html[pos+len(term):] pos += len(popover) return (html_new, pos)
def run(self): """ Thread run routine """ logger.trace('starting unconfiguration') for entry in slave_map: properties = slave_map[entry] if properties['state'] == 'running': _stop_slave(entry, properties) logger.trace('unconfigure done') self._mc.post_event(['unconfigure done'])
def delete_textelement(self, id_): log.trace('delete_textelement: start') try: cursor = self._conn.cursor() query = u"DELETE FROM textelement WHERE id_= '%s'" % id_ cursor.execute(query) self._conn.commit() except Exception as e: raise InternalError(_("Could not delete text element: "), query=query, source=e) log.trace('delete_textelement: end') return True
def delete_tag_id(self, tag_id): log.trace('delete_tag_id: start %s' % tag_id) try: cursor = self._conn.cursor() query = u"DELETE FROM tags WHERE tag_id = '%s'" % tag_id cursor.execute(query) self._conn.commit() except Exception as e: raise InternalError(_("Could not delete tag id: "), query=query, source=e) log.trace('delete_tag_id: end') return True
def create_validation_rule(self, database_name, schema_name, table_name, column_name, description, rule, param='', param2=''): """ Args: database_name(str): schema_name(str): table_name(str): column_name(str): description(str): rule(str): param(str): param2(str): Returns: integer when the rule successfully gets created. None when already exists. """ r = self.get_validation_rules(database_name, schema_name, table_name, column_name, description, rule, param, param2) assert len(r) <= 1 if r: log.warning((_("The same validation rule already exists: ") + u"{0},{1},{2},{3},{4},{5},{6},{7}" .format(database_name, schema_name, table_name, column_name, description, rule, param, param2))) return None query = u""" INSERT INTO validation_rule (id,database_name,schema_name,table_name, column_name,description,rule,param,param2) VALUES ((SELECT coalesce(max(id),0)+1 FROM validation_rule), '{0}','{1}','{2}','{3}','{4}','{5}','{6}','{7}'); """.format(database_name, schema_name, table_name, column_name, description, rule, '' if param is None else "%s" % param.replace("'", "''"), '' if param2 is None else "%s" % param2.replace("'", "''")) log.trace("create_validation_rule: %s" % query.replace('\n', '')) id = None try: cursor = self._conn.cursor() cursor.execute(query) cursor.execute("SELECT max(id) FROM validation_rule") id = cursor.fetchone()[0] self._conn.commit() except Exception as e: raise InternalError(_("Could not register validation rule: "), query=query, source=e) return id
def get_textelements(self, id_): log.trace('get_textelements: start') texts = [] try: cursor = self._conn.cursor() query = u"SELECT text_ FROM textelement WHERE id_= '%s'" % id_ for r in cursor.execute(query): texts.append(r[0]) except Exception as e: raise InternalError(_("Could not get text element: "), query=query, source=e) log.trace('get_textelements: end') return texts
def put_textelement(self, id_, text): log.trace('put_textelement: start') try: cursor = self._conn.cursor() query = (u"INSERT INTO textelement VALUES ('%s', '%s')" % (id_, text if text else '')) cursor.execute(query) self._conn.commit() except Exception as e: raise InternalError(_("Could not register text element: "), query=query, source=e) log.trace('put_textelement: end') return True
def add(self, column_name, label): cols = column_name.replace(' ', '').split(",") if len(cols) >= 2: for col in cols: self.add(col, label) return if column_name not in self._column_counter: self._column_counter[column_name] = {} if label not in self._column_counter[column_name]: self._column_counter[column_name][label] = 0 log.trace("ColumnValidationCounter#add(): %s %s done." % (column_name, label))
def _send_options(self, dcc_id, prev, new): diff = prev.symmetric_difference(new) for fg in function_groups: k = set(fg[0].keys()) if not k.isdisjoint(diff): data = reduce(operator.or_, [m for b, m in list(fg[0].items()) if b in new]) msg = self._make_loco_msg(dcc_id, fg[1], data) self.send_msg(msg) trace( 'controller', self, "setting loco %d functions %s msg %s", dcc_id, ','.join(["%d" % (f, ) for f in k.intersection(new)]) or 'none', self.show_msg(msg))
def merge(self, data1, data2): if data1 is None: return data2 for d1 in data1: for d2 in data2: if are_same_tables(d1, d2): data1.remove(d1) log.trace(u"add: %s.%s %s -> %s" % (d1['schema_name'], d1['table_name'], d1['timestamp'], d2['timestamp'])) log.trace(u"add: %s.%s" % (d2['schema_name'], d2['table_name'])) data1.extend(data2) return data1
def update_validation_rule(self, id, database_name, schema_name, table_name, column_name, description, rule, param=None, param2=None): """ Args: id(integer): database_name(str): schema_name(str): table_name(str): column_name(str): description(str): rule(str): param(str): param2(str): Returns: True when the rule successfully gets updated, otherwise False. """ query = u""" UPDATE validation_rule SET database_name = '{0}', schema_name = '{1}', table_name = '{2}', column_name = '{3}', description = '{4}', rule = '{5}', param = '{6}', param2 = '{7}' WHERE id = {8} """.format(database_name, schema_name, table_name, column_name, description, rule, '' if param is None else "%s" % param, '' if param2 is None else "%s" % param2, id) log.trace("update_validation_rule: %s" % query.replace('\n', '')) rowcount = 0 try: cursor = self._conn.cursor() cursor.execute(query) rowcount = cursor.rowcount self._conn.commit() except Exception as e: raise InternalError(_("Could not update validation rule: "), query=query, source=e) if rowcount == 0: return False return True
def _query_value_freqs(self, query, column_name, freqs): """Common code shared by PostgreSQL/MySQL/Oracle/MSSQL profilers to get the frequencies of the column. This function updates a dictionary, and does not return any value. Args: query(str): a query string to be executed on each database. column_name(str): column name. freqs(dict): a dictionary which holds the frequencies of the columns. This function updates this dictionary as output. """ rs = self.dbdriver.q2rs(query) for r in rs.resultset: log.trace(("_query_value_freqs: col %s val %s freq %d" % (column_name, _s2u(r[0]), _s2u(r[1])))) freqs[column_name].append([_s2u(r[0]), _s2u(r[1])])
def _query_column_datetypes(self, query): """Common code shared by PostgreSQL/MySQL/Oracle/MSSQL profilers to get data types of the columns. Args: query(str): a query string to be executed on each database. Returns: dict: {column_name, [type, len]} """ data_types = {} rs = self.dbdriver.q2rs(query) for r in rs.resultset: data_types[r[0]] = [r[1], r[2]] log.trace("_query_column_datetypes: " + unicode(r)) return data_types
def _query_column_names(self, query): """Common code shared by PostgreSQL/MySQL/Oracle/MSSQL profilers to get column names of the table. Args: query(str): a query string to be executed on each database. Returns: list: a list of the column names. """ rs = self.dbdriver.q2rs(query) column_names = [] for r in rs.resultset: column_names.append(_s2u(r[0])) log.trace("_query_column_names: " + unicode(r)) return column_names
def _query_column_cardinality(self, query, column_name, cardinalities): """Common code shared by PostgreSQL/MySQL/Oracle/MSSQL profilers to get cardinality of the column. This function updates a dictionary, and does not return any value. Args: query(str): a query string to be executed on each database. column_name(str): column name. cardinality(dict): a dictionary which holds the column cardinality. This function updates this dictionary as output. """ rs = self.dbdriver.q2rs(query) for r in rs.resultset: cardinalities[column_name] = int(r[0]) log.trace(("_query_column_cardinality: col %s cardinality %d" % (column_name, cardinalities[column_name])))
def __analyze_pings( self ): alive_nodes, masters, slave_links, used_memory = self.__collect_nodes_info() nonquorum_alive_nodes = sorted( map( lambda( k, v ): k, filter( lambda ( k, v ): v <= 1 or v <= self.total / 2, alive_nodes.iteritems() ) ) ) nonquorum_masters = sorted( map( lambda( k, v ): k, filter( lambda ( k, v ): v <= 1 or v <= self.total / 2, masters.iteritems() ) ) ) quorum_alive_nodes = sorted( map( lambda( k, v ): k, filter( lambda ( k, v ): v > 1 and v > self.total / 2, alive_nodes.iteritems() ) ) ) quorum_masters = sorted( map( lambda( k, v ): k, filter( lambda ( k, v ): v > 1 and v > self.total / 2, masters.iteritems() ) ) ) logger.trace( "Total failovers = %d" % self.total ) logger.trace( "NonQuorum Alive nodes = %s" % nodes_to_string( nonquorum_alive_nodes ) ) logger.trace( "NonQuorum Masters = %s" % nodes_to_string( nonquorum_masters ) ) logger.trace( "Quorum Alive nodes = %s" % nodes_to_string( quorum_alive_nodes ) ) logger.trace( "Quorum Masters = %s" % nodes_to_string( quorum_masters ) ) if quorum_alive_nodes: self.__process_quorum( quorum_alive_nodes, quorum_masters, slave_links, used_memory ) self.__process_nonquorum( nonquorum_alive_nodes, nonquorum_masters )
def _query_schema_names(self, query, ignores=[]): """Common code shared by PostgreSQL/MySQL/Oracle/MSSQL profilers to get schema names. Args: query(str): a query string to be executed on each database. ignores(list): a list of schema names to be ignored. Returns: list: a list of the schema names. """ rs = self.dbdriver.q2rs(query) schema_names = [] for r in rs.resultset: if r[0] not in ignores: schema_names.append(_s2u(r[0])) log.trace("_query_schema_names: " + unicode(r)) return schema_names
def put_tag(self, tag_id, tag_label): log.trace('put_tag: start %s %s' % (tag_id, tag_label)) if not tag_label: return False try: cursor = self._conn.cursor() query = (u"DELETE FROM tags WHERE tag_id = '%s' " u"AND tag_label = '%s'" % (tag_id, tag_label)) cursor.execute(query) query = (u"INSERT INTO tags VALUES ('%s', '%s')" % (tag_id, tag_label)) cursor.execute(query) self._conn.commit() except Exception as e: raise InternalError(_("Could not register tag: "), query=query, source=e) log.trace('put_tag: end') return True
def get_table(self, database_name, schema_name, table_name): """ Get a table record from the repository by object name. Args: database_name(str): database name schema_name(str): schema name table_name(str): table name Returns: a dictionary of table record. {table_record} """ assert database_name and schema_name and table_name table = None log.trace('get_table: start %s.%s.%s' % (database_name, schema_name, table_name)) query = """ SELECT data FROM repo WHERE database_name = '{0}' AND schema_name = '{1}' AND table_name = '{2}' ORDER BY created_at DESC LIMIT 1 """.format(database_name, schema_name, table_name) log.debug("get_table: query = %s" % query) try: cursor = self._conn.cursor() cursor.execute(query) r = cursor.fetchone() if r: table = json.loads(unicode(r[0])) except Exception as e: raise InternalError(_("Could not get table data: "), query=query, source=e) log.trace('get_table: end') return table
def incr(self, column_name, label): cols = column_name.replace(' ', '').split(",") if len(cols) >= 2: for col in cols: self.incr(col, label) return if column_name not in self._column_counter: m = ("ColumnValidationCounter#incr() key error: %s" % column_name) raise InternalError(m) if label not in self._column_counter[column_name]: m = ("ColumnValidationCounter#incr() key error: %s, %s" % (column_name, label)) raise InternalError(m) self._column_counter[column_name][label] += 1 m = ("ColumnValidationCounter#incr(): %s,%s done." % (column_name, label)) log.trace(m) return self._column_counter[column_name][label]
def append_table(self, tab): """ Update a table record if the same record (with same timestamp) already exist. Otherwise, append the table record to the repository. Args: tab: a dictionary of table record. Returns: True on success, otherwise False. """ assert (tab['database_name'] and tab['schema_name'] and tab['table_name'] and tab['timestamp']) query = None log.trace("append_table: start %s.%s.%s" % (tab['database_name'], tab['schema_name'], tab['table_name'])) try: if self.has_table_record(tab): query = """ UPDATE repo SET data = '%s' WHERE database_name = '{database_name}' AND schema_name = '{schema_name}' AND table_name = '{table_name}' AND created_at = datetime('{timestamp}') """.format(**tab) % DbProfilerFormatter.jsonize(tab).replace("'", "''") else: query = """ INSERT INTO repo VALUES ('{database_name}','{schema_name}','{table_name}', datetime('{timestamp}'), '%s') """.format(**tab) % DbProfilerFormatter.jsonize(tab).replace("'", "''") log.trace("append_table: INSERT") log.debug("append_table: query = %s" % query) assert self._conn cursor = self._conn.cursor() assert cursor cursor.execute(query) self._conn.commit() except Exception as e: raise InternalError(_("Could not register table data: "), query=query, source=e) # Remove all tag id/label pairs to replace with new ones. tagid = "%s.%s.%s" % (tab['database_name'], tab['schema_name'], tab['table_name']) self.delete_tag_id(tagid) if tab.get('tags'): for label in tab['tags']: self.put_tag(tagid, label) log.trace("append_table: end") return True
def listener(self, line): do_next_msg = False if line: trace('controller', self, "%s received '%s'", get_time_of_day(), line) line = line.strip() now = time.time() deadline = self.last_xmit_time + constants.nce_response_wait_time if self.last_xmit_time else 0.0 if line == '!': if self.last_msg: do_next_msg = True else: self.receive_errors += 1 log(self, 'unexpected ack') elif line and self.last_msg is None: self.receive_errors += 1 log(self, "%s unexpected error response '%s'", get_time_of_day(), line) elif (line=='' or line is None) \ and self.last_xmit_time \ and deadline < now \ and self.last_msg : # timeout trace('serial', self, '%s timeout waiting for response xmit %s now %s (%.6f) deadline %s (%.6f)', get_time_of_day(), \ self.last_xmit_time, now, now-self.last_xmit_time, deadline, deadline-self.last_xmit_time) self.timeouts += 1 do_next_msg = True elif line: self.receive_errors += 1 log (self, "%s error response '%s' for message '%s'", get_time_of_day(), \ line, self.show_msg(self.last_msg)) do_next_msg = True if do_next_msg: with self.lock: self.last_msg = None if self.pending: next = self.pending[0] self._transmit(next[0]) if next[1]: next[1]() self.pending = self.pending[1:]
def validate_record(self, column_names, column_values): validated_count = 0 failed_count = 0 assert len(column_names) == len(column_values) # new record validator for label in self.record_validators: validator = self.record_validators[label] validated_count += 1 try: if validator.validate(column_names, column_values) is False: log.trace("VALIDATION FAILED: %s %s %s %s" % (validator.label, unicode(validator.rule), validator.column_names, unicode(column_values))) self._column_counter.incr(validator.rule[0], validator.label) failed_count += 1 else: log.trace("VALIDATION OK: %s %s %s %s" % (validator.label, unicode(validator.rule), validator.column_names, unicode(column_values))) except ValidationError as e: log.error(u'%s' % e.value) log.trace("VALIDATION FAILED: %s %s %s %s" % (validator.label, unicode(validator.rule), validator.column_names, unicode(column_values))) self._column_counter.incr(validator.rule[0], validator.label) failed_count += 1 continue if failed_count > 0: return False return True
print >> sys.stderr, "Error while configuring logger:", e exit( 1 ) failover_id = str( uuid.uuid1() ) process_start = time.time() analyzer = PingsAnalyzer( failtime, len( failovers ) ) try: httpserver = HttpServer( port, analyzer ) except socket.error, e: print >> sys.stderr, "Error while starting http server:", e exit( 1 ) print "Replica process started. ReplicaProcessId = %s, Port = %d" % ( failover_id, port ) logger.trace( "Replica process started. ReplicaProcessId = %s, Port = %d" % ( failover_id, port ) ) become_daemon() write_pid( pidfile ) httpserver.start() lasttime = {} info = {} while True: for hostport in redises: r = redis.Redis( hostport[ 0 ], port = int( hostport[ 1 ] ), socket_timeout = check_delay / 2 ) role, master_host, master_port, used_memory = "", "", "", "" try: redis_info = r.info()
def none(data): # Uh...what? pass if __name__ == "__main__": # Unit test/example usage: import logger # Set the logging type you want to use (stdout logging): logger.setLogger(FileLogger(sys.stdout)) # Set the most verbose you want to log (TRACE, DEBUG, INFO, WARN, ERROR, FATAL, NONE) logger.setLogLevel(logger.TRACE) # Log a message: logger.log(logger.INFO, "logger!") time.sleep(0.01) # Alternatively, use logger.error("errrrr") logger.trace("some trace data: %d - %f - %s" % (5, 8.3, 'cows'))
def __init__(self): log.trace("Created a Table") self.points = [] return