def __init__(self, schema_name, table_name, caller=None,
validation_rules=None):
self.schema_name = schema_name
self.table_name = table_name
self.caller = caller
self._column_counter = (
ColumnValidationCounter.ColumnValidationCounter())
self.record_validators = {}
self.statistics_validators = {}
self.sql_validators = {}
self.descriptions = {}
assert validation_rules is None or isinstance(validation_rules, list)
num_rules = 0
if validation_rules:
for r in validation_rules:
log.trace("DbProfilerValidator: " + str(r))
assert len(r) == 9
if self.add_rule(r[0], r[1], r[2], r[3], r[4], r[5], r[6],
r[7], r[8]):
num_rules += 1
log.debug(
u"DbProfilerValidator: initialized with %d validation rules" %
num_rules)
def get_validation_rule(self, id):
"""
Args:
id(integer):
Returns:
tuple: (id,database_name,schema_name,table_name,column_name,
description,rule,param,param2) or None
"""
query = (u"SELECT id,database_name,schema_name,table_name,column_name,"
u"description,rule,param,param2 FROM validation_rule "
u"WHERE id = %d" % id)
log.trace("get_validation_rule: %s" % query.replace('\n', ''))
tup = None
try:
cursor = self._conn.cursor()
cursor.execute(query)
r = cursor.fetchone()
if r:
tup = tuple(r)
except Exception as e:
raise InternalError(_("Could not get validation rule: "),
query=query, source=e)
return tup
def get_table_list(self, database_name=None, schema_name=None,
table_name=None):
table_list = []
cond = []
if database_name:
cond.append("database_name = '%s'" % database_name)
if schema_name:
cond.append("schema_name = '%s'" % schema_name)
if table_name:
cond.append("table_name = '%s'" % table_name)
where = "WHERE (%s)" % " AND ".join(cond) if cond else ''
query = """
SELECT DISTINCT database_name, schema_name, table_name
FROM repo
{0}
ORDER BY database_name, schema_name, table_name
""".format(where)
log.trace("get_table_list: query = %s" % query)
try:
cursor = self._conn.cursor()
for r in cursor.execute(query):
table_list.append([r[0], r[1], r[2]])
except Exception as e:
log.error(_("Could not get data."), detail=unicode(e))
return None
return table_list
def validate_table(self, table_data):
validated_count = 0
failed_count = 0
# Run statistics validators.
for label in self.statistics_validators:
validator = self.statistics_validators[label]
log.info(_("Validating column statistics: %s") %
'; '.join(validator.rule))
validated_count += 1
try:
res = validator.validate(table_data)
except ValidationError as e:
log.error(u'%s' % e.value)
res = False
if res is False:
log.trace("VALIDATION FAILED: %s %s %s" %
(validator.label, unicode(validator.rule),
validator.column_names))
self._column_counter.incr(validator.rule[0], validator.label)
failed_count += 1
else:
log.trace("VALIDATION OK: %s %s %s" %
(validator.label, unicode(validator.rule),
validator.column_names))
return (validated_count, failed_count)
def run(self):
trace()
n = 0
s = 0
_max = 0
_min = None
for msg in self.iter:
trace()
assert issubclass(
type(msg),
WireMessage), "unexpected message type: %s" % str(type(msg))
n += 1
s += len(msg)
if len(msg) > _max:
_max = len(msg)
if not _min or _min > len(msg):
_min = len(msg)
self.file.write(msg)
self.file.close()
show("%d messages read" % n)
show("%d bytes read" % s)
show("%d = average message size" % int(s / n))
show("%d = minimum message size" % _min)
show("%d = maximum message size" % _max)
def run(self):
n = 0
peers = {}
for msg in self.iter:
assert issubclass(
type(msg),
BGPMessage), "unexpected message type: %s" % str(type(msg))
n += 1
bgp_msg = None
bgp_msg_type = -1
bmp_msg_type = msg.msg_type
#trace(str( msg.peer ))
peer = msg.peer['hash']
if not peer in peers:
show("adding new BMP peer context")
peers[peer] = bgplib.bgpcontext.BGP_context(msg.peer)
#peers[peer] = bgplib.bgpcontext.new_headless_context()
if not msg.msg is None:
bgp_msg = BGP_message(msg.msg)
peers[peer].consume(msg.msg)
bgp_msg_type = bgp_msg.bgp_type
trace("message %d type %d/%d" % (n, bmp_msg_type, bgp_msg_type))
show("%d messages read" % n)
for peer in peers.values():
info(str(peer))
def run(self):
show("run starts")
n = 0
for msg in self.next:
show("message received, type %s" % str(type(msg)))
n += 1
trace("process message %d" % n)
show("run ends - %d cycles" % n)
def run(self):
""" Thread run routine
"""
logger.trace('starting configuration')
# Start the local telescope state application
_start_slave('lts', slave_map['lts'])
logger.trace('configure done')
self._mc.post_event(['configure done'])
def filter_term2popover(html, pos, term, content, orig=None):
log.trace("filter_term2popover: %s" % term)
title = orig if orig else term
popover = (u'<a tabindex="0" data-toggle="popover" data-trigger="focus" '
u'data-html="true" title="{1}" data-content="{2}" '
u'class="glossary-term">{0}</a>'.format(
term, title, content))
html_new = html[0:pos] + popover + html[pos+len(term):]
pos += len(popover)
return (html_new, pos)
def run(self):
""" Thread run routine
"""
logger.trace('starting unconfiguration')
for entry in slave_map:
properties = slave_map[entry]
if properties['state'] == 'running':
_stop_slave(entry, properties)
logger.trace('unconfigure done')
self._mc.post_event(['unconfigure done'])
def run(self):
""" Thread run routine
"""
logger.trace('starting unconfiguration')
for entry in slave_map:
properties = slave_map[entry]
if properties['state'] == 'running':
_stop_slave(entry, properties)
logger.trace('unconfigure done')
self._mc.post_event(['unconfigure done'])
def delete_textelement(self, id_):
log.trace('delete_textelement: start')
try:
cursor = self._conn.cursor()
query = u"DELETE FROM textelement WHERE id_= '%s'" % id_
cursor.execute(query)
self._conn.commit()
except Exception as e:
raise InternalError(_("Could not delete text element: "),
query=query, source=e)
log.trace('delete_textelement: end')
return True
def delete_tag_id(self, tag_id):
log.trace('delete_tag_id: start %s' % tag_id)
try:
cursor = self._conn.cursor()
query = u"DELETE FROM tags WHERE tag_id = '%s'" % tag_id
cursor.execute(query)
self._conn.commit()
except Exception as e:
raise InternalError(_("Could not delete tag id: "),
query=query, source=e)
log.trace('delete_tag_id: end')
return True
def create_validation_rule(self, database_name, schema_name, table_name,
column_name, description, rule,
param='', param2=''):
"""
Args:
database_name(str):
schema_name(str):
table_name(str):
column_name(str):
description(str):
rule(str):
param(str):
param2(str):
Returns:
integer when the rule successfully gets created. None when already
exists.
"""
r = self.get_validation_rules(database_name, schema_name, table_name,
column_name, description, rule,
param, param2)
assert len(r) <= 1
if r:
log.warning((_("The same validation rule already exists: ") +
u"{0},{1},{2},{3},{4},{5},{6},{7}"
.format(database_name, schema_name, table_name,
column_name, description, rule,
param, param2)))
return None
query = u"""
INSERT INTO validation_rule (id,database_name,schema_name,table_name,
column_name,description,rule,param,param2)
VALUES ((SELECT coalesce(max(id),0)+1 FROM validation_rule),
'{0}','{1}','{2}','{3}','{4}','{5}','{6}','{7}');
""".format(database_name, schema_name, table_name, column_name, description,
rule,
'' if param is None else "%s" % param.replace("'", "''"),
'' if param2 is None else "%s" % param2.replace("'", "''"))
log.trace("create_validation_rule: %s" % query.replace('\n', ''))
id = None
try:
cursor = self._conn.cursor()
cursor.execute(query)
cursor.execute("SELECT max(id) FROM validation_rule")
id = cursor.fetchone()[0]
self._conn.commit()
except Exception as e:
raise InternalError(_("Could not register validation rule: "),
query=query, source=e)
return id
def get_textelements(self, id_):
log.trace('get_textelements: start')
texts = []
try:
cursor = self._conn.cursor()
query = u"SELECT text_ FROM textelement WHERE id_= '%s'" % id_
for r in cursor.execute(query):
texts.append(r[0])
except Exception as e:
raise InternalError(_("Could not get text element: "),
query=query, source=e)
log.trace('get_textelements: end')
return texts
def put_textelement(self, id_, text):
log.trace('put_textelement: start')
try:
cursor = self._conn.cursor()
query = (u"INSERT INTO textelement VALUES ('%s', '%s')" %
(id_, text if text else ''))
cursor.execute(query)
self._conn.commit()
except Exception as e:
raise InternalError(_("Could not register text element: "),
query=query, source=e)
log.trace('put_textelement: end')
return True
def add(self, column_name, label):
cols = column_name.replace(' ', '').split(",")
if len(cols) >= 2:
for col in cols:
self.add(col, label)
return
if column_name not in self._column_counter:
self._column_counter[column_name] = {}
if label not in self._column_counter[column_name]:
self._column_counter[column_name][label] = 0
log.trace("ColumnValidationCounter#add(): %s %s done." %
(column_name, label))
def _send_options(self, dcc_id, prev, new):
diff = prev.symmetric_difference(new)
for fg in function_groups:
k = set(fg[0].keys())
if not k.isdisjoint(diff):
data = reduce(operator.or_,
[m for b, m in list(fg[0].items()) if b in new])
msg = self._make_loco_msg(dcc_id, fg[1], data)
self.send_msg(msg)
trace(
'controller', self, "setting loco %d functions %s msg %s",
dcc_id,
','.join(["%d" % (f, )
for f in k.intersection(new)]) or 'none',
self.show_msg(msg))
def merge(self, data1, data2):
if data1 is None:
return data2
for d1 in data1:
for d2 in data2:
if are_same_tables(d1, d2):
data1.remove(d1)
log.trace(u"add: %s.%s %s -> %s" %
(d1['schema_name'], d1['table_name'],
d1['timestamp'], d2['timestamp']))
log.trace(u"add: %s.%s" %
(d2['schema_name'], d2['table_name']))
data1.extend(data2)
return data1
def update_validation_rule(self, id, database_name, schema_name,
table_name, column_name, description,
rule, param=None, param2=None):
"""
Args:
id(integer):
database_name(str):
schema_name(str):
table_name(str):
column_name(str):
description(str):
rule(str):
param(str):
param2(str):
Returns:
True when the rule successfully gets updated, otherwise False.
"""
query = u"""
UPDATE validation_rule
SET database_name = '{0}',
schema_name = '{1}',
table_name = '{2}',
column_name = '{3}',
description = '{4}',
rule = '{5}',
param = '{6}',
param2 = '{7}'
WHERE id = {8}
""".format(database_name, schema_name, table_name, column_name, description,
rule,
'' if param is None else "%s" % param,
'' if param2 is None else "%s" % param2, id)
log.trace("update_validation_rule: %s" % query.replace('\n', ''))
rowcount = 0
try:
cursor = self._conn.cursor()
cursor.execute(query)
rowcount = cursor.rowcount
self._conn.commit()
except Exception as e:
raise InternalError(_("Could not update validation rule: "),
query=query, source=e)
if rowcount == 0:
return False
return True
def _query_value_freqs(self, query, column_name, freqs):
"""Common code shared by PostgreSQL/MySQL/Oracle/MSSQL profilers
to get the frequencies of the column.
This function updates a dictionary, and does not return any value.
Args:
query(str): a query string to be executed on each database.
column_name(str): column name.
freqs(dict): a dictionary which holds the frequencies of the columns.
This function updates this dictionary as output.
"""
rs = self.dbdriver.q2rs(query)
for r in rs.resultset:
log.trace(("_query_value_freqs: col %s val %s freq %d" %
(column_name, _s2u(r[0]), _s2u(r[1]))))
freqs[column_name].append([_s2u(r[0]), _s2u(r[1])])
def _query_column_datetypes(self, query):
"""Common code shared by PostgreSQL/MySQL/Oracle/MSSQL profilers
to get data types of the columns.
Args:
query(str): a query string to be executed on each database.
Returns:
dict: {column_name, [type, len]}
"""
data_types = {}
rs = self.dbdriver.q2rs(query)
for r in rs.resultset:
data_types[r[0]] = [r[1], r[2]]
log.trace("_query_column_datetypes: " + unicode(r))
return data_types
def _query_column_names(self, query):
"""Common code shared by PostgreSQL/MySQL/Oracle/MSSQL profilers
to get column names of the table.
Args:
query(str): a query string to be executed on each database.
Returns:
list: a list of the column names.
"""
rs = self.dbdriver.q2rs(query)
column_names = []
for r in rs.resultset:
column_names.append(_s2u(r[0]))
log.trace("_query_column_names: " + unicode(r))
return column_names
def _query_column_cardinality(self, query, column_name, cardinalities):
"""Common code shared by PostgreSQL/MySQL/Oracle/MSSQL profilers
to get cardinality of the column.
This function updates a dictionary, and does not return any value.
Args:
query(str): a query string to be executed on each database.
column_name(str): column name.
cardinality(dict): a dictionary which holds the column cardinality.
This function updates this dictionary as output.
"""
rs = self.dbdriver.q2rs(query)
for r in rs.resultset:
cardinalities[column_name] = int(r[0])
log.trace(("_query_column_cardinality: col %s cardinality %d" %
(column_name, cardinalities[column_name])))
def __analyze_pings( self ):
alive_nodes, masters, slave_links, used_memory = self.__collect_nodes_info()
nonquorum_alive_nodes = sorted( map( lambda( k, v ): k, filter( lambda ( k, v ): v <= 1 or v <= self.total / 2, alive_nodes.iteritems() ) ) )
nonquorum_masters = sorted( map( lambda( k, v ): k, filter( lambda ( k, v ): v <= 1 or v <= self.total / 2, masters.iteritems() ) ) )
quorum_alive_nodes = sorted( map( lambda( k, v ): k, filter( lambda ( k, v ): v > 1 and v > self.total / 2, alive_nodes.iteritems() ) ) )
quorum_masters = sorted( map( lambda( k, v ): k, filter( lambda ( k, v ): v > 1 and v > self.total / 2, masters.iteritems() ) ) )
logger.trace( "Total failovers = %d" % self.total )
logger.trace( "NonQuorum Alive nodes = %s" % nodes_to_string( nonquorum_alive_nodes ) )
logger.trace( "NonQuorum Masters = %s" % nodes_to_string( nonquorum_masters ) )
logger.trace( "Quorum Alive nodes = %s" % nodes_to_string( quorum_alive_nodes ) )
logger.trace( "Quorum Masters = %s" % nodes_to_string( quorum_masters ) )
if quorum_alive_nodes:
self.__process_quorum( quorum_alive_nodes, quorum_masters, slave_links, used_memory )
self.__process_nonquorum( nonquorum_alive_nodes, nonquorum_masters )
def _query_schema_names(self, query, ignores=[]):
"""Common code shared by PostgreSQL/MySQL/Oracle/MSSQL profilers
to get schema names.
Args:
query(str): a query string to be executed on each database.
ignores(list): a list of schema names to be ignored.
Returns:
list: a list of the schema names.
"""
rs = self.dbdriver.q2rs(query)
schema_names = []
for r in rs.resultset:
if r[0] not in ignores:
schema_names.append(_s2u(r[0]))
log.trace("_query_schema_names: " + unicode(r))
return schema_names
def put_tag(self, tag_id, tag_label):
log.trace('put_tag: start %s %s' % (tag_id, tag_label))
if not tag_label:
return False
try:
cursor = self._conn.cursor()
query = (u"DELETE FROM tags WHERE tag_id = '%s' "
u"AND tag_label = '%s'" %
(tag_id, tag_label))
cursor.execute(query)
query = (u"INSERT INTO tags VALUES ('%s', '%s')" %
(tag_id, tag_label))
cursor.execute(query)
self._conn.commit()
except Exception as e:
raise InternalError(_("Could not register tag: "),
query=query, source=e)
log.trace('put_tag: end')
return True
def get_table(self, database_name, schema_name, table_name):
"""
Get a table record from the repository by object name.
Args:
database_name(str): database name
schema_name(str): schema name
table_name(str): table name
Returns:
a dictionary of table record. {table_record}
"""
assert database_name and schema_name and table_name
table = None
log.trace('get_table: start %s.%s.%s' %
(database_name, schema_name, table_name))
query = """
SELECT data
FROM repo
WHERE database_name = '{0}'
AND schema_name = '{1}'
AND table_name = '{2}'
ORDER BY created_at DESC
LIMIT 1
""".format(database_name, schema_name, table_name)
log.debug("get_table: query = %s" % query)
try:
cursor = self._conn.cursor()
cursor.execute(query)
r = cursor.fetchone()
if r:
table = json.loads(unicode(r[0]))
except Exception as e:
raise InternalError(_("Could not get table data: "),
query=query, source=e)
log.trace('get_table: end')
return table
def incr(self, column_name, label):
cols = column_name.replace(' ', '').split(",")
if len(cols) >= 2:
for col in cols:
self.incr(col, label)
return
if column_name not in self._column_counter:
m = ("ColumnValidationCounter#incr() key error: %s" % column_name)
raise InternalError(m)
if label not in self._column_counter[column_name]:
m = ("ColumnValidationCounter#incr() key error: %s, %s" %
(column_name, label))
raise InternalError(m)
self._column_counter[column_name][label] += 1
m = ("ColumnValidationCounter#incr(): %s,%s done." %
(column_name, label))
log.trace(m)
return self._column_counter[column_name][label]
def append_table(self, tab):
"""
Update a table record if the same record (with same timestamp)
already exist.
Otherwise, append the table record to the repository.
Args:
tab: a dictionary of table record.
Returns:
True on success, otherwise False.
"""
assert (tab['database_name'] and tab['schema_name'] and
tab['table_name'] and tab['timestamp'])
query = None
log.trace("append_table: start %s.%s.%s" %
(tab['database_name'], tab['schema_name'],
tab['table_name']))
try:
if self.has_table_record(tab):
query = """
UPDATE repo
SET data = '%s'
WHERE database_name = '{database_name}'
AND schema_name = '{schema_name}'
AND table_name = '{table_name}'
AND created_at = datetime('{timestamp}')
""".format(**tab) % DbProfilerFormatter.jsonize(tab).replace("'", "''")
else:
query = """
INSERT INTO repo VALUES ('{database_name}','{schema_name}','{table_name}',
datetime('{timestamp}'), '%s')
""".format(**tab) % DbProfilerFormatter.jsonize(tab).replace("'", "''")
log.trace("append_table: INSERT")
log.debug("append_table: query = %s" % query)
assert self._conn
cursor = self._conn.cursor()
assert cursor
cursor.execute(query)
self._conn.commit()
except Exception as e:
raise InternalError(_("Could not register table data: "),
query=query, source=e)
# Remove all tag id/label pairs to replace with new ones.
tagid = "%s.%s.%s" % (tab['database_name'], tab['schema_name'],
tab['table_name'])
self.delete_tag_id(tagid)
if tab.get('tags'):
for label in tab['tags']:
self.put_tag(tagid, label)
log.trace("append_table: end")
return True
def listener(self, line):
do_next_msg = False
if line:
trace('controller', self, "%s received '%s'", get_time_of_day(),
line)
line = line.strip()
now = time.time()
deadline = self.last_xmit_time + constants.nce_response_wait_time if self.last_xmit_time else 0.0
if line == '!':
if self.last_msg:
do_next_msg = True
else:
self.receive_errors += 1
log(self, 'unexpected ack')
elif line and self.last_msg is None:
self.receive_errors += 1
log(self, "%s unexpected error response '%s'", get_time_of_day(),
line)
elif (line=='' or line is None) \
and self.last_xmit_time \
and deadline < now \
and self.last_msg : # timeout
trace('serial', self, '%s timeout waiting for response xmit %s now %s (%.6f) deadline %s (%.6f)', get_time_of_day(), \
self.last_xmit_time, now, now-self.last_xmit_time, deadline, deadline-self.last_xmit_time)
self.timeouts += 1
do_next_msg = True
elif line:
self.receive_errors += 1
log (self, "%s error response '%s' for message '%s'", get_time_of_day(), \
line, self.show_msg(self.last_msg))
do_next_msg = True
if do_next_msg:
with self.lock:
self.last_msg = None
if self.pending:
next = self.pending[0]
self._transmit(next[0])
if next[1]:
next[1]()
self.pending = self.pending[1:]
def validate_record(self, column_names, column_values):
validated_count = 0
failed_count = 0
assert len(column_names) == len(column_values)
# new record validator
for label in self.record_validators:
validator = self.record_validators[label]
validated_count += 1
try:
if validator.validate(column_names, column_values) is False:
log.trace("VALIDATION FAILED: %s %s %s %s" %
(validator.label, unicode(validator.rule),
validator.column_names, unicode(column_values)))
self._column_counter.incr(validator.rule[0],
validator.label)
failed_count += 1
else:
log.trace("VALIDATION OK: %s %s %s %s" %
(validator.label, unicode(validator.rule),
validator.column_names, unicode(column_values)))
except ValidationError as e:
log.error(u'%s' % e.value)
log.trace("VALIDATION FAILED: %s %s %s %s" %
(validator.label, unicode(validator.rule),
validator.column_names, unicode(column_values)))
self._column_counter.incr(validator.rule[0], validator.label)
failed_count += 1
continue
if failed_count > 0:
return False
return True
print >> sys.stderr, "Error while configuring logger:", e
exit( 1 )
failover_id = str( uuid.uuid1() )
process_start = time.time()
analyzer = PingsAnalyzer( failtime, len( failovers ) )
try:
httpserver = HttpServer( port, analyzer )
except socket.error, e:
print >> sys.stderr, "Error while starting http server:", e
exit( 1 )
print "Replica process started. ReplicaProcessId = %s, Port = %d" % ( failover_id, port )
logger.trace( "Replica process started. ReplicaProcessId = %s, Port = %d" % ( failover_id, port ) )
become_daemon()
write_pid( pidfile )
httpserver.start()
lasttime = {}
info = {}
while True:
for hostport in redises:
r = redis.Redis( hostport[ 0 ], port = int( hostport[ 1 ] ), socket_timeout = check_delay / 2 )
role, master_host, master_port, used_memory = "", "", "", ""
try:
redis_info = r.info()
def none(data):
# Uh...what?
pass
if __name__ == "__main__":
# Unit test/example usage:
import logger
# Set the logging type you want to use (stdout logging):
logger.setLogger(FileLogger(sys.stdout))
# Set the most verbose you want to log (TRACE, DEBUG, INFO, WARN, ERROR, FATAL, NONE)
logger.setLogLevel(logger.TRACE)
# Log a message:
logger.log(logger.INFO, "logger!")
time.sleep(0.01)
# Alternatively, use
logger.error("errrrr")
logger.trace("some trace data: %d - %f - %s" % (5, 8.3, 'cows'))
def __init__(self):
log.trace("Created a Table")
self.points = []
return
