def _populate_translator_data_vdict(cls, translator, obj, parent_row_dict):
table = translator[cls.TABLE_NAME]
parent_key = translator.get(cls.PARENT_KEY, None)
id_col = translator.get(cls.ID_COL, None)
key_col = translator[cls.KEY_COL]
subtrans = translator[cls.TRANSLATOR]
if subtrans[cls.TRANSLATION_TYPE] == cls.VALUE:
extract_fn = subtrans.get(cls.EXTRACT_FN, None)
converted_items = tuple([(utils.value_to_congress(k),
cls._extract_value(v, extract_fn))
for k, v in obj.items()])
if id_col:
h = cls._compute_hash(converted_items)
new_tuples = [(table, (h, ) + i) for i in converted_items]
elif parent_key:
h = None
parent_key_value = parent_row_dict[parent_key]
new_tuples = [(table, (parent_key_value, ) + i)
for i in converted_items]
else:
h = None
new_tuples = [(table, i) for i in converted_items]
return new_tuples, h
else:
assert translator[cls.TRANSLATION_TYPE] in (cls.HDICT, cls.VDICT,
cls.LIST)
new_tuples = []
vdict_rows = []
for k, v in obj.items():
if v is None:
tuples = []
row_hash = []
if cls.ID_COL in subtrans:
row_hash = cls._compute_hash([])
else:
tuples, row_hash = cls.convert_obj(v, subtrans,
{key_col: k})
if tuples:
new_tuples.extend(tuples)
vdict_row = (k, )
if cls.need_column_for_subtable_id(subtrans):
vdict_row = vdict_row + (row_hash, )
vdict_rows.append(vdict_row)
h = None
if id_col:
h = cls._compute_hash(vdict_rows)
for vdict_row in vdict_rows:
if id_col:
new_tuples.append((table, (h, ) + vdict_row))
elif parent_key:
k = parent_row_dict[parent_key]
new_tuples.append((table, (k, ) + vdict_row))
else:
new_tuples.append((table, vdict_row))
return new_tuples, h
def _format_results_to_hdict(cls, results, translator, hdict_row):
"""Convert hdict row to translator format for hdict.
results - table row entries from subtables of a translator.
translator - is the translator to convert obj.
hdict_row - all the value fields of an hdict populated in a dict.
"""
field_translators = translator[cls.FIELD_TRANSLATORS]
table = translator[cls.TABLE_NAME]
id_col = translator.get(cls.ID_COL)
parent_key = translator.get(cls.PARENT_KEY)
new_row = []
for fieldtranslator in field_translators:
col = fieldtranslator.get(cls.COL, fieldtranslator[cls.FIELDNAME])
if col in hdict_row:
new_row.append(utils.value_to_congress(hdict_row[col]))
if id_col:
h = cls._compute_hash(new_row)
new_row = (h, ) + tuple(new_row)
elif parent_key:
h = None
parent_col_name = translator.get(cls.PARENT_COL_NAME,
cls.PARENT_KEY_COL_NAME)
new_row = (hdict_row[parent_col_name], ) + tuple(new_row)
else:
h = None
new_row = tuple(new_row)
results.append((table, new_row))
return results, h
def _format_results_to_hdict(cls, obj, results, translator, hdict_row):
"""Convert hdict row to translator format for hdict.
results - table row entries from subtables of a translator.
translator - is the translator to convert obj.
hdict_row - all the value fields of an hdict populated in a dict.
"""
field_translators = translator[cls.FIELD_TRANSLATORS]
table = translator[cls.TABLE_NAME]
id_col = translator.get(cls.ID_COL)
parent_key = translator.get(cls.PARENT_KEY)
new_row = []
for fieldtranslator in field_translators:
col = fieldtranslator.get(cls.COL,
fieldtranslator[cls.FIELDNAME])
if col in hdict_row:
new_row.append(utils.value_to_congress(hdict_row[col]))
if id_col:
h = cls._compute_id(id_col, obj, new_row)
new_row = (h,) + tuple(new_row)
elif parent_key:
h = None
parent_col_name = translator.get(cls.PARENT_COL_NAME,
cls.PARENT_KEY_COL_NAME)
new_row = (hdict_row[parent_col_name],) + tuple(new_row)
else:
h = None
new_row = tuple(new_row)
results.append((table, new_row))
return results, h
def test_value_to_congress(self):
self.assertEqual("abc", utils.value_to_congress("abc"))
self.assertEqual("True", utils.value_to_congress(True))
self.assertEqual("False", utils.value_to_congress(False))
self.assertEqual(0, utils.value_to_congress(0))
self.assertEqual(1, utils.value_to_congress(1))
self.assertEqual(123, utils.value_to_congress(123))
if sys.version < '3':
self.assertEqual(456.0, utils.value_to_congress(456.0))
def test_value_to_congress(self):
self.assertEqual("abc", utils.value_to_congress("abc"))
self.assertEqual("True", utils.value_to_congress(True))
self.assertEqual("False", utils.value_to_congress(False))
self.assertEqual(0, utils.value_to_congress(0))
self.assertEqual(1, utils.value_to_congress(1))
self.assertEqual(123, utils.value_to_congress(123))
if sys.version < '3':
self.assertEqual(456.0, utils.value_to_congress(456.0))
def _translate_snapshots(self, obj):
t_list = []
for s in obj:
stuple = (s.status, s.created_at, s.volume_id, s.size, s.id,
s.name)
row = list(stuple)
for v in row:
row[row.index(v)] = utils.value_to_congress(v)
t_list.append(tuple(row))
return t_list
def cfg_value_to_congress(value):
"""Sanitize values for congress
values of log formatting options typically contains
'%s' etc, which should not be put in datalog
"""
if isinstance(value, str):
value = value.replace('%', '')
if value is None:
return ''
return utils.value_to_congress(value)
def _translate_services(self, obj):
t_list = []
for s in obj:
stuple = (s.status, s.binary, s.zone, s.state, s.updated_at,
s.host, s.disabled_reason)
row = list(stuple)
for v in row:
row[row.index(v)] = utils.value_to_congress(v)
t_list.append(tuple(row))
return t_list
def _translate_volumes(self, obj):
t_list = []
for v in obj:
vtuple = (v.id, v.size, v.user_id, v.status, v.description, v.name,
v.bootable, v.created_at, v.volume_type)
row = list(vtuple)
for s in row:
row[row.index(s)] = utils.value_to_congress(s)
t_list.append(tuple(row))
return t_list
def _translate_snapshots(self, obj):
t_list = []
for s in obj:
stuple = (s.status, s.created_at, s.volume_id,
s.size, s.id, s.name)
row = list(stuple)
for v in row:
row[row.index(v)] = utils.value_to_congress(v)
t_list.append(tuple(row))
return t_list
def _translate_services(self, obj):
t_list = []
for s in obj:
stuple = (s.status, s.binary, s.zone,
s.state, s.updated_at, s.host,
s.disabled_reason)
row = list(stuple)
for v in row:
row[row.index(v)] = utils.value_to_congress(v)
t_list.append(tuple(row))
return t_list
def _translate_volumes(self, obj):
t_list = []
for v in obj:
vtuple = (v.id, v.size, v.user_id, v.status,
v.description, v.name, v.bootable,
v.created_at, v.volume_type)
row = list(vtuple)
for s in row:
row[row.index(s)] = utils.value_to_congress(s)
t_list.append(tuple(row))
return t_list
def _add_properties(self, obj_id, key, value):
"""Add a set of (obj_id, key, value) to properties table.
:param obj_id: uuid of object
:param key: property name. For the case value is a list, the
same key is used for multiple values.
:param value: property value. If value is a dict, the nested
properties will be mapped using dot notation.
"""
if value is None or value == "":
return
if isinstance(value, dict):
for k, v in value.iteritems():
new_key = key + "." + k
self._add_properties(obj_id, new_key, v)
elif isinstance(value, list):
if len(value) == 0:
return
for item in value:
self.state[self.PROPERTIES].add((obj_id, key, value_to_congress(item)))
else:
self.state[self.PROPERTIES].add((obj_id, key, value_to_congress(value)))
def _translate_meters(self, obj):
self.meters = []
key_to_index = self.meter_key_position_map()
max_meter_index = max(key_to_index.values()) + 1
t_list = []
for p in obj:
if type(p) != type(dict()):
p_dict = p.to_dict()
else:
p_dict = p
row = ['None'] * max_meter_index
for k, v in p_dict.items():
row[key_to_index[k]] = value_to_congress(v)
t_list.append(tuple(row))
self.meters = t_list
def _add_properties(self, obj_id, key, value):
"""Add a set of (obj_id, key, value) to properties table.
:param obj_id: uuid of object
:param key: property name. For the case value is a list, the
same key is used for multiple values.
:param value: property value. If value is a dict, the nested
properties will be mapped using dot notation.
"""
if value is None or value == '':
return
if isinstance(value, dict):
for k, v in value.items():
new_key = key + "." + k
self._add_properties(obj_id, new_key, v)
elif isinstance(value, list):
if len(value) == 0:
return
for item in value:
self.state[self.PROPERTIES].add(
(obj_id, key, utils.value_to_congress(item)))
else:
self.state[self.PROPERTIES].add(
(obj_id, key, utils.value_to_congress(value)))
def _translate_alarms(self, obj):
LOG.debug("Translating ALARM object %s", obj)
self.alarms = []
self.alarm_threshold_rule = []
key_to_index = self.alarm_key_position_map()
max_alarm_index = max(key_to_index.values()) + 1
t_list = []
t_thres_list = []
for k in obj:
if type(k) != type(dict()):
k_dict = k.to_dict()
else:
k_dict = k
row = ['None'] * max_alarm_index
for p, v in k_dict.items():
if p == 'threshold_rule':
threshold_rule_id = str(uuid.uuid1())
for s, t in v.items():
if type(t) != type(list()) and type(t) != type(dict()):
# FIXME(madhumohan): Dirty workaround. A cleaner
# approach is required to handled None object in
# the data
if t is None:
t = 'None'
row_thres_tuple = (threshold_rule_id, s, t)
t_thres_list.append(row_thres_tuple)
row[key_to_index['threshold_rule']] = threshold_rule_id
else:
if p in key_to_index:
row[key_to_index[p]] = value_to_congress(v)
else:
LOG.info("Ignoring unexpected dict key %s", p)
t_list.append(tuple(row))
LOG.debug("Generated alarm list %s", t_list)
LOG.debug("Generated threshold rule list %s", t_thres_list)
self.alarms = t_list
self.alarm_threshold_rule = t_thres_list
def _populate_translator_data_vdict(cls, translator, obj,
parent_row_dict):
table = translator[cls.TABLE_NAME]
parent_key = translator.get(cls.PARENT_KEY, None)
id_col = translator.get(cls.ID_COL, None)
key_col = translator[cls.KEY_COL]
subtrans = translator[cls.TRANSLATOR]
if subtrans[cls.TRANSLATION_TYPE] == cls.VALUE:
extract_fn = subtrans.get(cls.EXTRACT_FN, None)
converted_items = tuple([(utils.value_to_congress(k),
cls._extract_value(v, extract_fn))
for k, v in obj.items()])
if id_col:
h = cls._compute_id(id_col, obj, converted_items)
new_tuples = [(table, (h,) + i) for i in converted_items]
elif parent_key:
h = None
parent_key_value = parent_row_dict[parent_key]
new_tuples = [(table, (parent_key_value,) + i)
for i in converted_items]
else:
h = None
new_tuples = [(table, i) for i in converted_items]
return new_tuples, h
else:
assert translator[cls.TRANSLATION_TYPE] in (cls.HDICT,
cls.VDICT,
cls.LIST)
new_tuples = []
vdict_rows = []
for k, v in obj.items():
if v is None:
tuples = []
row_hash = []
if cls.ID_COL in subtrans:
row_hash = cls._compute_hash([])
else:
tuples, row_hash = cls.convert_obj(v, subtrans,
{key_col: k})
if tuples:
new_tuples.extend(tuples)
vdict_row = (k,)
if cls.need_column_for_subtable_id(subtrans):
vdict_row = vdict_row + (row_hash,)
vdict_rows.append(vdict_row)
h = None
if id_col:
h = cls._compute_id(id_col, obj, vdict_rows)
for vdict_row in vdict_rows:
if id_col:
new_tuples.append((table, (h,) + vdict_row))
elif parent_key:
k = parent_row_dict[parent_key]
new_tuples.append((table, (k,) + vdict_row))
else:
new_tuples.append((table, vdict_row))
return new_tuples, h
def _extract_value(cls, obj, extract_fn):
# Reads a VALUE object and returns (result_rows, h)
if extract_fn is None:
extract_fn = lambda x: x
return utils.value_to_congress(extract_fn(obj))
def _translate_events(self, obj):
LOG.debug("Translating EVENT object %s", obj)
self.events = []
self.event_traits = []
key_to_index = self.event_key_position_map()
max_event_index = max(key_to_index.values()) + 1
t_list = []
t_trait_list = []
# TODO(madhumohan): Need a modular implementation of the below loop for
# better readability and maintainability. Also for flexible translation
# all types of nested datastructure in the data.
for k in obj:
if type(k) != type(dict()):
k_dict = k.to_dict()
else:
k_dict = k
row = ['None'] * max_event_index
for p, v in k_dict.items():
if p == 'traits':
trait_id = str(uuid.uuid1())
for trait in k_dict[p]:
if trait['name'] == 'payload':
t_dict = eval(trait['value'])
for s, t in t_dict.items():
# FIXME(madhumohan): Dictionary items within the payload
# are handled as additional fields in the payload
# table. Need a better way to handle
# dictionaries or other structures within payload
# Nested dictionaries in the payload are skipped
# Lists within the dictionaries are also ignored
if type(t) == type(dict()):
for n, m in t.items():
if type(m) != type(dict()) and \
type(m) != type(list()):
# FIXME(madhumohan): Dirty workaround. A cleaner
# approach is required to handled None object in
# the data
if m is None:
m = 'None'
row_trait_tuple = \
(trait_id, n, m)
t_trait_list.append(
row_trait_tuple)
else:
if type(t) != type(list()):
# FIXME(madhumohan): Dirty workaround. A cleaner
# approach is required to handled None object in
# the data
if t is None:
t = 'None'
row_trait_tuple = (trait_id, s, t)
t_trait_list.append(row_trait_tuple)
row[key_to_index['traits']] = trait_id
else:
if p in key_to_index:
row[key_to_index[p]] = value_to_congress(v)
else:
LOG.info("Ignoring unexpected dict key %s", p)
t_list.append(tuple(row))
LOG.debug("Generated event list %s", t_list)
LOG.debug("Generated trait list %s", t_trait_list)
self.events = t_list
self.event_traits = t_trait_list
def _extract_value(cls, obj, extract_fn):
# Reads a VALUE object and returns (result_rows, h)
if extract_fn is None:
extract_fn = lambda x: x
return utils.value_to_congress(extract_fn(obj))
def convert_obj(cls, obj, translator, seen_tables=None,
parent_row_dict=None):
"""Takes an object and a translation descriptor. Returns two items:
(1) a list of tuples where the first element is the name of a table,
and the second element is a tuple to be inserted into the table, and
(2) if the translator specified an id-col, then return the id's value
here. The id is a hash that takes into account all the content of the
list of tuples. The hash can be used as a unique key to identify the
content in obj. Otherwise, return None here.
"""
def get_value(o, field, selector):
if selector == cls.DOT_SELECTOR:
if hasattr(o, field):
return getattr(o, field)
return None
elif selector == cls.DICT_SELECTOR:
if field in o:
return o[field]
return None
else:
raise AssertionError("Unexpected selector type: %s" %
(str(selector),))
def compute_hash(obj):
# This might turn out to be expensive.
# The datasource can contain a list which has an order, but
# Congress uses unordered sets internally for its tables which
# throw away a list's order. Since Congress throws away the
# order, this hash function needs to reimpose an order (by
# sorting) to ensure that two invocations of the hash function
# will always return the same result.
s = json.dumps(sorted(obj), sort_keys=True)
h = hashlib.md5(s).hexdigest()
return h
def extract_value(obj, extract_fn):
# Reads a VALUE object and returns (result_rows, h)
if extract_fn is None:
extract_fn = lambda x: x
return value_to_congress(extract_fn(obj))
if obj is None:
return None, None
if seen_tables is None:
seen_tables = []
cls.check_translation_type(translator.keys())
translation_type = translator[cls.TRANSLATION_TYPE]
if translation_type == cls.HDICT:
cls.check_params(translator.keys(), cls.HDICT_PARAMS)
table = translator[cls.TABLE_NAME]
parent_key = translator.get(cls.PARENT_KEY, None)
id_col = translator.get(cls.ID_COL, None)
selector = translator[cls.SELECTOR_TYPE]
field_translators = translator[cls.FIELD_TRANSLATORS]
if parent_key and id_col:
raise InvalidParamException(
'Specify at most one of %s or %s' %
(cls.PARENT_KEY, cls.ID_COL))
if table in seen_tables:
raise InvalidParamException('table (%s) used twice' % table)
new_results = [] # New tuples from this HDICT and sub containers.
hdict_row = {} # The content of the HDICT's new row.
def compare_subtranslator(x, y):
if cls.PARENT_KEY not in x[cls.TRANSLATOR] \
and cls.PARENT_KEY in y[cls.TRANSLATOR]:
return -1
elif cls.PARENT_KEY in x[cls.TRANSLATOR] \
and cls.PARENT_KEY not in y[cls.TRANSLATOR]:
return 1
else:
return cmp(x, y)
# Sort with fields lacking parent-key coming first so that the
# subtranslators that need a parent field will be able to get them
# from hdict_row.
sorted_translators = sorted(field_translators,
cmp=compare_subtranslator)
for field_translator in sorted_translators:
cls.check_params(field_translator.keys(),
cls.FIELD_TRANSLATOR_PARAMS)
field = field_translator[cls.FIELDNAME]
col_name = field_translator.get(cls.COL, field)
subtranslator = field_translator[cls.TRANSLATOR]
cls.check_translation_type(subtranslator.keys())
if subtranslator[cls.TRANSLATION_TYPE] == cls.VALUE:
cls.check_params(subtranslator.keys(), cls.VALUE_PARAMS)
extract_fn = subtranslator.get(cls.EXTRACT_FN, None)
v = extract_value(get_value(obj, field, selector),
extract_fn)
hdict_row[col_name] = v
else:
assert subtranslator[cls.TRANSLATION_TYPE] in (cls.HDICT,
cls.VDICT,
cls.LIST)
tuples = []
row_hash = None
v = get_value(obj, field, selector)
if v is None:
if cls.ID_COL in subtranslator:
row_hash = compute_hash([])
else:
next_seen_tables = seen_tables + [table]
tuples, row_hash = cls.convert_obj(v, subtranslator,
next_seen_tables,
hdict_row)
new_results.extend(tuples)
if cls.need_column_for_subtable_id(subtranslator):
hdict_row[col_name] = row_hash
new_row = []
for fieldtranslator in field_translators:
col = fieldtranslator.get(cls.COL,
fieldtranslator[cls.FIELDNAME])
if col in hdict_row:
new_row.append(value_to_congress(hdict_row[col]))
if id_col:
h = compute_hash(new_row)
new_row = (h,) + tuple(new_row)
elif parent_key:
h = None
new_row = (parent_row_dict[parent_key],) + tuple(new_row)
else:
h = None
new_row = tuple(new_row)
new_results.append((table, new_row))
return new_results, h
elif translation_type == cls.VDICT:
cls.check_params(translator.keys(), cls.VDICT_PARAMS)
table = translator[cls.TABLE_NAME]
parent_key = translator.get(cls.PARENT_KEY, None)
id_col = translator.get(cls.ID_COL, None)
key_col = translator[cls.KEY_COL]
subtrans = translator[cls.TRANSLATOR]
if parent_key and id_col:
raise InvalidParamException(
'Specify at most one of %s or %s' %
(cls.PARENT_KEY, cls.ID_COL))
if table in seen_tables:
raise InvalidParamException('table (%s) used twice' % table)
cls.check_translation_type(subtrans.keys())
if subtrans[cls.TRANSLATION_TYPE] == cls.VALUE:
cls.check_params(subtrans.keys(), cls.VALUE_PARAMS)
extract_fn = subtrans.get(cls.EXTRACT_FN, None)
converted_items = tuple([(value_to_congress(k),
extract_value(v, extract_fn))
for k, v in obj.items()])
if id_col:
h = compute_hash(converted_items)
new_tuples = [(table, (h,) + i) for i in converted_items]
elif parent_key:
h = None
parent_key_value = parent_row_dict[parent_key]
new_tuples = [(table, (parent_key_value,) + i)
for i in converted_items]
else:
h = None
new_tuples = [(table, i) for i in converted_items]
return new_tuples, h
else:
assert translator[cls.TRANSLATION_TYPE] in (cls.HDICT,
cls.VDICT,
cls.LIST)
new_tuples = []
vdict_rows = []
for k, v in obj.items():
if v is None:
tuples = []
row_hash = []
if cls.ID_COL in subtrans:
row_hash = compute_hash([])
else:
next_seen_tables = seen_tables + [table]
tuples, row_hash = cls.convert_obj(v, subtrans,
next_seen_tables,
{key_col: k})
if tuples:
new_tuples.extend(tuples)
vdict_row = (k,)
if cls.need_column_for_subtable_id(subtrans):
vdict_row = vdict_row + (row_hash,)
vdict_rows.append(vdict_row)
h = None
if id_col:
h = compute_hash(vdict_rows)
for vdict_row in vdict_rows:
if id_col:
new_tuples.append((table, (h,) + vdict_row))
elif parent_key:
k = parent_row_dict[parent_key]
new_tuples.append((table, (k,) + vdict_row))
else:
new_tuples.append((table, vdict_row))
return new_tuples, h
elif translation_type == cls.LIST:
cls.check_params(translator.keys(), cls.LIST_PARAMS)
table = translator[cls.TABLE_NAME]
parent_key = translator.get(cls.PARENT_KEY, None)
id_col = translator.get(cls.ID_COL, None)
subtrans = translator[cls.TRANSLATOR]
if parent_key and id_col:
raise InvalidParamException(
'Specify at most one of %s or %s' %
(cls.PARENT_KEY, cls.ID_COL))
if table in seen_tables:
raise InvalidParamException('table (%s) used twice' % table)
cls.check_translation_type(subtrans.keys())
if subtrans[cls.TRANSLATION_TYPE] == cls.VALUE:
cls.check_params(subtrans.keys(), cls.VALUE_PARAMS)
extract_fn = subtrans.get(cls.EXTRACT_FN, None)
converted_values = tuple([extract_value(o, extract_fn)
for o in obj])
if id_col:
h = compute_hash(converted_values)
new_tuples = [(table, (h, v)) for v in converted_values]
elif parent_key:
h = None
parent_key_value = parent_row_dict[parent_key]
new_tuples = [(table, (parent_key_value, v))
for v in converted_values]
else:
h = None
new_tuples = [(table, (v,)) for v in converted_values]
return new_tuples, h
else:
assert translator[cls.TRANSLATION_TYPE] in (cls.HDICT,
cls.VDICT,
cls.LIST)
new_tuples = []
row_hashes = []
for o in obj:
if o is None:
tuples = []
row_hash = []
if cls.ID_COL in subtrans:
row_hash = compute_hash([])
else:
next_seen_tables = seen_tables + [table]
tuples, row_hash = cls.convert_obj(o, subtrans,
next_seen_tables)
assert row_hash, "LIST's subtranslator must have row_hash"
assert cls.need_column_for_subtable_id(subtrans), \
"LIST's subtranslator should have id"
if tuples:
new_tuples.extend(tuples)
row_hashes.append(row_hash)
if id_col:
h = compute_hash(row_hashes)
else:
h = None
for row_hash in row_hashes:
if id_col:
new_tuples.append((table, (h, row_hash)))
elif parent_key:
new_tuples.append((table, (parent_row_dict[parent_key],
row_hash)))
else:
new_tuples.append((table, (row_hash,)))
return new_tuples, h
else:
raise AssertionError("unexpected translator type %s" %
translation_type)
