def test_get_events(self): data = self.get_json(self.PATH, headers=headers) self.assertEqual(len(data), 3) # We expect to get native UTC generated time back expected_generated = timeutils.strtime( at=timeutils.normalize_time(self.trait_time), fmt=timeutils._ISO8601_TIME_FORMAT) for event in data: self.assertTrue(event['event_type'] in ['Foo', 'Bar', 'Zoo']) self.assertEqual(4, len(event['traits'])) self.assertEqual(event['generated'], expected_generated) for trait_name in ['trait_A', 'trait_B', 'trait_C', 'trait_D']: self.assertTrue(trait_name in event['traits'])
def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter """ project_table = self.conn.table(self.PROJECT_TABLE) user_table = self.conn.table(self.USER_TABLE) resource_table = self.conn.table(self.RESOURCE_TABLE) meter_table = self.conn.table(self.METER_TABLE) # Make sure we know about the user and project if data['user_id']: user = user_table.row(data['user_id']) sources = _load_hbase_list(user, 's') # Update if source is new if data['source'] not in sources: user['f:s_%s' % data['source']] = "1" user_table.put(data['user_id'], user) project = project_table.row(data['project_id']) sources = _load_hbase_list(project, 's') # Update if source is new if data['source'] not in sources: project['f:s_%s' % data['source']] = "1" project_table.put(data['project_id'], project) rts = reverse_timestamp(data['timestamp']) resource = resource_table.row(data['resource_id']) new_meter = "%s!%s!%s" % (data['counter_name'], data['counter_type'], data['counter_unit']) new_resource = { 'f:resource_id': data['resource_id'], 'f:project_id': data['project_id'], 'f:user_id': data['user_id'], 'f:source': data["source"], # store meters with prefix "m_" 'f:m_%s' % new_meter: "1" } # store metadata fields with prefix "r_" if data['resource_metadata']: resource_metadata = dict( ('f:r_%s' % k, v) for (k, v) in data['resource_metadata'].iteritems()) new_resource.update(resource_metadata) # Update if resource has new information if new_resource != resource: meters = _load_hbase_list(resource, 'm') if new_meter not in meters: new_resource['f:m_%s' % new_meter] = "1" resource_table.put(data['resource_id'], new_resource) # Rowkey consists of reversed timestamp, meter and an md5 of # user+resource+project for purposes of uniqueness m = hashlib.md5() m.update("%s%s%s" % (data['user_id'], data['resource_id'], data['project_id'])) # We use reverse timestamps in rowkeys as they are sorted # alphabetically. row = "%s_%d_%s" % (data['counter_name'], rts, m.hexdigest()) # Convert timestamp to string as json.dumps won't ts = timeutils.strtime(data['timestamp']) record = { 'f:timestamp': ts, 'f:counter_name': data['counter_name'], 'f:counter_type': data['counter_type'], 'f:counter_volume': str(data['counter_volume']), 'f:counter_unit': data['counter_unit'], # TODO(shengjie) consider using QualifierFilter # keep dimensions as column qualifier for quicker look up # TODO(shengjie) extra dimensions need to be added as CQ 'f:user_id': data['user_id'], 'f:project_id': data['project_id'], 'f:resource_id': data['resource_id'], 'f:source': data['source'], # add in reversed_ts here for time range scan 'f:rts': str(rts) } # Don't want to be changing the original data object. data = copy.copy(data) data['timestamp'] = ts # Save original meter. record['f:message'] = json.dumps(data) meter_table.put(row, record)
def to_primitive(value, convert_instances=False, convert_datetime=True, level=0, max_depth=3): """Convert a complex object into primitives. Handy for JSON serialization. We can optionally handle instances, but since this is a recursive function, we could have cyclical data structures. To handle cyclical data structures we could track the actual objects visited in a set, but not all objects are hashable. Instead we just track the depth of the object inspections and don't go too deep. Therefore, convert_instances=True is lossy ... be aware. """ # handle obvious types first - order of basic types determined by running # full tests on nova project, resulting in the following counts: # 572754 <type 'NoneType'> # 460353 <type 'int'> # 379632 <type 'unicode'> # 274610 <type 'str'> # 199918 <type 'dict'> # 114200 <type 'datetime.datetime'> # 51817 <type 'bool'> # 26164 <type 'list'> # 6491 <type 'float'> # 283 <type 'tuple'> # 19 <type 'long'> if isinstance(value, _simple_types): return value if isinstance(value, datetime.datetime): if convert_datetime: return timeutils.strtime(value) else: return value # value of itertools.count doesn't get caught by nasty_type_tests # and results in infinite loop when list(value) is called. if type(value) == itertools.count: return six.text_type(value) # FIXME(vish): Workaround for LP bug 852095. Without this workaround, # tests that raise an exception in a mocked method that # has a @wrap_exception with a notifier will fail. If # we up the dependency to 0.5.4 (when it is released) we # can remove this workaround. if getattr(value, '__module__', None) == 'mox': return 'mock' if level > max_depth: return '?' # The try block may not be necessary after the class check above, # but just in case ... try: recursive = functools.partial(to_primitive, convert_instances=convert_instances, convert_datetime=convert_datetime, level=level, max_depth=max_depth) if isinstance(value, dict): return dict((k, recursive(v)) for k, v in value.iteritems()) elif isinstance(value, (list, tuple)): return [recursive(lv) for lv in value] # It's not clear why xmlrpclib created their own DateTime type, but # for our purposes, make it a datetime type which is explicitly # handled if isinstance(value, xmlrpclib.DateTime): value = datetime.datetime(*tuple(value.timetuple())[:6]) if convert_datetime and isinstance(value, datetime.datetime): return timeutils.strtime(value) elif hasattr(value, 'iteritems'): return recursive(dict(value.iteritems()), level=level + 1) elif hasattr(value, '__iter__'): return recursive(list(value)) elif convert_instances and hasattr(value, '__dict__'): # Likely an instance of something. Watch for cycles. # Ignore class member vars. return recursive(value.__dict__, level=level + 1) else: if any(test(value) for test in _nasty_type_tests): return six.text_type(value) return value except TypeError: # Class objects are tricky since they may define something like # __iter__ defined but it isn't callable as list(). return six.text_type(value)
def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter """ project_table = self.conn.table(self.PROJECT_TABLE) user_table = self.conn.table(self.USER_TABLE) resource_table = self.conn.table(self.RESOURCE_TABLE) meter_table = self.conn.table(self.METER_TABLE) # Make sure we know about the user and project if data['user_id']: user = user_table.row(data['user_id']) sources = _load_hbase_list(user, 's') # Update if source is new if data['source'] not in sources: user['f:s_%s' % data['source']] = "1" user_table.put(data['user_id'], user) project = project_table.row(data['project_id']) sources = _load_hbase_list(project, 's') # Update if source is new if data['source'] not in sources: project['f:s_%s' % data['source']] = "1" project_table.put(data['project_id'], project) rts = reverse_timestamp(data['timestamp']) resource = resource_table.row(data['resource_id']) new_meter = "%s!%s!%s" % ( data['counter_name'], data['counter_type'], data['counter_unit']) new_resource = {'f:resource_id': data['resource_id'], 'f:project_id': data['project_id'], 'f:user_id': data['user_id'], 'f:source': data["source"], # store meters with prefix "m_" 'f:m_%s' % new_meter: "1" } # store metadata fields with prefix "r_" if data['resource_metadata']: resource_metadata = dict(('f:r_%s' % k, v) for (k, v) in data['resource_metadata'].iteritems()) new_resource.update(resource_metadata) # Update if resource has new information if new_resource != resource: meters = _load_hbase_list(resource, 'm') if new_meter not in meters: new_resource['f:m_%s' % new_meter] = "1" resource_table.put(data['resource_id'], new_resource) # Rowkey consists of reversed timestamp, meter and an md5 of # user+resource+project for purposes of uniqueness m = hashlib.md5() m.update("%s%s%s" % (data['user_id'], data['resource_id'], data['project_id'])) # We use reverse timestamps in rowkeys as they are sorted # alphabetically. row = "%s_%d_%s" % (data['counter_name'], rts, m.hexdigest()) # Convert timestamp to string as json.dumps won't ts = timeutils.strtime(data['timestamp']) record = {'f:timestamp': ts, 'f:counter_name': data['counter_name'], 'f:counter_type': data['counter_type'], 'f:counter_volume': str(data['counter_volume']), 'f:counter_unit': data['counter_unit'], # TODO(shengjie) consider using QualifierFilter # keep dimensions as column qualifier for quicker look up # TODO(shengjie) extra dimensions need to be added as CQ 'f:user_id': data['user_id'], 'f:project_id': data['project_id'], 'f:resource_id': data['resource_id'], 'f:source': data['source'], # add in reversed_ts here for time range scan 'f:rts': str(rts) } # Don't want to be changing the original data object. data = copy.copy(data) data['timestamp'] = ts # Save original meter. record['f:message'] = json.dumps(data) meter_table.put(row, record)
def to_primitive(value, convert_instances=False, convert_datetime=True, level=0, max_depth=3): """Convert a complex object into primitives. Handy for JSON serialization. We can optionally handle instances, but since this is a recursive function, we could have cyclical data structures. To handle cyclical data structures we could track the actual objects visited in a set, but not all objects are hashable. Instead we just track the depth of the object inspections and don't go too deep. Therefore, convert_instances=True is lossy ... be aware. """ # handle obvious types first - order of basic types determined by running # full tests on nova project, resulting in the following counts: # 572754 <type 'NoneType'> # 460353 <type 'int'> # 379632 <type 'unicode'> # 274610 <type 'str'> # 199918 <type 'dict'> # 114200 <type 'datetime.datetime'> # 51817 <type 'bool'> # 26164 <type 'list'> # 6491 <type 'float'> # 283 <type 'tuple'> # 19 <type 'long'> if isinstance(value, _simple_types): return value if isinstance(value, datetime.datetime): if convert_datetime: return timeutils.strtime(value) else: return value # value of itertools.count doesn't get caught by nasty_type_tests # and results in infinite loop when list(value) is called. if type(value) == itertools.count: return six.text_type(value) # FIXME(vish): Workaround for LP bug 852095. Without this workaround, # tests that raise an exception in a mocked method that # has a @wrap_exception with a notifier will fail. If # we up the dependency to 0.5.4 (when it is released) we # can remove this workaround. if getattr(value, '__module__', None) == 'mox': return 'mock' if level > max_depth: return '?' # The try block may not be necessary after the class check above, # but just in case ... try: recursive = functools.partial(to_primitive, convert_instances=convert_instances, convert_datetime=convert_datetime, level=level, max_depth=max_depth) if isinstance(value, dict): return dict((k, recursive(v)) for k, v in value.iteritems()) elif isinstance(value, (list, tuple)): return [recursive(lv) for lv in value] # It's not clear why xmlrpclib created their own DateTime type, but # for our purposes, make it a datetime type which is explicitly # handled if xmlrpclib and isinstance(value, xmlrpclib.DateTime): value = datetime.datetime(*tuple(value.timetuple())[:6]) if convert_datetime and isinstance(value, datetime.datetime): return timeutils.strtime(value) elif isinstance(value, gettextutils.Message): return value.data elif hasattr(value, 'iteritems'): return recursive(dict(value.iteritems()), level=level + 1) elif hasattr(value, '__iter__'): return recursive(list(value)) elif convert_instances and hasattr(value, '__dict__'): # Likely an instance of something. Watch for cycles. # Ignore class member vars. return recursive(value.__dict__, level=level + 1) elif netaddr and isinstance(value, netaddr.IPAddress): return six.text_type(value) else: if any(test(value) for test in _nasty_type_tests): return six.text_type(value) return value except TypeError: # Class objects are tricky since they may define something like # __iter__ defined but it isn't callable as list(). return six.text_type(value)
def to_primitive(value, convert_instances=False, level=0): """Convert a complex object into primitives. Handy for JSON serialization. We can optionally handle instances, but since this is a recursive function, we could have cyclical data structures. To handle cyclical data structures we could track the actual objects visited in a set, but not all objects are hashable. Instead we just track the depth of the object inspections and don't go too deep. Therefore, convert_instances=True is lossy ... be aware. """ nasty = [ inspect.ismodule, inspect.isclass, inspect.ismethod, inspect.isfunction, inspect.isgeneratorfunction, inspect.isgenerator, inspect.istraceback, inspect.isframe, inspect.iscode, inspect.isbuiltin, inspect.isroutine, inspect.isabstract ] for test in nasty: if test(value): return unicode(value) # value of itertools.count doesn't get caught by inspects # above and results in infinite loop when list(value) is called. if type(value) == itertools.count: return unicode(value) # FIXME(vish): Workaround for LP bug 852095. Without this workaround, # tests that raise an exception in a mocked method that # has a @wrap_exception with a notifier will fail. If # we up the dependency to 0.5.4 (when it is released) we # can remove this workaround. if getattr(value, '__module__', None) == 'mox': return 'mock' if level > 3: return '?' # The try block may not be necessary after the class check above, # but just in case ... try: # It's not clear why xmlrpclib created their own DateTime type, but # for our purposes, make it a datetime type which is explicitly # handled if isinstance(value, xmlrpclib.DateTime): value = datetime.datetime(*tuple(value.timetuple())[:6]) if isinstance(value, (list, tuple)): o = [] for v in value: o.append( to_primitive(v, convert_instances=convert_instances, level=level)) return o elif isinstance(value, dict): o = {} for k, v in value.iteritems(): o[k] = to_primitive(v, convert_instances=convert_instances, level=level) return o elif isinstance(value, datetime.datetime): return timeutils.strtime(value) elif hasattr(value, 'iteritems'): return to_primitive(dict(value.iteritems()), convert_instances=convert_instances, level=level + 1) elif hasattr(value, '__iter__'): return to_primitive(list(value), convert_instances=convert_instances, level=level) elif convert_instances and hasattr(value, '__dict__'): # Likely an instance of something. Watch for cycles. # Ignore class member vars. return to_primitive(value.__dict__, convert_instances=convert_instances, level=level + 1) else: return value except TypeError, e: # Class objects are tricky since they may define something like # __iter__ defined but it isn't callable as list(). return unicode(value)
def to_primitive(value, convert_instances=False, level=0): """Convert a complex object into primitives. Handy for JSON serialization. We can optionally handle instances, but since this is a recursive function, we could have cyclical data structures. To handle cyclical data structures we could track the actual objects visited in a set, but not all objects are hashable. Instead we just track the depth of the object inspections and don't go too deep. Therefore, convert_instances=True is lossy ... be aware. """ nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod, inspect.isfunction, inspect.isgeneratorfunction, inspect.isgenerator, inspect.istraceback, inspect.isframe, inspect.iscode, inspect.isbuiltin, inspect.isroutine, inspect.isabstract] for test in nasty: if test(value): return unicode(value) # value of itertools.count doesn't get caught by inspects # above and results in infinite loop when list(value) is called. if type(value) == itertools.count: return unicode(value) # FIXME(vish): Workaround for LP bug 852095. Without this workaround, # tests that raise an exception in a mocked method that # has a @wrap_exception with a notifier will fail. If # we up the dependency to 0.5.4 (when it is released) we # can remove this workaround. if getattr(value, '__module__', None) == 'mox': return 'mock' if level > 3: return '?' # The try block may not be necessary after the class check above, # but just in case ... try: # It's not clear why xmlrpclib created their own DateTime type, but # for our purposes, make it a datetime type which is explicitly # handled if isinstance(value, xmlrpclib.DateTime): value = datetime.datetime(*tuple(value.timetuple())[:6]) if isinstance(value, (list, tuple)): o = [] for v in value: o.append(to_primitive(v, convert_instances=convert_instances, level=level)) return o elif isinstance(value, dict): o = {} for k, v in value.iteritems(): o[k] = to_primitive(v, convert_instances=convert_instances, level=level) return o elif isinstance(value, datetime.datetime): return timeutils.strtime(value) elif hasattr(value, 'iteritems'): return to_primitive(dict(value.iteritems()), convert_instances=convert_instances, level=level + 1) elif hasattr(value, '__iter__'): return to_primitive(list(value), convert_instances=convert_instances, level=level) elif convert_instances and hasattr(value, '__dict__'): # Likely an instance of something. Watch for cycles. # Ignore class member vars. return to_primitive(value.__dict__, convert_instances=convert_instances, level=level + 1) else: return value except TypeError, e: # Class objects are tricky since they may define something like # __iter__ defined but it isn't callable as list(). return unicode(value)