def iteritems(self): definitions = type(self).configuration_setting_definitions version = self.command.protocol_version return ifilter( lambda name_value1: name_value1[1] is not None, imap( lambda setting: (setting.name, setting.__get__(self)), ifilter( lambda setting: setting.is_supported_by_protocol(version), definitions)))
def iteritems(self): definitions = type(self).configuration_setting_definitions version = self.command.protocol_version return ifilter( lambda name_value1: name_value1[1] is not None, imap( lambda setting: (setting.name, setting.__get__(self)), ifilter( lambda setting: setting.is_supported_by_protocol(version), definitions)))
def iteritems(self): iteritems = SearchCommand.ConfigurationSettings.iteritems(self) version = self.command.protocol_version if version == 2: iteritems = ifilter(lambda name_value1: name_value1[0] != 'distributed', iteritems) if not self.distributed and self.type == 'streaming': iteritems = imap( lambda name_value: (name_value[0], 'stateful') if name_value[0] == 'type' else (name_value[0], name_value[1]), iteritems) return iteritems
def iteritems(self): iteritems = SearchCommand.ConfigurationSettings.iteritems(self) version = self.command.protocol_version if version == 2: iteritems = ifilter(lambda name_value1: name_value1[0] != 'distributed', iteritems) if not self.distributed and self.type == 'streaming': iteritems = imap( lambda name_value: (name_value[0], 'stateful') if name_value[0] == 'type' else (name_value[0], name_value[1]), iteritems) return iteritems
def validate_configuration_setting(specification, name, value): if not isinstance(value, specification.type): if isinstance(specification.type, type): type_names = specification.type.__name__ else: type_names = ', '.join(imap(lambda t: t.__name__, specification.type)) raise ValueError('Expected {} value, not {}={}'.format(type_names, name, repr(value))) if specification.constraint and not specification.constraint(value): raise ValueError('Illegal value: {}={}'.format(name, repr(value))) return value
def validate_configuration_setting(specification, name, value): if not isinstance(value, specification.type): if isinstance(specification.type, type): type_names = specification.type.__name__ else: type_names = ', '.join(imap(lambda t: t.__name__, specification.type)) raise ValueError('Expected {} value, not {}={}'.format(type_names, name, repr(value))) if specification.constraint and not specification.constraint(value): raise ValueError('Illegal value: {}={}'.format(name, repr(value))) return value
def iteritems(self): iteritems = SearchCommand.ConfigurationSettings.iteritems(self) version = self.command.protocol_version if version == 1: if self.required_fields is None: iteritems = ifilter(lambda name_value: name_value[0] != 'clear_required_fields', iteritems) else: iteritems = ifilter(lambda name_value2: name_value2[0] != 'distributed', iteritems) if not self.distributed: iteritems = imap( lambda name_value1: (name_value1[0], 'stateful') if name_value1[0] == 'type' else (name_value1[0], name_value1[1]), iteritems) return iteritems
def iteritems(self): iteritems = SearchCommand.ConfigurationSettings.iteritems(self) version = self.command.protocol_version if version == 1: if self.required_fields is None: iteritems = ifilter(lambda name_value: name_value[0] != 'clear_required_fields', iteritems) else: iteritems = ifilter(lambda name_value2: name_value2[0] != 'distributed', iteritems) if not self.distributed: iteritems = imap( lambda name_value1: (name_value1[0], 'stateful') if name_value1[0] == 'type' else (name_value1[0], name_value1[1]), iteritems) return iteritems
def __repr__(self): """ Converts the value of this instance to its string representation. The value of this ConfigurationSettings instance is represented as a string of comma-separated :code:`(name, value)` pairs. :return: String representation of this instance """ definitions = type(self).configuration_setting_definitions settings = imap( lambda setting: repr((setting.name, setting.__get__(self), setting.supporting_protocols)), definitions) return '[' + ', '.join(settings) + ']'
def __iter__(self): basedir = self._basedir name = self._name iterator = imap( lambda directory: Recording(os.path.join(basedir, directory, name) ), ifilter( lambda filename: os.path.isdir(os.path.join(basedir, filename) ), os.listdir(basedir))) return iterator
def __repr__(self): """ Converts the value of this instance to its string representation. The value of this ConfigurationSettings instance is represented as a string of comma-separated :code:`(name, value)` pairs. :return: String representation of this instance """ definitions = type(self).configuration_setting_definitions settings = imap( lambda setting: repr((setting.name, setting.__get__(self), setting.supporting_protocols)), definitions) return '[' + ', '.join(settings) + ']'
def search_results_info(self): """ Returns the search results info for this command invocation. The search results info object is created from the search results info file associated with the command invocation. :return: Search results info:const:`None`, if the search results info file associated with the command invocation is inaccessible. :rtype: SearchResultsInfo or NoneType """ if self._search_results_info is not None: return self._search_results_info if self._protocol_version == 1: try: path = self._input_header['infoPath'] except KeyError: return None else: assert self._protocol_version == 2 try: dispatch_dir = self._metadata.searchinfo.dispatch_dir except AttributeError: return None path = os.path.join(dispatch_dir, 'info.csv') try: with io.open(path, 'r') as f: reader = csv.reader(f, dialect=CsvDialect) fields = next(reader) values = next(reader) except IOError as error: if error.errno == 2: self.logger.error( 'Search results info file {} does not exist.'.format( json_encode_string(path))) return raise def convert_field(field): return (field[1:] if field[0] == '_' else field).replace('.', '_') decode = MetadataDecoder().decode def convert_value(value): try: return decode(value) if len(value) > 0 else value except ValueError: return value info = ObjectView( dict( imap( lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])), izip(fields, values)))) try: count_map = info.countMap except AttributeError: pass else: count_map = count_map.split(';') n = len(count_map) info.countMap = dict( izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2))) try: msg_type = info.msgType msg_text = info.msg except AttributeError: pass else: messages = ifilter( lambda t_m: t_m[0] or t_m[1], izip(msg_type.split('\n'), msg_text.split('\n'))) info.msg = [Message(message) for message in messages] del info.msgType try: info.vix_families = ElementTree.fromstring(info.vix_families) except AttributeError: pass self._search_results_info = info return info
def test_record_writer_with_random_data(self, save_recording=False): # Confirmed: [minint, maxint) covers the full range of values that xrange allows # RecordWriter writes apps in units of maxresultrows records. Default: 50,0000. # Partial results are written when the record count reaches maxresultrows. writer = RecordWriterV2(StringIO(), maxresultrows=10) # small for the purposes of this unit test test_data = OrderedDict() fieldnames = ['_serial', '_time', 'random_bytes', 'random_dict', 'random_integers', 'random_unicode'] test_data['fieldnames'] = fieldnames test_data['values'] = [] write_record = writer.write_record for serial_number in range(0, 31): values = [serial_number, time(), random_bytes(), random_dict(), random_integers(), random_unicode()] record = OrderedDict(izip(fieldnames, values)) #try: write_record(record) #except Exception as error: # self.fail(error) test_data['values'].append(values) # RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result # is produced or we're finished messages = [ ('debug', random_unicode()), ('error', random_unicode()), ('fatal', random_unicode()), ('info', random_unicode()), ('warn', random_unicode())] test_data['messages'] = messages for message_type, message_text in messages: writer.write_message(message_type, '{}', message_text) metrics = { 'metric-1': SearchMetric(1, 2, 3, 4), 'metric-2': SearchMetric(5, 6, 7, 8) } test_data['metrics'] = metrics for name, metric in six.iteritems(metrics): writer.write_metric(name, metric) self.assertEqual(writer._chunk_count, 3) self.assertEqual(writer._record_count, 1) self.assertGreater(writer._buffer.tell(), 0) self.assertEqual(writer._total_record_count, 30) self.assertListEqual(writer._fieldnames, fieldnames) self.assertListEqual(writer._inspector['messages'], messages) self.assertDictEqual( dict(ifilter(lambda k_v: k_v[0].startswith('metric.'), six.iteritems(writer._inspector))), dict(imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]), six.iteritems(metrics)))) writer.flush(finished=True) self.assertEqual(writer._chunk_count, 4) self.assertEqual(writer._record_count, 0) self.assertEqual(writer._buffer.tell(), 0) self.assertEqual(writer._buffer.getvalue(), '') self.assertEqual(writer._total_record_count, 31) self.assertRaises(AssertionError, writer.flush, finished=True, partial=True) self.assertRaises(AssertionError, writer.flush, finished='non-boolean') self.assertRaises(AssertionError, writer.flush, partial='non-boolean') self.assertRaises(AssertionError, writer.flush) self.assertRaises(RuntimeError, writer.write_record, {}) self.assertFalse(writer._ofile.closed) self.assertIsNone(writer._fieldnames) self.assertDictEqual(writer._inspector, OrderedDict()) # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling # RecordWriter.flush(partial=True). # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling # RecordWriter.flush(finish=True). if save_recording: cls = self.__class__ method = cls.test_record_writer_with_recordings base_path = os.path.join(self._recordings_path, '.'.join((cls.__name__, method.__name__, six.text_type(time())))) with gzip.open(base_path + '.input.gz', 'wb') as f: pickle.dump(test_data, f) with open(base_path + '.output', 'wb') as f: f.write(writer._ofile.getvalue()) return
def random_unicode(): return ''.join( imap(lambda x: six.unichr(x), random.sample(range(maxunicode), random.randint(0, max_length))))
def test_record_writer_with_random_data(self, save_recording=False): # Confirmed: [minint, maxint) covers the full range of values that xrange allows # RecordWriter writes apps in units of maxresultrows records. Default: 50,0000. # Partial results are written when the record count reaches maxresultrows. writer = RecordWriterV2( StringIO(), maxresultrows=10) # small for the purposes of this unit test test_data = OrderedDict() fieldnames = [ '_serial', '_time', 'random_bytes', 'random_dict', 'random_integers', 'random_unicode' ] test_data['fieldnames'] = fieldnames test_data['values'] = [] write_record = writer.write_record for serial_number in range(0, 31): values = [ serial_number, time(), random_bytes(), random_dict(), random_integers(), random_unicode() ] record = OrderedDict(izip(fieldnames, values)) #try: write_record(record) #except Exception as error: # self.fail(error) test_data['values'].append(values) # RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result # is produced or we're finished messages = [('debug', random_unicode()), ('error', random_unicode()), ('fatal', random_unicode()), ('info', random_unicode()), ('warn', random_unicode())] test_data['messages'] = messages for message_type, message_text in messages: writer.write_message(message_type, '{}', message_text) metrics = { 'metric-1': SearchMetric(1, 2, 3, 4), 'metric-2': SearchMetric(5, 6, 7, 8) } test_data['metrics'] = metrics for name, metric in six.iteritems(metrics): writer.write_metric(name, metric) self.assertEqual(writer._chunk_count, 3) self.assertEqual(writer._record_count, 1) self.assertGreater(writer._buffer.tell(), 0) self.assertEqual(writer._total_record_count, 30) self.assertListEqual(writer._fieldnames, fieldnames) self.assertListEqual(writer._inspector['messages'], messages) self.assertDictEqual( dict( ifilter(lambda k_v: k_v[0].startswith('metric.'), six.iteritems(writer._inspector))), dict( imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]), six.iteritems(metrics)))) writer.flush(finished=True) self.assertEqual(writer._chunk_count, 4) self.assertEqual(writer._record_count, 0) self.assertEqual(writer._buffer.tell(), 0) self.assertEqual(writer._buffer.getvalue(), '') self.assertEqual(writer._total_record_count, 31) self.assertRaises(AssertionError, writer.flush, finished=True, partial=True) self.assertRaises(AssertionError, writer.flush, finished='non-boolean') self.assertRaises(AssertionError, writer.flush, partial='non-boolean') self.assertRaises(AssertionError, writer.flush) self.assertRaises(RuntimeError, writer.write_record, {}) self.assertFalse(writer._ofile.closed) self.assertIsNone(writer._fieldnames) self.assertDictEqual(writer._inspector, OrderedDict()) # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling # RecordWriter.flush(partial=True). # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling # RecordWriter.flush(finish=True). if save_recording: cls = self.__class__ method = cls.test_record_writer_with_recordings base_path = os.path.join( self._recordings_path, '.'.join( (cls.__name__, method.__name__, six.text_type(time())))) with gzip.open(base_path + '.input.gz', 'wb') as f: pickle.dump(test_data, f) with open(base_path + '.output', 'wb') as f: f.write(writer._ofile.getvalue()) return
def _write_record(self, record): fieldnames = self._fieldnames if fieldnames is None: self._fieldnames = fieldnames = list(record.keys()) value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames) self._writerow(list(chain.from_iterable(value_list))) get_value = record.get values = [] for fieldname in fieldnames: value = get_value(fieldname, None) if value is None: values += (None, None) continue value_t = type(value) if issubclass(value_t, (list, tuple)): if len(value) == 0: values += (None, None) continue if len(value) > 1: value_list = value sv = '' mv = '$' for value in value_list: if value is None: sv += '\n' mv += '$;$' continue value_t = type(value) if value_t is not bytes: if value_t is bool: value = str(value.real) elif value_t is six.text_type: value = value elif isinstance( value, six.integer_types ) or value_t is float or value_t is complex: value = str(value) elif issubclass(value_t, (dict, list, tuple)): value = str(''.join( RecordWriter._iterencode_json(value, 0))) else: value = repr(value).encode( 'utf-8', errors='backslashreplace') sv += value + '\n' mv += value.replace('$', '$$') + '$;$' values += (sv[:-1], mv[:-2]) continue value = value[0] value_t = type(value) if value_t is bool: values += (str(value.real), None) continue if value_t is bytes: values += (value, None) continue if value_t is six.text_type: if six.PY2: value = value.encode('utf-8') values += (value, None) continue if isinstance(value, six.integer_types ) or value_t is float or value_t is complex: values += (str(value), None) continue if issubclass(value_t, dict): values += (str(''.join(RecordWriter._iterencode_json(value, 0))), None) continue values += (repr(value), None) self._writerow(values) self._pending_record_count += 1 if self.pending_record_count >= self._maxresultrows: self.flush(partial=True)
def random_unicode(): return ''.join(imap(lambda x: six.unichr(x), random.sample(range(maxunicode), random.randint(0, max_length))))
def fix_up(cls, values): is_configuration_setting = lambda attribute: isinstance(attribute, ConfigurationSetting) definitions = getmembers(cls, is_configuration_setting) i = 0 for name, setting in definitions: if setting._name is None: setting._name = name = six.text_type(name) else: name = setting._name validate, specification = setting._get_specification() backing_field_name = '_' + name if setting.fget is None and setting.fset is None and setting.fdel is None: value = setting._value if setting._readonly or value is not None: validate(specification, name, value) def fget(bfn, value): return lambda this: getattr(this, bfn, value) setting = setting.getter(fget(backing_field_name, value)) if not setting._readonly: def fset(bfn, validate, specification, name): return lambda this, value: setattr(this, bfn, validate(specification, name, value)) setting = setting.setter(fset(backing_field_name, validate, specification, name)) setattr(cls, name, setting) def is_supported_by_protocol(supporting_protocols): def is_supported_by_protocol(version): return version in supporting_protocols return is_supported_by_protocol del setting._name, setting._value, setting._readonly setting.is_supported_by_protocol = is_supported_by_protocol(specification.supporting_protocols) setting.supporting_protocols = specification.supporting_protocols setting.backing_field_name = backing_field_name definitions[i] = setting setting.name = name i += 1 try: value = values[name] except KeyError: continue if setting.fset is None: raise ValueError('The value of configuration setting {} is fixed'.format(name)) setattr(cls, backing_field_name, validate(specification, name, value)) del values[name] if len(values) > 0: settings = sorted(list(six.iteritems(values))) settings = imap(lambda n_v: '{}={}'.format(n_v[0], repr(n_v[1])), settings) raise AttributeError('Inapplicable configuration settings: ' + ', '.join(settings)) cls.configuration_setting_definitions = definitions
def __repr__(self): text = 'Option.View([' + ','.join(imap(lambda item: repr(item), six.itervalues(self))) + '])' return text
def random_unicode(): return ''.join( imap( lambda x: six.unichr(x), random.sample(range(MAX_NARROW_UNICODE), random.randint(0, max_length))))
def iteritems(self): iteritems = SearchCommand.ConfigurationSettings.iteritems(self) return imap( lambda name_value: (name_value[0], 'events' if name_value[0] == 'type' else name_value[1]), iteritems)
def iteritems(self): iteritems = SearchCommand.ConfigurationSettings.iteritems(self) return imap(lambda name_value: (name_value[0], 'events' if name_value[0] == 'type' else name_value[1]), iteritems)
def fix_up(cls, values): is_configuration_setting = lambda attribute: isinstance(attribute, ConfigurationSetting) definitions = getmembers(cls, is_configuration_setting) i = 0 for name, setting in definitions: if setting._name is None: setting._name = name = six.text_type(name) else: name = setting._name validate, specification = setting._get_specification() backing_field_name = '_' + name if setting.fget is None and setting.fset is None and setting.fdel is None: value = setting._value if setting._readonly or value is not None: validate(specification, name, value) def fget(bfn, value): return lambda this: getattr(this, bfn, value) setting = setting.getter(fget(backing_field_name, value)) if not setting._readonly: def fset(bfn, validate, specification, name): return lambda this, value: setattr(this, bfn, validate(specification, name, value)) setting = setting.setter(fset(backing_field_name, validate, specification, name)) setattr(cls, name, setting) def is_supported_by_protocol(supporting_protocols): def is_supported_by_protocol(version): return version in supporting_protocols return is_supported_by_protocol del setting._name, setting._value, setting._readonly setting.is_supported_by_protocol = is_supported_by_protocol(specification.supporting_protocols) setting.supporting_protocols = specification.supporting_protocols setting.backing_field_name = backing_field_name definitions[i] = setting setting.name = name i += 1 try: value = values[name] except KeyError: continue if setting.fset is None: raise ValueError('The value of configuration setting {} is fixed'.format(name)) setattr(cls, backing_field_name, validate(specification, name, value)) del values[name] if len(values) > 0: settings = sorted(list(six.iteritems(values))) settings = imap(lambda n_v: '{}={}'.format(n_v[0], repr(n_v[1])), settings) raise AttributeError('Inapplicable configuration settings: ' + ', '.join(settings)) cls.configuration_setting_definitions = definitions
def __repr__(self): text = 'Option.View([' + ','.join(imap(lambda item: repr(item), six.itervalues(self))) + '])' return text
def _write_record(self, record): fieldnames = self._fieldnames if fieldnames is None: self._fieldnames = fieldnames = list(record.keys()) value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames) self._writerow(list(chain.from_iterable(value_list))) get_value = record.get values = [] for fieldname in fieldnames: value = get_value(fieldname, None) if value is None: values += (None, None) continue value_t = type(value) if issubclass(value_t, (list, tuple)): if len(value) == 0: values += (None, None) continue if len(value) > 1: value_list = value sv = '' mv = '$' for value in value_list: if value is None: sv += '\n' mv += '$;$' continue value_t = type(value) if value_t is not bytes: if value_t is bool: value = str(value.real) elif value_t is six.text_type: value = value elif value_t is int or value_t is int or value_t is float or value_t is complex: value = str(value) elif issubclass(value_t, (dict, list, tuple)): value = str(''.join(RecordWriter._iterencode_json(value, 0))) else: value = repr(value).encode('utf-8', errors='backslashreplace') sv += value + '\n' mv += value.replace('$', '$$') + '$;$' values += (sv[:-1], mv[:-2]) continue value = value[0] value_t = type(value) if value_t is bool: values += (str(value.real), None) continue if value_t is bytes: values += (value, None) continue if value_t is six.text_type: if six.PY2: value = value.encode('utf-8') values += (value, None) continue if value_t is int or value_t is int or value_t is float or value_t is complex: values += (str(value), None) continue if issubclass(value_t, dict): values += (str(''.join(RecordWriter._iterencode_json(value, 0))), None) continue values += (repr(value), None) self._writerow(values) self._record_count += 1 if self._record_count >= self._maxresultrows: self.flush(partial=True)
def search_results_info(self): """ Returns the search results info for this command invocation. The search results info object is created from the search results info file associated with the command invocation. :return: Search results info:const:`None`, if the search results info file associated with the command invocation is inaccessible. :rtype: SearchResultsInfo or NoneType """ if self._search_results_info is not None: return self._search_results_info if self._protocol_version == 1: try: path = self._input_header['infoPath'] except KeyError: return None else: assert self._protocol_version == 2 try: dispatch_dir = self._metadata.searchinfo.dispatch_dir except AttributeError: return None path = os.path.join(dispatch_dir, 'info.csv') try: with io.open(path, 'r') as f: reader = csv.reader(f, dialect=CsvDialect) fields = next(reader) values = next(reader) except IOError as error: if error.errno == 2: self.logger.error('Search results info file {} does not exist.'.format(json_encode_string(path))) return raise def convert_field(field): return (field[1:] if field[0] == '_' else field).replace('.', '_') decode = MetadataDecoder().decode def convert_value(value): try: return decode(value) if len(value) > 0 else value except ValueError: return value info = ObjectView(dict(imap(lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])), izip(fields, values)))) try: count_map = info.countMap except AttributeError: pass else: count_map = count_map.split(';') n = len(count_map) info.countMap = dict(izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2))) try: msg_type = info.msgType msg_text = info.msg except AttributeError: pass else: messages = ifilter(lambda t_m: t_m[0] or t_m[1], izip(msg_type.split('\n'), msg_text.split('\n'))) info.msg = [Message(message) for message in messages] del info.msgType try: info.vix_families = ElementTree.fromstring(info.vix_families) except AttributeError: pass self._search_results_info = info return info