def iteritems(self): definitions = type(self).configuration_setting_definitions version = self.command.protocol_version return ifilter( lambda name_value1: name_value1[1] is not None, imap( lambda setting: (setting.name, setting.__get__(self)), ifilter( lambda setting: setting.is_supported_by_protocol(version), definitions)))
def iteritems(self): iteritems = SearchCommand.ConfigurationSettings.iteritems(self) version = self.command.protocol_version if version == 1: if self.required_fields is None: iteritems = ifilter(lambda name_value: name_value[0] != 'clear_required_fields', iteritems) else: iteritems = ifilter(lambda name_value2: name_value2[0] != 'distributed', iteritems) if not self.distributed: iteritems = imap( lambda name_value1: (name_value1[0], 'stateful') if name_value1[0] == 'type' else (name_value1[0], name_value1[1]), iteritems) return iteritems
def iteritems(self): iteritems = SearchCommand.ConfigurationSettings.iteritems(self) version = self.command.protocol_version if version == 2: iteritems = ifilter(lambda name_value1: name_value1[0] != 'distributed', iteritems) if not self.distributed and self.type == 'streaming': iteritems = imap( lambda name_value: (name_value[0], 'stateful') if name_value[0] == 'type' else (name_value[0], name_value[1]), iteritems) return iteritems
def __init__(self, name, action, phase, protocol_version): basedir = Recordings._prefix + six.text_type(protocol_version) if not os.path.isdir(basedir): raise ValueError( 'Directory "{}" containing recordings for protocol version {} does not exist' .format(protocol_version, basedir)) self._basedir = basedir self._name = '.'.join( ifilter(lambda part: part is not None, (name, action, phase)))
def __iter__(self): basedir = self._basedir name = self._name iterator = imap( lambda directory: Recording(os.path.join(basedir, directory, name) ), ifilter( lambda filename: os.path.isdir(os.path.join(basedir, filename) ), os.listdir(basedir))) return iterator
def search_results_info(self): """ Returns the search results info for this command invocation. The search results info object is created from the search results info file associated with the command invocation. :return: Search results info:const:`None`, if the search results info file associated with the command invocation is inaccessible. :rtype: SearchResultsInfo or NoneType """ if self._search_results_info is not None: return self._search_results_info if self._protocol_version == 1: try: path = self._input_header['infoPath'] except KeyError: return None else: assert self._protocol_version == 2 try: dispatch_dir = self._metadata.searchinfo.dispatch_dir except AttributeError: return None path = os.path.join(dispatch_dir, 'info.csv') try: with io.open(path, 'r') as f: reader = csv.reader(f, dialect=CsvDialect) fields = next(reader) values = next(reader) except IOError as error: if error.errno == 2: self.logger.error( 'Search results info file {} does not exist.'.format( json_encode_string(path))) return raise def convert_field(field): return (field[1:] if field[0] == '_' else field).replace('.', '_') decode = MetadataDecoder().decode def convert_value(value): try: return decode(value) if len(value) > 0 else value except ValueError: return value info = ObjectView( dict( imap( lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])), izip(fields, values)))) try: count_map = info.countMap except AttributeError: pass else: count_map = count_map.split(';') n = len(count_map) info.countMap = dict( izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2))) try: msg_type = info.msgType msg_text = info.msg except AttributeError: pass else: messages = ifilter( lambda t_m: t_m[0] or t_m[1], izip(msg_type.split('\n'), msg_text.split('\n'))) info.msg = [Message(message) for message in messages] del info.msgType try: info.vix_families = ElementTree.fromstring(info.vix_families) except AttributeError: pass self._search_results_info = info return info
def test_record_writer_with_random_data(self, save_recording=False): # Confirmed: [minint, maxint) covers the full range of values that xrange allows # RecordWriter writes apps in units of maxresultrows records. Default: 50,0000. # Partial results are written when the record count reaches maxresultrows. writer = RecordWriterV2( StringIO(), maxresultrows=10) # small for the purposes of this unit test test_data = OrderedDict() fieldnames = [ '_serial', '_time', 'random_bytes', 'random_dict', 'random_integers', 'random_unicode' ] test_data['fieldnames'] = fieldnames test_data['values'] = [] write_record = writer.write_record for serial_number in range(0, 31): values = [ serial_number, time(), random_bytes(), random_dict(), random_integers(), random_unicode() ] record = OrderedDict(izip(fieldnames, values)) #try: write_record(record) #except Exception as error: # self.fail(error) test_data['values'].append(values) # RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result # is produced or we're finished messages = [('debug', random_unicode()), ('error', random_unicode()), ('fatal', random_unicode()), ('info', random_unicode()), ('warn', random_unicode())] test_data['messages'] = messages for message_type, message_text in messages: writer.write_message(message_type, '{}', message_text) metrics = { 'metric-1': SearchMetric(1, 2, 3, 4), 'metric-2': SearchMetric(5, 6, 7, 8) } test_data['metrics'] = metrics for name, metric in six.iteritems(metrics): writer.write_metric(name, metric) self.assertEqual(writer._chunk_count, 3) self.assertEqual(writer._record_count, 1) self.assertGreater(writer._buffer.tell(), 0) self.assertEqual(writer._total_record_count, 30) self.assertListEqual(writer._fieldnames, fieldnames) self.assertListEqual(writer._inspector['messages'], messages) self.assertDictEqual( dict( ifilter(lambda k_v: k_v[0].startswith('metric.'), six.iteritems(writer._inspector))), dict( imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]), six.iteritems(metrics)))) writer.flush(finished=True) self.assertEqual(writer._chunk_count, 4) self.assertEqual(writer._record_count, 0) self.assertEqual(writer._buffer.tell(), 0) self.assertEqual(writer._buffer.getvalue(), '') self.assertEqual(writer._total_record_count, 31) self.assertRaises(AssertionError, writer.flush, finished=True, partial=True) self.assertRaises(AssertionError, writer.flush, finished='non-boolean') self.assertRaises(AssertionError, writer.flush, partial='non-boolean') self.assertRaises(AssertionError, writer.flush) self.assertRaises(RuntimeError, writer.write_record, {}) self.assertFalse(writer._ofile.closed) self.assertIsNone(writer._fieldnames) self.assertDictEqual(writer._inspector, OrderedDict()) # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling # RecordWriter.flush(partial=True). # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling # RecordWriter.flush(finish=True). if save_recording: cls = self.__class__ method = cls.test_record_writer_with_recordings base_path = os.path.join( self._recordings_path, '.'.join( (cls.__name__, method.__name__, six.text_type(time())))) with gzip.open(base_path + '.input.gz', 'wb') as f: pickle.dump(test_data, f) with open(base_path + '.output', 'wb') as f: f.write(writer._ofile.getvalue()) return
def search_results_info(self): """ Returns the search results info for this command invocation. The search results info object is created from the search results info file associated with the command invocation. :return: Search results info:const:`None`, if the search results info file associated with the command invocation is inaccessible. :rtype: SearchResultsInfo or NoneType """ if self._search_results_info is not None: return self._search_results_info if self._protocol_version == 1: try: path = self._input_header['infoPath'] except KeyError: return None else: assert self._protocol_version == 2 try: dispatch_dir = self._metadata.searchinfo.dispatch_dir except AttributeError: return None path = os.path.join(dispatch_dir, 'info.csv') try: with io.open(path, 'r') as f: reader = csv.reader(f, dialect=CsvDialect) fields = next(reader) values = next(reader) except IOError as error: if error.errno == 2: self.logger.error('Search results info file {} does not exist.'.format(json_encode_string(path))) return raise def convert_field(field): return (field[1:] if field[0] == '_' else field).replace('.', '_') decode = MetadataDecoder().decode def convert_value(value): try: return decode(value) if len(value) > 0 else value except ValueError: return value info = ObjectView(dict(imap(lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])), izip(fields, values)))) try: count_map = info.countMap except AttributeError: pass else: count_map = count_map.split(';') n = len(count_map) info.countMap = dict(izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2))) try: msg_type = info.msgType msg_text = info.msg except AttributeError: pass else: messages = ifilter(lambda t_m: t_m[0] or t_m[1], izip(msg_type.split('\n'), msg_text.split('\n'))) info.msg = [Message(message) for message in messages] del info.msgType try: info.vix_families = ElementTree.fromstring(info.vix_families) except AttributeError: pass self._search_results_info = info return info
def test_record_writer_with_random_data(self, save_recording=False): # Confirmed: [minint, maxint) covers the full range of values that xrange allows # RecordWriter writes apps in units of maxresultrows records. Default: 50,0000. # Partial results are written when the record count reaches maxresultrows. writer = RecordWriterV2(StringIO(), maxresultrows=10) # small for the purposes of this unit test test_data = OrderedDict() fieldnames = ['_serial', '_time', 'random_bytes', 'random_dict', 'random_integers', 'random_unicode'] test_data['fieldnames'] = fieldnames test_data['values'] = [] write_record = writer.write_record for serial_number in range(0, 31): values = [serial_number, time(), random_bytes(), random_dict(), random_integers(), random_unicode()] record = OrderedDict(izip(fieldnames, values)) #try: write_record(record) #except Exception as error: # self.fail(error) test_data['values'].append(values) # RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result # is produced or we're finished messages = [ ('debug', random_unicode()), ('error', random_unicode()), ('fatal', random_unicode()), ('info', random_unicode()), ('warn', random_unicode())] test_data['messages'] = messages for message_type, message_text in messages: writer.write_message(message_type, '{}', message_text) metrics = { 'metric-1': SearchMetric(1, 2, 3, 4), 'metric-2': SearchMetric(5, 6, 7, 8) } test_data['metrics'] = metrics for name, metric in six.iteritems(metrics): writer.write_metric(name, metric) self.assertEqual(writer._chunk_count, 3) self.assertEqual(writer._record_count, 1) self.assertGreater(writer._buffer.tell(), 0) self.assertEqual(writer._total_record_count, 30) self.assertListEqual(writer._fieldnames, fieldnames) self.assertListEqual(writer._inspector['messages'], messages) self.assertDictEqual( dict(ifilter(lambda k_v: k_v[0].startswith('metric.'), six.iteritems(writer._inspector))), dict(imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]), six.iteritems(metrics)))) writer.flush(finished=True) self.assertEqual(writer._chunk_count, 4) self.assertEqual(writer._record_count, 0) self.assertEqual(writer._buffer.tell(), 0) self.assertEqual(writer._buffer.getvalue(), '') self.assertEqual(writer._total_record_count, 31) self.assertRaises(AssertionError, writer.flush, finished=True, partial=True) self.assertRaises(AssertionError, writer.flush, finished='non-boolean') self.assertRaises(AssertionError, writer.flush, partial='non-boolean') self.assertRaises(AssertionError, writer.flush) self.assertRaises(RuntimeError, writer.write_record, {}) self.assertFalse(writer._ofile.closed) self.assertIsNone(writer._fieldnames) self.assertDictEqual(writer._inspector, OrderedDict()) # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling # RecordWriter.flush(partial=True). # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling # RecordWriter.flush(finish=True). if save_recording: cls = self.__class__ method = cls.test_record_writer_with_recordings base_path = os.path.join(self._recordings_path, '.'.join((cls.__name__, method.__name__, six.text_type(time())))) with gzip.open(base_path + '.input.gz', 'wb') as f: pickle.dump(test_data, f) with open(base_path + '.output', 'wb') as f: f.write(writer._ofile.getvalue()) return