def iteritems(self):
definitions = type(self).configuration_setting_definitions
version = self.command.protocol_version
return ifilter(
lambda name_value1: name_value1[1] is not None, imap(
lambda setting: (setting.name, setting.__get__(self)), ifilter(
lambda setting: setting.is_supported_by_protocol(version), definitions)))
def iteritems(self):
definitions = type(self).configuration_setting_definitions
version = self.command.protocol_version
return ifilter(
lambda name_value1: name_value1[1] is not None, imap(
lambda setting: (setting.name, setting.__get__(self)), ifilter(
lambda setting: setting.is_supported_by_protocol(version), definitions)))
def iteritems(self):
iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
version = self.command.protocol_version
if version == 2:
iteritems = ifilter(lambda name_value1: name_value1[0] != 'distributed', iteritems)
if not self.distributed and self.type == 'streaming':
iteritems = imap(
lambda name_value: (name_value[0], 'stateful') if name_value[0] == 'type' else (name_value[0], name_value[1]), iteritems)
return iteritems
def iteritems(self):
iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
version = self.command.protocol_version
if version == 2:
iteritems = ifilter(lambda name_value1: name_value1[0] != 'distributed', iteritems)
if not self.distributed and self.type == 'streaming':
iteritems = imap(
lambda name_value: (name_value[0], 'stateful') if name_value[0] == 'type' else (name_value[0], name_value[1]), iteritems)
return iteritems
def validate_configuration_setting(specification, name, value):
if not isinstance(value, specification.type):
if isinstance(specification.type, type):
type_names = specification.type.__name__
else:
type_names = ', '.join(imap(lambda t: t.__name__, specification.type))
raise ValueError('Expected {} value, not {}={}'.format(type_names, name, repr(value)))
if specification.constraint and not specification.constraint(value):
raise ValueError('Illegal value: {}={}'.format(name, repr(value)))
return value
def validate_configuration_setting(specification, name, value):
if not isinstance(value, specification.type):
if isinstance(specification.type, type):
type_names = specification.type.__name__
else:
type_names = ', '.join(imap(lambda t: t.__name__, specification.type))
raise ValueError('Expected {} value, not {}={}'.format(type_names, name, repr(value)))
if specification.constraint and not specification.constraint(value):
raise ValueError('Illegal value: {}={}'.format(name, repr(value)))
return value
def iteritems(self):
iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
version = self.command.protocol_version
if version == 1:
if self.required_fields is None:
iteritems = ifilter(lambda name_value: name_value[0] != 'clear_required_fields', iteritems)
else:
iteritems = ifilter(lambda name_value2: name_value2[0] != 'distributed', iteritems)
if not self.distributed:
iteritems = imap(
lambda name_value1: (name_value1[0], 'stateful') if name_value1[0] == 'type' else (name_value1[0], name_value1[1]), iteritems)
return iteritems
def iteritems(self):
iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
version = self.command.protocol_version
if version == 1:
if self.required_fields is None:
iteritems = ifilter(lambda name_value: name_value[0] != 'clear_required_fields', iteritems)
else:
iteritems = ifilter(lambda name_value2: name_value2[0] != 'distributed', iteritems)
if not self.distributed:
iteritems = imap(
lambda name_value1: (name_value1[0], 'stateful') if name_value1[0] == 'type' else (name_value1[0], name_value1[1]), iteritems)
return iteritems
def __repr__(self):
""" Converts the value of this instance to its string representation.
The value of this ConfigurationSettings instance is represented as a string of comma-separated
:code:`(name, value)` pairs.
:return: String representation of this instance
"""
definitions = type(self).configuration_setting_definitions
settings = imap(
lambda setting: repr((setting.name, setting.__get__(self), setting.supporting_protocols)), definitions)
return '[' + ', '.join(settings) + ']'
def __iter__(self):
basedir = self._basedir
name = self._name
iterator = imap(
lambda directory: Recording(os.path.join(basedir, directory, name)
),
ifilter(
lambda filename: os.path.isdir(os.path.join(basedir, filename)
), os.listdir(basedir)))
return iterator
def __repr__(self):
""" Converts the value of this instance to its string representation.
The value of this ConfigurationSettings instance is represented as a string of comma-separated
:code:`(name, value)` pairs.
:return: String representation of this instance
"""
definitions = type(self).configuration_setting_definitions
settings = imap(
lambda setting: repr((setting.name, setting.__get__(self), setting.supporting_protocols)), definitions)
return '[' + ', '.join(settings) + ']'
def search_results_info(self):
""" Returns the search results info for this command invocation.
The search results info object is created from the search results info file associated with the command
invocation.
:return: Search results info:const:`None`, if the search results info file associated with the command
invocation is inaccessible.
:rtype: SearchResultsInfo or NoneType
"""
if self._search_results_info is not None:
return self._search_results_info
if self._protocol_version == 1:
try:
path = self._input_header['infoPath']
except KeyError:
return None
else:
assert self._protocol_version == 2
try:
dispatch_dir = self._metadata.searchinfo.dispatch_dir
except AttributeError:
return None
path = os.path.join(dispatch_dir, 'info.csv')
try:
with io.open(path, 'r') as f:
reader = csv.reader(f, dialect=CsvDialect)
fields = next(reader)
values = next(reader)
except IOError as error:
if error.errno == 2:
self.logger.error(
'Search results info file {} does not exist.'.format(
json_encode_string(path)))
return
raise
def convert_field(field):
return (field[1:] if field[0] == '_' else field).replace('.', '_')
decode = MetadataDecoder().decode
def convert_value(value):
try:
return decode(value) if len(value) > 0 else value
except ValueError:
return value
info = ObjectView(
dict(
imap(
lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])),
izip(fields, values))))
try:
count_map = info.countMap
except AttributeError:
pass
else:
count_map = count_map.split(';')
n = len(count_map)
info.countMap = dict(
izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2)))
try:
msg_type = info.msgType
msg_text = info.msg
except AttributeError:
pass
else:
messages = ifilter(
lambda t_m: t_m[0] or t_m[1],
izip(msg_type.split('\n'), msg_text.split('\n')))
info.msg = [Message(message) for message in messages]
del info.msgType
try:
info.vix_families = ElementTree.fromstring(info.vix_families)
except AttributeError:
pass
self._search_results_info = info
return info
def test_record_writer_with_random_data(self, save_recording=False):
# Confirmed: [minint, maxint) covers the full range of values that xrange allows
# RecordWriter writes apps in units of maxresultrows records. Default: 50,0000.
# Partial results are written when the record count reaches maxresultrows.
writer = RecordWriterV2(StringIO(), maxresultrows=10) # small for the purposes of this unit test
test_data = OrderedDict()
fieldnames = ['_serial', '_time', 'random_bytes', 'random_dict', 'random_integers', 'random_unicode']
test_data['fieldnames'] = fieldnames
test_data['values'] = []
write_record = writer.write_record
for serial_number in range(0, 31):
values = [serial_number, time(), random_bytes(), random_dict(), random_integers(), random_unicode()]
record = OrderedDict(izip(fieldnames, values))
#try:
write_record(record)
#except Exception as error:
# self.fail(error)
test_data['values'].append(values)
# RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result
# is produced or we're finished
messages = [
('debug', random_unicode()),
('error', random_unicode()),
('fatal', random_unicode()),
('info', random_unicode()),
('warn', random_unicode())]
test_data['messages'] = messages
for message_type, message_text in messages:
writer.write_message(message_type, '{}', message_text)
metrics = {
'metric-1': SearchMetric(1, 2, 3, 4),
'metric-2': SearchMetric(5, 6, 7, 8)
}
test_data['metrics'] = metrics
for name, metric in six.iteritems(metrics):
writer.write_metric(name, metric)
self.assertEqual(writer._chunk_count, 3)
self.assertEqual(writer._record_count, 1)
self.assertGreater(writer._buffer.tell(), 0)
self.assertEqual(writer._total_record_count, 30)
self.assertListEqual(writer._fieldnames, fieldnames)
self.assertListEqual(writer._inspector['messages'], messages)
self.assertDictEqual(
dict(ifilter(lambda k_v: k_v[0].startswith('metric.'), six.iteritems(writer._inspector))),
dict(imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]), six.iteritems(metrics))))
writer.flush(finished=True)
self.assertEqual(writer._chunk_count, 4)
self.assertEqual(writer._record_count, 0)
self.assertEqual(writer._buffer.tell(), 0)
self.assertEqual(writer._buffer.getvalue(), '')
self.assertEqual(writer._total_record_count, 31)
self.assertRaises(AssertionError, writer.flush, finished=True, partial=True)
self.assertRaises(AssertionError, writer.flush, finished='non-boolean')
self.assertRaises(AssertionError, writer.flush, partial='non-boolean')
self.assertRaises(AssertionError, writer.flush)
self.assertRaises(RuntimeError, writer.write_record, {})
self.assertFalse(writer._ofile.closed)
self.assertIsNone(writer._fieldnames)
self.assertDictEqual(writer._inspector, OrderedDict())
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling
# RecordWriter.flush(partial=True).
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling
# RecordWriter.flush(finish=True).
if save_recording:
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(self._recordings_path, '.'.join((cls.__name__, method.__name__, six.text_type(time()))))
with gzip.open(base_path + '.input.gz', 'wb') as f:
pickle.dump(test_data, f)
with open(base_path + '.output', 'wb') as f:
f.write(writer._ofile.getvalue())
return
def random_unicode():
return ''.join(
imap(lambda x: six.unichr(x),
random.sample(range(maxunicode), random.randint(0, max_length))))
def test_record_writer_with_random_data(self, save_recording=False):
# Confirmed: [minint, maxint) covers the full range of values that xrange allows
# RecordWriter writes apps in units of maxresultrows records. Default: 50,0000.
# Partial results are written when the record count reaches maxresultrows.
writer = RecordWriterV2(
StringIO(),
maxresultrows=10) # small for the purposes of this unit test
test_data = OrderedDict()
fieldnames = [
'_serial', '_time', 'random_bytes', 'random_dict',
'random_integers', 'random_unicode'
]
test_data['fieldnames'] = fieldnames
test_data['values'] = []
write_record = writer.write_record
for serial_number in range(0, 31):
values = [
serial_number,
time(),
random_bytes(),
random_dict(),
random_integers(),
random_unicode()
]
record = OrderedDict(izip(fieldnames, values))
#try:
write_record(record)
#except Exception as error:
# self.fail(error)
test_data['values'].append(values)
# RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result
# is produced or we're finished
messages = [('debug', random_unicode()), ('error', random_unicode()),
('fatal', random_unicode()), ('info', random_unicode()),
('warn', random_unicode())]
test_data['messages'] = messages
for message_type, message_text in messages:
writer.write_message(message_type, '{}', message_text)
metrics = {
'metric-1': SearchMetric(1, 2, 3, 4),
'metric-2': SearchMetric(5, 6, 7, 8)
}
test_data['metrics'] = metrics
for name, metric in six.iteritems(metrics):
writer.write_metric(name, metric)
self.assertEqual(writer._chunk_count, 3)
self.assertEqual(writer._record_count, 1)
self.assertGreater(writer._buffer.tell(), 0)
self.assertEqual(writer._total_record_count, 30)
self.assertListEqual(writer._fieldnames, fieldnames)
self.assertListEqual(writer._inspector['messages'], messages)
self.assertDictEqual(
dict(
ifilter(lambda k_v: k_v[0].startswith('metric.'),
six.iteritems(writer._inspector))),
dict(
imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]),
six.iteritems(metrics))))
writer.flush(finished=True)
self.assertEqual(writer._chunk_count, 4)
self.assertEqual(writer._record_count, 0)
self.assertEqual(writer._buffer.tell(), 0)
self.assertEqual(writer._buffer.getvalue(), '')
self.assertEqual(writer._total_record_count, 31)
self.assertRaises(AssertionError,
writer.flush,
finished=True,
partial=True)
self.assertRaises(AssertionError, writer.flush, finished='non-boolean')
self.assertRaises(AssertionError, writer.flush, partial='non-boolean')
self.assertRaises(AssertionError, writer.flush)
self.assertRaises(RuntimeError, writer.write_record, {})
self.assertFalse(writer._ofile.closed)
self.assertIsNone(writer._fieldnames)
self.assertDictEqual(writer._inspector, OrderedDict())
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling
# RecordWriter.flush(partial=True).
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling
# RecordWriter.flush(finish=True).
if save_recording:
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(
self._recordings_path, '.'.join(
(cls.__name__, method.__name__, six.text_type(time()))))
with gzip.open(base_path + '.input.gz', 'wb') as f:
pickle.dump(test_data, f)
with open(base_path + '.output', 'wb') as f:
f.write(writer._ofile.getvalue())
return
def _write_record(self, record):
fieldnames = self._fieldnames
if fieldnames is None:
self._fieldnames = fieldnames = list(record.keys())
value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)),
fieldnames)
self._writerow(list(chain.from_iterable(value_list)))
get_value = record.get
values = []
for fieldname in fieldnames:
value = get_value(fieldname, None)
if value is None:
values += (None, None)
continue
value_t = type(value)
if issubclass(value_t, (list, tuple)):
if len(value) == 0:
values += (None, None)
continue
if len(value) > 1:
value_list = value
sv = ''
mv = '$'
for value in value_list:
if value is None:
sv += '\n'
mv += '$;$'
continue
value_t = type(value)
if value_t is not bytes:
if value_t is bool:
value = str(value.real)
elif value_t is six.text_type:
value = value
elif isinstance(
value, six.integer_types
) or value_t is float or value_t is complex:
value = str(value)
elif issubclass(value_t, (dict, list, tuple)):
value = str(''.join(
RecordWriter._iterencode_json(value, 0)))
else:
value = repr(value).encode(
'utf-8', errors='backslashreplace')
sv += value + '\n'
mv += value.replace('$', '$$') + '$;$'
values += (sv[:-1], mv[:-2])
continue
value = value[0]
value_t = type(value)
if value_t is bool:
values += (str(value.real), None)
continue
if value_t is bytes:
values += (value, None)
continue
if value_t is six.text_type:
if six.PY2:
value = value.encode('utf-8')
values += (value, None)
continue
if isinstance(value, six.integer_types
) or value_t is float or value_t is complex:
values += (str(value), None)
continue
if issubclass(value_t, dict):
values += (str(''.join(RecordWriter._iterencode_json(value,
0))),
None)
continue
values += (repr(value), None)
self._writerow(values)
self._pending_record_count += 1
if self.pending_record_count >= self._maxresultrows:
self.flush(partial=True)
def random_unicode():
return ''.join(imap(lambda x: six.unichr(x), random.sample(range(maxunicode), random.randint(0, max_length))))
def fix_up(cls, values):
is_configuration_setting = lambda attribute: isinstance(attribute, ConfigurationSetting)
definitions = getmembers(cls, is_configuration_setting)
i = 0
for name, setting in definitions:
if setting._name is None:
setting._name = name = six.text_type(name)
else:
name = setting._name
validate, specification = setting._get_specification()
backing_field_name = '_' + name
if setting.fget is None and setting.fset is None and setting.fdel is None:
value = setting._value
if setting._readonly or value is not None:
validate(specification, name, value)
def fget(bfn, value):
return lambda this: getattr(this, bfn, value)
setting = setting.getter(fget(backing_field_name, value))
if not setting._readonly:
def fset(bfn, validate, specification, name):
return lambda this, value: setattr(this, bfn, validate(specification, name, value))
setting = setting.setter(fset(backing_field_name, validate, specification, name))
setattr(cls, name, setting)
def is_supported_by_protocol(supporting_protocols):
def is_supported_by_protocol(version):
return version in supporting_protocols
return is_supported_by_protocol
del setting._name, setting._value, setting._readonly
setting.is_supported_by_protocol = is_supported_by_protocol(specification.supporting_protocols)
setting.supporting_protocols = specification.supporting_protocols
setting.backing_field_name = backing_field_name
definitions[i] = setting
setting.name = name
i += 1
try:
value = values[name]
except KeyError:
continue
if setting.fset is None:
raise ValueError('The value of configuration setting {} is fixed'.format(name))
setattr(cls, backing_field_name, validate(specification, name, value))
del values[name]
if len(values) > 0:
settings = sorted(list(six.iteritems(values)))
settings = imap(lambda n_v: '{}={}'.format(n_v[0], repr(n_v[1])), settings)
raise AttributeError('Inapplicable configuration settings: ' + ', '.join(settings))
cls.configuration_setting_definitions = definitions
def __repr__(self):
text = 'Option.View([' + ','.join(imap(lambda item: repr(item), six.itervalues(self))) + '])'
return text
def random_unicode():
return ''.join(
imap(
lambda x: six.unichr(x),
random.sample(range(MAX_NARROW_UNICODE),
random.randint(0, max_length))))
def iteritems(self):
iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
return imap(
lambda name_value:
(name_value[0], 'events'
if name_value[0] == 'type' else name_value[1]), iteritems)
def iteritems(self):
iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
return imap(lambda name_value: (name_value[0], 'events' if name_value[0] == 'type' else name_value[1]), iteritems)
def fix_up(cls, values):
is_configuration_setting = lambda attribute: isinstance(attribute, ConfigurationSetting)
definitions = getmembers(cls, is_configuration_setting)
i = 0
for name, setting in definitions:
if setting._name is None:
setting._name = name = six.text_type(name)
else:
name = setting._name
validate, specification = setting._get_specification()
backing_field_name = '_' + name
if setting.fget is None and setting.fset is None and setting.fdel is None:
value = setting._value
if setting._readonly or value is not None:
validate(specification, name, value)
def fget(bfn, value):
return lambda this: getattr(this, bfn, value)
setting = setting.getter(fget(backing_field_name, value))
if not setting._readonly:
def fset(bfn, validate, specification, name):
return lambda this, value: setattr(this, bfn, validate(specification, name, value))
setting = setting.setter(fset(backing_field_name, validate, specification, name))
setattr(cls, name, setting)
def is_supported_by_protocol(supporting_protocols):
def is_supported_by_protocol(version):
return version in supporting_protocols
return is_supported_by_protocol
del setting._name, setting._value, setting._readonly
setting.is_supported_by_protocol = is_supported_by_protocol(specification.supporting_protocols)
setting.supporting_protocols = specification.supporting_protocols
setting.backing_field_name = backing_field_name
definitions[i] = setting
setting.name = name
i += 1
try:
value = values[name]
except KeyError:
continue
if setting.fset is None:
raise ValueError('The value of configuration setting {} is fixed'.format(name))
setattr(cls, backing_field_name, validate(specification, name, value))
del values[name]
if len(values) > 0:
settings = sorted(list(six.iteritems(values)))
settings = imap(lambda n_v: '{}={}'.format(n_v[0], repr(n_v[1])), settings)
raise AttributeError('Inapplicable configuration settings: ' + ', '.join(settings))
cls.configuration_setting_definitions = definitions
def __repr__(self):
text = 'Option.View([' + ','.join(imap(lambda item: repr(item), six.itervalues(self))) + '])'
return text
def _write_record(self, record):
fieldnames = self._fieldnames
if fieldnames is None:
self._fieldnames = fieldnames = list(record.keys())
value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames)
self._writerow(list(chain.from_iterable(value_list)))
get_value = record.get
values = []
for fieldname in fieldnames:
value = get_value(fieldname, None)
if value is None:
values += (None, None)
continue
value_t = type(value)
if issubclass(value_t, (list, tuple)):
if len(value) == 0:
values += (None, None)
continue
if len(value) > 1:
value_list = value
sv = ''
mv = '$'
for value in value_list:
if value is None:
sv += '\n'
mv += '$;$'
continue
value_t = type(value)
if value_t is not bytes:
if value_t is bool:
value = str(value.real)
elif value_t is six.text_type:
value = value
elif value_t is int or value_t is int or value_t is float or value_t is complex:
value = str(value)
elif issubclass(value_t, (dict, list, tuple)):
value = str(''.join(RecordWriter._iterencode_json(value, 0)))
else:
value = repr(value).encode('utf-8', errors='backslashreplace')
sv += value + '\n'
mv += value.replace('$', '$$') + '$;$'
values += (sv[:-1], mv[:-2])
continue
value = value[0]
value_t = type(value)
if value_t is bool:
values += (str(value.real), None)
continue
if value_t is bytes:
values += (value, None)
continue
if value_t is six.text_type:
if six.PY2:
value = value.encode('utf-8')
values += (value, None)
continue
if value_t is int or value_t is int or value_t is float or value_t is complex:
values += (str(value), None)
continue
if issubclass(value_t, dict):
values += (str(''.join(RecordWriter._iterencode_json(value, 0))), None)
continue
values += (repr(value), None)
self._writerow(values)
self._record_count += 1
if self._record_count >= self._maxresultrows:
self.flush(partial=True)
def search_results_info(self):
""" Returns the search results info for this command invocation.
The search results info object is created from the search results info file associated with the command
invocation.
:return: Search results info:const:`None`, if the search results info file associated with the command
invocation is inaccessible.
:rtype: SearchResultsInfo or NoneType
"""
if self._search_results_info is not None:
return self._search_results_info
if self._protocol_version == 1:
try:
path = self._input_header['infoPath']
except KeyError:
return None
else:
assert self._protocol_version == 2
try:
dispatch_dir = self._metadata.searchinfo.dispatch_dir
except AttributeError:
return None
path = os.path.join(dispatch_dir, 'info.csv')
try:
with io.open(path, 'r') as f:
reader = csv.reader(f, dialect=CsvDialect)
fields = next(reader)
values = next(reader)
except IOError as error:
if error.errno == 2:
self.logger.error('Search results info file {} does not exist.'.format(json_encode_string(path)))
return
raise
def convert_field(field):
return (field[1:] if field[0] == '_' else field).replace('.', '_')
decode = MetadataDecoder().decode
def convert_value(value):
try:
return decode(value) if len(value) > 0 else value
except ValueError:
return value
info = ObjectView(dict(imap(lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])), izip(fields, values))))
try:
count_map = info.countMap
except AttributeError:
pass
else:
count_map = count_map.split(';')
n = len(count_map)
info.countMap = dict(izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2)))
try:
msg_type = info.msgType
msg_text = info.msg
except AttributeError:
pass
else:
messages = ifilter(lambda t_m: t_m[0] or t_m[1], izip(msg_type.split('\n'), msg_text.split('\n')))
info.msg = [Message(message) for message in messages]
del info.msgType
try:
info.vix_families = ElementTree.fromstring(info.vix_families)
except AttributeError:
pass
self._search_results_info = info
return info
