def run(self, initiator, conditions, **kwargs):
"""
Connects to NSQd instance specified by address, and evaluates
the conditions against every message received.
:param initiator:
:param conditions:
:return:
"""
# user can still initiate
initiator.execute()
reader = gnsq.Reader(self.topic, self.channel, self.address)
reader._funcy_messages = []
reader._funcy_take_n = self.take_n
@reader.on_message.connect
def handler(_r, message):
# each message finish and save it
message.finish()
_r._funcy_messages.append(message)
if len(_r._funcy_messages) == self.take_n:
_r.close()
reader.start()
logger.debug({
'class': self.__class__.__name__,
'num_messages': len(reader._funcy_messages),
})
return ValuesContainer(
reader._funcy_messages
)
def test_duplicate_values(self):
t = ListToDictByKey(type=None, by_key='test1')
self.assertEqual(
(
{
'test1': {
'test1': 'test1',
'test2': 'test2',
}
},
),
t.apply(
ValuesContainer(
[
{
'test1': 'test1',
'test2': 'test2',
},
{
'test1': 'test1',
'test2': 'test2',
}
]
)
).values()
)
def execute(self):
logger.debug({
'message': 'executing_command',
'command': self.command,
})
p = subprocess.Popen(self.command)
returncode = p.wait()
return ValuesContainer(returncode)
def execute(self):
logger.debug('%s', {
'message': 'publishing_message_nsq',
'body': self.message,
})
return ValuesContainer(
gnsq.Nsqd(address=self.nsqd_address).publish(
self.topic, json.dumps(self.message)))
def test_single_value(self):
extractor = DictExtractFields(type=None, fields=['test'])
values = ValuesContainer({
'hi': 'hello',
'test': 'ok',
}, )
self.assertEqual(({
'test': 'ok',
}, ),
extractor.apply(values).values())
def test_one_valid_value(self):
t = ParseJSON(type=None)
self.assertEqual(
(
{
'hi': 'there',
},
),
t.apply(
ValuesContainer('{"hi":"there"}')
).values()
)
def execute(self):
cur = self.conn.cursor()
results = []
try:
cur.execute(self.query)
results = cur.fetchall()
finally:
self.conn.close()
cur.close()
return ValuesContainer(results)
def apply(self, vs):
logger.info({
'class': self.__class__.__name__,
'values': vs.values(),
'fields_to_extract': self.fields
})
transformed = []
for v in vs.values():
ex = {}
for f in self.fields:
ex[f] = v[f]
transformed.append(ex)
return ValuesContainer(transformed)
def test_many_valid_values(self):
t = ParseJSON(type=None, value_property='body')
self.assertEqual(
(
{
'hi': 'there',
},
{
'hi1': 'there1',
}
),
t.apply(
ValuesContainer([
MagicMock(
body='{"hi":"there"}'
),
MagicMock(
body='{"hi1":"there1"}'
),
])
).values()
)
def test_multiple_values(self):
extractor = DictExtractFields(type=None, fields=[
'test',
'hi',
])
values = ValuesContainer((
{
'hi': 'hi',
'test': 'ok',
},
{
'hi': 'hi2',
'test': 'test2',
},
))
self.assertEqual(({
'test': 'ok',
'hi': 'hi',
}, {
'hi': 'hi2',
'test': 'test2',
}),
extractor.apply(values).values())
def execute(self):
return ValuesContainer(getattr(requests, self.method)(self.url))
def test_no_values(self):
t = ListToDictByKey(type=None, by_key='test')
self.assertEqual(
ValuesContainer({}).values(),
t.apply(EmptyValues()).values(),
)
def apply(self, vs):
transformed = []
for v in vs.values():
transformed.append(self.parse(v))
return ValuesContainer(transformed)
def apply(self, vs):
# what to do if a dictionary has the same key?!?!?
return ValuesContainer(
{v[self.by_key]: v for v in vs.values()}
)