def process(self, tup):
word = tup.values[0];
if (random() < 0.75):
storm.emit([word + 'lalala'], anchors=[tup])
storm.ack(tup)
else:
storm.log(word + ' randomly skipped!')
def initialize(self, conf, context):
self._conf = conf
self._context = context
self.count = 0
all_stock_codes = nse.get_stock_codes(cached=False)
self.quote_keys = all_stock_codes.keys()
storm.log("Spout instance starting...")
def received_message(self, message):
uid = str(message)
if len(uid) and _server:
log('[OutletWebSocket] Client connected: %s' % uid)
index = _server.manager.websockets.values().index(self)
_clients[uid] = _server.manager.websockets.keys()[index]
emitBolt(['connect', uid], stream='control')
def process(self, tup):
url = tup.values[0]
storm.log("HARing "+url)
output = self.get_har_with_image(url)
if output is not "FAIL":
storm.emit(output, anchors=[tup])
storm.ack(tup)
def nextTuple(self):
try:
input_tuple = None
try:
input_tuple = self.script._static_input.pop(0)
except AttributeError:
# there is no static input
pass
except IndexError:
# static input is empty - no more processing
return
outputs = self.script.process(input_tuple)
if outputs is None:
return
for streamname, output in outputs.iteritems():
result = output if isinstance(output, list) else [output]
storm.emit(result, stream=streamname)
storm.log(
"Dispel4Py ------> %s: emitted tuple %s to stream %s" %
(self.script.id, result, streamname))
except:
# logging the error but it should be passed to client somehow
storm.log("Dispel4Py ------> %s: %s" % (
self.scriptname,
traceback.format_exc(),
))
def emit(self, record):
from petrel import storm
msg = self.format(record)
for line in msg.split('\n'):
formatted_line = self.format_string % line
#print >> sys.stderr, "Calling storm.log with: %s" % formatted_line
storm.log('%s' % formatted_line)
def nextTuple(self):
try:
input_tuple = None
try:
if self.counter >= self.script._num_iterations:
return
except:
try:
input_tuple = self.script._static_input.pop(0)
except AttributeError:
# there is no static input
pass
except IndexError:
# static input is empty - no more processing
return
outputs = self.script.process(input_tuple)
if outputs is None:
return
for streamname, output in outputs.iteritems():
result = output if isinstance(output, list) else [output]
storm.emit(result, stream=streamname, id=self.counter)
storm.log("Dispel4Py ------> %s: emitted tuple %s to stream %s" % (self.script.id, result, streamname))
self.counter += 1
except:
# logging the error but it should be passed to client somehow
storm.log("Dispel4Py ------> %s: %s" % (self.scriptname, traceback.format_exc(), ))
def initialize(self, conf, context):
try:
self.counter = 0
self.modname = conf["dispel4py.module"]
self.scriptname = conf["dispel4py.script"]
scriptconfig = pickle.loads(str(conf['dispel4py.config'])) if 'dispel4py.config' in conf else {}
storm.log("Dispel4Py ------> loading script %s" % self.scriptname)
mod = import_module(self.modname)
self.script = getattr(mod, self.scriptname)()
for key, value in scriptconfig.iteritems():
storm.log("Dispel4Py ------> %s: setting attribute %s" % (self.scriptname, key))
setattr(self.script, key, value)
storm.log("Dispel4Py ------> loaded script %s" % self.scriptname)
# attach an output writer to each output connection
for outputname, output in self.script.outputconnections.iteritems():
output['writer'] = OutputWriter(self.scriptname, outputname)
# pre-processing if required
self.script.preprocess()
storm.log("Dispel4Py ------> %s: preprocess() completed." % (self.scriptname,))
except:
storm.log("Dispel4Py ------> %s: %s" % (self.scriptname, traceback.format_exc(),))
raise
def process(self, tup):
message_buf = tup.values[0]
message = json.loads(message_buf)
message_id = message.get('message_id')
if message_id == PUT_METRIC_DATA_MSG_ID:
metric_key = str(self.get_metric_key(message))
storm.emit([metric_key, message_buf])
elif message_id == PUT_METRIC_ALARM_MSG_ID:
metric_key = message.get('metric_key')
storm.emit([metric_key, message_buf])
elif message_id == DELETE_ALARMS_MSG_ID:
project_id = message.get('project_id')
alarmkeys = message.get('alarmkeys')
for alarmkey in alarmkeys:
try:
alarmkey_uuid = UUID(alarmkey)
metric_key = self.get_alarm_metric_key(alarmkey_uuid)
metric_key = str(metric_key)
if metric_key:
message['alarmkey'] = alarmkey
storm.emit([metric_key, json.dumps(message)])
except Exception as e:
storm.log("Alarm %s does not exists" % alarmkey)
storm.log(traceback.format_exc(e))
elif message_id == SET_ALARM_STATE_MSG_ID:
project_id = message.get('project_id')
alarm_name = message.get('alarm_name')
alarm_key = self.cass.get_metric_alarm_key(project_id,
alarm_name)
if alarm_key:
alarm = self.cass.get_metric_alarm(alarm_key)
metric_key = str(alarm.get('metric_key'))
storm.emit([metric_key, json.dumps(message)])
def process_set_alarm_state_msg(self, metric_key, message):
project_id = message.get('project_id')
alarm_name = message.get('alarm_name')
state_reason_data = message.get('state_reason_data')
if metric_key not in self.metrics:
self.metrics[metric_key] = MetricMonitor(metric_key, self.cass)
metric = self.metrics[metric_key]
ret = self.cass.get_metric_alarm_key(project_id, alarm_name)
if ret:
alarm_key = ret
try:
metricalarm = metric.alarms[alarm_key]
except KeyError:
storm.log("alarm key [%s] is found, but alarm is not found." % alarm_key)
return
else:
storm.log("alarm key [%s] is not found." % alarm_key)
return
metricalarm['state_reason'] = message.get('state_reason')
metricalarm['state_value'] = message.get('state_value')
metricalarm['state_reason_data'] = message.get('state_reason_data')
# write into database
alarm_columns = {'state_reason':message.get('state_reason'),
'state_value':message.get('state_value')}
if state_reason_data:
alarm_columns['state_reason_data'] = state_reason_data
self.cass.put_metric_alarm(alarm_key, alarm_columns)
def process(self, tup):
word = tup.values[0]
if (random() < 0.75):
storm.emit([word + 'lalala'], anchors=[tup])
storm.ack(tup)
else:
storm.log(word + ' randomly skipped!')
def received_message(self, message):
uid = str(message)
if len(uid) and _server:
log('[OutletWebSocket] Client connected: %s' % uid)
index = _server.manager.websockets.values().index(self)
_clients[uid] = _server.manager.websockets.keys()[index]
emitBolt(['connect', uid], stream='control')
def fail(self, msg_id):
tup, retries = self.buffer[msg_id]
if retries >= 5:
del self.buffer[msg_id]
log('[RabbitMQSpout] Message %s failed for good.' % msg_id)
else:
self.buffer[msg_id] = (tup, retries + 1)
emit(tup, id=msg_id)
def fail(self, cnt_id):
tup, retries = self.buffer[cnt_id]
if retries >= 5:
del self.buffer[cnt_id]
log('[ZonAPISpout] Message %s failed for good.' % cnt_id)
else:
self.buffer[cnt_id] = (tup, retries + 1)
emit(tup, id=cnt_id)
def fail(self, msg_id):
tup, retries = self.buffer[msg_id]
if retries >= 5:
del self.buffer[msg_id]
log('[RabbitMQSpout] Message %s failed for good.' % msg_id)
else:
self.buffer[msg_id] = (tup, retries + 1)
emit(tup, id=msg_id)
def fail(self, cnt_id):
tup, retries = self.buffer[cnt_id]
if retries >= 5:
del self.buffer[cnt_id]
log('[ZonAPISpout] Message %s failed for good.' % cnt_id)
else:
self.buffer[cnt_id] = (tup, retries + 1)
emit(tup, id=cnt_id)
def stormLogger(message):
'''
Logger implementation for Storm.
'''
try:
storm.log(str(message))
except:
# just ignore the exception
pass
def nextTuple(self):
if self.count < 2:
word = choice(words)
id = str(uuid4())
self.pending[id] = word
storm.rpcMetrics("my-custom-shellspout-metric", 1)
self.count = self.count + 1
storm.log("TesterSpout update my-custom-shellspout-metric "+str(self.count))
storm.emit([word], id=id)
def stormLogger(message):
'''
Logger implementation for Storm.
'''
try:
storm.log(str(message))
except:
# just ignore the exception
pass
def log_tweeter_error(tweep_error, sleep_time=2):
'''
:param tweep_error: Exception dealing with twitter to log to the parent process
:type api: tweepy.TweepError
:param sleep_time: time in seconds to sleep before continuing the execution
'''
# We have hit the REST API Rate limit for Twitter https://dev.twitter.com/docs/rate-limiting/1.1, no more tweets for some time
storm.log("Tweepy error {error}, sleeping for {secs} seconds in case Twitter rate limit has been hit".format(error=str(tweep_error), secs=sleep_time))
time.sleep(sleep_time)
def process(self, tup):
word = tup.values[0]
if self.counts.has_key(word):
count = self.counts[word]
else:
count = 0
count += 1
self.counts[word] = count
storm.log(str(word)+" "+str(count))
storm.emit([word, count])
def write(self, output):
result = output if isinstance(output, list) else [output]
try:
storm.emit(result, stream=self.streamname)
storm.log("Dispel4Py ------> %s: Emitted to stream %s." % (self.scriptname, self.streamname))
except TypeError:
# encode manually
encoded = encode_types(result)
storm.emit(encoded, stream=self.streamname)
storm.log("Dispel4Py ------> %s: Emitted to stream %s." % (self.scriptname, self.streamname))
def write(self, output):
result = output if isinstance(output, list) else [output]
try:
storm.emit(result, stream=self.streamname)
storm.log("Dispel4Py ------> Emitted to stream %s." % (self.scriptname, self.streamname))
except TypeError:
# encode manually
encoded = encode_types(result)
storm.emit(encoded, stream=self.streamname)
storm.log("Dispel4Py ------> Emitted to stream %s." % (self.scriptname, self.streamname))
def nextTuple(self):
try:
vb, msg = self.reader.response().next()
if msg and msg['opcode'] == 87:
tweet = json.loads(msg['value'])
tags = tweet['hashtags']
id = tweet['id']
storm.emit([tags[0], id, vb])
except StopIteration:
storm.log("resetting vbucket map")
self.reader.reset()
def process(self, tuple):
storm.log(tuple.component + ': ' + str(tuple.values))
# Write rules to local fs and reload
if tuple.component == 'rule-definitions':
fn = tuple.values[0].split('/')[-1]
self.write_rule(fn, tuple.values[1])
self.define_rule(fn)
# otherwise, apply each rule to each tuple
else:
for module in self.modules:
output = self.modules[module].handle(tuple)
storm.emit([output])
def process(self, tuple):
storm.log(tuple.component + ': ' + str(tuple.values))
# Write rules to local fs and reload
if tuple.component == 'rule-definitions':
fn = tuple.values[0].split('/')[-1]
self.write_rule(fn, tuple.values[1])
self.define_rule(fn)
# otherwise, apply each rule to each tuple
else:
for module in self.modules:
output = self.modules[module].handle(tuple)
storm.emit([output])
def process(self,tup):
# storm.log("Dispel4Py ------> %s: Received block." % (self.script.id, ))
try:
inputname = self.inputmapping[tup.component][tup.stream]
storm.log("Dispel4Py ------> %s: Received block at input '%s'" % (self.script.id, inputname, ))
# inputs = tup.values
inputs = decode_types(tup.values)
outputs = self.script.process( { inputname : inputs })
# storm.log("Dispel4Py ------> %s: Processing complete." % self.scriptname)
if outputs is None:
return
for streamname, output in outputs.iteritems():
result = output if isinstance(output, list) else [output]
try:
storm.emit(result, stream=streamname)
storm.log("Dispel4Py ------> %s: Emitted to stream %s: %s" % (self.script.id, streamname, str(result)[:200]))
except TypeError:
# encode manually
encoded = encode_types(result)
storm.emit(encoded, stream=streamname)
storm.log("Dispel4Py ------> %s: Emitted to stream %s" % (self.script.id, streamname))
# except:
# storm.log("%s: %s" % (self.script.id, traceback.format_exc()))
except:
storm.log("Dispel4Py ------> %s: %s" % (self.script.id, traceback.format_exc(), ))
def process(self, tup):
# storm.log("Dispel4Py ------> %s: Received block." % (self.script.id, ))
try:
inputname = self.inputmapping[tup.component][tup.stream]
storm.log("Dispel4Py ------> %s: Received block at input '%s'" % (
self.script.id,
inputname,
))
# inputs = tup.values
inputs = decode_types(tup.values)
outputs = self.script.process({inputname: inputs})
# storm.log("Dispel4Py ------> %s: Processing complete." % self.scriptname)
if outputs is None:
return
for streamname, output in outputs.iteritems():
result = output if isinstance(output, list) else [output]
try:
storm.emit(result, stream=streamname)
storm.log(
"Dispel4Py ------> %s: Emitted to stream %s: %s" %
(self.script.id, streamname, str(result)[:200]))
except TypeError:
# encode manually
encoded = encode_types(result)
storm.emit(encoded, stream=streamname)
storm.log("Dispel4Py ------> %s: Emitted to stream %s" %
(self.script.id, streamname))
# except:
# storm.log("%s: %s" % (self.script.id, traceback.format_exc()))
except:
storm.log("Dispel4Py ------> %s: %s" % (
self.script.id,
traceback.format_exc(),
))
def initialize(self, conf, context):
host = conf.get('zeit.recommend.elasticsearch.host', 'localhost')
port = conf.get('zeit.recommend.elasticsearch.port', 9200)
self.es = Elasticsearch(hosts=[{'host': host, 'port': port}])
self.match = re.compile('seite-[0-9]|komplettansicht').match
self.index = '%s-%s' % date.today().isocalendar()[:2]
ic = IndicesClient(self.es)
try:
if not ic.exists(self.index):
ic.create(self.index)
except ConnectionError, e:
log('[UserIndexBolt] ConnectionError, index unreachable: %s' % e)
return
def initialize(self, conf, context):
host = conf.get('zeit.recommend.elasticsearch.host', 'localhost')
port = conf.get('zeit.recommend.elasticsearch.port', 9200)
self.es = Elasticsearch(hosts=[{'host': host, 'port': port}])
self.match = re.compile('seite-[0-9]|komplettansicht').match
self.index = '%s-%s' % date.today().isocalendar()[:2]
ic = IndicesClient(self.es)
try:
if not ic.exists(self.index):
ic.create(self.index)
except ConnectionError, e:
log('[UserIndexBolt] ConnectionError, index unreachable: %s' % e)
return
def get_stats(tmp_stat):
try:
ret = dict(zip(self.cass.STATISTICS,
map(lambda x: x.values()[0], tmp_stat)))
for v in ret:
if v == None: v = float('nan')
except IndexError:
storm.log("index %s is not in DB." % time_idx)
ret = {'SampleCount' : float('nan'),
'Sum' : float('nan'),
'Average' : float('nan'),
'Minimum' : float('nan'),
'Maximum' : float('nan') }
return ret
def do_alarm_action(self, alarmkey, alarm, new_state, old_state,
query_date):
"""
parameter example:
alarmkey: f459c0e0-f927-481f-9158-deb8abe102a2
alarm: OrderedDict([('actions_enabled', False),
('alarm_actions', u'[]'),
('alarm_arn', u'arn:spcs:synaps:IaaS:alarm:TEST_\uc54c\ub78c_02'),
('alarm_configuration_updated_timestamp', datetime.datetime(2012, 8, 25, 10, 51, 38, 469000)),
('alarm_description', u''),
('alarm_name', u'TEST_\uc54c\ub78c_02'),
('comparison_operator', u'LessThanThreshold'),
('dimensions', u'{"instance_name": "test instance"}'),
('evaluation_periods', 2),
('insufficient_data_actions', u'[]'),
('metric_key', UUID('96f19ec9-673b-4237-ae66-1bfde526595c')),
('metric_name', u'test_metric'),
('namespace', u'SPCS/SYNAPSTEST'),
('ok_actions', u'[]'),
('period', 300),
('project_id', u'IaaS'),
('state_reason', u'Threshold Crossed: 2 datapoints were not less than the threshold(2.000000). The most recent datapoints: [55.25, 55.25].'),
('state_reason_data', u'{"startDate": "2012-08-25T10:30:00.000000", "period": 300, "threshold": 2.0, "version": "1.0", "statistic": "Average", "recentDatapoints": [55.25, 55.25], "queryDate": "2012-08-25T10:32:24.671991"}'),
('state_updated_timestamp', datetime.datetime(2012, 8, 25, 11, 39, 49, 657449)),
('state_value', 'OK'),
('statistic', u'Average'),
('threshold', 2.0),
('unit', u'Percent'),
('reason', u'Threshold Crossed: 3 datapoints were not less than the threshold(2.000000). The most recent datapoints: [75.0, 80.0, 67.625].'),
('reason_data', '{"startDate": "2012-08-25T11:37:00.000000", "period": 300, "threshold": 2.0, "version": "1.0", "statistic": "Average", "recentDatapoints": [75.0, 80.0, 67.625], "queryDate": "2012-08-25T11:39:49.657449"}')
])
new_state: {'stateReason': u'Threshold Crossed: 3 datapoints were not less than the threshold(2.000000). The most recent datapoints: [75.0, 80.0, 67.625].',
'stateValue': 'OK',
'stateReasonData': {'startDate': '2012-08-25T11:37:00.000000', 'period': 300, 'threshold': 2.0, 'version': '1.0', 'statistic': u'Average', 'recentDatapoints': [75.0, 80.0, 67.625], 'queryDate': '2012-08-25T11:39:49.657449'}}
old_state: {'stateReason': u'Insufficient Data: 1 datapoints were unknown.',
'stateReasonData': {u'startDate': u'2012-08-25T11:37:00.000000', u'period': 300, u'recentDatapoints': [55.25], u'version': u'1.0', u'statistic': u'Average', u'threshold': 2.0, u'queryDate': u'2012-08-25T11:39:26.261056'}, 'stateValue': 'INSUFFICIENT_DATA'}
"""
msg = {
'state': new_state['stateValue'],
'subject': "%s state has been changed from %s to %s at %s" %
(alarm['alarm_name'], old_state['stateValue'],
new_state['stateValue'], query_date),
'body': "%s at %s" % (new_state['stateReason'], query_date),
'query_date': query_date
}
storm.log("emit to Alarm Action: %s %s" % (alarmkey, msg))
storm.emit([str(alarmkey), json.dumps(msg)])
def process(self, tup):
title, url, text_content = tup.values
# print(title, url, type(text_content))
for clf in self.models:
new_counts = self.count_vect.transform([text_content])
new_tfidf = self.tfidf_transformer.transform(new_counts)
predicted = clf.predict(new_tfidf)
self._prediction.append(self.target_names[predicted])
storm.log(title + url + " is : ", self.target_names[predicted])
emit_tup = [title, url, "|".join(self._prediction)]
storm.log("Emit %s" % emit_tup)
storm.emit(emit_tup)
def process(self, tup):
student = tup.values[0]
gre_score = tup.values[1]
gpa_score = tup.values[2]
prestige = tup.values[3]
retInfo = tup.values[4]
storm.log('{}:{}:{}'.format(gre_score, gpa_score, prestige))
row = np.array([gre_score, gpa_score, prestige], dtype=float)
predArr = self.clf.predict(row)
pred = str(predArr[0])
storm.log('{}:{}:{}:{}:{}'.format(student, gre_score, gpa_score, pred, retInfo))
storm.emit([student, pred, retInfo])
def process(self, tup):
if tup.stream == 'control':
action, user = tup.values
if action == 'connect':
self.connections[user] = int(time.time())
elif action == 'disconnect':
del self.connections[user]
elif tup.stream == 'default':
user, paths = tup.values
if user in self.connections:
log('[MorelikethisBolt] Incoming: %s' % user)
recommendations = self.recommend(paths)
paths = list(set(paths))[:10]
emit([user, paths, recommendations])
def process(self, tup):
sensor_id, temperature = tup.values
new_ave = temperature
count = 1
# recompute running average
if sensor_id in self.running_averages:
ave, count = self.running_averages[sensor_id]
new_ave = (ave * count + temperature) / (count + 1)
count = count + 1
# store the running average
self.running_averages[sensor_id] = [new_ave, count]
storm.log("For sensor %s the new average is %f recorded over %d readings" % (sensor_id, new_ave, count))
storm.emit([sensor_id, new_ave], anchors=[tup])
def process(self, tup):
if tup.stream == 'control':
action, user = tup.values
if action == 'connect':
self.connections[user] = int(time.time())
elif action == 'disconnect':
del self.connections[user]
elif tup.stream == 'default':
user, paths = tup.values
if user in self.connections:
log('[RecommendationBolt] Incoming: %s' % user)
vector = self.expand(paths)
if self.folding:
self.fold(user, vector)
recommendations = self.recommend(vector).tolist()[:10]
paths = list(set(paths))[:10]
emit([user, paths, recommendations])
def process(self, tup):
if tup.stream == 'control':
action, user = tup.values
if action == 'connect':
self.connections[user] = int(time.time())
elif action == 'disconnect':
del self.connections[user]
elif tup.stream == 'default':
user, paths = tup.values
if user in self.connections:
log('[RecommendationBolt] Incoming: %s' % user)
vector = self.expand(paths)
if self.folding:
self.fold(user, vector)
recommendations = self.recommend(vector).tolist()[:10]
paths = list(set(paths))[:10]
emit([user, paths, recommendations])
def delete_metric_alarm(self, alarmkey):
"""
메모리 및 DB에서 알람을 삭제한다.
alarmkey:
alarmkey should be UUID
"""
try:
alarm = self.alarms.pop(alarmkey)
except KeyError:
storm.log("alarmkey %s doesn't exist" % alarmkey)
return
self.cass.delete_metric_alarm(alarmkey)
self.alarm_history_delete(alarmkey, alarm)
storm.log("delete alarm %s for metric %s" % (str(alarmkey), self.metric_key))
self.set_max_start_period(self.alarms)
def process(self, tup):
title, url, text_content = tup.values
# print(title, url, type(text_content))
new_counts = self.count_vect.transform([text_content])
new_tfidf = self.tfidf_transformer.transform(new_counts)
new_dmatrix = xgb.DMatrix(new_tfidf)
predicted = self.xgb.predict(new_dmatrix)
self._prediction = self.target_names[predicted.argmax()]
storm.log(title + url + " is : ", self._prediction)
emit_tup = [title, url, self._prediction]
storm.log("Emit %s" % emit_tup)
storm.emit(emit_tup)
def process(self, tup):
start_time = int(round(time.time() * 1000))
frame = base64.b64decode(tup.values[0])
original_time = tup.values[2]
sensorId = tup.values[1]
im = np.frombuffer(frame, count=frame_size_bytes, dtype=np.uint8)
im = im.reshape((frame_size[0], frame_size[1], 3))
frame = Tracking.ImageFrame(None, im)
targets = self.tracking.do(frame)
end_time = int(round(time.time() * 1000))
storm.log("Tracking Processing time: " + str(end_time - start_time))
targets_message = []
for target in targets:
targets_message.append({'found': 1, 'x': str(target.x), 'y': str(target.y), 'h': str(target.h), 'w': str(target.w)})
storm.log("sensor id: " + str(sensorId))
storm.emit([targets_message, sensorId, original_time])
def set_max_start_period(self, alarms):
msp = self.MAX_START_PERIOD
try:
self.MAX_START_PERIOD = (
max(v.get("period") for v in alarms.itervalues()) if alarms else FLAGS.get("max_start_period")
)
self.MIN_START_PERIOD = min(v.get("period") for v in alarms.itervalues()) if alarms else 0
except AttributeError:
msg = "alarm is not found in alarms."
self.log(msg)
return False
msg = "MAX_START_PERIOD is changed %s -> %s"
storm.log(msg % (str(msp), self.MAX_START_PERIOD))
if msp < self.MAX_START_PERIOD:
self.df = self.load_statistics()
elif msp > self.MAX_START_PERIOD:
self._reindex()
def process(self, tuple):
storm.log(tuple.component + ': ' + str(tuple.values))
#if filler arrives, add it to the window
if tuple.component == self.filler_component:
key = tuple.values[0]
payload = zlib.compress(tuple.values[1], self.compression_level)
if key in self.payloads: self.payloads[key].append(payload)
else: self.payloads[key] = [payload]
storm.log('Filler ' + key + ': ' + str(len(self.payloads[key])))
#otherwise check for match enrichment keys and emit the enriched event
else:
key = tuple.values[0]
storm.log('Non-filler: ' + str(tuple.values))
if key in self.payloads:
out = []
for payload in self.payloads[key]:
out.append(zlib.decompress(payload))
storm.log('Emitting ' + str(len(out)) + ' payloads for ' + key)
storm.emit([tuple.values, out])
def get_har_with_image(self, url, selectors=None):
"""Gets the raw HAR output from PhantomJs with rendered image(s)."""
tmp = "%s/%s.json" % ("/var/tmp", str(random.randint(0, 1000000)))
command = ["phantomjs", "netsniff-rasterize.js", url, tmp]
if selectors is not None:
command += selectors
for item in command:
storm.log("item: "+str(item))
har = Popen(command, stdout=PIPE, stderr=PIPE)
stdout, stderr = har.communicate()
if not os.path.exists(tmp):
storm.log("FAILED!")
return "FAIL"
with open(tmp, "r") as i:
output = i.read()
os.remove(tmp)
storm.log("GOT "+output)
return output
def log(self, msg):
storm.log(msg)
def fail(self, id):
log("emitting " + self.pending[id] + " on fail")
emit([self.pending[id]], id=id)
def nextTuple(self):
data = getData()
# data = ast.literal_eval(data)
log(data)
emit([data])
def nextTuple(self):
data = getData()
log(data)
emit([data])
def ack(self, cnt_id):
tup, retries = self.buffer[cnt_id]
del self.buffer[cnt_id]
log('[ZonAPISpout] Acknowledging content id-%s.' % cnt_id)
def initialize(self, stormconf, context):
try:
something_component_need = stormconf.get("something_component_need")
except Exception as e:
storm.log('init shell bolt error: %s'%traceback.format_exc())
