def run():
from revent import ReventClient
rc = ReventClient('scraper_timer')
with open('./sites.txt','r') as fh:
for line in fh.readlines():
line = line.strip()
print line
rc.fire('timer_scrape_blog', dict(blog_url=line))
def __init__(self,
app_name,
config,
base_context_mapping,
in_event,
handler,
out_event=None):
self.config = config
self.app_name = app_name
self.in_event = in_event
self.handler = handler
self.out_event = out_event
self.context = bubbles.build_context(base_context_mapping)
self.ReventMessage = ReventMessage
print 'context mapping: %s' % self.context.mapping
# sanity checks
assert in_event, "Must provide in_event"
assert handler, "Must provide handler"
# subscribe to our in_event
self.channel = '%s-%s-%s' % (app_name, in_event, self.handler.__name__)
self.rc = ReventClient(self.channel,
in_event,
verified=10,
**self.config.get('revent'))
# update the context to include the revent client and
# introspect module
self.context.add('revent', self.rc)
self.context.add('introspect', rc_introspect)
# create a connection to redis
self.redis = Redis(**self.config.get('redis'))
# add to context
self.context.add('redis', self.redis)
# make our redis namespace the same as our channel
self.redis_ns = 'App-%s' % self.app_name
# update the context to include all the underscore methods
# (but not dunderscore)
for name, value in getmembers(self):
if self.__include_in_context(name, value):
self.context.add(name, value)
# add config to context
self.context.add('config', self.config)
def __init__(self, app_name, config, base_context_mapping,
in_event, handler, out_event=None):
self.config = config
self.app_name = app_name
self.in_event = in_event
self.handler = handler
self.out_event = out_event
self.context = bubbles.build_context(base_context_mapping)
self.ReventMessage = ReventMessage
print 'context mapping: %s' % self.context.mapping
# sanity checks
assert in_event, "Must provide in_event"
assert handler, "Must provide handler"
# subscribe to our in_event
self.channel = '%s-%s-%s' % (app_name, in_event, self.handler.__name__)
self.rc = ReventClient(self.channel, in_event, verified=10,
**self.config.get('revent'))
# update the context to include the revent client and
# introspect module
self.context.add('revent', self.rc)
self.context.add('introspect', rc_introspect)
# create a connection to redis
self.redis = Redis(**self.config.get('redis'))
# add to context
self.context.add('redis', self.redis)
# make our redis namespace the same as our channel
self.redis_ns = 'App-%s' % self.app_name
# update the context to include all the underscore methods
# (but not dunderscore)
for name, value in getmembers(self):
if self.__include_in_context(name, value):
self.context.add(name, value)
# add config to context
self.context.add('config', self.config)
class AppHandler(object):
def __init__(self, app_name, config, base_context_mapping,
in_event, handler, out_event=None):
self.config = config
self.app_name = app_name
self.in_event = in_event
self.handler = handler
self.out_event = out_event
self.context = bubbles.build_context(base_context_mapping)
self.ReventMessage = ReventMessage
print 'context mapping: %s' % self.context.mapping
# sanity checks
assert in_event, "Must provide in_event"
assert handler, "Must provide handler"
# subscribe to our in_event
self.channel = '%s-%s-%s' % (app_name, in_event, self.handler.__name__)
self.rc = ReventClient(self.channel, in_event, verified=10,
**self.config.get('revent'))
# update the context to include the revent client and
# introspect module
self.context.add('revent', self.rc)
self.context.add('introspect', rc_introspect)
# create a connection to redis
self.redis = Redis(**self.config.get('redis'))
# add to context
self.context.add('redis', self.redis)
# make our redis namespace the same as our channel
self.redis_ns = 'App-%s' % self.app_name
# update the context to include all the underscore methods
# (but not dunderscore)
for name, value in getmembers(self):
if self.__include_in_context(name, value):
self.context.add(name, value)
# add config to context
self.context.add('config', self.config)
@staticmethod
def __include_in_context(name, value):
include = name.startswith('_') and not name.startswith('__')
return include
# helper methods for accessing natives
def _dict(self, name, default=None):
name = str(name) # redis demands ascii
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.Dict(*args)
def _sequence(self, name, default=None):
name = str(name)
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.Sequence(*args)
def _zset(self, name, default=None):
name = str(name)
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.ZSet(*args)
def _list(self, name, default=None):
name = str(name)
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.List(*args)
def _set(self, name, default=None):
name = str(name)
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.Set(*args)
def _string(self, name, default=None):
name = str(name)
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.Primitive(*args)
def _signal(self, name):
name = str(name)
return Signal(self.redis,
'%s:%s:signal' % (self.redis_ns, name))
def _stop(self):
print 'Stopping handler'
raise StopIteration
def __repr__(self):
return '<AppHandler %s=>%s=>%s>' % (self.in_event,
self.handler.__name__,
self.out_event or '')
def cycle(self, block=False, timeout=1):
# grab up our event
event = self.rc.read(block=block, timeout=timeout)
if event:
# wrap the handler in the current context
wrapped_handler = self._wrap_handler(event)
# call our handler
try:
print '[H] %s [E] %s' % (self.handler.__name__, event)
for result in wrapped_handler():
# see if this results calls for another event to be fired
result_event = self._build_result_event(event, result)
print '[%s] [%s] %s => %s' % (event, self, str(result), str(result_event))
# result event is event, event_data
if result_event:
self.rc.fire(*result_event)
except Exception:
print 'Handler Exception: %s %s' % (self.handler.__name__, event)
raise
# flush the prints
sys.stdout.flush()
# let them know we're done handling the event
self.rc.confirm(event)
def _wrap_handler(self, event):
"""
returns the handler wrapped in a context which
includes the current event's data
"""
context = self.context.copy()
context.update(
event_data=event.data,
event_name=event.event,
event_id=event.id,
event=event
)
context.update(**event.data)
return context.create_partial(self.handler)
def _build_result_event(self, event, result):
# if they didn't define an out event than we aren't putting
# off events even if we get a result
if not self.out_event:
return None
# if the result if an event, just fire it's info
if isinstance(result, self.ReventMessage):
return result.event, result.data
# if the reuslt is a true or false than it's a filter
# a false means don't re-fire the event, True means re-fire
# if we have an out event set than we'll fire the input event's
# data w/ our out event name
if result is True:
return self.out_event, event.data
# if the result if false than we're filting the message
if result is False:
return None
# if the reuslt is a dictionary than we're going to use that
# dict as the resulting event's data
if isinstance(result, dict):
return self.out_event, result
# if it's a two item tuple it's either new event
# or update to source event's data and out event
if isinstance(result, tuple) and len(result) == 2 \
and isinstance(result[0], (str, unicode)):
# if the second value is a dict, than it's a new event
# and the first value is the new event's event
if isinstance(result[1], dict):
return tuple(result)
else:
# a k/v pair to set in the previous
# events data (k,v)
# update the data in place, no one else should touch!
event.data[result[0]] = result[1]
return self.out_event, event.data
elif isinstance(result, tuple):
# if it's a tuple which is longer than 2 than it should be
# full of sub tuples, each sub tuble contains a key/value
# to be set
for k, v in result:
event.data[k] = v
return self.out_event, event.data
# if it's anything else we're going to update the source event's
# data to include these results and use resuling data as new
# events data
event_data = event.data.copy()
previous_results = event_data.setdefault('results', [])
if isinstance(result, (list, tuple)):
previous_results.extend(result)
else:
previous_results.append(result)
return self.out_event, event_data
class AppHandler(object):
def __init__(self,
app_name,
config,
base_context_mapping,
in_event,
handler,
out_event=None):
self.config = config
self.app_name = app_name
self.in_event = in_event
self.handler = handler
self.out_event = out_event
self.context = bubbles.build_context(base_context_mapping)
self.ReventMessage = ReventMessage
print 'context mapping: %s' % self.context.mapping
# sanity checks
assert in_event, "Must provide in_event"
assert handler, "Must provide handler"
# subscribe to our in_event
self.channel = '%s-%s-%s' % (app_name, in_event, self.handler.__name__)
self.rc = ReventClient(self.channel,
in_event,
verified=10,
**self.config.get('revent'))
# update the context to include the revent client and
# introspect module
self.context.add('revent', self.rc)
self.context.add('introspect', rc_introspect)
# create a connection to redis
self.redis = Redis(**self.config.get('redis'))
# add to context
self.context.add('redis', self.redis)
# make our redis namespace the same as our channel
self.redis_ns = 'App-%s' % self.app_name
# update the context to include all the underscore methods
# (but not dunderscore)
for name, value in getmembers(self):
if self.__include_in_context(name, value):
self.context.add(name, value)
# add config to context
self.context.add('config', self.config)
@staticmethod
def __include_in_context(name, value):
include = name.startswith('_') and not name.startswith('__')
return include
# helper methods for accessing natives
def _dict(self, name, default=None):
name = str(name) # redis demands ascii
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.Dict(*args)
def _sequence(self, name, default=None):
name = str(name)
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.Sequence(*args)
def _zset(self, name, default=None):
name = str(name)
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.ZSet(*args)
def _list(self, name, default=None):
name = str(name)
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.List(*args)
def _set(self, name, default=None):
name = str(name)
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.Set(*args)
def _string(self, name, default=None):
name = str(name)
args = [self.redis, '%s:%s' % (self.redis_ns, name)]
if default is not None:
args.append(default)
return rn.Primitive(*args)
def _signal(self, name):
name = str(name)
return Signal(self.redis, '%s:%s:signal' % (self.redis_ns, name))
def _stop(self):
print 'Stopping handler'
raise StopIteration
def __repr__(self):
return '<AppHandler %s=>%s=>%s>' % (
self.in_event, self.handler.__name__, self.out_event or '')
def cycle(self, block=False, timeout=1):
# grab up our event
event = self.rc.read(block=block, timeout=timeout)
if event:
# wrap the handler in the current context
wrapped_handler = self._wrap_handler(event)
# call our handler
try:
print '[H] %s [E] %s' % (self.handler.__name__, event)
for result in wrapped_handler():
# see if this results calls for another event to be fired
result_event = self._build_result_event(event, result)
print '[%s] [%s] %s => %s' % (event, self, str(result),
str(result_event))
# result event is event, event_data
if result_event:
self.rc.fire(*result_event)
except Exception:
print 'Handler Exception: %s %s' % (self.handler.__name__,
event)
raise
# flush the prints
sys.stdout.flush()
# let them know we're done handling the event
self.rc.confirm(event)
def _wrap_handler(self, event):
"""
returns the handler wrapped in a context which
includes the current event's data
"""
context = self.context.copy()
context.update(event_data=event.data,
event_name=event.event,
event_id=event.id,
event=event)
context.update(**event.data)
return context.create_partial(self.handler)
def _build_result_event(self, event, result):
# if they didn't define an out event than we aren't putting
# off events even if we get a result
if not self.out_event:
return None
# if the result if an event, just fire it's info
if isinstance(result, self.ReventMessage):
return result.event, result.data
# if the reuslt is a true or false than it's a filter
# a false means don't re-fire the event, True means re-fire
# if we have an out event set than we'll fire the input event's
# data w/ our out event name
if result is True:
return self.out_event, event.data
# if the result if false than we're filting the message
if result is False:
return None
# if the reuslt is a dictionary than we're going to use that
# dict as the resulting event's data
if isinstance(result, dict):
return self.out_event, result
# if it's a two item tuple it's either new event
# or update to source event's data and out event
if isinstance(result, tuple) and len(result) == 2 \
and isinstance(result[0], (str, unicode)):
# if the second value is a dict, than it's a new event
# and the first value is the new event's event
if isinstance(result[1], dict):
return tuple(result)
else:
# a k/v pair to set in the previous
# events data (k,v)
# update the data in place, no one else should touch!
event.data[result[0]] = result[1]
return self.out_event, event.data
elif isinstance(result, tuple):
# if it's a tuple which is longer than 2 than it should be
# full of sub tuples, each sub tuble contains a key/value
# to be set
for k, v in result:
event.data[k] = v
return self.out_event, event.data
# if it's anything else we're going to update the source event's
# data to include these results and use resuling data as new
# events data
event_data = event.data.copy()
previous_results = event_data.setdefault('results', [])
if isinstance(result, (list, tuple)):
previous_results.extend(result)
else:
previous_results.append(result)
return self.out_event, event_data
