def fetch_tweets():
while True:
try:
repl = req.get(url, timeout=timeout).content
tweet = json.loads(repl)
try:
loc = tweet["coordinates"]["coordinates"]
loc.reverse()
p = Pulse(
typename = "twitter",
title = tweet["user"]["name"],
data = tweet,
# stupid twitter returns reversed geoloc
location = loc,
timestamp_added = datetime.datetime.now(),
timestamp_created = parse(tweet["created_at"])
)
p.save()
print loc
except:
pass
except Timeout:
print "Się utło.."
break
def get_object_list(self, request):
"""Method getting a list of objects.
Example filters:
* getting from within box:
``?format=json&location__within_box=[[1, 50], [60, 56]]``
"""
filters = self.extract_filters(request)
all_objs = Pulse.objects().filter(typename='twitter')
return self.apply_filters(all_objs, filters)
def event_run(self):
keywords = ['warszawa', 'warsaw', 'warszawie', 'waw', 'hackwaw',
'stodola', 'hydrozagadka', 'palladium', 'proxima',
'fabryka', 'trzciny', 'sen', 'pszczoly', 'plan', 'arkadia', 'tarasy',
'sala', 'kongresowa', 'dom', 'kultury',
'festiwal', 'koncert', 'noc', 'impreza', 'party', 'mecz', 'manifestacja',
'prezentacja', 'wystawa', 'wystawy', 'projekcja', 'dyskusja', 'debata', 'spotkanie',
'w', 'na', 'o', 'we', 'i', 'a', '@']
event_iter = self.crawl_events((52.228641, 21.015558), 15000, custom_keywords=keywords)
crawl_time = datetime.now()
sys.stdout.write('crawling events\n')
sys.stdout.flush()
for evd in event_iter:
fbev = evd['model']
if not fbev.event_added:
ev = Event()
ev.title = fbev.name
ev.typename = 'facebook_event'
ev.location = fbev.location
ev.start_time = fbev.start_time
ev.end_time = fbev.end_time
ev.description = fbev.description
ev.image_url = None
ev.url = 'http://www.facebook.com/event/%s/' % fbev.fbid
ev.save()
fbev.event_added = True
fbev.event = ev
fbev.save()
msg = 'added %s, id=%s' % (ev.title, fbev.fbid)
sys.stdout.write('\n%s\n' % msg)
sys.stdout.flush()
else:
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('\n')
sys.stdout.flush()
sys.stdout.write('crawling event attendance\n')
sys.stdout.flush()
q = FBEvent.objects(attendance_pulses_added=False)
for fbev in q:
event_url = 'http://www.facebook.com/event/%s/' % fbev.fbid
ev = fbev.event
if ev is None:
ev_q = Event.objects(url=event_url)
if len(ev_q) == 1:
ev = ev_q[0]
if ev:
result = self.graph.get_event_members(fbev.fbid)
if 'attending' in result:
for fbid in result['attending']:
p = Pulse()
p.title = fbev.name
p.typename = 'facebook_attendance'
p.event = ev
p.data = {'uid': fbid, 'eid': fbev.fbid}
p.timestamp_added = datetime.now()
p.timestamp_created = fbev.start_time
p.location = fbev.location
p.url = event_url
p.save()
sys.stdout.write('.')
sys.stdout.flush()
fbev.attendance_pulses_added = True
if fbev.event is None:
fbev.event = ev
fbev.save()
sys.stdout.write('\n')
sys.stdout.flush()
