def fetch_tweets(): while True: try: repl = req.get(url, timeout=timeout).content tweet = json.loads(repl) try: loc = tweet["coordinates"]["coordinates"] loc.reverse() p = Pulse( typename = "twitter", title = tweet["user"]["name"], data = tweet, # stupid twitter returns reversed geoloc location = loc, timestamp_added = datetime.datetime.now(), timestamp_created = parse(tweet["created_at"]) ) p.save() print loc except: pass except Timeout: print "Się utło.." break
def get_object_list(self, request): """Method getting a list of objects. Example filters: * getting from within box: ``?format=json&location__within_box=[[1, 50], [60, 56]]`` """ filters = self.extract_filters(request) all_objs = Pulse.objects().filter(typename='twitter') return self.apply_filters(all_objs, filters)
def event_run(self): keywords = ['warszawa', 'warsaw', 'warszawie', 'waw', 'hackwaw', 'stodola', 'hydrozagadka', 'palladium', 'proxima', 'fabryka', 'trzciny', 'sen', 'pszczoly', 'plan', 'arkadia', 'tarasy', 'sala', 'kongresowa', 'dom', 'kultury', 'festiwal', 'koncert', 'noc', 'impreza', 'party', 'mecz', 'manifestacja', 'prezentacja', 'wystawa', 'wystawy', 'projekcja', 'dyskusja', 'debata', 'spotkanie', 'w', 'na', 'o', 'we', 'i', 'a', '@'] event_iter = self.crawl_events((52.228641, 21.015558), 15000, custom_keywords=keywords) crawl_time = datetime.now() sys.stdout.write('crawling events\n') sys.stdout.flush() for evd in event_iter: fbev = evd['model'] if not fbev.event_added: ev = Event() ev.title = fbev.name ev.typename = 'facebook_event' ev.location = fbev.location ev.start_time = fbev.start_time ev.end_time = fbev.end_time ev.description = fbev.description ev.image_url = None ev.url = 'http://www.facebook.com/event/%s/' % fbev.fbid ev.save() fbev.event_added = True fbev.event = ev fbev.save() msg = 'added %s, id=%s' % (ev.title, fbev.fbid) sys.stdout.write('\n%s\n' % msg) sys.stdout.flush() else: sys.stdout.write('.') sys.stdout.flush() sys.stdout.write('\n') sys.stdout.flush() sys.stdout.write('crawling event attendance\n') sys.stdout.flush() q = FBEvent.objects(attendance_pulses_added=False) for fbev in q: event_url = 'http://www.facebook.com/event/%s/' % fbev.fbid ev = fbev.event if ev is None: ev_q = Event.objects(url=event_url) if len(ev_q) == 1: ev = ev_q[0] if ev: result = self.graph.get_event_members(fbev.fbid) if 'attending' in result: for fbid in result['attending']: p = Pulse() p.title = fbev.name p.typename = 'facebook_attendance' p.event = ev p.data = {'uid': fbid, 'eid': fbev.fbid} p.timestamp_added = datetime.now() p.timestamp_created = fbev.start_time p.location = fbev.location p.url = event_url p.save() sys.stdout.write('.') sys.stdout.flush() fbev.attendance_pulses_added = True if fbev.event is None: fbev.event = ev fbev.save() sys.stdout.write('\n') sys.stdout.flush()