# select the database and collection based off config try: db = client["twitter"] # config.get("mongo", "database")] col = db["ptweets"] # ["geotweets"]#config.get("mongo", "collection")] except NoOptionError: logging.critical( "Cannot connect to MongoDB database and collection. Config incorrect?") sys.exit() # get the tweet cursor logging.info("Getting tweets...") cursor = col.find(no_cursor_timeout=True).sort('timestamp', 1) count = 0 tf = EventDetection( 'centre', 'timestamp', popmaploc='D:\ds\population\glds15ag.asc') # 'geo.coordinates') try: for doc in cursor: tf.process_tweet(doc) count += 1 if count % 100 == 0: logging.info( "Proc tweet %i by %s" % (count, doc['timestamp'])) # doc['user']['screen_name'])) logging.info(tf) except Exception as e: print(e) raise e
# connect to the MongoDB logging.info("Connecting to MongoDB...") client = MongoClient(config.get("mongo", "address"), config.getint("mongo", "port")) logging.info("Connected to MongoDB") # select the database and collection based off config try: db = client[config.get("mongo", "database")] col = db["geotweets"] #config.get("mongo", "collection")] except NoOptionError: logging.critical( "Cannot connect to MongoDB database and collection. Config incorrect?") sys.exit() # get the tweet cursor logging.info("Getting tweets...") cursor = db.tweets.find({'locinf.mi.test': {'$ne': None}}) count = 0 tf = EventDetection('geo.coordinates') ani = plotevents(tf) for doc in cursor: print("Proc tweet by", doc['user']['screen_name']) tf.process_tweet(doc) #if count % 50 == 0: # next(ani) count += 1
config = twieds.setup("logs/ed_test.log", "settings/locinf.ini", logging.DEBUG) # connect to the MongoDB logging.info("Connecting to MongoDB...") client = MongoClient(config.get("mongo", "address"), config.getint("mongo", "port")) logging.info("Connected to MongoDB") # select the database and collection based off config try: db = client[config.get("mongo", "database")] col = db["geotweets"]#config.get("mongo", "collection")] except NoOptionError: logging.critical("Cannot connect to MongoDB database and collection. Config incorrect?") sys.exit() # get the tweet cursor logging.info("Getting tweets...") cursor = db.tweets.find({'locinf.mi.test': {'$ne': None}}) count = 0 tf = EventDetection('geo.coordinates') ani = plotevents(tf) for doc in cursor: print("Proc tweet by", doc['user']['screen_name']) tf.process_tweet(doc) #if count % 50 == 0: # next(ani) count += 1
try: db = client["twitter"] col = db["ptweets"] except NoOptionError: logging.critical("Cannot connect to MongoDB database and collection. Config incorrect?") sys.exit() with open('D:/ds/code/workbooks/ukpoly.pkl', 'rb') as file: mp = pickle.load(file) # get the tweet cursor logging.info("Getting tweets...") cursor = col.find(no_cursor_timeout=True).sort('timestamp', 1) count = 0 tf = EventDetection('centre', 'timestamp', popmaploc='D:\ds\population\glds15ag.asc') try: for doc in cursor: if doc['centre'] is None or not mp.isInside(doc['centre'][0], doc['centre'][1]): continue tf.process_tweet(doc) count += 1 if count % 100 == 0: logging.info("Proc tweet %i by %s" % (count, doc['timestamp'])) logging.info(tf) except Exception as e: print(e) pass