import argparse
import logging
import pickle
import sys
from configparser import NoOptionError
from pymongo import MongoClient
from twied.eventec.eventdetection import EventDetection
import twieds
parser = argparse.ArgumentParser(description="Run the event detection")
parser.add_argument('output', help='the output file to write to')
args = parser.parse_args()
config = twieds.setup(None, "settings/locinf.ini", logging.INFO)
# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(config.get("mongo", "address"),
config.getint("mongo", "port"))
logging.info("Connected to MongoDB")
# select the database and collection based off config
try:
db = client["twitter"] # config.get("mongo", "database")]
col = db["ptweets"] # ["geotweets"]#config.get("mongo", "collection")]
except NoOptionError:
logging.critical(
"Cannot connect to MongoDB database and collection. Config incorrect?")
sys.exit()
import matplotlib.pyplot as plt
import networkx as nx
import pymongo
from mpl_toolkits.basemap import Basemap
from pymongo import MongoClient
from twied.labelprop.distance import geometric_mean
from twied.labelprop.inference import InferSL
import twieds
visited_users = {}
colors = {}
# must run this as a script
if __name__ == "__main__":
config = twieds.setup("logs/labelprop.log", "settings/locinf.ini")
# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(config.get("mongo", "address"), config.getint("mongo", "port"))
logging.info("Connected to MongoDB")
# select the database and collection based off config
try:
db = client[config.get("mongo", "database")]
collection = db["users"]
collection.create_index([('user.id', pymongo.ASCENDING)], unique=True)
collection.create_index([('user.screen_name', pymongo.ASCENDING)])
except NoOptionError:
logging.critical("Cannot connect to MongoDB database and collection. Config incorrect?")
sys.exit()
#!/usr/bin/python
"""
File to build an animation.
"""
import logging
import sys
from configparser import NoOptionError
from datetime import timedelta
from pymongo import MongoClient
import twieds
from polyani import plotevents_count
config = twieds.setup("logs/ed_test.log", "settings/locinf.ini", logging.DEBUG)
# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(config.get("mongo", "address"), config.getint("mongo", "port"))
logging.info("Connected to MongoDB")
# select the database and collection based off config
try:
db = client[config.get("mongo", "database")]
col = db["inftweets"]
except NoOptionError:
logging.critical("Cannot connect to MongoDB database and collection. Config incorrect?")
sys.exit()
# get the tweet cursor
logging.info("Getting tweets...")
self.tweetval.insert(1.0, fordoc)
self.output['text'] = "Got tweet successfully."
except Exception as e:
self.output['text'] = "ERROR in loading random doc: " + str(type(e).__name__)
raise e
def infer_tweet(self):
try:
doc = eval(self.tweetval.get(1.0, tk.END))
self.inferfunc(self, doc, self.inds)
except Exception as e:
self.output['text'] = "ERROR in parsing doc: " + str(type(e).__name__)
raise e
config = twieds.setup("logs/locinf.log", "settings/locinf.ini")
def add_ind(task):
ind = task[0]
field = task[1]
f_field = field[:50].encode('utf8') if isinstance(field, str) else field
start = time.clock()
logging.info("%10s <- Value: %-50s" % (type(ind).__name__[:-9], f_field))
result = ind.get_loc(field)
logging.info("%10s -> Took %.2f seconds. (returned: %i polys)" % (
type(ind).__name__[:-9], (time.clock() - start), len(result))
)
return result
#!/usr/bin/python
"""
Event detection test script.
"""
import logging
import sys
from configparser import NoOptionError
from pymongo import MongoClient
from twied.eventec.eventdetection import EventDetection
import twieds
from polyani import plotevents
config = twieds.setup("logs/ed_test.log", "settings/locinf.ini", logging.DEBUG)
# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(config.get("mongo", "address"),
config.getint("mongo", "port"))
logging.info("Connected to MongoDB")
# select the database and collection based off config
try:
db = client[config.get("mongo", "database")]
col = db["geotweets"] #config.get("mongo", "collection")]
except NoOptionError:
logging.critical(
"Cannot connect to MongoDB database and collection. Config incorrect?")
sys.exit()
import argparse
import logging
import pickle
import sys
from configparser import NoOptionError
from pymongo import MongoClient
from twied.eventec.eventdetection import EventDetection
import twieds
parser = argparse.ArgumentParser(description="Run the event detection")
parser.add_argument('output', help='the output file to write to')
args = parser.parse_args()
config = twieds.setup(None, "settings/locinf.ini", logging.INFO)
# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(config.get("mongo", "address"), config.getint("mongo", "port"))
logging.info("Connected to MongoDB")
# select the database and collection based off config
try:
db = client["twitter"]
col = db["ptweets"]
except NoOptionError:
logging.critical("Cannot connect to MongoDB database and collection. Config incorrect?")
sys.exit()
with open('D:/ds/code/workbooks/ukpoly.pkl', 'rb') as file:
#!/usr/bin/python
"""
For allocating ID's to each tweet in the database between a number.
"""
import argparse
import logging
import sys
from configparser import NoOptionError
from pymongo import MongoClient
import twieds
config = twieds.setup("logs/mi_alloc.log", "settings/locinf.ini")
parser = argparse.ArgumentParser(description="Allocates each tweet in a collection with a number in specified range.")
parser.add_argument('-field', help='the fieldname to store the inferred location in', required=True)
parser.add_argument('-num', type=int, help='the range [0, X] of id\'s to allocate', required=True)
parser.add_argument('-db', help='address of the database', default=config.get('mongo', 'address'))
parser.add_argument('-port', type=int, help='port of the database', default=config.get('mongo', 'port'))
parser.add_argument('-coll', help='database and collection name seperated by a dot (eg, twitter.tweets)', required=True)
args = parser.parse_args()
# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(args.db, args.port)
logging.info("Connected to MongoDB.")
# select the database and collection based off config
try:
db_n, col_n = args.coll.split(".")