Пример #1
0
import argparse
import logging
import pickle
import sys
from configparser import NoOptionError

from pymongo import MongoClient

from twied.eventec.eventdetection import EventDetection
import twieds

parser = argparse.ArgumentParser(description="Run the event detection")
parser.add_argument('output', help='the output file to write to')
args = parser.parse_args()

config = twieds.setup(None, "settings/locinf.ini", logging.INFO)

# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(config.get("mongo", "address"),
                     config.getint("mongo", "port"))
logging.info("Connected to MongoDB")

# select the database and collection based off config
try:
    db = client["twitter"]  # config.get("mongo", "database")]
    col = db["ptweets"]  # ["geotweets"]#config.get("mongo", "collection")]
except NoOptionError:
    logging.critical(
        "Cannot connect to MongoDB database and collection. Config incorrect?")
    sys.exit()
Пример #2
0
import matplotlib.pyplot as plt
import networkx as nx
import pymongo
from mpl_toolkits.basemap import Basemap
from pymongo import MongoClient

from twied.labelprop.distance import geometric_mean
from twied.labelprop.inference import InferSL
import twieds

visited_users = {}
colors = {}

# must run this as a script
if __name__ == "__main__":
    config = twieds.setup("logs/labelprop.log", "settings/locinf.ini")

    # connect to the MongoDB
    logging.info("Connecting to MongoDB...")
    client = MongoClient(config.get("mongo", "address"), config.getint("mongo", "port"))
    logging.info("Connected to MongoDB")

    # select the database and collection based off config
    try:
        db = client[config.get("mongo", "database")]
        collection = db["users"]
        collection.create_index([('user.id', pymongo.ASCENDING)], unique=True)
        collection.create_index([('user.screen_name', pymongo.ASCENDING)])
    except NoOptionError:
        logging.critical("Cannot connect to MongoDB database and collection. Config incorrect?")
        sys.exit()
Пример #3
0
#!/usr/bin/python
"""
File to build an animation.
"""
import logging
import sys
from configparser import NoOptionError
from datetime import timedelta

from pymongo import MongoClient

import twieds
from polyani import plotevents_count

config = twieds.setup("logs/ed_test.log", "settings/locinf.ini", logging.DEBUG)

# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(config.get("mongo", "address"), config.getint("mongo", "port"))
logging.info("Connected to MongoDB")

# select the database and collection based off config
try:
    db = client[config.get("mongo", "database")]
    col = db["inftweets"]
except NoOptionError:
    logging.critical("Cannot connect to MongoDB database and collection. Config incorrect?")
    sys.exit()

# get the tweet cursor
logging.info("Getting tweets...")
Пример #4
0
            self.tweetval.insert(1.0, fordoc)
            self.output['text'] = "Got tweet successfully."
        except Exception as e:
            self.output['text'] = "ERROR in loading random doc: " + str(type(e).__name__)
            raise e

    def infer_tweet(self):
        try:
            doc = eval(self.tweetval.get(1.0, tk.END))

            self.inferfunc(self, doc, self.inds)
        except Exception as e:
            self.output['text'] = "ERROR in parsing doc: " + str(type(e).__name__)
            raise e

config = twieds.setup("logs/locinf.log", "settings/locinf.ini")


def add_ind(task):
    ind = task[0]
    field = task[1]

    f_field = field[:50].encode('utf8') if isinstance(field, str) else field

    start = time.clock()
    logging.info("%10s <- Value: %-50s" % (type(ind).__name__[:-9], f_field))
    result = ind.get_loc(field)
    logging.info("%10s -> Took %.2f seconds. (returned: %i polys)" % (
        type(ind).__name__[:-9], (time.clock() - start), len(result))
    )
    return result
Пример #5
0
#!/usr/bin/python
"""
Event detection test script.
"""
import logging
import sys
from configparser import NoOptionError

from pymongo import MongoClient

from twied.eventec.eventdetection import EventDetection
import twieds

from polyani import plotevents

config = twieds.setup("logs/ed_test.log", "settings/locinf.ini", logging.DEBUG)

# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(config.get("mongo", "address"),
                     config.getint("mongo", "port"))
logging.info("Connected to MongoDB")

# select the database and collection based off config
try:
    db = client[config.get("mongo", "database")]
    col = db["geotweets"]  #config.get("mongo", "collection")]
except NoOptionError:
    logging.critical(
        "Cannot connect to MongoDB database and collection. Config incorrect?")
    sys.exit()
Пример #6
0
import argparse
import logging
import pickle
import sys
from configparser import NoOptionError

from pymongo import MongoClient

from twied.eventec.eventdetection import EventDetection
import twieds

parser = argparse.ArgumentParser(description="Run the event detection")
parser.add_argument('output', help='the output file to write to')
args = parser.parse_args()

config = twieds.setup(None, "settings/locinf.ini", logging.INFO)

# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(config.get("mongo", "address"), config.getint("mongo", "port"))
logging.info("Connected to MongoDB")

# select the database and collection based off config
try:
    db = client["twitter"]
    col = db["ptweets"]
except NoOptionError:
    logging.critical("Cannot connect to MongoDB database and collection. Config incorrect?")
    sys.exit()

with open('D:/ds/code/workbooks/ukpoly.pkl', 'rb') as file:
Пример #7
0
#!/usr/bin/python
"""
For allocating ID's to each tweet in the database between a number.
"""
import argparse
import logging
import sys
from configparser import NoOptionError

from pymongo import MongoClient

import twieds

config = twieds.setup("logs/mi_alloc.log", "settings/locinf.ini")

parser = argparse.ArgumentParser(description="Allocates each tweet in a collection with a number in specified range.")
parser.add_argument('-field', help='the fieldname to store the inferred location in', required=True)
parser.add_argument('-num', type=int, help='the range [0, X] of id\'s to allocate', required=True)
parser.add_argument('-db', help='address of the database', default=config.get('mongo', 'address'))
parser.add_argument('-port', type=int, help='port of the database', default=config.get('mongo', 'port'))
parser.add_argument('-coll', help='database and collection name seperated by a dot (eg, twitter.tweets)', required=True)
args = parser.parse_args()

# connect to the MongoDB
logging.info("Connecting to MongoDB...")
client = MongoClient(args.db, args.port)
logging.info("Connected to MongoDB.")

# select the database and collection based off config
try:
    db_n, col_n = args.coll.split(".")