def main(): args = docopt(__doc__) c = get_config() session = open_db() zodb_root = open_zodb(read_only=True) ids = session.query(Speech.id).all() all_words = set() for (id,) in progress.bar(ids, label="Progress: ", every=100): all_words = all_words.union(zodb_root.features['words'][id].keys()) print "Word set size: ", len(all_words) #print "Subtracting stopwords..." #nltk.download('stopwords') #langs = ['english', 'spanish'] #all_stopwords = reduce(lambda x, y: x | y, [set(nltk.corpus.stopwords.words(lng)) for lng in langs]) #all_stopwords = set(map(unidecode, all_stopwords)) #all_words = all_words - all_stopwords #print "Resulting word set size: ", len(all_words) print "Saving..." zodb_root = open_zodb() zodb_root.all_words = all_words transaction.commit() print "Done"
def main(): args = docopt(__doc__) feature_name = args['<feature_name>'] assert feature_name == 'words' assert args['<experimentset_name>'] in EXPERIMENT_SETS, '<experimentset_name> must be one of %s' % str(EXPERIMENT_SETS.keys()) c = get_config() experiment_set = EXPERIMENT_SETS[args['<experimentset_name>']](feature_name=feature_name) print "Computing foreground group sums using %d cores..." % c.num_cores pool = Pool(c.num_cores, init_worker) fg_groups = experiment_set.list_foreground_groups() cache = {} try: for group_name, sum_vector in progress.bar(pool.imap_unordered(ComputeForegroundGroupSumCallable(experiment_set), fg_groups), label="Progress ", expected_size=len(fg_groups)): cache[group_name] = sum_vector except KeyboardInterrupt: print "Terminating pool.." pool.terminate() pool.join() print "Computing background sums..." bg_groups = experiment_set.list_background_groups() for g in bg_groups: sum_vector = experiment_set.compute_background_group_sum(g, cache) cache[g] = sum_vector print "Saving sums to ZODB..." zodb_root = open_zodb(read_only=False) if getattr(zodb_root, 'group_sums', None) is None: zodb_root.group_sums = BTrees.OOBTree.OOBTree() transaction.commit() if feature_name not in zodb_root.group_sums: zodb_root.group_sums[feature_name] = BTrees.OOBTree.OOBTree() transaction.commit() for k, v in cache.iteritems(): zodb_root.group_sums[feature_name][k] = v transaction.commit() print "Creating output db tables..." create_db(c.resultsdb_url) session_out = open_db(c.resultsdb_url) print "Computing overrepresentation using %d cores..." % c.num_cores exps = experiment_set.list_experiments() cls = experiment_set.result_table_class() try: for fg, bg, results in progress.bar(pool.imap_unordered(ComputeOverrepresentedWordsCallable(experiment_set), exps), label="Progress ", expected_size=len(exps)): for w, odds, pval in results: c = cls(foreground_group_name=fg, background_group_name=bg, word=w, odds=odds, pval=pval) session_out.add(c) except KeyboardInterrupt: print "Terminating pool.." pool.terminate() pool.join() print "Committing..." session_out.commit() print "Done"
def open_db(db_url = None): "Returns an initialized Session object. If db_url is not specified, uses get_config().db_url" if db_url is None: from talkofactadb.config import get_config db_url = get_config().db_url e = create_engine(db_url) Session = sessionmaker(e) return Session()
def open_zodb(config=None, read_only=False): "Opens a Zope database and returns a root object. If config not specified get_config() is used." if config is None: from talkofactadb.config import get_config config = get_config() storage = ZODB.FileStorage.FileStorage(os.path.join(config.zodb_dir, 'zodb.fs'), read_only=read_only) db = ZODB.DB(storage) connection = db.open() root = connection.root return root
def main(): args = docopt(__doc__) c = get_config() e = create_engine(c.db_url) Base.metadata.drop_all(e) Base.metadata.create_all(e) Session = sessionmaker(e) s = Session() with gzip.open(os.path.join(c.textdb_dir, 'acta.csv.gz'), 'rb') as csv_file: reader = csv.reader(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) reader.next() # Skip header for row in progress.mill(reader, label='Writing to DB ', expected_size=10000, every=1000): sp = Speech(date=datetime.strptime(row[3], '%Y-%m-%d'), speaker_uri=unicode(row[1], 'utf-8'), hansard=unicode(row[0], 'utf-8'), speech=unicode(row[2], 'utf-8')) s.add(sp) print "Committing..." s.commit() print "Done"
def main(): args = docopt(__doc__) extractor_name = args['<feature_name>'] extractor = getattr(talkofactawords.extract, extractor_name, None) if extractor is None: print "Unknown extractor name" sys.exit(1) c = get_config() s = open_db() print "Preparing ZODB" zodb_root = open_zodb(read_only=False) if getattr(zodb_root, 'features', None) is None: zodb_root.features = BTrees.OOBTree.OOBTree() transaction.commit() if extractor_name not in zodb_root.features: zodb_root.features[extractor_name] = BTrees.OOBTree.OOBTree() transaction.commit() runner = TaskRunner(extractor) print "Querying database..." speeches = s.query(Speech).all() total_speeches = len(speeches) print "Computing using %d cores..." % c.num_cores pool = Pool(c.num_cores, init_worker) try: for i, (id, result) in enumerate(progress.bar(pool.imap_unordered(runner, speeches), label='Progress ', expected_size=total_speeches, every=100), 1): zodb_root.features[extractor_name][id] = result if i % 1000 == 0: transaction.commit() transaction.commit() except KeyboardInterrupt: print "Terminating pool.." pool.terminate() pool.join() print "Done"
def __init__(self, pval_cutoff=0.01, feature_name='words'): self.pval_cutoff = pval_cutoff self.feature_name = feature_name self.config = get_config()