def setupConnection():
"""
Create connection to configured SQLite database file.
The file will be created if it does not exist yet.
:return: DB connection object, which should be added SQLObject table
classes so they can access the db.
"""
dbPath = AppConf().get("SQL", "dbPath")
return builder()(dbPath)
storage, then pretty print the OneTab user data as JSON. See the docs/browsers_onetab_extraction.md file for instructions. """ import argparse import json import re import os import sys import plyvel from lib import BROWSER_PROFILE_DIRS from lib.config import AppConf conf = AppConf() # Path to OneTab data within a directory for a browser user. This # is for both Linux and Mac. FIREFOX_ONETAB = "browser-extension-data/[email protected]/storage.js" CHROME_ONETAB = "Local Storage/leveldb" # The plyvel docs recommend referencing LevelDB keys in the # binary form (which is how they are stored). LEVELDB_ONETAB_KEY = b'_chrome-extension://chphlpgkkbolifaimnlloiipkdnihall\x00\x01state' def parse_leveldb_bytes(data_bytes): """ Parse LevelDB OneTab data from bytes to dict.
# Allow imports to be done when executing this file directly.
sys.path.insert(
0,
os.path.abspath(
os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
),
)
from lib import database as db
from lib.config import AppConf
from lib.tweets import insertOrUpdateTweetBatch
from lib.db_query.tweets.categories import printAvailableCategories
conf = AppConf()
UTILITY_CAMPAIGN = conf.get("Labels", "fetchTweets")
def main():
"""
Command-line interface to fetch Tweet data for Profile Categories.
"""
parser = argparse.ArgumentParser(
description="""Fetch Tweets for Profiles utility. Filter Profiles
in the DB using a Category input, update them with new data
and insert or update the most recent Tweets for each.
Tweets are assigned to the '{0}' Campaign.""".format(
UTILITY_CAMPAIGN
)
)
os.path.pardir)),
)
import lib
import lib.text_handling
import lib.twitter_api.authentication
import lib.twitter_api.search
import lib.tweets
from lib import database as db
from lib.config import AppConf
from lib.db_query.tweets.campaigns import (
printAvailableCampaigns,
printCampaignsAndTweets,
)
from models import Campaign
conf = AppConf()
UTILITY_CATEGORY = UTILITY_CAMPAIGN = conf.get("Labels", "search")
# Create initial global API connection object, which needs to be set.
API_CONN = None
def search(query, pageCount=1, extended=True):
"""
Do a Search API query for one or more pages of tweets matching the query.
After every 100 or slightly fewer tweets, a new request will be done to
get the next page.
:param query: Query text to search on the Twitter API.
:param pageCount: Count pages of tweets to fetch. Each page contains 100
import os
import sys
# Allow imports to be done when executing this file directly.
sys.path.insert(
0,
os.path.abspath(
os.path.join(os.path.dirname(__file__), os.path.pardir,
os.path.pardir)),
)
from lib import database as db
from lib.config import AppConf
from lib.tweets import insertOrUpdateProfileBatch, assignProfileCategory
from lib.db_query.tweets.categories import printAvailableCategories
conf = AppConf()
UTILITY_CATEGORY = conf.get("Labels", "fetchProfiles")
INFLUENCER_CATEGORY = conf.get("Labels", "influencers")
def main():
"""
Command-line tool to add or update Profile records from list of Twitter
screen names.
Expects a list screen names, either from arguments list or to be read
from a specified text file. If custom Category is provided, then assign to
the Profiles.
:return: None
"""
import argparse
import csv
import sys
import os
sys.path.insert(
0,
os.path.abspath(
os.path.join(os.path.dirname(__file__), os.path.pardir,
os.path.pardir)),
)
from lib import database as db
from lib.config import AppConf
conf = AppConf()
REPORT_DIR = conf.get("Reporting", "reportingDir")
# TODO: Refactor to be in lib dir.
def write_csv(path, rows, append=False):
"""
Write a CSV file to a path with given rows and header from first row.
Default behavior is to overwrite an existing file. Append to existing file
if append is flag True. Either way, the header will only be added on a new
file. Appending is useful when adding sections to a report, but overwriting
is better when rerunning an entire report.
"""
if not rows:
print("No rows to write")
def __init__(self, *args, **kwargs):
super(TestConfig, self).__init__(*args, **kwargs)
self.conf = AppConf()
class TestConfig(TestCase):
def __init__(self, *args, **kwargs):
super(TestConfig, self).__init__(*args, **kwargs)
self.conf = AppConf()
def test_test_mode(self):
self.assertTrue(os.environ.get("TEST_MODE"))
db_path = self.conf.get("SQL", "dbPath")
self.assertTrue(db_path.endswith("test_db.sqlite"))
def test_check_paths(self):
self.conf.check_paths()
def test_staging_dir(self):
self.conf.stagingCSVs()
def test_read_values(self):
"""
Do no validation, just check the section and field can be read.
"""
self.conf.get("TwitterAuth", "consumerKey")
self.conf.get("TwitterAuth", "consumerSecret")
self.conf.get("TwitterAuth", "accessKey")
self.conf.get("TwitterAuth", "accessSecret")
