def create_database(module_name, design_document):
"""
Create a database for a module and store a design document.
:param module_name: the name of the module for which a database shall be created
:param design_document: a design document passed from the creating module
"""
# get the full database name from the module name and the database prefix
full_database_name = config.get('couchdb', 'database-prefix') + module_name
# configure the call to CouchDB
couch = couchdb.Server(config.get('couchdb', 'database-url'))
couch.resource.credentials = (config.get('couchdb', 'database-user'), config.get('couchdb', 'database-password'))
# create or select the databse
if full_database_name in couch:
db = couch[full_database_name]
logger.debug('found database %s' % full_database_name)
else:
db = couch.create(full_database_name)
logger.debug('created database %s' % full_database_name)
# replace the passed design document
if '_design/diarybot' in db:
del db['_design/diarybot']
db['_design/diarybot'] = json.loads(design_document)
def create_database(module_name, design_document):
"""
Create a database for a module and store a design document.
:param module_name: the name of the module for which a database shall be created
:param design_document: a design document passed from the creating module
"""
# get the full database name from the module name and the database prefix
full_database_name = config.get("couchdb", "database-prefix") + module_name
# configure the call to CouchDB
couch = couchdb.Server(config.get("couchdb", "database-url"))
couch.resource.credentials = (config.get("couchdb", "database-user"), config.get("couchdb", "database-password"))
# create or select the databse
if full_database_name in couch:
db = couch[full_database_name]
logger.debug("found database %s" % full_database_name)
else:
db = couch.create(full_database_name)
logger.debug("created database %s" % full_database_name)
# replace the passed design document
if "_design/diarybot" in db:
del db["_design/diarybot"]
db["_design/diarybot"] = json.loads(design_document)
def regular_import(self):
"""
Imports new tracks, beginning from the last tracks found in the local database
:return:
"""
first_local_track = self._get_first_track()
latest_local_track = self._get_latest_track()
# check if there are all old tracks imported.
# this can happen when the initial import become interrupted
logger.debug("first local stored track is from %s" % datetime.datetime.fromtimestamp(
int(first_local_track['date']['uts'])).isoformat()
)
params = {'method': 'user.getrecenttracks',
'user': config.get('lastfm', 'username'),
'api_key': config.get('lastfm', 'api_key'),
'format': 'json'}
opt_params = {'to': int(first_local_track['date']['uts']),
'limit': 1}
data = self.api_call(params, opt_params)
if '@attr' in data['recenttracks']:
track_count = int(data['recenttracks']['@attr']['total'])
else:
track_count = int(data['recenttracks']['total'])
if track_count > 0:
logger.debug("%s tracks are older as the oldest local track. they have to be imported" % track_count)
self._run_fetch_store(params, opt_params)
else:
logger.debug("all older tracks are imported")
logger.debug("latest local stored track is from %s" % datetime.datetime.fromtimestamp(
int(latest_local_track['date']['uts'])).isoformat())
# check if newer tracks have to be imported.
opt_params = {'from': int(latest_local_track['date']['uts']),
'limit': 1}
data = self.api_call(params, opt_params)
if '@attr' in data['recenttracks']:
track_count = int(data['recenttracks']['@attr']['total'])
else:
track_count = int(data['recenttracks']['total'])
if track_count > 0:
logger.debug("%s tracks are newer as the latest local track. they have to be imported." % track_count)
self._run_fetch_store(params, opt_params)
else:
logger.debug("all newer tracks are imported")
return True
def get_docs(last_id):
logger.debug("reading new entries from Skype's local database")
db_filename = expanduser(
"~") + '/Library/Application Support/Skype/' + config.get(
'skype', 'skype_username') + '/main.db'
conn = sqlite3.connect(db_filename)
conn.row_factory = sqlite3.Row
c = conn.cursor()
sql_statement = '''
SELECT author, from_dispname, timestamp, body_xml, chatname
FROM Messages
WHERE timestamp > {timestamp}
ORDER BY id
'''
c.execute(sql_statement.format(timestamp=last_id))
for entry in c.fetchall():
yield {
'date': datetime.fromtimestamp(entry['timestamp']).isoformat(),
'text': entry['body_xml'],
'author': entry['author'],
'display_name': entry['from_dispname'],
'chatname': entry['chatname']
}
def get_docs(last_id):
logger.debug("reading new entries from Skype's local database")
db_filename = expanduser("~") + '/Library/Application Support/Skype/' + config.get('skype', 'skype_username') + '/main.db'
conn = sqlite3.connect(db_filename)
conn.row_factory = sqlite3.Row
c = conn.cursor()
sql_statement = '''
SELECT author, from_dispname, timestamp, body_xml, chatname
FROM Messages
WHERE timestamp > {timestamp}
ORDER BY id
'''
c.execute(sql_statement.format(timestamp=last_id))
for entry in c.fetchall():
yield {
'date': datetime.fromtimestamp(entry['timestamp']).isoformat(),
'text': entry['body_xml'],
'author': entry['author'],
'display_name': entry['from_dispname'],
'chatname': entry['chatname']
}
def regular_import(self):
"""
Import new images and moves
Check first if all old media files were imported. This can happen when the import will be interrupted.
Then check if newer media files exists and import them
:return:
"""
latest_local_image = self._get_latest_media()
first_local_image = self._get_first_media()
# check if we have imported all images previews the oldes we have
url = self.base_url + "/users/self/media/recent/?access_token=%s&max_id=%s" % (
config.get('instagram', 'access_token'), first_local_image['id']
)
res = self._api_call(url)
if len(res['data']) > 0:
self._run_fetch({'min_id': first_local_image['id']})
# now import all newer images thince the newest we have
logger.debug("latest local stored image is from %s" % datetime.datetime.fromtimestamp(
int(latest_local_image['created_time'])).isoformat())
self._run_fetch({'min_id': latest_local_image['id'], 'max_timestamp': latest_local_image['created_time']})
return True
def get_database(module_name):
"""
Get a module's database from CouchDB.
:param module_name: the name of the module for which the database shall be returned
:return: the CouchDB database for the given module
"""
# get the full database name from the module name and the database prefix
full_database_name = config.get("couchdb", "database-prefix") + module_name
# configure the call to CouchDB
couch = couchdb.Server(config.get("couchdb", "database-url"))
couch.resource.credentials = (config.get("couchdb", "database-user"), config.get("couchdb", "database-password"))
# return the database
db = couch[full_database_name]
logger.debug("opening database %s" % full_database_name)
return db
def get_database(module_name):
"""
Get a module's database from CouchDB.
:param module_name: the name of the module for which the database shall be returned
:return: the CouchDB database for the given module
"""
# get the full database name from the module name and the database prefix
full_database_name = config.get('couchdb', 'database-prefix') + module_name
# configure the call to CouchDB
couch = couchdb.Server(config.get('couchdb', 'database-url'))
couch.resource.credentials = (config.get('couchdb', 'database-user'), config.get('couchdb', 'database-password'))
# return the database
db = couch[full_database_name]
logger.debug('opening database %s' % full_database_name)
return db
def _get_online_media_count(self):
"""
returns total online available media count
:return:
"""
# check how many entries in total available
url = self.base_url + "/users/self?access_token=%s" % config.get('instagram', 'access_token')
res = self._api_call(url)
return int(res['data']['counts']['media'])
def __init__(self):
try:
Module.__init__(self, "foursquare")
except:
return
self.client = Foursquare(access_token=config.get('foursquare', 'access_token'))
if not self.initial_import():
self.regular_import()
def initial_import(self):
"""
Imports all tracks at once if no import before were executed
:return: Boolean
"""
if not self._get_latest_track():
logger.debug("no local stored data found - inital import needed")
params = {'method': 'user.getrecenttracks',
'user': config.get('lastfm', 'username'),
'api_key': config.get('lastfm', 'api_key'),
'format': 'json'}
# check how many entries in total available
data = self.api_call(params, {'limit': 1})
logger.debug("%s tracks have to be imported" % (data['recenttracks']['@attr']['total']))
self._run_fetch_store(params)
return True
return False
def __init__(self):
try:
Module.__init__(self, "foursquare")
except:
return
self.client = Foursquare(
access_token=config.get('foursquare', 'access_token'))
if not self.initial_import():
self.regular_import()
def get_first_id(self):
"""
Query the Moves API for the first date for which data is stored for the user
:return: a date as string formatted "%Y%m%d"
"""
logger.debug("getting user's first date")
endpoint = '/user/profile'
data = {'access_token': config.get('moves', 'access_token')}
res = self.api_call(endpoint, data)
return res['profile']['firstDate']
def get_api(self):
"""
:return: an initialized Twitter API object
"""
logger.debug("logging in at Twitter")
# add a parser to access the raw JSON from the tweets
# from http://www.hongliangjie.com/2012/04/04/hack-tweepy-to-get-raw-json/
class RawJsonParser(tweepy.parsers.Parser):
def parse(self, method, payload):
return payload
#authorize twitter, initialize Tweepy
auth = tweepy.OAuthHandler(config.get('twitter', 'consumer_key'), config.get('twitter', 'consumer_secret'))
auth.set_access_token(config.get('twitter', 'access_token'), config.get('twitter', 'access_token_secret'))
api = tweepy.API(auth, parser=RawJsonParser())
# get screen name for later access to user timeline
self.screen_name = json.loads(api.me())['screen_name']
logger.debug("screen name: %s" % self.screen_name)
return api
def api_call(endpoint):
base_url = 'https://xboxapi.com'
headers = {"X-AUTH": config.get('xbox', 'api_key')}
url = base_url + endpoint
result = requests.get(url, headers=headers)
try:
result = result.json()
except ValueError:
logger.critical("error decoding JSON: %s" % result)
if 'error_message' in result:
logger.critical(result['error_message'])
return result
def api_call(endpoint):
base_url = 'https://xboxapi.com'
headers = {"X-AUTH": config.get('xbox', 'api_key')}
url = base_url + endpoint
result = requests.get(url, headers=headers)
try:
result = result.json()
except ValueError:
logger.critical("error decoding JSON: %s" % result)
if 'error_message' in result:
logger.critical(result['error_message'])
return result
def get_first_id(self):
"""
Query the Moves API for the first date for which data is stored for the user
:return: a date as string formatted "%Y%m%d"
"""
logger.debug("getting user's first date")
endpoint = '/user/profile'
data = {
'access_token': config.get('moves', 'access_token')
}
res = self.api_call(endpoint, data)
return res['profile']['firstDate']
def initial_import(self):
"""
Imports all tracks at once if no import before were executed
:return: Boolean
"""
if not self._get_latest_track():
logger.debug("no local stored data found - inital import needed")
params = {
'method': 'user.getrecenttracks',
'user': config.get('lastfm', 'username'),
'api_key': config.get('lastfm', 'api_key'),
'format': 'json'
}
# check how many entries in total available
data = self.api_call(params, {'limit': 1})
logger.debug("%s tracks have to be imported" %
(data['recenttracks']['@attr']['total']))
self._run_fetch_store(params)
return True
return False
def get_storyline(self, date):
logger.debug("querying story line for %s..." % date)
endpoint = '/user/storyline/daily/%s' % date
data = {
'access_token': config.get('moves', 'access_token'),
'trackPoints': 'true'
}
res = self.api_call(endpoint, data)
# the result is a list - get the first (and only) element and add the date as id
doc = res[0]
doc['_id'] = date.isoformat()
return doc
def get_storyline(self, date):
logger.debug("querying story line for %s..." % date)
endpoint = '/user/storyline/daily/%s' % date
data = {
'access_token': config.get('moves', 'access_token'),
'trackPoints': 'true'
}
res = self.api_call(endpoint, data)
# the result is a list - get the first (and only) element and add the date as id
doc = res[0]
doc['_id'] = date.isoformat()
return doc
def _run_fetch(self, params={}):
"""
fetches available media from instagram page by page
:param params: parameter to narrow the API result
:return: True
"""
record_count = 0
# at the first iteration we have to build the url by our self
# append additional parameters
url = self.base_url + "/users/self/media/recent/?access_token=%s" % config.get('instagram', 'access_token') + '&' + '&'.join(
["%s=%s" % (k, v) for k, v in params.iteritems()])
while True:
# build parameter set to get step by step all data
res = self._api_call(url)
# break here if there is nothing to import
if len(res['data']) < 1:
logger.debug("nothing to import.")
break
self._store_wattatchment(self.database, res['data'])
record_count += len(res['data'])
logger.debug("Stored %s of in total %s images in database" % (
record_count, (self.meta_online_image_count - self.meta_local_image_count)))
# get next url from api - if we are not at the end
if 'next_url' in res['pagination']:
url = res['pagination']['next_url']
else:
logger.debug("All images fetched.")
break
return True
import couchdb
from diarybot.utils.logger import logger
from diarybot.config import config
# connect to the CouchDB server
couch = couchdb.Server(config.get("couchdb", "database-url"))
couch.resource.credentials = (config.get("couchdb", "database-user"), config.get("couchdb", "database-password"))
# select the databases of Diary Bot
diarybot_databases = [
couch[db_name] for db_name in couch if db_name.startswith(config.get("couchdb", "database-prefix"))
]
logger.debug("performing maintenance for %d Diary Bot databases" % len(diarybot_databases))
# Clean up and compact each database and its design documents
for db in diarybot_databases:
logger.debug("cleaning up and compacting database %s" % db.name)
db.cleanup()
db.compact()
for design_document in db.view("_all_docs")["_design":"_design0"]:
# get the basename of the design document
design_document_name = design_document.id.split("/")[-1]
db.compact(design_document_name)
logger.debug("done")
import couchdb
from diarybot.utils.logger import logger
from diarybot.config import config
# connect to the CouchDB server
couch = couchdb.Server(config.get('couchdb', 'database-url'))
couch.resource.credentials = (config.get('couchdb', 'database-user'),
config.get('couchdb', 'database-password'))
# select the databases of Diary Bot
diarybot_databases = [
couch[db_name] for db_name in couch
if db_name.startswith(config.get('couchdb', 'database-prefix'))
]
logger.debug("performing maintenance for %d Diary Bot databases" %
len(diarybot_databases))
# Clean up and compact each database and its design documents
for db in diarybot_databases:
logger.debug("cleaning up and compacting database %s" % db.name)
db.cleanup()
db.compact()
for design_document in db.view('_all_docs')['_design':'_design0']:
# get the basename of the design document
design_document_name = design_document.id.split('/')[-1]
db.compact(design_document_name)
logger.debug("done")
def regular_import(self):
"""
Imports new tracks, beginning from the last tracks found in the local database
:return:
"""
first_local_track = self._get_first_track()
latest_local_track = self._get_latest_track()
# check if there are all old tracks imported.
# this can happen when the initial import become interrupted
logger.debug("first local stored track is from %s" %
datetime.datetime.fromtimestamp(
int(first_local_track['date']['uts'])).isoformat())
params = {
'method': 'user.getrecenttracks',
'user': config.get('lastfm', 'username'),
'api_key': config.get('lastfm', 'api_key'),
'format': 'json'
}
opt_params = {'to': int(first_local_track['date']['uts']), 'limit': 1}
data = self.api_call(params, opt_params)
if '@attr' in data['recenttracks']:
track_count = int(data['recenttracks']['@attr']['total'])
else:
track_count = int(data['recenttracks']['total'])
if track_count > 0:
logger.debug(
"%s tracks are older as the oldest local track. they have to be imported"
% track_count)
self._run_fetch_store(params, opt_params)
else:
logger.debug("all older tracks are imported")
logger.debug("latest local stored track is from %s" %
datetime.datetime.fromtimestamp(
int(latest_local_track['date']['uts'])).isoformat())
# check if newer tracks have to be imported.
opt_params = {
'from': int(latest_local_track['date']['uts']),
'limit': 1
}
data = self.api_call(params, opt_params)
if '@attr' in data['recenttracks']:
track_count = int(data['recenttracks']['@attr']['total'])
else:
track_count = int(data['recenttracks']['total'])
if track_count > 0:
logger.debug(
"%s tracks are newer as the latest local track. they have to be imported."
% track_count)
self._run_fetch_store(params, opt_params)
else:
logger.debug("all newer tracks are imported")
return True
