def create_database(module_name, design_document): """ Create a database for a module and store a design document. :param module_name: the name of the module for which a database shall be created :param design_document: a design document passed from the creating module """ # get the full database name from the module name and the database prefix full_database_name = config.get('couchdb', 'database-prefix') + module_name # configure the call to CouchDB couch = couchdb.Server(config.get('couchdb', 'database-url')) couch.resource.credentials = (config.get('couchdb', 'database-user'), config.get('couchdb', 'database-password')) # create or select the databse if full_database_name in couch: db = couch[full_database_name] logger.debug('found database %s' % full_database_name) else: db = couch.create(full_database_name) logger.debug('created database %s' % full_database_name) # replace the passed design document if '_design/diarybot' in db: del db['_design/diarybot'] db['_design/diarybot'] = json.loads(design_document)
def create_database(module_name, design_document): """ Create a database for a module and store a design document. :param module_name: the name of the module for which a database shall be created :param design_document: a design document passed from the creating module """ # get the full database name from the module name and the database prefix full_database_name = config.get("couchdb", "database-prefix") + module_name # configure the call to CouchDB couch = couchdb.Server(config.get("couchdb", "database-url")) couch.resource.credentials = (config.get("couchdb", "database-user"), config.get("couchdb", "database-password")) # create or select the databse if full_database_name in couch: db = couch[full_database_name] logger.debug("found database %s" % full_database_name) else: db = couch.create(full_database_name) logger.debug("created database %s" % full_database_name) # replace the passed design document if "_design/diarybot" in db: del db["_design/diarybot"] db["_design/diarybot"] = json.loads(design_document)
def regular_import(self): """ Imports new tracks, beginning from the last tracks found in the local database :return: """ first_local_track = self._get_first_track() latest_local_track = self._get_latest_track() # check if there are all old tracks imported. # this can happen when the initial import become interrupted logger.debug("first local stored track is from %s" % datetime.datetime.fromtimestamp( int(first_local_track['date']['uts'])).isoformat() ) params = {'method': 'user.getrecenttracks', 'user': config.get('lastfm', 'username'), 'api_key': config.get('lastfm', 'api_key'), 'format': 'json'} opt_params = {'to': int(first_local_track['date']['uts']), 'limit': 1} data = self.api_call(params, opt_params) if '@attr' in data['recenttracks']: track_count = int(data['recenttracks']['@attr']['total']) else: track_count = int(data['recenttracks']['total']) if track_count > 0: logger.debug("%s tracks are older as the oldest local track. they have to be imported" % track_count) self._run_fetch_store(params, opt_params) else: logger.debug("all older tracks are imported") logger.debug("latest local stored track is from %s" % datetime.datetime.fromtimestamp( int(latest_local_track['date']['uts'])).isoformat()) # check if newer tracks have to be imported. opt_params = {'from': int(latest_local_track['date']['uts']), 'limit': 1} data = self.api_call(params, opt_params) if '@attr' in data['recenttracks']: track_count = int(data['recenttracks']['@attr']['total']) else: track_count = int(data['recenttracks']['total']) if track_count > 0: logger.debug("%s tracks are newer as the latest local track. they have to be imported." % track_count) self._run_fetch_store(params, opt_params) else: logger.debug("all newer tracks are imported") return True
def get_docs(last_id): logger.debug("reading new entries from Skype's local database") db_filename = expanduser( "~") + '/Library/Application Support/Skype/' + config.get( 'skype', 'skype_username') + '/main.db' conn = sqlite3.connect(db_filename) conn.row_factory = sqlite3.Row c = conn.cursor() sql_statement = ''' SELECT author, from_dispname, timestamp, body_xml, chatname FROM Messages WHERE timestamp > {timestamp} ORDER BY id ''' c.execute(sql_statement.format(timestamp=last_id)) for entry in c.fetchall(): yield { 'date': datetime.fromtimestamp(entry['timestamp']).isoformat(), 'text': entry['body_xml'], 'author': entry['author'], 'display_name': entry['from_dispname'], 'chatname': entry['chatname'] }
def get_docs(last_id): logger.debug("reading new entries from Skype's local database") db_filename = expanduser("~") + '/Library/Application Support/Skype/' + config.get('skype', 'skype_username') + '/main.db' conn = sqlite3.connect(db_filename) conn.row_factory = sqlite3.Row c = conn.cursor() sql_statement = ''' SELECT author, from_dispname, timestamp, body_xml, chatname FROM Messages WHERE timestamp > {timestamp} ORDER BY id ''' c.execute(sql_statement.format(timestamp=last_id)) for entry in c.fetchall(): yield { 'date': datetime.fromtimestamp(entry['timestamp']).isoformat(), 'text': entry['body_xml'], 'author': entry['author'], 'display_name': entry['from_dispname'], 'chatname': entry['chatname'] }
def regular_import(self): """ Import new images and moves Check first if all old media files were imported. This can happen when the import will be interrupted. Then check if newer media files exists and import them :return: """ latest_local_image = self._get_latest_media() first_local_image = self._get_first_media() # check if we have imported all images previews the oldes we have url = self.base_url + "/users/self/media/recent/?access_token=%s&max_id=%s" % ( config.get('instagram', 'access_token'), first_local_image['id'] ) res = self._api_call(url) if len(res['data']) > 0: self._run_fetch({'min_id': first_local_image['id']}) # now import all newer images thince the newest we have logger.debug("latest local stored image is from %s" % datetime.datetime.fromtimestamp( int(latest_local_image['created_time'])).isoformat()) self._run_fetch({'min_id': latest_local_image['id'], 'max_timestamp': latest_local_image['created_time']}) return True
def get_database(module_name): """ Get a module's database from CouchDB. :param module_name: the name of the module for which the database shall be returned :return: the CouchDB database for the given module """ # get the full database name from the module name and the database prefix full_database_name = config.get("couchdb", "database-prefix") + module_name # configure the call to CouchDB couch = couchdb.Server(config.get("couchdb", "database-url")) couch.resource.credentials = (config.get("couchdb", "database-user"), config.get("couchdb", "database-password")) # return the database db = couch[full_database_name] logger.debug("opening database %s" % full_database_name) return db
def get_database(module_name): """ Get a module's database from CouchDB. :param module_name: the name of the module for which the database shall be returned :return: the CouchDB database for the given module """ # get the full database name from the module name and the database prefix full_database_name = config.get('couchdb', 'database-prefix') + module_name # configure the call to CouchDB couch = couchdb.Server(config.get('couchdb', 'database-url')) couch.resource.credentials = (config.get('couchdb', 'database-user'), config.get('couchdb', 'database-password')) # return the database db = couch[full_database_name] logger.debug('opening database %s' % full_database_name) return db
def _get_online_media_count(self): """ returns total online available media count :return: """ # check how many entries in total available url = self.base_url + "/users/self?access_token=%s" % config.get('instagram', 'access_token') res = self._api_call(url) return int(res['data']['counts']['media'])
def __init__(self): try: Module.__init__(self, "foursquare") except: return self.client = Foursquare(access_token=config.get('foursquare', 'access_token')) if not self.initial_import(): self.regular_import()
def initial_import(self): """ Imports all tracks at once if no import before were executed :return: Boolean """ if not self._get_latest_track(): logger.debug("no local stored data found - inital import needed") params = {'method': 'user.getrecenttracks', 'user': config.get('lastfm', 'username'), 'api_key': config.get('lastfm', 'api_key'), 'format': 'json'} # check how many entries in total available data = self.api_call(params, {'limit': 1}) logger.debug("%s tracks have to be imported" % (data['recenttracks']['@attr']['total'])) self._run_fetch_store(params) return True return False
def __init__(self): try: Module.__init__(self, "foursquare") except: return self.client = Foursquare( access_token=config.get('foursquare', 'access_token')) if not self.initial_import(): self.regular_import()
def get_first_id(self): """ Query the Moves API for the first date for which data is stored for the user :return: a date as string formatted "%Y%m%d" """ logger.debug("getting user's first date") endpoint = '/user/profile' data = {'access_token': config.get('moves', 'access_token')} res = self.api_call(endpoint, data) return res['profile']['firstDate']
def get_api(self): """ :return: an initialized Twitter API object """ logger.debug("logging in at Twitter") # add a parser to access the raw JSON from the tweets # from http://www.hongliangjie.com/2012/04/04/hack-tweepy-to-get-raw-json/ class RawJsonParser(tweepy.parsers.Parser): def parse(self, method, payload): return payload #authorize twitter, initialize Tweepy auth = tweepy.OAuthHandler(config.get('twitter', 'consumer_key'), config.get('twitter', 'consumer_secret')) auth.set_access_token(config.get('twitter', 'access_token'), config.get('twitter', 'access_token_secret')) api = tweepy.API(auth, parser=RawJsonParser()) # get screen name for later access to user timeline self.screen_name = json.loads(api.me())['screen_name'] logger.debug("screen name: %s" % self.screen_name) return api
def api_call(endpoint): base_url = 'https://xboxapi.com' headers = {"X-AUTH": config.get('xbox', 'api_key')} url = base_url + endpoint result = requests.get(url, headers=headers) try: result = result.json() except ValueError: logger.critical("error decoding JSON: %s" % result) if 'error_message' in result: logger.critical(result['error_message']) return result
def get_first_id(self): """ Query the Moves API for the first date for which data is stored for the user :return: a date as string formatted "%Y%m%d" """ logger.debug("getting user's first date") endpoint = '/user/profile' data = { 'access_token': config.get('moves', 'access_token') } res = self.api_call(endpoint, data) return res['profile']['firstDate']
def initial_import(self): """ Imports all tracks at once if no import before were executed :return: Boolean """ if not self._get_latest_track(): logger.debug("no local stored data found - inital import needed") params = { 'method': 'user.getrecenttracks', 'user': config.get('lastfm', 'username'), 'api_key': config.get('lastfm', 'api_key'), 'format': 'json' } # check how many entries in total available data = self.api_call(params, {'limit': 1}) logger.debug("%s tracks have to be imported" % (data['recenttracks']['@attr']['total'])) self._run_fetch_store(params) return True return False
def get_storyline(self, date): logger.debug("querying story line for %s..." % date) endpoint = '/user/storyline/daily/%s' % date data = { 'access_token': config.get('moves', 'access_token'), 'trackPoints': 'true' } res = self.api_call(endpoint, data) # the result is a list - get the first (and only) element and add the date as id doc = res[0] doc['_id'] = date.isoformat() return doc
def _run_fetch(self, params={}): """ fetches available media from instagram page by page :param params: parameter to narrow the API result :return: True """ record_count = 0 # at the first iteration we have to build the url by our self # append additional parameters url = self.base_url + "/users/self/media/recent/?access_token=%s" % config.get('instagram', 'access_token') + '&' + '&'.join( ["%s=%s" % (k, v) for k, v in params.iteritems()]) while True: # build parameter set to get step by step all data res = self._api_call(url) # break here if there is nothing to import if len(res['data']) < 1: logger.debug("nothing to import.") break self._store_wattatchment(self.database, res['data']) record_count += len(res['data']) logger.debug("Stored %s of in total %s images in database" % ( record_count, (self.meta_online_image_count - self.meta_local_image_count))) # get next url from api - if we are not at the end if 'next_url' in res['pagination']: url = res['pagination']['next_url'] else: logger.debug("All images fetched.") break return True
import couchdb from diarybot.utils.logger import logger from diarybot.config import config # connect to the CouchDB server couch = couchdb.Server(config.get("couchdb", "database-url")) couch.resource.credentials = (config.get("couchdb", "database-user"), config.get("couchdb", "database-password")) # select the databases of Diary Bot diarybot_databases = [ couch[db_name] for db_name in couch if db_name.startswith(config.get("couchdb", "database-prefix")) ] logger.debug("performing maintenance for %d Diary Bot databases" % len(diarybot_databases)) # Clean up and compact each database and its design documents for db in diarybot_databases: logger.debug("cleaning up and compacting database %s" % db.name) db.cleanup() db.compact() for design_document in db.view("_all_docs")["_design":"_design0"]: # get the basename of the design document design_document_name = design_document.id.split("/")[-1] db.compact(design_document_name) logger.debug("done")
import couchdb from diarybot.utils.logger import logger from diarybot.config import config # connect to the CouchDB server couch = couchdb.Server(config.get('couchdb', 'database-url')) couch.resource.credentials = (config.get('couchdb', 'database-user'), config.get('couchdb', 'database-password')) # select the databases of Diary Bot diarybot_databases = [ couch[db_name] for db_name in couch if db_name.startswith(config.get('couchdb', 'database-prefix')) ] logger.debug("performing maintenance for %d Diary Bot databases" % len(diarybot_databases)) # Clean up and compact each database and its design documents for db in diarybot_databases: logger.debug("cleaning up and compacting database %s" % db.name) db.cleanup() db.compact() for design_document in db.view('_all_docs')['_design':'_design0']: # get the basename of the design document design_document_name = design_document.id.split('/')[-1] db.compact(design_document_name) logger.debug("done")
def regular_import(self): """ Imports new tracks, beginning from the last tracks found in the local database :return: """ first_local_track = self._get_first_track() latest_local_track = self._get_latest_track() # check if there are all old tracks imported. # this can happen when the initial import become interrupted logger.debug("first local stored track is from %s" % datetime.datetime.fromtimestamp( int(first_local_track['date']['uts'])).isoformat()) params = { 'method': 'user.getrecenttracks', 'user': config.get('lastfm', 'username'), 'api_key': config.get('lastfm', 'api_key'), 'format': 'json' } opt_params = {'to': int(first_local_track['date']['uts']), 'limit': 1} data = self.api_call(params, opt_params) if '@attr' in data['recenttracks']: track_count = int(data['recenttracks']['@attr']['total']) else: track_count = int(data['recenttracks']['total']) if track_count > 0: logger.debug( "%s tracks are older as the oldest local track. they have to be imported" % track_count) self._run_fetch_store(params, opt_params) else: logger.debug("all older tracks are imported") logger.debug("latest local stored track is from %s" % datetime.datetime.fromtimestamp( int(latest_local_track['date']['uts'])).isoformat()) # check if newer tracks have to be imported. opt_params = { 'from': int(latest_local_track['date']['uts']), 'limit': 1 } data = self.api_call(params, opt_params) if '@attr' in data['recenttracks']: track_count = int(data['recenttracks']['@attr']['total']) else: track_count = int(data['recenttracks']['total']) if track_count > 0: logger.debug( "%s tracks are newer as the latest local track. they have to be imported." % track_count) self._run_fetch_store(params, opt_params) else: logger.debug("all newer tracks are imported") return True