def load(self, config=DEFAULT_CONFIG):
"""
Load the required config from the cache or configuration file if not available
"""
# check to see if the config is currently cached
dataMap = memcache.get(
cachehelper.createCacheKey("twitter-config", config))
# if the dataMap is not cached, then load it from the yaml in the filesystem
if dataMap is None:
# open the required configuration file (using the conf directory)
# TODO: fix this rather silly path
fHandle = open('../../conf/' + config + '.yaml')
# now read the configuration information, and then close the file
try:
dataMap = yaml.load(fHandle)
# save the datamap to the cache
memcache.set(
cachehelper.createCacheKey("twitter-config", config),
dataMap)
finally:
fHandle.close()
# update the data based on the configuration data
self.consumerKey = dataMap.get('consumerKey', None)
self.consumerSecret = dataMap.get('consumerSecret', None)
# add some logging
logging.debug("consumerKey = %s", self.consumerKey)
def findOrCreate(searchName):
# convert the rulename to lower case
searchName = searchName.lower()
# look for the twawl rule in the cache first
fnresult = memcache.get(cachehelper.createCacheKey("twawlrule", searchName))
if fnresult is not None:
logging.debug("Returned twawlrule %s from the cache", searchName)
return fnresult
# look for the specified rule where the rule name is a match
query = TwawlRule.gql("WHERE ruleName = :name", name=searchName)
# get the result from the query
fnresult = query.get()
# if we couldn't find the rule, then create a new one
if fnresult == None:
fnresult = TwawlRule(ruleName=searchName)
fnresult.put()
# add the rule to the cache
if not memcache.add(cachehelper.createCacheKey("twawlrule", searchName), fnresult):
logging.error("Unable to write the twawl rule %s to the cache", searchName)
return fnresult
def findOrCreate(searchName):
# convert the rulename to lower case
searchName = searchName.lower()
# look for the twawl rule in the cache first
fnresult = memcache.get(
cachehelper.createCacheKey("twawlrule", searchName))
if (fnresult is not None):
logging.debug("Returned twawlrule %s from the cache", searchName)
return fnresult
# look for the specified rule where the rule name is a match
query = TwawlRule.gql("WHERE ruleName = :name", name=searchName)
# get the result from the query
fnresult = query.get()
# if we couldn't find the rule, then create a new one
if (fnresult == None):
fnresult = TwawlRule(ruleName=searchName)
fnresult.put()
# add the rule to the cache
if not memcache.add(
cachehelper.createCacheKey("twawlrule", searchName), fnresult):
logging.error("Unable to write the twawl rule %s to the cache",
searchName)
return fnresult
def load(self, config = DEFAULT_PROXY_CONFIG):
"""
Load the required config from the cache or configuration file if not available
"""
# check to see if the config is currently cached
self.configurations = memcache.get(cachehelper.createCacheKey("twitter-config", config))
# if the dataMap is not cached, then load it from the yaml in the filesystem
if self.configurations is None:
# open the required configuration file (using the conf directory)
fHandle = open('conf/' + config + '.yaml')
# now read the configuration information, and then close the file
try:
tmpConfigurations = yaml.load(fHandle)
# validate the configuration
validateConfig(tmpConfigurations)
# save the datamap to the cache
self.configurations = tmpConfigurations
memcache.set(cachehelper.createCacheKey("twitter-config", config), self.configurations)
finally:
fHandle.close()
def load(self, config = DEFAULT_CONFIG):
"""
Load the required config from the cache or configuration file if not available
"""
# check to see if the config is currently cached
dataMap = memcache.get(cachehelper.createCacheKey("twitter-config", config))
# if the dataMap is not cached, then load it from the yaml in the filesystem
if dataMap is None:
# open the required configuration file (using the conf directory)
# TODO: fix this rather silly path
fHandle = open('../../conf/' + config + '.yaml')
# now read the configuration information, and then close the file
try:
dataMap = yaml.load(fHandle)
# save the datamap to the cache
memcache.set(cachehelper.createCacheKey("twitter-config", config), dataMap)
finally:
fHandle.close()
# update the data based on the configuration data
self.consumerKey = dataMap.get('consumerKey', None)
self.consumerSecret = dataMap.get('consumerSecret', None)
# add some logging
logging.debug("consumerKey = %s", self.consumerKey)
def find(searchRule, searchDate):
"""
This static method will be used to find the TwawlHistory. First hitting the cache
for information and then checking the databsae is not available.
"""
# check the cache for the twawl history record
fnresult = memcache.get(cachehelper.createCacheKey("twawlHistory", searchRule.ruleName, searchDate.isoformat()))
# if found, return the value
if fnresult is not None:
logging.debug("returned TwawlHistory %s from the cache", fnresult)
return fnresult
# look for the specified date in the database
query = TwawlHistory.gql("WHERE rule = :rule AND searchDate = :date", rule=searchRule, date=searchDate)
# go looking for the result
return query.get()
def update(self, highTweet, tweetsIncrement):
"""
This method is used to update the details of the twawl rule and then clear the cache
"""
# update the last search and high tweet id of the rule
self.lastSearch = datetime.datetime.utcnow()
self.highTweetId = highTweet
# update the total tweets of the rule
if self.totalTweets is None:
self.totalTweets = tweetsIncrement
else:
self.totalTweets = self.totalTweets + tweetsIncrement
# save the rule to the database
self.put()
# add an info log entry about the number of tweets processed
logging.info("successfully processed %s tweets, high tweet id now %s", tweetsIncrement, highTweet)
# clear the cache key for the update
memcache.delete(cachehelper.createCacheKey("twawlrule", self.ruleName))
def find(searchRule, searchDate):
"""
This static method will be used to find the TwawlHistory. First hitting the cache
for information and then checking the databsae is not available.
"""
# check the cache for the twawl history record
fnresult = memcache.get(
cachehelper.createCacheKey("twawlHistory", searchRule.ruleName,
searchDate.isoformat()))
# if found, return the value
if fnresult is not None:
logging.debug("returned TwawlHistory %s from the cache", fnresult)
return fnresult
# look for the specified date in the database
query = TwawlHistory.gql("WHERE rule = :rule AND searchDate = :date",
rule=searchRule,
date=searchDate)
# go looking for the result
return query.get()
def update(self, highTweet, tweetsIncrement):
"""
This method is used to update the details of the twawl rule and then clear the cache
"""
# update the last search and high tweet id of the rule
self.lastSearch = datetime.datetime.utcnow()
self.highTweetId = highTweet
# update the total tweets of the rule
if (self.totalTweets is None):
self.totalTweets = tweetsIncrement
else:
self.totalTweets = self.totalTweets + tweetsIncrement
# save the rule to the database
self.put()
# add an info log entry about the number of tweets processed
logging.info("successfully processed %s tweets, high tweet id now %s",
tweetsIncrement, highTweet)
# clear the cache key for the update
memcache.delete(cachehelper.createCacheKey("twawlrule", self.ruleName))
