def main(configFile=None):
''' Deletes all subscriptions from the Instagram server. Typically called
on a new dotcloud push just to make sure its all clear.'''
print "----------->", configFile
# Get the config information into a single object
p = getConfigParameters(configFile)
# Get the client and secret keys
api = InstagramAPI(client_id=p.client, client_secret=p.secret)
# Get all current subs
subs = api.list_subscriptions()
# For each active sub, delete it
if subs['meta']['code'] == 200:
for sub in subs['data']:
if sub['type'] == 'subscription':
deleted = api.delete_subscriptions(id=int(sub['id']))
# Final check - make sure they're definitely all gone
subs = api.list_subscriptions()
if len(subs['data']) == 0:
success = True
else:
success = False
else:
success = False
def main(configFile=None):
''' Deletes all subscriptions from the Instagram server. Typically called
on a new dotcloud push just to make sure its all clear.'''
print "----------->", configFile
# Get the config information into a single object
p = getConfigParameters(configFile)
# Get the client and secret keys
api = InstagramAPI(client_id=p.client, client_secret=p.secret)
# Get all current subs
subs = api.list_subscriptions()
# For each active sub, delete it
if subs['meta']['code'] == 200:
for sub in subs['data']:
if sub['type'] == 'subscription':
deleted = api.delete_subscriptions(id=int(sub['id']))
# Final check - make sure they're definitely all gone
subs = api.list_subscriptions()
if len(subs['data']) == 0:
success = True
else:
success = False
else:
success = False
def main(cleanup, ageOff, protectedSubs=None):
''' Coordinates which element to cleanup - subscriptions including remote delete of subs from instagram
or event data - getting rid of the documents that contain the media and metadata'''
# If there is a command to remove or keep protected subs, use it
if protectedSubs:
protectedSubs = bool(protectedSubs)
# Get the config information
os.chdir('/home/dotcloud/code/')
cwd = os.getcwd()
cfgs = os.path.join(cwd, 'config/crowded.cfg')
p = getConfigParameters(cfgs)
# The mongo bits
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
# Whether to cleanup the subscriptions and subscription collection
if cleanup == 'subs':
subsCollHandle=dbh[p.subsCollection]
res = ageOffSubscriptions(p, subsCollHandle, ageOff=ageOff, protectedSubs=protectedSubs)
print datetime.datetime.utcnow(), res
# Or the events metadata collection
elif cleanup == 'events':
evCollHandle=dbh[p.eventsCollection]
res = ageOffMetadata(evCollHandle, ageOff=ageOff, protectMedia=protectedSubs)
print datetime.datetime.utcnow(), res
else:
print 'Provide either subs or events as the first argument depending on what you want to clean up.'
def testGetCurrentTags(self):
import mdb
cwd = os.getcwd()
parent = os.path.dirname(cwd)
cfgs = os.path.join(parent, 'config/twitterCrowded.cfg')
p = getConfigParameters(cfgs)
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost,
port=p.dbPort,
db=p.db,
user=p.dbUser,
password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
print "Failed to connect to mongo."
sys.exit(1)
res = cf.getCurrentTags(evCollHandle, 'shitbrick')
self.assertEquals(res, ['hellworld', 'fuckhole', 'shitbrick'])
# Quick test chucked in
results = cf.getQueryBBox(evCollHandle)
print results
def main(configFile, subscriptionType, source):
''' Coordinates the retrieval of public CCTV camera URLs to crowded. '''
# Get the config information into a single object
p = getConfigParameters(configFile)
#////////////////////////////////////////////////////////
if source == 'cctv':
url = p.tflUrl
elif source == 'youtube':
url = p.socialiseUrl
elif source == 'flickr':
url = p.viewFinderUrl
# More sources here and adds to the config file
#////////////////////////////////////////////////////////
# Mongo connection parameters
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
collHandle = dbh['subs']
evCollHandle = dbh['events']
# Get the active subs
activeSubs = getActiveSubs(collHandle, type=subscriptionType)
# Barf at this point if there's nothing in subs
if not activeSubs or len(activeSubs) < 1:
mdb.close(c, dbh)
return None
# For each active active subscription, query by geography
for aSub in activeSubs:
print 'ASUB:', aSub
if subscriptionType == 'geography':
lon, lat = aSub['loc']
radius = float(aSub['radius'])
media = queryByGeo(url, lat, lon, radius)
elif subscriptionType == 'tag':
tag = aSub['objectId']
media = queryByTag(url, tag)
# For each of the images, update the correct event url list
for image in media:
# Mod the datetime into a python dt
try:
img = datetime.datetime.strptime(image['captured'], "%Y-%m-%dT%H:%M:%S")
except Exception, e:
img = datetime.datetime.strptime(image['published'], "%Y-%m-%dT%H:%M:%S")
image['dt'] = img
success = updateEvents(evCollHandle, aSub['objectId'], image)
if success == None:
print "Failed to update event ID '%s' with media: \n %s" %(aSub['objectId'], image)
def main(configFile):
''' Holds it all together '''
# Get the config information into a single object. p also gets passed into the listener
p = getConfigParameters(configFile)
# Handle the mongo connection and authentication
c, dbh, collection, emoCollection = getMongoHandles(p)
# Open the english dictionary file and loop it
try:
f = open(os.path.join(p.sourcePath, p.enPlainFile), 'r')
except Exception, e:
handleErrors(p, e)
def main(configFile=None):
''' Takes the dotcloud default admin privs, authorises on the db,
creates the user I've specified and returns. '''
# Get the parameters that were set up by dotcloud
dcParams = getEnvironment()
print "got DC environment settings."
reParams = getRedisEnvironment()
print "got redis environment settings."
# Authenticate on the admin db
try:
c, adminDbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db='admin', user=dcParams.adminUser, password=dcParams.adminPass)
print 'got handle'
except:
print "Failed to get handle under admin."
# Authentication of the administrator
#try:
# auth = adminDbh.authenticate(dcParams.adminUser, dcParams.adminPass)
#except Exception, e:
# print "Failed to authenticate with mongo db."
# print e
# Create a new user
p = getConfigParameters(configFile)
# Switch the database handle to that being used from the admin one
dbh = c[p.db]
success = dbh.add_user(p.dbUser, p.dbPassword)
c.disconnect()
try:
# Authenticate on the admin db
c, dbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db=p.db, user=p.dbUser, password=p.dbPassword)
print 'Connected to the normal db: %s' %(p.db)
except:
logging.critical("Failed to connect to db and get handle as user.", exc_info=True)
sys.exit()
# Write out the new information to the regular config file
try:
writeConfigFile(configFile, dcParams)
print 'Writing out mongo config info.'
writeConfigFileRedis(configFile, reParams)
print 'Writing out redis config'
except:
logging.critical("Failed in writing params back to config file.", exc_info=True)
mdb.close(c, dbh)
def main(configFile=None):
''' Builds the collections and indexes needed. '''
# Get the config information into a single object
p = getConfigParameters(configFile)
# Get a db handle
if p.verbose==True:
print "---- Geting Mongo Handle."
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db)
try:
auth = dbh.authenticate(p.dbUser, p.dbPassword)
except Exception, e:
print "Failed to authenticate with mongo db."
print e
def main(configFile=None):
''' Builds the collections and indexes needed. '''
# Get the config information into a single object
p = getConfigParameters(configFile)
try:
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
except:
logging.warning("Failed to connect to db and get handle.", exc_info=True)
# The collections provided and create them and their indexes
for coll in p.collections:
collHandle = buildCollection(dbh, p, coll['collection'])
indexes = buildIndexes(p, coll, collHandle)
mdb.close(c, dbh)
def setUp(self):
''' sets up parameters before each test'''
# The base url to hit
self.url = 'http://localhost:8044'
# Config information from a relative path
configPath = os.path.join(wsDir, 'config')
configFile = 'testsWordslang.cfg'
self.p = getConfigParameters(os.path.join(configPath, configFile))
self.c, dbh, collection, emoCollection = getMongoHandles(self.p)
# Build a document and insert it
doc1 = {"pho" : "sml",
"slang" : ["smlie", "smiiile", "2smile", "smiiileee", "smilee",
"usmile", "smilie", "simle", "smillleee", "smillle",
"smille", "smileee", "*s*", "smle", "smilert", "smiile",
"smil", "smie", ": )", "(-:"],
"word" : "smile"}
doc2 = {"pho" : "st",
"slang" : [ "sosad", "ssad", "saaad", "saaddd",
"saaadd", "sadd", "saadd", "saddd",
"saddy", "saad", "saaaddd", ": (", ": ["],
"word" : "sad"}
doc3 = {"pho" : "unh",
"slang" : [ "sad"],
"word" : "unhappy"}
# Insert a couple of documents
id1 = collection.insert(doc1)
id2 = collection.insert(doc2)
id3 = collection.insert(doc3)
self.document1 = doc1
self.document2 = doc2
self.document3 = doc3
# Documents to go into the emoticons collection
doc4 = {"word" : "smile",
"emo" : [":-)"]}
id4 = emoCollection.insert(doc4)
self.document4 = doc4
def main(configFile=None):
''' Builds the collections and indexes needed. '''
# Get the config information into a single object
p = getConfigParameters(configFile)
try:
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
except:
logging.warning("Failed to connect to db and get handle.", exc_info=True)
# The collections provided and create them and their indexes
for coll in p.collections:
print "Building Collections and indexes: %s" %coll
collHandle = buildCollection(dbh, p, coll['collection'])
indexes = buildIndexes(p, coll, collHandle)
mdb.close(c, dbh)
def testvalidMediaCheck_NO_STANDARD_RESOLUTION(self):
''' Makes sure the media is correctly formatted before being pushed in'''
testCase = {"low_resolution" : "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_6.jpg",
"source" : "instagram",
"dt" : "2012-10-04T19:06:04Z",
"caption" : "NOH8 CAMPAGIN #selfmade #wtwd #ducttape #chicago",
"thumbnail" : "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_5.jpg",
"objectId" : 'foobar'
}
cwd = os.getcwd()
parent = os.path.dirname(cwd)
cfgs = os.path.join(parent, 'config/crowded.cfg')
p = getConfigParameters(cfgs)
nm = crowdedWorker.validMediaCheck(p, testCase)
self.assertEquals(len(nm['errors']), 1)
print nm['errors']
def main(configFile, file="norm"):
""" Holds it all together """
# Get the config information into a single object. p also gets passed into the listener
p = getConfigParameters(configFile)
# Handle the mongo connection and authentication
c, dbh, collection, emoCollection = getMongoHandles(p)
if file == "norm":
fileToProcess = p.enNormalisedFile
elif file == "slang":
fileToProcess = p.slangFile
# Open the english dictionary file and loop it
try:
f = open(os.path.join(p.sourcePath, fileToProcess), "r")
except Exception, e:
print e
handleErrors(p, e)
def main(cleanup, ageOff, protectedSubs=None):
''' Coordinates which element to cleanup - subscriptions including remote delete of subs from instagram
or event data - getting rid of the documents that contain the media and metadata'''
# If there is a command to remove or keep protected subs, use it
if protectedSubs:
protectedSubs = bool(protectedSubs)
# Get the config information
os.chdir('/home/dotcloud/code/')
cwd = os.getcwd()
cfgs = os.path.join(cwd, 'config/crowded.cfg')
p = getConfigParameters(cfgs)
# The mongo bits
c, dbh = mdb.getHandle(host=p.dbHost,
port=p.dbPort,
db=p.db,
user=p.dbUser,
password=p.dbPassword)
# Whether to cleanup the subscriptions and subscription collection
if cleanup == 'subs':
subsCollHandle = dbh[p.subsCollection]
res = ageOffSubscriptions(p,
subsCollHandle,
ageOff=ageOff,
protectedSubs=protectedSubs)
print datetime.datetime.utcnow(), res
# Or the events metadata collection
elif cleanup == 'events':
evCollHandle = dbh[p.eventsCollection]
res = ageOffMetadata(evCollHandle,
ageOff=ageOff,
protectMedia=protectedSubs)
print datetime.datetime.utcnow(), res
else:
print 'Provide either subs or events as the first argument depending on what you want to clean up.'
def testvalidMediaCheck_NO_STANDARD_RESOLUTION(self):
''' Makes sure the media is correctly formatted before being pushed in'''
testCase = {
"low_resolution":
"http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_6.jpg",
"source": "instagram",
"dt": "2012-10-04T19:06:04Z",
"caption": "NOH8 CAMPAGIN #selfmade #wtwd #ducttape #chicago",
"thumbnail":
"http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_5.jpg",
"objectId": 'foobar'
}
cwd = os.getcwd()
parent = os.path.dirname(cwd)
cfgs = os.path.join(parent, 'config/crowded.cfg')
p = getConfigParameters(cfgs)
nm = crowdedWorker.validMediaCheck(p, testCase)
self.assertEquals(len(nm['errors']), 1)
print nm['errors']
def testGetCurrentTags(self):
import mdb
cwd = os.getcwd()
parent = os.path.dirname(cwd)
cfgs = os.path.join(parent, 'config/twitterCrowded.cfg')
p = getConfigParameters(cfgs)
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
print "Failed to connect to mongo."
sys.exit(1)
res = cf.getCurrentTags(evCollHandle, 'shitbrick')
self.assertEquals(res, ['hellworld','fuckhole', 'shitbrick'])
# Quick test chucked in
results = cf.getQueryBBox(evCollHandle)
print results
def main(configFile=None):
''' Takes the dotcloud default admin privs, authorises on the db,
creates the user I've specified and returns. '''
# Get the parameters that were set up by dotcloud
dcParams = getEnvironment()
# Authenticate on the admin db
try:
c, dbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db='admin', user=dcParams.adminUser, password=dcParams.adminPass)
except:
logging.critical('Failed to connect to database as admin.')
sys.exit()
# Create a new user
p = getConfigParameters(configFile)
# Switch the database handle to that being used from the admin one
dbh = c[p.db]
success = dbh.add_user(p.dbUser, p.dbPassword)
c.disconnect()
try:
# Authenticate on the admin db
c, dbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db=p.db, user=p.dbUser, password=p.dbPassword)
except:
logging.critical("Failed to connect to db and get handle as user.", exc_info=True)
sys.exit()
# Write out the new information to the regular config file
try:
writeConfigFile(configFile, dcParams)
print "----- writing out new config parameters."
except:
logging.critical("Failed in writing params back to config file.", exc_info=True)
mdb.close(c, dbh)
def main(configFile=None):
''' Takes the dotcloud default admin privs, authorises on the db,
creates the user I've specified and returns. '''
# Get the parameters that were set up by dotcloud
dcParams = getEnvironment()
logging.info("Mongo Params:\n%s\n%s\n%s\n%s" %(dcParams.mongoHost, dcParams.mongoPort, dcParams.adminUser, dcParams.adminPass))
# Authenticate on the admin db
try:
c, dbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db='admin', user=dcParams.adminUser, password=dcParams.adminPass)
except:
logging.critical('Failed to connect to database as admin.')
sys.exit()
# Create a new user
p = getConfigParameters(configFile)
# Switch the database handle to that being used from the admin one
dbh = c[p.db]
success = dbh.add_user(p.dbUser, p.dbPassword)
c.disconnect()
try:
# Authenticate on the admin db
c, dbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db=p.db, user=p.dbUser, password=p.dbPassword)
except:
logging.critical("Failed to connect to db and get handle as user.", exc_info=True)
sys.exit()
# Write out the new information to the regular config file
try:
writeConfigFile(configFile, dcParams)
except:
logging.critical("Failed in writing params back to config file.", exc_info=True)
mdb.close(c, dbh)
if mediaOnly:
try:
q.put(json.dumps(tweet))
except:
logging.critical("Failed to put tweet on redis. This tweet: \n%s" % (tweet), exc_info=True)
except tweetstream.ConnectionError:
logging.critical("Disconnected from twitter", exc_info=True)
# ------------------------------------------------------------------------------------------
if __name__ == "__main__":
# Command Line Options
parser = optparse.OptionParser()
parser.add_option("-c", "--config", dest="config", help="The absolute path to the config file")
parser.add_option("-m", "--media_only", dest="media", action="store_true", help="true if only want media tweets.")
(opts, args) = parser.parse_args()
# Check for the config file
if not opts.config:
print "Must provide a config file location. \n"
parser.print_help()
exit(-1)
# Config File Parameters
p = getConfigParameters(opts.config)
main(p, mediaOnly=opts.media)
#
#
# To do:
# -----
# - Is it possible to support multiple wsgi files for a single application?
# - If so, break out the admin functions.
# - Look into using twisted as the framework or web server so that it is FAST to receive the POSTs
# from external sources
# Extract the instagram bits from this code.
#///////////////////////////////////////////////////////////////////////////////////////////////
os.chdir('/home/dotcloud/current/')
cwd = os.getcwd()
cfgs = os.path.join(cwd, 'config/crowded.cfg')
p = getConfigParameters(cfgs)
# The mongo bits
c, dbh = mdb.getHandle(host=p.dbHost,
port=p.dbPort,
db=p.db,
user=p.dbUser,
password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
logFile = os.path.join(p.errorPath, p.errorFile)
logging.basicConfig(filename=logFile,
format='%(levelname)s:: \t%(asctime)s %(message)s',
level='DEBUG')
#------------------------------------------------------------------------------------------------
if directory.find('.git') == -1:
sys.path.append(directory)
#============================================================================================
import configureDatabase
import setupDatabase
import baseUtils
# Get the config file
configFile = sys.argv[1]
# Where this is being run -
site = sys.argv[2]
p = baseUtils.getConfigParameters(configFile)
# Setup the error logging
logFile = os.path.join(p.errorPath, p.errorFile)
logging.basicConfig(filename=logFile,
format='%(levelname)s:: \t%(asctime)s %(message)s',
level='DEBUG')
# Configuring the dotcloud settings for mongodb
if site == 'dotcloud':
logging.debug('---- Configuring the dotcloud settings for mongodb')
print '---- Configuring the dotcloud settings for mongodb'
configureDatabase.main(configFile)
elif site == 'local':
logging.debug('Skipping all dotcloud configuration.')
print 'Skipping all dotcloud configuration. '
if directory.find('.git') == -1:
sys.path.append(directory)
#============================================================================================
import configureDatabase
import setupDatabase
import baseUtils
# Get the config file
configFile = sys.argv[1]
# Where this is being run -
site = sys.argv[2]
p = baseUtils.getConfigParameters(configFile)
# Setup the error logging
logFile = os.path.join(p.errorPath, p.errorFile)
logging.basicConfig(filename=logFile, format='%(levelname)s:: \t%(asctime)s %(message)s', level='DEBUG')
# Configuring the dotcloud settings for mongodb
if site == 'dotcloud':
configureDatabase.main(configFile)
elif site == 'local':
pass
# Setup the database
print '---- Setting up and populating database'
setupDatabase.main(configFile)
logging.debug("Tag-based message dispatched: %s" % (success))
if queryBBoxes:
tweetGeos = cf.matchesCurrentGeos(queryBBoxes, tweet)
for tweetGeo in tweetGeos:
success = dispatchTweet(p, tweet, tweetGeo)
logging.debug("Geo-based message dispatched: %s" % (success))
#--------------------------------------------------------------------------------
if __name__ == "__main__":
configFile = sys.argv[1]
# first argument is the config file path
if not configFile:
print 'no Config file provided. Exiting.'
sys.exit()
else:
# Get the config information into a single object
p = getConfigParameters(configFile)
# Logging config
logFile = os.path.join(p.errorPath, p.redisConsumerLog)
logging.basicConfig(filename=logFile,
format='%(levelname)s:: \t%(asctime)s %(message)s',
level=p.logLevel)
main(p)
if not opts.config or not opts.eventId:
print "Must provide a config file location and an object id. \n"
parser.print_help()
exit(-1)
# Check for the bbox
if not opts.n and not opts.s and not opts.e and not opts.w and not opts.tag:
print "Must provide either tag or -n & s & e & w \n"
parser.print_help()
exit(-1)
else:
bbox = {'n':opts.n,'s':opts.s,'e':opts.e,'w':opts.w}
#Config File Parameters
logging.basicConfig()
p = getConfigParameters(opts.config)
x = True
while x == True:
pass
#main(p, event=opts.eventId, tag=opts.tag, bbox=bbox, mediaOnly=opts.media)
"""
{"data":[{"standard_resolution" : "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_7.jpg",
"low_resolution" : "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_6.jpg",
"source" : "instagram",
"dt" : "2012-10-04T19:06:04Z",
"caption" : "This actually worked as a POST",
"thumbnail" : "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_5.jpg",
"objectId" : "snorty",
except:
on_error(message='failed import of bottle.')
# Custom
try:
from baseUtils import getConfigParameters, getMongoHandles, handleErrors, decodeEncode
import mdb
except:
on_error(message='failed import of baseUtils and mdb.\n')
# Config information from a relative path
configFile = '/home/dotcloud/code/config/wordslang.cfg'
try:
# Get the config information into a single object. p also gets passed into the listener
p = getConfigParameters(configFile)
except Exception, e:
on_error(message='%s \n failed on getConfigParameters.\n' %e)
# Handle the mongo connection and authentication
try:
c, dbh, collection = getMongoHandles(p)
except Exception, e:
#on_error(message='db: %s host: %s port: %s user: %s pssd: %s \n' %(p.db, p.dbHost, p.dbPort, p.dbUser, p.dbPassword))
on_error(message='Failed to get mongo handles.\n')
#on_error(message='%s\n' %e)
@route('/check1')
def check1():
''' '''
#
# To do:
# -----
# - Is it possible to support multiple wsgi files for a single application?
# - If so, break out the admin functions.
# - Look into using twisted as the framework or web server so that it is FAST to receive the POSTs
# from external sources
# Extract the instagram bits from this code.
#///////////////////////////////////////////////////////////////////////////////////////////////
os.chdir('/home/dotcloud/current/')
cwd = os.getcwd()
cfgs = os.path.join(cwd, 'config/crowded.cfg')
p = getConfigParameters(cfgs)
# The mongo bits
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
logFile = os.path.join(p.errorPath, p.errorFile)
logging.basicConfig(filename=logFile, format='%(levelname)s:: \t%(asctime)s %(message)s', level='DEBUG')
#------------------------------------------------------------------------------------------------
def on_error(errFile='errors.txt', message=None):
''' Handles an error message '''
f = open(os.path.join(cwd, errFile), 'a')
f.write(message + '\n')
def buildSubscription(event):
''' Builds a new subscription based on an GET called event'''
# Placeholder for doing this by users/algorithm?
user = '******'
cwd = os.getcwd()
cfgs = os.path.join(cwd, 'config/crowded.cfg')
p = getConfigParameters(cfgs)
#print "Config Filepath in buildSubscription: ", cfgs
# The mongo bits
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
subsCollHandle = dbh[p.subsCollection]
evCollHandle = dbh[p.eventsCollection]
# Check whether we definitely need a new subscription or not
checked = checkForExistingSubs(p, subsCollHandle, event)
# If the subscription doesn't already exist,
if checked['exists'] == False:
# Get the client and secret keys
api = InstagramAPI(client_id=p.client, client_secret=p.secret)
# If it's a geo-based subscription
if event['object'] == 'geography':
res = api.create_subscription(object='geography', lat=event['lat'], lng=event['lon'], radius=event['radius'],
aspect='media', callback_url=p.subBaseUrl)
print "Geo Subscription setup: %s" %res
# A tag-based subscription
elif event['object'] == 'tag':
res = api.create_subscription(object='tag', object_id=event['tag'], aspect='media', callback_url=p.subBaseUrl)
print "Tag Subscription setup: %s" %res
# Just in case
else:
print 'Didnt setup a subscription'
res = None
# Update the subscription collection
if res and res['meta']['code']==200:
data = res['data']
subType = data['object']
objectId = data['object_id']
subId = data['id']
aspect = data['aspect']
success = updateSubs(subsCollHandle, subType, subId, objectId, aspect, event, user)
# Build the response
response = {'success' : True,
'objectId' : objectId,
'object' : subType,
'url' : "%s/%s" %(p.baseUrl, success)}
# Insert a blank document to populate
_id = buildEventPlaceholder(evCollHandle, subType, event, objectId)
# Something failed in the subscription build...?
else:
print '='*40
print 'Failed here. No event placeholder or subscription updated.'
print res
print '='*40
response = {'success' : False,
'objectId' : checked['objectId'],
'object' : checked['object'],
'url' : "%s/%s" %(p.baseUrl, checked['objectId'])}
# A valid subscription already exists
elif checked['exists'] == True:
response = {'success' : True,
'objectId' : checked['objectId'],
'object' : checked['object'],
'url' : "%s/%s" %(p.baseUrl, checked['objectId'])}
# Close the connection/handle
mdb.close(c, dbh)
return response
def main(configFile=None):
''' Coordinates the management functions
Command line called, typically from a CRON.'''
# Get the config file
p = getConfigParameters(configFile)
# Logging config
logFile = os.path.join(p.errorPath, p.errorFile)
logging.basicConfig(filename=logFile,
format='%(levelname)s:: \t%(asctime)s %(message)s',
level=p.logLevel)
# Streaming client
connClientPath = os.path.dirname(p.errorPath)
p.streamClient = os.path.join(connClientPath, 'src/connectionClient.py')
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost,
port=p.dbPort,
db=p.db,
user=p.dbUser,
password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
mgmtCollHandle = dbh[p.mgmtCollection]
logging.debug("Connected and authenticated on the db.")
except:
logging.critical('Failed to connect to db and authenticate.',
exc_info=True)
sys.exit()
# Create a new management document if needed
initialOID = setInitialPid(mgmtCollHandle)
# Get the current events from crowded
crowdedEvents = getCrowdedEvents(p)
# Get the events currently stored by this app
myEvents = getLocalEvents(p, evCollHandle)
# Compare the 2 sets of events: what's old and new?
oldEvents, newEvents = checkEvents(crowdedEvents, myEvents)
# Expire old events from db, so that the new stream reflects the correct interest
for oldEvent in oldEvents:
print oldEvent
logging.debug('Expiring Old Event in DB: %s' % (oldEvent))
res = expireOldEvent(evCollHandle, oldEvent)
# Create new item in the db
for newEvent in newEvents:
logging.debug('Creating New Event in DB: %s' % (newEvent))
res = createLocalEvent(evCollHandle, newEvent)
# Get the old process ID and kill it off
pid = getPid(mgmtCollHandle)
logging.debug('Current PID: %s' % (pid))
# Only continue if there is a change in the events
if len(oldEvents) > 0 or len(newEvents) > 0:
if pid:
logging.debug('Killing old process with ID: %s' % (pid))
res = killOldProcess(pid)
# Now create the new one
newPid = processNewEvent(p)
logging.debug('Creating a new process with PID: %s' % (newPid))
# Update the current process id in mongo
res = storePid(mgmtCollHandle, newPid)
logging.debug('Stored the new PID: %s' % (res))
mdb.close(c, dbh)
logging.shutdown()
