Пример #1
0
def main(configFile=None):
    ''' Deletes all subscriptions from the Instagram server. Typically called
        on a new dotcloud push just to make sure its all clear.'''

    print "----------->", configFile
    # Get the config information into a single object
    p = getConfigParameters(configFile)
    
        # Get the client and secret keys
    api = InstagramAPI(client_id=p.client, client_secret=p.secret)
    
    # Get all current subs
    subs = api.list_subscriptions()

    # For each active sub, delete it
    if subs['meta']['code'] == 200:
        
        for sub in subs['data']:
            
            if sub['type'] == 'subscription':
                deleted = api.delete_subscriptions(id=int(sub['id']))
        
        # Final check - make sure they're definitely all gone
        subs = api.list_subscriptions()
        if len(subs['data']) == 0:
            success = True
        else:
            success = False
    
    else:
        success = False
Пример #2
0
def main(configFile=None):
    ''' Deletes all subscriptions from the Instagram server. Typically called
        on a new dotcloud push just to make sure its all clear.'''

    print "----------->", configFile
    # Get the config information into a single object
    p = getConfigParameters(configFile)

    # Get the client and secret keys
    api = InstagramAPI(client_id=p.client, client_secret=p.secret)

    # Get all current subs
    subs = api.list_subscriptions()

    # For each active sub, delete it
    if subs['meta']['code'] == 200:

        for sub in subs['data']:

            if sub['type'] == 'subscription':
                deleted = api.delete_subscriptions(id=int(sub['id']))

        # Final check - make sure they're definitely all gone
        subs = api.list_subscriptions()
        if len(subs['data']) == 0:
            success = True
        else:
            success = False

    else:
        success = False
Пример #3
0
def main(cleanup, ageOff, protectedSubs=None):
    ''' Coordinates which element to cleanup - subscriptions including remote delete of subs from instagram
        or event data - getting rid of the documents that contain the media and metadata'''
    
    # If there is a command to remove or keep protected subs, use it
    if protectedSubs:
        protectedSubs = bool(protectedSubs)
    
    # Get the config information
    os.chdir('/home/dotcloud/code/')
    cwd  = os.getcwd()
    cfgs = os.path.join(cwd, 'config/crowded.cfg')
    p = getConfigParameters(cfgs)
    
    # The mongo bits
    c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)

    # Whether to cleanup the subscriptions and subscription collection
    if cleanup == 'subs':
        subsCollHandle=dbh[p.subsCollection]
        res = ageOffSubscriptions(p, subsCollHandle, ageOff=ageOff, protectedSubs=protectedSubs)
        print datetime.datetime.utcnow(), res
    
    # Or the events metadata collection
    elif cleanup == 'events':
        evCollHandle=dbh[p.eventsCollection]
        res = ageOffMetadata(evCollHandle, ageOff=ageOff, protectMedia=protectedSubs)
        print datetime.datetime.utcnow(), res
        
    else:
        print 'Provide either subs or events as the first argument depending on what you want to clean up.'
Пример #4
0
    def testGetCurrentTags(self):

        import mdb
        cwd = os.getcwd()
        parent = os.path.dirname(cwd)
        cfgs = os.path.join(parent, 'config/twitterCrowded.cfg')
        p = getConfigParameters(cfgs)

        # The mongo bits
        try:
            c, dbh = mdb.getHandle(host=p.dbHost,
                                   port=p.dbPort,
                                   db=p.db,
                                   user=p.dbUser,
                                   password=p.dbPassword)
            evCollHandle = dbh[p.eventsCollection]
        except:
            print "Failed to connect to mongo."
            sys.exit(1)

        res = cf.getCurrentTags(evCollHandle, 'shitbrick')
        self.assertEquals(res, ['hellworld', 'fuckhole', 'shitbrick'])

        # Quick test chucked in
        results = cf.getQueryBBox(evCollHandle)
        print results
Пример #5
0
def main(configFile, subscriptionType, source):
    ''' Coordinates the retrieval of public CCTV camera URLs to crowded. '''
    
    # Get the config information into a single object
    p = getConfigParameters(configFile)
    
    #////////////////////////////////////////////////////////
    if source == 'cctv':
        url = p.tflUrl
    elif source == 'youtube':
        url = p.socialiseUrl
    elif source == 'flickr':
        url = p.viewFinderUrl
    # More sources here and adds to the config file
    #////////////////////////////////////////////////////////
        
    # Mongo connection parameters
    c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
    collHandle = dbh['subs']
    evCollHandle = dbh['events']
    
    # Get the active subs
    activeSubs = getActiveSubs(collHandle, type=subscriptionType)

    # Barf at this point if there's nothing in subs
    if not activeSubs or len(activeSubs) < 1:
        mdb.close(c, dbh)
        return None

    # For each active active subscription, query by geography
    for aSub in activeSubs:
        
        print 'ASUB:', aSub
        if subscriptionType == 'geography':
            lon, lat = aSub['loc']
            radius = float(aSub['radius'])
            media = queryByGeo(url, lat, lon, radius)
        
        elif subscriptionType == 'tag':
            tag = aSub['objectId']
            media = queryByTag(url, tag)
        
        # For each of the images, update the correct event url list
        for image in media:
            # Mod the datetime into a python dt
            try:
                img = datetime.datetime.strptime(image['captured'], "%Y-%m-%dT%H:%M:%S")
            except Exception, e:
                img = datetime.datetime.strptime(image['published'], "%Y-%m-%dT%H:%M:%S")
            image['dt'] = img    
            
            success = updateEvents(evCollHandle, aSub['objectId'], image)
            if success == None:
                print "Failed to update event ID '%s' with media: \n %s" %(aSub['objectId'], image)
Пример #6
0
def main(configFile):
    ''' Holds it all together '''
    
    # Get the config information into a single object. p also gets passed into the listener
    p = getConfigParameters(configFile)

    # Handle the mongo connection and authentication
    c, dbh, collection, emoCollection = getMongoHandles(p)

    # Open the english dictionary file and loop it
    try:
        f = open(os.path.join(p.sourcePath, p.enPlainFile), 'r')
    except Exception, e:
        handleErrors(p, e)
Пример #7
0
def main(configFile=None):
    ''' Takes the dotcloud default admin privs, authorises on the db, 
        creates the user I've specified and returns. '''
    
    # Get the parameters that were set up by dotcloud
    dcParams = getEnvironment()
    print "got DC environment settings."
    reParams = getRedisEnvironment()
    print "got redis environment settings."
    
    # Authenticate on the admin db
    try:
        c, adminDbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db='admin', user=dcParams.adminUser, password=dcParams.adminPass)
        print 'got handle'
    except:
        print "Failed to get handle under admin."
    # Authentication of the administrator
    #try:
    #    auth = adminDbh.authenticate(dcParams.adminUser, dcParams.adminPass)
    #except Exception, e:
    #    print "Failed to authenticate with mongo db."
    #    print e
    
    # Create a new user
    p = getConfigParameters(configFile)
    # Switch the database handle to that being used from the admin one
    dbh = c[p.db]
    success = dbh.add_user(p.dbUser, p.dbPassword)
    c.disconnect()
    
    try:
        # Authenticate on the admin db
        c, dbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db=p.db, user=p.dbUser, password=p.dbPassword)
        print 'Connected to the normal db: %s' %(p.db)
    except:
        logging.critical("Failed to connect to db and get handle as user.", exc_info=True)
        sys.exit()
    
    # Write out the new information to the regular config file
    try:
        writeConfigFile(configFile, dcParams)
        print 'Writing out mongo config info.'
        writeConfigFileRedis(configFile, reParams)
        print 'Writing out redis config'
    except:
        logging.critical("Failed in writing params back to config file.", exc_info=True)
    
    mdb.close(c, dbh)
Пример #8
0
def main(configFile=None):
    ''' Builds the collections and indexes needed. '''

    # Get the config information into a single object
    p = getConfigParameters(configFile)

    # Get a db handle
    if p.verbose==True:
        print "---- Geting Mongo Handle."
    c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db)
    
    try:
        auth = dbh.authenticate(p.dbUser, p.dbPassword)
    except Exception, e:
        print "Failed to authenticate with mongo db."
        print e
Пример #9
0
def main(configFile=None):
    ''' Builds the collections and indexes needed. '''

    # Get the config information into a single object
    p = getConfigParameters(configFile)

    try:
        c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
    except:
        logging.warning("Failed to connect to db and get handle.", exc_info=True)

    # The collections provided and create them and their indexes
    for coll in p.collections:
        collHandle = buildCollection(dbh, p, coll['collection'])
        indexes = buildIndexes(p, coll, collHandle)
    
    mdb.close(c, dbh)
Пример #10
0
    def setUp(self):
        ''' sets up parameters before each test'''
        
        # The base url to hit
        self.url = 'http://localhost:8044'

        # Config information from a relative path
        configPath = os.path.join(wsDir, 'config')
        configFile = 'testsWordslang.cfg'
        self.p = getConfigParameters(os.path.join(configPath, configFile))
        
        self.c, dbh, collection, emoCollection = getMongoHandles(self.p)

        # Build a document and insert it 
        doc1 = {"pho" : "sml",
                 "slang" : ["smlie", "smiiile", "2smile", "smiiileee", "smilee",
                            "usmile", "smilie", "simle", "smillleee", "smillle",
                            "smille", "smileee", "*s*", "smle", "smilert", "smiile",
                            "smil", "smie", ": )", "(-:"],
                 "word" : "smile"}
        
        doc2 = {"pho" : "st",
                 "slang" : [ "sosad", "ssad", "saaad", "saaddd",
                        "saaadd", "sadd", "saadd", "saddd",
                        "saddy", "saad", "saaaddd", ": (", ": ["],
                "word" : "sad"}
        
        doc3 = {"pho" : "unh",
                 "slang" : [ "sad"],
                "word" : "unhappy"}

                
        # Insert a couple of documents
        id1 = collection.insert(doc1) 
        id2 = collection.insert(doc2)
        id3 = collection.insert(doc3)
        self.document1 = doc1
        self.document2 = doc2
        self.document3 = doc3
        
        # Documents to go into the emoticons collection
        doc4 = {"word" : "smile",
                "emo"  : [":-)"]}
        id4 = emoCollection.insert(doc4)
        self.document4 = doc4
Пример #11
0
def main(configFile=None):
    ''' Builds the collections and indexes needed. '''

    # Get the config information into a single object
    p = getConfigParameters(configFile)

    try:
        c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
    except:
        logging.warning("Failed to connect to db and get handle.", exc_info=True)

    # The collections provided and create them and their indexes
    for coll in p.collections:
        print "Building Collections and indexes: %s" %coll
        collHandle = buildCollection(dbh, p, coll['collection'])
        indexes = buildIndexes(p, coll, collHandle)
    
    mdb.close(c, dbh)
Пример #12
0
    def testvalidMediaCheck_NO_STANDARD_RESOLUTION(self):
        ''' Makes sure the media is correctly formatted before being pushed in'''
    
        testCase = {"low_resolution" : "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_6.jpg",
                    "source" : "instagram",
                    "dt" : "2012-10-04T19:06:04Z",
                    "caption" : "NOH8 CAMPAGIN #selfmade #wtwd #ducttape #chicago",
                    "thumbnail" : "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_5.jpg",
                    "objectId" : 'foobar'
                    }

        cwd = os.getcwd()
        parent = os.path.dirname(cwd)
        cfgs = os.path.join(parent, 'config/crowded.cfg')
        p = getConfigParameters(cfgs)
        
        nm = crowdedWorker.validMediaCheck(p, testCase)
        self.assertEquals(len(nm['errors']), 1)
        print nm['errors']
Пример #13
0
def main(configFile, file="norm"):
    """ Holds it all together """

    # Get the config information into a single object. p also gets passed into the listener
    p = getConfigParameters(configFile)

    # Handle the mongo connection and authentication
    c, dbh, collection, emoCollection = getMongoHandles(p)

    if file == "norm":
        fileToProcess = p.enNormalisedFile
    elif file == "slang":
        fileToProcess = p.slangFile

    # Open the english dictionary file and loop it
    try:
        f = open(os.path.join(p.sourcePath, fileToProcess), "r")
    except Exception, e:
        print e
        handleErrors(p, e)
Пример #14
0
def main(cleanup, ageOff, protectedSubs=None):
    ''' Coordinates which element to cleanup - subscriptions including remote delete of subs from instagram
        or event data - getting rid of the documents that contain the media and metadata'''

    # If there is a command to remove or keep protected subs, use it
    if protectedSubs:
        protectedSubs = bool(protectedSubs)

    # Get the config information
    os.chdir('/home/dotcloud/code/')
    cwd = os.getcwd()
    cfgs = os.path.join(cwd, 'config/crowded.cfg')
    p = getConfigParameters(cfgs)

    # The mongo bits
    c, dbh = mdb.getHandle(host=p.dbHost,
                           port=p.dbPort,
                           db=p.db,
                           user=p.dbUser,
                           password=p.dbPassword)

    # Whether to cleanup the subscriptions and subscription collection
    if cleanup == 'subs':
        subsCollHandle = dbh[p.subsCollection]
        res = ageOffSubscriptions(p,
                                  subsCollHandle,
                                  ageOff=ageOff,
                                  protectedSubs=protectedSubs)
        print datetime.datetime.utcnow(), res

    # Or the events metadata collection
    elif cleanup == 'events':
        evCollHandle = dbh[p.eventsCollection]
        res = ageOffMetadata(evCollHandle,
                             ageOff=ageOff,
                             protectMedia=protectedSubs)
        print datetime.datetime.utcnow(), res

    else:
        print 'Provide either subs or events as the first argument depending on what you want to clean up.'
Пример #15
0
    def testvalidMediaCheck_NO_STANDARD_RESOLUTION(self):
        ''' Makes sure the media is correctly formatted before being pushed in'''

        testCase = {
            "low_resolution":
            "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_6.jpg",
            "source": "instagram",
            "dt": "2012-10-04T19:06:04Z",
            "caption": "NOH8 CAMPAGIN #selfmade #wtwd #ducttape #chicago",
            "thumbnail":
            "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_5.jpg",
            "objectId": 'foobar'
        }

        cwd = os.getcwd()
        parent = os.path.dirname(cwd)
        cfgs = os.path.join(parent, 'config/crowded.cfg')
        p = getConfigParameters(cfgs)

        nm = crowdedWorker.validMediaCheck(p, testCase)
        self.assertEquals(len(nm['errors']), 1)
        print nm['errors']
Пример #16
0
    def testGetCurrentTags(self):
        
        import mdb
        cwd = os.getcwd()
        parent = os.path.dirname(cwd)
        cfgs = os.path.join(parent, 'config/twitterCrowded.cfg')
        p = getConfigParameters(cfgs)

        # The mongo bits
        try:
            c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
            evCollHandle = dbh[p.eventsCollection]    
        except:
            print "Failed to connect to mongo."
            sys.exit(1)

        res = cf.getCurrentTags(evCollHandle, 'shitbrick')
        self.assertEquals(res, ['hellworld','fuckhole', 'shitbrick'])

        # Quick test chucked in
        results = cf.getQueryBBox(evCollHandle)
        print results
Пример #17
0
def main(configFile=None):
    ''' Takes the dotcloud default admin privs, authorises on the db, 
        creates the user I've specified and returns. '''
    
    # Get the parameters that were set up by dotcloud
    dcParams = getEnvironment()
    
    # Authenticate on the admin db
    try:
        c, dbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db='admin', user=dcParams.adminUser, password=dcParams.adminPass)
    except:
        logging.critical('Failed to connect to database as admin.')
        sys.exit()

    # Create a new user
    p = getConfigParameters(configFile)
    # Switch the database handle to that being used from the admin one
    dbh = c[p.db]
    success = dbh.add_user(p.dbUser, p.dbPassword)
    c.disconnect()
    
    try:
        # Authenticate on the admin db
        c, dbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db=p.db, user=p.dbUser, password=p.dbPassword)
    except:
        logging.critical("Failed to connect to db and get handle as user.", exc_info=True)
        sys.exit()
    
    # Write out the new information to the regular config file
    try:
        writeConfigFile(configFile, dcParams)
        print "----- writing out new config parameters."
    except:
        logging.critical("Failed in writing params back to config file.", exc_info=True)
    
    mdb.close(c, dbh)
Пример #18
0
def main(configFile=None):
    ''' Takes the dotcloud default admin privs, authorises on the db, 
        creates the user I've specified and returns. '''
    
    # Get the parameters that were set up by dotcloud
    dcParams = getEnvironment()
    logging.info("Mongo Params:\n%s\n%s\n%s\n%s" %(dcParams.mongoHost, dcParams.mongoPort, dcParams.adminUser, dcParams.adminPass))
       
    # Authenticate on the admin db
    try:
        c, dbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db='admin', user=dcParams.adminUser, password=dcParams.adminPass)
    except:
        logging.critical('Failed to connect to database as admin.')
        sys.exit()
        
    # Create a new user
    p = getConfigParameters(configFile)
    # Switch the database handle to that being used from the admin one
    dbh = c[p.db]
    success = dbh.add_user(p.dbUser, p.dbPassword)
    c.disconnect()

    try:
        # Authenticate on the admin db
        c, dbh = mdb.getHandle(host=dcParams.mongoHost, port=dcParams.mongoPort, db=p.db, user=p.dbUser, password=p.dbPassword)
    except:
        logging.critical("Failed to connect to db and get handle as user.", exc_info=True)
        sys.exit()
        
    # Write out the new information to the regular config file
    try:
        writeConfigFile(configFile, dcParams)
    except:
        logging.critical("Failed in writing params back to config file.", exc_info=True)
        
    mdb.close(c, dbh)
Пример #19
0
                    if mediaOnly:
                        try:
                            q.put(json.dumps(tweet))
                        except:
                            logging.critical("Failed to put tweet on redis. This tweet: \n%s" % (tweet), exc_info=True)

        except tweetstream.ConnectionError:
            logging.critical("Disconnected from twitter", exc_info=True)


# ------------------------------------------------------------------------------------------

if __name__ == "__main__":

    # Command Line Options
    parser = optparse.OptionParser()
    parser.add_option("-c", "--config", dest="config", help="The absolute path to the config file")
    parser.add_option("-m", "--media_only", dest="media", action="store_true", help="true if only want media tweets.")
    (opts, args) = parser.parse_args()

    # Check for the config file
    if not opts.config:
        print "Must provide a config file location. \n"
        parser.print_help()
        exit(-1)

    # Config File Parameters
    p = getConfigParameters(opts.config)

    main(p, mediaOnly=opts.media)
Пример #20
0
#
#
# To do:
# -----
# - Is it possible to support multiple wsgi files for a single application?
# - If so, break out the admin functions.
# - Look into using twisted as the framework or web server so that it is FAST to receive the POSTs
#   from external sources
#   Extract the instagram bits from this code.

#///////////////////////////////////////////////////////////////////////////////////////////////

os.chdir('/home/dotcloud/current/')
cwd = os.getcwd()
cfgs = os.path.join(cwd, 'config/crowded.cfg')
p = getConfigParameters(cfgs)

# The mongo bits
c, dbh = mdb.getHandle(host=p.dbHost,
                       port=p.dbPort,
                       db=p.db,
                       user=p.dbUser,
                       password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]

logFile = os.path.join(p.errorPath, p.errorFile)
logging.basicConfig(filename=logFile,
                    format='%(levelname)s:: \t%(asctime)s %(message)s',
                    level='DEBUG')

#------------------------------------------------------------------------------------------------
Пример #21
0
        if directory.find('.git') == -1:
            sys.path.append(directory)

#============================================================================================

import configureDatabase
import setupDatabase
import baseUtils

# Get the config file
configFile = sys.argv[1]

# Where this is being run -
site = sys.argv[2]

p = baseUtils.getConfigParameters(configFile)

# Setup the error logging
logFile = os.path.join(p.errorPath, p.errorFile)
logging.basicConfig(filename=logFile,
                    format='%(levelname)s:: \t%(asctime)s %(message)s',
                    level='DEBUG')

# Configuring the dotcloud settings for mongodb
if site == 'dotcloud':
    logging.debug('---- Configuring the dotcloud settings for mongodb')
    print '---- Configuring the dotcloud settings for mongodb'
    configureDatabase.main(configFile)
elif site == 'local':
    logging.debug('Skipping all dotcloud configuration.')
    print 'Skipping all dotcloud configuration. '
Пример #22
0
        if directory.find('.git') == -1:
            sys.path.append(directory)

#============================================================================================

import configureDatabase
import setupDatabase
import baseUtils

# Get the config file
configFile = sys.argv[1]

# Where this is being run - 
site = sys.argv[2]

p = baseUtils.getConfigParameters(configFile)

# Setup the error logging
logFile = os.path.join(p.errorPath, p.errorFile)
logging.basicConfig(filename=logFile, format='%(levelname)s:: \t%(asctime)s %(message)s', level='DEBUG')

# Configuring the dotcloud settings for mongodb
if site == 'dotcloud':
    configureDatabase.main(configFile)
elif site == 'local':
    pass

# Setup the database
print '---- Setting up and populating database'
setupDatabase.main(configFile)
Пример #23
0
                logging.debug("Tag-based message dispatched: %s" % (success))

        if queryBBoxes:
            tweetGeos = cf.matchesCurrentGeos(queryBBoxes, tweet)
            for tweetGeo in tweetGeos:
                success = dispatchTweet(p, tweet, tweetGeo)
                logging.debug("Geo-based message dispatched: %s" % (success))


#--------------------------------------------------------------------------------

if __name__ == "__main__":

    configFile = sys.argv[1]

    # first argument is the config file path
    if not configFile:
        print 'no Config file provided. Exiting.'
        sys.exit()
    else:
        # Get the config information into a single object
        p = getConfigParameters(configFile)

    # Logging config
    logFile = os.path.join(p.errorPath, p.redisConsumerLog)
    logging.basicConfig(filename=logFile,
                        format='%(levelname)s:: \t%(asctime)s %(message)s',
                        level=p.logLevel)

    main(p)
Пример #24
0
    if not opts.config or not opts.eventId:
        print "Must provide a config file location and an object id. \n"
        parser.print_help()
        exit(-1)
    
    # Check for the bbox
    if not opts.n and not opts.s and not opts.e and not opts.w and not opts.tag:
        print "Must provide either tag or -n & s & e & w \n"
        parser.print_help()
        exit(-1)    
    else:
        bbox = {'n':opts.n,'s':opts.s,'e':opts.e,'w':opts.w}

    #Config File Parameters
    logging.basicConfig()
    p = getConfigParameters(opts.config)
    
    x = True
    while x == True:
        pass
    
    #main(p, event=opts.eventId, tag=opts.tag, bbox=bbox, mediaOnly=opts.media)

"""
{"data":[{"standard_resolution" : "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_7.jpg",
          "low_resolution" : "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_6.jpg",
           "source" : "instagram",
           "dt" : "2012-10-04T19:06:04Z",
           "caption" : "This actually worked as a POST",
           "thumbnail" : "http://distilleryimage1.s3.amazonaws.com/8b74d2ee0e5611e2adc122000a1de653_5.jpg",
           "objectId" : "snorty",
Пример #25
0
except:
    on_error(message='failed import of bottle.')
    
# Custom
try:
    from baseUtils import getConfigParameters, getMongoHandles, handleErrors, decodeEncode
    import mdb
except:
    on_error(message='failed import of baseUtils and mdb.\n')
    
# Config information from a relative path
configFile = '/home/dotcloud/code/config/wordslang.cfg'

try:
    # Get the config information into a single object. p also gets passed into the listener
    p = getConfigParameters(configFile)
except Exception, e:
    on_error(message='%s \n failed on getConfigParameters.\n' %e)
    
# Handle the mongo connection and authentication
try:
    c, dbh, collection = getMongoHandles(p)
except Exception, e:
    #on_error(message='db: %s host: %s port: %s user: %s pssd: %s \n' %(p.db, p.dbHost, p.dbPort, p.dbUser, p.dbPassword))
    on_error(message='Failed to get mongo handles.\n')
    #on_error(message='%s\n' %e)
    
@route('/check1')
def check1():
    '''    '''
    
Пример #26
0
#
# To do:
# -----
# - Is it possible to support multiple wsgi files for a single application?
# - If so, break out the admin functions.
# - Look into using twisted as the framework or web server so that it is FAST to receive the POSTs
#   from external sources
#   Extract the instagram bits from this code.


#///////////////////////////////////////////////////////////////////////////////////////////////

os.chdir('/home/dotcloud/current/')
cwd = os.getcwd()
cfgs = os.path.join(cwd, 'config/crowded.cfg')
p = getConfigParameters(cfgs)

# The mongo bits
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection] 

logFile = os.path.join(p.errorPath, p.errorFile)
logging.basicConfig(filename=logFile, format='%(levelname)s:: \t%(asctime)s %(message)s', level='DEBUG')

#------------------------------------------------------------------------------------------------

def on_error(errFile='errors.txt', message=None):
    ''' Handles an error message '''
    
    f = open(os.path.join(cwd, errFile), 'a')
    f.write(message + '\n')
Пример #27
0
def buildSubscription(event):
    ''' Builds a new subscription based on an GET called event'''

    # Placeholder for doing this by users/algorithm?
    user = '******'

    cwd = os.getcwd()
    cfgs = os.path.join(cwd, 'config/crowded.cfg')
    p = getConfigParameters(cfgs)

    #print "Config Filepath in buildSubscription: ", cfgs

    # The mongo bits
    c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
    subsCollHandle = dbh[p.subsCollection]
    evCollHandle   = dbh[p.eventsCollection]
                
    # Check whether we definitely need a new subscription or not    
    checked = checkForExistingSubs(p, subsCollHandle, event)
    
    # If the subscription doesn't already exist, 
    if checked['exists'] == False:
        
        # Get the client and secret keys
        api = InstagramAPI(client_id=p.client, client_secret=p.secret)
    
        # If it's a geo-based subscription
        if event['object'] == 'geography':
            res = api.create_subscription(object='geography', lat=event['lat'], lng=event['lon'], radius=event['radius'],
                                    aspect='media', callback_url=p.subBaseUrl)
            print "Geo Subscription setup: %s" %res
        # A tag-based subscription
        elif event['object'] == 'tag':
            res = api.create_subscription(object='tag', object_id=event['tag'], aspect='media', callback_url=p.subBaseUrl)
            print "Tag Subscription setup: %s" %res
        # Just in case
        else:
            print 'Didnt setup a subscription' 
            res = None
    
        # Update the subscription collection 
        if res and res['meta']['code']==200:
    
            data = res['data']
            subType  = data['object'] 
            objectId = data['object_id'] 
            subId    = data['id']
            aspect   = data['aspect']
            success = updateSubs(subsCollHandle, subType, subId, objectId, aspect, event, user)
            
            # Build the response 
            response = {'success'  : True,
                        'objectId' : objectId,
                        'object'   : subType,
                        'url'      : "%s/%s" %(p.baseUrl, success)}
        
            # Insert a blank document to populate
            _id = buildEventPlaceholder(evCollHandle, subType, event, objectId)
            
        # Something failed in the subscription build...?
        else:
            print '='*40
            print 'Failed here. No event placeholder or subscription updated.'
            print res
            print '='*40
            response = {'success'  : False,
                        'objectId' : checked['objectId'],
                        'object'   : checked['object'],
                        'url'      : "%s/%s" %(p.baseUrl, checked['objectId'])}
    
    # A valid subscription already exists 
    elif checked['exists'] == True:
        response = {'success'  : True,
                    'objectId' : checked['objectId'],
                    'object'   : checked['object'],
                    'url'      : "%s/%s" %(p.baseUrl, checked['objectId'])}

    # Close the connection/handle
    mdb.close(c, dbh)

    return response
Пример #28
0
def main(configFile=None):
    ''' Coordinates the management functions
        Command line called, typically from a CRON.'''

    # Get the config file
    p = getConfigParameters(configFile)

    # Logging config
    logFile = os.path.join(p.errorPath, p.errorFile)
    logging.basicConfig(filename=logFile,
                        format='%(levelname)s:: \t%(asctime)s %(message)s',
                        level=p.logLevel)

    # Streaming client
    connClientPath = os.path.dirname(p.errorPath)
    p.streamClient = os.path.join(connClientPath, 'src/connectionClient.py')

    # The mongo bits
    try:
        c, dbh = mdb.getHandle(host=p.dbHost,
                               port=p.dbPort,
                               db=p.db,
                               user=p.dbUser,
                               password=p.dbPassword)
        evCollHandle = dbh[p.eventsCollection]
        mgmtCollHandle = dbh[p.mgmtCollection]
        logging.debug("Connected and authenticated on the db.")
    except:
        logging.critical('Failed to connect to db and authenticate.',
                         exc_info=True)
        sys.exit()

    # Create a new management document if needed
    initialOID = setInitialPid(mgmtCollHandle)

    # Get the current events from crowded
    crowdedEvents = getCrowdedEvents(p)

    # Get the events currently stored by this app
    myEvents = getLocalEvents(p, evCollHandle)

    # Compare the 2 sets of events: what's old and new?
    oldEvents, newEvents = checkEvents(crowdedEvents, myEvents)

    # Expire old events from db, so that the new stream reflects the correct interest
    for oldEvent in oldEvents:
        print oldEvent
        logging.debug('Expiring Old Event in DB: %s' % (oldEvent))
        res = expireOldEvent(evCollHandle, oldEvent)

    # Create new item in the db
    for newEvent in newEvents:
        logging.debug('Creating New Event in DB: %s' % (newEvent))
        res = createLocalEvent(evCollHandle, newEvent)

    # Get the old process ID and kill it off
    pid = getPid(mgmtCollHandle)
    logging.debug('Current PID: %s' % (pid))

    # Only continue if there is a change in the events
    if len(oldEvents) > 0 or len(newEvents) > 0:

        if pid:
            logging.debug('Killing old process with ID: %s' % (pid))
            res = killOldProcess(pid)

        # Now create the new one
        newPid = processNewEvent(p)
        logging.debug('Creating a new process with PID: %s' % (newPid))

        # Update the current process id in mongo
        res = storePid(mgmtCollHandle, newPid)
        logging.debug('Stored the new PID: %s' % (res))

    mdb.close(c, dbh)
    logging.shutdown()