def retrieveTweets(self, ID, Q, geoCode): '''retrieveTweets from twitter and store the feeds into MongoDB ''' passCnt = 0 logger.debug('retrieve tweets') #fetch the latest since_id and pass it in next twitter call #since_id = mongoInt.retrieveSinceID(ID) twits = twitterInt.retrieveTweets(Q, geoCode) map( lambda tw: tw.update({ 'created_time': timegm( time.gmtime( time.strptime(tw['created_at'], "%a %b %d %H:%M:%S +0000 %Y"))) }), twits) #map(lambda tw:tw.update({'created_time': int(time.gmtime(time.strptime(tw['created_at'],"%a %b %d %H:%M:%S +0000 %Y")))}),twits) #twits = twitterInt.retrieveTweetsBasedHashtag(Q) #if geoCode: # twits.extend(twitterInt.retrieveTweetBasedLocation(geoCode)) logger.debug( 'storing tweets of twitter of both location based on keyword mongoDb' ) #twits=sparkInt.wowFieldTrueOrFalse(twits) if len(twits): passCnt += mongoInt.insertFeedData(ID, twits) else: if not mongoInt.createCollection(ID): logger.warn('unable to create collection in mongodb') #page_sanitized = json_util.dumps(twits) # below returning to be removed has to be done from mongoDB only return twits
def createUserNode(self, decodedFBJson): ''' latitude & longitude also needs to be stored in neo4j for retrival of reveland tweets ''' #decodedFBJson=json.loads(decodedFBJson) remove true for now #if mongoInt.insertFBUserLoginData(decodedFBJson) or True: mongoInt.insertFBUserLoginData(decodedFBJson) neo4jInt.createUserNode(graphDB, decodedFBJson, 'user') interestList = ['hometown', 'location', 'work', 'education'] for intr in interestList: if 'work' in intr: keyIs = 'employer' elif 'education' in intr: keyIs = 'school' if intr in decodedFBJson: if isinstance(decodedFBJson[intr], list): for itm in decodedFBJson[intr]: if itm.get('type') == None: itm['type'] = keyIs data = facebookInt.getIdLocation(itm[keyIs]['id']) logger.debug('Facebook get address using id:%s', data) if 'location' in data: itm[keyIs].update(data['location']) else: data = facebookInt.getIdLocation(decodedFBJson[intr]['id']) logger.debug('Facebook get address using id:%s', data) decodedFBJson[intr].update(data['location']) #add IF check whther interest is part of data provided neo4jInt.createInterestNode(graphDB, decodedFBJson, intr) #creating mongoDb interest nodes with ID as thy are unique if not mongoInt.createCollection(decodedFBJson['id']): logger.warn('unable to create collection in mongodb') else: logger.debug('user key doesnot exists') logger.debug('dataFb decodedFBJson:%s', decodedFBJson) #else: #logger.debug('user already exists hence skipping the neo4J creation of nodes & interest') #once the nodes are created lets fetch the feeds return 1
def createUserNode(self,decodedFBJson): ''' latitude & longitude also needs to be stored in neo4j for retrival of reveland tweets ''' #decodedFBJson=json.loads(decodedFBJson) remove true for now #if mongoInt.insertFBUserLoginData(decodedFBJson) or True: mongoInt.insertFBUserLoginData(decodedFBJson) neo4jInt.createUserNode(graphDB,decodedFBJson,'user') interestList = ['hometown','location','work','education'] for intr in interestList: if 'work' in intr: keyIs='employer' elif 'education' in intr: keyIs='school' if intr in decodedFBJson: if isinstance(decodedFBJson[intr],list): for itm in decodedFBJson[intr]: if itm.get('type') == None: itm['type'] = keyIs data = facebookInt.getIdLocation(itm[keyIs]['id']) logger.debug('Facebook get address using id:%s',data) if 'location' in data: itm[keyIs].update(data['location']) else: data = facebookInt.getIdLocation(decodedFBJson[intr]['id']) logger.debug('Facebook get address using id:%s',data) decodedFBJson[intr].update(data['location']) #add IF check whther interest is part of data provided neo4jInt.createInterestNode(graphDB,decodedFBJson,intr) #creating mongoDb interest nodes with ID as thy are unique if not mongoInt.createCollection(decodedFBJson['id']): logger.warn('unable to create collection in mongodb') else: logger.debug('user key doesnot exists') logger.debug('dataFb decodedFBJson:%s',decodedFBJson) #else: #logger.debug('user already exists hence skipping the neo4J creation of nodes & interest') #once the nodes are created lets fetch the feeds return 1
def retrieveTweets(self,ID,Q,geoCode): '''retrieveTweets from twitter and store the feeds into MongoDB ''' passCnt = 0 logger.debug('retrieve tweets') #fetch the latest since_id and pass it in next twitter call #since_id = mongoInt.retrieveSinceID(ID) twits = twitterInt.retrieveTweets(Q,geoCode) map(lambda tw:tw.update({'created_time': str(int(time.mktime(time.strptime(tw['created_at'],"%a %b %d %H:%M:%S +0000 %Y"))))}),twits) #twits = twitterInt.retrieveTweetsBasedHashtag(Q) #if geoCode: # twits.extend(twitterInt.retrieveTweetBasedLocation(geoCode)) logger.debug('storing tweets of twitter of both location based on keyword mongoDb') #twits=sparkInt.wowFieldTrueOrFalse(twits) if len(twits): passCnt += mongoInt.insertFeedData(ID,twits) else: if not mongoInt.createCollection(ID): logger.warn('unable to create collection in mongodb') #page_sanitized = json_util.dumps(twits) # below returning to be removed has to be done from mongoDB only return twits