def retrieveTweets(self, ID, Q, geoCode):
'''retrieveTweets from twitter and store the feeds into MongoDB
'''
passCnt = 0
logger.debug('retrieve tweets')
#fetch the latest since_id and pass it in next twitter call
#since_id = mongoInt.retrieveSinceID(ID)
twits = twitterInt.retrieveTweets(Q, geoCode)
map(
lambda tw: tw.update({
'created_time':
timegm(
time.gmtime(
time.strptime(tw['created_at'],
"%a %b %d %H:%M:%S +0000 %Y")))
}), twits)
#map(lambda tw:tw.update({'created_time': int(time.gmtime(time.strptime(tw['created_at'],"%a %b %d %H:%M:%S +0000 %Y")))}),twits)
#twits = twitterInt.retrieveTweetsBasedHashtag(Q)
#if geoCode:
# twits.extend(twitterInt.retrieveTweetBasedLocation(geoCode))
logger.debug(
'storing tweets of twitter of both location based on keyword mongoDb'
)
#twits=sparkInt.wowFieldTrueOrFalse(twits)
if len(twits):
passCnt += mongoInt.insertFeedData(ID, twits)
else:
if not mongoInt.createCollection(ID):
logger.warn('unable to create collection in mongodb')
#page_sanitized = json_util.dumps(twits)
# below returning to be removed has to be done from mongoDB only
return twits
def createUserNode(self, decodedFBJson):
'''
latitude & longitude also needs to be stored in neo4j for retrival of reveland tweets
'''
#decodedFBJson=json.loads(decodedFBJson) remove true for now
#if mongoInt.insertFBUserLoginData(decodedFBJson) or True:
mongoInt.insertFBUserLoginData(decodedFBJson)
neo4jInt.createUserNode(graphDB, decodedFBJson, 'user')
interestList = ['hometown', 'location', 'work', 'education']
for intr in interestList:
if 'work' in intr:
keyIs = 'employer'
elif 'education' in intr:
keyIs = 'school'
if intr in decodedFBJson:
if isinstance(decodedFBJson[intr], list):
for itm in decodedFBJson[intr]:
if itm.get('type') == None:
itm['type'] = keyIs
data = facebookInt.getIdLocation(itm[keyIs]['id'])
logger.debug('Facebook get address using id:%s', data)
if 'location' in data:
itm[keyIs].update(data['location'])
else:
data = facebookInt.getIdLocation(decodedFBJson[intr]['id'])
logger.debug('Facebook get address using id:%s', data)
decodedFBJson[intr].update(data['location'])
#add IF check whther interest is part of data provided
neo4jInt.createInterestNode(graphDB, decodedFBJson, intr)
#creating mongoDb interest nodes with ID as thy are unique
if not mongoInt.createCollection(decodedFBJson['id']):
logger.warn('unable to create collection in mongodb')
else:
logger.debug('user key doesnot exists')
logger.debug('dataFb decodedFBJson:%s', decodedFBJson)
#else:
#logger.debug('user already exists hence skipping the neo4J creation of nodes & interest')
#once the nodes are created lets fetch the feeds
return 1
def createUserNode(self,decodedFBJson):
'''
latitude & longitude also needs to be stored in neo4j for retrival of reveland tweets
'''
#decodedFBJson=json.loads(decodedFBJson) remove true for now
#if mongoInt.insertFBUserLoginData(decodedFBJson) or True:
mongoInt.insertFBUserLoginData(decodedFBJson)
neo4jInt.createUserNode(graphDB,decodedFBJson,'user')
interestList = ['hometown','location','work','education']
for intr in interestList:
if 'work' in intr:
keyIs='employer'
elif 'education' in intr:
keyIs='school'
if intr in decodedFBJson:
if isinstance(decodedFBJson[intr],list):
for itm in decodedFBJson[intr]:
if itm.get('type') == None:
itm['type'] = keyIs
data = facebookInt.getIdLocation(itm[keyIs]['id'])
logger.debug('Facebook get address using id:%s',data)
if 'location' in data:
itm[keyIs].update(data['location'])
else:
data = facebookInt.getIdLocation(decodedFBJson[intr]['id'])
logger.debug('Facebook get address using id:%s',data)
decodedFBJson[intr].update(data['location'])
#add IF check whther interest is part of data provided
neo4jInt.createInterestNode(graphDB,decodedFBJson,intr)
#creating mongoDb interest nodes with ID as thy are unique
if not mongoInt.createCollection(decodedFBJson['id']):
logger.warn('unable to create collection in mongodb')
else:
logger.debug('user key doesnot exists')
logger.debug('dataFb decodedFBJson:%s',decodedFBJson)
#else:
#logger.debug('user already exists hence skipping the neo4J creation of nodes & interest')
#once the nodes are created lets fetch the feeds
return 1
def retrieveTweets(self,ID,Q,geoCode):
'''retrieveTweets from twitter and store the feeds into MongoDB
'''
passCnt = 0
logger.debug('retrieve tweets')
#fetch the latest since_id and pass it in next twitter call
#since_id = mongoInt.retrieveSinceID(ID)
twits = twitterInt.retrieveTweets(Q,geoCode)
map(lambda tw:tw.update({'created_time': str(int(time.mktime(time.strptime(tw['created_at'],"%a %b %d %H:%M:%S +0000 %Y"))))}),twits)
#twits = twitterInt.retrieveTweetsBasedHashtag(Q)
#if geoCode:
# twits.extend(twitterInt.retrieveTweetBasedLocation(geoCode))
logger.debug('storing tweets of twitter of both location based on keyword mongoDb')
#twits=sparkInt.wowFieldTrueOrFalse(twits)
if len(twits):
passCnt += mongoInt.insertFeedData(ID,twits)
else:
if not mongoInt.createCollection(ID):
logger.warn('unable to create collection in mongodb')
#page_sanitized = json_util.dumps(twits)
# below returning to be removed has to be done from mongoDB only
return twits
