def get_facebook_identity(profile):
profile = format_url(profile)
pid, username = get_id_or_username(profile)
if not pid and not username:
return None
response = check_if_exists(pid, username)
if response:
return response
path = pid or username
response = http_request('', "GET", FACEBOOK_HOST, '/%s' % path, {})
time.sleep(1)
if response.is_error() or 'error' in response['raw_data']:
return None
response = simplejson.loads(response['raw_data'])
username = ''
if 'username' in response:
username = unicode(response['username'])
identity = {
'time' : datetime.now(),
'profile_id' : int(response['id']),
'profile_user' : username,
'response' : response,
}
facebook_identities = get_mongo_collection(FACEBOOK_IDENTITY_COLLECTION)
facebook_identities.insert(identity)
return response
def check_if_exists(pid, username):
facebook_identities = get_mongo_collection(FACEBOOK_IDENTITY_COLLECTION)
identity = None
if pid:
identity = facebook_identities.find_one({'profile_id' : int(pid)})
if username:
identity = facebook_identities.find_one(
{'profile_user' : unicode(username)})
if identity:
return identity['response']
else:
return None
def process_job(self, job):
info = simplejson.loads(job.body)
email = info['email']
username = info['username']
response = self._soundcloud(username, email)
if response.is_error():
return 'not'
profiles = get_mongo_collection(info['collection'])
location = 'network_candidates.' + SOUNDCLOUD
profiles.find_and_modify({'_id' : ObjectId(info['id'])},
{'$push' : {location : response}})
return 'ok'
def setUp(self):
self.scavenger = LinkedinScavenger(NAME, IN, OUT)
thread.start_new_thread(self.scavenger.run, ())
self.collection = get_mongo_collection(COLLECTION)
social_profile = self.collection.SocialProfile()
social_profile["time"] = datetime.now()
social_profile["email"] = unicode(EMAIL)
social_profile.save()
self.json = {}
self.json["email"] = EMAIL
self.json["id"] = str(social_profile["_id"])
self.json["collection"] = COLLECTION
def setUp(self):
self.scavenger = TwitterScavenger(NAME, IN, OUT)
thread.start_new_thread(self.scavenger.run, ())
self.collection = get_mongo_collection(COLLECTION)
social_profile = self.collection.SocialProfile()
social_profile['time'] = datetime.now()
social_profile['email'] = unicode(EMAIL)
social_profile.save()
self.json = {}
self.json['email'] = EMAIL
self.json['id'] = str(social_profile['_id'])
self.json['collection'] = COLLECTION
def __init__(self):
self.scavengers = []
self.processing_profiles = {}
self.email_beanstalk = beanstalkc.Connection()
self.response_beanstalk = beanstalkc.Connection()
self.username_beanstalk = beanstalkc.Connection()
self.username_beanstalk.use(USERNAME_QUEUE)
self.response_beanstalk.watch(RESPONSE_QUEUE)
thread.start_new_thread(self.watch_responses, ())
self.profiles = get_mongo_collection(MONGO_COLLECTION)
for key in scavengers_dict:
scavenger = scavengers_dict[key](key, key + '_in', RESPONSE_QUEUE)
self.scavengers.append(scavenger)
thread.start_new_thread(scavenger.run, ())
from pinterest_scavenger import PinterestScavenger
from linkedin_scavenger import LinkedinScavenger
from base.mongodb_utils import get_mongo_collection
from datetime import datetime
networks = {
'lastfm': LastfmScavenger,
'soundcloud': SoundcloudScavenger,
'twitter': TwitterScavenger,
'aboutme': AboutmeScavenger,
'pinterest': PinterestScavenger,
'linkedin': LinkedinScavenger,
}
if __name__=="__main__":
profiles = get_mongo_collection('test')
social_profile = profiles.SocialProfile()
jsn = {}
jsn['email'] = '*****@*****.**'
jsn['username'] = '******'
jsn['collection'] = 'test'
social_profile['email'] = unicode(jsn['email'])
social_profile['time'] = datetime.now()
social_profile.save()
jsn['id'] = str(social_profile['_id'])
scavs = []
for k, v in networks.iteritems():
sc = v(k, k + "_in", "out")
scavs.append(sc)