def get_facebook_identity(profile): profile = format_url(profile) pid, username = get_id_or_username(profile) if not pid and not username: return None response = check_if_exists(pid, username) if response: return response path = pid or username response = http_request('', "GET", FACEBOOK_HOST, '/%s' % path, {}) time.sleep(1) if response.is_error() or 'error' in response['raw_data']: return None response = simplejson.loads(response['raw_data']) username = '' if 'username' in response: username = unicode(response['username']) identity = { 'time' : datetime.now(), 'profile_id' : int(response['id']), 'profile_user' : username, 'response' : response, } facebook_identities = get_mongo_collection(FACEBOOK_IDENTITY_COLLECTION) facebook_identities.insert(identity) return response
def check_if_exists(pid, username): facebook_identities = get_mongo_collection(FACEBOOK_IDENTITY_COLLECTION) identity = None if pid: identity = facebook_identities.find_one({'profile_id' : int(pid)}) if username: identity = facebook_identities.find_one( {'profile_user' : unicode(username)}) if identity: return identity['response'] else: return None
def process_job(self, job): info = simplejson.loads(job.body) email = info['email'] username = info['username'] response = self._soundcloud(username, email) if response.is_error(): return 'not' profiles = get_mongo_collection(info['collection']) location = 'network_candidates.' + SOUNDCLOUD profiles.find_and_modify({'_id' : ObjectId(info['id'])}, {'$push' : {location : response}}) return 'ok'
def setUp(self): self.scavenger = LinkedinScavenger(NAME, IN, OUT) thread.start_new_thread(self.scavenger.run, ()) self.collection = get_mongo_collection(COLLECTION) social_profile = self.collection.SocialProfile() social_profile["time"] = datetime.now() social_profile["email"] = unicode(EMAIL) social_profile.save() self.json = {} self.json["email"] = EMAIL self.json["id"] = str(social_profile["_id"]) self.json["collection"] = COLLECTION
def setUp(self): self.scavenger = TwitterScavenger(NAME, IN, OUT) thread.start_new_thread(self.scavenger.run, ()) self.collection = get_mongo_collection(COLLECTION) social_profile = self.collection.SocialProfile() social_profile['time'] = datetime.now() social_profile['email'] = unicode(EMAIL) social_profile.save() self.json = {} self.json['email'] = EMAIL self.json['id'] = str(social_profile['_id']) self.json['collection'] = COLLECTION
def __init__(self): self.scavengers = [] self.processing_profiles = {} self.email_beanstalk = beanstalkc.Connection() self.response_beanstalk = beanstalkc.Connection() self.username_beanstalk = beanstalkc.Connection() self.username_beanstalk.use(USERNAME_QUEUE) self.response_beanstalk.watch(RESPONSE_QUEUE) thread.start_new_thread(self.watch_responses, ()) self.profiles = get_mongo_collection(MONGO_COLLECTION) for key in scavengers_dict: scavenger = scavengers_dict[key](key, key + '_in', RESPONSE_QUEUE) self.scavengers.append(scavenger) thread.start_new_thread(scavenger.run, ())
from pinterest_scavenger import PinterestScavenger from linkedin_scavenger import LinkedinScavenger from base.mongodb_utils import get_mongo_collection from datetime import datetime networks = { 'lastfm': LastfmScavenger, 'soundcloud': SoundcloudScavenger, 'twitter': TwitterScavenger, 'aboutme': AboutmeScavenger, 'pinterest': PinterestScavenger, 'linkedin': LinkedinScavenger, } if __name__=="__main__": profiles = get_mongo_collection('test') social_profile = profiles.SocialProfile() jsn = {} jsn['email'] = '*****@*****.**' jsn['username'] = '******' jsn['collection'] = 'test' social_profile['email'] = unicode(jsn['email']) social_profile['time'] = datetime.now() social_profile.save() jsn['id'] = str(social_profile['_id']) scavs = [] for k, v in networks.iteritems(): sc = v(k, k + "_in", "out") scavs.append(sc)