def delete_expired_results(): """ Delete results more than _days_to_keep_result. Sites including expired results are retested. """ worker.start_job() db_session = worker.get_session() tested_sites = set() expiry = datetime.utcnow() - timedelta(days=_days_to_keep_result) expired_results = db_session.query(Result).filter( Result.created_at < expiry).all() for result in expired_results: # Don't delete permanent image files if result.image_file.name in _permanent_images: result.image_file = None result.image_file_id = None db_session.flush() db_session.delete(result) if result.site_id not in tested_sites: tracker_id = 'tracker.{}'.format(random_string(10)) test_site.enqueue(result.site_id, tracker_id) tested_sites.add(result.site_id) db_session.commit() worker.finish_job()
def scrape_profile(site, usernames, stub=False, labels={}): """ Scrape a twitter or instagram account. """ redis = worker.get_redis() worker.start_job() try: if site == 'twitter': profiles = scrape_twitter_account(usernames, stub, labels) elif site == 'instagram': profiles = [] for username in usernames: profile = scrape_instagram_account(username, stub) profiles.append(profile) else: raise ScrapeException('No scraper exists for site: {}'.format(site)) for profile in profiles: redis.publish('profile', json.dumps(profile)) worker.finish_job() except requests.exceptions.HTTPError as he: response = he.response message = { 'usernames': usernames, 'site': site, 'code': response.status_code } if response.status_code == 404: message['error'] = 'Does not exist on {}.'.format(site) else: message['error'] = 'Cannot communicate with {} ({})' \ .format(site, response.status_code) message_str = json.dumps(message) redis.publish('profile', message_str) sys.stderr.write('{}\n'.format(message_str)) sys.stderr.write('{}\n'.format(response.text)) except ScrapeException as se: message = { 'usernames': usernames, 'site': site, 'error': se.message, } redis.publish('profile', json.dumps(message)) except: message = { 'usernames': usernames, 'site': site, 'error': 'Unknown error while fetching profile.', } redis.publish('profile', json.dumps(message)) raise
def delete_profile_posts(profile_id): ''' Delete profile posts. ''' worker.start_job() session = worker.get_session() solr = worker.get_solr() query = solr.Q(solr.Q(type_s='Post') & solr.Q(profile_id_i=profile_id)) solr.delete_by_query(query=query) solr.commit() worker.finish_job()
def delete_expired_archives(): """ Delete archives older than _days_to_keep_archive. """ worker.start_job() db_session = worker.get_session() expiry = datetime.utcnow() - timedelta(days=_days_to_keep_archive) db_session.query(Archive).filter(Archive.created_at < expiry).delete() db_session.commit() worker.finish_job()
def index_profile(profile_id): ''' Index a profile. ''' worker.start_job() session = worker.get_session() solr = worker.get_solr() profile = session.query(Profile).filter(Profile.id == profile_id).one() solr.add(app.index.make_profile_doc(profile)) solr.commit() worker.finish_job()
def scrape_avatar(id_, site, url): """ Get an twitter avatar from ``url`` and save it to the Profile identified by ``id_``. """ worker.start_job() redis = worker.get_redis() db_session = worker.get_session() avatar = None profile = db_session.query(Profile).filter(Profile.id == id_).first() if profile is None: raise ValueError('No profile exists with id={}'.format(id_)) if site == 'twitter': # Twitter points you to a scaled image by default, but we can # get the original resolution by removing "_normal" from the URL. # # See: https://dev.twitter.com/overview/general/user-profile-images-and-banners url = url.replace('_normal', '') # Update Avatar if it's already stored in the db for profile_avatar in profile.avatars: if profile_avatar.upstream_url == url: profile_avatar.end_date = datetime.today() avatar = profile_avatar break # Otherwise, scrape the new Avatar and append to the profile if avatar is None: response = requests.get(url) response.raise_for_status() if 'content-type' in response.headers: mime = response.headers['content-type'] else: mime = 'application/octet-stream' image = response.content avatar = Avatar(url, mime, image) profile.avatars.append(avatar) profile.current_avatar = avatar db_session.commit() worker.finish_job() redis.publish('avatar', json.dumps({ 'id': id_, 'thumb_url': '/api/file/' + str(avatar.thumb_file.id), 'url': '/api/file/' + str(avatar.file.id), }))
def sleep_determinate(period): ''' Sleep for a specified period of time with progress updates.''' total_sleep = 0 worker.start_job(total=int(math.ceil(period))) while total_sleep < period: time.sleep(1) total_sleep += 1 worker.update_job(current=total_sleep) worker.finish_job()
def create_archive(username, category_id, tracker_id, user_id): """ Archive summary of results in the database and store a zip archive in the data directory. """ redis = worker.get_redis() worker.start_job() db_session = worker.get_session() found_count = 0 not_found_count = 0 error_count = 0 results = (db_session.query(Result).options(subqueryload( Result.image_file)).filter(Result.tracker_id == tracker_id).all()) site_count = len(results) # Generate zip file filename = re.sub('[\W_]+', '', username) # Strip non-alphanumeric char zip_file_id = create_zip(filename, results, user_id) for result in results: if result.status == 'e': error_count += 1 elif result.status == 'f': found_count += 1 elif result.status == 'n': not_found_count += 1 archive = Archive(tracker_id=tracker_id, username=username, category_id=category_id, site_count=site_count, found_count=found_count, not_found_count=not_found_count, error_count=error_count, zip_file_id=zip_file_id, user_id=user_id) # Write to db db_session.add(archive) db_session.commit() # Publish message = { 'id': archive.id, 'name': archive.username, 'status': 'created', 'archive': archive.as_dict(), } redis.publish('archive', json.dumps(message)) worker.finish_job()
def scrape_avatar(id_, site, url): ''' Get an twitter avatar from ``url`` and save it to the Profile identified by ``id_``. ''' worker.start_job() redis = worker.get_redis() db_session = worker.get_session() avatar = None profile = db_session.query(Profile).filter(Profile.id==id_).first() if profile is None: raise ValueError('No profile exists with id={}'.format(id_)) if site == 'twitter': # Twitter points you to a scaled image by default, but we can # get the original resolution by removing "_normal" from the URL. # # See: https://dev.twitter.com/overview/general/user-profile-images-and-banners url = url.replace('_normal', '') # Update Avatar if it's already stored in the db for profile_avatar in profile.avatars: if profile_avatar.upstream_url == url: profile_avatar.end_date = datetime.today() avatar = profile_avatar # Otherwise, scrape the new Avatar and append to the profile if avatar is None: response = requests.get(url) response.raise_for_status() if 'content-type' in response.headers: mime = response.headers['content-type'] else: mime = 'application/octet-stream' image = response.content avatar = Avatar(url, mime, image) profile.avatars.append(avatar) db_session.commit() worker.finish_job() redis.publish('avatar', json.dumps({ 'id': id_, 'thumb_url': '/api/file/' + str(avatar.thumb_file.id), 'url': '/api/file/' + str(avatar.file.id), }))
def check_username(username, site_id, category_id, total, tracker_id, request_timeout=10, test=False): """ Check if `username` exists on the specified site. """ worker.start_job() redis = worker.get_redis() db_session = worker.get_session() # Make a splash request. site = db_session.query(Site).get(site_id) # Check site. splash_result = _splash_username_request(username, site, request_timeout) image_file = _save_image(db_session, splash_result) # Save result to DB. result = Result(tracker_id=tracker_id, site_name=splash_result['site']['name'], site_url=splash_result['url'], status=splash_result['status'], image_file_id=image_file.id, error=splash_result['error']) db_session.add(result) db_session.commit() if not test: # Notify clients of the result. current = redis.incr(tracker_id) result_dict = result.as_dict() result_dict['current'] = current # result_dict['image_file_url'] = image_file.url() # result_dict['image_name'] = image_file.name result_dict['total'] = total redis.publish('result', json.dumps(result_dict)) # If this username search is complete, then queue an archive job. if current == total: app.queue.schedule_archive(username, category_id, tracker_id) worker.finish_job() return result.id
def index_posts(post_ids): ''' Index a collection of posts. ''' worker.start_job() session = worker.get_session() solr = worker.get_solr() post_query = session.query(Post, Profile) \ .join(Post.author) \ .filter(Post.id.in_(post_ids)) for post, author in post_query: solr.add(app.index.make_post_doc(post, author)) solr.commit() worker.finish_job()
def scrape_profile(site, username): ''' Scrape a twitter or instagram account. ''' redis = worker.get_redis() worker.start_job() try: if site == 'twitter': profile = scrape_twitter_account(username) elif site == 'instagram': profile = scrape_instagram_account(username) else: raise ScrapeException('No scraper exists for site: {}'.format(site)) redis.publish('profile', json.dumps(profile)) worker.finish_job() except requests.exceptions.HTTPError as he: response = he.response message = {'username': username, 'site': site, 'code': response.status_code} if response.status_code == 404: message['error'] = 'Does not exist on Twitter.' else: message['error'] = 'Cannot communicate with Twitter ({})' \ .format(response.status_code) redis.publish('profile', json.dumps(message)) except ScrapeException as se: message = { 'username': username, 'site': site, 'error': se.message, } redis.publish('profile', json.dumps(message)) except Exception as e: message = { 'username': username, 'site': site, 'error': 'Unknown error while fetching profile.', } redis.publish('profile', json.dumps(message)) raise
def scrape_instagram_relations(id_): ''' Fetch friends and followers for the Instagram user identified by `id_`. The number of friends and followers to fetch is configured in Admin. ''' redis = worker.get_redis() db = worker.get_session() profile = db.query(Profile).filter(Profile.id==id_).first() proxies = _get_proxies(db) friends_results = 0 followers_results = 0 #max_results = _get_max_relations(db)['instagram'] max_results = get_config(db, 'max_relations_instagram', required=True).value try: max_results = int(max_results) except: raise ScrapeException( 'Value of max_relations_instagram must be an integer' ) friends_params = {} followers_params = {} total_results = max_results*2 if profile is None: raise ValueError('No profile exists with id={}'.format(id_)) # Get friends currently stored in db for this profile. friends_query = \ db.query(Profile.upstream_id) \ .join(\ profile_join_self, \ (profile_join_self.c.friend_id == Profile.id) ) \ .filter(profile_join_self.c.follower_id == id_) current_friends_ids = [friend.upstream_id for friend in friends_query] # Get followers currently stored in db for this profile. followers_query = \ db.query(Profile.upstream_id) \ .join(\ profile_join_self, \ (profile_join_self.c.follower_id == Profile.id) ) \ .filter(profile_join_self.c.friend_id == id_) current_followers_ids = [follower.upstream_id for follower in followers_query] worker.start_job(total=total_results) # Get friend IDs. friends_url = 'https://api.instagram.com/v1/users/{}/follows' \ .format(profile.upstream_id) while friends_results < max_results: # Get friends from Instagram API friends_response = requests.get( friends_url, params=friends_params, proxies=proxies, verify=False ) friends_response.raise_for_status() pagination = friends_response.json()['pagination'] for friend in friends_response.json()['data']: # Only store friends that are not already in db. if friend['id'] not in current_friends_ids: related_profile = Profile( 'instagram', friend['id'], friend['username'], is_stub=True ) db.add(related_profile) try: db.commit() except IntegrityError: db.rollback() related_profile = db \ .query(Profile) \ .filter(Profile.site=='instagram') \ .filter(Profile.upstream_id==friend['id']) \ .one() related_profile.name = friend['full_name'] profile.friends.append(related_profile) friends_results += 1 worker.update_job(current=friends_results) if friends_results == max_results: break # If there are more results, set the cursor paramater, otherwise finish if 'next_cursor' in pagination: friends_params['cursor'] = pagination['next_cursor'] else: break # No more results # Get follower IDs. followers_url = 'https://api.instagram.com/v1/users/{}/followed-by' \ .format(profile.upstream_id) # Get followers from Instagram API while followers_results < max_results: # Get friends from Instagram API followers_response = requests.get( followers_url, params=followers_params, proxies=proxies, verify=False ) followers_response.raise_for_status() pagination = followers_response.json()['pagination'] for follower in followers_response.json()['data']: # Only store followers that are not already in db. if follower['id'] not in current_followers_ids: related_profile = Profile( 'instagram', follower['id'], follower['username'], is_stub=True ) db.add(related_profile) try: db.commit() except IntegrityError: db.rollback() related_profile = db \ .query(Profile) \ .filter(Profile.site=='instagram') \ .filter(Profile.upstream_id==follower['id']) \ .one() related_profile.name = follower['full_name'] profile.followers.append(related_profile) followers_results += 1 worker.update_job(current=friends_results + followers_results) if followers_results == max_results: break # If there are more results, set the cursor paramater, otherwise finish if 'next_cursor' in pagination: followers_params['cursor'] = pagination['next_cursor'] else: break # No more results worker.finish_job() redis.publish('profile_relations', json.dumps({'id': id_}))
def scrape_twitter_posts(id_, recent): ''' Fetch tweets for the user identified by id_. Checks tweets already stored in db, and will only fetch older or newer tweets depending on value of the boolean argument 'recent', e.g. recent=True will return recent tweets not already stored in the db. The number of tweets to fetch is configured in the Admin. ''' db = worker.get_session() #max_results = _get_max_posts(db)['twitter'] max_results = get_config(db, 'max_posts_twitter', required=True).value try: max_results = int(max_results) except: raise ScrapeException('Value of max_posts_twitter must be an integer') worker.start_job(total=max_results) redis = worker.get_redis() author = db.query(Profile).filter(Profile.id==id_).first() proxies = _get_proxies(db) results = 0 max_id = None more_results = True count = 200 if author is None: raise ValueError('No profile exists with id={}'.format(id_)) # Get posts currently stored in db for this profile. post_query = db.query(Post) \ .filter(Post.author_id == id_) \ .order_by(Post.upstream_created.desc()) url = 'https://api.twitter.com/1.1/statuses/user_timeline.json' params = {'count': count, 'user_id': author.upstream_id} if post_query.count() > 0: # Only fetch posts newer than those already stored in db if recent: since_id = post_query[0].upstream_id params['since_id'] = str(since_id) # Only fetch posts older than those already stored in db else: max_id = post_query[post_query.count() -1].upstream_id params['max_id'] = str(max_id) while more_results: response = requests.get( url, params=params, proxies=proxies, verify=False ) response.raise_for_status() post_ids = list() tweets = response.json() if len(tweets) == 0: more_results = False if len(tweets) < count: more_results = False for tweet in tweets: # Twitter API result set includes the tweet with the max_id/since_id # so ignore it. if tweet['id_str'] != max_id: post = Post( author, tweet['id_str'], dateutil.parser.parse(tweet['created_at']), tweet['text'] ) if tweet['lang'] is not None: post.language = tweet['lang'] if tweet['coordinates'] is not None: post.latitude, post.longitude = tweet['coordinates'] place = tweet['place'] if place is not None: # Set longitude/latitude to the center the of bounding polygon. total_lon = 0 total_lat = 0 num_coords = 0 for lon, lat in place['bounding_box']['coordinates'][0]: total_lon += lon total_lat += lat num_coords += 1 post.longitude = total_lon / num_coords post.latitude = total_lat / num_coords # Set location to string identifying the place. post.location = '{}, {}'.format( place['full_name'], place['country'] ) db.add(post) db.flush() post_ids.append(post.id) # Set the max_id to the last tweet to get the next set of # results max_id = tweet['id_str'] params['max_id'] = max_id results += 1 worker.update_job(current=results) if results == max_results: more_results = False break db.commit() worker.finish_job() redis.publish('profile_posts', json.dumps({'id': id_})) app.queue.schedule_index_posts(post_ids)
def scrape_twitter_relations(id_): ''' Fetch friends and followers for the Twitter user identified by `id_`. The number of friends and followers to fetch is configured in Admin. ''' redis = worker.get_redis() db = worker.get_session() profile = db.query(Profile).filter(Profile.id==id_).first() proxies = _get_proxies(db) #max_results = _get_max_relations(db)['twitter'] max_results = get_config(db, 'max_relations_twitter', required=True).value try: max_results = int(max_results) except: raise ScrapeException( 'Value of max_relations_twitter must be an integer' ) friends_results = 0 friends_ids = [] followers_results = 0 followers_ids = [] friends_cursor = -1 followers_cursor = -1 if profile is None: raise ValueError('No profile exists with id={}'.format(id_)) params = { 'count': 5000, 'user_id': profile.upstream_id, 'stringify_ids': True, } # Get friends currently stored in db for this profile. friends_query = \ db.query(Profile.upstream_id) \ .join(\ profile_join_self, \ (profile_join_self.c.friend_id == Profile.id) ) \ .filter(profile_join_self.c.follower_id == id_) current_friends_ids = [friend.upstream_id for friend in friends_query] # Get followers currently stored in db for this profile. followers_query = \ db.query(Profile.upstream_id) \ .join(\ profile_join_self, \ (profile_join_self.c.follower_id == Profile.id) ) \ .filter(profile_join_self.c.friend_id == id_) current_followers_ids = [follower.upstream_id for follower in followers_query] ## Get friend IDs. friends_url = 'https://api.twitter.com/1.1/friends/ids.json' params['cursor'] = friends_cursor while friends_results < max_results: friends_response = requests.get( friends_url, params=params, proxies=proxies, verify=False ) friends_response.raise_for_status() # Ignore friends already in the db for friend_id in friends_response.json()['ids']: if friend_id not in current_friends_ids: friends_ids.append(friend_id) friends_results += 1 if friends_results == max_results: break friends_cursor = friends_response.json()['next_cursor'] if friends_cursor == 0: break # No more results else: params['cursor'] = friends_cursor # Get follower IDs. followers_url = 'https://api.twitter.com/1.1/followers/ids.json' params['cursor'] = followers_cursor while followers_results < max_results: followers_response = requests.get( followers_url, params=params, proxies=proxies, verify=False ) followers_response.raise_for_status() # Ignore followers already in the db for follower_id in followers_response.json()['ids']: if follower_id not in current_followers_ids: followers_ids.append(follower_id) followers_results += 1 if followers_results == max_results: break followers_cursor = followers_response.json()['next_cursor'] if followers_cursor == 0: break # No more results else: params['cursor'] = followers_cursor # Get username for each of the friend/follower IDs and create # a relationship in QuickPin. user_ids = [(uid, 'friend') for uid in friends_ids] + \ [(uid, 'follower') for uid in followers_ids] worker.start_job(total=len(user_ids)) chunk_size = 100 for chunk_start in range(0, len(user_ids), chunk_size): chunk_end = chunk_start + chunk_size chunk = user_ids[chunk_start:chunk_end] chunk_lookup = {id_:relation for id_,relation in chunk} lookup_url = 'https://api.twitter.com/1.1/users/lookup.json' lookup_response = requests.post( lookup_url, proxies=_get_proxies(db), verify=False, data={'user_id': ','.join(chunk_lookup.keys())} ) lookup_response.raise_for_status() relations = lookup_response.json() for related_dict in relations: uid = related_dict['id_str'] username = related_dict['screen_name'] related_profile = Profile('twitter', uid, username, is_stub=True) db.add(related_profile) try: db.commit() except IntegrityError: # Already exists: use the existing profile. db.rollback() related_profile = db \ .query(Profile) \ .filter(Profile.site=='twitter') \ .filter(Profile.upstream_id==uid) \ .one() _twitter_populate_profile(related_dict, related_profile) relation = chunk_lookup[uid] if relation == 'friend': profile.friends.append(related_profile) else: # relation == 'follower': profile.followers.append(related_profile) db.commit() worker.update_job(current=chunk_end) db.commit() worker.finish_job() redis.publish('profile_relations', json.dumps({'id': id_}))
def sleep_indeterminate(period): ''' Sleep for a specified period of time with no progress updates.''' worker.start_job() time.sleep(period) worker.finish_job()
def scrape_instagram_posts(id_, recent): """ Fetch instagram posts for the user identified by id_. Checks posts already stored in db, and will only fetch older or newer posts depending on value of the boolean argument 'recent', e.g. recent=True will return recent posts not already stored in the db. The number of posts to fetch is configured in the Admin. """ redis = worker.get_redis() db = worker.get_session() author = db.query(Profile).filter(Profile.id == id_).first() proxies = _get_proxies(db) max_results = get_config(db, 'max_posts_instagram', required=True).value try: max_results = int(max_results) except: raise ScrapeException('Value of max_posts_instagram must be an integer') min_id = None results = 0 params = {} if author is None: raise ValueError('No profile exists with id={}'.format(id_)) url = 'https://api.instagram.com/v1/users/{}/media/recent' \ .format(author.upstream_id) # Get last post currently stored in db for this profile. post_query = db.query(Post) \ .filter(Post.author_id == id_) \ .order_by(Post.upstream_created.desc()) \ if post_query.count() > 0: # Only fetch posts newer than those already stored in db if recent: min_id = post_query[0].upstream_id params['min_id'] = str(min_id) # Only fetch posts older than those already stored in db else: max_id = post_query[post_query.count() - 1].upstream_id params['max_id'] = str(max_id) worker.start_job(total=max_results) while results < max_results: response = requests.get( url, params=params, proxies=proxies, verify=False ) response.raise_for_status() post_ids = list() response_json = response.json()['data'] pagination = response.json()['pagination'] # Instagram API result includes post with min_id so remove it response_json[:] = [d for d in response_json if d.get('id') != min_id] for gram in response_json: if gram['caption'] is not None: text = gram['caption']['text'] else: text = None post = Post( author, gram['id'], datetime.fromtimestamp(int(gram['created_time'])), text ) if gram['location'] is not None: if 'latitude' in gram['location']: post.latitude = gram['location']['latitude'] post.longitude = gram['location']['longitude'] if 'name' in gram['location']: post.location = gram['location']['name'] if 'street_address' in gram['location']: post.location += ' ' + gram['location']['street_address'] if 'images' in gram: image_url = gram['images']['standard_resolution']['url'] name = os.path.basename(urlparse(image_url).path) img_response = requests.get(image_url, verify=False) mime = img_response.headers['Content-type'] image = img_response.content post.attachments.append(File(name, mime, image)) db.add(post) db.flush() post_ids.append(post.id) worker.update_job(current=results) results += 1 if results == max_results: break # If there are more results, set the max_id param, otherwise finish if 'next_max_id' in pagination: params['max_id'] = pagination['next_max_id'] else: break db.commit() worker.finish_job() redis.publish('profile_posts', json.dumps({'id': id_})) app.queue.schedule_index_posts(post_ids)
def scrape_instagram_relations(id_): """ Fetch friends and followers for the Instagram user identified by `id_`. The number of friends and followers to fetch is configured in Admin. """ redis = worker.get_redis() db = worker.get_session() profile = db.query(Profile).filter(Profile.id==id_).first() proxies = _get_proxies(db) friends_results = 0 followers_results = 0 max_results = get_config(db, 'max_relations_instagram', required=True).value try: max_results = int(max_results) except: raise ScrapeException( 'Value of max_relations_instagram must be an integer' ) friends_params = {} followers_params = {} total_results = max_results*2 if profile is None: raise ValueError('No profile exists with id={}'.format(id_)) # Get friends currently stored in db for this profile. friends_query = \ db.query(Profile.upstream_id) \ .join(\ profile_join_self, \ (profile_join_self.c.friend_id == Profile.id) ) \ .filter(profile_join_self.c.follower_id == id_) current_friends_ids = [friend.upstream_id for friend in friends_query] # Get followers currently stored in db for this profile. followers_query = \ db.query(Profile.upstream_id) \ .join(\ profile_join_self, \ (profile_join_self.c.follower_id == Profile.id) ) \ .filter(profile_join_self.c.friend_id == id_) current_followers_ids = [follower.upstream_id for follower in followers_query] worker.start_job(total=total_results) # Get friend IDs. friends_url = 'https://api.instagram.com/v1/users/{}/follows' \ .format(profile.upstream_id) while friends_results < max_results: # Get friends from Instagram API friends_response = requests.get( friends_url, params=friends_params, proxies=proxies, verify=False ) friends_response.raise_for_status() pagination = friends_response.json()['pagination'] for friend in friends_response.json()['data']: # Only store friends that are not already in db. if friend['id'] not in current_friends_ids: related_profile = Profile( 'instagram', friend['id'], friend['username'], is_stub=True ) db.add(related_profile) try: db.commit() except IntegrityError: db.rollback() related_profile = db \ .query(Profile) \ .filter(Profile.site=='instagram') \ .filter(Profile.upstream_id==friend['id']) \ .one() related_profile.name = friend['full_name'] profile.friends.append(related_profile) friends_results += 1 worker.update_job(current=friends_results) if friends_results == max_results: break # If there are more results, set the cursor paramater, otherwise finish if 'next_cursor' in pagination: friends_params['cursor'] = pagination['next_cursor'] else: break # No more results # Get follower IDs. followers_url = 'https://api.instagram.com/v1/users/{}/followed-by' \ .format(profile.upstream_id) # Get followers from Instagram API while followers_results < max_results: # Get friends from Instagram API followers_response = requests.get( followers_url, params=followers_params, proxies=proxies, verify=False ) followers_response.raise_for_status() pagination = followers_response.json()['pagination'] for follower in followers_response.json()['data']: # Only store followers that are not already in db. if follower['id'] not in current_followers_ids: related_profile = Profile( 'instagram', follower['id'], follower['username'], is_stub=True ) db.add(related_profile) try: db.commit() except IntegrityError: db.rollback() related_profile = db \ .query(Profile) \ .filter(Profile.site=='instagram') \ .filter(Profile.upstream_id==follower['id']) \ .one() related_profile.name = follower['full_name'] profile.followers.append(related_profile) followers_results += 1 worker.update_job(current=friends_results + followers_results) if followers_results == max_results: break # If there are more results, set the cursor paramater, otherwise finish if 'next_cursor' in pagination: followers_params['cursor'] = pagination['next_cursor'] else: break # No more results worker.finish_job() redis.publish('profile_relations', json.dumps({'id': id_}))
def scrape_twitter_relations(id_): """ Fetch friends and followers for the Twitter user identified by `id_`. The number of friends and followers to fetch is configured in Admin. """ redis = worker.get_redis() db = worker.get_session() profile = db.query(Profile).filter(Profile.id==id_).first() proxies = _get_proxies(db) max_results = get_config(db, 'max_relations_twitter', required=True).value try: max_results = int(max_results) except: raise ScrapeException( 'Value of max_relations_twitter must be an integer' ) friends_results = 0 friends_ids = [] followers_results = 0 followers_ids = [] friends_cursor = -1 followers_cursor = -1 if profile is None: raise ValueError('No profile exists with id={}'.format(id_)) params = { 'count': 5000, 'user_id': profile.upstream_id, 'stringify_ids': True, } # Get friends currently stored in db for this profile. friends_query = \ db.query(Profile.upstream_id) \ .join(\ profile_join_self, \ (profile_join_self.c.friend_id == Profile.id) ) \ .filter(profile_join_self.c.follower_id == id_) current_friends_ids = [friend.upstream_id for friend in friends_query] # Get followers currently stored in db for this profile. followers_query = \ db.query(Profile.upstream_id) \ .join(\ profile_join_self, \ (profile_join_self.c.follower_id == Profile.id) ) \ .filter(profile_join_self.c.friend_id == id_) current_followers_ids = [follower.upstream_id for follower in followers_query] ## Get friend IDs. friends_url = 'https://api.twitter.com/1.1/friends/ids.json' params['cursor'] = friends_cursor while friends_results < max_results: friends_response = requests.get( friends_url, params=params, proxies=proxies, verify=False, headers=TWITTER_HEADERS ) friends_response.raise_for_status() # Ignore friends already in the db for friend_id in friends_response.json()['ids']: if friend_id not in current_friends_ids: friends_ids.append(friend_id) friends_results += 1 if friends_results == max_results: break friends_cursor = friends_response.json()['next_cursor'] if friends_cursor == 0: break # No more results else: params['cursor'] = friends_cursor # Get follower IDs. followers_url = 'https://api.twitter.com/1.1/followers/ids.json' params['cursor'] = followers_cursor while followers_results < max_results: followers_response = requests.get( followers_url, params=params, proxies=proxies, verify=False, headers=TWITTER_HEADERS, ) followers_response.raise_for_status() # Ignore followers already in the db for follower_id in followers_response.json()['ids']: if follower_id not in current_followers_ids: followers_ids.append(follower_id) followers_results += 1 if followers_results == max_results: break followers_cursor = followers_response.json()['next_cursor'] if followers_cursor == 0: break # No more results else: params['cursor'] = followers_cursor # Get username for each of the friend/follower IDs and create # a relationship in QuickPin. user_ids = [(uid, 'friend') for uid in friends_ids] + \ [(uid, 'follower') for uid in followers_ids] worker.start_job(total=len(user_ids)) chunk_size = 100 for chunk_start in range(0, len(user_ids), chunk_size): chunk_end = chunk_start + chunk_size chunk = user_ids[chunk_start:chunk_end] chunk_lookup = {id_:relation for id_,relation in chunk} lookup_url = 'https://api.twitter.com/1.1/users/lookup.json' lookup_response = requests.post( lookup_url, proxies=_get_proxies(db), verify=False, headers=TWITTER_HEADERS, data={'user_id': ','.join(chunk_lookup.keys())} ) lookup_response.raise_for_status() relations = lookup_response.json() for related_dict in relations: uid = related_dict['id_str'] username = related_dict['screen_name'] related_profile = Profile('twitter', uid, username, is_stub=True) db.add(related_profile) try: db.commit() except IntegrityError: # Already exists: use the existing profile. db.rollback() related_profile = db \ .query(Profile) \ .filter(Profile.site=='twitter') \ .filter(Profile.upstream_id==uid) \ .one() _twitter_populate_profile(related_dict, related_profile) relation = chunk_lookup[uid] if relation == 'friend': profile.friends.append(related_profile) else: # relation == 'follower': profile.followers.append(related_profile) db.commit() worker.update_job(current=chunk_end) db.commit() worker.finish_job() redis.publish('profile_relations', json.dumps({'id': id_}))
def scrape_twitter_posts(id_, recent): """ Fetch tweets for the user identified by id_. Checks tweets already stored in db, and will only fetch older or newer tweets depending on value of the boolean argument 'recent', e.g. recent=True will return recent tweets not already stored in the db. The number of tweets to fetch is configured in the Admin. """ db = worker.get_session() max_results = get_config(db, 'max_posts_twitter', required=True).value try: max_results = int(max_results) except: raise ScrapeException('Value of max_posts_twitter must be an integer') worker.start_job(total=max_results) redis = worker.get_redis() author = db.query(Profile).filter(Profile.id==id_).first() proxies = _get_proxies(db) results = 0 max_id = None more_results = True count = 200 if author is None: raise ValueError('No profile exists with id={}'.format(id_)) # Get posts currently stored in db for this profile. post_query = db.query(Post) \ .filter(Post.author_id == id_) \ .order_by(Post.upstream_created.desc()) url = 'https://api.twitter.com/1.1/statuses/user_timeline.json' params = {'count': count, 'user_id': author.upstream_id} if post_query.count() > 0: # Only fetch posts newer than those already stored in db if recent: since_id = post_query[0].upstream_id params['since_id'] = str(since_id) # Only fetch posts older than those already stored in db else: max_id = post_query[post_query.count() -1].upstream_id params['max_id'] = str(max_id) while more_results: response = requests.get( url, params=params, proxies=proxies, verify=False, headers=TWITTER_HEADERS, ) response.raise_for_status() post_ids = list() tweets = response.json() if len(tweets) == 0: more_results = False if len(tweets) < count: more_results = False for tweet in tweets: # Twitter API result set includes the tweet with the max_id/since_id # so ignore it. if tweet['id_str'] != max_id: post = Post( author, tweet['id_str'], dateutil.parser.parse(tweet['created_at']), tweet['text'] ) if tweet['lang'] is not None: post.language = tweet['lang'] if tweet['coordinates'] is not None: post.latitude, post.longitude = tweet['coordinates'] place = tweet['place'] if place is not None: # Set longitude/latitude to the center the of bounding polygon. total_lon = 0 total_lat = 0 num_coords = 0 for lon, lat in place['bounding_box']['coordinates'][0]: total_lon += lon total_lat += lat num_coords += 1 post.longitude = total_lon / num_coords post.latitude = total_lat / num_coords # Set location to string identifying the place. post.location = '{}, {}'.format( place['full_name'], place['country'] ) db.add(post) db.flush() post_ids.append(post.id) # Set the max_id to the last tweet to get the next set of # results max_id = tweet['id_str'] params['max_id'] = max_id results += 1 worker.update_job(current=results) if results == max_results: more_results = False break db.commit() worker.finish_job() redis.publish('profile_posts', json.dumps({'id': id_})) app.queue.schedule_index_posts(post_ids)
def check_username(username, site_id, category_id, total, tracker_id, user_id, test=False): """ Check if `username` exists on the specified site. """ worker.start_job() redis = worker.get_redis() db_session = worker.get_session() # Get site site = db_session.query(Site).get(site_id) # Check site for username splash_result = _splash_username_request(username, site) # Save image file image_file = _save_image(db_session=db_session, scrape_result=splash_result, user_id=user_id, censor=site.censor_images) # Save result to DB. result = Result(tracker_id=tracker_id, site_id=splash_result['site']['id'], site_name=splash_result['site']['name'], site_url=splash_result['url'], status=splash_result['status'], image_file_id=image_file.id, username=username, error=splash_result['error'], user_id=user_id) if result.status == 'f': result.html = splash_result['html'] db_session.add(result) db_session.commit() if not test: # Notify clients of the result. current = redis.incr(tracker_id) result_dict = result.as_dict() result_dict['current'] = current # result_dict['image_file_url'] = image_file.url() # result_dict['image_name'] = image_file.name result_dict['total'] = total redis.publish('result', json.dumps(result_dict)) # If this username search is complete, then queue an archive job. if current == total: description = 'Archiving results ' \ 'for username "{}"'.format(username) worker.archive.create_archive.enqueue( username=username, category_id=category_id, tracker_id=tracker_id, jobdesc=description, timeout=_redis_worker['archive_timeout'], user_id=user_id) worker.finish_job() return result.id
def scrape_instagram_posts(id_, recent): ''' Fetch instagram posts for the user identified by id_. Checks posts already stored in db, and will only fetch older or newer posts depending on value of the boolean argument 'recent', e.g. recent=True will return recent posts not already stored in the db. The number of posts to fetch is configured in the Admin. ''' redis = worker.get_redis() db = worker.get_session() author = db.query(Profile).filter(Profile.id==id_).first() proxies = _get_proxies(db) max_results = get_config(db, 'max_posts_instagram', required=True).value try: max_results = int(max_results) except: raise ScrapeException('Value of max_posts_instagram must be an integer') min_id = None more_results = True results = 0 params = {} if author is None: raise ValueError('No profile exists with id={}'.format(id_)) url = 'https://api.instagram.com/v1/users/{}/media/recent' \ .format(author.upstream_id) # Get last post currently stored in db for this profile. post_query = db.query(Post) \ .filter(Post.author_id == id_) \ .order_by(Post.upstream_created.desc()) \ if post_query.count() > 0: # Only fetch posts newer than those already stored in db if recent: min_id = post_query[0].upstream_id params['min_id'] = str(min_id) # Only fetch posts older than those already stored in db else: max_id = post_query[post_query.count() -1].upstream_id params['max_id'] = str(max_id) worker.start_job(total=max_results) logging.warning('WORKER max results: {}'.format(max_results)) while results < max_results: response = requests.get( url, params=params, proxies=proxies, verify=False ) response.raise_for_status() post_ids = list() response_json = response.json()['data'] pagination = response.json()['pagination'] # Instagram API result includes post with min_id so remove it response_json[:] = [d for d in response_json if d.get('id') != min_id] for gram in response_json: if gram['caption'] is not None: text = gram['caption']['text'] else: text = None post = Post( author, gram['id'], datetime.fromtimestamp(int(gram['created_time'])), text ) if gram['location'] is not None: if 'latitude' in gram['location']: post.latitude = gram['location']['latitude'] post.longitude = gram['location']['longitude'] if 'name' in gram['location']: post.location = gram['location']['name'] if 'street_address' in gram['location']: post.location += ' ' + gram['location']['street_address'] if 'images' in gram: image_url = gram['images']['standard_resolution']['url'] name = os.path.basename(urlparse(image_url).path) img_response = requests.get(image_url, verify=False) mime = img_response.headers['Content-type'] image = img_response.content post.attachments.append(File(name, mime, image)) db.add(post) db.flush() post_ids.append(post.id) worker.update_job(current=results) results += 1 if results == max_results: break # If there are more results, set the max_id param, otherwise finish if 'next_max_id' in pagination: params['max_id'] = pagination['next_max_id'] else: break db.commit() worker.finish_job() redis.publish('profile_posts', json.dumps({'id': id_})) app.queue.schedule_index_posts(post_ids)