def proxies(): fake = Faker() requests = [] for server in dictionary['SERVERS']: requests.append(get( 'https://www.facebook.com/directory/pages/%(alphabet)s' % { 'alphabet': choice(list(ascii_uppercase) + ['others']), }, headers={ 'User-Agent': fake.user_agent(), }, proxies={ 'https': server, }, timeout=15, )) responses = map(requests) for index, _ in enumerate(dictionary['SERVERS']): status = 'OK' if ( not responses[index] or not responses[index].status_code == 200 or 'Security Check' in responses[index].text ): status = 'Not OK' log.write(10, '%(server)-46s %(status)s' % { 'server': dictionary['SERVERS'][index], 'status': status, }, 1)
def decorated_function(*args, **kwargs): log.write(10, '%(function)s()' % { 'function': function.__name__ }, indent) timer.start(function.__name__) output = function(*args, **kwargs) timer.stop(function.__name__) log.write(10, '%(seconds).3f seconds' % { 'seconds': timer.get_seconds(function.__name__), }, indent) return output
def twitter_1(id): log.write(10, id, 1) with closing(database.session()) as session: handle = session.query(models.handle).get(id) log.write(10, handle.screen_name, 1) tweets = [] for tweet in twitter.get_tweets('from:%(screen_name)s' % { 'screen_name': handle.screen_name, }): tweets.append(tweet) for tweet in twitter.get_tweets('@%(screen_name)s' % { 'screen_name': handle.screen_name, }): tweets.append(tweet) seven_days_ago = datetime.now() - timedelta(days=7) for tweet in tweets: if tweet['text'].startswith('@'): continue if tweet['text'].startswith('I posted'): continue if utilities.is_retweet(tweet['text']): continue if tweet['created_at'] <= seven_days_ago: continue instance = session.query(models.tweet).get(tweet['id']) if not instance: instance = models.tweet(**{ 'id': tweet['id'], }) instance.created_at = tweet['created_at'] instance.favorites = tweet['favorites'] instance.media = tweet['media'] instance.retweets = tweet['retweets'] instance.text = tweet['text'] instance.user_name = tweet['user_name'] instance.user_profile_image_url = tweet['user_profile_image_url'] instance.user_screen_name = tweet['user_screen_name'] session.add(instance) session.commit() session.refresh(handle) log.write(10, len(tweets), 2)
def profiles_and_profiles_discussions_and_profiles_likes(): with closing(database.postgresql['session']()) as session: log.write(10, 'profiles', 1) for profile in session.query( models.profile, ).filter(or_( models.profile.numbers_likes > 0, models.profile.numbers_discussions_absolute > 0, models.profile.numbers_discussions_relative > 0, )).all(): log.write(10, profile.id, 2) log.write(10, 'profiles_discussions', 3) for index, profile_discussion in enumerate( profile.discussions.order_by('timestamp desc')[1:] ): log.write(10, index + 2, 4) session.delete(profile_discussion) session.commit() log.write(10, 'profiles_likes', 3) for index, profile_like in enumerate( profile.likes.order_by('timestamp desc')[1:] ): log.write(10, index + 2, 4) session.delete(profile_like) session.commit() log.write(10, 'profiles_discussions', 1) for profile_discussion in session.query( models.profile_discussion ).all(): profile_discussion.profile.refresh() session.add(profile_discussion) session.commit() log.write(10, profile_discussion.profile.id, 2) log.write(10, 'profiles_likes', 1) for profile_like in session.query(models.profile_like).all(): profile_like.profile.refresh() session.add(profile_like) session.commit() log.write(10, profile_like.profile.id, 2)