def proxies():
fake = Faker()
requests = []
for server in dictionary['SERVERS']:
requests.append(get(
'https://www.facebook.com/directory/pages/%(alphabet)s' % {
'alphabet': choice(list(ascii_uppercase) + ['others']),
},
headers={
'User-Agent': fake.user_agent(),
},
proxies={
'https': server,
},
timeout=15,
))
responses = map(requests)
for index, _ in enumerate(dictionary['SERVERS']):
status = 'OK'
if (
not responses[index]
or
not responses[index].status_code == 200
or
'Security Check' in responses[index].text
):
status = 'Not OK'
log.write(10, '%(server)-46s %(status)s' % {
'server': dictionary['SERVERS'][index],
'status': status,
}, 1)
def decorated_function(*args, **kwargs):
log.write(10, '%(function)s()' % {
'function': function.__name__
}, indent)
timer.start(function.__name__)
output = function(*args, **kwargs)
timer.stop(function.__name__)
log.write(10, '%(seconds).3f seconds' % {
'seconds': timer.get_seconds(function.__name__),
}, indent)
return output
def twitter_1(id):
log.write(10, id, 1)
with closing(database.session()) as session:
handle = session.query(models.handle).get(id)
log.write(10, handle.screen_name, 1)
tweets = []
for tweet in twitter.get_tweets('from:%(screen_name)s' % {
'screen_name': handle.screen_name,
}):
tweets.append(tweet)
for tweet in twitter.get_tweets('@%(screen_name)s' % {
'screen_name': handle.screen_name,
}):
tweets.append(tweet)
seven_days_ago = datetime.now() - timedelta(days=7)
for tweet in tweets:
if tweet['text'].startswith('@'):
continue
if tweet['text'].startswith('I posted'):
continue
if utilities.is_retweet(tweet['text']):
continue
if tweet['created_at'] <= seven_days_ago:
continue
instance = session.query(models.tweet).get(tweet['id'])
if not instance:
instance = models.tweet(**{
'id': tweet['id'],
})
instance.created_at = tweet['created_at']
instance.favorites = tweet['favorites']
instance.media = tweet['media']
instance.retweets = tweet['retweets']
instance.text = tweet['text']
instance.user_name = tweet['user_name']
instance.user_profile_image_url = tweet['user_profile_image_url']
instance.user_screen_name = tweet['user_screen_name']
session.add(instance)
session.commit()
session.refresh(handle)
log.write(10, len(tweets), 2)
def profiles_and_profiles_discussions_and_profiles_likes():
with closing(database.postgresql['session']()) as session:
log.write(10, 'profiles', 1)
for profile in session.query(
models.profile,
).filter(or_(
models.profile.numbers_likes > 0,
models.profile.numbers_discussions_absolute > 0,
models.profile.numbers_discussions_relative > 0,
)).all():
log.write(10, profile.id, 2)
log.write(10, 'profiles_discussions', 3)
for index, profile_discussion in enumerate(
profile.discussions.order_by('timestamp desc')[1:]
):
log.write(10, index + 2, 4)
session.delete(profile_discussion)
session.commit()
log.write(10, 'profiles_likes', 3)
for index, profile_like in enumerate(
profile.likes.order_by('timestamp desc')[1:]
):
log.write(10, index + 2, 4)
session.delete(profile_like)
session.commit()
log.write(10, 'profiles_discussions', 1)
for profile_discussion in session.query(
models.profile_discussion
).all():
profile_discussion.profile.refresh()
session.add(profile_discussion)
session.commit()
log.write(10, profile_discussion.profile.id, 2)
log.write(10, 'profiles_likes', 1)
for profile_like in session.query(models.profile_like).all():
profile_like.profile.refresh()
session.add(profile_like)
session.commit()
log.write(10, profile_like.profile.id, 2)
