def make_new_users(self, query):
mongo_app = connections.mongo()
mongo_app.connect_to_mongo()
mongo_app.db.users.remove({
'$and': [{
'created_year': {
'$ne': query['created_year']
}
}, {
'created_day': {
'$ne': query['created_day']
}
}]
})
users = []
for member in mongo_app.db.users.find({
'$and': [{
'status': query['status']
}, {
'created_year': query['created_year']
}, {
'created_day': query['created_day']
}, {
'made_user': query['made_user']
}]
}):
users.append(member)
keys = connections.twitter_api_keys()
keys.get_existing_keys(no_of_keys=1)
api = keys.keys[0].connect_to_twitter()
try:
new_user = api.get_user(id=users[0]['screen_name'])
except Exception as e:
if e.reason[:22] == 'Failed to send request':
st.write('No Internet 😴')
st.stop()
if e.api_code == 50:
st.write('User not found, please check the name properly 🥱')
st.stop()
for member in users:
new_user = User()
new_user.make_new_user(member, api)
self.users.append(new_user)
def is_repeated(self):
count = 0
existing_keys = connections.twitter_api_keys()
existing_keys.get_existing_keys()
for key in existing_keys.keys:
if (self.key.api_key == key.api_key
or self.key.api_secrete_key == key.api_secrete_key
or self.key.access_token == key.access_token or
self.key.access_token_secret == key.access_token_secret):
count = 1
break
if count == 0:
return False
else:
return True
def is_user_added(self):
if (st.button('Done')):
if (self.screen_name == None or self.core_user == None
or self.status == None or self.screen_name == ''):
st.write('Please enter details properly 🎅👵')
st.stop()
else:
try:
keys = connections.twitter_api_keys()
keys.get_existing_keys(no_of_keys=1)
api = keys.keys[0].connect_to_twitter()
new_user = api.get_user(id=self.screen_name)
except Exception as e:
if e.reason[:22] == 'Failed to send request':
st.write('No Internet 😴')
st.stop()
if e.api_code == 50:
st.write(
'User not found, please check the name properly 🥱')
st.stop()
user = self.make_json()
mongo_app = connections.mongo()
mongo_app.connect_to_mongo()
mongo_app.db.users.insert(user)
#self.push_to_users_collection(user)
msg = f'@{self.screen_name} is added, Is Core user : {self.core_user}, @{self.screen_name} will be {self.status}, date is {self.created_day}, Add the users and press on Make users button to add users sofar'
st.success(msg)
def tweets_celery_collector(self, query):
#connecting to db
mongo_app = connections.mongo()
mongo_app.connect_to_mongo()
#today
today = datetime.date.today()
#connecting to twitter
api_keys = connections.twitter_api_keys()
api_keys.get_existing_keys(no_of_keys=1)
api = api_keys.keys[0].connect_to_twitter()
#setting time period
todaydt = datetime.datetime(today.year, today.month, today.day, 0, 0, 0)
startDate = todaydt - datetime.timedelta(days=query['days'])
endDate = todaydt - datetime.timedelta(days=2)
#set status in db
update_query = {
'$set': {
f"last_day.{query['source']}": int(today.strftime('%j')),
f"tweets_status.{query['source']}": 'collecting',
f"duration.{query['source']}": f"{startDate} - {endDate}"
}
}
mongo_app.db.reports.update({'report': 'data_collection'}, update_query)
#getting users
users = user_objects.user_set()
payload = query['user_payload']
users.get_users(payload)
#collect tweets
tweets = []
i = 0
j = 1
count = 0
for user in users.users:
i = i + 1
try:
tmpTweets = api.user_timeline(user.user['screen_name'])
except tweepy.TweepError as e:
try:
if e.reason[0:22] == 'Failed to send request':
return ('No Network')
except:
print('Skipping' + user.user['screen_name'])
print(i, user.user['screen_name'], j)
for tweet in tmpTweets:
if tweet.created_at < endDate and tweet.created_at > startDate:
tweet._json['created_day'] = int(
tweet.created_at.strftime('%j'))
tweet._json['created_year'] = tweet.created_at.year
tweet._json['tweeted_hour'] = int(
tweet._json['created_at'][11:13])
tweets.append(tweet)
try:
while (tmpTweets[-1].created_at > startDate):
print("Last Tweet @", tmpTweets[-1].created_at,
" - fetching some more")
i = i + 1
try:
tmpTweets = api.user_timeline(user.user['screen_name'],
max_id=tmpTweets[-1].id)
except tweepy.TweepError as e:
try:
if e.reason[0:22] == 'Failed to send request':
return ('No Network')
except:
print('Skipping' + user.user['screen_name'])
print(i, user.user['screen_name'], j)
for tweet in tmpTweets:
if tweet.created_at < endDate and tweet.created_at > startDate:
tweet._json['created_day'] = int(
tweet.created_at.strftime('%j'))
tweet._json['created_year'] = tweet.created_at.year
tweet._json['tweeted_hour'] = int(
tweet._json['created_at'][11:13])
tweets.append(tweet)
except IndexError:
print('*=*=*=*= NO TWEETS BY *=*=*=*=*=', user, j)
j = j + 1
#updating status in reports
mongo_app.db.reports.update({'report': 'data_collection'}, {
'$set': {
f"tweets_user.{query['source']}": user.user['screen_name']
}
})
#pulling json part of tweets status collected
tweets_json = []
for status in tweets:
tweets_json.append(status._json)
#removing duplicates
ids = {}
duplicates = []
index = 0
for tweet in tweets_json:
if tweet['id'] in ids:
duplicates.append(index)
else:
ids[tweet['id']] = 0
index += 1
for index in sorted(duplicates, reverse=True):
del tweets[index]
#inserting to database
i = 0
for tweet in tweets_json:
tweet['downloaded_day_year'] = int(today.strftime('%j'))
tweet['downloaded_year'] = int(today.year)
if '_id' in tweet:
tweet.pop('_id')
print(i)
if not mongo_app.db.tweets.find({'id': tweet['id']}).count() > 0:
mongo_app.db.tweets.insert_one(tweet)
count += 1
i = i + 1
#set status in db
mongo_app.db.reports.update({'report': 'data_collection'}, {
'$set': {
f"tweets_status.{query['source']}": 'collected',
f"quantity.{query['source']}": count
}
})
def user_friends_celery_collector(self, query):
#connecting to db
mongo_app = connections.mongo()
mongo_app.connect_to_mongo()
#today
today = datetime.date.today()
#connecting to twitter
api_keys = connections.twitter_api_keys()
api_keys.get_existing_keys()
#set status in db
update_query = {
'$set': {
f"last_day.{query['source']}": int(today.strftime('%j')),
f"user_friends_status.{query['source']}": 'collecting'
}
}
mongo_app.db.reports.update({'report': 'data_collection'}, update_query)
#getting users
users = user_objects.user_set()
payload = query['user_friends_payload']
users.get_users(payload)
#checking if it stopped in between
reports = []
for report in mongo_app.db.reports.find({'report': 'data_collection'}):
reports.append(report)
if len(reports) != 0:
if reports[0]['friends_user']['type'] == query['type']:
if reports[0]['friends_user']['user_no'] == len(users.users) - 1:
user_no = 0
elif reports[0]['friends_user']['user_no'] == 0:
user_no = 0
else:
user_no = reports[0]['friends_user']['user_no'] - 1
else:
user_no = 0
key_no = 0
api = api_keys.keys[key_no].connect_to_twitter_no_wait()
while user_no < len(users.users):
try:
i = 0
friends = []
for id in tweepy.Cursor(api.friends_ids,
screen_name=users.users[user_no].
user['screen_name']).items():
friends.append({'id': id})
i = i + 1
#Printing Status
print(i, users.users[user_no].user['screen_name'], user_no)
#updating dB
mongo_app.db.community.update_one(
{'_id': users.user_ids[user_no]},
{'$set': {
'friends_id': friends
}})
#writing status to reports
mongo_app.db.reports.update({'report': 'data_collection'}, {
'$set': {
'friends_user': {
'screen_name':
users.users[user_no].user['screen_name'],
'user_no': user_no,
'type': query['type']
}
}
})
user_no = user_no + 1
except tweepy.RateLimitError:
time.sleep(60)
key_no = (key_no + 1) % (len(api_keys.keys))
api = api_keys.keys[key_no].connect_to_twitter_no_wait()
print(key_no)
print(88)
except Exception as e:
try:
if e.reason[0:22] == 'Failed to send request':
return ('No Network')
except:
print('skipping ' +
users.users[user_no].user['screen_name'])
print(e)
user_no += 1
#set status in db
mongo_app.db.reports.update(
{'report': 'data_collection'},
{'$set': {
f"user_friends_status.{query['source']}": 'collected'
}})