def part(self):
try:
self.seen = time.time()
db.save_user(self)
return True
except Exception, e:
traceback.print_exc()
def create_user():
if not request.is_json:
return "Wrong JSON\n"
content = request.get_json()
print(content)
db.save_user(content)
return "OK\n"
async def on_member_update(self, before, after):
secure = client.get_channel(chan)
for role_for_plex in roleid:
role = after.guild.get_role(int(role_for_plex))
if (role in after.roles and role not in before.roles):
await after.send(
'Welcome To ' + PLEX_SERVER_NAME +
'. Just reply with your email so we can add you to Plex!')
await after.send(
'I will wait 10 minutes for your message, if you do not send it by then I will cancel the command.'
)
def check(m):
return m.author == after and not m.guild
try:
email = await client.wait_for('message',
timeout=600,
check=check)
except asyncio.TimeoutError:
await after.send(
'Timed Out. Message Server Admin So They Can Add You Manually.'
)
return
else:
await asyncio.sleep(5)
await after.send(
'Got it we will be processing your email shortly')
print(email.content) #make it go to a log channel
plexname = str(email.content)
if plexadd(plexname):
if auto_remove_user:
db.save_user(str(after.id), email.content)
await asyncio.sleep(20)
await after.send('You have Been Added To Plex!')
await secure.send(plexname + ' ' + after.mention +
' was added to plex')
else:
await after.send(
'There was an error adding this email address. Message Server Admin.'
)
return
elif (role not in after.roles and role in before.roles):
if auto_remove_user:
try:
user_id = after.id
email = db.get_useremail(user_id)
plexremove(email)
deleted = db.delete_user(user_id)
if deleted:
print("Removed {} from db".format(email))
await secure.send(plexname + ' ' + after.mention +
' was removed from plex')
else:
print("Cannot remove this user from db.")
except:
print("Cannot remove this user from plex.")
return
def current_user():
data = make_request("/user")
if data and data.get("email", None):
db.save_user(data["login"], data.get("email", None), data.get("name", None))
elif data:
email = make_request("/user/emails")[0]
data["email"] = email
db.save_user(data["login"], email, data.get("name", None))
return data
def enroll():
if request.method == 'GET':
return render_template('enroll.html'), 200
else:
username = request.form.get('form-username', default='user')
password = request.form.get('form-password', default='pass')
if db.get_pass(username):
return 'existed'
else:
db.save_user(username, password)
return 'ok'
def user_info(username):
existing = db.user(username)
if existing:
return existing, True
u = "http://github.com/api/v2/json/user/show/" + username
data = json.load(urlopen(u))
if "error" in data:
raise Error("GitHub error: " + repr(data["error"]))
data = data["user"]
db.save_user(username, data.get("email", None), data.get("name", None))
return data, False
def signup():
if current_user.is_authenticated:
return redirect(url_for('home'))
message = ''
if request.method == 'POST':
username = request.form.get('username')
email = request.form.get('email')
password = request.form.get('password')
try:
save_user(username, email, password)
return redirect(url_for('login'))
except DuplicateKeyError:
message = 'User is already exist!'
return render_template('signup.html', message=message)
def signup():
if current_user.is_authenticated:
return redirect(url_for('home'))
if request.method=='POST':
username=request.form.get('username')
email=request.form.get('email')
password_input=request.form.get('password')
try:
save_user(username,email,password_input)
flash('user created successfully !','success')
return redirect(url_for('login'))
except DuplicateKeyError:
flash('user already exist !','danger')
return render_template('signup.html')
def signup():
if current_user.is_authenticated:
return redirect(url_for("home"))
message = ""
if request.method == "POST":
username = request.form.get("username")
email = request.form.get("email")
password = request.form.get("password")
try:
save_user(username, email, password)
return redirect(url_for("login"))
except DuplicateKeyError:
message = "User already exists!"
return render_template("signup.html", message=message)
def signup():
message = ''
if request.method == "POST":
username = request.form.get('username')
password = request.form.get('password')
email_address = request.form.get('email')
user = get_user(username)
message = 'ã…¤'
if user:
message = '{} already exist'.format(username)
else:
save_user(username, password, email_address)
return redirect(url_for('login'))
return render_template('signup.html', message=message)
def search_tweets_referring_meps(database):
# get candidate meps screen names
meps = pd.read_csv('gr_candidate_meps.csv')
twitter_meps = meps[meps['Twitter username'].str.startswith(
'@')]['Screen Name'].str[1:]
# find tweets mentioning the candidates accounts
operators = ["@", "to:", "#"]
for mep in twitter_meps:
tweets = []
for op in operators:
first_attempt = True
tmp_tweets = []
while first_attempt or len(tmp_tweets) > 0:
if first_attempt:
tmp_tweets = api.search(q=op + mep,
count=100,
include_entities=True,
tweet_mode='extended')
first_attempt = False
else:
tmp_tweets = api.search(q=op + mep,
count=100,
include_entities=True,
tweet_mode='extended',
max_id=tweets[-1].id)[1:]
tweets.extend(tmp_tweets)
print mep, len(tweets)
for tweet in tweets:
# save the user who wrote the tweet
db_user_id = db.get_user(tweet.user.id_str,
database) or db.save_user(
tweet.user, database)
# save the tweet
if not db.get_tweet(tweet.id_str, database):
db.save_tweet(tweet, db_user_id, database)
def signup():
if current_user.is_authenticated:
return render_template('index.html')
message = ' '
if request.method == 'POST':
username = request.form.get('username')
email = request.form.get('email')
password = request.form.get('password')
try:
db.save_user(username, email, password)
flash('Registered successfully. Please login.', 'success')
return redirect(url_for('login'))
except DuplicateKeyError:
message = "User already exists!"
return render_template('signup.html', message=message)
def signup():
message = ''
if request.method == 'POST':
username = request.form.get('username')
password_input = request.form.get('password')
email_input = request.form.get('email')
box_name_input = request.form.get('box_name')
user = save_user(username, email_input, password_input, box_name_input)
if user:
save_box(box_name_input)
app.logger.info(
f"{username} added with box {box_name_input} linked to its account."
)
login_user(user)
return redirect(f"box/{user.box_name}/")
else:
if not get_box(box_name_input):
app.logger.info(f"{username} already used")
message = "Username already exist, please try with another one."
else:
app.logger.info("Box already in use")
message = "Box name already registered."
return render_template('signup.html', message=message)
def signup():
#if current_user.is_authenticated:
form = RegisterForm()
#if current_user.is_authenticated:
# return redirect(url_for('chatshoppie'))
message = ''
if request.method == 'POST' and form.validate_on_submit():
username = str(request.form.get('name')).strip()
email = str(request.form.get('email')).strip()
mobile = str(request.form.get('mobile')).strip()
password = str(request.form.get('password')).strip()
print(username)
print("name")
# username = username
key = email
token = s.dumps(key, salt='email-confirm')
#zoho email
try:
isUserPresent = save_user(username, email, mobile, password)
print('isuser present ', isUserPresent)
if (isUserPresent):
flash("**A User has already registered with this email",
'danger')
return redirect(url_for('signup'))
else:
msg = Message('Chatshoppie',
sender='*****@*****.**',
recipients=[key])
link = url_for('confirm_email', token=token, _external=True)
msg.subject = "Account Confirmation Email from Chatshoppie.com"
msg.html = render_template('/email.html',
username=username,
link=link,
key=key)
mail.send(msg)
return render_template("thankyou.html")
except:
print('exception ::')
message = "User already exists!"
flash("**Unable to register. Please contact Admin", 'danger')
#return render_template("register.html",async_mode=socketio.async_mode,form=form)
return render_template("register.html", form=form)
async def on_message(self, message):
secure = client.get_channel(chan)
if message.author.id == self.user.id:
return
if message.author.id == ownerid:
if message.content.startswith('-dbadd'):
mgs = message.content.replace('-dbadd ', '')
try:
mgs = mgs.split(' ')
email = mgs[0]
bad_chars = ['<', '>', '@', '!']
user_id = mgs[1]
for i in bad_chars:
user_id = user_id.replace(i, '')
db.save_user(user_id, email)
await message.channel.send('The user has been added to db!'
)
await secure.send(email + ' ' + mgs[1] +
' was added to plex')
except:
await message.channel.send('Cannot add this user to db.')
print("Cannot add this user to db.")
await message.delete()
if str(message.channel) == str(secure):
if message.content.startswith('-plexadd'):
mgs = message.content.replace('-plexadd ', '')
if plexadd(mgs):
await message.channel.send(
'The email has been added! {0.author.mention}'.format(
message))
else:
message.channel.send(
'Error Check Logs! {0.author.mention}'.format(message))
if message.content.startswith('-plexrm'):
mgs = message.content.replace('-plexrm ', '')
if plexremove(mgs):
await message.channel.send(
'The email has been removed! {0.author.mention}'.
format(message))
else:
message.channel.send(
'Error Check Logs! {0.author.mention}'.format(message))
def signup():
if current_user.is_authenticated:
print(current_user)
return redirect(url_for('home')) # checks if user is already logged in
# user does not need to log in again
message = ''
if request.method == 'POST':
username = request.form.get('username')
email = request.form.get('email')
password_input = request.form.get('password')
img = request.form.get('imgdata')
app.logger.info(img)
try:
save_user(username, email, password_input, img)
return redirect(url_for('login'))
except DuplicateKeyError:
message = "User Already Exists"
return render_template('signup.html', message=message)
def signup():
if current_user.is_authenticated:
return redirect(url_for('home'))
message = ''
if request.method == 'POST':
username = request.form.get('username')
email = request.form.get('email')
password = request.form.get('password')
usn = request.form.get('usn')
name = request.form.get('name')
branch = request.form.get('branch')
try:
save_user(username, email, password, usn, name, branch)
return redirect(url_for('login'))
except DuplicateKeyError:
message = "User already exists!"
return render_template('registration.html', message=message)
async def on_member_update(self, before, after):
secure = client.get_channel(chan)
for role_for_plex in roleid:
role = after.guild.get_role(int(role_for_plex))
if (role in after.roles and role not in before.roles):
await after.send("Welcome to Neptune! 🔱 Reply to this message with the email linked to your Plex account and I'll send you an invite.")
await after.send("If I don't hear from you in 10 minutes, I'll cancel this request.")
def check(m):
return m.author == after and not m.guild
try:
email = await client.wait_for('message', timeout=600, check=check)
except asyncio.TimeoutError:
await after.send("Time's up! ⏰ Contact the admin to have them add you manually.")
return
else:
await asyncio.sleep(5)
await after.send("Thanks! You'll get an email with your invitation soon. 💌")
print(email.content) #make it go to a log channel
plexname = str(email.content)
if plexadd(plexname):
if auto_remove_user:
db.save_user(str(after.id), email.content)
await asyncio.sleep(20)
await after.send("You're in! 🎉 Enjoy your time with Neptune and let us know if you have any questions.")
await secure.send(plexname + ' ' + after.mention + ' was added to plex')
else:
await after.send('There was an error adding this email address. Message the admin for help.')
return
elif(role not in after.roles and role in before.roles):
if auto_remove_user:
try:
user_id = after.id
email = db.get_useremail(user_id)
plexremove(email)
deleted = db.delete_user(user_id)
if deleted:
print("Removed {} from db".format(email))
await secure.send(plexname + ' ' + after.mention + ' was removed from plex')
else:
print("Cannot remove this user from db.")
except:
print("Cannot remove this user from plex.")
return
def on_text(self, update, context):
username = update.effective_user.name
last_cmd = context.user_data.get('last_cmd', '')
if last_cmd == 'login':
res = finance_requests.login(update.message.text)
if res:
db.save_user(username, res['api_key'])
context.user_data['api_key'] = res['api_key']
self.on_start(update, context)
else:
context.user_data['api_key'] = ''
self.reply(update, 'Bad api key. Retry again.')
context.user_data['last_cmd'] = ''
elif last_cmd == 'add_income__title':
context.user_data['income'] = {}
context.user_data['income']['title'] = update.message.text
self.on_add_income__add_category(update, context)
elif last_cmd == 'add_income__category':
context.user_data['income']['selectedCategories'] = [
update.message.text
]
self.on_add_income__add_amount(update, context)
elif last_cmd == 'add_income__amount':
context.user_data['income']['amount'] = update.message.text
self.on_add_income__save(update, context)
elif last_cmd == 'add_expense__title':
context.user_data['expense'] = {}
context.user_data['expense']['title'] = update.message.text
self.on_add_expense__add_category(update, context)
elif last_cmd == 'add_expense__category':
context.user_data['expense']['selectedCategories'] = [
update.message.text
]
self.on_add_expense__add_amount(update, context)
elif last_cmd == 'add_expense__amount':
context.user_data['expense']['amount'] = update.message.text
self.on_add_expense__save(update, context)
else:
self.reply(update,
f'''I don't know "{update.message.text}" command :(''')
context.user_data['last_cmd'] = ''
def add_user():
json_input = request.get_json(force=True)
try:
token = json_input['token']
child = json_input['child']
username = child['username']
password = child['password']
email = child['email']
except Exception as err:
return Response(400, str(err)).get_json()
try:
save_user(username, email, password)
except DuplicateKeyError as err:
return Response(400, str(err)).get_json()
return Response(200, 'created a user').get_json()
def callback():
# Get authorization code Google sent back to you
code = request.args.get("code")
# Find out what URL to hit to get tokens that allow you to ask for
# things on behalf of a user
google_provider_cfg = get_google_provider_cfg()
token_endpoint = google_provider_cfg["token_endpoint"]
# Prepare and send a request to get tokens! Yay tokens!
token_url, headers, body = client.prepare_token_request(
token_endpoint,
authorization_response=request.url,
redirect_url=request.base_url,
code=code)
token_response = requests.post(
token_url,
headers=headers,
data=body,
auth=(app.config['GOOGLE_CLIENT_ID'],
app.config['GOOGLE_CLIENT_SECRET']),
)
# Parse the tokens!
client.parse_request_body_response(json.dumps(token_response.json()))
# Now that you have tokens (yay) let's find and hit the URL
# from Google that gives you the user's profile information,
# including their Google profile image and email
userinfo_endpoint = google_provider_cfg["userinfo_endpoint"]
uri, headers, body = client.add_token(userinfo_endpoint)
userinfo_response = requests.get(uri, headers=headers, data=body)
# You want to make sure their email is verified.
# The user authenticated with Google, authorized your
# app, and now you've verified their email through Google!
if userinfo_response.json().get("email_verified"):
unique_id = userinfo_response.json()["sub"]
users_email = userinfo_response.json()["email"]
users_name = userinfo_response.json()["given_name"]
else:
return "User email not available or not verified by Google.", 400
# Create a user in your db with the information provided
# by Google
user = {'id': unique_id, 'name': users_name, 'email': users_email}
session_user = User(id_=unique_id, name=users_name, email=users_email)
dbuser = db.get_user(unique_id)
# Doesn't exist? Add it to the database.
if not dbuser:
resp = db.save_user(user)
# Begin user session by logging the user in
login_user(session_user)
# Send user back to homepage
return redirect(url_for('home'))
def signup():
if current_user.is_authenticated:
return redirect(url_for('home'))
message = ''
if request.method == "POST":
username = request.form.get('username')
email = request.form.get('email')
password = request.form.get('password')
# signup 폼으로 작성한 데이터를 저장하고, 성공하면, login 페이지로 이동
try:
save_user(username, email, password)
return redirect(url_for('login'))
# 이미 있는 사용자라면, message 로 해당 문구를 보여준다.
except DuplicateKeyError:
message = "User already exists!"
return render_template('signup.html', message=message)
def signup():
if current_user.is_authenticated:
return redirect(url_for('UserPage'))
message = ''
if request.method == 'POST':
username = request.form.get('username')
email = request.form.get('email')
password = request.form.get('password')
contact = request.form.get('contact')
dob = request.form.get('dob')
gender = request.form.get("gender")
file = request.files["profile_pic"]
file.save(os.path.join(app.config['UPLOAD_DIR'], file.filename))
try:
save_user(username, email, password, contact, dob, gender,
file.filename)
return redirect(url_for('login'))
except DuplicateKeyError:
message = "User already exists!"
return render_template('signup.html', message=message)
def _save_to_db(self, session):
"""Save a given session to the database. """
cxn = self.get_cxn()
c = cxn.cursor()
del self[session.id]
c.execute(
'''
INSERT INTO user_sessions
(user_id, session_id, remote_addr, creation_time, access_time)
VALUES (%s, %s, %s, %s, %s)
''',
(
session.user.id,
session.id,
session.get_remote_address(),
session.get_creation_time(),
session.get_access_time(),
),
)
db.save_user(c, session.user)
cxn.commit()
def autolike_users(self, max_count):
"""
Start autoliking tinder users
:param max_count: maximum number of users to like
:return: summary of how many users were liked and matched in json
"""
matches = 0
number_of_users = 0
for user in self.session.nearby_users(max_count):
db.save_user(user, [])
db.update_user_first_shown_date(user.id)
number_of_users += 1
match = user.like()
db.update_user_swipe_date(user.id, True)
if match:
matches += 1
print "%s match=%s %d/%d %f" % (user, match,
matches, number_of_users,
float(matches) / number_of_users)
time.sleep(
0.2) # To prevent spamming the server with too many requests
print("Matched with %d/%d users\n" % (matches, number_of_users))
return json.dumps({"users": number_of_users, "matched": matches})
def index():
form = UserForm()
if form.validate_on_submit():
try:
save_user(form.name.data)
except DuplicateKeyError:
print("name error")
return render_template("index.html",
form=form,
message="User already exists")
if form.create.data:
try:
save_room(form.room.data, form.name.data)
except DuplicateKeyError:
print("room exists")
remove_user(form.name.data)
return render_template(
"index.html",
form=form,
message="Cannot create room which already exists")
if form.join.data:
if not room_exists(form.room.data):
print("room doesn't exists")
remove_user(form.name.data)
return render_template(
"index.html",
form=form,
message="Cannot join room which doesn't exists")
add_member(form.room.data, form.name.data)
return redirect(
url_for("lobby", user=form.name.data, room=form.room.data))
return render_template("index.html", form=form, message='')
def create_account():
ip = request.environ.get('HTTP_X_REAL_IP', request.remote_addr)
json_input = request.get_json(force=True)
try:
username = json_input['username']
password = json_input['password']
email = json_input['email']
full_name = json_input['name']
current_app.logger.info('{} trying to create a new account, {}'.format(
request.remote_addr, username))
if len(username) == 0 or len(password) == 0 or len(email) == 0 or len(
full_name) == 0:
raise KeyError('Empty field.')
except KeyError as e:
return jsonify(
{'Error':
'Invalid request: Missing required field. {}'.format(e)}), 400
except TypeError as e:
return jsonify(
{'Error':
'Invalid request: Must be a json/dict. {}'.format(e)}), 400
regex_email = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
if not re.match(regex_email, email):
return jsonify({'Error': 'Invalid email address.'}), 400
if not re.match("^[A-Za-z ]*$", full_name):
return jsonify({'Error': 'Invalid full name.'}), 400
if re.match("^[A-Za-z_]*$", username):
if len(password) < 6:
return jsonify({'Error': 'Password must be 6+ characters.'}), 400
try:
user_id = save_user(username, email, password, full_name)
current_app.logger.info('{} created a new account, {}'.format(
request.remote_addr, username))
add_log_event(200, user_id, 'Signup', ip_address=ip)
return jsonify({'Success': 'User created.'.format(user_id)}), 200
except DuplicateKeyError:
return jsonify({'Error':
'Username or email is already in use.'}), 400
else:
return jsonify({'Error': 'Bad username.'}), 400
return jsonify({'Error': ''}), 500
def get_recommendations(self):
"""
Get new recommendations from tinder
:return: Recommendations as JSON
"""
ret = []
recommendations = self.session.nearby_users()
for counter, recommendation in enumerate(recommendations):
user = db.save_user(recommendation, [])
db.update_user_first_shown_date(recommendation.id)
print(recommendation)
ret.append(db_utils.database_user_to_object(user))
if counter == 30:
break
return jsonpickle.dumps(ret)
def signup():
if current_user.is_authenticated:
return redirect(url_for('home'))
message = ''
if request.method == 'POST':
username = request.form.get('username')
email = request.form.get('email')
mobile = request.form.get('mobile')
password = request.form.get('password')
try:
isUserPresent = save_user(username, email,mobile, password)
print('isuser present ',isUserPresent)
if (isUserPresent):
message = "User already exists!"
else:
return redirect(url_for('login'))
except:
print('exception ::')
message = "User already exists!"
return render_template('signup.html', message=message)
def set_points(self, am):
self.points = int(am)
db.save_user(self)
return self.points
def add_points(self, am):
self.points += int(am)
if not isinstance(self.points, int):
self.points = int(self.points)
db.save_user(self)
return self.points
def save(self):
"""Save the user to the database (or create if it is not present)"""
if self.id:
db.save_user(self.id, self.email, self.password)
else:
self.id = db.create_user(self.email, self.password)
def create(id_, name, email):
db.save_user(id, name, email)
async def on_message(self, message):
secure = client.get_channel(chan)
if message.author.id == self.user.id:
return
if message.author.guild_permissions.administrator:
if message.content.startswith('-plex add'):
mgs = message.content.replace('-plex add ','')
if plexadd(mgs):
await secure.send('The email has been added! {0.author.mention}'.format(message))
else:
await secure.send('Error Check Logs! {0.author.mention}'.format(message))
if message.content.startswith('-plex rm'):
mgs = message.content.replace('-plex rm ','')
if plexremove(mgs):
await secure.send('The email has been removed! {0.author.mention}'.format(message))
else:
await secure.send('Error Check Logs! {0.author.mention}'.format(message))
if message.author.guild_permissions.administrator:
if message.content.startswith('-db add'):
mgs = message.content.replace('-db add ','')
try:
mgs = mgs.split(' ')
email = mgs[0]
if(plexadd(email)):
await message.channel.send('User has been added to plex.')
else:
await message.channel.send('Error adding email to plex. Ignore this if the email already exist in plex.')
bad_chars = ['<','>','@','!']
user_id = mgs[1]
for i in bad_chars:
user_id = user_id.replace(i, '')
db.save_user(user_id, email)
await message.channel.send(email + ' ' + mgs[1] + ' was added to the database.')
except:
await message.channel.send('Cannot add this user to db.')
print("Cannot add this user to db.")
#await message.delete()
if message.author.guild_permissions.administrator:
if message.content.startswith('-db ls') or message.content.startswith('-db rm'):
embed = discord.Embed(title='Invitarr Database.')
all = db.read_useremail()
table = texttable.Texttable()
table.set_cols_dtype(["t", "t", "t"])
table.set_cols_align(["c", "c", "c"])
header = ("#", "Name", "Email")
table.add_row(header)
for index, peoples in enumerate(all):
index = index + 1
id = int(peoples[1])
dbuser = client.get_user(id)
dbemail = peoples[2]
try:
username = dbuser.name
except:
username = "******"
embed.add_field(name=f"**{index}. {username}**", value=dbemail+'\n', inline=False)
table.add_row((index, username, dbemail))
if message.content.startswith('-db ls'):
total = str(len(all))
if(len(all)>25):
f = open("db.txt", "w")
f.write(table.draw())
f.close()
await message.channel.send("Database too large! Total: {total}".format(total = total),file=discord.File('db.txt'))
else:
await message.channel.send(embed = embed)
else:
try:
position = message.content.replace("-db rm", "")
position = int(position) - 1
id = all[position][1]
email = db.get_useremail(id)
deleted = db.delete_user(id)
if deleted:
print("Removed {} from db".format(email))
await secure.send("Removed {} from db".format(email))
else:
print("Cannot remove this user from db.")
except Exception as e:
print(e)
if message.content.startswith('-help'):
embed = discord.Embed(title='Invitarr Bot Commands', description='Made by [Sleepingpirates](https://github.com/Sleepingpirates/Invitarr), [Join Discord Server](https://discord.gg/vcxCytN)')
embed.add_field(name='-plex add <email>', value='This command is used to add an email to plex', inline=False)
embed.add_field(name='-plex rm <email>', value='This command is used to remove an email from plex', inline=False)
embed.add_field(name='-db ls', value='This command is used list Invitarrs database', inline=False)
embed.add_field(name='-db add <email> <@user>', value='This command is used to add exsisting users email and discord id to the DB. Bot tries to add email to plex', inline=False)
embed.add_field(name='-db rm <position>', value='This command is used remove a record from the Db. Use -db ls to determine record position. ex: -db rm 1', inline=False)
await message.channel.send(embed = embed)
def increment_commands(self):
self.commands += 1
db.save_user(self)
return self.commands
def set_nick(self, nick):
self.last_nick = nick
db.save_user(self)
def set_rank(self, rank):
self.rank = rank
db.save_user(self)
return self.rank
def save(self):
"""Save the user to the database (or create if it is not present)"""
if self.id:
db.save_user(self.id, self.email, self.password)
else:
self.id = db.create_user(self.email, self.password)
def get_tweets(mps_file,
mps_db,
country="UK",
context_free=False,
start_date=datetime(2019, 2, 20, 0, 0, 0),
end_date=datetime.now()):
"""
retrieve the tweets of the MPs of each country
:return:
"""
# create json file to dump tweets
dir_path = os.path.dirname(os.path.realpath(__file__))
if country == "UK":
fname = os.path.join(dir_path, 'json_files',
'tweets_%s_to_%s.json' % (start_date, end_date))
else: # if country == "GR"
fname = os.path.join(dir_path, 'json_files',
'tweets_anndate_to_%s.json' % end_date)
with open(fname, 'a'):
os.utime(fname, None)
# parse date columns to datetime
dateparse = lambda x: pd.datetime.strptime(x, '%d/%m/%Y')
# read mps csv file
if country == "UK":
# ip patching
mps_file = os.path.join(dir_path, mps_file)
mps = pd.read_csv(mps_file)
politicians = mps[mps['Screen Name'].str.startswith('@')]
users_tuples = zip(politicians['Screen Name'], politicians['Party'],
[start_date] * politicians.shape[0])
else: # if country == "GR"
meps = pd.read_csv(mps_file,
parse_dates=['Announcement date'],
date_parser=dateparse)
politicians = meps[meps['Screen Name'].str.startswith('@')]
users_tuples = zip(politicians['Screen Name'], politicians['Party'], [
ann_date.to_pydatetime()
for ann_date in politicians['Announcement date']
])
for username, party, start_datetime in users_tuples:
while True:
try:
user = api.get_user(screen_name=username)
except Exception, ex:
print 'I just caught the exception: %s' % ex
time.sleep(20)
continue
break
print "Retrieve tweets from MP {} of {} party from {} to {}".format(
username, party, start_datetime, end_date)
# # if user exists in db continue
# if db.get_user(user.id_str, mps_db): continue
# save user in db
db_user_id = db.get_user(user.id_str, mps_db) or db.save_user(
user, mps_db, party, start_datetime, is_mp=True)
if user.protected: continue
statuses = []
# retrieve most recent statuses
max_id = None
while True:
while True:
try:
tmp_statuses = api.user_timeline(screen_name=username,
count=100,
tweet_mode="extended",
max_id=max_id)
except Exception, ex:
print 'I just caught the exception: %s' % ex
time.sleep(20)
continue
break
if max_id is not None:
# remove duplicate tweet with the id == max_id, because we got it on the previous Search
tmp_statuses = tmp_statuses[1:]
for status in tmp_statuses:
# if the status date is bigger than start date append it
if start_datetime < status.created_at < end_date:
# save status in db
status_text = get_text(status._json)
if not db.get_tweet(status.id_str, database):
db.save_tweet(status,
db_user_id,
mps_db,
text=status_text)
statuses.append(status._json)
if not context_free:
# get all the replies to this tweet and the replies of the replies recursively
for reply in get_replies(status):
ruser = reply.user
ruser_is_mp = "@{}".format(
ruser.screen_name) in politicians
ruser_party = mps.loc[mps['Screen Name'] == "@{}".format(ruser.screen_name)]["Party"] \
if ruser_is_mp else None
# save ruser
db_ruser_id = db.get_user(ruser.id_str, mps_db) or \
db.save_user(ruser, mps_db, ruser_party, start_datetime, is_mp=ruser_is_mp)
# save the reply
if not db.get_tweet(reply.id_str, database):
print(
"save reply {} from user {} who replies to {}"
.format(reply.id, ruser.screen_name,
reply.in_reply_to_status_id))
db.save_tweet(reply, db_ruser_id, mps_db)
statuses.append(reply._json)
if len(tmp_statuses
) == 0 or tmp_statuses[-1].created_at < start_datetime:
break
else:
max_id = tmp_statuses[-1].id
def update(self):
self.update_seen()
db.save_user(self)