def apiClientEdit(api_client_id):
# pdb.set_trace()
# this page will be for creating API Clients
try:
client = db_session.query(User_Api_Client).filter_by(id=api_client_id)
if client.one():
if request.method == 'POST':
client.update({
"name": request.form['name'], "api_id": aes_encrypt(request.form['api_id']), "auth_id": aes_encrypt(request.form['auth_id'])})
db_session.commit()
flash('%s was sucessfully updated!' % request.form['name'])
return redirect(url_for('apiClientEdit', api_client_id=api_client_id))
else:
# pdb.set_trace()
apiClient = client.one()
id = apiClient.id
api_id = aes_decrypt(apiClient.api_id)
auth_id = aes_decrypt(apiClient.auth_id)
name = apiClient.name
return render_template('/apiClients/apiClientEdit.html', id=id, api_id=api_id, auth_id=auth_id, clientName=name)
else:
flash('API Client Id %s not found!' % api_client_id)
return render_template('/apiClients/apiClients.html')
except:
print('some error could not save transaction, rolling back')
db_session.rollback()
db_session.commit()
flash('API Client Id %s not found!' % api_client_id)
return render_template('/apiClients/apiClients.html')
def customerEdit(customer_id):
# pdb.set_trace()
# this page will be for creating API Clients
try:
customer = db_session.query(User_Customer).filter_by(id=customer_id)
if customer.first():
if request.method == 'POST':
customer.update({
"name": request.form['customerName'], "phone": request.form['customerPhone'], "email": request.form['customerEmail'], 'status': request.form['customerStatus']})
db_session.commit()
flash('%s was sucessfully updated!' %
request.form['customerName'])
# return redirect(url_for('customerEdit', customer_id=customer.one().id))
return redirect(url_for('Customers', customers=db_session.query(User_Customer).filter_by(
user_id=current_user.id).all()))
else:
# pdb.set_trace()
return render_template('/customers/customerEdit.html', customer=customer.one())
else:
flash('Customer with Id %s not found!' % customer_id)
# return render_template('/customers/customers.html')
return redirect(url_for('Customers', customers=db_session.query(User_Customer).filter_by(
user_id=current_user.id).all()))
except:
db_session.rollback()
db_session.commit()
flash('Something went wrong, rolling back transactions!')
return redirect(url_for('Customers', customers=db_session.query(User_Customer).filter_by(
user_id=current_user.id).all()))
def newInboundMessage():
""" This page will be for all inbound messages, check if customer exists if so, check content of message if == "UNSUBSCRIBE", change user status.
If customer does not exist add to database.
"""
try:
print ('requestform', request.form)
tel = str(request.form['To'])
user = db_session.query(User).filter_by(
phone=tel).one()
if user:
customer = db_session.query(
User_Customer).filter_by(phone=request.form['From']).first()
if customer:
if request.form['Text'] == "UNSUBSCRIBE":
customer.status = "UNSUBSCRIBED"
else:
customer = User_Customer(
name='UNKNOWN', phone=request.form['From'], user_id=user.id, status="SUBSCRIBED")
db_session.add(customer)
db_session.commit()
newMessage = Message(
user_id=user.id, user_customer_id=customer.id, message_uuid=request.form['MessageUUID'], message=request.form['Text'], direction="inbound", status="RECEIVED",
units=request.form["Units"],
total_rate=request.form["TotalRate"],
total_amount=request.form["TotalAmount"], error_code="200", message_time=datetime.datetime.now())
db_session.add(newMessage)
db_session.commit()
return jsonify(newMessage.serialize)
except:
print('Something went wrong, rolling back transaction!')
db_session.rollback()
db_session.commit()
def add_asso_group():
if request.method == 'POST':
try:
request_key = request.form.keys()[0]
get_user_id = int(request_key.split('-')[1])
get_group_id = int(request.form.get(request_key, ''))
assouser = db_session.query(GitUser).filter_by(
id=get_user_id).first()
assogroup = db_session.query(GitGroup).filter_by(
id=get_group_id).first()
if assogroup not in assouser.git_group:
assouser.git_group.append(assogroup)
assos = db_session.query(asso_group_user).all()
if (get_group_id, get_user_id) not in assos:
try:
db_session.add(assouser)
db_session.commit()
print '====> Store (%s) to DB successfully.' % assouser.name
assogroup_file = BASE_DIR + '/conf/groups/' \
+ assogroup.name + '.conf'
gconf_dict = {}
with open(assogroup_file, 'rb') as f:
lines = f.readlines()
for line in lines:
gconf_group = line.split('=')[0].lstrip(
'@').strip()
gconf_user = line.split('=')[1].split()
gconf_dict[gconf_group] = gconf_user
allowed_ext_user = []
for _ in allowed_ext:
allowed_ext_user.append('.'.join(
assouser.name.split('.')[:-1]))
for _ in set(allowed_ext_user):
if gconf_dict.has_key(
assogroup.name) and _ not in gconf_dict[
assogroup.name]:
gconf_dict[assogroup.name].append(str(_))
f = open(assogroup_file, 'wb')
f.truncate()
for _ in gconf_dict:
f.write('@' + _ + ' = ' +
' '.join(gconf_dict[_]))
f.close()
except TypeError:
db_session.rollback()
print '====> TypeError Need string or buffer, list found.'
finally:
db_session.close()
else:
print '====> The record(%s) already exists.' % assouser.name
except IndexError:
print '====> IndexError because of out of range.'
return redirect(url_for('user'))
def collect(self, query, context, waiting_time, count, number_of_attempts):
if query == '':
return
last_since = -1
tweets_collect = []
list_ids = []
for i in range(0, number_of_attempts):
print("Collecting " + query + "... Attempt: " + str(i))
if i == 0:
results = self.twython.search(q=query, count=count, lang='pt')
else:
results = self.twython.search(q=query,
count=count,
lang='pt',
since_id=last_since)
count_control = 0
for t in results['statuses']:
tweet = self.twitter.get_tweet_data(t)
if count_control == 0:
last_since = tweet['object_id']
count_control += 1
try:
tweet_instance = AllTweets(tweet['object_id'],
tweet['user_name'],
tweet['text'],
tweet['date_formated'],
tweet['user_rt'], tag, context)
db_session.add(tweet_instance)
db_session.commit()
print("Saved " + str(tweet['object_id']))
except exc.IntegrityError as e:
print("The tweet " + str(tweet['object_id']) +
" has already on database")
db_session.rollback()
except Exception as e:
raise Exception("Database Error: " + str(e))
#waiting_time in seconds
print("Waiting Sleep Time ...")
time.sleep(waiting_time)
def apiClientDelete(api_client_id):
try:
client = db_session.query(User_Api_Client).get(api_client_id)
db_session.delete(client)
db_session.commit()
flash("%s was sucessfully Deleted!" % client.name)
clients = db_session.query(User_Api_Client).filter_by(
user_id=current_user.id).all()
return render_template('/apiClients/apiClients.html', clients=clients)
except:
flash("Something went wrong rolling back transaction!" % client.name)
print('some error could not save transaction, rolling back')
db_session.rollback()
db_session.commit()
def remove_asso_group():
if request.method == 'POST':
try:
request_key = request.form.keys()[0]
get_user_id = int(request_key.split('-')[1])
get_group_id = request.form.get(request_key, '')
assouser = db_session.query(GitUser).filter_by(
id=get_user_id).first()
assogroup = db_session.query(GitGroup).filter_by(
id=get_group_id).first()
assouser.git_group.remove(assogroup)
db_session.add(assouser)
db_session.commit()
print '====> Sponge (%s) from DB successfully.' % assouser.name
assogroup_file = BASE_DIR + '/conf/groups/' \
+ assogroup.name + '.conf'
gconf_dict = {}
with open(assogroup_file, 'rb') as f:
lines = f.readlines()
for line in lines:
gconf_group = line.split('=')[0].lstrip('@').strip()
gconf_user = line.split('=')[1].split()
gconf_dict[gconf_group] = gconf_user
allowed_ext_user = []
for _ in allowed_ext:
allowed_ext_user.append('.'.join(
assouser.name.split('.')[:-1]))
for _ in set(allowed_ext_user):
if gconf_dict.has_key(
assogroup.name) and _ in gconf_dict[assogroup.name]:
gconf_dict[assogroup.name].remove(str(_))
f = open(assogroup_file, 'wb')
f.truncate()
for _ in gconf_dict:
f.write('@' + _ + ' = ' + ' '.join(gconf_dict[_]))
f.close()
except IndexError:
db_session.rollback()
print '====> IndexError Out of range.'
except ValueError:
db_session.rollback()
print '====> ValueError Not in list.'
finally:
db_session.close()
return redirect(url_for('user'))
def newCustomer():
# This page will be for creating a new Customer
if request.method == 'POST':
newCustomer = User_Customer(
name=request.form['name'], email=request.form['email'], phone=request.form['phone'], user_id=current_user.id, status="SUBSCRIBED")
db_session.add(newCustomer)
try:
db_session.commit()
flash('%s was sucessfully added!' % request.form['name'])
return redirect(url_for('newCustomer'))
except:
print('some error could not save transaction, rolling back')
db_session.rollback()
db_session.commit()
else:
# pdb.set_trace()
return render_template('/customers/newCustomer.html')
def statusMessage():
try:
print ('requestform', request.form)
messageUuid = str(request.form['MessageUUID'])
message = db_session.query(Message).filter_by(
message_uuid=messageUuid)
message.update({"status": request.form["Status"],
"units": int(request.form["Units"] or 1),
"total_rate": request.form["TotalRate" or "0.0"],
"total_amount": request.form["TotalAmount"] or "0.0",
"message_time": datetime.datetime.now()})
db_session.commit()
print(message.one().serialize)
return jsonify(message.one().serialize)
except:
print('some error could not save transaction, rolling back')
db_session.rollback()
db_session.commit()
def customerDelete(customer_id):
try:
customer = db_session.query(User_Customer).get(customer_id)
db_session.delete(customer)
db_session.commit()
flash("%s was sucessfully Deleted!" % customer.name)
customers = db_session.query(User_Customer).filter_by(
user_id=current_user.id).all()
return render_template('/customers/customers.html', customers=customers)
except:
print('some error could not save transaction, rolling back')
db_session.rollback()
db_session.commit()
flash("Encountered some errors, rolling back transactions")
customers = db_session.query(User_Customer).filter_by(
user_id=current_user.id).all()
return render_template('/customers/customers.html', customers=customers)
def add_asso_subrepo():
if request.method == 'POST':
try:
request_key = request.form.keys()[0]
get_repo_name = request_key.split('-')[1]
get_subrepo_name = request.form.get(request_key, '')
assosubrepo = db_session.query(GitSubRepo).filter_by(
name=get_subrepo_name).first()
assorepo = db_session.query(GitRepo).filter_by(
name=get_repo_name).first()
if assosubrepo.main_repo_id != assorepo.id:
try:
assosubrepo.main_repo_id = assorepo.id
db_session.commit()
assorepo_file = BASE_DIR + '/conf/repos/' \
+ assorepo.name + '.conf'
rconf_dict = {}
with open(assorepo_file, 'rb') as f:
lines = f.readlines()
for line in lines:
rconf_repo = line.split('=')[0].lstrip('@').strip()
rconf_subrepo = line.split('=')[1].split()
rconf_dict[rconf_repo] = rconf_subrepo
if rconf_dict.has_key(
assorepo.name
) and get_subrepo_name not in rconf_dict[assorepo.name]:
rconf_dict[assorepo.name].append(
str(re.sub('~', '/', get_subrepo_name)))
f = open(assorepo_file, 'wb')
f.truncate()
for _ in rconf_dict:
f.write('@' + _ + ' = ' + ' '.join(rconf_dict[_]))
f.close()
except TypeError:
db_session.rollback()
print '====> TypeError Need string or buffer, list found.'
finally:
db_session.close()
except IndexError:
db_session.rollback()
print '====> IndexError Out of range.'
return redirect(url_for('repo'))
def newMessage():
try:
# This page will be for creating a new Customer
customers = db_session.query(
User_Customer).filter_by(user_id=current_user.id).all()
if request.method == 'POST':
apiClient = db_session.query(User_Api_Client).filter_by(
user_id=current_user.id).all()[0]
destinationNumbers = list(map(lambda x: db_session.query(User_Customer).filter_by(
id=int(x)).one().phone, request.form.getlist('customerSelect')))
# Commented out to not make unnecessary api calls
client = plivo.RestClient(
auth_id=aes_decrypt(apiClient.api_id), auth_token=aes_decrypt(apiClient.auth_id))
response = client.messages.create(
src=current_user.phone,
dst="<".join(destinationNumbers),
text=request.form['message'],
url="https://sms-messages.herokuapp.com/message/status")
# message_uuid = []
# for i in destinationNumbers:
# message_uuid.append(datetime.datetime.now())
# response = {'message_uuid': message_uuid}
# uncomment below when using api
for i, uuid in enumerate(response.message_uuid, 0):
# uncomment below when not using api
# for i, uuid in enumerate(response['message_uuid'], 0):
newMessage = Message(
user_id=current_user.id, user_customer_id=int(request.form.getlist('customerSelect')[i]), message_uuid=uuid, message=request.form['message'], direction="outbound")
db_session.add(newMessage)
db_session.commit()
flash('The message was sucessfully created!')
return redirect(url_for('newMessage', customers=customers))
else:
return render_template('/messages/newMessage.html', customers=customers)
except:
print('some error could not save transaction, rolling back')
db_session.rollback()
db_session.commit()
flash('There was some errors and some of your messages my have Failed!')
return render_template('/messages/newMessage.html', customers=customers)
def on_success(self, data):
tweet = self.twitter.get_tweet_data(data)
try:
tweet_instance = AllTweets(tweet['object_id'], tweet['user_name'],
tweet['text'], tweet['date_formated'],
tweet['user_rt'], self.hashtag,
self.context)
db_session.add(tweet_instance)
db_session.commit()
except exc.IntegrityError as e:
print("The tweet " + str(tweet['object_id']) +
" has already on database")
db_session.rollback()
except Exception as e:
raise Exception("Database Error: " + str(e))
print("Saved: " + str(tweet['object_id']))
def messageDelete(message_id):
try:
message = db_session.query(Message).get(message_id)
db_session.delete(message)
db_session.commit()
flash("The message '%s' from %s was sucessfully Deleted!" %
(message.message, message.customer.name))
sms_messages = db_session.query(Message).filter_by(
user_id=current_user.id).order_by(Message.id.desc()).all()
return render_template('/messages/messages.html', sms_messages=sms_messages)
except:
print('some error could not save transaction, rolling back')
db_session.rollback()
db_session.commit()
flash("Something went wrong, rolling back transaction!")
sms_messages = db_session.query(Message).filter_by(
user_id=current_user.id).order_by(Message.id.desc()).all()
return render_template('/messages/messages.html', sms_messages=sms_messages)
def remove_asso_subrepo():
if request.method == 'POST':
try:
request_key = request.form.keys()[0]
get_repo_name = request_key.split('-')[1]
get_subrepo_name = request.form.get(request_key, '')
assorepo = db_session.query(GitRepo).filter_by(
name=get_repo_name).first()
assosubrepo = db_session.query(GitSubRepo).filter_by(
name=get_subrepo_name).first()
assosubrepo.main_repo_id = None
db_session.add(assosubrepo)
db_session.commit()
print '====> Sponge (%s) from DB successfully.' % assosubrepo.name
assorepo_file = BASE_DIR + '/conf/repos/' \
+ assorepo.name + '.conf'
rconf_dict = {}
with open(assorepo_file, 'rb') as f:
lines = f.readlines()
for line in lines:
rconf_repo = line.split('=')[0].lstrip('@').strip()
rconf_subrepo = line.split('=')[1].split()
rconf_dict[rconf_repo] = rconf_subrepo
if rconf_dict.has_key(assorepo.name) and '/'.join(
get_subrepo_name.split('~')) in rconf_dict[assorepo.name]:
rconf_dict[assorepo.name].remove(
str(re.sub('~', '/', get_subrepo_name)))
f = open(assorepo_file, 'wb')
f.truncate()
for _ in rconf_dict:
f.write('@' + _ + ' = ' + ' '.join(rconf_dict[_]))
f.close()
except IndexError:
db_session.rollback()
print '====> IndexError Out of range.'
except ValueError:
db_session.rollback()
print '====> ValueError Not in list.'
finally:
db_session.close()
return redirect(url_for('repo'))
def group():
groupdict = {}
groupfile = item_traversal('conf/groups')
try:
groupfile = groupfile['conf/groups']
if groupfile:
for _ in groupfile:
if allowed_file(_, allowed_ext):
filepath = BASE_DIR + '/conf/groups/' + _
with open(filepath, 'rb') as f:
content = f.readlines()
for _ in content:
group_name = (((_.strip('\n')).split('='))[0]
).strip().lstrip('@')
group_member = ((_.strip('\n')).split('='))[1]
groupdict[group_name] = group_member
except:
groupfile = []
if request.method == 'POST':
new_group_name = request.form.get('addGroupName', '')
query_group = db_session.query(GitGroup).filter_by(name=new_group_name)
if not db_session.query(
query_group.exists()
).scalar() and new_group_name.strip() != '' and \
new_group_name.strip() != 'all':
try:
new_group = GitGroup(name=new_group_name)
db_session.add(new_group)
db_session.commit()
group_init(new_group_name)
except:
db_session.rollback()
finally:
db_session.close()
return redirect(url_for('group'))
return render_template(
'group.html',
groupdict=groupdict,
)
def apiClient():
# pdb.set_trace()
# this page will be for creating API Clients
if request.method == 'POST':
newClient = User_Api_Client(
name=request.form['name'], api_id=aes_encrypt(request.form['api_id']), auth_id=aes_encrypt(request.form['auth_id']), user_id=current_user.id)
print('new_client=', newClient.api_id, newClient.auth_id)
db_session.add(newClient)
try:
db_session.commit()
flash('%s was sucessfully added!' % request.form['name'])
return redirect(url_for('apiClient'))
except:
print('some error could not save transaction, rolling back')
db_session.rollback()
db_session.commit()
else:
# pdb.set_trace()
return render_template('/apiClients/apiClientNew.html')
def repo():
repodict = {}
repofile = item_traversal('conf/repos')
try:
repofile = repofile['conf/repos']
except:
repofile = []
for _ in repofile:
if allowed_file(_, allowed_ext):
filepath = BASE_DIR + '/conf/repos/' + _
with open(filepath, 'rb') as f:
content = f.readlines()
for _ in content:
if '=' in _:
repo_name = (((
_.strip('\n')).split('='))[0]).strip().lstrip('@')
repo_member = ((_.strip('\n')).split('='))[1]
repodict[repo_name] = repo_member
subrepodict = {}
subrepoitem = {}
subrepofile = item_traversal('conf/subrepos')
try:
subrepofile = subrepofile['conf/subrepos']
except:
subrepofile = []
for _ in subrepofile:
if allowed_file(_, allowed_ext):
filepath = BASE_DIR + '/conf/subrepos/' + _
with open(filepath, 'rb') as f:
first_line = f.readline().strip('\n')
subrepo_name = first_line[4:].strip()
content = f.readlines()
for _ in content:
if '=' in _:
subrepo_item = (((
_.strip('\n')).split('='))[0]).strip().lstrip('@')
subrepo_member = ((_.strip('\n')).split('='))[1]
subrepoitem[subrepo_item] = subrepo_member
subrepodict[subrepo_name] = subrepoitem.copy()
subrepotorepodict = {}
assignedlist = []
assignedgroup = {}
for gitsubrepo in GitSubRepo.query.filter_by(main_repo_id=None).all():
subrepotorepodict[gitsubrepo.id] = gitsubrepo.name
for i in repodict.keys():
queries = GitSubRepo.query.join(GitRepo).filter(
GitRepo.name == i).all()
for query in queries:
assignedlist.append(query.name)
assignedgroup[i] = assignedlist
assignedlist = []
if request.args:
tab_page = request.args.get('tab_page', '')
subRepoName = request.args.get('subRepoName', '')
else:
tab_page = 0
subRepoName = ''
subitemdict = {}
select_subrepo = ''
grouptosubrepodict = {}
for gitgroup in db_session.query(GitGroup).all():
grouptosubrepodict[gitgroup.id] = gitgroup.name
grouptosubrepodict[0] = u'all'
if request.method == 'GET':
if 'subRepoName' in request.args.keys() and \
'tab_page' in request.args.keys():
select_subrepo = str(request.args.get('subRepoName', ''))
subitemdict = subrepodict[select_subrepo]
if request.method == 'POST':
if 'addMainRepoName' in request.form.keys():
get_repo_name = request.form.get('addMainRepoName', '')
new_repo_name = ''
if get_repo_name.strip() != '':
if get_repo_name[0] == '%' and get_repo_name[-1] == '%':
new_repo_name = get_repo_name.strip('%')
elif not (get_repo_name[0] == '%'
and get_repo_name[-1] == '%'):
if not get_repo_name.endswith('_repo'):
get_repo_name = get_repo_name + '_repo'
new_repo_name = get_repo_name
query_repo = db_session.query(GitRepo).filter_by(
name=new_repo_name)
if not db_session.query(query_repo.exists()).scalar(
) and new_repo_name.strip() != '':
try:
new_repo = GitRepo(name=new_repo_name)
db_session.add(new_repo)
db_session.commit()
repo_init(new_repo_name)
except:
db_session.rollback()
finally:
db_session.close()
return redirect(url_for('repo'))
elif 'addSubRepoName' in request.form.keys():
new_sub_repo_name = request.form.get('addSubRepoName', '')
if '/' in new_sub_repo_name:
new_sub_repo_name = '~'.join(new_sub_repo_name.split('/'))
query_repo = db_session.query(GitSubRepo).filter_by(
name=new_sub_repo_name)
if not db_session.query(query_repo.exists()).scalar(
) and new_sub_repo_name.strip() != '':
try:
new_sub_repo = GitSubRepo(name=new_sub_repo_name)
db_session.add(new_sub_repo)
db_session.commit()
sub_repo_init(new_sub_repo_name)
except:
db_session.rollback()
finally:
db_session.close()
return redirect(url_for('repo'))
elif 'subRepoName' in request.form.keys():
tab_page = 1
select_subrepo = str(request.form.get('subRepoName', ''))
subitemdict = subrepodict[select_subrepo]
return render_template(
'repo.html',
#~ repofile=repofile,
repodict=repodict,
subrepodict=subrepodict,
subrepotorepodict=subrepotorepodict,
assignedgroup=assignedgroup,
tab_page=tab_page,
select_subrepo=select_subrepo,
subitemdict=subitemdict,
grouptosubrepodict=grouptosubrepodict,
subRepoName=subRepoName,
)
def user():
status_dict = {
0: 'Upload successfully.',
1: 'Not a pub key file.',
2: 'The pub key already exists.',
3: 'Please choose a pub key file.',
4: 'Error while writing into db.',
5: 'Please check if `keydir` exists.',
}
#~ keydict = item_traversal('keydir')
userdict = {}
grouptouserdict = {}
assignedlist = []
assignedgroup = {}
upload_status = {'status': '-1'}
for gituser in db_session.query(GitUser).all():
userdict[gituser.id] = gituser.name
for gitgroup in db_session.query(GitGroup).all():
grouptouserdict[gitgroup.id] = gitgroup.name
for i in userdict.keys():
queries = GitGroup.query.filter(GitGroup.git_user.any(id=i)).all()
for query in queries:
assignedlist.append(query.name)
assignedgroup[i] = assignedlist
assignedlist = []
f = open('log.log', 'ab+')
if request.method == 'POST' and request.files:
file = request.files.get('keyFile', '')
if file:
filename = secure_filename(file.filename)
filepath = os.path.join(KEY_DIR, filename)
if not allowed_file(filename, allowed_ext):
upload_status['status'] = 1
print >> f, filename + ' is not a pub key.'
return render_template(
'upload.html',
upload_status=upload_status,
)
if os.path.exists(filepath):
upload_status['status'] = 2
return render_template(
'upload.html',
upload_status=upload_status,
)
try:
new_user = GitUser(name=filename)
db_session.add(new_user)
file.save(filepath)
db_session.commit()
upload_status['status'] = 0
print >> f, filename + ' has been saved locally.'
return render_template(
'upload.html',
upload_status=upload_status,
)
except IOError:
db_session.rollback()
upload_status['status'] = 5
return render_template(
'upload.html',
upload_status=upload_status,
)
except:
db_session.rollback()
upload_status['status'] = 4
return render_template(
'upload.html',
upload_status=upload_status,
)
finally:
db_session.close()
else:
upload_status['status'] = 3
return render_template(
'upload.html',
upload_status=upload_status,
)
f.close()
return render_template(
'user.html',
status_dict=status_dict,
userdict=userdict,
grouptouserdict=grouptouserdict,
assignedgroup=assignedgroup,
)
def collect(self, query, context, waiting_time, count, number_of_attempts):
def find(word, letter):
for l in word:
if l == letter:
return True
return False
if query == '':
return
geocoder = Geocoder(
access_token=
"pk.eyJ1IjoiZ3Jhbmhva2FpbiIsImEiOiJjam8wbXdyeDIwMXprM3Bubm04ZnUyYzRqIn0.XKzTzY7qAr44O8hU2M2agA"
)
tweets_collect = []
list_ids = []
for i in range(0, number_of_attempts):
print("Collecting " + query + "... Attempt: " + str(i))
if i == 0:
results = self.twython.search(q=query, count=count, lang='pt')
else:
results = self.twython.search(q=query,
count=count,
lang='pt',
max_id=last_since)
#count_control = 0
tweets = results['statuses']
indice = len(tweets) - 1
last_tweet = tweets[indice]
last_since = last_tweet['id']
for t in results['statuses']:
tweet = self.twitter.get_tweet_data(t)
try:
user_ = t['user']
_location = user_['location']
saveFile = open('raw.json', 'a')
if _location == None:
#print (':( User %s is not sharing location'%t['user']['screen_name'])
saveFile.close()
else:
if find(_location, ',') == False:
#print (':( User %s is sharing wrong location'%t['user']['screen_name'])
saveFile.close()
else:
coordinates = geocoder.forward(_location)
if coordinates.geojson()['features'] == []:
#print (':( User %s is sharing wrong city'%t['user']['screen_name'])
saveFile.close()
else:
mapboxResponse = coordinates.geojson(
)['features'][0]
coordinates_ = mapboxResponse['geometry'][
'coordinates']
t['geo'] = {
"type":
"Point",
"coordinates":
(coordinates_[1], coordinates_[0])
}
novo_tweet = json.dumps(t)
saveFile.write(novo_tweet + '\n')
saveFile.close()
#Collecting all here
tweet_instance = AllTweets(
tweet['object_id'], tweet['user_name'],
tweet['text'], tweet['date_formated'],
tweet['user_rt'], tag, context)
db_session.add(tweet_instance)
db_session.commit()
print("Saved " + str(tweet['object_id']))
except exc.IntegrityError as e:
print("The tweet " + str(tweet['object_id']) +
" has already on database")
db_session.rollback()
except Exception as e:
raise Exception("Database Error: " + str(e))
#waiting_time in seconds
print("Waiting Sleep Time ...")
time.sleep(waiting_time)
tweet_id = row['tweet_id']
try:
tweet = tw.show_status(id=tweet_id)
tweet_processed = t.get_tweet_data(tweet)
tweet_instance = AllTweets(tweet_processed['object_id'],
tweet_processed['user_name'],
tweet_processed['text'],
tweet_processed['date_formated'],
tweet_processed['user_rt'], tag,
context)
db_session.add(tweet_instance)
db_session.commit()
count = count + 1
print("\t " + str(count) + " tweets collected")
except TwythonRateLimitError as e:
print("Limit Error. Waiting 15 minutes")
time.sleep(900)
except TwythonError as e:
print("\tERROR: " + str(tweet_id) + " not collected: " + str(e))
except IntegrityError as e:
print("\tERROR: " + str(tweet_id) +
" not inserted at db: tweet id is already in database")
db_session.rollback()
except Exception as e:
print("\ERROR: " + str(tweet_id + ": " + str(e)))
def clear_tables():
try:
num_rows_deleted = db_session.query(HashtagsGraph).delete()
db_session.commit()
print("HashtagsGraph: " + str(num_rows_deleted) + " rows deleted.")
except Exception as e:
print("ERROR: " + str(e))
db_session.rollback()
try:
num_rows_deleted = db_session.query(BigramTrigram).delete()
db_session.commit()
print("BigramTrigram: " + str(num_rows_deleted) + " rows deleted.")
except Exception as e:
print("ERROR: " + str(e))
db_session.rollback()
try:
num_rows_deleted = db_session.query(Usuarios).delete()
db_session.commit()
print("Usuarios: " + str(num_rows_deleted) + " rows deleted.")
except Exception as e:
print("ERROR: " + str(e))
db_session.rollback()
try:
num_rows_deleted = db_session.query(Hashtags).delete()
db_session.commit()
print("Hashtags: " + str(num_rows_deleted) + " rows deleted.")
except Exception as e:
print("ERROR: " + str(e))
db_session.rollback()
try:
num_rows_deleted = db_session.query(UsuariosCitados).delete()
db_session.commit()
print("UsuariosCitados: " + str(num_rows_deleted) + " rows deleted.")
except Exception as e:
print("ERROR: " + str(e))
db_session.rollback()
try:
num_rows_deleted = db_session.query(Termos).delete()
db_session.commit()
print("Termos: " + str(num_rows_deleted) + " rows deleted.")
except Exception as e:
print("ERROR: " + str(e))
db_session.rollback()