def handle(self):
self.is_run = True
req_queue = Queue()
rcv_queue = Queue()
config = configparser.ConfigParser()
config.read("config.ini")
db_host = config.get("DB", "HOST")
db_port = int(config.get("DB", "PORT"))
db_user = config.get("DB", "USER")
db_pw = config.get("DB", "PASSWORD")
db_name = config.get("DB", "DB_NAME")
db_charset = config.get("DB", "CHAR_SET")
db_handler_thd = DbHandler(req_queue, rcv_queue, db_host, db_port, db_user, db_pw, db_name, db_charset)
db_handler_thd.start()
req_thd = threading.Thread(target=self._request_listener, args=(req_queue,))
req_thd.start()
rcv_thd = threading.Thread(target=self._receive_listener, args=(rcv_queue,))
rcv_thd.start()
req_thd.join()
rcv_thd.join()
db_handler_thd.is_run = False
def __init__(self):
self.current_user = None
self.current_stock = None # Stores the current stock that the user is watching.
self.stock_dict = {} # {stock_id: stock_ticker}
self.current_df = None # Stores the current dataframe. Changes if the user changes the stock.
self.stock_change = True # Checks if the user changes the current stock.
self.time_change = False # Checks if the user changes the timeframe.
self.timeperiod = '1M' # Changes the graph
self.dbh = DbHandler() # Initialises a database handler
def __init__(self):
"""
Initializes model instance and gets needed data from instagram
:return: None
"""
print('starting session and getting headers ... ', end='')
self.session = requests.Session()
self.get_initial_headers()
print(colored('DONE', 'green'))
self.db_handler = DbHandler()
def __init__(self, db_handler: DbHandler):
self.db_handler = db_handler
self.https_proxies = db_handler.select_proxies()
self.proxy_number = 1
self.proxy_dict = {
"https": "https://" + self.https_proxies[self.proxy_number][0]
}
def run(self):
while True:
notes = DbHandler.get_note_by_current_time()
if len(notes) > 0:
for item in notes:
text = ""
text = text + ("Дата напоминания: " + str(item[1]) + "\n" +
"Содержимое: " + str(item[3]) + "\n\n")
vk.method(
'messages.send', {
'user_id': item[4],
'message': text,
'random_id': random.getrandbits(64)
})
DbHandler.delete_note(item[0])
time.sleep(0.01)
def new_message(self, message):
DbHandler.add_user(self._USER_ID, self._USERNAME)
if message.upper() == self._COMMANDS[0]:
self.send_message(
f"Привет-привет, {self._USERNAME}! \n " +
f"Список команд: \n\n" +
f"ПРИВЕТ - получить список команд \n\n" +
f"ЗАПОМНИ - создать напоминание. Например, запомни/Вынести мусор/2020-12-20 10:00/ \n\n"
+
f"ПОКАЖИ - показать список всех напоминаний. Если ввести ПОКАЖИ, то выведутся"
+
f" все заметки за все время, если ввести ПОКАЖИ/2020-10-10/, "
f"то выведутся все заметки на эту дату \n\n" +
f"ПОВЕСЕЛИ - получить случайный мем")
# f"ХОЧУ - получить список доступных жанров мемов \n\n" +
# f"МНЕ НРАВИТСЯ - задать свои предпочтения в мемах \n\n" +
elif str(message).upper().startswith(self._COMMANDS[1]):
command = str(message).split('/')
DbHandler.add_note(self._USER_ID, command[1], command[2])
self.send_message("Готово, бро!")
elif str(message).upper().startswith(self._COMMANDS[2]):
command = str(message).split('/')
if len(command) == 1:
self.send_message(DbHandler.get_notes_by_user_id(
self._USER_ID))
else:
self.send_message(
DbHandler.get_notes_by_user_id(self._USER_ID, command[1]))
elif message.upper() == self._COMMANDS[3]:
self.send_attachment(DbHandler.get_random_image())
else:
self.send_message("Неизвестная команда")
def setUp(self):
# create a connection to test database
self.db = DbHandler()
# create a test client
self.app = app.test_client()
# propagate exceptions to the test client
self.app.testing = True
# default pre-configured user
self.user_id = "aaa10022-38b0-4a1a-95af-776f35aa2b8f"
class MainWindow(QWidget):
def __init__(self):
super().__init__()
self.db = DbHandler()
self.initUI()
def initUI(self):
grid = QGridLayout()
grid.setSpacing(1)
self.balance = QLabel(f'Balance: {self.db.get_balance()}')
grid.addWidget(self.balance, 0, 0)
self.expense_table = QTableWidget(self)
self.load_db_data()
grid.addWidget(self.expense_table, 1, 0)
refresh_expenses_btn = QPushButton('Refresh Expenses', self)
refresh_expenses_btn.clicked.connect(self.load_db_data)
grid.addWidget(refresh_expenses_btn, 2, 0)
add_expense_btn = QPushButton('Add Expense', self)
add_expense_btn.clicked.connect(self.add_expense_window)
grid.addWidget(add_expense_btn, 3, 0)
self.setLayout(grid)
self.setGeometry(300, 300, 600, 500)
self.setWindowTitle('Review')
self.show()
def load_db_data(self):
self.balance.setText(f'Balance: {self.db.get_balance()}')
self.expense_table.setRowCount(0)
self.expense_table.setRowCount(5)
self.expense_table.setColumnCount(0)
self.expense_table.setColumnCount(5)
self.expense_table.setHorizontalHeaderLabels(
['Amount', 'Description', 'Category', 'Date', 'Balance'])
expenses = self.db.get_all_expenses()
for row_idx, row_data in enumerate(expenses):
self.expense_table.insertRow(row_idx)
for column_idx, data in enumerate(row_data):
if column_idx > 0:
self.expense_table.setItem(row_idx, column_idx - 1,
QTableWidgetItem(str(data)))
def add_expense_window(self):
self.add_expense_popup = AddExpensePopup()
def do_POST(self):
content_length = int(self.headers['Content-Length'])
post_data = self.rfile.read(content_length).decode('UTF-8')
if not PostHandler.check_validity(post_data):
self.set_response(405, "message", "Wrong input")
else:
values = PostHandler.unpack_values(post_data)
secret_data = DbHandler.post_secret(values["secret"],
values["expireAfter"],
values["expireAfterViews"])
self.set_response(200, "data", secret_data)
async def test_containing_db(loop):
async def write_data_to_db(record, recid, tags, db=None):
await db.insert_to_table('records', record)
for tag in tags:
try:
await db.insert_to_table('tags', {'tagname': tag},
ignore_conflict=[
'tagname',
])
except UniqueViolationError:
pass
try:
await db.insert_to_table('records_tags', {
'tagname': tag,
'recordid': recid
})
except UniqueViolationError:
pass
aio_jikan = AioJikan(loop=loop)
cfg = Config()
pg_pool = await asyncpg.create_pool(cfg.DB_ADDRESS)
db = DbHandler(pg_pool=pg_pool, cfg=cfg)
year_2000 = 2000
seasons = ['winter', 'spring', 'summer', 'fall']
for year in range(2015, 2019):
for season in seasons:
print(f'[+] reading {season} in {year}')
season_year = await aio_jikan.season(year=year, season=season)
for item in season_year['anime']:
title = item['title']
title_tag = ('_'.join(re.findall(r'\W?(\w+)\W?',
title))).lower()
recid = recid_gen()
record = {
'recordid': recid,
'username': '******',
'record_headline': title,
'record_text': f'{title} ({season} {year})'
}
tags = [title_tag, str(year), season]
await write_data_to_db(record, recid, tags, db=db)
await aio_jikan.close()
await pg_pool.close()
def main():
conf = read_config()
path_watchers = []
global db
db = DbHandler()
for c in conf['sync']:
ssh_clients.append(
SshClient(c['server'], c['dirs']['to'], c['dirs']['from'],
c['key_file']))
path_watchers.append(
PathWatcher(c['dirs']['from'], ssh_clients[-1], db))
signal.signal(signal.SIGINT, clean_up)
signal.signal(signal.SIGTERM, clean_up)
def do_GET(self):
# ignoring request for icon while using browser
if "favicon.ico" not in self.path:
path = self.path
if "/v1/secret/" in path:
requested_hash = path.lstrip("/v1/secret/")
secret = DbHandler.get_secret(requested_hash)
if not secret:
self.set_response(404, "message", "Secret not found")
else:
self.set_response(200, "data", secret)
else:
self.set_response(405, "message",
"You've reached wrong endpoint.")
def __init__(self):
self.db = DbHandler()
self.config = configurator.get_config()
self.args = self.parse_arguments()
self.initialize_logger()
_id = self.get_id()
self.r = praw.Reddit(
user_agent="PrawWallpaperDownloader 1.0.0 by /u/Pusillus",
client_id=_id["id"],
client_secret=_id["secret"])
self.n_posts = 0
self.albums = 0
self.succeeded = 0
self.failed = 0
self.skipped = 0
self.deleted = 0
self.notify = False
self.posts = []
self.que = queue.Queue()
self.downloaded_images = []
class Session:
"""Stores all of the current session variables"""
def __init__(self):
self.current_user = None
self.current_stock = None # Stores the current stock that the user is watching.
self.stock_dict = {} # {stock_id: stock_ticker}
self.current_df = None # Stores the current dataframe. Changes if the user changes the stock.
self.stock_change = True # Checks if the user changes the current stock.
self.time_change = False # Checks if the user changes the timeframe.
self.timeperiod = '1M' # Changes the graph
self.dbh = DbHandler() # Initialises a database handler
def update(self):
self.stock_dict = self.dbh.get_user_stocks(self)
def current_stock_ticker(self):
"""Returns the current stock ticker"""
try:
return self.stock_dict[self.current_stock]
except AttributeError:
return None
def __init__(self):
self.db = DbHandler()
self.config = configurator.get_config()
self.args = self.parse_arguments()
self.initialize_logger()
_id = self.get_id()
self.r = praw.Reddit(user_agent="PrawWallpaperDownloader 1.0.0 by /u/Pusillus", client_id=_id["id"], client_secret=_id["secret"])
self.n_posts = 0
self.albums = 0
self.succeeded = 0
self.failed = 0
self.skipped = 0
self.deleted = 0
self.notify = False
self.posts = []
self.que = queue.Queue()
self.downloaded_images = []
class DbHandlerTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def setUp(self):
# create a connection to test database
self.db = DbHandler()
# default pre-configured user
self.user_id = "aaa10022-38b0-4a1a-95af-776f35aa2b8f"
def tearDown(self):
pass
def test_get_participant(self):
participant = self.db.get_participant(self.user_id)
self.assertNotEqual(participant, None)
self.assertEqual(participant[1], self.user_id)
def __init__(self):
QWidget.__init__(self)
self.db = DbHandler()
self.initUI()
def __init__(self):
super().__init__()
self.db = DbHandler()
self.initUI()
class AddExpensePopup(QWidget):
def __init__(self):
QWidget.__init__(self)
self.db = DbHandler()
self.initUI()
def initUI(self):
grid = QGridLayout()
grid.setSpacing(1)
amount_title = QLabel('Amount ')
grid.addWidget(amount_title, 0, 0)
self.amount = QLineEdit(self)
grid.addWidget(self.amount, 0, 1)
description_title = QLabel('Description ')
grid.addWidget(description_title, 1, 0)
self.description = QLineEdit(self)
grid.addWidget(self.description, 1, 1)
date_title = QLabel('Date (yyyy-mm-dd) ')
grid.addWidget(date_title, 2, 0)
self.date = QLineEdit(self)
self.date.setPlaceholderText(str(datetime.date.today()))
grid.addWidget(self.date, 2, 1)
category_title = QLabel('Category ')
grid.addWidget(category_title, 3, 0)
self.category = QComboBox(self)
self.category.addItem("food and drink")
self.category.addItem("entertainment")
self.category.addItem("self edification")
self.category.addItem("video game")
self.category.addItem("travel")
self.category.addItem("bills")
self.category.addItem("other")
grid.addWidget(self.category, 3, 1)
self.submit_button = QPushButton('Add Expense', self)
self.submit_button.clicked.connect(self.add_expense)
grid.addWidget(self.submit_button, 4, 2)
self.setLayout(grid)
self.setGeometry(300, 300, 500, 200)
self.setWindowTitle('Add Expense')
self.amount.setFocus()
self.show()
def add_expense(self):
amount = self.amount.text()
description = self.description.text()
date = self.get_date()
category = str(self.category.currentText())
if self.are_valid_entries(amount, date):
self.db.add_expense(amount, description, category, date)
self.reset_add_expense_widgets()
else:
# ADD POPUP WARNING
print("INVALID ENTRIES")
def get_date(self):
if self.date.text() == "":
return str(datetime.date.today())
return self.date.text()
def print_expense_data(self, amount, description, category, date):
print(f"amount: {amount}")
print(f"description: {description}")
print(f"date: {date}")
print(f"category: {category}")
def are_valid_entries(self, amount, date):
try:
float(amount)
except ValueError:
print("INVALID AMOUNT")
return False
try:
datetime.datetime.strptime(date, "%Y-%m-%d")
except ValueError:
print("INVALID DATE")
return False
return True
def reset_add_expense_widgets(self):
""" Resets all widgets in AddExpensePopup window so another expense can be added."""
self.amount.setText("")
self.description.setText("")
self.date.setText(str(datetime.date.today()))
import time
from flask import Flask
from flask import request
from db_handler import DbHandler
from flask_cors import CORS
app = Flask(__name__)
db_handler = DbHandler()
CORS(app)
@app.route("/")
def hello_world():
return "<p>Hello, World!</p>"
@app.route("/subscribe", methods=['POST'])
def subscribe():
global db_handler
print(request.json)
email = request.json['email']
application = request.json['application']
print(email)
print(application)
before = time.time()
db_handler.subscribe(email, application)
after = time.time()
return f"<p>{(after-before)*1000}ms</p>"
@app.route("/unsubscribe", methods=['POST'])
from crawler import Spider
from db_handler import DbHandler
import scrapy
from scrapy.crawler import CrawlerProcess
# input_handler = XlsHandler('./hubspot_list.xlsx','Meta Data')
db = DbHandler()
process = CrawlerProcess(
{'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'})
temp_url_list = db.getUrls()
domain_list = db.getDomains()
url_list = []
for item in temp_url_list:
if item != None:
for url in item.split(";"):
url_list.append("https://" + url)
print(domain_list)
print(url_list)
class MySpider(Spider):
allowed_domains = domain_list
start_urls = url_list
process.crawl(MySpider)
def setUp(self):
# create a connection to test database
self.db = DbHandler()
# default pre-configured user
self.user_id = "aaa10022-38b0-4a1a-95af-776f35aa2b8f"
class Model:
def __init__(self):
"""
Initializes model instance and gets needed data from instagram
:return: None
"""
print('starting session and getting headers ... ', end='')
self.session = requests.Session()
self.get_initial_headers()
print(colored('DONE', 'green'))
self.db_handler = DbHandler()
def get_initial_headers(self):
"""
Sends to instagram initial requests to get needed http headers to session
:return: None
"""
q = 'https://www.instagram.com/accounts/login/ajax/'
res = self.session.get(q)
auth_headers = {
'referer': 'https://www.instagram.com/accounts/login',
'X-CSRFToken': res.cookies['csrftoken']
}
auth = {
'username': '******',
'enc_password':
'******',
'optIntoOneTap': 'false'
}
res = self.session.post(q, data=auth, headers=auth_headers)
self.session.headers['X-CSRFToken'] = res.cookies['csrftoken']
def specify_needed_headers_to_session(self):
"""
Specifies all needed http headers to session
:return: None
"""
self.session.headers[
'User-Agent'] = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0'
self.session.headers['Accept'] = '*/*'
self.session.headers['Accept-Language'] = 'en-US,en;q=0.5'
self.session.headers['Accept-Encoding'] = 'gzip, deflate, br'
self.session.headers[
'X-IG-WWW-Claim'] = 'hmac.AR3Kny3HF-Th32yVAevSRIsFGdIwF_BK--Utv75fBqFu_g5b'
self.session.headers['X-IG-App-ID'] = '936619743392459'
self.session.headers['X-Requested-With'] = 'XMLHttpRequest'
self.session.headers['DNT'] = '1'
self.session.headers['Connection'] = 'keep-alive'
self.session.headers['Pragma'] = 'no-cache'
self.session.headers['Cache-Control'] = 'no-cache'
self.session.headers['Cookie'] = [
f'{x.name}={x.value}; ' for x in self.session.cookies
].__str__().replace('[', '').replace(']',
'').replace(',',
'').replace('\'', '')
def get_overall_account_information(self, username):
"""
Downloads overall information abount account, like user_id,
count of followers, count of following accounts
:param username: client instagram username
:return: tuple with (user_id, followers_count, following_count) if account exists,
and (None, None, None) otherwise
"""
user_id = None
followers_info = None
following_info = None
self.session.headers[
'Referer'] = f'https://www.instagram.com/{username}/'
overall_account_info_url = f'https://www.instagram.com/{username}/?__a=1'
res = self.session.get(overall_account_info_url,
headers=self.session.headers)
self.session.headers['X-CSRFToken'] = res.cookies['csrftoken']
if res.status_code == 200:
res = res.json()
else:
return user_id, followers_info, following_info
try:
user_id = res['graphql']['user']['id']
followers_info = res['graphql']['user']['edge_followed_by'][
'count']
following_info = res['graphql']['user']['edge_follow']['count']
except KeyError as kerr:
pass
return user_id, followers_info, following_info
def get_unfollowers(self, user_id, username, download=True):
"""
Gets list of accounts who doesn't follow client
:param user_id: user instagram id
:param username: instagram username
:param download: flag which says if its need to download data from instagram, or get from dump
:return: list of tuples with accounts who doesn't follow user in format (user_id, username)
"""
not_following_back_wo_whitelist = [] # wo - without
whitelist = self.get_user_whitelist(user_id)
not_following_back = self.get_full_list_of_unfollowers(
user_id=user_id, username=username, download=download)
if whitelist.__len__() > 0:
not_following_back_wo_whitelist = self.get_not_following_back_wo_whitelist(
not_following_back, whitelist)
else:
not_following_back_wo_whitelist = not_following_back
return not_following_back_wo_whitelist
def get_full_list_of_unfollowers(self, user_id, username, download=True):
"""
Returns full list of accounts who doesn't follow back user
:param user_id: user's instagram id
:param username: user's instagram username
:param download: boolean flag, indicates data need to be downloaded, or got from dump
:return: full list of accounts eho doesn't follow user back
"""
not_following_back = []
if download:
print('\n')
print('-' * 30)
followers = self.download_followers_list(user_id, username)
following = self.download_accounts_user_following_list(
user_id, username)
print(f'followers: {followers.__len__()}')
print(f'following: {following.__len__()}')
self.dump_followers(user_id, followers, following)
print('calculating rats ... ', end='')
for followee in following:
if followee not in followers:
not_following_back.append(followee)
print(colored('DONE', 'green'))
self.dump_full_not_following_back_list(user_id, not_following_back)
else:
with open(
f'{config.WORKING_DIR.replace("scripts", "")}client_data/{user_id}/{user_id}_not_following_back_full.pickle',
'rb') as f:
not_following_back = pickle.load(f)
return not_following_back
def get_user_whitelist(self, client_instagram_id):
"""
Returns list with instagram account ids which was added by user
for specified account as whitelist
:param client_instagram_id: instagram account id
:return: list with instagram ids, 'whitelist'
"""
whitelist = self.db_handler.get_whitelist_for_instagram_id(
client_instagram_id)
return whitelist
def get_not_following_back_wo_whitelist(self, not_following_back,
whitelist):
"""
Deletes from list of accounts who doesn't follow back those, who r in whitelist
:param not_following_back: list of accounts who doesn't follow back
:param whitelist: list of user's whitelist
:return: list of accounts who doesn't follow, with whitelist excluded
"""
not_following_back_wo_whitelist = []
for account in not_following_back:
if account[0] not in [acc[0].__str__() for acc in whitelist]:
not_following_back_wo_whitelist.append(account)
return not_following_back_wo_whitelist
def download_followers_list(self, user_id, username):
"""
Downloading from instagram list of client's followers
:param user_id: client instagram id
:param username: client username
:return: list with tuples of client's followers i.e. (instagram_id, username)
"""
followers = []
end_cursor = ''
has_next_page = True
self.session.headers[
'Referer'] = f'https://www.instagram.com/{username}/followers'
first_page_url = f'https://www.instagram.com/graphql/query/?query_hash=c76146de99bb02f6415203be841dd25a&' \
f'variables=%7B%22id%22%3A%22{user_id}%22%2C%22include_reel%22%3Atrue%2C%22' \
f'fetch_mutual%22%3Atrue%2C%22first%22%3A24%7D'
res = self.session.get(first_page_url, headers=self.session.headers)
self.session.headers['X-CSRFToken'] = res.cookies['csrftoken']
print('getting followers ... ', end='')
while has_next_page:
try:
has_next_page = bool(
res.json()['data']['user']['edge_followed_by']['page_info']
['has_next_page'])
except KeyError as kerr:
pass
try:
for node in res.json(
)['data']['user']['edge_followed_by']['edges']:
followers.append(
(node['node']['id'], node['node']['username']))
except Exception as err:
print()
if has_next_page:
end_cursor = res.json()['data']['user']['edge_followed_by'][
'page_info']['end_cursor'].replace('==', '')
next_page = f'https://www.instagram.com/graphql/query/?query_hash=c76146de99bb02f6415203be841dd25a' \
f'&variables=%7B%22id%22%3A%22{user_id}%22%2C%22' \
f'include_reel%22%3Atrue%2C%22fetch_mutual%22%3Afalse%2C%22first%22%3A14%2C%22' \
f'after%22%3A%22{end_cursor}%3D%3D%22%7D'
# time.sleep(randint(2,5)) # delay, prevents instagram ban
res = self.session.get(next_page, headers=self.session.headers)
self.session.headers['X-CSRFToken'] = res.cookies['csrftoken']
else:
print(colored('DONE', 'green'))
return followers
def download_accounts_user_following_list(self, user_id, username):
"""
Downloading from instagram list of users which client follows
:param user_id: client instagram id
:param username: client username
:return: list with tuples of following users by client i.e. (instagram_id, username)
"""
following = []
end_cursor = ''
has_next_page = True
first_page = f'https://www.instagram.com/graphql/query/?query_hash=d04b0a864b4b54837c0d870b0e77e076&' \
f'variables=%7B%22id%22%3A%22{user_id}%22%2C%22include_reel%22%3Atrue%2C%22' \
f'fetch_mutual%22%3Afalse%2C%22first%22%3A24%7D'
res = self.session.get(first_page, headers=self.session.headers)
self.session.headers['X-CSRFToken'] = res.cookies['csrftoken']
print('getting following ... ', end='')
while has_next_page:
try:
has_next_page = bool(res.json()['data']['user']['edge_follow']
['page_info']['has_next_page'])
except KeyError as kerr:
pass
for node in res.json()['data']['user']['edge_follow']['edges']:
following.append(
(node['node']['id'], node['node']['username']))
if has_next_page:
end_cursor = res.json()['data']['user']['edge_follow'][
'page_info']['end_cursor'].replace('==', '')
next_page = f'https://www.instagram.com/graphql/query/?query_hash=d04b0a864b4b54837c0d870b0e77e076' \
f'&variables=%7B%22id%22%3A%22{user_id}%22%2C%22include_reel%22%3Atrue%2C%22' \
f'fetch_mutual%22%3Afalse%2C%22first%22%3A12%2C%22after%22%3A%22{end_cursor}%3D%3D%22%7D'
# time.sleep(randint(2, 5)) # delay, prevents instagram ban
res = self.session.get(next_page, headers=self.session.headers)
self.session.headers['X-CSRFToken'] = res.cookies['csrftoken']
else:
print(colored('DONE', 'green'))
return following
def dump_followers(self, user_id, followers, following):
"""
Dumps to disk lists with followers and following people using pickle module,
to directory with user instagram id, which located at /client_data dir.
:param user_id: instagram user id of client
:param followers: list with tuples of (follower_id, follower_nickname)
:param following: list with tuples of (following_id, following_nickname)
:return: None
"""
if not os.path.exists(
config.WORKING_DIR.replace('scripts',
f'client_data/{user_id}')):
try:
os.mkdir(
config.WORKING_DIR.replace('scripts',
f'client_data/{user_id}'))
except OSError as oserr:
print(colored(oserr, 'red'))
with open(
config.WORKING_DIR.replace(
"scripts",
f"/client_data/{user_id}/{user_id}_followers.pickle"),
'wb') as f:
pickle.dump(followers, f)
with open(
config.WORKING_DIR.replace(
"scripts",
f"/client_data/{user_id}/{user_id}_followed.pickle"),
'wb') as f:
pickle.dump(following, f)
def dump_full_not_following_back_list(self, user_id, not_following_back):
"""
Dumps to disk list of people who doesn't follow back user with pickle module
:param user_id: user instagram id
:param not_following_back: list with tuples of (user_id, username)
:return: None
"""
path = config.WORKING_DIR.replace(
'scripts',
f'client_data/{user_id}/{user_id}_not_following_back_full.pickle')
with open(path, 'wb') as f:
pickle.dump(not_following_back, f)
def dump_not_following_without_whitelist(self, user_id,
not_following_back_wo_whitelist):
"""
Dumps data about accounts who doesb't follow user back to pickle file,
with whitelist excluded
:param user_id: user instagram id
:param not_following_back_wo_whitelist: list with users who doesn't follow user back
:return: None
"""
path = f'{config.WORKING_DIR.replace("scripts", "")}client_data/{user_id}/{user_id}' \
f'_not_following_back_without_whitelist.pickle'
with open(path, 'wb') as f:
pickle.dump(not_following_back_wo_whitelist, f)
def add_account_to_whitelist(self, client_id, client_telegram_id,
account_id):
"""
Calls db handler's method which adds account to whitelist
:param client_id: client instagram id
:param client_telegram_id:client telegram id
:param account_id: instagram id of account which need 2 b added to whitelist
:return:None
"""
self.db_handler.add_account_to_whitelist(client_id, client_telegram_id,
account_id)
class Scraper:
"""
The scraper, pretty much does all the hard work.
When the Scraper is initialized it wil parse commandline arguments.
utilizes DbHandler.py,´and configurator.py
"""
def __init__(self):
self.db = DbHandler()
self.config = configurator.get_config()
self.args = self.parse_arguments()
self.initialize_logger()
_id = self.get_id()
self.r = praw.Reddit(
user_agent="PrawWallpaperDownloader 1.0.0 by /u/Pusillus",
client_id=_id["id"],
client_secret=_id["secret"])
self.n_posts = 0
self.albums = 0
self.succeeded = 0
self.failed = 0
self.skipped = 0
self.deleted = 0
self.notify = False
self.posts = []
self.que = queue.Queue()
self.downloaded_images = []
@staticmethod
def get_id():
if os.path.isfile('client_secret.json'):
with open('client_secret.json', 'r') as id_file:
return json.loads("".join(id_file.readlines()))
else:
logging.error('Client_secret.json not found, exiting')
sys.exit('Unable to locate client_secret.json.\n'
'Please have a look at README.md '
'and follow the instructions')
def parse_arguments(self):
"""Parse arguments from commandline"""
parser = argparse.ArgumentParser()
parser.add_argument("-s",
"--subreddit",
help="specify subreddit to scrape",
default=self.config['Sub'])
parser.add_argument(
"-se",
"--section",
help=
"specify section of subreddit to scrape (hot, top, rising or new)",
default=self.config['section'])
parser.add_argument("-l",
"--limit",
help="set amount of posts to sift through "
"(default " + self.config['Limit'] + ")",
default=int(self.config['Limit']),
type=int)
parser.add_argument(
"--log",
help="save a log of wallpapers downloaded/skipped/failed",
action="store_true",
default=False)
parser.add_argument(
"-re",
"--redownload",
help="attempt to download all the links in the database",
action="store_true",
default=False)
parser.add_argument("-v",
"--verbose",
help="increase output detail",
action="store_true",
default=False)
parser.add_argument('-nc',
"--noclean",
help="Skip cleaning off small images (Cleaning: " +
self.config['Clean'] + ")",
action="store_true",
default=not self.config.getboolean('Clean'))
parser.add_argument(
'-ns',
'--nosort',
help="Skip sorting out previously downloaded images (Sorting: {})".
format(self.config['sort']),
action="store_true",
default=not self.config.getboolean('Sort'))
parser.add_argument('-na',
'--noalbum',
help='Skip imgur albums',
action='store_true',
default=not self.config.getboolean('Albums'))
parser.add_argument('-t',
'--threads',
help='Amount of threads for downloading images',
default=int(self.config['Threads']),
type=int)
parser.add_argument('-con',
'--configure',
help="Change settings",
action='store_true',
default=False)
parser.add_argument(
'-rlock',
'--ratiolock',
help="Sort out images with incorrect aspect ratio, 0 for no lock, "
"1 for full lock (Ratio lock: {})".format(
self.config['ratiolock']),
default=float(self.config['ratiolock']),
type=float)
parser.add_argument('-q',
'--search',
help="Scrape by search term",
default=False,
type=str)
args = parser.parse_args()
if args.ratiolock < 0 or args.ratiolock > 1:
sys.exit(
"Incorrect ratio lock, please keep it between 0.0 and 1.0 (Currently {})"
.format(args.ratiolock))
return args
def initialize_logger(self):
handlers = []
if self.args.log:
# Windows default encoding for text files isn't UTF-8 (it's ANSI afaik)
# So we need to create a custom FileHandler which opens the text file in UTF-8
file_handler = logging.FileHandler(filename='papers.log',
mode='w',
encoding="utf8")
handlers.append(file_handler)
if self.args.verbose:
# Create stream handler pointing to stdout (terminal) and add it to handlers.
stream_handler = logging.StreamHandler(stream=sys.stdout)
handlers.append(stream_handler)
elif not self.args.log:
handlers.append(
logging.StreamHandler(
stream=open(os.devnull, 'w', encoding="utf-8")))
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(message)s',
datefmt='%d/%m/%y %H:%M:%S:',
handlers=handlers)
logging.info('Logger started')
settings = "Arguments:\n"
for key, val in zip(vars(self.args).keys(), vars(self.args).values()):
settings += "{}: {}\n".format(key, val)
logging.info(settings)
def get_submissions(self, subreddit):
"""
Get submissions from reddit
Takes a subreddit object from PRAW as argument
Returns list of PRAW submission objects
"""
section = self.args.section.lower().strip()
limit = self.args.limit
if self.args.search:
return subreddit.search(self.args.search)
elif section == "top":
return subreddit.top(limit=limit)
elif section == "new":
return subreddit.new(limit=limit)
elif section == "rising":
return subreddit.rising(limit=limit)
else:
if section != "hot":
logging.warning("Unknown section, defaulting to hot")
print("Unknown section, defaulting to hot")
return subreddit.hot(limit=limit)
def extract_submission_data(self, submission):
"""
Exctract direct image links, and relevant data from a PRAW submission
object
Takes a PRAW submission object as arguments and appends a dictionary
in the following format to self.posts:
{"url": image-link,
"title": submission title,
"author": author of the submission (reddit user name),
"parent_id": None (only used for images in albums)}
If the submission link to an album this function will instead return
an album dictionary for further processing later in the process
album dictionary format:
{"url": link to imgur album,
"title": submission title,
"author": submission author (reddit username}
"""
url = submission.url
# Check for author
if not submission.author:
author = '[User Deleted]'
else:
author = str(submission.author)
# Direct jpg and png links
if url.endswith(".jpg") or url.endswith(".png"):
context = {
"url": url,
"title": submission.title,
"author": author,
"parent_id": None
}
self.posts.append(context)
# Imgur support
elif ("imgur.com" in url) and ("/a/" not in url):
if url.endswith("/new"):
url = url.rsplit("/", 1)[0]
id = url.rsplit("/", 1)[1].rsplit(".", 1)[0]
link = "http://i.imgur.com/" + id + ".jpg"
context = {
"url": link,
"title": submission.title,
"author": author,
"parent_id": None
}
self.posts.append(context)
# Album support
elif ("imgur.com" in url) and ("/a/" in url):
album_context = {
"url": url,
"title": submission.title,
"author": author
}
return album_context
def handle_submissions(self, submissions):
"""Get and sort posts from reddit"""
albums = [] # Array to hold all the album elements for later.
for submission in submissions:
album = self.extract_submission_data(submission)
if album:
albums.append(album)
# Extract all image links from the imgur albums
if not self.args.noalbum:
self.handle_albums(albums)
# Save amount of valid images
self.n_posts = len(self.posts)
# Sort out previously downloaded images
if not self.args.nosort:
if int(self.config["MaxAge"]) == 0:
self.posts = self.db.sort_links(self.posts)
else:
self.posts = self.db.sort_links(
self.posts, age_limit=self.config["MaxAge"])
self.skipped = self.n_posts - len(self.posts)
def handle_albums(self, albums):
"""Extract all links from a list of imgur albums"""
logging.info('Extracting albums...')
albums = self.db.sort_albums(albums)
n_albums = len(albums)
for _id, album in enumerate(albums):
print("\rHandling album: {}/{}".format(_id + 1, n_albums), end='')
logging.info('Handling album {}/{}'.format(_id + 1, n_albums))
# Download imgur album
res = requests.get(album["url"])
try:
res.raise_for_status()
except Exception as exc:
self.handle_error(exc, album)
continue
# Parse through the html fetching all link elements
soup = bs4.BeautifulSoup(res.text, 'html.parser')
link_elements = soup.select('a.zoom')
# Insert link to get id
album['length'] = len(link_elements)
album_id = self.db.insert_album(album)
if len(link_elements) > 0:
for a_id, ele in enumerate(link_elements):
# Put the data in context for later
context = {
"url": "http:" + ele.get('href'),
"title": album["title"],
"parent_id": album_id,
"id": a_id,
"author": album["author"]
}
self.posts.append(context)
self.albums += 1
print() # Add missing newline from printing album nr
def handle_error(self, err, post):
"""Handles error stats and prints a message if verbose is enabled"""
self.failed += 1
logging.error('Error occurred at:{} {}: {}'.format(
post["title"], type(err), str(err)))
def grab_image(self, download_folder, bar):
"""
Worker function for downloading images, keeps pulling a new link
from image que, downloads it, and saves it, untill the que is empty.
Takes 2 arguments:
download_folder: Path to desired save folder
bar: PyCLIBar to be stepped after image has been downloaded
"""
while True:
try:
submission = self.que.get(block=False)
except queue.Empty:
logging.info('Download thread done... Stopping')
return
# Try to download image
logging.info('Downloading image {}'.format(submission["title"]))
try:
response = requests.get(submission["url"], timeout=10)
response.raise_for_status()
except Exception as exc:
self.handle_error(exc, submission)
# Try to determine file format from headers fall back to trying to
# determine it from url if content-type is missing.
# Most modern operating systems will open the file regardless
# of format suffix
try:
type_header = response.headers['Content-Type'].split('/')
if type_header[0] == 'image':
content_type = type_header[1]
else:
# Sometimes the content-type is incorrect
# try to guess it from URL
if submission["url"].endswith('.png'):
content_type = "png"
else:
content_type = "jpg"
except KeyError and IndexError:
# Missing content-type header, guess from link
if submission["url"].endswith('.png'):
content_type = "png"
else:
content_type = "jpg"
# content-headers describe .jpg images with jpeg
if content_type == 'jpeg':
image_format = '.jpg'
else:
image_format = '.' + content_type
# If there's an id key in the submission it's from an album and
# should be suffixed with it's position within that album
if 'id' in submission:
file_path = os.path.join(
download_folder,
re.sub(r'[\\/:*?"<>|]', '', submission["title"][:25]) +
'_' + str(submission['id'] + 1) + image_format)
else:
file_path = os.path.join(
download_folder,
re.sub(r'[\\/:*?"<>|]', '', submission["title"][:25]) +
image_format)
# Try to save the image to disk
try:
with open(file_path, 'wb') as image:
for chunk in response.iter_content(4096):
image.write(chunk)
self.succeeded += 1
self.downloaded_images.append(file_path)
bar.step()
except Exception as exc:
self.handle_error(exc, submission)
def update_screen(self, bar):
"""
Keeps refreshing the CLIbar every .5 seconds as llong as self.notify
is true, always run as a seperate thread
"""
while self.notify:
print("{}".format(bar.get_progress_bar()), flush=True, end='\r')
time.sleep(0.5)
logging.info('Notify thread stopping')
return
def download_images(self):
"""Create folders and try to download/save the image links
in self.posts, assumes all links are image links"""
# Stop if there's not posts to download
if len(self.posts) < 1:
logging.info('No images to download, stopping')
sys.exit("No new images to download.")
logging.info('Starting download')
# Make folders
logging.info('Creating folders')
os.makedirs("Downloads", exist_ok=True)
download_folder = os.path.join("Downloads", self.args.subreddit)
os.makedirs(download_folder, exist_ok=True)
for post in self.posts:
self.que.put(post)
# Create progress bar
bar = CLIBar(_max=len(self.posts))
threads = []
print("Starting {} threads".format(self.args.threads))
for n in range(min(len(self.posts), self.args.threads)):
logging.info('Starting thread: {}'.format(n))
thread = threading.Thread(target=self.grab_image,
args=(download_folder, bar))
thread.start()
threads.append(thread)
print("Downloading images")
bar.start()
self.notify = True
logging.info('Starting notify thread')
threading.Thread(target=self.update_screen, args=(bar, )).start()
logging.info('Waiting for download threads to finish')
for thread in threads:
try:
thread.join()
except KeyboardInterrupt:
# Don't know how to handle this, ideas?
pass
logging.info('Done, telling notify thread to stop')
self.notify = False
def print_stats(self):
"""Print download stats to console"""
print()
new_images = self.succeeded - self.deleted
print('Albums: {}\nImages downloaded: {}/{} \nSkipped: {}\n'
'Failed: {}\nDeleted: {}\n'
'New images: {}'.format(self.albums, self.succeeded,
self.n_posts, self.skipped, self.failed,
self.deleted, new_images))
def save_posts(self):
"""Save posts currently in self.posts to database"""
for post in self.posts:
self.db.insert_link(post)
self.db.save_changes()
def clean_up(self):
"""Examines all downloaded images, deleting duds"""
print('\nCleaning up')
min_ratio = (int(self.config['MinWidth']) /
int(self.config['MinHeight'])) * self.args.ratiolock
max_ratio = (int(self.config['MinWidth']) /
int(self.config['MinHeight'])) * (2 - self.args.ratiolock)
if float(self.args.ratiolock) <= 0:
max_ratio = 100000
logging.info(
"Ratio settings: RatioLock: {} | MinRatio: {} | MaxRatio: {}".
format(self.args.ratiolock, min_ratio, max_ratio))
for image_path in self.downloaded_images:
try:
logging.info("Checking: {}".format(image_path))
image = Image.open(image_path)
image_ratio = image.size[0] / image.size[1]
logging.info("Image size: {}x{}".format(
image.size[0], image.size[1]))
logging.info("Image ratio: {}".format(image_ratio))
except OSError:
continue
# Check for size.
if image.size[0] < int(self.config['MinWidth'])\
or image.size[1] < int(self.config['MinHeight']):
image.close()
try:
os.remove(image_path)
logging.info(
'Removing image due to size: {}'.format(image_path))
self.deleted += 1
except PermissionError as e:
logging.warning('Error deleting image: {}: {}: {}'.format(
image_path, type(e), str(e)))
print('\nCan\'t delete ' + image_path +
' image is currently in use')
continue
else:
logging.info("Image size ok, checking ratio")
# Check for ratio
if min_ratio > image_ratio or max_ratio > max_ratio:
logging.info(
'Removing image due to ratio: {}'.format(image_path))
image.close()
try:
os.remove(image_path)
self.deleted += 1
except PermissionError as e:
logging.warning('Error deleting image: {}: {}: {}'.format(
image_path, type(e), str(e)))
print('\nCan\'t delete ' + image_path +
' image is currently in use')
else:
logging.info('Ratio ok.')
image.close()
def re_download(self):
"""Attempts to re-download all links in the database"""
self.posts = self.db.get_posts()
self.n_posts = len(self.posts)
self.download_images()
if len(self.downloaded_images) > 0 and not self.args.noclean:
self.clean_up()
self.print_stats()
def run(self):
"""Run the scraper"""
try:
if not self.args.search:
print('Getting {} posts from the {} section of {}'.format(
self.args.limit, self.args.section, self.args.subreddit))
else:
print('Searching {} for {}'.format(self.args.subreddit,
self.args.search))
# Get submissions from sub
submissions = self.get_submissions(
self.r.subreddit(self.args.subreddit))
# Handle the submissions and add them to self.posts
# Sorts out previously downloaded images and extracts imgur albums
self.handle_submissions(submissions)
except RequestException:
sys.exit('\nError connecting to reddit, please check your '
'internet connection')
except ResponseException:
sys.exit('\nError connecting to reddit.\n'
'Probably because you wrote an invalid subreddit name.\n'
'If that\'s not the case it\'s probably an'
'invalid client_secret.json file\nPlease see README.md '
'about how to set up the file properly.')
except Exception as e:
sys.exit('\nAn unknown error occurred:\n{}: {}'.format(
type(e), str(e)))
# Download all images in self.posts.
self.download_images()
self.save_posts()
if len(self.downloaded_images) > 0 and not self.args.noclean:
self.clean_up()
self.print_stats()
def __init__(self):
self.db = DbHandler()
class TitsaBot:
CUSTOM_OR_DEFAULT, INSERT_CUSTOM, TRANVIA, BROADCAST_TEXT = range(4)
def __init__(self):
config = configparser.ConfigParser()
config.read('bot_config.ini')
bot_token = config.get("TELEGRAM", "token")
titsa_idApp = config.get("TITSA", "idApp")
self.adminId = config.get("ADMIN", "chatId")
self.apiHandler = ApiHandler(titsa_idApp)
self.dbHandler = DbHandler()
self.transportThread = OpenTransitThread(
"http://www.titsa.com/Google_transit.zip", 7 * 24 * 60 * 60)
self.transportThread.start()
updater = Updater(token=bot_token, use_context=True)
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.INFO)
b1 = telegram.KeyboardButton("⭐ Consultar favorito ⭐")
b2 = telegram.KeyboardButton("✖️ Borrar favorito ✖️")
b3 = telegram.KeyboardButton("🚊 Tranvia de Tenerife 🚊")
b4 = telegram.KeyboardButton("📍 Ver paradas cercanas 📍",
request_location=True)
self.keyboard = telegram.ReplyKeyboardMarkup([[b3, b4], [b1, b2]],
resize_keyboard=True)
h1 = MessageHandler(Filters.regex(r"^.+Consultar favorito.+$"),
self.favKeyBoard)
h2 = MessageHandler(Filters.regex(u"^\U0001F68F.+(\d{4})"),
self.replyToFav)
h3 = MessageHandler(Filters.regex(r"^.+Borrar favorito.+$"),
self.favKeyBoard)
h4 = MessageHandler(Filters.regex(u"^\u2716.+(\d{4})"), self.eraseFav)
updater.dispatcher.add_handler(CommandHandler("start", self.start))
updater.dispatcher.add_handler(
MessageHandler(Filters.regex(r"^\d{4}$"), self.responder_a_codigo))
updater.dispatcher.add_handler(h1)
updater.dispatcher.add_handler(h2)
updater.dispatcher.add_handler(h3)
updater.dispatcher.add_handler(h4)
updater.dispatcher.add_handler(
ConversationHandler(
entry_points=[
MessageHandler(Filters.regex(r"^.+Tranvia de Tenerife.+$"),
self.listStops),
],
states={
TitsaBot.TRANVIA:
[MessageHandler(Filters.all, self.queryTram)]
},
fallbacks=[]))
updater.dispatcher.add_handler(
MessageHandler(Filters.location, self.nearStops))
updater.dispatcher.add_handler(
ConversationHandler(
entry_points=[
CommandHandler("broadcast", self.newBroadcast),
],
states={
TitsaBot.BROADCAST_TEXT:
[MessageHandler(Filters.all, self.broadcast)]
},
fallbacks=[]))
updater.dispatcher.add_handler(
ConversationHandler(entry_points=[
CommandHandler("addFav",
self.addFavCommand,
pass_args=True,
pass_user_data=True),
CallbackQueryHandler(self.addFavQuery,
pattern=r"^\d{4}$",
pass_user_data=True)
],
states={
TitsaBot.CUSTOM_OR_DEFAULT: [
CallbackQueryHandler(
self.setFavNameOption,
pass_user_data=True)
],
TitsaBot.INSERT_CUSTOM: [
MessageHandler(Filters.text,
self.customName,
pass_user_data=True)
]
},
fallbacks=[h1, h2, h3, h4]))
updater.dispatcher.add_handler(
ConversationHandler(entry_points=[
CallbackQueryHandler(self.reloadStationQuery,
pattern=r"^Repetir \d{4}$",
pass_user_data=True)
],
states={
TitsaBot.CUSTOM_OR_DEFAULT: [
CallbackQueryHandler(
self.reloadStationQuery,
pass_user_data=True)
]
},
fallbacks=[h1, h2, h3, h4]))
updater.dispatcher.add_handler(
CallbackQueryHandler(self.sendStopAndLocation,
pattern=r"^Locate \d{4}$",
pass_user_data=True))
updater.start_polling()
updater.idle()
self.dbHandler.save()
self.transportThread.stop()
def build_minutes_text(self, status, bus=True):
if status is not None:
text = "🚏 *" + status.name + "* 🚏\n\n"
sorted_lines = sorted(status.minutes.items(),
key=lambda line: int(line[1][0]["minutes"]))
print(sorted_lines)
for linea, data in sorted_lines:
for entry in data:
emoji = "🚊*" if not bus else "🚍*"
text += emoji + linea + "* (" + entry["dest"] + \
"): "+ entry["minutes"] + " minutos \n"
return text, 1
else:
text = "⚠ Parada no encontrada o sin pasos registrados ⚠"
return text, 0
def start(self, update: Update, context: CallbackContext):
context.bot.send_message(
chat_id=update.message.chat_id,
text=
"Hola! Gracias por usarme! \nEnvíame el código de una parada :)",
reply_markup=self.keyboard)
def addFavCommand(self, update: Update, context: CallbackContext):
return self.newFavMethod(update.message.from_user.id, context.args[0],
context.bot, update.message.chat.id,
context.user_data)
def addFavQuery(self, update: Update, context: CallbackContext):
return self.newFavMethod(update.callback_query.from_user.id,
update.callback_query.data, context.bot,
update.callback_query.message.chat.id,
context.user_data)
def reloadStationQuery(self, update: Update, context: CallbackContext):
codeText = update.callback_query.data.replace("Repetir ", "")
self.stationQuery(context.bot, update.callback_query.message.chat_id,
update.callback_query.from_user.id, codeText)
def newFavMethod(self, user, station, bot, chat, user_data):
if self.dbHandler.check_duplicate(user, station):
bot.send_message(chat_id=chat,
text="Ya está en favoritos",
reply_markup=None)
return -1
else:
logging.info(msg="New fav required user:%s id:%s" %
(user, station))
stationName = StopsHandler.stationName(station)
text = "Nombre: " + stationName + "\n¿Quiere usar otro?"
b1 = telegram.InlineKeyboardButton(text="Sí", callback_data="si")
b2 = telegram.InlineKeyboardButton(text="No", callback_data="no")
b3 = telegram.InlineKeyboardButton(text="❌Cancelar",
callback_data="cancel")
bot.send_message(chat_id=chat,
text=text,
reply_markup=telegram.InlineKeyboardMarkup(
[[b1, b2, b3]]))
user_data["currentFavStationId"] = station
user_data["currentFavStationName"] = stationName
return TitsaBot.CUSTOM_OR_DEFAULT
def responder_a_codigo(self, update: Update, context: CallbackContext):
self.stationQuery(context.bot, update.message.chat_id,
update.message.from_user.id, update.message.text)
def stationQuery(self, bot, chat_id, user_id, text):
if not self.dbHandler.check_duplicate_user(chat_id):
self.dbHandler.addUser(chat_id)
logging.info(msg="Message %s" % (text))
if text.isdigit() and len(text) == 4:
texto = self.build_minutes_text(self.apiHandler.new_request(text),
True)[0]
button = telegram.InlineKeyboardButton(
text="⭐ Añadir a favoritos ⭐", callback_data=text)
buttonReload = telegram.InlineKeyboardButton(
text="🔃 Repetir consulta 🔃", callback_data="Repetir " + text)
if not self.dbHandler.check_duplicate(user_id, text):
keyboard = telegram.InlineKeyboardMarkup([[button],
[buttonReload]])
else:
keyboard = telegram.InlineKeyboardMarkup([[buttonReload]])
bot.send_message(chat_id=chat_id,
text=texto,
parse_mode=telegram.ParseMode.MARKDOWN,
reply_markup=keyboard)
else:
bot.send_message(chat_id=chat_id, text="Código inválido")
bot.send_message(chat_id=chat_id,
text="Escribe el número de parada",
reply_markup=self.keyboard)
def setFavNameOption(self, update: Update, context: CallbackContext):
logging.info(
msg="Answer for the fav question: user:%s reply:%s" %
(update.callback_query.from_user.id, update.callback_query.data))
if update.callback_query.data == "no":
self.dbHandler.addUserFav(
update.callback_query.from_user.id,
context.user_data["currentFavStationId"],
context.user_data["currentFavStationName"])
text = "*Favorito añadido*\n" + context.user_data[
"currentFavStationName"] + "(" + context.user_data[
"currentFavStationId"] + ")"
context.bot.edit_message_text(
text,
update.callback_query.message.chat.id,
update.callback_query.message.message_id,
reply_markup=None,
parse_mode=telegram.ParseMode.MARKDOWN)
context.bot.send_message(
chat_id=update.callback_query.message.chat.id,
text="Escribe el número de parada",
reply_markup=self.keyboard)
return -1
elif update.callback_query.data == "si":
context.bot.edit_message_text(
"¿Qué nombre?",
update.callback_query.message.chat.id,
update.callback_query.message.message_id,
reply_markup=None)
return TitsaBot.INSERT_CUSTOM
else:
context.bot.delete_message(
update.callback_query.message.chat.id,
update.callback_query.message.message_id, None)
context.user_data.pop("currentFavStationId", None)
context.user_data.pop("currentFavStationName", None)
context.bot.send_message(
chat_id=update.callback_query.message.chat.id,
text="Escribe el número de parada",
reply_markup=self.keyboard)
return -1
def customName(self, update: Update, context: CallbackContext):
logging.info(msg="Custom name: user:%s reply:%s" %
(update.message.from_user.id, update.message.text))
self.dbHandler.addUserFav(update.message.from_user.id,
context.user_data["currentFavStationId"],
update.message.text)
text = "*Favorito añadido*\n" + update.message.text + "(" + context.user_data[
"currentFavStationId"] + ")"
context.bot.send_message(update.message.chat.id,
text=text,
reply_markup=self.keyboard,
parse_mode=telegram.ParseMode.MARKDOWN)
return -1
def listStops(self, update: Update, context: CallbackContext):
if not self.dbHandler.check_duplicate_user(update.message.chat_id):
self.dbHandler.addUser(update.message.chat_id)
logging.info(msg="Listing tram stations")
stations = self.apiHandler.tranvia_stations()
if stations is not None and len(stations) > 0:
buttons = []
for station in stations.items():
buttons.append([
telegram.KeyboardButton(u"🚋" + station[0] + " (" +
station[1] + ")")
])
context.bot.send_message(
update.message.chat.id,
text="Elige estación",
reply_markup=telegram.ReplyKeyboardMarkup(buttons),
resize_keyboard=True)
return TitsaBot.TRANVIA
return -1
def queryTram(self, update: Update, context: CallbackContext):
p = re.compile(u"^\U0001F68B.+(\w{3})")
stop = p.search(update.message.text).group(1)
status = self.apiHandler.tranvia_request(stop)
texto = self.build_minutes_text(status, False)[0]
context.bot.send_message(chat_id=update.message.chat_id,
text=texto,
parse_mode=telegram.ParseMode.MARKDOWN,
reply_markup=self.keyboard)
return -1
def favKeyBoard(self, update: Update, context: CallbackContext):
logging.info(msg="Fav request from user %s" %
(update.message.from_user.id))
stations = self.dbHandler.getUserFavs(update.message.from_user.id)
if len(stations) > 0:
buttons = []
emoji = u"🚏" if not "Borrar" in update.message.text else u"✖️"
for station in stations:
buttons.append([
telegram.KeyboardButton(emoji + station[1] + " (" +
station[0] + ")")
])
context.bot.send_message(
update.message.chat.id,
text="Elige estación",
reply_markup=telegram.ReplyKeyboardMarkup(buttons),
resize_keyboard=True)
else:
context.bot.send_message(update.message.chat.id,
text="No tienes favoritos",
reply_markup=self.keyboard,
resize_keyboard=True)
def replyToFav(self, update: Update, context: CallbackContext):
p = re.compile(u"^\U0001F68F.+(\d{4})")
code = p.search(update.message.text).group(1)
update.message.text = code
self.responder_a_codigo(update, context)
def eraseFav(self, update: Update, context: CallbackContext):
p = re.compile(u"^\u2716.+(\d{4})")
code = p.search(update.message.text).group(1)
self.dbHandler.deleteUserFav(update.message.from_user.id, code)
context.bot.send_message(update.message.chat.id,
text="Favorito eliminado",
reply_markup=self.keyboard,
resize_keyboard=True)
def broadcast(self, update: Update, context: CallbackContext):
logging.info(msg="Broadcasting message %s" % update.message.text)
if (update.message.chat.id == int(self.adminId)):
for user in self.dbHandler.getAllUsers():
logging.info(msg="Broadcasted to %s" % (user))
context.bot.send_message(str(user),
text=update.message.text,
reply_markup=self.keyboard,
resize_keyboard=True)
return -1
def newBroadcast(self, update: Update, context: CallbackContext):
return TitsaBot.BROADCAST_TEXT
def nearStops(self, update: Update, context: CallbackContext):
logging.info(msg="Nearest from user %s" %
(update.message.from_user.id))
stations = StopsHandler.nearestStops(
4, float(update.message.location.latitude),
float(update.message.location.longitude))
if stations is not None:
buttons = []
for station in stations:
buttons.append([
telegram.InlineKeyboardButton(
text=f"{station.name} ({station.id})",
callback_data=f"Locate {station.id}")
])
context.bot.send_message(
update.message.chat.id,
text="Estas son tus estaciones cercanas",
reply_markup=telegram.InlineKeyboardMarkup(buttons))
else:
context.bot.send_message(update.message.chat.id,
text="⚠ No hay información disponible ⚠",
reply_markup=self.keyboard,
resize_keyboard=True)
def sendStopAndLocation(self, update: Update, context: CallbackContext):
context.bot.answer_callback_query(update.callback_query.id)
logging.info(msg="Requested from nearest( user %s)" %
(update.callback_query.message.chat_id))
id = update.callback_query.data.replace("Locate ", "")
location = StopsHandler.stopLocation(id)
context.bot.send_location(update.callback_query.message.chat_id,
latitude=location[0],
longitude=location[1])
self.stationQuery(context.bot, update.callback_query.message.chat_id,
update.callback_query.from_user.id, id)
from flask import Flask, request, Response
import json
from db_handler import DbHandler
from auth_handler import AuthHandler
from qc.qc_handler import QcHandler
from util import log
app = Flask(__name__)
# TODO move to config file (along with configs in DbHandler and AuthHandler mayhaps)
app.config['MAIN_RECORDINGS_PATH'] = '/data/eyra/recordings'
dbHandler = DbHandler(app)
authHandler = AuthHandler(
app) # sets up /auth/login @app.route and @login_required()
qcHandler = QcHandler(app, dbHandler)
# SUBMISSION ROUTES
@app.route('/submit/general/<method>', methods=['POST'])
@authHandler.login_required()
def submit_general(method):
"""
supports everything in the client-server API
right now, /submit/general/{device,instuctor}
requires sender to be authenticated with JWT, see auth_handler.py
from discord import message
from discord.ext import commands
from discord.ext.commands.errors import MissingRequiredArgument
from dotenv import load_dotenv
from db_handler import DbHandler
load_dotenv()
TOKEN = os.getenv('DISCORD_TOKEN')
HOST = os.getenv('HOST')
USER = os.getenv('USER')
PASSWORD = os.getenv('PASSWORD')
DATABASE = os.getenv('DATABASE')
bot = commands.Bot(command_prefix='!')
db = DbHandler(HOST, USER, PASSWORD, DATABASE)
@bot.event
async def on_ready():
print(f"{bot.user.name} has connected to Discord!")
@bot.command(
name="add",
help=
"Προσθέτω εργασία. Το date να είναι της μόρφης 2021-03-25, ενώ η ώρα 21:00:00."
)
async def add_command(ctx, assignment_name, class_name, turn_in_date,
turn_in_time):
author = ctx.message.author
from http.server import HTTPServer
from server import MyServer
from os import path
from db_handler import DbHandler
if __name__ == "__main__":
if not path.exists("task_db.db"):
DbHandler.initialize()
server_address = ("localhost", 8000)
server = HTTPServer(server_address, MyServer)
print("Starting http server on Localhost:8000")
server.serve_forever()
class Scraper:
"""
The scraper, pretty much does all the hard work.
When the Scraper is initialized it wil parse commandline arguments.
utilizes DbHandler.py,´and configurator.py
"""
def __init__(self):
self.db = DbHandler()
self.config = configurator.get_config()
self.args = self.parse_arguments()
self.initialize_logger()
_id = self.get_id()
self.r = praw.Reddit(user_agent="PrawWallpaperDownloader 1.0.0 by /u/Pusillus", client_id=_id["id"], client_secret=_id["secret"])
self.n_posts = 0
self.albums = 0
self.succeeded = 0
self.failed = 0
self.skipped = 0
self.deleted = 0
self.notify = False
self.posts = []
self.que = queue.Queue()
self.downloaded_images = []
@staticmethod
def get_id():
if os.path.isfile('client_secret.json'):
with open('client_secret.json', 'r') as id_file:
return json.loads("".join(id_file.readlines()))
else:
logging.error('Client_secret.json not found, exiting')
sys.exit('Unable to locate client_secret.json.\n'
'Please have a look at README.md '
'and follow the instructions')
def parse_arguments(self):
"""Parse arguments from commandline"""
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--subreddit",
help="specify subreddit to scrape",
default=self.config['Sub'])
parser.add_argument("-se", "--section",
help="specify section of subreddit to scrape (hot, top, rising or new)",
default=self.config['section'])
parser.add_argument("-l", "--limit",
help="set amount of posts to sift through "
"(default " + self.config['Limit'] + ")",
default=int(self.config['Limit']), type=int)
parser.add_argument("--log",
help="save a log of wallpapers downloaded/skipped/failed",
action="store_true", default=False)
parser.add_argument("-re", "--redownload",
help="attempt to download all the links in the database",
action="store_true", default=False)
parser.add_argument("-v", "--verbose", help="increase output detail",
action="store_true",
default=False)
parser.add_argument('-nc', "--noclean", help="Skip cleaning off small images (Cleaning: " + self.config['Clean'] + ")",
action="store_true", default= not self.config.getboolean('Clean'))
parser.add_argument('-ns', '--nosort', help="Skip sorting out previously downloaded images (Sorting: {})".format(self.config['sort']),
action="store_true", default= not self.config.getboolean('Sort'))
parser.add_argument('-na', '--noalbum', help='Skip imgur albums',
action='store_true', default= not self.config.getboolean('Albums'))
parser.add_argument('-t', '--threads', help='Amount of threads for downloading images',
default=int(self.config['Threads']), type=int)
parser.add_argument('-con', '--configure', help="Change settings",
action='store_true', default=False)
parser.add_argument('-rlock', '--ratiolock',
help="Sort out images with incorrect aspect ratio, 0 for no lock, "
"1 for full lock (Ratio lock: {})".format(self.config['ratiolock']),
default=float(self.config['ratiolock']), type=float)
parser.add_argument('-q', '--search', help="Scrape by search term", default=False,
type=str)
args = parser.parse_args()
if args.ratiolock < 0 or args.ratiolock > 1:
sys.exit("Incorrect ratio lock, please keep it between 0.0 and 1.0 (Currently {})".format(args.ratiolock))
return args
def initialize_logger(self):
handlers = []
if self.args.log:
# Windows default encoding for text files isn't UTF-8 (it's ANSI afaik)
# So we need to create a custom FileHandler which opens the text file in UTF-8
file_handler = logging.FileHandler(filename='papers.log', mode='w', encoding="utf8")
handlers.append(file_handler)
if self.args.verbose:
# Create stream handler pointing to stdout (terminal) and add it to handlers.
stream_handler = logging.StreamHandler(stream=sys.stdout)
handlers.append(stream_handler)
elif not self.args.log:
handlers.append(logging.StreamHandler(stream=open(os.devnull, 'w', encoding="utf-8")))
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(message)s',
datefmt='%d/%m/%y %H:%M:%S:',
handlers=handlers)
logging.info('Logger started')
settings = "Arguments:\n"
for key, val in zip(vars(self.args).keys(), vars(self.args).values()):
settings += "{}: {}\n".format(key, val)
logging.info(settings)
def get_submissions(self, subreddit):
"""
Get submissions from reddit
Takes a subreddit object from PRAW as argument
Returns list of PRAW submission objects
"""
section = self.args.section.lower().strip()
limit = self.args.limit
if self.args.search:
return subreddit.search(self.args.search)
elif section == "top":
return subreddit.top(limit=limit)
elif section == "new":
return subreddit.new(limit=limit)
elif section == "rising":
return subreddit.rising(limit=limit)
else:
if section != "hot":
logging.warning("Unknown section, defaulting to hot")
print("Unknown section, defaulting to hot")
return subreddit.hot(limit=limit)
def extract_submission_data(self, submission):
"""
Exctract direct image links, and relevant data from a PRAW submission
object
Takes a PRAW submission object as arguments and appends a dictionary
in the following format to self.posts:
{"url": image-link,
"title": submission title,
"author": author of the submission (reddit user name),
"parent_id": None (only used for images in albums)}
If the submission link to an album this function will instead return
an album dictionary for further processing later in the process
album dictionary format:
{"url": link to imgur album,
"title": submission title,
"author": submission author (reddit username}
"""
url = submission.url
# Check for author
if not submission.author:
author = '[User Deleted]'
else:
author = str(submission.author)
# Direct jpg and png links
if url.endswith(".jpg") or url.endswith(".png"):
context = {"url": url,
"title": submission.title,
"author": author,
"parent_id": None}
self.posts.append(context)
# Imgur support
elif ("imgur.com" in url) and ("/a/" not in url):
if url.endswith("/new"):
url = url.rsplit("/", 1)[0]
id = url.rsplit("/", 1)[1].rsplit(".", 1)[0]
link = "http://i.imgur.com/" + id + ".jpg"
context = {"url": link,
"title": submission.title,
"author": author,
"parent_id": None}
self.posts.append(context)
# Album support
elif ("imgur.com" in url) and ("/a/" in url):
album_context = {"url": url,
"title": submission.title,
"author": author}
return album_context
def handle_submissions(self, submissions):
"""Get and sort posts from reddit"""
albums = [] # Array to hold all the album elements for later.
for submission in submissions:
album = self.extract_submission_data(submission)
if album:
albums.append(album)
# Extract all image links from the imgur albums
if not self.args.noalbum:
self.handle_albums(albums)
# Save amount of valid images
self.n_posts = len(self.posts)
# Sort out previously downloaded images
if not self.args.nosort:
if int(self.config["MaxAge"]) == 0:
self.posts = self.db.sort_links(self.posts)
else:
self.posts = self.db.sort_links(self.posts, age_limit=self.config["MaxAge"])
self.skipped = self.n_posts - len(self.posts)
def handle_albums(self, albums):
"""Extract all links from a list of imgur albums"""
logging.info('Extracting albums...')
albums = self.db.sort_albums(albums)
n_albums = len(albums)
for _id, album in enumerate(albums):
print("\rHandling album: {}/{}".format(_id+1, n_albums), end='')
logging.info('Handling album {}/{}'.format(_id+1, n_albums))
# Download imgur album
res = requests.get(album["url"])
try:
res.raise_for_status()
except Exception as exc:
self.handle_error(exc, album)
continue
# Parse through the html fetching all link elements
soup = bs4.BeautifulSoup(res.text, 'html.parser')
link_elements = soup.select('a.zoom')
# Insert link to get id
album['length'] = len(link_elements)
album_id = self.db.insert_album(album)
if len(link_elements) > 0:
for a_id, ele in enumerate(link_elements):
# Put the data in context for later
context = {"url": "http:" + ele.get('href'),
"title": album["title"],
"parent_id": album_id,
"id": a_id,
"author": album["author"]}
self.posts.append(context)
self.albums += 1
print() # Add missing newline from printing album nr
def handle_error(self, err, post):
"""Handles error stats and prints a message if verbose is enabled"""
self.failed += 1
logging.error('Error occurred at:{} {}: {}'.format(post["title"],
type(err),
str(err)))
def grab_image(self, download_folder, bar):
"""
Worker function for downloading images, keeps pulling a new link
from image que, downloads it, and saves it, untill the que is empty.
Takes 2 arguments:
download_folder: Path to desired save folder
bar: PyCLIBar to be stepped after image has been downloaded
"""
while True:
try:
submission = self.que.get(block=False)
except queue.Empty:
logging.info('Download thread done... Stopping')
return
# Try to download image
logging.info('Downloading image {}'.format(submission["title"]))
try:
response = requests.get(submission["url"], timeout=10)
response.raise_for_status()
except Exception as exc:
self.handle_error(exc, submission)
# Try to determine file format from headers fall back to trying to
# determine it from url if content-type is missing.
# Most modern operating systems will open the file regardless
# of format suffix
try:
type_header = response.headers['Content-Type'].split('/')
if type_header[0] == 'image':
content_type = type_header[1]
else:
# Sometimes the content-type is incorrect
# try to guess it from URL
if submission["url"].endswith('.png'):
content_type = "png"
else:
content_type = "jpg"
except KeyError and IndexError:
# Missing content-type header, guess from link
if submission["url"].endswith('.png'):
content_type = "png"
else:
content_type = "jpg"
# content-headers describe .jpg images with jpeg
if content_type == 'jpeg':
image_format = '.jpg'
else:
image_format = '.' + content_type
# If there's an id key in the submission it's from an album and
# should be suffixed with it's position within that album
if 'id' in submission:
file_path = os.path.join(download_folder,
re.sub(r'[\\/:*?"<>|]',
'',
submission["title"][:25
]) + '_' + str(submission['id']+1) + image_format)
else:
file_path = os.path.join(download_folder,
re.sub(r'[\\/:*?"<>|]',
'',
submission["title"][:25]) + image_format)
# Try to save the image to disk
try:
with open(file_path, 'wb') as image:
for chunk in response.iter_content(4096):
image.write(chunk)
self.succeeded += 1
self.downloaded_images.append(file_path)
bar.step()
except Exception as exc:
self.handle_error(exc, submission)
def update_screen(self, bar):
"""
Keeps refreshing the CLIbar every .5 seconds as llong as self.notify
is true, always run as a seperate thread
"""
while self.notify:
print("{}".format(bar.get_progress_bar()),
flush=True, end='\r')
time.sleep(0.5)
logging.info('Notify thread stopping')
return
def download_images(self):
"""Create folders and try to download/save the image links
in self.posts, assumes all links are image links"""
# Stop if there's not posts to download
if len(self.posts) < 1:
logging.info('No images to download, stopping')
sys.exit("No new images to download.")
logging.info('Starting download')
# Make folders
logging.info('Creating folders')
os.makedirs("Downloads", exist_ok=True)
download_folder = os.path.join("Downloads", self.args.subreddit)
os.makedirs(download_folder, exist_ok=True)
for post in self.posts:
self.que.put(post)
# Create progress bar
bar = CLIBar(_max=len(self.posts))
threads = []
print("Starting {} threads".format(self.args.threads))
for n in range(min(len(self.posts), self.args.threads)):
logging.info('Starting thread: {}'.format(n))
thread = threading.Thread(target=self.grab_image,
args=(download_folder, bar))
thread.start()
threads.append(thread)
print("Downloading images")
bar.start()
self.notify = True
logging.info('Starting notify thread')
threading.Thread(target=self.update_screen,
args=(bar, )).start()
logging.info('Waiting for download threads to finish')
for thread in threads:
try:
thread.join()
except KeyboardInterrupt:
# Don't know how to handle this, ideas?
pass
logging.info('Done, telling notify thread to stop')
self.notify = False
def print_stats(self):
"""Print download stats to console"""
print()
new_images = self.succeeded-self.deleted
print('Albums: {}\nImages downloaded: {}/{} \nSkipped: {}\n'
'Failed: {}\nDeleted: {}\n'
'New images: {}'.format(self.albums,
self.succeeded,
self.n_posts,
self.skipped,
self.failed,
self.deleted,
new_images))
def save_posts(self):
"""Save posts currently in self.posts to database"""
for post in self.posts:
self.db.insert_link(post)
self.db.save_changes()
def clean_up(self):
"""Examines all downloaded images, deleting duds"""
print('\nCleaning up')
min_ratio = (int(self.config['MinWidth'])/int(self.config['MinHeight']))*self.args.ratiolock
max_ratio = (int(self.config['MinWidth'])/int(self.config['MinHeight']))*(2-self.args.ratiolock)
if float(self.args.ratiolock) <= 0:
max_ratio = 100000
logging.info("Ratio settings: RatioLock: {} | MinRatio: {} | MaxRatio: {}".format(self.args.ratiolock,
min_ratio, max_ratio))
for image_path in self.downloaded_images:
try:
logging.info("Checking: {}".format(image_path))
image = Image.open(image_path)
image_ratio = image.size[0] / image.size[1]
logging.info("Image size: {}x{}".format(image.size[0], image.size[1]))
logging.info("Image ratio: {}".format(image_ratio))
except OSError:
continue
# Check for size.
if image.size[0] < int(self.config['MinWidth'])\
or image.size[1] < int(self.config['MinHeight']):
image.close()
try:
os.remove(image_path)
logging.info('Removing image due to size: {}'.format(image_path))
self.deleted += 1
except PermissionError as e:
logging.warning('Error deleting image: {}: {}: {}'.format(
image_path, type(e), str(e)))
print('\nCan\'t delete ' + image_path + ' image is currently in use')
continue
else:
logging.info("Image size ok, checking ratio")
# Check for ratio
if min_ratio > image_ratio or max_ratio > max_ratio:
logging.info('Removing image due to ratio: {}'.format(image_path))
image.close()
try:
os.remove(image_path)
self.deleted += 1
except PermissionError as e:
logging.warning('Error deleting image: {}: {}: {}'.format(
image_path, type(e), str(e)))
print('\nCan\'t delete ' + image_path + ' image is currently in use')
else:
logging.info('Ratio ok.')
image.close()
def re_download(self):
"""Attempts to re-download all links in the database"""
self.posts = self.db.get_posts()
self.n_posts = len(self.posts)
self.download_images()
if len(self.downloaded_images) > 0 and not self.args.noclean:
self.clean_up()
self.print_stats()
def run(self):
"""Run the scraper"""
try:
if not self.args.search:
print('Getting {} posts from the {} section of {}'
.format(self.args.limit, self.args.section, self.args.subreddit))
else:
print('Searching {} for {}'.format(self.args.subreddit, self.args.search))
# Get submissions from sub
submissions = self.get_submissions(self.r.subreddit(self.args.subreddit))
# Handle the submissions and add them to self.posts
# Sorts out previously downloaded images and extracts imgur albums
self.handle_submissions(submissions)
except RequestException:
sys.exit('\nError connecting to reddit, please check your '
'internet connection')
except ResponseException:
sys.exit('\nError connecting to reddit.\n'
'Probably because you wrote an invalid subreddit name.\n'
'If that\'s not the case it\'s probably an'
'invalid client_secret.json file\nPlease see README.md '
'about how to set up the file properly.')
except Exception as e:
sys.exit('\nAn unknown error occurred:\n{}: {}'.format(type(e),
str(e)))
# Download all images in self.posts.
self.download_images()
self.save_posts()
if len(self.downloaded_images) > 0 and not self.args.noclean:
self.clean_up()
self.print_stats()
def __init__(self):
config = configparser.ConfigParser()
config.read('bot_config.ini')
bot_token = config.get("TELEGRAM", "token")
titsa_idApp = config.get("TITSA", "idApp")
self.adminId = config.get("ADMIN", "chatId")
self.apiHandler = ApiHandler(titsa_idApp)
self.dbHandler = DbHandler()
self.transportThread = OpenTransitThread(
"http://www.titsa.com/Google_transit.zip", 7 * 24 * 60 * 60)
self.transportThread.start()
updater = Updater(token=bot_token, use_context=True)
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.INFO)
b1 = telegram.KeyboardButton("⭐ Consultar favorito ⭐")
b2 = telegram.KeyboardButton("✖️ Borrar favorito ✖️")
b3 = telegram.KeyboardButton("🚊 Tranvia de Tenerife 🚊")
b4 = telegram.KeyboardButton("📍 Ver paradas cercanas 📍",
request_location=True)
self.keyboard = telegram.ReplyKeyboardMarkup([[b3, b4], [b1, b2]],
resize_keyboard=True)
h1 = MessageHandler(Filters.regex(r"^.+Consultar favorito.+$"),
self.favKeyBoard)
h2 = MessageHandler(Filters.regex(u"^\U0001F68F.+(\d{4})"),
self.replyToFav)
h3 = MessageHandler(Filters.regex(r"^.+Borrar favorito.+$"),
self.favKeyBoard)
h4 = MessageHandler(Filters.regex(u"^\u2716.+(\d{4})"), self.eraseFav)
updater.dispatcher.add_handler(CommandHandler("start", self.start))
updater.dispatcher.add_handler(
MessageHandler(Filters.regex(r"^\d{4}$"), self.responder_a_codigo))
updater.dispatcher.add_handler(h1)
updater.dispatcher.add_handler(h2)
updater.dispatcher.add_handler(h3)
updater.dispatcher.add_handler(h4)
updater.dispatcher.add_handler(
ConversationHandler(
entry_points=[
MessageHandler(Filters.regex(r"^.+Tranvia de Tenerife.+$"),
self.listStops),
],
states={
TitsaBot.TRANVIA:
[MessageHandler(Filters.all, self.queryTram)]
},
fallbacks=[]))
updater.dispatcher.add_handler(
MessageHandler(Filters.location, self.nearStops))
updater.dispatcher.add_handler(
ConversationHandler(
entry_points=[
CommandHandler("broadcast", self.newBroadcast),
],
states={
TitsaBot.BROADCAST_TEXT:
[MessageHandler(Filters.all, self.broadcast)]
},
fallbacks=[]))
updater.dispatcher.add_handler(
ConversationHandler(entry_points=[
CommandHandler("addFav",
self.addFavCommand,
pass_args=True,
pass_user_data=True),
CallbackQueryHandler(self.addFavQuery,
pattern=r"^\d{4}$",
pass_user_data=True)
],
states={
TitsaBot.CUSTOM_OR_DEFAULT: [
CallbackQueryHandler(
self.setFavNameOption,
pass_user_data=True)
],
TitsaBot.INSERT_CUSTOM: [
MessageHandler(Filters.text,
self.customName,
pass_user_data=True)
]
},
fallbacks=[h1, h2, h3, h4]))
updater.dispatcher.add_handler(
ConversationHandler(entry_points=[
CallbackQueryHandler(self.reloadStationQuery,
pattern=r"^Repetir \d{4}$",
pass_user_data=True)
],
states={
TitsaBot.CUSTOM_OR_DEFAULT: [
CallbackQueryHandler(
self.reloadStationQuery,
pass_user_data=True)
]
},
fallbacks=[h1, h2, h3, h4]))
updater.dispatcher.add_handler(
CallbackQueryHandler(self.sendStopAndLocation,
pattern=r"^Locate \d{4}$",
pass_user_data=True))
updater.start_polling()
updater.idle()
self.dbHandler.save()
self.transportThread.stop()
class InputValidator:
def __init__(self):
self.db = DbHandler()
# internal methods to check validity of inputs
def _is_user_id_valid(self, user_id):
# check if user id is a valid UUIDv4
try:
uuid_obj = uuid.UUID(user_id, version=4)
except ValueError:
return False
# check if UUID object is the same as the input string
if str(uuid_obj) != user_id:
return False
# check if we have a participant for the input user id
data = self.db.get_participant(user_id)
if data is None:
return False
return True
def _is_ref_domain_valid(self, ref_domain):
ref_domains = [elem[0] for elem in self.db.get_all_ref_domains()]
if ref_domain in ref_domains:
return True
return False
def _is_created_domain_valid(self, created_domain):
# check if special characters are in created domain
if any([
char in created_domain for char in [
"_", "<", ">", "&", "'", "\"", "=", "[", "]", "(", ")",
"%", "$", "?", "#", "*", "+", "/", " ", ",", ";", ":"
]
]):
return False
return True
def _is_valid_integer(self, test_string):
try:
int(test_string)
return True
except ValueError:
return False
def _is_rated_domain_valid(self, rated_domain):
test_domains = [elem[0] for elem in self.db.get_all_test_domains()]
if rated_domain in test_domains:
return True
ref_domains = [elem[0] for elem in self.db.get_all_ref_domains()]
if rated_domain in ref_domains:
return True
step1_domains = [elem[0] for elem in self.db.get_all_step1_domains()]
if rated_domain in step1_domains:
return True
return False
def check_input_user_id(self, user_id):
if self._is_user_id_valid(user_id) is False:
return {"result": False, "message": "Invalid user ID!"}
return {"result": True, "message": ""}
def check_last_unfinished_step(self, user_id, step_id):
if os.environ["CHECK_LAST_UNFINISHED_STEP"] == "1":
data = self.db.get_participant(user_id)
current_step = "step1"
if data[7] == 1:
current_step = "step2"
if data[8] == 1:
current_step = "step3"
if data[9] == 1:
current_step = "step4"
if data[10] == 1:
current_step = "step5"
if data[11] == 1:
current_step = "questionnaire"
if data[12] == 1:
current_step = "final_notes"
if current_step != step_id:
return {"result": True, "current_step": current_step}
else:
return {"result": False, "current_step": "test"}
return {"result": False, "current_step": ""}
def check_log_time(self, user_id, time_type, time):
valid_time_types = ["experiment_start_time", "experiment_end_time",\
"step1_start_time", "step1_end_time",\
"step2_start_time", "step2_end_time",\
"step3_start_time", "step3_end_time",\
"step4_start_time", "step4_end_time",\
"step5_start_time", "step5_end_time",\
"questionnaire_start_time", "questionnaire_end_time"]
if self._is_user_id_valid(user_id) is False:
return {"result": False, "message": "Invalid user ID!"}
if time_type not in valid_time_types:
return {"result": False, "message": "Invalid time type!"}
if self._is_valid_integer(time) is False:
return {"result": False, "message": "Invalid current time!"}
return {"result": True, "message": ""}
def check_input_set_step_finished(self, user_id, step_id):
if self._is_user_id_valid(user_id) is False:
return {"result": False, "message": "Invalid user ID!"}
if step_id not in ["step1", "step2", "step3", "step4", "step5"]:
return {"result": False, "message": "Invalid step ID!"}
return {"result": True, "message": ""}
def check_input_step1_result(self, user_id, ref_domain, created_domain,
elapsed_time, domain_position):
if self.db.get_count("step1", user_id)[0] >= 10:
return {
"result": False,
"message": "Already created maximum number of domains!"
}
if self.db.is_step_finished(user_id, "finished_step1") is not None:
return {"result": False, "message": "Step is already finished!"}
if self._is_user_id_valid(user_id) is False:
return {"result": False, "message": "Invalid user ID!"}
if self._is_ref_domain_valid(ref_domain) is False:
return {"result": False, "message": "Invalid reference domain!"}
if self._is_created_domain_valid(created_domain) is False:
return {"result": False, "message": "Invalid created domain!"}
if self._is_valid_integer(elapsed_time) is False:
return {"result": False, "message": "Invalid elapsed time!"}
if self._is_valid_integer(domain_position) is False:
return {"result": False, "message": "Invalid domain position!"}
return {"result": True, "message": ""}
def check_input_step2_result(self, user_id, ref_domain, squatting_technique, squatting_technique_infos,\
created_domain, elapsed_time, domain_position, squatting_techniques_order):
valid_squatting_techniques = [
"wrong_tld", "homograph", "typosquatting", "combosquatting",
"subdomain"
]
valid_squatting_technique_infos = ["", "prepend_www", "omit_character", "duplicate_character",\
"swap_characters", "replace_qwerty", "none_before_none_behind",\
"term_before_none_behind", "chars_before_none_behind",\
"none_before_term_behind", "none_before_chars_behind",\
"term_before_term_behind", "chars_before_chars_behind",\
"chars_before_term_behind", "term_before_chars_behind"]
if self.db.get_count("step2", user_id)[0] >= 10:
return {
"result": False,
"message": "Already created maximum number of domains!"
}
if self.db.is_step_finished(user_id, "finished_step2") is not None:
return {"result": False, "message": "Step is already finished!"}
if self._is_user_id_valid(user_id) is False:
return {"result": False, "message": "Invalid user ID!"}
if self._is_ref_domain_valid(ref_domain) is False:
return {"result": False, "message": "Invalid reference domain!"}
if squatting_technique not in valid_squatting_techniques:
return {"result": False, "message": "Invalid squatting technique!"}
if squatting_technique_infos not in valid_squatting_technique_infos:
return {
"result": False,
"message": "Invalid squatting technique infos!"
}
if self._is_created_domain_valid(created_domain) is False:
return {"result": False, "message": "Invalid created domain!"}
if self._is_valid_integer(elapsed_time) is False:
return {"result": False, "message": "Invalid elapsed time!"}
if self._is_valid_integer(domain_position) is False:
return {"result": False, "message": "Invalid domain position!"}
if set(squatting_techniques_order.split(",")) != set(
valid_squatting_techniques):
return {
"result": False,
"message": "Invalid order of squatting techniques!"
}
return {"result": True, "message": ""}
def check_input_step3_result(self, user_id, rated_domain, type,
elapsed_time, rating, domain_position):
if self.db.get_step3_rated_domains_count(user_id) >= 30:
return {
"result": False,
"message": "Already created maximum number of domains!"
}
if self.db.is_step_finished(user_id, "finished_step3") is not None:
return {"result": False, "message": "Step is already finished!"}
if self._is_user_id_valid(user_id) is False:
return {"result": False, "message": "Invalid user ID!"}
if self._is_rated_domain_valid(rated_domain) is False:
return {"result": False, "message": "Invalid rated domain!"}
if type not in ["step1", "ref_domain", "phishing_domain"]:
return {"result": False, "message": "Invalid domain type!"}
if self._is_valid_integer(elapsed_time) is False:
return {"result": False, "message": "Invalid elapsed time!"}
if rating not in ["1", "2", "3", "4", "5"]:
return {"result": False, "message": "Invalid rating!"}
if int(domain_position) < 1 or int(domain_position) > 30:
return {"result": False, "message": "Invalid domain position!"}
return {"result": True, "message": ""}
def check_input_step4_result(self, user_id, created_domain, elapsed_time,
domain_position):
if self.db.get_count("step4", user_id)[0] >= 10:
return {
"result": False,
"message": "Already created maximum number of domains!"
}
if self.db.is_step_finished(user_id, "finished_step4") is not None:
return {"result": False, "message": "Step is already finished!"}
if self._is_user_id_valid(user_id) is False:
return {"result": False, "message": "Invalid user ID!"}
if self._is_created_domain_valid(created_domain) is False:
return {"result": False, "message": "Invalid created domain!"}
if self._is_valid_integer(elapsed_time) is False:
return {"result": False, "message": "Invalid elapsed time!"}
if self._is_valid_integer(domain_position) is False:
return {"result": False, "message": "Invalid domain position!"}
return {"result": True, "message": ""}
def check_input_step5_result(self, user_id, selected_domains, elapsed_time,
counter):
if self.db.get_count("step5", user_id)[0] >= 10:
return {
"result": False,
"message": "Already created maximum number of domains!"
}
if self.db.is_step_finished(user_id, "finished_step5") is not None:
return {"result": False, "message": "Step is already finished!"}
if self._is_user_id_valid(user_id) is False:
return {"result": False, "message": "Invalid user ID!"}
if selected_domains.count(",") != 30 or selected_domains.count(
";") != 9:
return {"result": False, "message": "Invalid selected domains!"}
if self._is_valid_integer(elapsed_time) is False:
return {"result": False, "message": "Invalid elapsed time!"}
if self._is_valid_integer(counter) is False:
return {"result": False, "message": "Invalid counter!"}
return {"result": True, "message": ""}
def check_input_questionnaire_result(self, user_id, age, gender_current, education, origin,\
f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12,\
f13, f14, f15, f16, attention_test1, attention_test2):
countries = [elem[0] for elem in self.db.get_countries()]
if self._is_user_id_valid(user_id) is False:
return {"result": False, "message": "Invalid user ID!"}
if age not in [
"18_25", "26_35", "36_45", "46_55", "over_55", "not_answer"
]:
return {"result": False, "message": "Invalid age!"}
if gender_current not in [
"male", "female", "non_binary", "transgender", "other",
"not_answer"
]:
return {"result": False, "message": "Invalid current gender!"}
if education not in ["less_high_school", "high_school", "associate", "no_degree",\
"bachelor", "master", "over_master", "not_answer"]:
return {"result": False, "message": "Invalid education!"}
if origin not in countries:
return {"result": False, "message": "Invalid origin!"}
sebis_valid_answers = [
"never", "rarely", "sometimes", "often", "always"
]
if f1 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f1!"}
if f2 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f2!"}
if f3 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f3!"}
if f4 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f4!"}
if f5 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f5!"}
if f6 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f6!"}
if f7 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f7!"}
if f8 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f8!"}
if f9 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f9!"}
if f10 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f10!"}
if f11 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f11!"}
if f12 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f12!"}
if f13 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f13!"}
if f14 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f14!"}
if f15 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f15!"}
if f16 not in sebis_valid_answers:
return {"result": False, "message": "Invalid f16!"}
if attention_test1 not in sebis_valid_answers:
return {"result": False, "message": "Invalid attention 1!"}
if attention_test2 not in sebis_valid_answers:
return {"result": False, "message": "Invalid attention 2!"}
return {"result": True, "message": ""}
def check_input_final_notes_feedback(self, user_id, feedback):
if self._is_user_id_valid(user_id) is False:
return {"result": False, "message": "Invalid user ID!"}
if len(feedback) > 500:
feedback_length = len(feedback) - 500
return {
"result":
False,
"message":
"Feedback is " + str(feedback_length) + " characters too long!"
}
if any([
char in feedback for char in [
"<", ">", "&", "'", "\"", "=", "(", ")", "%", "$", "?",
"#", "*", "+", "/"
]
]):
return {"result": False, "message": "Not allowed characters used!"}
return {"result": True, "message": ""}
