class Config:
"""
Class responsible for reading the configurations file
"""
def __init__(self):
self.log = Logging("Config")
self.config_parser = configparser.SafeConfigParser()
self.read_file()
def read_file(self):
"""
Read the content of the configurations file
"""
try:
self.config_parser.read("src/config.ini")
except configparser.Error as err:
self.log.error(err)
def get(self, section):
"""
Return the specified section of the configurations file
as a dict
"""
try:
return self.config_parser._sections[section]
except configparser.Error as err:
self.log.error(err)
return []
class Request:
"""
Handles the requests
"""
def __init__(self, url):
self.log = Logging("Requests")
self.url = url
self.decode_gzip = lambda response: zlib.decompress(
response, 16 + zlib.MAX_WBITS)
def header(self, header_type):
"""
Returns the get header stored in the json file
"""
try:
header_file = open("src/headers/%s.json" % header_type, "r")
header_obj = json.loads(header_file.read())
return header_obj
except (OSError, IOError) as err:
self.log.error(err)
return {'': ''}
def request_page(self):
"""
Makes the actual request, return a soup
"""
try:
req = get(self.url, headers=self.header('get'))
while req.status_code != 200:
req = get(self.url, headers=self.header('get'))
if req.encoding == 'gzip':
return self.decode_gzip(req.text)
else:
return req.text
except exceptions.TooManyRedirects:
self.log.error("Request too many redirections on url <%s>" %
self.url)
return "Error"
except exceptions.Timeout:
self.log.error("Request timeout on url <%s>" % self.url)
return "Error"
except exceptions.RequestException as err:
self.log.error(err)
return "Error"
def get_json(self, send_data=None):
"""
Send json post and expects a json return
"""
if send_data is None:
send_data = []
try:
req = post(self.url,
headers=self.header('post'),
data=json.dumps(send_data))
return req.json()
except Exception as err:
self.log.error(err)
def soup(self):
"""
Request wrapper, to garante request is successful
"""
request_result = self.request_page()
while request_result is "Error":
request_result = self.request_page()
return BeautifulSoup(request_result, "html5lib")
class Database(metaclass=Singleton):
"""
Class responsible for connecting and making
operations on the database, it uses a singleton
to maintain a always open connection,
thus saving resources
"""
def __init__(self):
conf = Config()
database_conf = conf.get('database')
self.host = database_conf['host']
self.name = database_conf['name']
self.user = database_conf['user']
self.passwd = database_conf['passwd']
self.log = Logging("Database")
self.conn = self.connect()
def connect(self):
"""
Responsible for opening the connection to the database
and setting the charset
"""
try:
conn = pymysql.connect(self.host,
self.user,
self.passwd,
self.name)
cursor = conn.cursor()
conn.set_charset('utf8')
cursor.execute('SET NAMES utf8;')
cursor.execute('SET CHARACTER SET utf8;')
cursor.execute('SET character_set_connection=utf8;')
return conn
except pymysql.MySQLError as err:
self.log.error(err)
return False
def disconnect(self):
"""
Responsible for closing the connection to the database
"""
self.conn.close()
def execute(self, query, params=None):
"""
Responsible for executing queries on the database
"""
if params is None:
params = []
if not self.conn.open:
self.conn = self.connect()
try:
cursor = self.conn.cursor()
cursor.execute(query, params)
self.conn.commit()
return cursor.fetchall()
except pymysql.MySQLError as err:
self.log.error(err)
return []
def last_inserted_id(self):
try:
query = """SELECT LAST_INSERT_ID()"""
return self.execute(query)[0][0]
except Exception as err:
self.log.error(err)