def __init__(self):
self.logger = logging.getLogger("semitan_resources_fetcher")
file_handler = logging.FileHandler(LOGGING_FILE)
formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
file_handler.setFormatter(formatter)
self.logger.addHandler(file_handler)
self.logger.setLevel(logging.DEBUG)
self.parser = TanParser(self.logger)
self.connection = connector.connect(
user=MYSQL_USER, password=MYSQL_PASSWORD, host=MYSQL_HOST, database=MYSQL_DATABASE
)
class ResourcesFetcher:
def __init__(self):
self.logger = logging.getLogger("semitan_resources_fetcher")
file_handler = logging.FileHandler(LOGGING_FILE)
formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
file_handler.setFormatter(formatter)
self.logger.addHandler(file_handler)
self.logger.setLevel(logging.DEBUG)
self.parser = TanParser(self.logger)
self.connection = connector.connect(
user=MYSQL_USER, password=MYSQL_PASSWORD, host=MYSQL_HOST, database=MYSQL_DATABASE
)
def register_map_in_db(self, name, path, width, height):
cursor = self.connection.cursor()
cursor.execute("""SELECT map_id FROM Line WHERE id = %s""", (name,))
row = cursor.fetchone()
if row != None:
if row[0] == None:
data_tileset = (path, 1, 0)
cursor.execute("""INSERT INTO Tileset (path, validated, cacheTime) VALUES(%s, %s, %s)""", data_tileset)
cursor.execute("""SELECT LAST_INSERT_ID()""")
id_tileset = cursor.fetchone()[0]
data_image = (id_tileset, 0, 0, width, height, None)
cursor.execute(
"""
INSERT INTO Image (tileset_id, topCoord, leftCoord, width, height, newImage_id)
VALUES (%s, %s, %s, %s, %s, %s)""",
data_image,
)
cursor.execute("""SELECT LAST_INSERT_ID()""")
id_image = cursor.fetchone()[0]
cursor.execute("""UPDATE Line SET map_id = %s WHERE id = %s""", (id_image, name))
self.logger.info("line " + name + " map registered in db")
else:
cursor.execute("""SELECT newImage_id FROM Image WHERE id = %s""", (row[0],))
new_image_id = cursor.fetchone()[0]
if new_image_id == None:
data_tileset = (path, 0, 0)
cursor.execute(
"""INSERT INTO Tileset (path, validated, cacheTime) VALUES(%s, %s, %s)""", data_tileset
)
cursor.execute("""SELECT LAST_INSERT_ID()""")
id_tileset = cursor.fetchone()[0]
data_image = (id_tileset, 0, 0, width, height, None)
cursor.execute(
"""
INSERT INTO Image (tileset_id, topCoord, leftCoord, width, height, newImage_id)
VALUES (%s, %s, %s, %s, %s, %s)""",
data_image,
)
cursor.execute("""SELECT LAST_INSERT_ID()""")
id_image = cursor.fetchone()[0]
cursor.execute("""UPDATE Image SET newImage_id = %s WHERE id = %s""", (id_image, row[0]))
self.logger.info("line " + name + " map updated in db")
else:
self.logger.info("new version of the " + name + " map available: " + path)
self.connection.commit()
cursor.close()
def get_html_content(self, url):
try:
httpResponse = urllib.request.urlopen(url)
encoding = httpResponse.getheader("Content-Type").split("=")[1]
content = httpResponse.read().decode(encoding)
except urllib.error.HTTPError as e:
self.logger.error("the server couldn't fulfill the request." + "Error code: " + e.code)
except urllib.error.URLError as e:
self.logger.error("impossible to reach a server." + "Reason: " + e.reason)
return content
def convert_pdf_to_png(self, pdf_filename, png_filename):
with Image(filename=pdf_filename + "[0]", resolution=IMAGE_RESOLUTION) as image:
image.compression_quality = IMAGE_COMPRESSION_QUALITY
if pdf_filename == GLOBAL_MAP_PDF_FILE:
image.resize(GLOBAL_MAP_WIDTH, GLOBAL_MAP_HEIGHT)
else:
image.resize(LINE_MAP_WIDTH, LINE_MAP_HEIGHT)
image.save(filename=png_filename)
self.logger.info(pdf_filename + " converted to " + png_filename)
return (image.width, image.height)
def get_maps_as_png(self):
cursor = self.connection.cursor()
try:
content = self.get_html_content(BUS_SCHEDULES_URL)
self.parser.feed(content)
for line, url in self.parser.schedules_urls.items():
cursor.execute("""SELECT map_id FROM Line WHERE id = %s""", (line,))
row = cursor.fetchone()
filename_suffix = ""
if row != None:
if row[0] != None:
filename_suffix = "_tmp"
pdf_filename = SCHEDULES_FOLDER + line + ".pdf"
map_filename = MAPS_FOLDER + line + filename_suffix + ".png"
try:
urllib.request.urlretrieve(url, pdf_filename)
except urllib.error.HTTPError as e:
self.logger.error(
"impossible to download the schedule of line " + line + ". Error code: " + e.code
)
except urllib.error.URLError as e:
self.logger.error(
"impossible to download the schedule of line " + line + ". Reason: " + e.reason
)
except Exception as e:
self.logger.error(
"impossible to download the schedule of line " + line + ". Message: " + e.message
)
else:
self.logger.info("line " + line + " schedule downloaded in: " + pdf_filename)
(width, height) = self.convert_pdf_to_png(pdf_filename, map_filename)
self.register_map_in_db(line, map_filename, width, height)
else:
self.logger.info("no line named " + line + " in db, " + line + " will be skipped.")
except:
self.logger.error("impossible to reach " + BUS_SCHEDULES_URL)
def get_global_map_as_png(self):
try:
urllib.request.urlretrieve(GLOBAL_MAP_URL, GLOBAL_MAP_PDF_FILE)
except urllib.error.HTTPError as e:
self.logger.error("impossible to download the global map. Error code: " + e.code)
except urllib.error.URLError as e:
self.logger.error("impossible to download the global map. Reason: " + e.reason)
except Exception as e:
self.logger.error("impossible to download the global map. Message: " + e.message)
else:
self.logger.info("global map successfully downloaded")
self.convert_pdf_to_png(GLOBAL_MAP_PDF_FILE, GLOBAL_MAP_PNG_FILE)
def get_logos(self):
try:
urllib.request.urlretrieve(LOGOS_URL, LOGOS_ZIP_FILE)
except urllib.error.HTTPError as e:
self.logger.error("impossible to download the logos archive. Error code: " + e.code)
except urllib.error.URLError as e:
self.logger.error("impossible to download the logos archive. Reason: " + e.reason)
except Exception as e:
self.logger.error("impossible to download the logos archive. Message: " + e.message)
else:
self.logger.error("logos archive successfully downloaded")
main_zip = zipfile.ZipFile(LOGOS_ZIP_FILE)
for subfile in main_zip.namelist():
if subfile.split(".")[-1] == "zip":
subfile_data = io.BytesIO(main_zip.open(subfile).read())
nested_zip = zipfile.ZipFile(subfile_data)
for subfile in nested_zip.namelist():
nested_zip.extract(subfile, LOGOS_FOLDER)
self.logger.info("logos " + subfile + " extracted to " + LOGOS_FOLDER)
def fetch_resources(self):
# pass
self.get_maps_as_png()
# self.get_global_map_as_png()
self.get_logos()
def __del__(self):
self.connection.close()