def __init__(self, name=''):
conf = com_config.Config()
self.config = conf.getconfig()
self.logger = logging.Logger(name, logging.DEBUG)
self.logger.name = name
# Formatter
formatterfile = logging.Formatter(
'%(asctime)s %(levelname)s : %(name)s - %(message)s',
datefmt='%d/%m/%Y %H:%M:%S')
formatterconsole = colorlog.ColoredFormatter(
'%(asctime)s %(log_color)s%(levelname)s : %(name)s - %(message)s',
datefmt='%d/%m/%Y %H:%M:%S',
log_colors={
'DEBUG': 'white',
'INFO': 'green',
'WARNING': 'bold_yellow',
'ERROR': 'bold_red',
'CRITICAL': 'bold_red'
})
# First logger (file)
self.logger.setLevel(logging.DEBUG)
file_handler = RotatingFileHandler(
self.config['LOGGER']['logfile'], 'a',
int(self.config['LOGGER']['logfilesize']), 1)
file_handler.setLevel(int(self.config['LOGGER']['levelfile']))
file_handler.setFormatter(formatterfile)
self.logger.addHandler(file_handler)
# second logger (console)
steam_handler = logging.StreamHandler()
steam_handler.setLevel(int(self.config['LOGGER']['levelconsole']))
steam_handler.setFormatter(formatterconsole)
self.logger.addHandler(steam_handler)
def __init__(self):
conf = com_config.Config()
self.config = conf.getconfig()
self.logger = com_logger.Logger('Export Data')
sqlite = com_sqlite.SQLite(True)
self.connexion = sqlite.connection
self.cursor = sqlite.cursor
def __init__(self, lock, delay):
super().__init__()
conf = com_config.Config()
config = conf.getconfig()
self.delay = delay
self.lock = lock
self.database = config['SQLITE']['database']
self.instance = com_camera.Camera('PICTURE')
# GPIO
self.gpioinout = com_gpio_inout.GPIOINOT()
def __init__(self, copy=False):
conf = com_config.Config()
self.config = conf.getconfig()
if copy:
self.connection = sqlite3.Connection(
self.config['SQLITE']['databasecopy'])
else:
self.connection = sqlite3.Connection(
self.config['SQLITE']['database'])
self.cursor = self.connection.cursor()
def __init__(self, delay):
super().__init__()
self.delay = delay
self.hFunction = self.getpicture
self.thread = Timer(self.delay, self.handle_function)
conf = com_config.Config()
config = conf.getconfig()
self.database = config['SQLITE']['database']
self.instance = com_camera.Camera('PICTURE')
# GPIO
self.gpioinout = com_gpio_inout.GPIOINOT()
def __init__(self):
# Log
self.logger = com_logger.Logger('POI')
# Config
conf = com_config.Config()
self.config = conf.getconfig()
# Get database name
self.database = self.config['SQLITE']['database']
# Database
self.connection = sqlite3.Connection(self.database)
self.cursor = self.connection.cursor()
def __init__(self, name, lock, port, delay, delayread, delayws, counter, ledport, infiny = False):
super().__init__()
conf = com_config.Config()
config = conf.getconfig()
self.name = name
self.port = port
self.ledport = ledport
self.counter = counter
self.delay = delay
self.delayread = delayread
self.delayws = delayws
self.lock = lock
self.infiny = infiny
self.database = config['SQLITE']['database']
def __init__(self):
self.gpio = com_gpio.GPIODialog('LED ACQUISITION')
self.gpio.setmodebcm()
conf = com_config.Config()
self.config = conf.getconfig()
# LED ACQUISITION
self.led_acquisition = int(self.config['GPIO']['LED_ACQUISITION'])
self.gpio.setup(self.led_acquisition, self.gpio.OUT)
# INPUT ACQUISITION
self.input_acquisition = int(self.config['GPIO']['INPUT_ACQUISITION'])
self.gpio.setuppud(self.input_acquisition, self.gpio.IN,
self.gpio.PUD_DOWN)
def __init__(self, mode):
if PiCamera is not None:
self.imgName = 'PIC' + str(randint(1000, 9999)) + '_'
conf = com_config.Config()
config = conf.getconfig()
self.logger = com_logger.Logger('CAMERA')
self.camera = PiCamera()
if mode == 'PICTURE':
self.camera.resolution = (
int(config['CAMERA']['pic_resolution_x']),
int(config['CAMERA']['pic_resolution_y']))
self.logger.info('Camera mode PICTURE: ' +
config['CAMERA']['pic_resolution_x'] + ' ' +
config['CAMERA']['pic_resolution_y'])
if mode == 'VIDEO':
self.camera.resolution = (
int(config['CAMERA']['vid_resolution_x']),
int(config['CAMERA']['vid_resolution_y']))
self.logger.debug('Init Camera mode VIDEO: ' +
config['CAMERA']['vid_resolution_x'] + ' ' +
config['CAMERA']['vid_resolution_y'])
self.camera.framerate = int(config['CAMERA']['framerate'])
self.camera.rotation = int(config['CAMERA']['rotation'])
# self.camera.brightness = int(config['CAMERA']['brightness'])
# self.camera.contrast = int(config['CAMERA']['contrast'])
if len(config['CAMERA']['image_effect']) > 0:
self.camera.image_effect = config['CAMERA']['image_effect']
self.camera.exposure_mode = config['CAMERA']['exposure_mode']
self.camera.meter_mode = config['CAMERA']['meter_mode']
self.camera.awb_mode = config['CAMERA']['awb']
if len(config['CAMERA']['raw']) > 0:
self.camera.raw_format = config['CAMERA']['raw']
self.path = config['CAMERA']['picture_path']
self.camera.iso = int(config['CAMERA']['ISO'])
self.quality = int(config['CAMERA']['jpegquality'])
self.config = config
self.index = 0
self.last = time()
def __init__(self):
self.gpio = com_gpio.GPIODialog('LED ACQUISITION')
self.gpio.setmodebcm()
conf = com_config.Config()
self.config = conf.getconfig()
# LED ACQUISITION
self.led_acquisition = int(self.config['GPIO']['LED_ACQUISITION'])
self.gpio.setup(self.led_acquisition, self.gpio.OUT)
# START ACQUISITION
self.start_acquisition = int(self.config['GPIO']['START_ACQUISITION'])
self.gpio.setuppud(self.start_acquisition, self.gpio.IN,
self.gpio.PUD_DOWN)
# CONFIG ACQUISITION
self.config_acquisition = int(
self.config['GPIO']['CONFIG_ACQUISITION'])
self.gpio.setuppud(self.config_acquisition, self.gpio.IN,
self.gpio.PUD_DOWN)
def send_mail_gmail(subject, table, filename = ''):
conf = com_config.Config()
config = conf.getconfig()
logger = com_logger.Logger('Email')
logger.info('Sending email')
msg = MIMEMultipart()
body = "".join(str(l) for l in table)
body = body.replace("', '","").replace("['","").replace("']","").replace("', \"","").replace("\", '","")
msg['From'] = config['EMAIL']['from']
msg['To'] = config['EMAIL']['to']
msg['Subject'] = subject
msg.attach(MIMEText(body, 'html'))
try:
if len(filename) > 0:
attachment = open("./" + filename, "rb")
part = MIMEBase('application', 'octet-stream')
part.set_payload(attachment.read())
encoders.encode_base64(part)
part.add_header('Content-Disposition', "attachment; filename= %s" % filename)
msg.attach(part)
logger.debug('Try to connect SMTP')
server = smtplib.SMTP('smtp.gmail.com', 587)
logger.debug('Connected to SMTP')
server.starttls()
logger.debug('Try to connect mail box')
server.login(config['EMAIL']['from'], config['EMAIL']['password'])
text = msg.as_string()
logger.debug('Connected to mailbox')
logger.debug('Try to send mail')
server.sendmail(config['EMAIL']['from'], config['EMAIL']['to'], text)
logger.debug('Mail sent')
server.quit()
except Exception as exp:
logger.critical('Error sending mail' + str(exp))
def __init__(self):
conf = com_config.Config()
self.config = conf.getconfig()
font_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../fonts',
'FreeSans.ttf'))
img_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../images',
'tempicone.png'))
self.tempicone = Image.open(img_path).transform(device.size, Image.AFFINE, (1, 0, 0, 0, 1, 0), Image.BILINEAR) \
.convert(device.mode)
img_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../images',
'humicone.png'))
self.humicone = Image.open(img_path).transform(device.size, Image.AFFINE, (1, 0, 0, 0, 1, 0), Image.BILINEAR) \
.convert(device.mode)
self.smallfont = ImageFont.truetype(font_path, 10)
self.normalfont = ImageFont.truetype(font_path, 14)
self.bigfont = ImageFont.truetype(font_path, 27)
def __init__(self):
conf = com_config.Config()
self.config = conf.getconfig()
self.fileutils = fileutils.FileUtils()
# coding=utf-8
"""
Auteur: Bruno DELATTRE
Date : 12/08/2016
"""
import scapperRTL
import scraperBBC
import scraperFranceCulture
import scraperFranceInter
from lib import com_config, com_logger
conf = com_config.Config()
conf.setconfig()
config = conf.getconfig()
logger = com_logger.Logger('Main')
logger.info(config['APPLICATION']['name'] + ' ' + config['APPLICATION']['version'])
logger.info('Start')
# Scrap ! INFO: soup.findall(attrs={'class':None or value})
fc = scraperFranceCulture.FranceCulture()
fc.scrapxml()
fi = scraperFranceInter.FranceInter()
fi.scrapxml()
bbc = scraperBBC.BBC()
bbc.scrap()
rtl = scapperRTL.RTL()
def __init__(self):
conf = com_config.Config()
self.config = conf.getconfig()
self.logger = com_logger.Logger('SQLite')
self.database = self.config['SQLITE']['database']
def scrap():
logger = com_logger.Logger('Scraper')
url_start = "https://www.leboncoin.fr/annonces/offres/haute_normandie/?o="
# https://www.leboncoin.fr/annonces/offres/?th=1
# https: // www.leboncoin.fr/annonces/offres/haute_normandie/?th = 1 & q = vtt
url_search = "&q="
url_end = "&it=1" # vide sinon recherche uniquement dans le titre "&it=1"
search_list = []
# Get Things to search from config file
conf = com_config.Config()
config = conf.getconfig()
for i in range(1, 50):
strthings = 'things' + str(i)
try:
if len(config['SEARCH'][strthings]) != 0:
search_list.append(config['SEARCH'][strthings])
except Exception as exp:
logger.info('Things stop at:' + str(exp))
break
max_browse = int(config['SEARCH']['max_browse'])
logger.info('Start extraction')
for search_item in search_list:
tab = search_item.split(",")
prix_min = int(tab[1])
prix_max = int(tab[2])
index = 1
contenuhtml = []
logger.info('Search: ' + urllib.parse.unquote(tab[0]))
urlok = 0
try:
urlok = requests.get(
url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end).status_code
except Exception as exp:
logger.error('URL: ' + url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end)
logger.error(str(exp))
while urlok == 200 and index <= max_browse:
try:
url = urllib.request.urlopen(
url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end).read()
except Exception as exp:
logger.error('URL: ' + url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end)
logger.error(str(exp))
break
soup = BeautifulSoup(url, "html.parser")
soup.prettify()
if len(soup.find_all("section", class_="tabsContent block-white dontSwitch")) == 0:
break
for section in soup.find_all("section", class_="tabsContent block-white dontSwitch"):
for li in section.find_all("li"):
link = li.find("a", class_="list_item clearfix trackable")["href"]
idx = \
li.find("a", class_="list_item clearfix trackable")["data-info"].strip().split(",", 7)[2].split(
":", 2)[1].replace('"', "").strip()
# container = source.find('div', attrs={'id':'dlbox'})
imglink = ''
try:
imglink = li.find("span", class_="lazyload")["data-imgsrc"]
except Exception as exp:
logger.debug('No image: ' + str(exp))
pass
for item in li.find_all("section", class_="item_infos"):
titre = urllib.parse.unquote(
item.find("h2", class_="item_title").text.replace("\n", "").replace("\t", ""))
category = urllib.parse.unquote(
item.find("p", itemprop="category").text.replace("\n", "").replace("\t", ""))
localisation = urllib.parse.unquote(
item.find("p", itemprop="availableAtOrFrom").text.replace("\n", "").replace("\t", ""))
try:
prix = int(item.find("h3", class_="item_price").text.encode("ASCII", 'ignore').strip())
except AttributeError:
prix = 0
except ValueError:
prix = 0
if (prix >= prix_min) and (prix <= prix_max):
if com_sqlite.select(idx) == 0: # Item not yet present in database
logger.info('Find: ' + titre.strip())
com_sqlite.insert(idx)
contenuhtml = mailcontent(contenuhtml, imglink, link, prix, titre, category,
localisation)
index += 1
logger.debug('Page : ' + str(index))
if len(contenuhtml) > 0:
contenu = Scraper.mailfull(urllib.parse.unquote(tab[0] + " Prix: " + str(prix_min) + "-" + str(prix_max) + ' €'), contenuhtml)
com_email.send_mail_gmail(urllib.parse.unquote(tab[0]), contenu)
logger.info('Mail sent')
logger.info('End extraction')
def connect():
conf = com_config.Config()
config = conf.getconfig()
con = sqlite3.connect(config['SQLITE']['database'])
cursor = con.cursor()
return con, cursor
def __init__(self):
conf = com_config.Config()
self.config = conf.getconfig()
def __init__(self):
conf = com_config.Config()
self.config = conf.getconfig()
self.connection = sqlite3.Connection(self.config['SQLITE']['database'])
self.cursor = self.connection.cursor()
def __init__(self):
conf = com_config.Config()
self.config = conf.getconfig()
self.logger = com_logger.Logger('Mail')
