def scrapxml(self):
logger = com_logger.Logger('RTL')
table = []
for conf in self.config['URLRTL']:
url = self.config['URLRTL'][str(conf)]
logger.info('Check URL:' + url)
if requests.get(url).status_code == 200:
url = urllib.request.urlopen(url).read()
soup = BeautifulSoup(url, "lxml")
soup.prettify()
for item_list in soup.find_all("item"):
a_name = self.fileutils.replace(item_list.find("title").text)
a_link = item_list.find("guid").text
logger.info('Find: ' + a_name)
if com_sqlite.select(a_name) != a_name:
base_dir = os.path.dirname(os.path.abspath(__file__))
db_path = os.path.join(base_dir, self.config['DIRDOWNLOAD']['DIR'])
urllib.request.urlretrieve(a_link, db_path + "/" + a_name + ".mp3")
logger.info('Downloaded: ' + a_name)
com_sqlite.insert(a_name)
table.append(a_name)
mail = com_email.Mail()
mail.send_mail_gmail("RTL: " + a_name, table)
break
def getconfigacquisition(self):
state = self.gpio.getio(self.config_acquisition)
if state:
logger = com_logger.Logger('STOP_ACQUISITION')
logger.debug('INPUT Config')
return state
def getacquisition(self):
state = self.gpio.getio(self.input_acquisition)
if state:
logger = com_logger.Logger('INPUT_ACQUISITION')
logger.debug('INPUT ' + str(state))
return state
def getstart(self):
state = self.gpio.getio(self.start_acquisition)
if state:
logger = com_logger.Logger('START_ACQUISITION')
logger.debug('INPUT Start')
return state
def scrap(self):
logger = com_logger.Logger('BBC')
table = []
try:
for conf in self.config['URLBBC']:
url = self.config['URLBBC'][str(conf)]
logger.info('Check URL: ' + url)
if requests.get(url).status_code == 200:
url = urllib.request.urlopen(url).read()
soup = BeautifulSoup(url, "html.parser")
soup.prettify()
for ul_list in soup.find_all("ul", class_="list-unstyled"):
div_test = ul_list.find_all("div",
class_="programme__body")
if len(div_test) > 0:
for li_list in ul_list.find_all("li", class_=""):
for div_list in li_list.find_all(
"div", class_="programme__body"):
span_name = div_list.find(
"span", class_="programme__title ")
a_name = self.fileutils.replace(
span_name.find("span").text)
div_link = div_list.find(
"div",
class_=
"popup__content popup__content--download br-box-subtle br-subtle-link-onbg br-subtle-link-onborder"
)
if div_link is not None:
a_link = div_link.find(
"a",
class_=
"link-complex popup__list__item island--squashed br-subtle-bg-ontext br-subtle-bg-onbg--hover br-subtle-link-ontext--hover"
)["href"]
logger.info('Find: ' + a_name)
if com_sqlite.select(a_name) != a_name:
logger.info('Download: ' + a_name)
base_dir = os.path.dirname(
os.path.abspath(__file__))
db_path = os.path.join(
base_dir,
self.config['DIRDOWNLOAD']
['DIR'])
urllib.request.urlretrieve(
a_link, db_path + "/" +
a_name + ".mp3")
logger.info('Downloaded !')
com_sqlite.insert(a_name)
table.append(a_name)
mail = com_email.Mail()
mail.send_mail_gmail(
"BBC: " + a_name, table)
break
except Exception as exp:
logger.error(repr(exp))
def is_plugged(function):
def plugged(*original_args, **original_kwargs):
return function(*original_args, **original_kwargs)
if not PiCamera:
logger = com_logger.Logger('CAMERA')
logger.warning('Camera not plugged')
return plugged
def is_plugged(function):
def plugged(*original_args, **original_kwargs):
return function(*original_args, **original_kwargs)
if not GPIO:
logger = com_logger.Logger('GPIO')
logger.warning('GPIO not present')
return plugged
def settime(self, mode, utc):
if not self.initTime:
logger = com_logger.Logger('TIME UTC')
if mode >= 2:
subprocess.call("timedatectl set-time '" + str(utc) + "'",
shell=True)
logger.info('Time set to: ' + str(utc))
self.initTime = True
return self.initTime
def __init__(self):
conf = com_config.Config()
self.config = conf.getconfig()
self.logger = com_logger.Logger('Export Data')
sqlite = com_sqlite.SQLite(True)
self.connexion = sqlite.connection
self.cursor = sqlite.cursor
def read(self):
self.trigger()
time.sleep(0.2)
# Log
logger = com_logger.Logger('DHT22')
logger.debug('Read Temp: ' + str(self.temperature())[:4] + ' Hum: ' +
str(self.humidity())[:4])
return self.temperature(), self.humidity()
def __init__(self, name=''):
self.importlib = GPIO
self.logger = com_logger.Logger(name)
# self.setwarnings(False)
self.IN = GPIO.IN if GPIO is not None else None
self.OUT = GPIO.OUT if GPIO is not None else None
self.LOW = GPIO.LOW if GPIO is not None else None
self.HIGH = GPIO.HIGH if GPIO is not None else None
self.PUD_UP = GPIO.PUD_UP if GPIO is not None else None
self.PUD_DOWN = GPIO.PUD_DOWN if GPIO is not None else None
self.RISING = GPIO.RISING if GPIO is not None else None
def __init__(self):
# Log
self.logger = com_logger.Logger('POI')
# Config
conf = com_config.Config()
self.config = conf.getconfig()
# Get database name
self.database = self.config['SQLITE']['database']
# Database
self.connection = sqlite3.Connection(self.database)
self.cursor = self.connection.cursor()
def send_mail_gmail(self, subject, table, filename=''):
htmlheader = """\
<html>
<head></head>
<body>
<H2>Nouvelles informations</H2><br><br>
<p>
"""
htmlfooter = """\
</p>
</body>
</html>
"""
for line in table:
htmlheader += line + "<br>"
htmlheader += htmlfooter
msg = MIMEMultipart()
msg['To'] = self.config['EMAIL']['to']
msg['From'] = self.config['EMAIL']['from']
msg['Subject'] = subject
msg.attach(MIMEText(htmlheader, 'html'))
if filename:
attachment = open("./" + filename, "rb")
part = MIMEBase('application', 'octet-stream')
part.set_payload(attachment.read())
encoders.encode_base64(part)
part.add_header('Content-Disposition',
"attachment; filename= %s" % filename)
msg.attach(part)
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(self.config['EMAIL']['from'],
self.config['EMAIL']['password'])
text = msg.as_string()
server.sendmail(self.config['EMAIL']['from'],
self.config['EMAIL']['to'], text)
server.quit()
logger = com_logger.Logger('Mail')
logger.info('Mail sent')
def __init__(self, mode):
if PiCamera is not None:
self.imgName = 'PIC' + str(randint(1000, 9999)) + '_'
conf = com_config.Config()
config = conf.getconfig()
self.logger = com_logger.Logger('CAMERA')
self.camera = PiCamera()
if mode == 'PICTURE':
self.camera.resolution = (
int(config['CAMERA']['pic_resolution_x']),
int(config['CAMERA']['pic_resolution_y']))
self.logger.info('Camera mode PICTURE: ' +
config['CAMERA']['pic_resolution_x'] + ' ' +
config['CAMERA']['pic_resolution_y'])
if mode == 'VIDEO':
self.camera.resolution = (
int(config['CAMERA']['vid_resolution_x']),
int(config['CAMERA']['vid_resolution_y']))
self.logger.debug('Init Camera mode VIDEO: ' +
config['CAMERA']['vid_resolution_x'] + ' ' +
config['CAMERA']['vid_resolution_y'])
self.camera.framerate = int(config['CAMERA']['framerate'])
self.camera.rotation = int(config['CAMERA']['rotation'])
# self.camera.brightness = int(config['CAMERA']['brightness'])
# self.camera.contrast = int(config['CAMERA']['contrast'])
if len(config['CAMERA']['image_effect']) > 0:
self.camera.image_effect = config['CAMERA']['image_effect']
self.camera.exposure_mode = config['CAMERA']['exposure_mode']
self.camera.meter_mode = config['CAMERA']['meter_mode']
self.camera.awb_mode = config['CAMERA']['awb']
if len(config['CAMERA']['raw']) > 0:
self.camera.raw_format = config['CAMERA']['raw']
self.path = config['CAMERA']['picture_path']
self.camera.iso = int(config['CAMERA']['ISO'])
self.quality = int(config['CAMERA']['jpegquality'])
self.config = config
self.index = 0
self.last = time()
def getlocalisation(self):
logger = com_logger.Logger('GPS')
self.mode = 0
try:
# Get gps position
packet = gpsd.get_current()
# Debug info GPS
logger.debug('mode:' + str(packet.mode))
logger.debug('lon:' + str(packet.lon))
logger.debug('lat:' + str(packet.lat))
logger.debug('alt:' + str(packet.alt))
logger.debug('hspeed:' + str(packet.hspeed))
# logger.debug('vspeed:' + str(packet.speed_vertical()))
logger.debug('lon +/-:' + str(packet.error['x']))
logger.debug('lat +/-:' + str(packet.error['y']))
logger.debug('alt +/-:' + str(packet.error['v']))
logger.debug('sats:' + str(packet.sats))
logger.debug('track:' + str(packet.track))
logger.debug('time:' + str(packet.time))
# See the inline docs for GpsResponse for the available data
self.altitude = 0.0
self.mode = packet.mode
if self.mode >= 2:
self.sats = packet.sats
self.track = packet.track
self.longitude = packet.lon
self.latitude = packet.lat
self.timeutc = packet.time
self.hspeed = round(packet.hspeed * 1.60934, 1)
self.lonprecision = packet.error['x']
self.latprecision = packet.error['y']
if self.mode >= 3:
self.altitude = packet.altitude()
self.altprecision = packet.error['v']
# self.vspeed = packet.speed_vertical()
except:
logger.error('GPS exception')
def scrap(self):
logger = com_logger.Logger('RTL')
for conf in self.config['URLRTL']:
url = self.config['URLRTL'][str(conf)]
logger.info('Check URL: ' + url)
if requests.get(url).status_code == 200:
url = urllib.request.urlopen(url).read()
soup = BeautifulSoup(url, "html.parser")
soup.prettify()
table = []
for div_list in soup.find_all("div", class_ = "timeline-post shift"):
try:
div_item = div_list.find("div", class_ = "post-fig brd brd-grey")
a_link = div_item.find("a", class_ = "post-link")["href"]
if requests.get(a_link).status_code == 200:
a_link = urllib.request.urlopen(a_link).read()
soupnext = BeautifulSoup(a_link, "html.parser")
soupnext.prettify()
div_mp3 = soupnext.find("figcaption", class_ = "figcaption article-mdl cf")
a_name = self.fileutils.replace(div_mp3.find("span", class_ = "legend").text)
a_link = div_mp3.find("a", class_ = "dl icon icon-download")["href"]
logger.info('Find: ' + a_name)
if com_sqlite.select(a_name) != a_name:
logger.info('Download: ' + a_name)
base_name = os.path.dirname(os.path.abspath(__file__))
db_path = os.path.join(base_name, self.config['DIRDOWNLOAD']['DIR'])
urllib.request.urlretrieve(a_link, db_path + "/" + a_name + ".mp3")
logger.info('Downloaded !')
com_sqlite.insert(a_name)
table.append(a_name)
mail = com_email.Mail()
mail.send_mail_gmail("RTL: " + a_name, table)
break
except Exception as exp:
logger.error(repr(exp))
def send_mail_gmail(subject, table, filename=""):
config = com_config.getConfig()
logger = com_logger.Logger('Email')
logger.log.info('Sending email')
msg = MIMEMultipart()
body=''
for line in table:
body += line + "<br>"
msg['From'] = config['EMAIL']['from']
msg['To'] = config['EMAIL']['to']
msg['Subject'] = subject
msg.attach(MIMEText(body, 'html'))
try:
if len(filename) > 0:
attachment = open("./" + filename, "rb")
part = MIMEBase('application', 'octet-stream')
part.set_payload((attachment).read())
encoders.encode_base64(part)
part.add_header('Content-Disposition', "attachment; filename= %s" % filename)
msg.attach(part)
logger.log.debug('Try to connect SMTP')
server = smtplib.SMTP('smtp.gmail.com', 587)
logger.log.debug('Connected to SMTP')
server.starttls()
logger.log.debug('Try to connect mail box')
server.login(config['EMAIL']['from'], config['EMAIL']['password'])
text = msg.as_string()
logger.log.debug('Connected to mailbox')
logger.log.debug('Try to send mail')
server.sendmail(config['EMAIL']['from'], config['EMAIL']['to'], text)
logger.log.debug('Mail sent')
server.quit()
except:
logger.log.critical('Error sending mail')
def getIFFT():
logger = com_logger.Logger('GetCommand')
config = com_config.getConfig()
logger.log.info('Getting command')
command = ''
try:
logger.log.debug('Try to connect IMAP')
connection = imaplib.IMAP4_SSL('imap.gmail.com')
logger.log.debug('Connected to IMAP')
logger.log.debug('Try to login mailbox')
connection.login(config['EMAIL']['username'], config['EMAIL']['password'])
logger.log.debug('Connected to mailbox')
rv, data = connection.select('INBOX')
if rv == 'OK':
logger.log.debug('Check command to mailbox')
resp, items = connection.search(None, '(UNSEEN FROM "' + config['CCM'][
'mailfrom'] + '")') # (FROM "Doug" SUBJECT "test message 2"
items = items[0].split()
for emailid in items:
resp, data = connection.fetch(emailid, "(BODY[HEADER.FIELDS (SUBJECT)])")
msg = email.message_from_string(str(data[0][1]))
subject = (msg["b'Subject"].replace('\\n', '').replace('\\r', '').replace("'", ''))
logger.log.debug(subject)
lst = subject.split(':')
if len(lst) == 2 and lst[0] == config['CCM']['mailsubject']:
logger.log.debug('Command: ' + lst[1])
command = lst[1]
else:
logger.log.debug('Mail with no command')
connection.close()
connection.logout()
logger.log.debug('Logout IMAP')
except socket.gaierror:
logger.log.critical('Error get command')
finally:
return command
Autor: Bruno
Date : 09/06/2017
"""
import datetime
from lib import com_config, com_logger
from utils import POI
# Config
conf = com_config.Config()
conf.setconfig()
config = conf.getconfig()
# Log
logger = com_logger.Logger()
logger.info('Application start')
# POI
poi = POI.POI()
# Import RadarFile
# Start time
start = datetime.datetime.now()
poi.importpoi()
# Stop time
end = datetime.datetime.now()
logger.info('Import duration: ' + str(end - start))
logger.info('Application stop')
"""
Auteur: Bruno DELATTRE
Date : 12/08/2016
"""
import scapperRTL
import scraperBBC
import scraperFranceCulture
import scraperFranceInter
from lib import com_config, com_logger
conf = com_config.Config()
conf.setconfig()
config = conf.getconfig()
logger = com_logger.Logger('Main')
logger.info(config['APPLICATION']['name'] + ' ' + config['APPLICATION']['version'])
logger.info('Start')
# Scrap ! INFO: soup.findall(attrs={'class':None or value})
fc = scraperFranceCulture.FranceCulture()
fc.scrapxml()
fi = scraperFranceInter.FranceInter()
fi.scrapxml()
bbc = scraperBBC.BBC()
bbc.scrap()
rtl = scapperRTL.RTL()
rtl.scrapxml()
def __init__(self, name='', file=''):
self.logger = com_logger.Logger(name, file)
def __init__(self):
conf = com_config.Config()
self.config = conf.getconfig()
self.logger = com_logger.Logger('Mail')
def run(self):
logger = com_logger.Logger('Camera Thread')
logger.info('Start')
self.getpicture()
logger.info('Stop')
def __init__(self):
self.logger = com_logger.Logger('FILE')
def __init__(self):
conf = com_config.Config()
self.config = conf.getconfig()
self.logger = com_logger.Logger('SQLite')
self.database = self.config['SQLITE']['database']
def __init__(self, connection, cursor):
# Log
self.logger = com_logger.Logger()
self.connection = connection
self.cursor = cursor
def setacquisition(self, state):
if self.led_acquisition > 0:
self.gpio.setio(self.led_acquisition, state)
logger = com_logger.Logger('LED_ACQUISITION')
logger.debug('LED ' + str(state))
def scrap():
logger = com_logger.Logger('Scraper')
url_start = "https://www.leboncoin.fr/annonces/offres/haute_normandie/?o="
# https://www.leboncoin.fr/annonces/offres/?th=1
# https: // www.leboncoin.fr/annonces/offres/haute_normandie/?th = 1 & q = vtt
url_search = "&q="
url_end = "&it=1" # vide sinon recherche uniquement dans le titre "&it=1"
search_list = []
# Get Things to search from config file
conf = com_config.Config()
config = conf.getconfig()
for i in range(1, 50):
strthings = 'things' + str(i)
try:
if len(config['SEARCH'][strthings]) != 0:
search_list.append(config['SEARCH'][strthings])
except Exception as exp:
logger.info('Things stop at:' + str(exp))
break
max_browse = int(config['SEARCH']['max_browse'])
logger.info('Start extraction')
for search_item in search_list:
tab = search_item.split(",")
prix_min = int(tab[1])
prix_max = int(tab[2])
index = 1
contenuhtml = []
logger.info('Search: ' + urllib.parse.unquote(tab[0]))
urlok = 0
try:
urlok = requests.get(
url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end).status_code
except Exception as exp:
logger.error('URL: ' + url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end)
logger.error(str(exp))
while urlok == 200 and index <= max_browse:
try:
url = urllib.request.urlopen(
url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end).read()
except Exception as exp:
logger.error('URL: ' + url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end)
logger.error(str(exp))
break
soup = BeautifulSoup(url, "html.parser")
soup.prettify()
if len(soup.find_all("section", class_="tabsContent block-white dontSwitch")) == 0:
break
for section in soup.find_all("section", class_="tabsContent block-white dontSwitch"):
for li in section.find_all("li"):
link = li.find("a", class_="list_item clearfix trackable")["href"]
idx = \
li.find("a", class_="list_item clearfix trackable")["data-info"].strip().split(",", 7)[2].split(
":", 2)[1].replace('"', "").strip()
# container = source.find('div', attrs={'id':'dlbox'})
imglink = ''
try:
imglink = li.find("span", class_="lazyload")["data-imgsrc"]
except Exception as exp:
logger.debug('No image: ' + str(exp))
pass
for item in li.find_all("section", class_="item_infos"):
titre = urllib.parse.unquote(
item.find("h2", class_="item_title").text.replace("\n", "").replace("\t", ""))
category = urllib.parse.unquote(
item.find("p", itemprop="category").text.replace("\n", "").replace("\t", ""))
localisation = urllib.parse.unquote(
item.find("p", itemprop="availableAtOrFrom").text.replace("\n", "").replace("\t", ""))
try:
prix = int(item.find("h3", class_="item_price").text.encode("ASCII", 'ignore').strip())
except AttributeError:
prix = 0
except ValueError:
prix = 0
if (prix >= prix_min) and (prix <= prix_max):
if com_sqlite.select(idx) == 0: # Item not yet present in database
logger.info('Find: ' + titre.strip())
com_sqlite.insert(idx)
contenuhtml = mailcontent(contenuhtml, imglink, link, prix, titre, category,
localisation)
index += 1
logger.debug('Page : ' + str(index))
if len(contenuhtml) > 0:
contenu = Scraper.mailfull(urllib.parse.unquote(tab[0] + " Prix: " + str(prix_min) + "-" + str(prix_max) + ' €'), contenuhtml)
com_email.send_mail_gmail(urllib.parse.unquote(tab[0]), contenu)
logger.info('Mail sent')
logger.info('End extraction')
"""
main.py v 1.1.5
Auteur: Bruno DELATTRE
Date : 03/12/2016
"""
from acquisition import thread_acquisition_camera_timer
from lib import com_config, com_gpio_inout, com_logger
# Config
conf = com_config.Config()
conf.setconfig()
config = conf.getconfig()
# Log
logger = com_logger.Logger('TIMELAPSE')
logger.info(config['APPLICATION']['name'] + ' ' + config['APPLICATION']['version'] + ' ' + 'Start')
# Init
gpioinout = com_gpio_inout.GPIOINOT()
delayqcquition = 0
tabdelay = [4, 6, 6, 10, 12]
timedelay = 3
gpioinout.blink(0.2, 3)
# Waiting for Init acquisition
logger.info('Wait for start')
logger.info('Delay: ' + str(tabdelay[delayqcquition] + timedelay))
while not gpioinout.getstart():
# Check config
if gpioinout.getconfigacquisition():
def __init__(self, connection, cursor):
self.connection = connection
self.cursor = cursor
self.logger = com_logger.Logger('DHT22 DAL')
