def scrapxml(self): logger = com_logger.Logger('RTL') table = [] for conf in self.config['URLRTL']: url = self.config['URLRTL'][str(conf)] logger.info('Check URL:' + url) if requests.get(url).status_code == 200: url = urllib.request.urlopen(url).read() soup = BeautifulSoup(url, "lxml") soup.prettify() for item_list in soup.find_all("item"): a_name = self.fileutils.replace(item_list.find("title").text) a_link = item_list.find("guid").text logger.info('Find: ' + a_name) if com_sqlite.select(a_name) != a_name: base_dir = os.path.dirname(os.path.abspath(__file__)) db_path = os.path.join(base_dir, self.config['DIRDOWNLOAD']['DIR']) urllib.request.urlretrieve(a_link, db_path + "/" + a_name + ".mp3") logger.info('Downloaded: ' + a_name) com_sqlite.insert(a_name) table.append(a_name) mail = com_email.Mail() mail.send_mail_gmail("RTL: " + a_name, table) break
def getconfigacquisition(self): state = self.gpio.getio(self.config_acquisition) if state: logger = com_logger.Logger('STOP_ACQUISITION') logger.debug('INPUT Config') return state
def getacquisition(self): state = self.gpio.getio(self.input_acquisition) if state: logger = com_logger.Logger('INPUT_ACQUISITION') logger.debug('INPUT ' + str(state)) return state
def getstart(self): state = self.gpio.getio(self.start_acquisition) if state: logger = com_logger.Logger('START_ACQUISITION') logger.debug('INPUT Start') return state
def scrap(self): logger = com_logger.Logger('BBC') table = [] try: for conf in self.config['URLBBC']: url = self.config['URLBBC'][str(conf)] logger.info('Check URL: ' + url) if requests.get(url).status_code == 200: url = urllib.request.urlopen(url).read() soup = BeautifulSoup(url, "html.parser") soup.prettify() for ul_list in soup.find_all("ul", class_="list-unstyled"): div_test = ul_list.find_all("div", class_="programme__body") if len(div_test) > 0: for li_list in ul_list.find_all("li", class_=""): for div_list in li_list.find_all( "div", class_="programme__body"): span_name = div_list.find( "span", class_="programme__title ") a_name = self.fileutils.replace( span_name.find("span").text) div_link = div_list.find( "div", class_= "popup__content popup__content--download br-box-subtle br-subtle-link-onbg br-subtle-link-onborder" ) if div_link is not None: a_link = div_link.find( "a", class_= "link-complex popup__list__item island--squashed br-subtle-bg-ontext br-subtle-bg-onbg--hover br-subtle-link-ontext--hover" )["href"] logger.info('Find: ' + a_name) if com_sqlite.select(a_name) != a_name: logger.info('Download: ' + a_name) base_dir = os.path.dirname( os.path.abspath(__file__)) db_path = os.path.join( base_dir, self.config['DIRDOWNLOAD'] ['DIR']) urllib.request.urlretrieve( a_link, db_path + "/" + a_name + ".mp3") logger.info('Downloaded !') com_sqlite.insert(a_name) table.append(a_name) mail = com_email.Mail() mail.send_mail_gmail( "BBC: " + a_name, table) break except Exception as exp: logger.error(repr(exp))
def is_plugged(function): def plugged(*original_args, **original_kwargs): return function(*original_args, **original_kwargs) if not PiCamera: logger = com_logger.Logger('CAMERA') logger.warning('Camera not plugged') return plugged
def is_plugged(function): def plugged(*original_args, **original_kwargs): return function(*original_args, **original_kwargs) if not GPIO: logger = com_logger.Logger('GPIO') logger.warning('GPIO not present') return plugged
def settime(self, mode, utc): if not self.initTime: logger = com_logger.Logger('TIME UTC') if mode >= 2: subprocess.call("timedatectl set-time '" + str(utc) + "'", shell=True) logger.info('Time set to: ' + str(utc)) self.initTime = True return self.initTime
def __init__(self): conf = com_config.Config() self.config = conf.getconfig() self.logger = com_logger.Logger('Export Data') sqlite = com_sqlite.SQLite(True) self.connexion = sqlite.connection self.cursor = sqlite.cursor
def read(self): self.trigger() time.sleep(0.2) # Log logger = com_logger.Logger('DHT22') logger.debug('Read Temp: ' + str(self.temperature())[:4] + ' Hum: ' + str(self.humidity())[:4]) return self.temperature(), self.humidity()
def __init__(self, name=''): self.importlib = GPIO self.logger = com_logger.Logger(name) # self.setwarnings(False) self.IN = GPIO.IN if GPIO is not None else None self.OUT = GPIO.OUT if GPIO is not None else None self.LOW = GPIO.LOW if GPIO is not None else None self.HIGH = GPIO.HIGH if GPIO is not None else None self.PUD_UP = GPIO.PUD_UP if GPIO is not None else None self.PUD_DOWN = GPIO.PUD_DOWN if GPIO is not None else None self.RISING = GPIO.RISING if GPIO is not None else None
def __init__(self): # Log self.logger = com_logger.Logger('POI') # Config conf = com_config.Config() self.config = conf.getconfig() # Get database name self.database = self.config['SQLITE']['database'] # Database self.connection = sqlite3.Connection(self.database) self.cursor = self.connection.cursor()
def send_mail_gmail(self, subject, table, filename=''): htmlheader = """\ <html> <head></head> <body> <H2>Nouvelles informations</H2><br><br> <p> """ htmlfooter = """\ </p> </body> </html> """ for line in table: htmlheader += line + "<br>" htmlheader += htmlfooter msg = MIMEMultipart() msg['To'] = self.config['EMAIL']['to'] msg['From'] = self.config['EMAIL']['from'] msg['Subject'] = subject msg.attach(MIMEText(htmlheader, 'html')) if filename: attachment = open("./" + filename, "rb") part = MIMEBase('application', 'octet-stream') part.set_payload(attachment.read()) encoders.encode_base64(part) part.add_header('Content-Disposition', "attachment; filename= %s" % filename) msg.attach(part) server = smtplib.SMTP('smtp.gmail.com', 587) server.starttls() server.login(self.config['EMAIL']['from'], self.config['EMAIL']['password']) text = msg.as_string() server.sendmail(self.config['EMAIL']['from'], self.config['EMAIL']['to'], text) server.quit() logger = com_logger.Logger('Mail') logger.info('Mail sent')
def __init__(self, mode): if PiCamera is not None: self.imgName = 'PIC' + str(randint(1000, 9999)) + '_' conf = com_config.Config() config = conf.getconfig() self.logger = com_logger.Logger('CAMERA') self.camera = PiCamera() if mode == 'PICTURE': self.camera.resolution = ( int(config['CAMERA']['pic_resolution_x']), int(config['CAMERA']['pic_resolution_y'])) self.logger.info('Camera mode PICTURE: ' + config['CAMERA']['pic_resolution_x'] + ' ' + config['CAMERA']['pic_resolution_y']) if mode == 'VIDEO': self.camera.resolution = ( int(config['CAMERA']['vid_resolution_x']), int(config['CAMERA']['vid_resolution_y'])) self.logger.debug('Init Camera mode VIDEO: ' + config['CAMERA']['vid_resolution_x'] + ' ' + config['CAMERA']['vid_resolution_y']) self.camera.framerate = int(config['CAMERA']['framerate']) self.camera.rotation = int(config['CAMERA']['rotation']) # self.camera.brightness = int(config['CAMERA']['brightness']) # self.camera.contrast = int(config['CAMERA']['contrast']) if len(config['CAMERA']['image_effect']) > 0: self.camera.image_effect = config['CAMERA']['image_effect'] self.camera.exposure_mode = config['CAMERA']['exposure_mode'] self.camera.meter_mode = config['CAMERA']['meter_mode'] self.camera.awb_mode = config['CAMERA']['awb'] if len(config['CAMERA']['raw']) > 0: self.camera.raw_format = config['CAMERA']['raw'] self.path = config['CAMERA']['picture_path'] self.camera.iso = int(config['CAMERA']['ISO']) self.quality = int(config['CAMERA']['jpegquality']) self.config = config self.index = 0 self.last = time()
def getlocalisation(self): logger = com_logger.Logger('GPS') self.mode = 0 try: # Get gps position packet = gpsd.get_current() # Debug info GPS logger.debug('mode:' + str(packet.mode)) logger.debug('lon:' + str(packet.lon)) logger.debug('lat:' + str(packet.lat)) logger.debug('alt:' + str(packet.alt)) logger.debug('hspeed:' + str(packet.hspeed)) # logger.debug('vspeed:' + str(packet.speed_vertical())) logger.debug('lon +/-:' + str(packet.error['x'])) logger.debug('lat +/-:' + str(packet.error['y'])) logger.debug('alt +/-:' + str(packet.error['v'])) logger.debug('sats:' + str(packet.sats)) logger.debug('track:' + str(packet.track)) logger.debug('time:' + str(packet.time)) # See the inline docs for GpsResponse for the available data self.altitude = 0.0 self.mode = packet.mode if self.mode >= 2: self.sats = packet.sats self.track = packet.track self.longitude = packet.lon self.latitude = packet.lat self.timeutc = packet.time self.hspeed = round(packet.hspeed * 1.60934, 1) self.lonprecision = packet.error['x'] self.latprecision = packet.error['y'] if self.mode >= 3: self.altitude = packet.altitude() self.altprecision = packet.error['v'] # self.vspeed = packet.speed_vertical() except: logger.error('GPS exception')
def scrap(self): logger = com_logger.Logger('RTL') for conf in self.config['URLRTL']: url = self.config['URLRTL'][str(conf)] logger.info('Check URL: ' + url) if requests.get(url).status_code == 200: url = urllib.request.urlopen(url).read() soup = BeautifulSoup(url, "html.parser") soup.prettify() table = [] for div_list in soup.find_all("div", class_ = "timeline-post shift"): try: div_item = div_list.find("div", class_ = "post-fig brd brd-grey") a_link = div_item.find("a", class_ = "post-link")["href"] if requests.get(a_link).status_code == 200: a_link = urllib.request.urlopen(a_link).read() soupnext = BeautifulSoup(a_link, "html.parser") soupnext.prettify() div_mp3 = soupnext.find("figcaption", class_ = "figcaption article-mdl cf") a_name = self.fileutils.replace(div_mp3.find("span", class_ = "legend").text) a_link = div_mp3.find("a", class_ = "dl icon icon-download")["href"] logger.info('Find: ' + a_name) if com_sqlite.select(a_name) != a_name: logger.info('Download: ' + a_name) base_name = os.path.dirname(os.path.abspath(__file__)) db_path = os.path.join(base_name, self.config['DIRDOWNLOAD']['DIR']) urllib.request.urlretrieve(a_link, db_path + "/" + a_name + ".mp3") logger.info('Downloaded !') com_sqlite.insert(a_name) table.append(a_name) mail = com_email.Mail() mail.send_mail_gmail("RTL: " + a_name, table) break except Exception as exp: logger.error(repr(exp))
def send_mail_gmail(subject, table, filename=""): config = com_config.getConfig() logger = com_logger.Logger('Email') logger.log.info('Sending email') msg = MIMEMultipart() body='' for line in table: body += line + "<br>" msg['From'] = config['EMAIL']['from'] msg['To'] = config['EMAIL']['to'] msg['Subject'] = subject msg.attach(MIMEText(body, 'html')) try: if len(filename) > 0: attachment = open("./" + filename, "rb") part = MIMEBase('application', 'octet-stream') part.set_payload((attachment).read()) encoders.encode_base64(part) part.add_header('Content-Disposition', "attachment; filename= %s" % filename) msg.attach(part) logger.log.debug('Try to connect SMTP') server = smtplib.SMTP('smtp.gmail.com', 587) logger.log.debug('Connected to SMTP') server.starttls() logger.log.debug('Try to connect mail box') server.login(config['EMAIL']['from'], config['EMAIL']['password']) text = msg.as_string() logger.log.debug('Connected to mailbox') logger.log.debug('Try to send mail') server.sendmail(config['EMAIL']['from'], config['EMAIL']['to'], text) logger.log.debug('Mail sent') server.quit() except: logger.log.critical('Error sending mail')
def getIFFT(): logger = com_logger.Logger('GetCommand') config = com_config.getConfig() logger.log.info('Getting command') command = '' try: logger.log.debug('Try to connect IMAP') connection = imaplib.IMAP4_SSL('imap.gmail.com') logger.log.debug('Connected to IMAP') logger.log.debug('Try to login mailbox') connection.login(config['EMAIL']['username'], config['EMAIL']['password']) logger.log.debug('Connected to mailbox') rv, data = connection.select('INBOX') if rv == 'OK': logger.log.debug('Check command to mailbox') resp, items = connection.search(None, '(UNSEEN FROM "' + config['CCM'][ 'mailfrom'] + '")') # (FROM "Doug" SUBJECT "test message 2" items = items[0].split() for emailid in items: resp, data = connection.fetch(emailid, "(BODY[HEADER.FIELDS (SUBJECT)])") msg = email.message_from_string(str(data[0][1])) subject = (msg["b'Subject"].replace('\\n', '').replace('\\r', '').replace("'", '')) logger.log.debug(subject) lst = subject.split(':') if len(lst) == 2 and lst[0] == config['CCM']['mailsubject']: logger.log.debug('Command: ' + lst[1]) command = lst[1] else: logger.log.debug('Mail with no command') connection.close() connection.logout() logger.log.debug('Logout IMAP') except socket.gaierror: logger.log.critical('Error get command') finally: return command
Autor: Bruno Date : 09/06/2017 """ import datetime from lib import com_config, com_logger from utils import POI # Config conf = com_config.Config() conf.setconfig() config = conf.getconfig() # Log logger = com_logger.Logger() logger.info('Application start') # POI poi = POI.POI() # Import RadarFile # Start time start = datetime.datetime.now() poi.importpoi() # Stop time end = datetime.datetime.now() logger.info('Import duration: ' + str(end - start)) logger.info('Application stop')
""" Auteur: Bruno DELATTRE Date : 12/08/2016 """ import scapperRTL import scraperBBC import scraperFranceCulture import scraperFranceInter from lib import com_config, com_logger conf = com_config.Config() conf.setconfig() config = conf.getconfig() logger = com_logger.Logger('Main') logger.info(config['APPLICATION']['name'] + ' ' + config['APPLICATION']['version']) logger.info('Start') # Scrap ! INFO: soup.findall(attrs={'class':None or value}) fc = scraperFranceCulture.FranceCulture() fc.scrapxml() fi = scraperFranceInter.FranceInter() fi.scrapxml() bbc = scraperBBC.BBC() bbc.scrap() rtl = scapperRTL.RTL() rtl.scrapxml()
def __init__(self, name='', file=''): self.logger = com_logger.Logger(name, file)
def __init__(self): conf = com_config.Config() self.config = conf.getconfig() self.logger = com_logger.Logger('Mail')
def run(self): logger = com_logger.Logger('Camera Thread') logger.info('Start') self.getpicture() logger.info('Stop')
def __init__(self): self.logger = com_logger.Logger('FILE')
def __init__(self): conf = com_config.Config() self.config = conf.getconfig() self.logger = com_logger.Logger('SQLite') self.database = self.config['SQLITE']['database']
def __init__(self, connection, cursor): # Log self.logger = com_logger.Logger() self.connection = connection self.cursor = cursor
def setacquisition(self, state): if self.led_acquisition > 0: self.gpio.setio(self.led_acquisition, state) logger = com_logger.Logger('LED_ACQUISITION') logger.debug('LED ' + str(state))
def scrap(): logger = com_logger.Logger('Scraper') url_start = "https://www.leboncoin.fr/annonces/offres/haute_normandie/?o=" # https://www.leboncoin.fr/annonces/offres/?th=1 # https: // www.leboncoin.fr/annonces/offres/haute_normandie/?th = 1 & q = vtt url_search = "&q=" url_end = "&it=1" # vide sinon recherche uniquement dans le titre "&it=1" search_list = [] # Get Things to search from config file conf = com_config.Config() config = conf.getconfig() for i in range(1, 50): strthings = 'things' + str(i) try: if len(config['SEARCH'][strthings]) != 0: search_list.append(config['SEARCH'][strthings]) except Exception as exp: logger.info('Things stop at:' + str(exp)) break max_browse = int(config['SEARCH']['max_browse']) logger.info('Start extraction') for search_item in search_list: tab = search_item.split(",") prix_min = int(tab[1]) prix_max = int(tab[2]) index = 1 contenuhtml = [] logger.info('Search: ' + urllib.parse.unquote(tab[0])) urlok = 0 try: urlok = requests.get( url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end).status_code except Exception as exp: logger.error('URL: ' + url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end) logger.error(str(exp)) while urlok == 200 and index <= max_browse: try: url = urllib.request.urlopen( url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end).read() except Exception as exp: logger.error('URL: ' + url_start + str(index) + url_search + urllib.parse.quote(tab[0]) + url_end) logger.error(str(exp)) break soup = BeautifulSoup(url, "html.parser") soup.prettify() if len(soup.find_all("section", class_="tabsContent block-white dontSwitch")) == 0: break for section in soup.find_all("section", class_="tabsContent block-white dontSwitch"): for li in section.find_all("li"): link = li.find("a", class_="list_item clearfix trackable")["href"] idx = \ li.find("a", class_="list_item clearfix trackable")["data-info"].strip().split(",", 7)[2].split( ":", 2)[1].replace('"', "").strip() # container = source.find('div', attrs={'id':'dlbox'}) imglink = '' try: imglink = li.find("span", class_="lazyload")["data-imgsrc"] except Exception as exp: logger.debug('No image: ' + str(exp)) pass for item in li.find_all("section", class_="item_infos"): titre = urllib.parse.unquote( item.find("h2", class_="item_title").text.replace("\n", "").replace("\t", "")) category = urllib.parse.unquote( item.find("p", itemprop="category").text.replace("\n", "").replace("\t", "")) localisation = urllib.parse.unquote( item.find("p", itemprop="availableAtOrFrom").text.replace("\n", "").replace("\t", "")) try: prix = int(item.find("h3", class_="item_price").text.encode("ASCII", 'ignore').strip()) except AttributeError: prix = 0 except ValueError: prix = 0 if (prix >= prix_min) and (prix <= prix_max): if com_sqlite.select(idx) == 0: # Item not yet present in database logger.info('Find: ' + titre.strip()) com_sqlite.insert(idx) contenuhtml = mailcontent(contenuhtml, imglink, link, prix, titre, category, localisation) index += 1 logger.debug('Page : ' + str(index)) if len(contenuhtml) > 0: contenu = Scraper.mailfull(urllib.parse.unquote(tab[0] + " Prix: " + str(prix_min) + "-" + str(prix_max) + ' €'), contenuhtml) com_email.send_mail_gmail(urllib.parse.unquote(tab[0]), contenu) logger.info('Mail sent') logger.info('End extraction')
""" main.py v 1.1.5 Auteur: Bruno DELATTRE Date : 03/12/2016 """ from acquisition import thread_acquisition_camera_timer from lib import com_config, com_gpio_inout, com_logger # Config conf = com_config.Config() conf.setconfig() config = conf.getconfig() # Log logger = com_logger.Logger('TIMELAPSE') logger.info(config['APPLICATION']['name'] + ' ' + config['APPLICATION']['version'] + ' ' + 'Start') # Init gpioinout = com_gpio_inout.GPIOINOT() delayqcquition = 0 tabdelay = [4, 6, 6, 10, 12] timedelay = 3 gpioinout.blink(0.2, 3) # Waiting for Init acquisition logger.info('Wait for start') logger.info('Delay: ' + str(tabdelay[delayqcquition] + timedelay)) while not gpioinout.getstart(): # Check config if gpioinout.getconfigacquisition():
def __init__(self, connection, cursor): self.connection = connection self.cursor = cursor self.logger = com_logger.Logger('DHT22 DAL')