def doWork(self): '''重写WorkRequest类的线程执行函数,此函数将在线程池中执行''' logger.debug('Start storage`s doWork...') if not self.__initDB(): logger.error('Storage thread is stop.') return conn = sqlite3.connect(self.__dbPath) cur = conn.cursor() while True: try: # 从data队列获取数据并插入数据库 if self.__dataQueue.qsize() > 0: data = self.__dataQueue.get() sqlInsert = '''INSERT INTO zspider(url, time, depth) VALUES ('%s', '%s', %d)''' % ( data.url, data.time, data.depth) cur.execute(sqlInsert) conn.commit() else: time.sleep(1) except Exception, e: logger.error('Database operate exception: %s ', str(e)) continue # 检测退出事件 if self.__exitEvent.is_set(): cur.close() conn.close() logger.info('Storage model quit...') return
def record_info(self, car: Car): mileage = car.status.timed_odometer.mileage level = car.status.get_energy('Electric').level level_fuel = car.status.get_energy('Fuel').level charge_date = car.status.get_energy('Electric').updated_at moving = car.status.kinetic.moving longitude = car.status.last_position.geometry.coordinates[0] latitude = car.status.last_position.geometry.coordinates[1] altitude = car.status.last_position.geometry.coordinates[2] date = car.status.last_position.properties.updated_at if date is None: date = charge_date logger.debug("vin:%s longitude:%s latitude:%s date:%s mileage:%s level:%s charge_date:%s level_fuel:" "%s moving:%s", car.vin, longitude, latitude, date, mileage, level, charge_date, level_fuel, moving) Database.record_position(self.weather_api, car.vin, mileage, latitude, longitude, altitude, date, level, level_fuel, moving) self.abrp.call(car, Database.get_last_temp(car.vin)) try: charging_status = car.status.get_energy('Electric').charging.status charging_mode = car.status.get_energy('Electric').charging.charging_mode Charging.record_charging(car, charging_status, charge_date, level, latitude, longitude, self.country_code, charging_mode) logger.debug("charging_status:%s ", charging_status) except AttributeError: logger.error("charging status not available from api")
def load_otp(self, force_new=False): otp_session = load_otp() if otp_session is None or force_new: logger.error("Please redo otp config") return False self.otp = otp_session return True
def send_message_by_nick(self, nick, word): user_openid = self.get_openid(nick) if user_openid: if not self.send_message(word, user_openid): logger.error("send message to %s failed.", nick) else: logger.error('[*] 此用户不存在')
def doWork(self): '''重写WorkRequest类的线程执行函数,此函数将在线程池中执行''' logger.debug('Start storage`s doWork...') if not self.__initDB(): logger.error('Storage thread is stop.') return conn = sqlite3.connect(self.__dbPath) cur = conn.cursor() while True: try: # 从data队列获取数据并插入数据库 if self.__dataQueue.qsize() > 0: data = self.__dataQueue.get() sqlInsert = '''INSERT INTO zspider(url, time, depth) VALUES ('%s', '%s', %d)''' % (data.url, data.time, data.depth) cur.execute(sqlInsert) conn.commit() else: time.sleep(1) except Exception, e: logger.error('Database operate exception: %s ', str(e)) continue # 检测退出事件 if self.__exitEvent.is_set(): cur.close() conn.close() logger.info('Storage model quit...') return
def refresh_token(self): try: # pylint: disable=protected-access self.manager._refresh_token() self.save_config() except RequestException as e: logger.error("Can't refresh token %s", e) sleep(60)
def refresh_token_now(self): try: self._refresh_token() for refresh_callback in self.refresh_callbacks: refresh_callback() return True except RequestException as e: logger.error("Can't refresh token %s", e) return False
def set_chargings_price(conn, start_at, price): if isinstance(start_at, str): start_at = Database.convert_datetime_from_string(start_at) update = conn.execute("UPDATE battery SET price=? WHERE start_at=?", (price, start_at)).rowcount == 1 conn.commit() if not update: logger.error("Can't find line to update in the database") return update
def calc_power(self): try: item = findlatest(self.items) logger.info('==item_latest=={}'.format(item)) e = EnergyItem(item) e.calc_item() self.power = e.power except Exception as e: logger.error(str(e)) raise PowerCalculationException(str(e))
def get_power(mac): power_watt = 0 try: raw = _get_json(mac) power_watt = _cook_json_forpower(raw) power_watt = round(power_watt, 3) except Exception as e: logger.error(str(e)) return -1 return power_watt
def capture_diffs_in_battery_table(timestamp, data, data_previous): # pylint: disable=unused-variable if timestamp is None: raise PreventUpdate diff_data = diff_dashtable(data, data_previous, "start_at") for changed_line in diff_data: if changed_line['column_name'] == 'price': conn = Database.get_db() if not Database.set_chargings_price( conn, changed_line['start_at'], changed_line['current_value']): logger.error("Can't find line to update in the database") conn.close() return ""
def get_country(latitude, longitude, country_code_default): try: location = reverse_geocode.search([(latitude, longitude)])[0] country_code = location["country_code"] return country_code except (UnicodeDecodeError, IndexError): logger.error("Can't find country for %s %s", latitude, longitude) # return None country_code = country_code_default logger.warning( "Using country of origin : %s (wrong co2 when traveling abroad)", country_code) return country_code
def get_price(self, start, end, consumption): prices = [] date = start res = None if not (start is None or end is None): while date < end: prices.append(self.get_instant_price(date)) date = date + timedelta(minutes=30) try: res = round(consumption * mean(prices), 2) except (TypeError, StatisticsError): logger.error("Can't get_price of charge, check config") return res
def __isLegalDomain(self, url): '''判断url是否为站内链接''' try: host = urlparse.urlparse(url).hostname if host == None: return False domain = host.split('.')[::-1] for index in range(len(self.__legalDomain)): if not domain[index] == self.__legalDomain[index]: return False except Exception,e: logger.error('isLegalDomain() exception: %s', str(e)) return False
def __ignoreSuffix(self, linkList): '''过滤url后缀,如rar,zip,jpg等文件''' tmpList = [] for url in linkList: try: path = urlparse.urlparse(url)[2] suffix = path.split('.')[-1] suffix = suffix.lower() if suffix not in SUFFIX_LIST: tmpList.append(url) except Exception,e: logger.error('ignoreSuffix() exception : %s, URL : %s', str(e), url) continue
def get_remote_access_token(self, password): try: res = self.manager.post(REMOTE_URL + self.client_id, json={"grant_type": "password", "password": password}, headers=self.headers) data = res.json() self.remote_access_token = data["access_token"] self.remote_refresh_token = data["refresh_token"] return res except RequestException as e: logger.error("Can't refresh remote token %s", e) sleep(60) return None
def remove_file(response): torch.cuda.empty_cache() try: os.remove(whole_path_wav) except Exception as error: logger.error(f"Error removing file {whole_path_wav}") if ext != 'wav': try: os.remove(whole_path_ext) except Exception as error: logger.error(f"Error removing file {whole_path_ext}") return response
def _get(self, url, api=None): request = urllib2.Request(url=url) request.add_header('Referer', 'https://wx.qq.com/') if api == 'webwxgetvoice': request.add_header('Range', 'bytes=0-') if api == 'webwxgetvideo': request.add_header('Range', 'bytes=0-') try: response = urllib2.urlopen(request) data = response.read() # logging.debug(url) return data except urllib2.HTTPError, e: logger.error('HTTPError = ' + str(e.code))
def start_remote_control(self): if self.args.remote_disable: logger.info("mqtt disabled") elif not self.args.web_conf or path.isfile(OTP_CONFIG_NAME): if self.myp.remote_client.mqtt_client is not None: self.myp.remote_client.mqtt_client.disconnect() try: self.myp.remote_client.start() if self.args.charge_control: Config.chc = ChargeControls.load_config( self.myp, name=self.args.charge_control) Config.chc.init() self.myp.start_refresh_thread() except ConfigException: logger.error("start_remote_control failed redo otp config")
def get_vehicle_info(self, vin): res = None car = self.vehicles_list.get_car_by_vin(vin) for _ in range(0, 2): try: res = self.api().get_vehicle_status(car.vehicle_id, extension=["odometer"]) if res is not None: car.status = res if self._record_enabled: self.record_info(car) return res except (ApiException, InvalidHeader) as ex: logger.error("get_vehicle_info: ApiException: %s", ex) logger.debug(exc_info=True) car.status = res return res
def _post(self, url, params, jsonfmt=True): # logger.debug("%s,%s,cookie:%s", url, params, self.cookie) if jsonfmt: request = urllib2.Request(url=url, data=json.dumps(params)) request.add_header('ContentType', 'application/json; charset=UTF-8') else: request = urllib2.Request(url=url, data=urllib.urlencode(params)) try: response = urllib2.urlopen(request) data = response.read() if jsonfmt: return json.loads(data, object_hook=_decode_dict) return data except urllib2.HTTPError, e: logger.error('HTTPError = ' + str(e.code))
def get_temp(latitude: str, longitude: str, api_key: str) -> float: try: if not (latitude is None or longitude is None or api_key is None): weather_rep = requests.get("https://api.openweathermap.org/data/2.5/onecall", params={"lat": latitude, "lon": longitude, "exclude": "minutely,hourly,daily,alerts", "appid": api_key, "units": "metric"}) temp = weather_rep.json()["current"]["temp"] logger.debug("Temperature :%fc", temp) return temp except ConnectionError: logger.error("Can't connect to openweathermap :", exc_info=True) except KeyError: logger.error("Unable to get temperature from openweathermap :", exc_info=True) return None
def update_trips(): global trips, chargings, cached_layout, min_date, max_date, min_millis, max_millis, step, marks logger.info("update_data") conn = Database.get_db(update_callback=False) Database.add_altitude_to_db(conn) conn.close() min_date = None max_date = None if CONFIG.is_good: car = CONFIG.myp.vehicles_list[0] # todo handle multiple car try: trips_by_vin = Trips.get_trips(Cars([car])) trips = trips_by_vin[car.vin] assert len(trips) > 0 min_date = trips[0].start_at max_date = trips[-1].start_at figures.get_figures(trips[0].car) except (AssertionError, KeyError): logger.debug("No trips yet") figures.get_figures(Car("vin", "vid", "brand")) try: chargings = Charging.get_chargings() assert len(chargings) > 0 if min_date: min_date = min(min_date, chargings[0]["start_at"]) max_date = max(max_date, chargings[-1]["start_at"]) else: min_date = chargings[0]["start_at"] max_date = chargings[-1]["start_at"] except AssertionError: logger.debug("No chargings yet") if min_date is None: return # update for slider try: logger.debug("min_date:%s - max_date:%s", min_date, max_date) min_millis = web.utils.unix_time_millis(min_date) max_millis = web.utils.unix_time_millis(max_date) step = (max_millis - min_millis) / 100 marks = web.utils.get_marks_from_start_end(min_date, max_date) cached_layout = None # force regenerate layout figures.get_figures(car) except (ValueError, IndexError): logger.error("update_trips (slider): %s", exc_info=True) except AttributeError: logger.debug("position table is probably empty :", exc_info=True) return
def start(self): if self.load_otp(): self.mqtt_client = mqtt.Client(clean_session=True, protocol=mqtt.MQTTv311) if environ.get("MQTT_LOG", "0") == "1": self.mqtt_client.enable_logger(logger=logger) if self._refresh_remote_token(): self.mqtt_client.tls_set_context() self.mqtt_client.on_connect = self.__on_mqtt_connect self.mqtt_client.on_message = self.__on_mqtt_message self.mqtt_client.on_disconnect = self._on_mqtt_disconnect self.mqtt_client.connect(MQTT_SERVER, 8885, 60) self.mqtt_client.loop_start() self.__keep_mqtt() return self.mqtt_client.is_connected() logger.error("Can't configure MQTT Client") return False
def query(): if request.method == 'GET': return render_template('iotapi.html') iot_username = request.form.get('iot_username') iot_password = request.form.get('iot_password') email_input = request.form.get('email_input') try: email_input = email_input.strip() except Exception as e: logger.error('email_input: ' + str(e)) email_input = request.form.get('email_input') logger.info('==iot_username=={}'.format(iot_username)) logger.info('==email_input=={}'.format(email_input)) errno = 0 errmsg = None account_info = {} subscribe_list = [] extattr_list = [] try: account_info, subscribe_list, extattr_list = get_from_iotapi( iot_username, iot_password, email_input) except Exception as e: errno = 2 errmsg = str(e) # logger.error('==errno=={}'.format(errno)) logger.error('==errmsg=={}'.format(errmsg)) logger.error('==account_info=={}'.format(account_info)) logger.error('==subscribe_list=={}'.format(subscribe_list)) logger.error('==extattr_list=={}'.format(extattr_list)) params = { "iot_username": iot_username, "iot_password": iot_password, "email_pre": email_input, "query": True, "errno": errno, "errmsg": errmsg, "account_info": account_info, "subscribe_list": subscribe_list, "extattr_list": extattr_list, } return render_template('iotapi.html', **params)
def __initDB(self): '''初始化数据库文件路径,并创建数据库''' try: dbDir = os.getcwd() + '/db/' if not os.path.exists(dbDir): os.makedirs(dbDir) self.__dbPath = dbDir + self.__dbName conn = sqlite3.connect(self.__dbPath) sqlCreateTable = '''CREATE TABLE IF NOT EXISTS zspider( id integer primary key, url text, html text, time text, depth integer)''' conn.execute(sqlCreateTable) conn.close() logger.debug('Create database success.') return True except Exception, e: logger.error('Init database error : %s', str(e)) return False
def synccheck(self): url = 'https://' + self.syncHost + '/cgi-bin/mmwebwx-bin/synccheck?' + urllib.urlencode({'r': int(time.time()), 'sid': self.sid, 'uin': self.uin, 'skey': self.skey, 'deviceid': self.deviceId, 'synckey': self.synckey, '_': int(time.time()), }) data = self._get(url) logger.debug("sync check: <%s>", data) if data == '': return [-1, -1] pm = re.search(r"window.synccheck={retcode:\"(\d+)\",selector:\"(\d+)\"}", data) if not pm: logger.error("invalid sync check response:%s", data) return pm.groups()
def capture_diffs_in_battery_table(timestamp, data, data_previous): if timestamp is None: raise PreventUpdate diff_data = diff_dashtable(data, data_previous, "start_at") for changed_line in diff_data: if changed_line['column_name'] == 'price': conn = Database.get_db() if not Database.set_chargings_price( conn, dash_date_to_datetime(changed_line['start_at']), changed_line['current_value']): logger.error( "Can't find line to update in the database") else: logger.debug("update price %s of %s", changed_line['current_value'], changed_line['start_at']) conn.close() return "" # don't need to update dashboard
def add_altitude_to_db(conn): max_pos_by_req = 100 nb_null = conn.execute( "SELECT COUNT(1) FROM position WHERE altitude IS NULL " "and longitude IS NOT NULL AND latitude IS NOT NULL;").fetchone( )[0] if nb_null > max_pos_by_req: logger.warning( "There is %s to fetch from API, it can take some time", nb_null) try: while True: res = conn.execute( "SELECT DISTINCT latitude,longitude FROM position WHERE altitude IS NULL " "and longitude IS NOT NULL AND latitude IS NOT NULL LIMIT ?;", (max_pos_by_req, )).fetchall() nb_res = len(res) if nb_res > 0: logger.debug("add altitude for %s positions point", nb_null) nb_null -= nb_res locations_str = "" for line in res: locations_str += str(line[0]) + "," + str( line[1]) + "|" locations_str = locations_str[:-1] res = requests.get( "https://api.opentopodata.org/v1/srtm30m", params={"locations": locations_str}) data = res.json()["results"] for line in data: conn.execute( "UPDATE position SET altitude=? WHERE latitude=? and longitude=?", (line["elevation"], line["location"]["lat"], line["location"]["lng"])) conn.commit() if nb_res == 100: sleep(1) # API is limited to 1 call by sec else: break except (ValueError, KeyError, requests.exceptions.RequestException): logger.error("Can't get altitude from API")
def load_app(self) -> bool: my_logger(handler_level=int(self.args.debug)) if self.args.config: self.config_name = self.args.config if path.isfile(self.config_name): self.myp = MyPSACC.load_config(name=self.config_name) elif self.args.web_conf: self.is_good = False return False else: raise FileNotFoundError(self.config_name) atexit.register(self.save_config) self.myp.set_record(self.args.record) Charging.elec_price = ElecPrice.read_config() if self.args.offline: logger.info("offline mode") self.is_good = True else: self.is_good = False try: self.is_good = self.myp.manager.refresh_token_now() if self.is_good: logger.info(str(self.myp.get_vehicles())) except OAuthError: if self.args.mail and self.args.password: self.myp.connect(self.args.mail, self.args.password) logger.info(str(self.myp.get_vehicles())) self.is_good = True else: self.is_good = False logger.error( "Please reconnect by going to config web page") if self.args.refresh: self.myp.info_refresh_rate = self.args.refresh * 60 if self.is_good: self.myp.start_refresh_thread() if self.is_good: self.start_remote_control() elif not self.args.web_conf: raise ConnectionError self.save_config() return True
def __on_mqtt_message(self, client, userdata, msg): try: logger.info("mqtt msg received: %s %s", msg.topic, msg.payload) logger.debug("client: %s userdata: %s", client, userdata) data = json.loads(msg.payload) charge_info = None if msg.topic.startswith(MQTT_RESP_TOPIC): if "return_code" not in data: logger.debug("mqtt msg hasn't return code") elif data["return_code"] == "400": self._refresh_remote_token(force=True) if self.last_request: logger.warning( "last request is send again, token was expired") last_request = self.last_request self.last_request = None self.publish(last_request, store=False) else: logger.error( "Last request might have been send twice without success" ) elif data["return_code"] != "0": logger.error('%s : %s', data["return_code"], data.get("reason", "?")) elif msg.topic.startswith(MQTT_EVENT_TOPIC): charge_info = data["charging_state"] self.precond_programs[ data["vin"]] = data["precond_state"]["programs"] if charge_info is not None and charge_info['remaining_time'] != 0: try: car = self.vehicles_list.get_car_by_vin( vin=msg.topic.split("/")[-1]) if car and car.status.get_energy( 'Electric').charging.status != INPROGRESS: # fix a psa server bug where charge beginning without status api being properly updated logger.warning("charge begin but API isn't updated") sleep(60) self.wakeup(data["vin"]) except (IndexError, AttributeError, RateLimitException): logger.exception("on_mqtt_message:") except KeyError: logger.exception("on_mqtt_message:")
def update_trips(): global trips, chargings, cached_layout, min_date, max_date, min_millis, max_millis, step, marks logger.info("update_data") conn = Database.get_db(update_callback=False) Database.add_altitude_to_db(conn) conn.close() min_date = None max_date = None try: trips_by_vin = Trips.get_trips(myp.vehicles_list) trips = next(iter(trips_by_vin.values())) # todo handle multiple car assert len(trips) > 0 min_date = trips[0].start_at max_date = trips[-1].start_at except (StopIteration, AssertionError): logger.debug("No trips yet") try: chargings = Charging.get_chargings() assert len(chargings) > 0 if min_date: min_date = min(min_date, chargings[0]["start_at"]) max_date = max(max_date, chargings[-1]["start_at"]) else: min_date = chargings[0]["start_at"] max_date = chargings[-1]["start_at"] except AssertionError: logger.debug("No chargings yet") if min_date is None: return # update for slider try: logger.debug("min_date:%s - max_date:%s", min_date, max_date) min_millis = figures.unix_time_millis(min_date) max_millis = figures.unix_time_millis(max_date) step = (max_millis - min_millis) / 100 marks = figures.get_marks_from_start_end(min_date, max_date) cached_layout = None # force regenerate layout except (ValueError, IndexError): logger.error("update_trips (slider): %s", exc_info=True) except AttributeError: logger.debug("position table is probably empty :", exc_info=True) return
def _refresh_remote_token(self, force=False): bad_remote_token = self.remoteCredentials.refresh_token is None if not force and not bad_remote_token and self.remoteCredentials.last_update: last_update: datetime = self.remoteCredentials.last_update if (datetime.now() - last_update).total_seconds() < MQTT_TOKEN_TTL: return True try: self.manager.refresh_token_now() if bad_remote_token: logger.error("remote_refresh_token isn't defined") else: res = self.manager.post( REMOTE_URL + self.account_info.client_id, json={ "grant_type": "refresh_token", "refresh_token": self.remoteCredentials.refresh_token }, headers=self.headers) data = res.json() logger.debug("refresh_remote_token: %s", data) if "access_token" in data: self.remoteCredentials.access_token = data["access_token"] self.remoteCredentials.refresh_token = data[ "refresh_token"] bad_remote_token = False else: logger.error( "can't refresh_remote_token: %s\n Create a new one", data) bad_remote_token = True if bad_remote_token: otp_code = self.get_otp_code() res = self.get_remote_access_token(otp_code) self.remote_token_last_update = datetime.now() self.mqtt_client.username_pw_set( "IMA_OAUTH_ACCESS_TOKEN", self.remoteCredentials.access_token) return True except (RequestException, RateLimitException) as e: logger.exception("Can't refresh remote token %s", e) sleep(60) return False