def doWork(self):
'''重写WorkRequest类的线程执行函数，此函数将在线程池中执行'''
logger.debug('Start storage`s doWork...')
if not self.__initDB():
logger.error('Storage thread is stop.')
return
conn = sqlite3.connect(self.__dbPath)
cur = conn.cursor()
while True:
try:
# 从data队列获取数据并插入数据库
if self.__dataQueue.qsize() > 0:
data = self.__dataQueue.get()
sqlInsert = '''INSERT INTO zspider(url, time, depth) VALUES ('%s', '%s', %d)''' % (
data.url, data.time, data.depth)
cur.execute(sqlInsert)
conn.commit()
else:
time.sleep(1)
except Exception, e:
logger.error('Database operate exception: %s ', str(e))
continue
# 检测退出事件
if self.__exitEvent.is_set():
cur.close()
conn.close()
logger.info('Storage model quit...')
return
def record_info(self, car: Car):
mileage = car.status.timed_odometer.mileage
level = car.status.get_energy('Electric').level
level_fuel = car.status.get_energy('Fuel').level
charge_date = car.status.get_energy('Electric').updated_at
moving = car.status.kinetic.moving
longitude = car.status.last_position.geometry.coordinates[0]
latitude = car.status.last_position.geometry.coordinates[1]
altitude = car.status.last_position.geometry.coordinates[2]
date = car.status.last_position.properties.updated_at
if date is None:
date = charge_date
logger.debug("vin:%s longitude:%s latitude:%s date:%s mileage:%s level:%s charge_date:%s level_fuel:"
"%s moving:%s", car.vin, longitude, latitude, date, mileage, level, charge_date, level_fuel,
moving)
Database.record_position(self.weather_api, car.vin, mileage, latitude, longitude, altitude, date, level,
level_fuel, moving)
self.abrp.call(car, Database.get_last_temp(car.vin))
try:
charging_status = car.status.get_energy('Electric').charging.status
charging_mode = car.status.get_energy('Electric').charging.charging_mode
Charging.record_charging(car, charging_status, charge_date, level, latitude, longitude, self.country_code,
charging_mode)
logger.debug("charging_status:%s ", charging_status)
except AttributeError:
logger.error("charging status not available from api")
def load_otp(self, force_new=False):
otp_session = load_otp()
if otp_session is None or force_new:
logger.error("Please redo otp config")
return False
self.otp = otp_session
return True
def send_message_by_nick(self, nick, word):
user_openid = self.get_openid(nick)
if user_openid:
if not self.send_message(word, user_openid):
logger.error("send message to %s failed.", nick)
else:
logger.error('[*] 此用户不存在')
def doWork(self):
'''重写WorkRequest类的线程执行函数，此函数将在线程池中执行'''
logger.debug('Start storage`s doWork...')
if not self.__initDB():
logger.error('Storage thread is stop.')
return
conn = sqlite3.connect(self.__dbPath)
cur = conn.cursor()
while True:
try:
# 从data队列获取数据并插入数据库
if self.__dataQueue.qsize() > 0:
data = self.__dataQueue.get()
sqlInsert = '''INSERT INTO zspider(url, time, depth) VALUES ('%s', '%s', %d)''' % (data.url, data.time, data.depth)
cur.execute(sqlInsert)
conn.commit()
else:
time.sleep(1)
except Exception, e:
logger.error('Database operate exception: %s ', str(e))
continue
# 检测退出事件
if self.__exitEvent.is_set():
cur.close()
conn.close()
logger.info('Storage model quit...')
return
def refresh_token(self):
try:
# pylint: disable=protected-access
self.manager._refresh_token()
self.save_config()
except RequestException as e:
logger.error("Can't refresh token %s", e)
sleep(60)
def refresh_token_now(self):
try:
self._refresh_token()
for refresh_callback in self.refresh_callbacks:
refresh_callback()
return True
except RequestException as e:
logger.error("Can't refresh token %s", e)
return False
def set_chargings_price(conn, start_at, price):
if isinstance(start_at, str):
start_at = Database.convert_datetime_from_string(start_at)
update = conn.execute("UPDATE battery SET price=? WHERE start_at=?",
(price, start_at)).rowcount == 1
conn.commit()
if not update:
logger.error("Can't find line to update in the database")
return update
def calc_power(self):
try:
item = findlatest(self.items)
logger.info('==item_latest=={}'.format(item))
e = EnergyItem(item)
e.calc_item()
self.power = e.power
except Exception as e:
logger.error(str(e))
raise PowerCalculationException(str(e))
def get_power(mac):
power_watt = 0
try:
raw = _get_json(mac)
power_watt = _cook_json_forpower(raw)
power_watt = round(power_watt, 3)
except Exception as e:
logger.error(str(e))
return -1
return power_watt
def capture_diffs_in_battery_table(timestamp, data, data_previous): # pylint: disable=unused-variable
if timestamp is None:
raise PreventUpdate
diff_data = diff_dashtable(data, data_previous, "start_at")
for changed_line in diff_data:
if changed_line['column_name'] == 'price':
conn = Database.get_db()
if not Database.set_chargings_price( conn, changed_line['start_at'],
changed_line['current_value']):
logger.error("Can't find line to update in the database")
conn.close()
return ""
def get_country(latitude, longitude, country_code_default):
try:
location = reverse_geocode.search([(latitude, longitude)])[0]
country_code = location["country_code"]
return country_code
except (UnicodeDecodeError, IndexError):
logger.error("Can't find country for %s %s", latitude, longitude)
# return None
country_code = country_code_default
logger.warning(
"Using country of origin : %s (wrong co2 when traveling abroad)",
country_code)
return country_code
def get_price(self, start, end, consumption):
prices = []
date = start
res = None
if not (start is None or end is None):
while date < end:
prices.append(self.get_instant_price(date))
date = date + timedelta(minutes=30)
try:
res = round(consumption * mean(prices), 2)
except (TypeError, StatisticsError):
logger.error("Can't get_price of charge, check config")
return res
def __isLegalDomain(self, url):
'''判断url是否为站内链接'''
try:
host = urlparse.urlparse(url).hostname
if host == None:
return False
domain = host.split('.')[::-1]
for index in range(len(self.__legalDomain)):
if not domain[index] == self.__legalDomain[index]:
return False
except Exception,e:
logger.error('isLegalDomain() exception: %s', str(e))
return False
def __ignoreSuffix(self, linkList):
'''过滤url后缀，如rar，zip，jpg等文件'''
tmpList = []
for url in linkList:
try:
path = urlparse.urlparse(url)[2]
suffix = path.split('.')[-1]
suffix = suffix.lower()
if suffix not in SUFFIX_LIST:
tmpList.append(url)
except Exception,e:
logger.error('ignoreSuffix() exception : %s, URL : %s', str(e), url)
continue
def get_remote_access_token(self, password):
try:
res = self.manager.post(REMOTE_URL + self.client_id,
json={"grant_type": "password", "password": password},
headers=self.headers)
data = res.json()
self.remote_access_token = data["access_token"]
self.remote_refresh_token = data["refresh_token"]
return res
except RequestException as e:
logger.error("Can't refresh remote token %s", e)
sleep(60)
return None
def remove_file(response):
torch.cuda.empty_cache()
try:
os.remove(whole_path_wav)
except Exception as error:
logger.error(f"Error removing file {whole_path_wav}")
if ext != 'wav':
try:
os.remove(whole_path_ext)
except Exception as error:
logger.error(f"Error removing file {whole_path_ext}")
return response
def _get(self, url, api=None):
request = urllib2.Request(url=url)
request.add_header('Referer', 'https://wx.qq.com/')
if api == 'webwxgetvoice':
request.add_header('Range', 'bytes=0-')
if api == 'webwxgetvideo':
request.add_header('Range', 'bytes=0-')
try:
response = urllib2.urlopen(request)
data = response.read()
# logging.debug(url)
return data
except urllib2.HTTPError, e:
logger.error('HTTPError = ' + str(e.code))
def start_remote_control(self):
if self.args.remote_disable:
logger.info("mqtt disabled")
elif not self.args.web_conf or path.isfile(OTP_CONFIG_NAME):
if self.myp.remote_client.mqtt_client is not None:
self.myp.remote_client.mqtt_client.disconnect()
try:
self.myp.remote_client.start()
if self.args.charge_control:
Config.chc = ChargeControls.load_config(
self.myp, name=self.args.charge_control)
Config.chc.init()
self.myp.start_refresh_thread()
except ConfigException:
logger.error("start_remote_control failed redo otp config")
def get_vehicle_info(self, vin):
res = None
car = self.vehicles_list.get_car_by_vin(vin)
for _ in range(0, 2):
try:
res = self.api().get_vehicle_status(car.vehicle_id, extension=["odometer"])
if res is not None:
car.status = res
if self._record_enabled:
self.record_info(car)
return res
except (ApiException, InvalidHeader) as ex:
logger.error("get_vehicle_info: ApiException: %s", ex)
logger.debug(exc_info=True)
car.status = res
return res
def _post(self, url, params, jsonfmt=True):
# logger.debug("%s,%s,cookie:%s", url, params, self.cookie)
if jsonfmt:
request = urllib2.Request(url=url, data=json.dumps(params))
request.add_header('ContentType', 'application/json; charset=UTF-8')
else:
request = urllib2.Request(url=url, data=urllib.urlencode(params))
try:
response = urllib2.urlopen(request)
data = response.read()
if jsonfmt:
return json.loads(data, object_hook=_decode_dict)
return data
except urllib2.HTTPError, e:
logger.error('HTTPError = ' + str(e.code))
def get_temp(latitude: str, longitude: str, api_key: str) -> float:
try:
if not (latitude is None or longitude is None or api_key is None):
weather_rep = requests.get("https://api.openweathermap.org/data/2.5/onecall",
params={"lat": latitude, "lon": longitude,
"exclude": "minutely,hourly,daily,alerts",
"appid": api_key,
"units": "metric"})
temp = weather_rep.json()["current"]["temp"]
logger.debug("Temperature :%fc", temp)
return temp
except ConnectionError:
logger.error("Can't connect to openweathermap :", exc_info=True)
except KeyError:
logger.error("Unable to get temperature from openweathermap :", exc_info=True)
return None
def update_trips():
global trips, chargings, cached_layout, min_date, max_date, min_millis, max_millis, step, marks
logger.info("update_data")
conn = Database.get_db(update_callback=False)
Database.add_altitude_to_db(conn)
conn.close()
min_date = None
max_date = None
if CONFIG.is_good:
car = CONFIG.myp.vehicles_list[0] # todo handle multiple car
try:
trips_by_vin = Trips.get_trips(Cars([car]))
trips = trips_by_vin[car.vin]
assert len(trips) > 0
min_date = trips[0].start_at
max_date = trips[-1].start_at
figures.get_figures(trips[0].car)
except (AssertionError, KeyError):
logger.debug("No trips yet")
figures.get_figures(Car("vin", "vid", "brand"))
try:
chargings = Charging.get_chargings()
assert len(chargings) > 0
if min_date:
min_date = min(min_date, chargings[0]["start_at"])
max_date = max(max_date, chargings[-1]["start_at"])
else:
min_date = chargings[0]["start_at"]
max_date = chargings[-1]["start_at"]
except AssertionError:
logger.debug("No chargings yet")
if min_date is None:
return
# update for slider
try:
logger.debug("min_date:%s - max_date:%s", min_date, max_date)
min_millis = web.utils.unix_time_millis(min_date)
max_millis = web.utils.unix_time_millis(max_date)
step = (max_millis - min_millis) / 100
marks = web.utils.get_marks_from_start_end(min_date, max_date)
cached_layout = None # force regenerate layout
figures.get_figures(car)
except (ValueError, IndexError):
logger.error("update_trips (slider): %s", exc_info=True)
except AttributeError:
logger.debug("position table is probably empty :", exc_info=True)
return
def start(self):
if self.load_otp():
self.mqtt_client = mqtt.Client(clean_session=True,
protocol=mqtt.MQTTv311)
if environ.get("MQTT_LOG", "0") == "1":
self.mqtt_client.enable_logger(logger=logger)
if self._refresh_remote_token():
self.mqtt_client.tls_set_context()
self.mqtt_client.on_connect = self.__on_mqtt_connect
self.mqtt_client.on_message = self.__on_mqtt_message
self.mqtt_client.on_disconnect = self._on_mqtt_disconnect
self.mqtt_client.connect(MQTT_SERVER, 8885, 60)
self.mqtt_client.loop_start()
self.__keep_mqtt()
return self.mqtt_client.is_connected()
logger.error("Can't configure MQTT Client")
return False
def query():
if request.method == 'GET':
return render_template('iotapi.html')
iot_username = request.form.get('iot_username')
iot_password = request.form.get('iot_password')
email_input = request.form.get('email_input')
try:
email_input = email_input.strip()
except Exception as e:
logger.error('email_input: ' + str(e))
email_input = request.form.get('email_input')
logger.info('==iot_username=={}'.format(iot_username))
logger.info('==email_input=={}'.format(email_input))
errno = 0
errmsg = None
account_info = {}
subscribe_list = []
extattr_list = []
try:
account_info, subscribe_list, extattr_list = get_from_iotapi(
iot_username, iot_password, email_input)
except Exception as e:
errno = 2
errmsg = str(e)
# logger.error('==errno=={}'.format(errno))
logger.error('==errmsg=={}'.format(errmsg))
logger.error('==account_info=={}'.format(account_info))
logger.error('==subscribe_list=={}'.format(subscribe_list))
logger.error('==extattr_list=={}'.format(extattr_list))
params = {
"iot_username": iot_username,
"iot_password": iot_password,
"email_pre": email_input,
"query": True,
"errno": errno,
"errmsg": errmsg,
"account_info": account_info,
"subscribe_list": subscribe_list,
"extattr_list": extattr_list,
}
return render_template('iotapi.html', **params)
def __initDB(self):
'''初始化数据库文件路径，并创建数据库'''
try:
dbDir = os.getcwd() + '/db/'
if not os.path.exists(dbDir):
os.makedirs(dbDir)
self.__dbPath = dbDir + self.__dbName
conn = sqlite3.connect(self.__dbPath)
sqlCreateTable = '''CREATE TABLE IF NOT EXISTS zspider(
id integer primary key, url text, html text, time text, depth integer)'''
conn.execute(sqlCreateTable)
conn.close()
logger.debug('Create database success.')
return True
except Exception, e:
logger.error('Init database error : %s', str(e))
return False
def __initDB(self):
'''初始化数据库文件路径，并创建数据库'''
try:
dbDir = os.getcwd() + '/db/'
if not os.path.exists(dbDir):
os.makedirs(dbDir)
self.__dbPath = dbDir + self.__dbName
conn = sqlite3.connect(self.__dbPath)
sqlCreateTable = '''CREATE TABLE IF NOT EXISTS zspider(
id integer primary key, url text, html text, time text, depth integer)'''
conn.execute(sqlCreateTable)
conn.close()
logger.debug('Create database success.')
return True
except Exception, e:
logger.error('Init database error : %s', str(e))
return False
def synccheck(self):
url = 'https://' + self.syncHost + '/cgi-bin/mmwebwx-bin/synccheck?' + urllib.urlencode({'r': int(time.time()),
'sid': self.sid,
'uin': self.uin,
'skey': self.skey,
'deviceid': self.deviceId,
'synckey': self.synckey,
'_': int(time.time()),
})
data = self._get(url)
logger.debug("sync check: <%s>", data)
if data == '':
return [-1, -1]
pm = re.search(r"window.synccheck={retcode:\"(\d+)\",selector:\"(\d+)\"}", data)
if not pm:
logger.error("invalid sync check response:%s", data)
return pm.groups()
def capture_diffs_in_battery_table(timestamp, data, data_previous):
if timestamp is None:
raise PreventUpdate
diff_data = diff_dashtable(data, data_previous, "start_at")
for changed_line in diff_data:
if changed_line['column_name'] == 'price':
conn = Database.get_db()
if not Database.set_chargings_price(
conn,
dash_date_to_datetime(changed_line['start_at']),
changed_line['current_value']):
logger.error(
"Can't find line to update in the database")
else:
logger.debug("update price %s of %s",
changed_line['current_value'],
changed_line['start_at'])
conn.close()
return "" # don't need to update dashboard
def add_altitude_to_db(conn):
max_pos_by_req = 100
nb_null = conn.execute(
"SELECT COUNT(1) FROM position WHERE altitude IS NULL "
"and longitude IS NOT NULL AND latitude IS NOT NULL;").fetchone(
)[0]
if nb_null > max_pos_by_req:
logger.warning(
"There is %s to fetch from API, it can take some time",
nb_null)
try:
while True:
res = conn.execute(
"SELECT DISTINCT latitude,longitude FROM position WHERE altitude IS NULL "
"and longitude IS NOT NULL AND latitude IS NOT NULL LIMIT ?;",
(max_pos_by_req, )).fetchall()
nb_res = len(res)
if nb_res > 0:
logger.debug("add altitude for %s positions point",
nb_null)
nb_null -= nb_res
locations_str = ""
for line in res:
locations_str += str(line[0]) + "," + str(
line[1]) + "|"
locations_str = locations_str[:-1]
res = requests.get(
"https://api.opentopodata.org/v1/srtm30m",
params={"locations": locations_str})
data = res.json()["results"]
for line in data:
conn.execute(
"UPDATE position SET altitude=? WHERE latitude=? and longitude=?",
(line["elevation"], line["location"]["lat"],
line["location"]["lng"]))
conn.commit()
if nb_res == 100:
sleep(1) # API is limited to 1 call by sec
else:
break
except (ValueError, KeyError, requests.exceptions.RequestException):
logger.error("Can't get altitude from API")
def load_app(self) -> bool:
my_logger(handler_level=int(self.args.debug))
if self.args.config:
self.config_name = self.args.config
if path.isfile(self.config_name):
self.myp = MyPSACC.load_config(name=self.config_name)
elif self.args.web_conf:
self.is_good = False
return False
else:
raise FileNotFoundError(self.config_name)
atexit.register(self.save_config)
self.myp.set_record(self.args.record)
Charging.elec_price = ElecPrice.read_config()
if self.args.offline:
logger.info("offline mode")
self.is_good = True
else:
self.is_good = False
try:
self.is_good = self.myp.manager.refresh_token_now()
if self.is_good:
logger.info(str(self.myp.get_vehicles()))
except OAuthError:
if self.args.mail and self.args.password:
self.myp.connect(self.args.mail, self.args.password)
logger.info(str(self.myp.get_vehicles()))
self.is_good = True
else:
self.is_good = False
logger.error(
"Please reconnect by going to config web page")
if self.args.refresh:
self.myp.info_refresh_rate = self.args.refresh * 60
if self.is_good:
self.myp.start_refresh_thread()
if self.is_good:
self.start_remote_control()
elif not self.args.web_conf:
raise ConnectionError
self.save_config()
return True
def __on_mqtt_message(self, client, userdata, msg):
try:
logger.info("mqtt msg received: %s %s", msg.topic, msg.payload)
logger.debug("client: %s userdata: %s", client, userdata)
data = json.loads(msg.payload)
charge_info = None
if msg.topic.startswith(MQTT_RESP_TOPIC):
if "return_code" not in data:
logger.debug("mqtt msg hasn't return code")
elif data["return_code"] == "400":
self._refresh_remote_token(force=True)
if self.last_request:
logger.warning(
"last request is send again, token was expired")
last_request = self.last_request
self.last_request = None
self.publish(last_request, store=False)
else:
logger.error(
"Last request might have been send twice without success"
)
elif data["return_code"] != "0":
logger.error('%s : %s', data["return_code"],
data.get("reason", "?"))
elif msg.topic.startswith(MQTT_EVENT_TOPIC):
charge_info = data["charging_state"]
self.precond_programs[
data["vin"]] = data["precond_state"]["programs"]
if charge_info is not None and charge_info['remaining_time'] != 0:
try:
car = self.vehicles_list.get_car_by_vin(
vin=msg.topic.split("/")[-1])
if car and car.status.get_energy(
'Electric').charging.status != INPROGRESS:
# fix a psa server bug where charge beginning without status api being properly updated
logger.warning("charge begin but API isn't updated")
sleep(60)
self.wakeup(data["vin"])
except (IndexError, AttributeError, RateLimitException):
logger.exception("on_mqtt_message:")
except KeyError:
logger.exception("on_mqtt_message:")
def update_trips():
global trips, chargings, cached_layout, min_date, max_date, min_millis, max_millis, step, marks
logger.info("update_data")
conn = Database.get_db(update_callback=False)
Database.add_altitude_to_db(conn)
conn.close()
min_date = None
max_date = None
try:
trips_by_vin = Trips.get_trips(myp.vehicles_list)
trips = next(iter(trips_by_vin.values())) # todo handle multiple car
assert len(trips) > 0
min_date = trips[0].start_at
max_date = trips[-1].start_at
except (StopIteration, AssertionError):
logger.debug("No trips yet")
try:
chargings = Charging.get_chargings()
assert len(chargings) > 0
if min_date:
min_date = min(min_date, chargings[0]["start_at"])
max_date = max(max_date, chargings[-1]["start_at"])
else:
min_date = chargings[0]["start_at"]
max_date = chargings[-1]["start_at"]
except AssertionError:
logger.debug("No chargings yet")
if min_date is None:
return
# update for slider
try:
logger.debug("min_date:%s - max_date:%s", min_date, max_date)
min_millis = figures.unix_time_millis(min_date)
max_millis = figures.unix_time_millis(max_date)
step = (max_millis - min_millis) / 100
marks = figures.get_marks_from_start_end(min_date, max_date)
cached_layout = None # force regenerate layout
except (ValueError, IndexError):
logger.error("update_trips (slider): %s", exc_info=True)
except AttributeError:
logger.debug("position table is probably empty :", exc_info=True)
return
def _refresh_remote_token(self, force=False):
bad_remote_token = self.remoteCredentials.refresh_token is None
if not force and not bad_remote_token and self.remoteCredentials.last_update:
last_update: datetime = self.remoteCredentials.last_update
if (datetime.now() - last_update).total_seconds() < MQTT_TOKEN_TTL:
return True
try:
self.manager.refresh_token_now()
if bad_remote_token:
logger.error("remote_refresh_token isn't defined")
else:
res = self.manager.post(
REMOTE_URL + self.account_info.client_id,
json={
"grant_type": "refresh_token",
"refresh_token": self.remoteCredentials.refresh_token
},
headers=self.headers)
data = res.json()
logger.debug("refresh_remote_token: %s", data)
if "access_token" in data:
self.remoteCredentials.access_token = data["access_token"]
self.remoteCredentials.refresh_token = data[
"refresh_token"]
bad_remote_token = False
else:
logger.error(
"can't refresh_remote_token: %s\n Create a new one",
data)
bad_remote_token = True
if bad_remote_token:
otp_code = self.get_otp_code()
res = self.get_remote_access_token(otp_code)
self.remote_token_last_update = datetime.now()
self.mqtt_client.username_pw_set(
"IMA_OAUTH_ACCESS_TOKEN", self.remoteCredentials.access_token)
return True
except (RequestException, RateLimitException) as e:
logger.exception("Can't refresh remote token %s", e)
sleep(60)
return False
