class CarLogo:
DATABASE = 'car.db'
def __init__(self):
self.db = DB()
self.path = os.path.dirname(os.path.realpath(__file__))
self.images_path = os.path.join(self.path, 'images_path')
self.host = "http://www.chebiaow.com"
self.headers = {
'Connection': 'keep-alive',
'user-agent': ('Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36')
}
def check_dir(self):
if not os.path.exists(self.images_path):
os.mkdir(self.images_path)
def get_response(self, url, params=None):
try:
r = requests.get(url, headers=self.headers, params=params, timeout=15)
except:
pass
soup = BeautifulSoup(r.text, "lxml")
return soup
def create_url(self):
_url_format = "http://www.chebiaow.com/logo/{}.html"
for uppercase in au:
try:
soup = self.get_response(_url_format.format(uppercase))
_cars = soup.find("ul", {"class": "cb-list"}).findAll('li')
for car in _cars:
# self.car_info()
t = threading.Thread(target=self.car_info, args=(urljoin(self.host, car.div.a['href']),))
time.sleep(0.1)
t.start()
except:
pass
def car_info(self, url):
sm.acquire()
try:
soup = self.get_response(url)
left_index = soup.find("div", {"class": "xq-left"}).findAll('p')
name = left_index[0].text
image_byte = requests.get(left_index[1].img['src'], headers=self.headers).content
right_index = soup.find("ul", {"class": "xq-right"}).findAll('li')
founded = right_index[3].span.text
models = right_index[5].span.text
website = right_index[7].span.text
print("Insert Car Logo {}".format(name))
_sql = "insert into car_logo(name,image,founded,models,website) values (?,?,?,?,?)"
self.db.insert(_sql, (name, Binary(image_byte), founded, models, website))
except Exception as error:
print(error)
sm.release()
def make_table(self, load_history_trade_data=False):
DbMaker.create_table(table_name=self.table_name)
if load_history_trade_data:
df = pd.read_csv(self.csv_file_path, index_col=None)
df = df[['date', 'close']]
# unix timestamp转datetime
df['date'] = df['date'].apply(lambda x: datetime.fromtimestamp(x))
df['close_forecast'] = ''
df.columns = ['time', 'close', 'close_forecast']
DbMaker.update_table(table_name=self.table_name,
close_df=df,
close_forecast_df=None)
def __init__(self):
self.db = DB()
self.path = os.path.dirname(os.path.realpath(__file__))
self.images_path = os.path.join(self.path, 'images_path')
self.host = "http://www.chebiaow.com"
self.headers = {
'Connection': 'keep-alive',
'user-agent': ('Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36')
}
def __update_table(self, realtime_df, predicts):
realtime_df = realtime_df.loc[:, ['date', 'close']]
realtime_df['date'] = realtime_df['date'].apply(
lambda x: datetime.fromtimestamp(x))
realtime_df['close_forecast'] = ''
realtime_df.columns = ['time', 'close', 'close_forecast']
latest_time = realtime_df.tail(1)['time'].values[0]
logging.info('Latest time of trade is {}'.format(
pd.to_datetime(str(latest_time)).strftime('%Y-%m-%d %H:%M:%S')))
period = self.trade_data_opts['period']
# 计算real_time_df最后一个时间之后output_size个周期
# 返回格式为: [[time1,'',predict1], [time2,'',predict2]...]
predicts_data = [[
latest_time + np.timedelta64(((x + 1) * period), 'm'), '',
predicts[x]
] for x in range(0, self.model_opts['output_size'])]
close_forecast_df = pd.DataFrame(
predicts_data, columns=['time', 'close', 'close_forecast'])
# 将real_time_df最后一个时间之后output_size个周期的预测数据append到df中
# df = df.append(pd.DataFrame(predicts_data, columns=['time', 'close', 'close_forecast']), ignore_index=True)
DbMaker.update_table(self.table_name, realtime_df, close_forecast_df)
def __dump_table(self):
DbMaker.dump_table(self.table_name)