def stopGamesWithRegex():
"""we do not support Regex rules anymore.
Mark all games using them as finished - until somebody
complains. So for now always return False"""
if not Internal.db.hasTable('usedrule'):
return
usedRegexRulesets = Query("select distinct ruleset from usedrule "
"where definition not like 'F%' "
"and definition not like 'O%' "
"and definition not like 'int%' "
"and definition not like 'bool%' "
"and definition<>'' "
"and definition not like 'XEAST9X%'").records
usedRegexRulesets = list(unicode(x[0]) for x in usedRegexRulesets)
if not usedRegexRulesets:
return
openRegexGames = Query("select id from game "
"where endtime is null "
"and ruleset in (%s)" % ','.join(usedRegexRulesets)).records
openRegexGames = list(x[0] for x in openRegexGames)
if not openRegexGames:
return
logInfo(
'Marking games using rules with regular expressions as finished: %s' %
openRegexGames)
for openGame in openRegexGames:
endtime = datetime.datetime.now().replace(
microsecond=0).isoformat()
Query('update game set endtime=? where id=?', (endtime, openGame))
def _loginReallyFailed(self, failure):
"""login failed, not fixable by adding missing user"""
msg = None
if not isAlive(Internal.mainWindow):
raise CancelledError
if failure.check(CancelledError):
pass
elif failure.check(twisted.internet.error.TimeoutError):
msg = i18n('Server %1 did not answer', self.url)
elif failure.check(twisted.internet.error.ConnectionRefusedError):
msg = i18n('Server %1 refused connection', self.url)
elif failure.check(twisted.internet.error.ConnectionLost):
msg = i18n('Server %1 does not run a kajongg server', self.url)
elif failure.check(twisted.internet.error.DNSLookupError):
msg = i18n('Address for server %1 cannot be found', self.url)
elif failure.check(twisted.internet.error.ConnectError):
msg = i18n(
'Login to server %1 failed: You have no network connection',
self.url)
else:
msg = 'Login to server {} failed: {}/{} Callstack:{}'.format(
self.url, failure.value.__class__.__name__,
failure.getErrorMessage(), failure.getTraceback())
# Maybe the server is running but something is wrong with it
if self.url and self.url.useSocket:
if removeIfExists(socketName()):
logInfo(
i18n('removed stale socket <filename>%1</filename>',
socketName()))
msg += '\n\n\n' + i18n('Please try again')
self.dlg = None
if msg:
logWarning(msg)
raise CancelledError
def __upgrade(self):
"""upgrade the structure of an existing kajongg database"""
try:
Internal.db = DBHandle(self.path)
if isPython3:
allVersions = list(['4.13.0', '8300'])
else:
allVersions = list(['4.13.0', '8200'])
assert allVersions[-1] == str(Internal.defaultPort), '{} != {}'.format(
allVersions[-1], str(Internal.defaultPort))
# skip versions before current db versions:
currentVersion = self.__currentVersion()
while allVersions and allVersions[0] <= currentVersion:
allVersions = allVersions[1:]
for version in allVersions:
currentVersion = self.__currentVersion()
with Internal.db: # transaction
updateMethodName = 'updateToVersion{}'.format(version.replace('.', '_'))
if hasattr(self, updateMethodName):
getattr(self, updateMethodName)()
Query('UPDATE general SET schemaversion=?', (version,))
logInfo(m18n('Database %1 updated from schema %2 to %3',
Internal.db.path, currentVersion, version), showDialog=True)
except sqlite3.Error as exc:
logException(
u'opening %s: %s' %
(unicodeString(self.path), exc.message))
finally:
Internal.db.close(silent=True)
def __startLocalServer(self):
"""start a local server"""
try:
args = self.__findServerProgram()
if self.useSocket or os.name == 'nt': # for nt --socket tells the server to bind to 127.0.0.1
args.append('--socket=%s' % socketName())
if removeIfExists(socketName()):
logInfo(
i18n('removed stale socket <filename>%1</filename>',
socketName()))
if not self.useSocket:
args.append('--port=%d' % self.port)
if self.isLocalGame:
args.append('--db={}'.format(
os.path.normpath(os.path.join(appdataDir(), 'local3.db'))))
if Debug.argString:
args.append('--debug=%s' % Debug.argString)
if os.name == 'nt':
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
else:
startupinfo = None
process = subprocess.Popen(
args, startupinfo=startupinfo) # , shell=os.name == 'nt')
if Debug.connections:
logDebug(
i18n(
'started the local kajongg server: pid=<numid>%1</numid> %2',
process.pid, ' '.join(args)))
except OSError as exc:
exc.filename = ' '.join(args)
logException(exc)
def downloadGoogleTrendsData(geoIdArray):
pytrend = TrendReq()
maxLength = len(geoIdArray)
progress = 0
tooManyRequestGeoIdArray = []
for geoId in geoIdArray:
printProgressBar(progress, maxLength,
"Downloading Google Trends data for " + geoId)
geoIdForGoolge = geoId
#special cases
if geoId == "GB":
geoIdForGoolge = "GB-ENG"
if(geoIdForGoolge != "" and geoIdForGoolge != None and geoIdForGoolge != "N/A"):
try:
pytrend.build_payload(kw_list=['/m/01cpyy'], timeframe='2019-11-01 ' + datetime.datetime.now().strftime("%Y-%m-%d"), geo=geoIdForGoolge)
interest_over_time_df = pytrend.interest_over_time()
interest_over_time_df.to_csv("../dat/temp/googleTrends/" + geoId + ".csv")
except pytrends.exceptions.ResponseError as error:
if error.response.status_code == 429:
log.logInfo("Google is mad at us that we make so much requests so were waiting for a minute before continuing (geoId will be redownloaded at the end)")
tooManyRequestGeoIdArray.append(geoId)
time.sleep(60)
else:
log.logError("Downloading googletrends for geoId " + geoId + " failed - Errormessage was: " + str(error))
except Exception as error:
log.logWarning("Downloading googletrends for geoId " + geoId + " failed - Errormessage was: " + str(error))
progress += 1
if len(tooManyRequestGeoIdArray) > 0:
downloadGoogleTrendsData(tooManyRequestGeoIdArray)
def update(self, **kwargs):
table = kwargs['table']
# del kwargs['table']
kwargs.pop('table')
where = kwargs['where']
kwargs.pop('where')
sql = 'update %s set ' % table
for k, v in kwargs.items():
sql += "%s='%s'," % (k, v)
sql = sql.rstrip(',')
sql += ' where %s' % where
print(sql)
try:
# 执行SQL语句
self.cursor.execute(sql)
# 提交到数据库执行
self.conn.commit()
# 影响的行数
rowcount = self.cursor.rowcount
except Exception as e:
log.logInfo(repr(e), threading.current_thread().name + ".log")
print(e)
# 发生错误时回滚
self.conn.rollback()
return rowcount
def __init__(self):
logInfo('Starting pygame')
pygame.init()
self.screen = pygame.display.set_mode(self.DFT_SCREEN_SIZE)
self.graph = UIGraph(self.screen, (35, 21), (800, 600))
self.status = UIStatus(self.screen, (0, 0), (835, 20))
self.trigger = UITrigger(self.screen, (0, 21), (35, 600))
def processUserInput(gui, serial):
global expectingData
scaleGraphLUT = {
pygame.K_UP: (0, 0.1),
pygame.K_DOWN: (0, -0.1),
pygame.K_LEFT: (-0.1, 0),
pygame.K_RIGHT: (0.1, 0)
}
posGraphLUT = {pygame.K_a: (10, 0), pygame.K_d: (-10, 0)}
scaleTriggerLUT = {pygame.K_i: 0.1, pygame.K_j: -0.1}
freqLUT = {pygame.K_z: 10000, pygame.K_x: -10000}
samplesLUT = {pygame.K_n: -100, pygame.K_m: 100}
for event in pygame.event.get():
if event.type == pygame.QUIT:
logInfo('Exiting...')
pygame.quit()
exit(0)
elif event.type == pygame.KEYDOWN:
if event.key in scaleGraphLUT:
gui.graph.incScale(scaleGraphLUT[event.key])
return True
elif event.key in scaleTriggerLUT:
gui.trigger.incTriggerLevel(scaleTriggerLUT[event.key])
serial.setTriggerLevel(gui.trigger.triggerLevel)
return True
elif event.key in posGraphLUT:
gui.graph.incPos(posGraphLUT[event.key])
return True
elif event.key in freqLUT:
gui.graph.incFreq(freqLUT[event.key])
serial.setPrecision(gui.graph.freq)
return True
elif event.key in samplesLUT:
gui.graph.incNumberOfSamples(samplesLUT[event.key])
if serial.setNumberOfSamples(gui.graph.numberOfSamples):
gui.graph.incNumberOfSamples(samplesLUT[event.key] * (-1))
return True
elif event.key == pygame.K_SPACE:
if serial.triggerNow() == 0:
expectingData = True
return True
elif event.key == pygame.K_o:
serial.turnOff()
return True
elif event.key == pygame.K_p:
if serial.trigMode() == 0:
expectingData = True
return True
elif event.type == pygame.MOUSEBUTTONDOWN:
if event.button == 4:
gui.graph.incScale((0.5, 0))
return True
elif event.button == 5:
gui.graph.incScale((-0.5, 0))
return True
return False
def __init__(self, host, port, user, password, database): # 构造函数
try:
self.conn = pymysql.connect(host=host,
port=port,
user=user,
password=password,
database=database,
charset='utf8')
self.cursor = self.conn.cursor()
except Exception as e:
log.logInfo(repr(e), threading.current_thread().name + ".log")
print(e)
def getSuningFinalRequest(text):
#分析得知，参数都存储在一段js中。所以从js中提取. 而且是一个json字符串，因此可以将其转成json的dict进行获取。
##转成json 更方便快捷，访问获取更简单。不要直接操作字符串，用find从字符串中提取。
index = text.find("var sn = sn || {") + len("var sn = sn || {") - 1
json_text = text[index:].split(';')[0]
#去除 "itemDomain":"//"+document.location.hostname,
json_text = json_text.replace(
'"itemDomain":"//"+document.location.hostname, ', '')
#去除/* */ 注释
pattern = re.compile(r"/\*.*?\*/")
json_text = pattern.sub("", json_text)
#json.loads(str) 中的json字符串的key只能为字符串形式，否则解析出错。 这里cuxiaoSeq":{voucherTitle:1,lhvou，key并不是一个字符串
json_text = re.sub(r'"cuxiaoSeq":{.*?},', "", json_text)
json_param = json.loads(json_text)
#注意使用贪婪和非贪婪方式。 贪婪方式，会一直到最后一个*/. 非贪婪方式，值找到第一个。
# pattern = re.compile(r'/\*.*\*/')
# result1 = pattern.findall(json_text)
# result1
# ['/*默认关闭，true为打开*/ "hasBottomFixed":false, /*默认关闭，true为打开*/ "hasTopFixed":false, /*默认关闭，true为打开*/']
# pattern = re.compile(r'/\*.*?\*/')
# result1 = pattern.findall(json_text)
# result1
# ['/*默认关闭，true为打开*/', '/*默认关闭，true为打开*/', '/*默认关闭，true为打开*/']
partNumber = json_param.get("partNumber")
vendorCode = json_param.get("vendorCode")
category1 = json_param.get("category1")
catenIds = json_param.get("catenIds")
weight = json_param.get("weight")
brandId = json_param.get("brandId")
#https://pas.suning.com/nspcsale_0_000000010606656239_000000010606656239_0000000000_190_755_7550101_20089_1000051_9051_10346_Z001___R1901001_0.463_0___000060021__.html?callback=pcData
# "partNumber":"000000010606656239"
# "vendorCode":"0000000000"
# _190_755_7550101 固定
# "category1":"20089"
# _1000051_9051_10346_Z001 固定
# "catenIds":"R1901001"
# "weight":"0.463"
# 后面这个0 不影响。
# "brandId":"000060021"
httpRequest = ('https://pas.suning.com/nspcsale_0_' + partNumber + '_' +
partNumber + '_' + vendorCode + '_190_755_7550101_' +
category1 + '_1000051_9051_10346_Z001___' + catenIds + '_' +
weight + '_0___' + brandId +
'__.html?callback=pcData').replace(' ', '')
log.logInfo(httpRequest, log_file)
return httpRequest
def test_transaction(self):
sql1 = "insert into STORE(link,name,type,addr,budget_pm,act_budget_pm,act_budget_am,pay,e_pay,score,tel,seat,start_date)values('https://tabelog.com/tokyo/A1301/A130101/13203796/','CHIUnE （チウネ）','イノベーティブ・フュージョン、モダンフレンチ','東京都中央区銀座1-22-12 マドリガル銀座 1F大きな地図を見る周辺のお店を探す このお店は「岐阜市弥八町5-1」から移転しています。 ※移転前の情報は最新のものとは異なります。 移転前の店舗情報を見る','￥50,000～￥59,999','￥50,000～￥59,999','￥60,000～￥79,999','1','0','4.57','03-6228-6928','6席 （カウンター席）','2016年12月27日')"
sql2 = "insert into STORE(id,link,name,type,addr,budget_pm,act_budget_pm,act_budget_am,pay,e_pay,score,tel,seat,start_date)values('1597','hello','CHIUnE （チウネ）','イノベーティブ・フュージョン、モダンフレンチ','東京都中央区銀座1-22-12 マドリガル銀座 1F大きな地図を見る周辺のお店を探す このお店は「岐阜市弥八町5-1」から移転しています。 ※移転前の情報は最新のものとは異なります。 移転前の店舗情報を見る','￥50,000～￥59,999','￥50,000～￥59,999','￥60,000～￥79,999','1','0','4.57','03-6228-6928','6席 （カウンター席）','2016年12月27日')"
try:
# 执行SQL语句
self.cursor.execute(sql1)
self.cursor.execute(sql2)
# 提交到数据库执行
self.conn.commit()
except Exception as e:
# 发生错误时回滚
log.logInfo(repr(e), threading.current_thread().name + ".log")
print(e)
self.conn.rollback()
def raw_insert(self, sql):
res = 0
print(sql)
try:
# 执行SQL语句
self.cursor.execute(sql)
# 提交到数据库执行
self.conn.commit()
# 获取自增id
res = self.cursor.lastrowid
except Exception as e:
# 发生错误时回滚
log.logInfo(repr(e), threading.current_thread().name + ".log")
print(e)
self.conn.rollback()
return res
def selectAll(self, **kwargs):
table = kwargs['table']
field = 'field' in kwargs and kwargs['field'] or '*'
where = 'where' in kwargs and 'where ' + kwargs['where'] or ''
order = 'order' in kwargs and 'order by ' + kwargs['order'] or ''
sql = 'select %s from %s %s %s ' % (field, table, where, order)
print(sql)
try:
# 执行SQL语句
self.cursor.execute(sql)
# 使用 fetchone() 方法获取单条数据.
data = self.cursor.fetchall()
except Exception as e:
log.logInfo(repr(e), threading.current_thread().name + ".log")
print(e)
# 发生错误时回滚
self.conn.rollback()
return data
def checkPings(self):
"""are all clients still alive? If not log them out"""
since = elapsedSince(self.lastPing)
if self.srvUsers and since > 30:
if Debug.quit:
logDebug('no ping since {} seconds but we still have users:{}'.
format(elapsedSince(self.lastPing), self.srvUsers))
if not self.srvUsers and since > 30:
# no user at all since 30 seconds, but we did already have a user
self.__stopAfterLastDisconnect()
for user in self.srvUsers:
if elapsedSince(user.lastPing) > 60:
logInfo(
'No messages from %s since 60 seconds, clearing connection now'
% user.name)
user.mind = None
self.logout(user)
reactor.callLater(10, self.checkPings)
def delete(self, **kwargs):
table = kwargs['table']
where = kwargs['where']
sql = 'DELETE FROM %s where %s' % (table, where)
print(sql)
try:
# 执行SQL语句
self.cursor.execute(sql)
# 提交到数据库执行
self.conn.commit()
# 影响的行数
rowcount = self.cursor.rowcount
except Exception as e:
log.logInfo(repr(e), threading.current_thread().name + ".log")
print(e)
# 发生错误时回滚
self.conn.rollback()
return rowcount
def __init__(self, table, temp=False):
dummy, dummy, function, dummy = traceback.extract_stack()[-2]
self.outstanding = 0
self.calledBy = function
if not temp:
self.garbageCollection()
self.table = table
self.requests = []
self.callbackMethod = None
self.__callbackArgs = None
self.completed = False
if not temp:
DeferredBlock.blocks.append(self)
if not DeferredBlock.blockWarned:
if len([x for x in DeferredBlock.blocks if x.table == table]) > 10:
DeferredBlock.blockWarned = True
logInfo('We have %d DBlocks:' % len(DeferredBlock.blocks))
for block in DeferredBlock.blocks:
logInfo(str(block))
def __init__(self, table, temp=False):
dummy, dummy, function, dummy = traceback.extract_stack()[-2]
self.outstanding = 0
self.calledBy = function
if not temp:
self.garbageCollection()
self.table = table
self.requests = []
self.callbackMethod = None
self.__callbackArgs = None
self.completed = False
if not temp:
DeferredBlock.blocks.append(self)
if not DeferredBlock.blockWarned:
if len([x for x in DeferredBlock.blocks if x.table == table
]) > 10:
DeferredBlock.blockWarned = True
logInfo('We have %d DBlocks:' % len(DeferredBlock.blocks))
for block in DeferredBlock.blocks:
logInfo(str(block))
def getPriceByDriverHeadless():
chrome_options = Options()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome(
"chromedriver", chrome_options=chrome_options) # open with Chrome
driver.set_page_load_timeout(20) #(5)
while True:
config_dict = loadConfig()
#dict 的迭代
for item in config_dict:
try:
print(timeStampToTime(time.time()), "open...",
config_dict[item].get("URL"))
driver.get(config_dict[item].get("URL"))
print(timeStampToTime(time.time()), "open done")
except Exception as e:
print("Except:", e)
print("加载页面太慢，直接停止加载，继续下一步操作")
print(timeStampToTime(time.time()),
config_dict[item].get("URL"))
try:
# 更好的方式
ele = driver.find_element_by_xpath(
'//div[@class="dd"]/span[@class="p-price"]/span[contains(@class,"price")]'
)
#print(ele)
log.logInfo(driver.title + ": " + ele.text, log_file)
if float(ele.text) < config_dict[item].get("ExpectedPrice"):
#发送邮件提醒
print(driver.title)
print(ele.text)
sendmail.sendEmail(
config_dict[item].get("Email", "*****@*****.**"),
config_dict[item].get("URL") + " " + driver.title +
":" + ele.text)
except Exception as e:
print(e)
log.logInfo("find_element_by_xpath error")
time.sleep(30)
def get_html(url, headers=user_config.headers):
# print(url)
time.sleep(0.02)
try:
response = requests.get(url, headers=headers, timeout=20)
# res.content b类型 res.text 字符类型。
# encoding：http 头部， 页面 charset，猜测。
if response.status_code == 200:
return response.text
elif response.status_code == 400: #结束循环
return "EOF"
else:
# TODO
print("页面获取错误：状态码{}".format(response.status_code))
return "ERR"
except Exception as e:
#TODO 异常处理除了打印供调试的信息外。还应该提供相应的结果给调用者或用户。
# raise ConnectionError('获取网页失败')
log.logInfo(repr(e), threading.current_thread().name + ".log")
print(e)
return "ERR"
def test_sql(self):
#python中二进制 串。
b_data = b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00H\x00H\x00\x00\xff\xfe\x00+Optimized by JPEGmini 3.12.0.2\xff\xd9'
try:
#原生底层的sql语句。
t_data = 'hello"world'.replace("\"", "\\\"")
sql = "insert into test(big_mb)values(\"%s\")" % t_data
print(sql)
self.cursor.execute("insert into test(big_mb)values(\"%s\")" %
t_data)
self.cursor.execute(
"insert into test(big_mb)values(\"b'123abc'\")"
) #这里实际上插入的是字符串，字符串为b'123abc'。insert into test(big_mb)values("b'123abc'")
self.cursor.execute(
"insert into test(big_mb)values(b'0101')"
) # sql 语句 insert into test(big_mb)values(b'0101'). 插入bit。
self.cursor.execute(
"insert into test(big_mb)values(0xff12ee0011ff)") # 插入byte。
self.cursor.execute(
"insert into test(big_mb)values(1234)") # 插入数值。
#带参数api 也是把 b'' 串中的b去掉，变成str （其实就是decode('ascii','surrogateescape')）。 而且实际也是%拼接的str。 带参数的api，底层也是转str。 所以和自己进行转原生str的sql语句一样，而且它的灵活性其实不高。 所以python实际无法直接插入sql中的b类型。除非自己转成01拼接／
#注意 python中的b''和sql中的b''不太一样。 python中十六进制\xff和sql中的十六进制0xff也不太一样。
#底层是调用 def escape_bytes(value, mapping=None):
# return "'%s'" % value.decode("ascii", "surrogateescape").translate(_escape_table)
self.cursor.execute("insert into test(big_mb)values(%s)", b_data)
#所以python可以直接拼接原生sql语句，进行插入。而不要使用库中带参数的api，因为转化的不是很好。
# 提交到数据库执行
self.conn.commit()
# 获取自增id
res = self.cursor.lastrowid
except Exception as e:
# 发生错误时回滚
log.logInfo(repr(e), threading.current_thread().name + ".log")
print(e)
self.conn.rollback()
return res
def getJDFinalRequest(text):
# https://c0.3.cn/stock?skuId=1592448&cat=670,677,11303&venderId=1000000328&area=19_1607_3155_0&buyNum=1&callback=jQuery3990626
skuIdStr = 'skuid:'
index = text.find(skuIdStr) + len(skuIdStr)
skuId = text[index:index + 20].split(',')[0]
#print(skuId)
catStr = 'cat: ['
index = text.find(catStr) + len(catStr)
cat = text[index:index + 20].split(']')[0]
#print(cat)
venderIdStr = 'venderId:'
index = text.find(venderIdStr) + len(venderIdStr)
venderId = text[index:index + 20].split(',')[0]
#print(venderId)
#log.logInfo("sku:" + skuId + ",cat:" + cat + ",venderId:" + venderId)
httpRequest = (
'https://c0.3.cn/stock?skuId=' + skuId + '&cat=' + cat + '&venderId=' +
venderId +
'&area=19_1607_3155_0&buyNum=1&callback=jQuery3990626').replace(
' ', '')
log.logInfo(httpRequest, log_file)
return httpRequest
def insert(self, **kwargs):
res = 0
table = kwargs['table']
del kwargs['table']
sql = "insert into %s(" % table
fields = ""
values = ""
for k, v in kwargs.items():
fields += "%s," % k
#因为这里对于二进制 bytes串，% 拼接后，实际上存入的是 b'' 形式的字符串。 如果要想存二进制，应该用带参数的api，或者这里要先decode成str。
#不过对于二进制转str 或用带参数的api（其背后也是转str），因为图片二进制不一定能完全转成str，因此有一定可能性转报错。
#解决转错误的办法 加上 surrogateescape。
if type(v).__name__ == "bytes":
v = v.replace(b"\"", b"\\\"")
if type(v).__name__ == "str":
v.replace(
"\"", "\\\""
) # 因为下面sql语句是 insert into xx() values("",""). 值是以双引号的，为了避免值内容中的双引号,要转义。
values += "\"%s\"," % v #最好要转义。 sql 和 python一样 字符串可以用单引号和双引号（很多脚本语言也是这样）。 但应该保持良好的编码习惯。 都统一用双引号将字符串引用起来，字符串中有双引号应该转义。
fields = fields.rstrip(',')
values = values.rstrip(',')
# 最后sql是"" 包起来的字符串。 对于b 字节串， % 连接后 变成 b'' 这样的字符串。
sql = sql + fields + ")values(" + values + ")"
# print(sql)
try:
# 执行SQL语句
self.cursor.execute(sql)
# 提交到数据库执行
self.conn.commit()
# 获取自增id
res = self.cursor.lastrowid
except Exception as e:
log.logInfo(
repr(e) + "\t" + sql,
threading.current_thread().name + ".log")
print(e)
# 发生错误时回滚
self.conn.rollback()
return res
def loadConfig():
#获取文件最后修改时间戳 float
try:
modify_t = os.path.getmtime(config_file)
except FileNotFoundError as e:
log.logInfo(config_file + "FileNotFound")
exit(1)
#修改时间没有变动，表明不需要重新读取配置
if cofig_lastDate == modify_t:
print("配置文件未改变")
return config_dict
else:
try:
with open(config_file, 'r') as f:
#read() 读取全部。返回类型str
config_str = f.read()
# 注意json标准语法中是不支持单引号。所以配置文件中的字符串应该都是双引号而不是单引号。 https://blog.csdn.net/u012063507/article/details/71554775
config_dict = json.loads(config_str)
return config_dict
except:
log.logInfo('读取配置文件 json数据异常')
exit(1)
def processModuleRequest(moduleConfig, moduleType, paginaId):
log.logDebug(
'Checking processing for module ' + str(moduleType) + ' from ' + str(paginaId) + ': ' + str(moduleConfig))
if moduleType not in apiClassInstance:
log.logDebug('Module ' + str(moduleType) + ' not to be postprocessed')
return
if moduleType not in moduleCache:
log.logError('Cache undefined for module type ' + str(moduleType))
return
log.logDebug('Checking cache for module ' + str(moduleType) + ' from ' + str(paginaId) + ': ' + str(moduleConfig))
for key in moduleCache[moduleType].keys():
if moduleConfig == key:
log.logInfo('Request already stored in cache: ' + str(moduleCache[moduleType][moduleConfig]))
return moduleCache[moduleType].get(key)
log.logDebug('Processing request not found in cache for ' + str(moduleType))
moduleConfigDict = ast.literal_eval(moduleConfig)
if pluginIdLabel not in moduleConfigDict:
log.logError('Parameter ' + str(pluginIdLabel) + ' not found')
return
else:
pluginType = moduleConfigDict[pluginIdLabel]
# noinspection PyBroadException
try:
response = apiClassInstance[moduleType][pluginType](paginaId, moduleConfig)
requestResponse = response.processPlugin()
log.logDebug('Storing request in cache for ' + str(moduleType) + ': ' + str(requestResponse))
moduleCache[moduleType][moduleConfig] = requestResponse
return requestResponse
except:
log.logError('Error trying to make this plugin: ' + pluginType + ' from: ' + moduleType)
return
def __upgrade(self):
"""upgrade the structure of an existing kajongg database"""
try:
Internal.db = DBHandle(self.path)
allVersions = list(['4.13.0', '8300', '8301'])
assert allVersions[-1] == str(Internal.defaultPort), '{} != {}'.format(
allVersions[-1], str(Internal.defaultPort))
# skip versions before current db versions:
currentVersion = self.__currentVersion()
while allVersions and allVersions[0] <= currentVersion:
allVersions = allVersions[1:]
for version in allVersions:
currentVersion = self.__currentVersion()
with Internal.db: # transaction
updateMethodName = 'updateToVersion{}'.format(version.replace('.', '_'))
if hasattr(self, updateMethodName):
getattr(self, updateMethodName)()
Query('UPDATE general SET schemaversion=?', (version,))
logInfo(i18n('Database %1 updated from schema %2 to %3',
Internal.db.path, currentVersion, version), showDialog=True)
except sqlite3.Error as exc:
logException('opening %s: %s' % (self.path, exc.message))
finally:
Internal.db.close(silent=True)
def get_html(url, proxies=None):
print(url)
#有的网站会反爬。 尽量模拟构造成浏览器的header， User-Agent等。 关键还是在User-Agent
# headers = {}
# headers['Host'] = 'www.stats.gov.cn'
# headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
# headers['Accept']= 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3'
# headers['Accept-Encoding'] = 'gzip, deflate' #如果是压缩的，这个header不可少
# headers['Accept-Language'] = 'zh-CN,zh;q=0.9,en;q=0.8'
# headers['Cache-Control'] = 'max-age=0'
headers = {
'Host': 'www.stats.gov.cn',
'Connection': 'keep-alive',
'Cache-Control': 'max-age=0',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'If-None-Match': '1c98-580baa54b4840-gzip',
'If-Modified-Since': 'Thu, 31 Jan 2019 05:53:29 GMT'
}
req = request.Request(url, headers=headers)
#也可以req = request.Request(url） 然后再通过req.add_header()去添加
if proxies is not None:
proxy_handler = request.ProxyHandler(proxies)
opener = request.build_opener(proxy_handler, request.HTTPHandler)
request.install_opener(opener)
flag = True
count = 0
furl = {}
while flag and count < 3:
try:
furl = urlopen(req, timeout=15)
flag = False
except Exception as e:
print(str(e) + ',' + url)
log.logInfo(str(e) + ',' + url)
count += 1
#return None
if flag:
return None
# furl = urlopen(req)
if furl.code != 200:
print('get ' + url + ' code:' + str(furl.code))
return None
text_b = furl.read()
furl.close()
#print(text_b[:6])
#压缩过的。http://www.01happy.com/python-request-url-gbk-decode/
# gzip 压缩 解压缩 https://blog.csdn.net/qq_35899407/article/details/91383983
if text_b[:6] == b'\x1f\x8b\x08\x00\x00\x00':
#print('gzip')
text_b = gzip.decompress(text_b)
#print(text_b)
charset = 'utf-8'
index = text_b.find("charset".encode())
if "GBK".encode() in text_b[index:index +
30] or "gbk".encode() in text_b[index:index +
30]:
charset = 'gbk'
elif "utf-8".encode() in text_b[index:index + 30] or "UTF-8".encode(
) in text_b[index:index + 30]:
charset = 'utf-8'
elif 'gb2312'.encode() in text_b[index:index + 30] or 'GB2312'.encode(
) in text_b[index:index + 30]:
charset = 'gb2312'
#https://bbs.csdn.net/topics/390632652?page=1
#对于响应的编码问题，requests会从两方面去获取编码信息。 (其实我们自己处理的话也应该这样的方式)
#1.首先从包的头部中的 Content-Type获取编码信息，将获取到编码信息放在req.encoding. 如果Content-Type没法得到编码信息，
#encoding置为默认的ISO-8859-1。
#2.其次会从获取到的html中获取编码信息。 一般html中的meta元素中会有页面编码的声明。requests会放在apparent_encoding中。
#requests 的text 和 content 。 text实际上是由req2.content.decode(req2.encoding,'ignore')得来的。
# 若果规范的话，可以直接根据1 2来得出编码信息。 而很多时候不规范，所以，这里自己直接根据页面获取编码信息。
# 编码信息：http头部Content-Type的声明，html页面meta的声明，html本身的实际编码方式。
# 有时候不规范，声明的和实际的不一致 很容易导致解析乱码的。
#对于统计局官网真是坑。
#首先其请求的回应包中的头部的Content-type中也没有表明是什么编码，导致requests库也识别不到编码信息，req.encoding用默认的’ISO-8859-1‘，这也导致
#requests库解码出现问题。
#其次其网页真正的编码时GBK，但是其网页内声明的是gb2312，所以用gb2312进行解码 有些字符是由乱码的。req.apparent_encoding成了GB2312。用这个解码也成问题。
#所以只能手动写死GBK。 坑。 这里就是声明的和实际的不一致，那么根据声明的来解码当然就乱码了。
#print(charset)
charset = 'GBK' # 对于统计局官网，写死。
text = text_b.decode(charset, 'ignore')
#print(text)
time.sleep(2) #不要频繁. 避免服务器作妖，尽量每次请求后sleep
return text
def getPriceByDriver():
#注意驱动问题。一般最好和chrome版本一致。否则会出现莫名其妙的问题。
#这里用chrome已经是最新版本了。所以更新驱动。放置在当前目录。或者覆盖掉/usr/local/bin/chromedriver
#webdriver.Chrome('./chromedriver') 具体路径，从具体路径去加载驱动。
#webdriver.Chrome("chromedriver") 会从默认的路径顺序去加载。 /usr/local/bin/chromedriver
#webdriver.Chrome()<===> webdriver.Chrome("chromedriver") 看构造函数就清楚了
driver = webdriver.Chrome(
"/usr/local/bin/chromedriver") # open with Chrome
#driver.implicitly_wait(5) #设置等待时间都不起作用。什么显示 隐式时间都不行。
#因为driver.get 加载页面是同步阻塞的。因此有时候元素已经出来，但是还要完全等待完，没有必要，因此做超时处理。
driver.set_page_load_timeout(20) #(5)
while True:
config_dict = loadConfig()
#dict 的迭代
for item in config_dict:
# print(config_dict[item])
#locator = (By.XPATH, '//div[@class="dd"]/span[@class="p-price"]/span[contains(@class,"price")]')
#locator = (By.ID,"banner-miaosha")
#由于css中有多个属性，因此直接使用css selector 来获取
#使用xpath 也可以，要将带有空格也填入，不能只填入部分，否则找不到。
#但是基于jd网页上的class是根据商品的id合成的。因此直接填入不会通用。
# driver.find_element_by_xpath('//div[@class="dd"]/span[@class="p-price"]/span[@class="price J-p-6287165"]')
#因此先找到父节点(缩小范围)，再找下面那个带有price class 的节点。
#<span class="price J-p-6287165">329.00</span>
#ele_p = driver.find_element_by_xpath('//div[@class="dd"]/span[@class="p-price"]') # ele_p.text ￥329.00
#ele = ele_p.find_element_by_css_selector('.price') #ele.text 329
try:
print("open...")
print(timeStampToTime(time.time()),
config_dict[item].get("URL"))
driver.get(config_dict[item].get("URL"))
#不起作用
#WebDriverWait(driver, 10, 0.5).until(EC.presence_of_element_located(locator))
print("open done") #driver.get() 是同步阻塞的。要等网页完全打开才进行下一步。
print(timeStampToTime(time.time()))
except Exception as e:
print("Except:", e)
print("加载页面太慢，直接停止加载，继续下一步操作")
print(timeStampToTime(time.time()),
config_dict[item].get("URL"))
#不需要做，timeout后自动停止加载了。
#下面对chrome都不起作用
#driver.execute_script('window.stop ? window.stop() : document.execCommand("Stop");')
#driver.execute_script("window.stop()")
# finally:
# pass
# driver.close()
try:
# 更好的方式
ele = driver.find_element_by_xpath(
'//div[@class="dd"]/span[@class="p-price"]/span[contains(@class,"price")]'
)
#print(ele)
log.logInfo(driver.title + ": " + ele.text, log_file)
if float(ele.text) < config_dict[item].get("ExpectedPrice"):
#发送邮件提醒
print(driver.title)
print(ele.text)
sendmail.sendEmail(
config_dict[item].get("Email", "*****@*****.**"),
config_dict[item].get("URL") + " " + driver.title +
":" + ele.text)
except Exception as e:
print(e)
log.logInfo("find_element_by_xpath error")
time.sleep(30)
def getPriceBySource():
price_httprequest = {}
history_price = {}
goods_title = {}
while True:
config_dict = loadConfig()
#dict 的迭代. item ->key.
for item in config_dict:
#text 二进制 b'xxxx'
# 网页 <meta http-equiv="Content-Type" content="text/html; charset=gbk">
# 明明是gbk，但是有些网页死活decode出问题，所以只能暂时ingore方式。
if (price_httprequest.get(item) == None):
fUrl = urlopen(config_dict[item].get("URL"))
if fUrl.getcode() != 200:
log.logInfo(config_dict[item].get("URL") +
' urlopen status code not 200')
fUrl.close()
continue
text_b = fUrl.read()
index = text_b.find("charset".encode())
if "GBK".encode() in text_b[index:index + 30] or "gbk".encode(
) in text_b[index:index + 30]:
charset = 'gbk'
elif "utf-8".encode() in text_b[index:index +
30] or "UTF-8".encode(
) in text_b[index:index + 30]:
charset = 'utf-8'
text = text_b.decode(charset, 'ignore')
if ("jd.com" in config_dict[item].get("URL")):
#<meta http-equiv="Content-Type" content="text/html; charset=gbk">
httpRequest = getJDFinalRequest(text)
elif ("suning.com" in config_dict[item].get("URL")):
#<meta charset="UTF-8" />
httpRequest = getSuningFinalRequest(text)
fUrl.close() #注意关闭，不然太多的链接会导致Connection问题
price_httprequest[item] = httpRequest
index = text.find('<title>')
title = text[index:index + 20] #含中文
goods_title[item] = title
#如果每个商品的skuid，cat ，verderId不会改变，那么完全不用每次都发送上面的请求，可以在第一次发送然后，然后下次直接下面的请求。
#一般来说是不会变动的。
try:
fRequest = urlopen(price_httprequest.get(item))
if fRequest.getcode() != 200:
log.logInfo(
price_httprequest.get(item) + " status not 200")
fRequest.close()
continue
if ("jd.com" in config_dict[item].get("URL")):
priceText = fRequest.read().decode('gbk')
price = getJDPrice(priceText)
elif ("suning.com" in config_dict[item].get("URL")):
#会将转义字符decode之后又多了\.
priceText = fRequest.read().decode()
price = getSuningPrice(priceText)
fRequest.close()
log.logInfo(goods_title[item] + ": " + price, log_file)
#为避免一天内发送多次请求，做一个历史低价策略。当出现低于历史低价时，才再发送。
low_price = history_price.get(config_dict[item].get("URL"),
1000000.0)
if float(price.strip('"')) < config_dict[item].get(
"ExpectedPrice") and float(
price.strip('"')) < low_price:
#记录在历史价格文件中
with open("history_p.txt", "a+") as f:
f.write(
timeStampToTime(time.time()) + "\t" +
config_dict[item].get("URL") + " " +
goods_title[item] + ": " + price + "\n")
history_price[config_dict[item].get("URL")] = float(
price.strip('"'))
print(price)
#发送邮件提醒
#sendmail.sendEmail(config_dict[item].get("Email"), config_dict[item].get("URL") +" " + title + ": " + price)
sendmail.sendEmailBylocal(
config_dict[item].get("Email", "*****@*****.**"),
goods_title[item], config_dict[item].get("URL") + " " +
goods_title[item] + ": " + price)
except Exception as e:
print(e)
log.logInfo("urlopen Error" + price_httprequest.get(item))
time.sleep(30 * 60) #30minutes
import os
import string
import glob
import settings
import log
import libconfig
import libclient
#### MAIN ####
print("Starting 3D Dashboard backend...")
if os.path.isfile(settings.configFile):
print("Loading configuration from file " + settings.configFile)
libconfig.loadConfig(settings.configFile)
log.start("backend")
log.logInfo('Settings log path in ' + settings.logPath)
log.logInfo('Settings clients path in ' + settings.clientsPath)
log.logInfo('Settings output path in ' + settings.outputPath)
log.logInfo('Settings output filename is ' + settings.outputJSFilename)
log.logInfo('Settings output plotly image path is ' + settings.plotlyPath)
clientList = glob.glob(settings.clientsPath + "/*.cfg")
for c in clientList:
client_process = libclient.Client()
result = client_process.processClientConfigFile(clientFile=c)
# log.logDebug(list(result))
from pathlib import Path
import plot
import draw
import load
import download
import preprocessing
import log
import os
scriptPath = os.path.dirname(os.path.realpath(__file__))
os.chdir(scriptPath)
log.logInfo("Scriptpath: " + str(scriptPath))
def main():
log.logInfo("Creating Directories")
createAllDir()
log.logInfo("Downloading corona cases")
download.downloadCoronaCases()
log.logInfo("Loading corona cases into memory")
coronaCases = load.loadCoronaCases()
coronaCasesByDay = load.loadCoronaCases("date")
log.logInfo("Downloading world popoulation")
download.downloadWorldPopulation()
log.logInfo("Loading world popoulation into memory")
population = load.loadPopulationGroupedByYear()
log.logInfo("Downloading health spending data")
def main():
log.logInfo("Creating Directories")
createAllDir()
log.logInfo("Downloading corona cases")
download.downloadCoronaCases()
log.logInfo("Loading corona cases into memory")
coronaCases = load.loadCoronaCases()
coronaCasesByDay = load.loadCoronaCases("date")
log.logInfo("Downloading world popoulation")
download.downloadWorldPopulation()
log.logInfo("Loading world popoulation into memory")
population = load.loadPopulationGroupedByYear()
log.logInfo("Downloading health spending data")
download.downloadHealthSpendingPerCapita()
log.logInfo("Loading health spending data into memory")
healthSpendingDict = load.loadHealthSpendingPerCapita()
log.logInfo("Plotting topFlop health spender")
plot.plotTopFlopHealthSpendingCoronaCases(
preprocessing.getTopFlopCountries(coronaCases, healthSpendingDict, 3),
preprocessing.extractCountryPopulationForYear(population, "2020"))
log.logInfo("Downloading country borders")
download.downloadCountryBorders()
log.logInfo("Downloading Google trends data")
download.downloadGoogleTrendsData(coronaCases.keys())
log.logInfo("Loading Google trends data into memory")
googleTrends = load.loadGoogleTrendsData()
log.logInfo("Creating Plots")
plot.plotCaseGoogleTrends(coronaCases, googleTrends)
log.logInfo("Drawing corona case maps")
preprocessing.generateCoronaCaseWorldMaps(coronaCasesByDay)
log.logInfo("Downloading Gini-Coefficient")
download.downloadGiniCoefficient()
log.logInfo("Loading Gini-Coefficient data into memory")
giniCoefficient = load.loadGiniData()
log.logInfo("Creating Gini-Coefficient Plots")
plot.plotGiniData(giniCoefficient)
log.logInfo("Creating Gini-Coefficient grouped dictionary")
newestGiniCoefficientDict = preprocessing.getNewestGiniCoefficientDict(
giniCoefficient)
log.logInfo("Creating top flop gini coef. plot")
plot.plotTopFlopGiniCoefficientOverview(newestGiniCoefficientDict)
log.logInfo("Creating gini coef map")
preprocessing.generateGiniCoefficientMap(newestGiniCoefficientDict)
log.logInfo("Creating gini-cases coef map")
preprocessing.generateGiniCoronaMap(
coronaCases, newestGiniCoefficientDict,
preprocessing.extractCountryPopulationForYear(population, "2020"))
log.logInfo("Creating health spending per capita map")
preprocessing.generateHealthSpendingMap(healthSpendingDict)
def raw_test(self):
res = 0
try:
# 执行SQL语句
sql = "insert into test(t_t)values(%s)"
sql2 = "insert into test(big_b)values(%s)"
sql3 = "insert into test(t_mt)values(%s)"
sql4 = "insert into test(big_mb)values(%s)"
# self.cursor.execute("insert into test(big_mb)values(%s)" % b"123abc")
# self.cursor.execute("insert into test(big_mb)values('%s')" % b"123abc") # 底层有bug。底层瞎转义，对引号转义有点问题。
# self.cursor.execute("insert into test(big_mb)values('b'123abc'')")
self.cursor.execute("insert into test(big_mb)values(\"b'123abc'\")"
) #注意在python语法中写sql的原生语句 要注意引号。 这里实际上插入的是字符串。
# python底层最终的sql形式 是bytes， b'insert into test(big_mb)values("b\'123abc\'")'
self.cursor.execute(
"insert into test(big_mb)values(b'0101')"
) #sql 语句 insert into test(big_mb)values(b'0101') 插入bit。
self.cursor.execute(
"insert into test(big_mb)values(0xff12)") # 插入byte。
#存入二进制数据。 这里字符串不管有没有转成bytes，底层都会帮你转bytes。
#传入的sql参数都会变成str，然后在最后执行sql语句时会转成b"" 让Mysql底层执行. execute ->cursors.query->Connection.query
#这里也会将python中的十六进制形式中 转成sql语句中的十六进制。
self.cursor.execute(
sql, pymysql.Binary("\x00\x78 123abc")
) # pymysql.Binary 没什么用，只是bytes(). 而且时无法对str进行转，pymysql.Binary中少了指定编码方式如utf-8。
self.cursor.execute(
sql, "\x00\x78 123abc中") # pymysql.Binary(b"abc123"))
self.cursor.execute(
sql, b"\x00\x78 123abc") # pymysql.Binary(b"abc123"))
self.cursor.execute(
sql2, "\x00\x78123abc中") # pymysql.Binary(b"abc123"))
self.cursor.execute(
sql2, b"\x00\x78123abc") # pymysql.Binary(b"abc123"))
self.cursor.execute(
sql3, "\x00\x78123abc中") # pymysql.Binary(b"abc123"))
self.cursor.execute(
sql3, b"\x00\x78123abc") # pymysql.Binary(b"abc123"))
self.cursor.execute(
sql4, "\x00\x78123abc中") # pymysql.Binary(b"abc123"))
self.cursor.execute(
sql4, b"\x00\x78123abc") # pymysql.Binary(b"abc123"))
data1 = b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00H\x00H\x00\x00\xff\xfe\x00+Optimized by JPEGmini 3.12.0.2\'"\xcbX\xdbj-\xdc[S\xff\xd9'
#字符串
data2 = '\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00H\x00H\x00\x00\xff\xfe\x00+Optimized by JPEGmini 3.12.0.2\'"\xcbX\xdbj-\xdc[S\xff\xd9'
self.cursor.execute(sql4, data1)
self.cursor.execute(sql4, data2)
sql5 = "insert into test(t_t)values(\"b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01'\")" #二进制形式的字符串
sql6 = "insert into test(t_t)values('\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01')" #
sql7 = "insert into test(t_t)values(\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01)" #
sql8 = "insert into test(t_t)values(\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01)" #
#(1366, "Incorrect string value: '\\xFF\\xD8\\xFF\\xE0\\x00\\x10...' for column 't_mt' at row 1")
# self.cursor.execute(sql3,data1)
self.cursor.execute(sql3, data2)
# self.cursor.execute(sql3,b"\xff\xd8\xff\xe0" ) #b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00H")
# 提交到数据库执行
self.conn.commit()
# 获取自增id
res = self.cursor.lastrowid
except Exception as e:
# 发生错误时回滚
log.logInfo(repr(e), threading.current_thread().name + ".log")
print(e)
self.conn.rollback()
return res
# Common 3D Dashboard processes
###############################################################
# Author: Manel Muñiz (initial version 30/01/2020)
# Last updated: 05/02/2020 (Manel Muñiz)
###############################################################
import commonlibClient as commonLib
import settings
import log
import os
import libconfig
#### MAIN ####
print("Starting common 3D Dashboard backend...")
if os.path.isfile(settings.configCommonFile):
print("Loading configuration from file " + settings.configCommonFile)
libconfig.loadConfig(settings.configCommonFile)
log.start("common")
log.logInfo('Settings log path in ' + settings.logPath)
log.logInfo('Settings clients path in ' + settings.clientsPath)
log.logInfo('Settings output path in ' + settings.configFile)
log.logInfo('Settings output filename is ' + settings.outputJSFilename)
configFile = str(settings.clientsPath + "common.cfg")
commonLib.processCommonFile(configFile)
