def crawl_sale_by_search(self, args):
""" 根据商圈或社区爬取一页在售房源 """
search_key, page = args
url_page = self.base_url + f"ershoufang/pg{page}rs{search_key}/"
content = self.request_fn(url_page)
soup = BeautifulSoup(content, self.bs4_parser)
logging.debug('@crawl_sale_by_search: {0} - page - {1}: {2}'.format(search_key, page, url_page))
session = DBSession()
for ul_tag in soup.find_all("ul", class_="sellListContent"):
for item_tag in ul_tag.find_all("li"):
try:
info_dict = self.parse_sale_content(item_tag)
logging.debug('@crawl_sale_by_search: {0} - page - {1}: {2}'.format(search_key, page, info_dict))
sale_info = SaleInfo(**info_dict)
if not sale_info.house_id or not sale_info.community_id or not sale_info.district:
continue
session.add(sale_info)
except Exception as e:
session.rollback()
logging.exception('@crawl_sale_by_search: {0} - page - {1}: {2}'.format(search_key, page, e))
time.sleep(3)
session.commit()
session.close()
logging.info('@crawl_sale_by_search: {0} - page - {1} complete.'.format(search_key, page))
def crawl_community_by_district(self, args):
""" 根据区县爬取一页小区信息 """
district, page = args
url_page = self.base_url + f"xiaoqu/{district}/pg{page}/"
content = self.request_fn(url_page)
soup = BeautifulSoup(content, self.bs4_parser)
logging.debug('@crawl_community_by_district: {0} - page - {1}: {2}'.format(district, page, url_page))
session = DBSession()
for ul_tag in soup.find_all("ul", class_="listContent"):
for item_tag in ul_tag.find_all("li"):
try:
info_dict = self.parse_community_content(item_tag)
query = session.query(CommunityInfo).filter(CommunityInfo.id == info_dict['id'])
if query.first():
query.update(info_dict)
else:
session.add(CommunityInfo(**info_dict))
session.commit()
logging.debug('@crawl_community_by_district: {0} - page - {1}: {2}'.format(district, page, info_dict))
except Exception as e:
session.rollback()
logging.exception('@crawl_community_by_district: {0} - page - {1}: {2}'.format(district, page, e))
time.sleep(3)
session.close()
logging.info('@crawl_community_by_district: {0} - page - {1} complete.'.format(district, page))
def _save_payment_transaction(self, tx_hash, miners, amounts):
amounts_str = []
for amount in amounts:
amounts_str.append(str(amount))
db_session = DBSession()
try:
pt = PaymentTransaction()
data = {
"miners": miners,
"amounts": amounts_str,
}
pt.transaction_data = json.dumps(data)
pt.transaction_hash = tx_hash
# 0: failed, 1: success, 2: pending
pt.transaction_status(2)
db_session.add(pt)
db_session.commit()
except Exception as e:
self._logger.warning(f'save payment transaction fail! err:{e}')
finally:
db_session.rollback()
def DetailforEachOid(self, name, message):
# Oid = 1.1
if name == 'insert':
''' Step 1. All information about input detail '''
tmpProjectName = ""
tmpProjectFunc = ""
tmpKernel = ""
tmpCpuCoreNum = ""
tmpSerialNum = ""
tmpZCBM = ""
tmpMemory = ""
tmpCpuType = ""
tmpModel = ""
tmpHostName = ""
tmpOS = ""
tmpManufacturer = ""
tmpEthInfo = {}
tmpTimestamp = int(round(time.time()))
Resultofbody = {}
for key,value in message.items():
if key == 'Status':
if value != 'Success':
return dict(Status='False', msg='Message check Failed.')
if type(message['Result']).__name__ == 'str':
Resultofbody = changeDict().strtodict(message['Result'])
else:
Resultofbody = message['Result']
for keys,values in Resultofbody.items():
if keys == 'Project':
for K,V in Resultofbody[keys].items():
if K == 'Name':
tmpProjectName = self.changestr(V)
elif K == 'Func':
tmpProjectFunc = self.changestr(V)
elif keys == 'HwInfo':
for KK,VV in Resultofbody[keys].items():
if KK == 'Kernel':
tmpKernel = self.changestr(VV)
elif KK == 'CpuCoreNum':
tmpCpuCoreNum = self.changestr(VV)
elif KK == 'SN':
tmpSerialNum = self.changestr(VV)
elif KK == 'ZCBM':
tmpZCBM = self.changestr(VV)
elif KK == 'Memory':
tmpMemory = self.changestr(VV)
elif KK == 'CpuType':
tmpCpuType = self.changestr(VV)
elif KK == 'Model':
tmpModel = self.changestr(VV)
elif KK == 'HostName':
tmpHostName = self.changestr(VV)
elif KK == 'OS':
tmpOS = self.changestr(VV)
elif KK == 'Manufacturer':
tmpManufacturer = self.changestr(VV)
elif keys == 'EthInfo':
for eachline in Resultofbody[keys]:
tmpStatus = ''
tmpip = ''
tmpmask = ''
tmpethname = ''
for KKK, VVV in eachline.items():
if KKK == 'status':
tmpStatus = self.changestr(VVV)
elif KKK == 'ip':
tmpip = self.changestr(VVV)
elif KKK == 'mask':
tmpmask = self.changestr(VVV)
elif KKK == 'ethname':
tmpethname = self.changestr(VVV)
tmpEthInfo[tmpethname] = dict(status=tmpStatus, ip=tmpip, mask=tmpmask)
if tmpZCBM == '':
return dict(Status='False', msg='Input Server has not ZCBM.')
''' get eth detail '''
tmpEthDict = {}
for key,value in tmpEthInfo.items():
if key == 'eth0':
getSearchofeth = DBSession.query(Ethdetail).filter(Ethdetail.ip == value['ip'], Ethdetail.mask == value['mask']).first()
if getSearchofeth:
tmpEthDict['eth0'] = getSearchofeth.eid
else:
getethcount = DBSession.query(Ethdetail).count()
getethcount = (getethcount + 1)
DBSession.add(Ethdetail(getethcount,value['status'],value['ip'],value['mask'],'eth0'))
tmpEthDict['eth0'] = getethcount
elif key == 'eth1':
getSearchofethone = DBSession.query(Ethdetail).filter(Ethdetail.ip == value['ip'], Ethdetail.mask == value['mask']).first()
if getSearchofethone:
tmpEthDict['eth1'] = getSearchofethone.eid
else:
getethcountone = DBSession.query(Ethdetail).count()
getethcountone = (getethcountone + 1)
DBSession.add(Ethdetail(getethcountone,value['status'],value['ip'],value['mask'],'eth1'))
tmpEthDict['eth1'] = getethcountone
''' Step 2. check server information exist. '''
getSearchofHardware = DBSession.query(AssetForAgent).filter(AssetForAgent.ZCBM == tmpZCBM).first()
if getSearchofHardware:
try:
if int(getSearchofHardware.Timestamp) < message['SendTime']:
DBSession.delete(getSearchofHardware)
DBSession.commit()
tmpeth0 = ""
tmpeth1 = ""
for key,value in tmpEthDict.items():
if key == 'eth0':
tmpeth0 = value
elif key == 'eth1':
tmpeth1 = value
getCountofeth = DBSession.query(EthInfo).count()
getCountofeth = (getCountofeth + 1)
DBSession.add(EthInfo(getCountofeth,tmpeth0,tmpeth1,'None','None'))
DBSession.add(AssetForAgent(tmpProjectName, tmpProjectFunc, tmpKernel, tmpCpuCoreNum, tmpSerialNum, tmpZCBM, tmpMemory, tmpCpuType, tmpModel, tmpHostName, tmpOS, tmpManufacturer, message['SendTime']))
DBSession.commit()
getTmpid = DBSession.query(AssetForAgent).filter_by(ZCBM = tmpZCBM).first()
if getTmpid:
Tmpid = getTmpid.Hid
else:
DBSession.rollback()
return dict(Status='False', msg='flush assetforagent Error.')
getCountofrelation = DBSession.query(AssetidtoEid).count()
getCountofrelation = int(getCountofrelation + 1)
DBSession.add(AssetidtoEid(getCountofrelation, Tmpid, getCountofeth))
DBSession.commit()
return dict(Status='Success')
else:
return dict(Status='Success', msg='Input Hostname Need not fresh.')
except Exception, e:
DBSession.commit()
return dict(Status='False', msg=str(e))
else:
try:
tmpHidforinsert = ""
''' insert into table'''
DBSession.add(AssetForAgent(tmpProjectName, tmpProjectFunc, tmpKernel, tmpCpuCoreNum, tmpSerialNum, tmpZCBM, tmpMemory, tmpCpuType, tmpModel, tmpHostName, tmpOS, tmpManufacturer, message['SendTime']))
DBSession.commit()
getHid = DBSession.query(AssetForAgent).filter_by(ZCBM = tmpZCBM).first()
if getHid:
tmpHidforinsert = getHid.Hid
else:
DBSession.rollback()
return dict(Status='False', msg='insert into assetforagent error.')
for key,value in tmpEthDict.items():
if key == 'eth0':
tmpeth0 = value
elif key == 'eth1':
tmpeth1 = value
getCountofeth = DBSession.query(EthInfo).count()
getCountofeth = (getCountofeth + 1)
DBSession.add(EthInfo(getCountofeth,tmpeth0,tmpeth1,'None','None'))
getCountofrelation = DBSession.query(AssetidtoEid).count()
getCountofrelation = int(getCountofrelation + 1)
DBSession.add(AssetidtoEid(getCountofrelation, tmpHidforinsert, getCountofeth))
except Exception, e:
DBSession.rollback()
return dict(Status='False', msg=str(e))
DBSession.commit()
return dict(Status='Success')
def teardownDatabase():
DBSession.rollback()
metadata.drop_all(engine)
def _save_payments_info(self, tx_receipt, miners, amounts, miner_pool_amount):
db_session = DBSession()
try:
# update transaction status
pt = db_session.query(PaymentTransaction)\
.filter_by(transaction_hash=self._web3.toHex(tx_receipt["transactionHash"])).first()
pt.transaction_status(tx_receipt["status"])
db_session.add(pt)
# save payments
if pt.status == PaymentTransaction.SUCCESS:
miner_payments = db_session.query(PaymentSummary).all()
miner_round_payments = db_session.query(RoundPaymentSummary)\
.filter_by(mining_round = config.MINING_ROUND).all()
payments_map = {}
for payment in miner_payments:
payments_map[payment.holder] = payment
round_payments_map = {}
for round_payment in miner_round_payments:
if round_payments_map.get(round_payment.pool_name) is None:
round_payments_map[round_payment.pool_name] = {}
round_payments_map[round_payment.pool_name][round_payment.holder] = round_payment
for miner, rewards in miner_pool_amount.items():
for pool_name, reward in rewards.items():
# save round payments
rp = RoundPayment()
rp.mining_round = config.MINING_ROUND
rp.pool_name = pool_name
rp.holder = miner.lower()
rp.amount = Decimal(reward)
rp.transaction_id = pt.id
db_session.add(rp)
# update round payment summaries
pool_round_summary = round_payments_map.get(rp.pool_name)
miner_round_summary = None
if pool_round_summary is not None:
miner_round_summary = pool_round_summary.get(rp.holder)
if miner_round_summary is not None:
miner_round_summary.paid_amount += rp.amount
else:
miner_round_summary = RoundPaymentSummary()
miner_round_summary.pool_name = pool_name
miner_round_summary.mining_round = rp.mining_round
miner_round_summary.holder = rp.holder
miner_round_summary.paid_amount = rp.amount
db_session.add(miner_round_summary)
for i in range(len(miners)):
# save payments
p = Payment()
p.holder = miners[i].lower()
p.amount = amounts[i]
p.pay_time = datetime.datetime.utcnow()
p.transaction_id = pt.id
db_session.add(p)
# update payment summaries
payment_summary = payments_map.get(p.holder)
if payment_summary is not None:
payment_summary.paid_amount += p.amount
else:
payment_summary = PaymentSummary()
payment_summary.holder = p.holder
payment_summary.paid_amount = p.amount
db_session.add(payment_summary)
else:
self._logger.warning(
f"transaction not success! tx_receipt:{tx_receipt}")
db_session.commit()
except Exception as e:
self._logger.warning(f'save payment info fail! err:{e}')
# raise exception for _check_pending_transactions
raise
finally:
db_session.rollback()
def searchfinisheddoingeventofbefore(self, startpoint):
tmpArray = []
if type(startpoint).__name__ != 'int':
startpoint = int(startpoint)
try:
getsearchfinisheddoingeventofall = DBSession.query(EventFinished).filter((EventFinished.Eid < startpoint)).order_by(desc(EventFinished.Eid)).all()
if getsearchfinisheddoingeventofall:
if len(getsearchfinisheddoingeventofall) > 10:
for eachline in range(10):
# GamePYname
tmpGamePYname = EventSearch().searchGamelistAboutPYname(getsearchfinisheddoingeventofall[eachline].GameID)
if tmpGamePYname['Status'] != 'Success':
GamePYname = 'None'
else:
GamePYname = tmpGamePYname['FullName']
# OidPYname
tmpOidPYname = EventSearch().searchOIDdetailinTemplate(getsearchfinisheddoingeventofall[eachline].Oid)
if tmpOidPYname['Status'] != 'Success':
OidPYname = 'None'
else:
OidPYname = tmpOidPYname['TemplateName']
# Data
newData = base64Data().decode64(getsearchfinisheddoingeventofall[eachline].Data)
newData = eval(newData)
newData = json.dumps(newData)
# Timestamp
closeTime = getsearchfinisheddoingeventofall[eachline].CloseTime
newclosetime = TimeBasic().timeControl(getsearchfinisheddoingeventofall[eachline].CloseTime, 5)
# OccurTime
occurTime = getsearchfinisheddoingeventofall[eachline].OccurTime
newoccurTime = TimeBasic().timeControl(getsearchfinisheddoingeventofall[eachline].OccurTime, 5)
# DealTime
tmpDealTime = TimeBasic().TimeMinus(getsearchfinisheddoingeventofall[eachline].OccurTime, getsearchfinisheddoingeventofall[eachline].CloseTime)
# username
username = getsearchfinisheddoingeventofall[eachline].Username
# close information
Detail = getsearchfinisheddoingeventofall[eachline].Detail
tmpArray.append(dict(occurTime = occurTime, occurPYtime = newoccurTime, DealTime = tmpDealTime, closeTime = closeTime, closePYtime = newclosetime, Username = username, CloseDetail=Detail, Eid = getsearchfinisheddoingeventofall[eachline].Eid, GameID = getsearchfinisheddoingeventofall[eachline].GameID, GamePYname = GamePYname, Oid = getsearchfinisheddoingeventofall[eachline].Oid, OidPYname = OidPYname, Data = newData))
else:
for eachline in getsearchfinisheddoingeventofall:
# GamePYname
tmpGamePYname = EventSearch().searchGamelistAboutPYname(eachline.GameID)
if tmpGamePYname['Status'] != 'Success':
GamePYname = 'None'
else:
GamePYname = tmpGamePYname['FullName']
# OidPYname
tmpOidPYname = EventSearch().searchOIDdetailinTemplate(eachline.Oid)
if tmpOidPYname['Status'] != 'Success':
OidPYname = 'None'
else:
OidPYname = tmpOidPYname['TemplateName']
# Data
newData = base64Data().decode64(eachline.Data)
newData = eval(newData)
newData = json.dumps(newData)
# Timestamp
closeTime = eachline.CloseTime
newclosetime = TimeBasic().timeControl(eachline.CloseTime, 5)
# OccurTime
occurTime = eachline.OccurTime
newoccurTime = TimeBasic().timeControl(eachline.OccurTime, 5)
# DealTime
tmpDealTime = TimeBasic().TimeMinus(eachline.OccurTime, eachline.CloseTime)
# username
username = eachline.Username
# close information
Detail = eachline.Detail
tmpArray.append(dict(occurTime = occurTime, occurPYtime = newoccurTime, DealTime = tmpDealTime, closeTime = closeTime, closePYtime = newclosetime, Username = username, CloseDetail=Detail, Eid = eachline.Eid, GameID = eachline.GameID, GamePYname = GamePYname, Oid = eachline.Oid, OidPYname = OidPYname, Data = newData))
else:
return dict(Status='False', msg='MySQL could not found any thing in Eventfinshed.')
except Exception, e:
DBSession.rollback()
return dict(Status='False', msg=str(e))
def searchdoingeventofall(self):
tmpArray = []
try:
getsearchofdoingeventall = DBSession.query(EventAlarmDoing).order_by(desc(EventAlarmDoing.Eid)).all()
if getsearchofdoingeventall:
if len(getsearchofdoingeventall) > 10:
for eachline in range(10):
# GamePYname
tmpGamePYname = EventSearch().searchGamelistAboutPYname(getsearchofdoingeventall[eachline].GameID)
if tmpGamePYname['Status'] != 'Success':
GamePYname = 'None'
else:
GamePYname = tmpGamePYname['FullName']
# OidPYname
tmpOidPYname = EventSearch().searchOIDdetailinTemplate(getsearchofdoingeventall[eachline].Oid)
if tmpOidPYname['Status'] != 'Success':
OidPYname = 'None'
else:
OidPYname = tmpOidPYname['TemplateName']
# Data
newData = base64Data().decode64(getsearchofdoingeventall[eachline].Data)
newData = eval(newData)
newData = json.dumps(newData)
# EventName
newEventName = base64Data().decode64(getsearchofdoingeventall[eachline].EventName)
# opTime
tmpOpTime = getsearchofdoingeventall[eachline].Timestamp
tmpOpPYTime = TimeBasic().timeControl(tmpOpTime, 5)
# OccurTime
tmpOccurTime = getsearchofdoingeventall[eachline].OccurTime
tmpOccurPYTime = TimeBasic().timeControl(tmpOccurTime, 5)
# ToUser
tmpUser = EventTransportExpand().searcheventindesigntoother(getsearchofdoingeventall[eachline].Eid)
if tmpUser['Status'] == 'Success':
tmpOpuser = tmpUser['ToUser']
else:
tmpOpuser = '******'
tmpArray.append(dict(OperationTime = tmpOpTime, OperationPYTime = tmpOpPYTime, OccurTime = tmpOccurTime, OccurPYTime = tmpOccurPYTime, opUser = tmpOpuser, Eid = getsearchofdoingeventall[eachline].Eid, GameID = getsearchofdoingeventall[eachline].GameID, GamePYname = GamePYname, Oid = getsearchofdoingeventall[eachline].Oid, OidPYname = OidPYname, eventGrade = getsearchofdoingeventall[eachline].eventGrade, Data = newData, EventName = newEventName))
else:
for eachline in getsearchofdoingeventall:
# GamePYname
tmpGamePYname = EventSearch().searchGamelistAboutPYname(eachline.GameID)
if tmpGamePYname['Status'] != 'Success':
GamePYname = 'None'
else:
GamePYname = tmpGamePYname['FullName']
# OidPYname
tmpOidPYname = EventSearch().searchOIDdetailinTemplate(eachline.Oid)
if tmpOidPYname['Status'] != 'Success':
OidPYname = 'None'
else:
OidPYname = tmpOidPYname['TemplateName']
# Data
newData = base64Data().decode64(eachline.Data)
newData = eval(newData)
newData = json.dumps(newData)
# EventName
newEventName = base64Data().decode64(eachline.EventName)
# opTime
tmpOpTime = eachline.Timestamp
tmpOpPYTime = TimeBasic().timeControl(tmpOpTime, 5)
# OccurTime
tmpOccurTime = eachline.OccurTime
tmpOccurPYTime = TimeBasic().timeControl(tmpOccurTime, 5)
# ToUser
tmpUser = EventTransportExpand().searcheventindesigntoother(eachline.Eid)
if tmpUser['Status'] == 'Success':
tmpOpuser = tmpUser['ToUser']
else:
tmpOpuser = '******'
tmpArray.append(dict(OperationTime = tmpOpTime, OperationPYTime = tmpOpPYTime, OccurTime = tmpOccurTime, OccurPYTime = tmpOccurPYTime, opUser = tmpOpuser, Eid = eachline.Eid, GameID = eachline.GameID, GamePYname = GamePYname, Oid = eachline.Oid, OidPYname = OidPYname, eventGrade = eachline.eventGrade, Data = newData, EventName = newEventName))
else:
return dict(Status='False', msg='MySQL could not found any thing in Eventalarmdoing.')
except Exception, e:
DBSession.rollback()
return dict(Status='False', msg=str(e))
def teardownDatabase():
DBSession.rollback()
metadata.drop_all(engine)
