def crawl_sale_by_search(self, args): """ 根据商圈或社区爬取一页在售房源 """ search_key, page = args url_page = self.base_url + f"ershoufang/pg{page}rs{search_key}/" content = self.request_fn(url_page) soup = BeautifulSoup(content, self.bs4_parser) logging.debug('@crawl_sale_by_search: {0} - page - {1}: {2}'.format(search_key, page, url_page)) session = DBSession() for ul_tag in soup.find_all("ul", class_="sellListContent"): for item_tag in ul_tag.find_all("li"): try: info_dict = self.parse_sale_content(item_tag) logging.debug('@crawl_sale_by_search: {0} - page - {1}: {2}'.format(search_key, page, info_dict)) sale_info = SaleInfo(**info_dict) if not sale_info.house_id or not sale_info.community_id or not sale_info.district: continue session.add(sale_info) except Exception as e: session.rollback() logging.exception('@crawl_sale_by_search: {0} - page - {1}: {2}'.format(search_key, page, e)) time.sleep(3) session.commit() session.close() logging.info('@crawl_sale_by_search: {0} - page - {1} complete.'.format(search_key, page))
def crawl_community_by_district(self, args): """ 根据区县爬取一页小区信息 """ district, page = args url_page = self.base_url + f"xiaoqu/{district}/pg{page}/" content = self.request_fn(url_page) soup = BeautifulSoup(content, self.bs4_parser) logging.debug('@crawl_community_by_district: {0} - page - {1}: {2}'.format(district, page, url_page)) session = DBSession() for ul_tag in soup.find_all("ul", class_="listContent"): for item_tag in ul_tag.find_all("li"): try: info_dict = self.parse_community_content(item_tag) query = session.query(CommunityInfo).filter(CommunityInfo.id == info_dict['id']) if query.first(): query.update(info_dict) else: session.add(CommunityInfo(**info_dict)) session.commit() logging.debug('@crawl_community_by_district: {0} - page - {1}: {2}'.format(district, page, info_dict)) except Exception as e: session.rollback() logging.exception('@crawl_community_by_district: {0} - page - {1}: {2}'.format(district, page, e)) time.sleep(3) session.close() logging.info('@crawl_community_by_district: {0} - page - {1} complete.'.format(district, page))
def _save_payment_transaction(self, tx_hash, miners, amounts): amounts_str = [] for amount in amounts: amounts_str.append(str(amount)) db_session = DBSession() try: pt = PaymentTransaction() data = { "miners": miners, "amounts": amounts_str, } pt.transaction_data = json.dumps(data) pt.transaction_hash = tx_hash # 0: failed, 1: success, 2: pending pt.transaction_status(2) db_session.add(pt) db_session.commit() except Exception as e: self._logger.warning(f'save payment transaction fail! err:{e}') finally: db_session.rollback()
def DetailforEachOid(self, name, message): # Oid = 1.1 if name == 'insert': ''' Step 1. All information about input detail ''' tmpProjectName = "" tmpProjectFunc = "" tmpKernel = "" tmpCpuCoreNum = "" tmpSerialNum = "" tmpZCBM = "" tmpMemory = "" tmpCpuType = "" tmpModel = "" tmpHostName = "" tmpOS = "" tmpManufacturer = "" tmpEthInfo = {} tmpTimestamp = int(round(time.time())) Resultofbody = {} for key,value in message.items(): if key == 'Status': if value != 'Success': return dict(Status='False', msg='Message check Failed.') if type(message['Result']).__name__ == 'str': Resultofbody = changeDict().strtodict(message['Result']) else: Resultofbody = message['Result'] for keys,values in Resultofbody.items(): if keys == 'Project': for K,V in Resultofbody[keys].items(): if K == 'Name': tmpProjectName = self.changestr(V) elif K == 'Func': tmpProjectFunc = self.changestr(V) elif keys == 'HwInfo': for KK,VV in Resultofbody[keys].items(): if KK == 'Kernel': tmpKernel = self.changestr(VV) elif KK == 'CpuCoreNum': tmpCpuCoreNum = self.changestr(VV) elif KK == 'SN': tmpSerialNum = self.changestr(VV) elif KK == 'ZCBM': tmpZCBM = self.changestr(VV) elif KK == 'Memory': tmpMemory = self.changestr(VV) elif KK == 'CpuType': tmpCpuType = self.changestr(VV) elif KK == 'Model': tmpModel = self.changestr(VV) elif KK == 'HostName': tmpHostName = self.changestr(VV) elif KK == 'OS': tmpOS = self.changestr(VV) elif KK == 'Manufacturer': tmpManufacturer = self.changestr(VV) elif keys == 'EthInfo': for eachline in Resultofbody[keys]: tmpStatus = '' tmpip = '' tmpmask = '' tmpethname = '' for KKK, VVV in eachline.items(): if KKK == 'status': tmpStatus = self.changestr(VVV) elif KKK == 'ip': tmpip = self.changestr(VVV) elif KKK == 'mask': tmpmask = self.changestr(VVV) elif KKK == 'ethname': tmpethname = self.changestr(VVV) tmpEthInfo[tmpethname] = dict(status=tmpStatus, ip=tmpip, mask=tmpmask) if tmpZCBM == '': return dict(Status='False', msg='Input Server has not ZCBM.') ''' get eth detail ''' tmpEthDict = {} for key,value in tmpEthInfo.items(): if key == 'eth0': getSearchofeth = DBSession.query(Ethdetail).filter(Ethdetail.ip == value['ip'], Ethdetail.mask == value['mask']).first() if getSearchofeth: tmpEthDict['eth0'] = getSearchofeth.eid else: getethcount = DBSession.query(Ethdetail).count() getethcount = (getethcount + 1) DBSession.add(Ethdetail(getethcount,value['status'],value['ip'],value['mask'],'eth0')) tmpEthDict['eth0'] = getethcount elif key == 'eth1': getSearchofethone = DBSession.query(Ethdetail).filter(Ethdetail.ip == value['ip'], Ethdetail.mask == value['mask']).first() if getSearchofethone: tmpEthDict['eth1'] = getSearchofethone.eid else: getethcountone = DBSession.query(Ethdetail).count() getethcountone = (getethcountone + 1) DBSession.add(Ethdetail(getethcountone,value['status'],value['ip'],value['mask'],'eth1')) tmpEthDict['eth1'] = getethcountone ''' Step 2. check server information exist. ''' getSearchofHardware = DBSession.query(AssetForAgent).filter(AssetForAgent.ZCBM == tmpZCBM).first() if getSearchofHardware: try: if int(getSearchofHardware.Timestamp) < message['SendTime']: DBSession.delete(getSearchofHardware) DBSession.commit() tmpeth0 = "" tmpeth1 = "" for key,value in tmpEthDict.items(): if key == 'eth0': tmpeth0 = value elif key == 'eth1': tmpeth1 = value getCountofeth = DBSession.query(EthInfo).count() getCountofeth = (getCountofeth + 1) DBSession.add(EthInfo(getCountofeth,tmpeth0,tmpeth1,'None','None')) DBSession.add(AssetForAgent(tmpProjectName, tmpProjectFunc, tmpKernel, tmpCpuCoreNum, tmpSerialNum, tmpZCBM, tmpMemory, tmpCpuType, tmpModel, tmpHostName, tmpOS, tmpManufacturer, message['SendTime'])) DBSession.commit() getTmpid = DBSession.query(AssetForAgent).filter_by(ZCBM = tmpZCBM).first() if getTmpid: Tmpid = getTmpid.Hid else: DBSession.rollback() return dict(Status='False', msg='flush assetforagent Error.') getCountofrelation = DBSession.query(AssetidtoEid).count() getCountofrelation = int(getCountofrelation + 1) DBSession.add(AssetidtoEid(getCountofrelation, Tmpid, getCountofeth)) DBSession.commit() return dict(Status='Success') else: return dict(Status='Success', msg='Input Hostname Need not fresh.') except Exception, e: DBSession.commit() return dict(Status='False', msg=str(e)) else: try: tmpHidforinsert = "" ''' insert into table''' DBSession.add(AssetForAgent(tmpProjectName, tmpProjectFunc, tmpKernel, tmpCpuCoreNum, tmpSerialNum, tmpZCBM, tmpMemory, tmpCpuType, tmpModel, tmpHostName, tmpOS, tmpManufacturer, message['SendTime'])) DBSession.commit() getHid = DBSession.query(AssetForAgent).filter_by(ZCBM = tmpZCBM).first() if getHid: tmpHidforinsert = getHid.Hid else: DBSession.rollback() return dict(Status='False', msg='insert into assetforagent error.') for key,value in tmpEthDict.items(): if key == 'eth0': tmpeth0 = value elif key == 'eth1': tmpeth1 = value getCountofeth = DBSession.query(EthInfo).count() getCountofeth = (getCountofeth + 1) DBSession.add(EthInfo(getCountofeth,tmpeth0,tmpeth1,'None','None')) getCountofrelation = DBSession.query(AssetidtoEid).count() getCountofrelation = int(getCountofrelation + 1) DBSession.add(AssetidtoEid(getCountofrelation, tmpHidforinsert, getCountofeth)) except Exception, e: DBSession.rollback() return dict(Status='False', msg=str(e)) DBSession.commit() return dict(Status='Success')
def teardownDatabase(): DBSession.rollback() metadata.drop_all(engine)
def _save_payments_info(self, tx_receipt, miners, amounts, miner_pool_amount): db_session = DBSession() try: # update transaction status pt = db_session.query(PaymentTransaction)\ .filter_by(transaction_hash=self._web3.toHex(tx_receipt["transactionHash"])).first() pt.transaction_status(tx_receipt["status"]) db_session.add(pt) # save payments if pt.status == PaymentTransaction.SUCCESS: miner_payments = db_session.query(PaymentSummary).all() miner_round_payments = db_session.query(RoundPaymentSummary)\ .filter_by(mining_round = config.MINING_ROUND).all() payments_map = {} for payment in miner_payments: payments_map[payment.holder] = payment round_payments_map = {} for round_payment in miner_round_payments: if round_payments_map.get(round_payment.pool_name) is None: round_payments_map[round_payment.pool_name] = {} round_payments_map[round_payment.pool_name][round_payment.holder] = round_payment for miner, rewards in miner_pool_amount.items(): for pool_name, reward in rewards.items(): # save round payments rp = RoundPayment() rp.mining_round = config.MINING_ROUND rp.pool_name = pool_name rp.holder = miner.lower() rp.amount = Decimal(reward) rp.transaction_id = pt.id db_session.add(rp) # update round payment summaries pool_round_summary = round_payments_map.get(rp.pool_name) miner_round_summary = None if pool_round_summary is not None: miner_round_summary = pool_round_summary.get(rp.holder) if miner_round_summary is not None: miner_round_summary.paid_amount += rp.amount else: miner_round_summary = RoundPaymentSummary() miner_round_summary.pool_name = pool_name miner_round_summary.mining_round = rp.mining_round miner_round_summary.holder = rp.holder miner_round_summary.paid_amount = rp.amount db_session.add(miner_round_summary) for i in range(len(miners)): # save payments p = Payment() p.holder = miners[i].lower() p.amount = amounts[i] p.pay_time = datetime.datetime.utcnow() p.transaction_id = pt.id db_session.add(p) # update payment summaries payment_summary = payments_map.get(p.holder) if payment_summary is not None: payment_summary.paid_amount += p.amount else: payment_summary = PaymentSummary() payment_summary.holder = p.holder payment_summary.paid_amount = p.amount db_session.add(payment_summary) else: self._logger.warning( f"transaction not success! tx_receipt:{tx_receipt}") db_session.commit() except Exception as e: self._logger.warning(f'save payment info fail! err:{e}') # raise exception for _check_pending_transactions raise finally: db_session.rollback()
def searchfinisheddoingeventofbefore(self, startpoint): tmpArray = [] if type(startpoint).__name__ != 'int': startpoint = int(startpoint) try: getsearchfinisheddoingeventofall = DBSession.query(EventFinished).filter((EventFinished.Eid < startpoint)).order_by(desc(EventFinished.Eid)).all() if getsearchfinisheddoingeventofall: if len(getsearchfinisheddoingeventofall) > 10: for eachline in range(10): # GamePYname tmpGamePYname = EventSearch().searchGamelistAboutPYname(getsearchfinisheddoingeventofall[eachline].GameID) if tmpGamePYname['Status'] != 'Success': GamePYname = 'None' else: GamePYname = tmpGamePYname['FullName'] # OidPYname tmpOidPYname = EventSearch().searchOIDdetailinTemplate(getsearchfinisheddoingeventofall[eachline].Oid) if tmpOidPYname['Status'] != 'Success': OidPYname = 'None' else: OidPYname = tmpOidPYname['TemplateName'] # Data newData = base64Data().decode64(getsearchfinisheddoingeventofall[eachline].Data) newData = eval(newData) newData = json.dumps(newData) # Timestamp closeTime = getsearchfinisheddoingeventofall[eachline].CloseTime newclosetime = TimeBasic().timeControl(getsearchfinisheddoingeventofall[eachline].CloseTime, 5) # OccurTime occurTime = getsearchfinisheddoingeventofall[eachline].OccurTime newoccurTime = TimeBasic().timeControl(getsearchfinisheddoingeventofall[eachline].OccurTime, 5) # DealTime tmpDealTime = TimeBasic().TimeMinus(getsearchfinisheddoingeventofall[eachline].OccurTime, getsearchfinisheddoingeventofall[eachline].CloseTime) # username username = getsearchfinisheddoingeventofall[eachline].Username # close information Detail = getsearchfinisheddoingeventofall[eachline].Detail tmpArray.append(dict(occurTime = occurTime, occurPYtime = newoccurTime, DealTime = tmpDealTime, closeTime = closeTime, closePYtime = newclosetime, Username = username, CloseDetail=Detail, Eid = getsearchfinisheddoingeventofall[eachline].Eid, GameID = getsearchfinisheddoingeventofall[eachline].GameID, GamePYname = GamePYname, Oid = getsearchfinisheddoingeventofall[eachline].Oid, OidPYname = OidPYname, Data = newData)) else: for eachline in getsearchfinisheddoingeventofall: # GamePYname tmpGamePYname = EventSearch().searchGamelistAboutPYname(eachline.GameID) if tmpGamePYname['Status'] != 'Success': GamePYname = 'None' else: GamePYname = tmpGamePYname['FullName'] # OidPYname tmpOidPYname = EventSearch().searchOIDdetailinTemplate(eachline.Oid) if tmpOidPYname['Status'] != 'Success': OidPYname = 'None' else: OidPYname = tmpOidPYname['TemplateName'] # Data newData = base64Data().decode64(eachline.Data) newData = eval(newData) newData = json.dumps(newData) # Timestamp closeTime = eachline.CloseTime newclosetime = TimeBasic().timeControl(eachline.CloseTime, 5) # OccurTime occurTime = eachline.OccurTime newoccurTime = TimeBasic().timeControl(eachline.OccurTime, 5) # DealTime tmpDealTime = TimeBasic().TimeMinus(eachline.OccurTime, eachline.CloseTime) # username username = eachline.Username # close information Detail = eachline.Detail tmpArray.append(dict(occurTime = occurTime, occurPYtime = newoccurTime, DealTime = tmpDealTime, closeTime = closeTime, closePYtime = newclosetime, Username = username, CloseDetail=Detail, Eid = eachline.Eid, GameID = eachline.GameID, GamePYname = GamePYname, Oid = eachline.Oid, OidPYname = OidPYname, Data = newData)) else: return dict(Status='False', msg='MySQL could not found any thing in Eventfinshed.') except Exception, e: DBSession.rollback() return dict(Status='False', msg=str(e))
def searchdoingeventofall(self): tmpArray = [] try: getsearchofdoingeventall = DBSession.query(EventAlarmDoing).order_by(desc(EventAlarmDoing.Eid)).all() if getsearchofdoingeventall: if len(getsearchofdoingeventall) > 10: for eachline in range(10): # GamePYname tmpGamePYname = EventSearch().searchGamelistAboutPYname(getsearchofdoingeventall[eachline].GameID) if tmpGamePYname['Status'] != 'Success': GamePYname = 'None' else: GamePYname = tmpGamePYname['FullName'] # OidPYname tmpOidPYname = EventSearch().searchOIDdetailinTemplate(getsearchofdoingeventall[eachline].Oid) if tmpOidPYname['Status'] != 'Success': OidPYname = 'None' else: OidPYname = tmpOidPYname['TemplateName'] # Data newData = base64Data().decode64(getsearchofdoingeventall[eachline].Data) newData = eval(newData) newData = json.dumps(newData) # EventName newEventName = base64Data().decode64(getsearchofdoingeventall[eachline].EventName) # opTime tmpOpTime = getsearchofdoingeventall[eachline].Timestamp tmpOpPYTime = TimeBasic().timeControl(tmpOpTime, 5) # OccurTime tmpOccurTime = getsearchofdoingeventall[eachline].OccurTime tmpOccurPYTime = TimeBasic().timeControl(tmpOccurTime, 5) # ToUser tmpUser = EventTransportExpand().searcheventindesigntoother(getsearchofdoingeventall[eachline].Eid) if tmpUser['Status'] == 'Success': tmpOpuser = tmpUser['ToUser'] else: tmpOpuser = '******' tmpArray.append(dict(OperationTime = tmpOpTime, OperationPYTime = tmpOpPYTime, OccurTime = tmpOccurTime, OccurPYTime = tmpOccurPYTime, opUser = tmpOpuser, Eid = getsearchofdoingeventall[eachline].Eid, GameID = getsearchofdoingeventall[eachline].GameID, GamePYname = GamePYname, Oid = getsearchofdoingeventall[eachline].Oid, OidPYname = OidPYname, eventGrade = getsearchofdoingeventall[eachline].eventGrade, Data = newData, EventName = newEventName)) else: for eachline in getsearchofdoingeventall: # GamePYname tmpGamePYname = EventSearch().searchGamelistAboutPYname(eachline.GameID) if tmpGamePYname['Status'] != 'Success': GamePYname = 'None' else: GamePYname = tmpGamePYname['FullName'] # OidPYname tmpOidPYname = EventSearch().searchOIDdetailinTemplate(eachline.Oid) if tmpOidPYname['Status'] != 'Success': OidPYname = 'None' else: OidPYname = tmpOidPYname['TemplateName'] # Data newData = base64Data().decode64(eachline.Data) newData = eval(newData) newData = json.dumps(newData) # EventName newEventName = base64Data().decode64(eachline.EventName) # opTime tmpOpTime = eachline.Timestamp tmpOpPYTime = TimeBasic().timeControl(tmpOpTime, 5) # OccurTime tmpOccurTime = eachline.OccurTime tmpOccurPYTime = TimeBasic().timeControl(tmpOccurTime, 5) # ToUser tmpUser = EventTransportExpand().searcheventindesigntoother(eachline.Eid) if tmpUser['Status'] == 'Success': tmpOpuser = tmpUser['ToUser'] else: tmpOpuser = '******' tmpArray.append(dict(OperationTime = tmpOpTime, OperationPYTime = tmpOpPYTime, OccurTime = tmpOccurTime, OccurPYTime = tmpOccurPYTime, opUser = tmpOpuser, Eid = eachline.Eid, GameID = eachline.GameID, GamePYname = GamePYname, Oid = eachline.Oid, OidPYname = OidPYname, eventGrade = eachline.eventGrade, Data = newData, EventName = newEventName)) else: return dict(Status='False', msg='MySQL could not found any thing in Eventalarmdoing.') except Exception, e: DBSession.rollback() return dict(Status='False', msg=str(e))