def call_auction(self, auction_id): auction = Auction.one(auction_id) if auction is None: return { "status": "error", "errors": ["This auction does not exist."], } return auction.end()
def query_all_bids_for_auction(self, auction_id): auction = Auction.one(auction_id) if auction is None: return { "status": "error", "errors": ["This auction does not exist."], } return auction.get_all_submitted_bids()
def submit_bid_for_auction(self, auction_id, bid_price): auction = Auction.one(auction_id) if auction is None: return { "status": "error", "errors": ["This auction does not exist."], } return auction.process_bid(bid_price, self.id)
def query_all_live_auctions(self, involved_only=False): def filter_func(auc_record): return auc_record.get("status") == const.AUCTION_STATUS_IN_PROGRESS auctions = Auction.all(filter_func) if involved_only: bids = Bid.all(lambda x: x["participant_id"] == self.id) involved_auction_ids = set([b.auction_id for b in bids]) auctions = [a for a in auctions if a.id in involved_auction_ids] return auctions
def create_auction(self, item_name): item = Item.one(item_name) if item is None: return { "status": "error", "errors": ["Item does not exist."], } def filter_func(auc_record): return auc_record.get("status") != const.AUCTION_STATUS_CALLED_FAIL if Auction.all(filter_func): return { "status": "error", "errors": ["Auction already exists for this item."], } auction = Auction(item_name) auction.save() return { "status": "success", "auction_id": auction.id, }
def unstage_item_from_auction(self, item_name): item = Item.one(item_name) if item is None: return { "status": "error", "errors": ["Item does not exist."], } if item.status != const.ITEM_STATUS_STAGED: return { "status": "error", "errors": ["Item is not staged for auction."], } def filter_func(auc_record): item_name_met = auc_record.get("item_name") == item_name auc_status_met = auc_record.get("status") == const.AUCTION_STATUS_CREATED return item_name_met and auc_status_met ret = Auction.all(filter_func) if ret: auction = ret[0] auction.delete() else: return { "status": "error", "errors": [ "Auction for this item already progressed, and cannot " "unstage the item at this time." ], } item.status = const.ITEM_STATUS_AVAILABLE item.save() return { "status": "success", "item_name": item.name, }
def _runAuction(self, gteeOfferPrice, gteeBidPrice, haveQ): """ Clears an auction to determine the quantity and price for each offer/bid. """ pOffers = [offer for offer in self.offers if not offer.reactive] pBids = [bid for bid in self.bids if not bid.reactive] # Clear offer/bid quantities and prices. auction = Auction(self.case, pOffers, pBids, self.auctionType, gteeOfferPrice, gteeBidPrice, self.limits) auction.run() # Separate auction for reactive power. if haveQ: qOffers = [offer for offer in self.offers if offer.reactive] qBids = [bid for bid in self.bids if bid.reactive] # Too complicated to scale with mixed bids/offers (only # auction_types LAO and FIRST_PRICE allowed) qAuction = Auction(self.case, qOffers, qBids, self.auctionType, gteeOfferPrice, gteeBidPrice, self.limits) qAuction.run()
def demo_auction(): """Run a random auction. Stop after max_bids bids.""" # number of bids to accept max_bids = 10 auction = Auction("Vacation to Ko Samui") auction.start() print("Starting", auction, "with min bidding increment", auction.increment) #auction.bid("initial", 20) for n in range(0,max_bids): bidder = get_bidder(auction) amount = int(auction.best_bid() + 10*random.randint(-2,5) + 10) print_and_bid(auction,bidder,amount) # pause the auction if n == max_bids/2: print(">>> stop()") auction.stop() elif n == max_bids/2 + 1: print(">>> start()") auction.start() print("The winner is", auction.winner(), "with a bid of", auction.best_bid())
def detail_parse(self, **kwargs): auction = Auction(source='ali') auction.auction_name = kwargs['auction_name'] auction.auctionId = kwargs['auction_id'] auction.biddingState = kwargs['status'] auction.curPrice = kwargs['current_price'] auction.evalPrice = kwargs['evalprice'] auction.startShootingDate = kwargs['start_time'] auction.endShootingDate = kwargs['end_time'] auction.participantsNumber = kwargs['partnumber'] auction.visitCount = kwargs['visitCount'] detail_url = kwargs['url'] auction.url = detail_url try: # todo 调用黄村粮的方法进行切割获取城市,区域,小区名,经纬度等 cut_info = CutMatch.to_match('上海', kwargs['auction_name']) # 切割后匹配库中的城市 auction.matchCity = cut_info['matchCity'] # 切割后匹配库中的区域 auction.matchRegion = cut_info['matchRegion'] # 切割后匹配库中的小区名称 auction.matchName = cut_info['matchName'] # 切割后匹配库中的地址 auction.matchAddress = cut_info['matchAddress'] # 切割后的房号 auction.roomNum = cut_info['cutRoomnum'] # 切割后的楼号 auction.houseNum = cut_info['cutHousenum'] # 切割后的城市 auction.cutCity = cut_info['cutCity'] # 切割后的区域 auction.cutRegion = cut_info['cutRegion'] # 切割后的小区名称 auction.cutName = cut_info['cutName'] # 切割后的地址 auction.cutAddress = cut_info['cutAddress'] # 切割后跑高德接口得到的经纬度 auction.lat = cut_info['mapLat'] auction.lng = cut_info['mapLng'] except Exception as e: log.error(e) return try: detail_res = requests.get(url=detail_url, headers=self.headers) except Exception as e: log.error('url={}, e={}'.format(detail_url, e)) return html = etree.HTML(detail_res.content.decode('gbk')) try: title = html.xpath( '//div[contains(@class,"pm-main clearfix")]/h1/text()' )[0].strip() except: log.error('没有标题 url={}'.format(detail_url)) return auctionStage = re.search('【(.*?)】', title).group(1) auction.auctionStage = auctionStage auction.auctionCount = self.get_auctionCount(auctionStage) startPrice = re.search('起拍价¥(.*?) ,', detail_res.content.decode('gbk')).group(1) bond = re.search('保 证 金.*?J_Price">(.*?)</span', detail_res.content.decode('gbk'), re.S | re.M).group(1) comm_url = 'http://sf.taobao.com/json/getGovItemSummary.htm?itemId={}'.format( kwargs['auction_id']) res = requests.get(comm_url, headers=self.headers) try: auction.area = float( int(res.json()['props']['area']['value']) / 100) except: pass images = html.xpath("//div[@class='pm-pic pm-s80 ']/a/img/@src") image_list = [] for image_url in images: big_img = image_url.replace('_80x80.jpg', '') image = qiniufetch(big_img, big_img) image_list.append(image) auction.houseImgUrls = image_list auction.startPrice = float(float(startPrice.replace(',', '')) / 10000) auction.bond = float(float(bond.replace(',', '').strip()) / 10000) if kwargs['status'] == '已成交': if re.search( '失败|流拍', html.xpath('//h1[@class="bid-fail"]/text()')[0]) is None: auction.update() else: auction.biddingState = '流拍' auction.update() else: auction.update()
def detail_parse(auction_res, auction_type, html_type, auction_id): con = auction_res.json() auction = Auction(source=source, auction_type=auction_type) auction.source_html = con auction.html_type = html_type auction.auction_id = auction_id auction.auction_name = con['object_title'] auction.start_auction_price = con['start_price'] auction.assess_value = con['appraise_price'] auction.earnest_money = con['bond_price'] auction.court = con['court_name'] auction_time = con['start_time'] location = con['location'] auction.auction_time = datetime.datetime.strptime( auction_time, "%Y-%m-%d %H:%M:%S") province, city, region = location.split(' ') auction.province = province auction.city = city auction.region = region if html_type == '房产': auction.floor = con['detail']['house_floor'] auction.area = con['detail']['gross_floor_area'] elif html_type == '土地': auction.area = con['detail']['l_land_area'] auction.insert_db()
from auction import Input from auction import Auction import util auction_parameters = Input() test = Auction(auction_parameters) market_prices, buyer_profits, seller_profits = test.run() market_prices, round_avgs, seller_avgs, avg_buyer_profit_per_round, avg_seller_profit_per_round, avg_market_price = util.process_data( market_prices, auction_parameters, buyer_profits, seller_profits) print(f"Market Prices:") for i, round_outcome in enumerate(market_prices): print(f"round {i}: {round_outcome}") print() print( f"Buyer profits for all {auction_parameters.num_round} rounds: \n{buyer_profits}\n" f"AVG Buyer profits per round: \n{avg_buyer_profit_per_round}\n" f"Seller profits for all {auction_parameters.num_round} rounds: \n{seller_profits}\n" f"AVG Seller profits per round: {avg_seller_profit_per_round}\n" f"AVG market price: {avg_market_price}\n")
def query_all_auctions(self, status_code=None): auctions = Auction.all() if status_code: auctions = [a for a in auctions if a.status == status_code] return auctions
def crawler_detail_page(self, auction_id, province_name, city_name, type_name, auction_type): detail_url = 'http://www.chinesesfpm.com/index/index/info/biao_id/' + auction_id res = requests.get(detail_url) tree = etree.HTML(res.text) a = Auction(source=source, auction_type=auction_type) a.auction_id = auction_id a.auction_name = tree.xpath( '/html/body/div/div[6]/div/div[2]/div[1]/div[1]/text()')[0] a.html_type = type_name auction_time = tree.xpath( '/html/body/div/div[6]/div/div[2]/div[1]/div[2]/div[2]/div[2]/text()' )[0] auction_time_ = re.search('开始时间: (.*?)$', auction_time, re.S | re.M).group(1) a.auction_time = datetime.datetime.strptime(auction_time_, "%Y年%m月%d日 %H时%M分%S秒") a.province = province_name a.city = city_name a.info = [ tree.xpath('string(//*[@id="f4"])'), tree.xpath('string(//*[@id="f6"])') ] start_auction_price = \ tree.xpath('/html/body/div/div[6]/div/div[2]/div[1]/div[2]/div[2]/div[5]/div[1]/em[3]/text()')[0] s = start_auction_price.encode('utf-8').decode() a.start_auction_price = float( re.search('起拍价: ¥(.*)', s, re.S | re.M).group(1)) court = tree.xpath( '/html/body/div/div[6]/div/div[2]/div[1]/div[2]/div[2]/div[5]/div[2]/em[1]/text()' )[0] a.court = re.search('拍卖机构:(.*)', court, re.S | re.M).group(1) a.source_html = res.text a.insert_db()
def test_new_bid(self, tell): product = Auction().new_product(product_id=189, initial_bid=1000, step=50, duration=5) Auction().product_chat_request(product.vk_product_id, 213) obj = MessageProcessor().process_object( 'market_comment_new', { "id": 3, "date": 1508607991, "text": "#bid 1000", "from_id": 213, "item_id": 189, "market_owner_id": -155 }) self.assertEqual(obj.amount, 1000) obj = MessageProcessor().process_object('message_new', { "id": 47, "out": 0, "body": "#bid 2000", "user_id": 213 }) self.assertEqual(obj.amount, 2000) obj = MessageProcessor().process_object( 'message_new', { "id": 6, "user_id": 213, "body": "PRIVET", "attachments": [{ "type": "link", "link": { "url": "https://m.vk.com/landings/moneysend", "title": "3000 ?4??4??1?.", "caption": "Денежный перевод", } }], "group_id": 155403696 }) self.assertEqual(obj.amount, 3000) obj = MessageProcessor().process_object( 'market_comment_new', { "id": 3, "date": 1508607991, "text": "#bid 123", "from_id": 213, "item_id": 189, "market_owner_id": -155 }) self.assertFalse(obj) obj = MessageProcessor().process_object( 'market_comment_new', { "id": 3, "date": 1508607991, "text": "#bid 123000", "from_id": 213, "item_id": 189, "market_owner_id": -155 }) self.assertTrue(obj)
def get_detail(self, aution_url, aution_id, aution_time, region_name, city_name, html_type, auction_type): info = [] aution = Auction(source, auction_type) response = requests.get(aution_url, headers=self.headers) try: html = response.text tree = etree.HTML(html) aution.auction_id = aution_id aution.region = region_name aution.city = city_name aution.source_html = html aution.html_type = html_type try: aution.start_auction_price = float( tree.xpath('//*[@id="Price_Start"]/text()')[0].replace( ',', '')) except Exception as e: aution.start_auction_price = None if 'item2' in aution_url: aution.auction_name = tree.xpath( '//div[@class="d-m-title"]/b/text()')[0] aution.auction_level = tree.xpath( '//div[@class="d-m-tb"]/table[1]/tr[1]/td[2]/text()')[0] try: assess_value = tree.xpath( '//div[@class="d-m-tb"]/table[1]/tr[4]/td[1]/text()' )[0] aution.assess_value = float( re.search('(\d+),?(\d+)', assess_value, re.S | re.M).group(1).replace(',', '')) except Exception as e: aution.assess_value = None earnest_money = tree.xpath( '//div[@class="d-m-tb"]/table[1]/tr[3]/td[2]/text()')[0] aution.earnest_money = float( re.search('(\d+),?(\d+)', earnest_money, re.S | re.M).group(1).replace(',', '')) court = tree.xpath('//td[@class="pr7"]/text()')[0] aution.court = re.search('法院:(.*?)$', court, re.S | re.M).group(1) aution.contacts = tree.xpath('//td[@valign="top"]/text()')[0] phone_number = tree.xpath('//td[@colspan="2"]/text()')[0] try: aution.phone_number = re.search('联系电话:(.*?)$', phone_number, re.S | re.M).group(1) except Exception as e: aution.phone_number = None info.append( tree.xpath( 'string(//div[@class="panel-con"]/div[@class="d-block"][2])' )) info.append( tree.xpath( 'string(//div[@class="panel-con"]/div[@class="d-article d-article2"][3])' )) aution.info = info if aution_time: aution.auction_time = datetime.datetime.strptime( aution_time, "%Y-%m-%d %H:%M:%S") else: aution.auction_name = tree.xpath( '//div[@class="DivItemName"]/text()')[0] aution.auction_level = tree.xpath( '/html/body/div[1]/div[7]/div[2]/div[1]/div[2]/div[4]/li[4]/text()' )[0] try: assess_value = tree.xpath( '/html/body/div[1]/div[7]/div[2]/div[1]/div[2]/div[4]/li[5]/text()' )[0] aution.assess_value = float( re.search('(\d+),?(\d+)', assess_value, re.S | re.M).group(1).replace(',', '')) except Exception as e: aution.assess_value = None earnest_money = tree.xpath( '/html/body/div[1]/div[7]/div[2]/div[1]/div[2]/div[4]/li[6]/text()' )[0] aution.earnest_money = float( re.search('(\d+),?(\d+)', earnest_money, re.S | re.M).group(1).replace(',', '')) court = tree.xpath( '/html/body/div[1]/div[7]/div[2]/div[1]/div[2]/div[4]/li[8]/text()' )[0] aution.court = re.search('法院:(.*?)$', court, re.S | re.M).group(1) area = tree.xpath( '/html/body/div[1]/div[7]/div[2]/div[1]/div[2]/div[4]/li[2]/text()' )[0] aution.area = float( re.search('(\d+)\.(\d+)', area, re.S | re.M).group(1).replace(',', '')) info.append(tree.xpath('string(//div[@id="Tab1"])')) info.append( tree.xpath('string(//div[@class="bootstrap-table"])')) aution.info = info if aution_time: aution.auction_time = datetime.datetime.strptime( aution_time, "%Y-%m-%d %H:%M:%S") aution.insert_db() except Exception as e: log.error('解析错误,url="{}",e="{}"'.format(aution_url, e))
import numpy as np import matplotlib.pyplot as plt from auction import CostFn, Auction from utils import create_cost_fn_coef # single bidder cost_fns = [CostFn(create_cost_fn_coef(2, 3), 3)] auction = Auction(2, cost_fns).run() assert (auction.actual_soln == np.array([2])).all() assert (auction.approx_soln == np.array([2])).all() assert auction.actual_cost == 17 # two of same bidder cost_fns = [ CostFn(create_cost_fn_coef(2, 3), 3), CostFn(create_cost_fn_coef(2, 3), 3) ] auction = Auction(4, cost_fns).run() assert (auction.actual_soln == np.array( [3, 1])).all() or (auction.actual_soln == np.array([1, 3])).all() assert (auction.approx_soln == np.array( [3, 1])).all() or (auction.approx_soln == np.array([1, 3])).all() assert auction.actual_cost == 32 # one bidder with cheaper cost structure cost_fns = [ CostFn(create_cost_fn_coef(4, 4), 4), CostFn(create_cost_fn_coef(2, 3), 3) ] assert (auction.actual_soln == np.array([1, 3])).all() assert (auction.approx_soln == np.array([1, 3])).all()
def query_latest_summary_for_item(self, item_name): item = Item.one(item_name) if item is None: return { "status": "error", "errors": ["Item does not exist."], } item_info = { "name": item.name, "reserved_price": item.reserved_price, "status_code": item.status, "status_name": const.ITEM_STATUS_NAMES[item.status], "created_at": item.created_at, "updated_at": item.updated_at, } auction = None auction_info = None if item.status > const.ITEM_STATUS_AVAILABLE: # We will exclude any past failed auctions in summary because # presumably that's not good for auction. def filter_func(auc_record): item_name_met = auc_record.get("item_name") == item_name auc_status_met = auc_record.get("status") != const.AUCTION_STATUS_CALLED_FAIL return item_name_met and auc_status_met ret = Auction.all(filter_func) if ret: auction = ret[0] auction_info = { "id": auction.id, "status_code": auction.status, "status_name": const.AUCTION_STATUS_NAMES[auction.status], "highest_bid_id": auction.highest_bid_id, "winning_bid_id": auction.winning_bid_id, "created_at": auction.created_at, "started_at": auction.started_at, "closed_at": auction.closed_at, } bid_id = None if auction and auction.winning_bid_id: bid_id = auction.winning_bid_id elif auction and auction.highest_bid_id: bid_id = auction.highest_bid_id bid_info = None if bid_id: prevailing_bid = Bid.one(bid_id) if prevailing_bid: bid_info = { "id": prevailing_bid.id, "offer_price": prevailing_bid.offer_price, "participant_id": prevailing_bid.participant_id, "submitted_at": prevailing_bid.submitted_at, } return { "status": "success", "item": item_info, "auction": auction_info, "prevailing_bid": bid_info, }
def get_list_info(self, url_page, html_type, auction_type): response = requests.get(url_page, headers=self.headers) html = response.text tree = etree.HTML(html) div_list = tree.xpath('//div[@class="sflistdiv"]') for i in div_list: info = [] auction = Auction(source, auction_type) auction.province = '上海' auction.city = '上海' auction.html_type = html_type auction.source_html = html auction_id = i.xpath('div[@class="sflistdivn2"]/div[@class="f20hei"]/a/@href')[0].split('/')[-1] is_exist = coll.find_one({'auction_id': str(auction_id), 'source': source}) if is_exist: log.info('id已存在,id="{}"'.format(str(auction_id))) continue auction.auction_id = auction_id try: auction_name_ = i.xpath('div[@class="sflistdivn2"]/div[@class="f20hei"]/a/text()')[0] except Exception as e: auction_name_ = '' region = i.xpath('div[@class="sflistdivn2"]/div[@class="sflistban"]/text()')[0] auction.region = re.search(' - (.*?)$', region, re.S | re.M).group(1) auction_time_ = i.xpath('div[@class="sflistdivn2"]/div[@class="sflisttime"]/text()')[0] address = i.xpath('div[@class="sflistdivn2"]/div[@class="sflistcan"]/text()')[3].encode().decode() auction.auction_name = auction_name_ + address try: auction_time = re.search('拍卖时间:(.*?)$', auction_time_, re.S | re.M).group(1) auction.auction_time = datetime.datetime.strptime(auction_time, "%y.%m.%d") except Exception as e: auction.auction_time = None info.append(i.xpath('string(div[@class="sflistdivn2"])')) area_ = i.xpath('div[@class="sflistdivn2"]/div[@class="sflistcan"]/span[1]/text()')[0] auction.area = re.search('面积:(.*?)$', area_, re.S | re.M).group(1) floor = i.xpath('div[@class="sflistdivn2"]/div[@class="sflistcan"]/span[3]/text()')[0] auction.floor = re.search('楼层:(.*?)$', floor, re.S | re.M).group(1) start_auction_price = i.xpath('//div[@class="f34hong"]/text()')[0] auction.start_auction_price = float( re.search('(\d+),?(\d+)', start_auction_price, re.S | re.M).group(1).replace(',', '')) * 10000 auction.insert_db()
def get_info(self, url): response = requests.get(url=url, headers=self.headers) html = etree.HTML(response.text) print(url) wrong_list = [] try: wrong = html.xpath("//div[@class='dialog']/h1/text()")[0] wrong_list.append(wrong) except Exception as e: print(e) if "We're sorry, but something went wrong." not in wrong_list: title = html.xpath("//div[@class='title']/text()")[0] start_price = html.xpath( "//table[@class='item-attrs']//tr[1]/td[2]/text()")[0] assess_price = html.xpath( "//table[@class='item-attrs']//tr[1]/td[4]/text()")[0] ensure_price = html.xpath( "//table[@class='item-attrs']//tr[1]/td[6]/text()")[0] auction_id = re.search( "http://auction\.qdauction\.com/items/(\d+)", url).group(1) auction = Auction(source=source, auction_type=auction_type) auction.auction_name = title auction.start_auction_price = start_price auction.assess_value = assess_price auction.earnest_money = ensure_price auction.auction_id = auction_id try: time = html.xpath("//tr[@class='deal']/td[4]/text()")[0] Auction.auction_time = datetime.datetime.strptime( time, "%Y-%m-%d %H:%M:%S") except Exception as e: print(e) auction.source_html = response.text auction.city = '青岛' auction.html_type = '其他' auction.insert_db()
def gen_token(): curve = PriceSupplyCurve(factor=0.000001, base_price=1) auction = Auction() beneficiary = Beneficiary(issuance_fraction=.2) ct = Mint(curve, beneficiary, auction) return ct
def start_crawler(self): for type_num in type_list: page_num = self.get_page(type_num.code) for page in range(1, int(page_num) + 1): url = 'http://auction.jd.com/getJudicatureList.html?page=' + str( page) + '&limit=40&childrenCateId=' + type_num.code try: response = s.get(url, headers=self.headers) html = response.json() try: for info in html['ls']: auction = Auction( source=source, auction_type=type_num.auction_type) auction.html_type = type_num.html_type auction.auction_name = info['title'] # 商品名 auction.assess_value = info[ 'assessmentPrice'] # 评估值 try: auction.province = info['province'] # 省 auction.city = info['city'] # 城市 except Exception as e: auction.province = None auction.city = None auction.auction_time = datetime.datetime.fromtimestamp( int(info['startTime']) / 1000) # 评估值 auction.earnest_money = info['currentPrice'] # 保证金 auction.auction_id = str(info['id']) # 商品id is_exist = coll.find_one({ 'auction_id': str(info['id']), 'source': source }) if is_exist: log.info('id已存在,id="{}"'.format(str( info['id']))) continue self.get_detail(str(info['id']), auction) except Exception as e: log.error('解析错误,url="{}"'.format(url)) except Exception as e: log.error('请求错误,url="{}"'.format(url))
def auction(self): if not getattr(self, '_auction', None): self._auction = Auction() return self._auction
wb.save(fname) def output_all(self): self.output_hist_to_xlsx("output_master_aggregate.xlsx") for ix, puid in enumerate(self.puids): self.output_policy_info_to_xlsx( "output_policy_info_{}.xlsx".format(puid), ix) self.output_time_logged_to_xlsx("output_time_spent_in_seconds.xlsx") if __name__ == "__main__": t_start = time.time() print("{:.2f} sec: start loading simulator".format(time.time() - t_start)) sim = Simulator() param, attrs = Auction.read_init_xlsx("auction_ini_01.xlsx") auc = Auction(param, attrs) aucts = auc.generate_sample() # aucts = sl.load_auction_p("auction_01.p") # loading from a snapshot example. sim.read_in_auction(aucts) print("{:.2f} sec: finished loading simulator".format(time.time() - t_start)) for t in range(param['max iteration']): sim_res = sim.step() print("{:.2f} sec: simulation iter {}, auction happened? {}".format( time.time() - t_start, t, sim_res)) sim.output_all() print("{:.2f} sec: created output files.".format(time.time() - t_start)) pass
def get_detail(self, id_, auction_time, html_type, auction_type, province, city, region): auction = Auction(source=source, auction_type=auction_type) auction.html_type = html_type auction.auction_type = auction_type auction.province = province auction.city = city auction.region = region detail_url = 'http://www1.rmfysszc.gov.cn/Handle/' + id_ + '.shtml' try: response = requests.get(detail_url, headers=self.headers) html = response.content.decode() auction.source_html = html info_list = [] try: if 'GetRecord()' in html: tree = etree.HTML(html) auction.auction_name = tree.xpath( '//div[@id="Title"]/h1/text()')[0] start_auction_price = tree.xpath( '//*[@id="price"]/div[1]/span/text()')[0] auction.start_auction_price = self.get_float( start_auction_price) assess_value = tree.xpath( '//*[@id="bg1"]/div[1]/table/tr[1]/td/span[2]/text()' )[0] try: auction.assess_value = self.get_float(assess_value) except Exception as e: auction.assess_value = None earnest_money = tree.xpath( '//*[@id="bg1"]/div[1]/table/tr[2]/td/span[2]/text()' )[0] auction.earnest_money = self.get_float(earnest_money) announcement_date = tree.xpath( '//*[@id="bg1"]/div[1]/table/tr[3]/td/span/text()')[0] announcement_date_ = re.search(': (.*?)$', announcement_date, re.S | re.M).group(1) auction.announcement_date = datetime.datetime.strptime( announcement_date_, "%Y.%m.%d") auction_level = tree.xpath( '//*[@id="bg1"]/div[1]/table/tr[4]/td/span/text()')[0] auction.auction_level = re.search(': (.*?)$', auction_level, re.S | re.M).group(1) court = tree.xpath( '//*[@id="bg1"]/div[2]/table/tr[1]/td/span/text()')[0] auction.court = re.search(': (.*?)$', court, re.S | re.M).group(1) info_list.append( tree.xpath( 'string(//*[@id="bdjs11"])').encode().decode()) info_list.append( tree.xpath( 'string(//*[@id="jjjl"])').encode().decode()) contacts = tree.xpath( '//*[@id="bg1"]/div[2]/table/tr[2]/td/span/text()')[0] auction.contacts = re.search(': (.*?)$', contacts, re.S | re.M).group(1) phone_number = tree.xpath( '//*[@id="bg1"]/div[2]/table/tr[3]/td/span/text()')[0] auction.phone_number = re.search(': (.*?)$', phone_number, re.S | re.M).group(1) auction.info = info_list try: auction.build_type = tree.xpath( '//*[@id="bdjs11"]/table[1]/tr[2]/td[4]/text()')[0] except Exception as e: auction.build_type = None auction.auction_id = id_ auction.auction_time = self.get_date(date=auction_time) auction.insert_db() elif 'bmnumber()' in html: tree = etree.HTML(html) auction.auction_name = tree.xpath( '//div[@id="Title"]/h1/text()')[0] start_auction_price = tree.xpath( '//*[@id="price"]/div[1]/span/text()')[0] auction.start_auction_price = self.get_float( start_auction_price) assess_value = tree.xpath( '//*[@id="bg1"]/div[1]/table/tr[1]/td/span[2]/text()' )[0] auction.assess_value = self.get_float(assess_value) earnest_money = tree.xpath( '//*[@id="bg1"]/div[1]/table/tr[2]/td/span[2]/text()' )[0] auction.earnest_money = self.get_float(earnest_money) announcement_date = tree.xpath( '//*[@id="bg1"]/div[1]/table/tr[3]/td/span/text()')[0] announcement_date_ = re.search(': (.*?)$', announcement_date, re.S | re.M).group(1) auction.announcement_date = datetime.datetime.strptime( announcement_date_, "%Y-%m-%d") auction_level = tree.xpath( '//*[@id="bg1"]/div[1]/table/tr[4]/td/span/text()')[0] auction.auction_level = re.search(': (.*?)$', auction_level, re.S | re.M).group(1) court = tree.xpath( '//*[@id="bg1"]/div[2]/table/tr[1]/td/span/text()')[0] auction.court = re.search(': (.*?)$', court, re.S | re.M).group(1) info_list.append( tree.xpath( 'string(//*[@id="bdjs"])').encode().decode()) contacts = tree.xpath( '//*[@id="bg1"]/div[2]/table/tr[2]/td/span/text()')[0] auction.contacts = re.search(': (.*?)$', contacts, re.S | re.M).group(1) phone_number = tree.xpath( '//*[@id="bg1"]/div[2]/table/tr[3]/td/span/text()')[0] auction.phone_number = re.search(': (.*?)$', phone_number, re.S | re.M).group(1) auction.info = info_list try: auction.build_type = tree.xpath( '//*[@id="bdjs11"]/table[1]/tr[2]/td[4]/text()')[0] except Exception as e: auction.build_type = None auction.auction_id = id_ auction.auction_time = self.get_date(date=auction_time) auction.insert_db() else: tree = etree.HTML(html) auction.auction_name = tree.xpath( '//*[@id="xmgg"]/div/div[1]/text()')[0] assess_value = tree.xpath( '/html/body/div[6]/table/tr/td/ul/li[3]/span/text()' )[0] auction.assess_value = self.get_float(assess_value) announcement_date = tree.xpath( '/html/body/div[6]/table/tr/td/ul/li[2]/span/text()' )[0] try: auction.announcement_date = datetime.datetime.strptime( announcement_date, "%Y-%m-%d") except Exception as e: auction.announcement_date = datetime.datetime.strptime( announcement_date, "%Y/%m/%d") auction.court = tree.xpath( '/html/body/div[6]/table/tr/td/ul/li[1]/span/text()' )[0] info_list.append( tree.xpath( 'string(//*[@id="bdxx"]/div)').encode().decode()) info_list.append( tree.xpath('string(//*[@id="tjzl"]/div/div[2])'). encode().decode()) auction.contacts = tree.xpath( '/html/body/div[6]/table/tr/td/ul/li[4]/span/text()' )[0] auction.phone_number = tree.xpath( '/html/body/div[6]/table/tr/td/ul/li[5]/span/text()' )[0] auction.info = info_list try: auction.build_type = tree.xpath( '//*[@id="bdxx"]/div/div[2]/table/tr[2]/td[3]/text()' )[0] except Exception as e: auction.build_type = None auction.auction_id = id_ auction.auction_time = self.get_date(date=auction_time) auction.insert_db() except Exception as e: log.error('解析错误,url="{}",e="{}"'.format(detail_url, e)) except Exception as e: log.error('详情页请求错误,url="{}",e="{}"'.format(detail_url, e))
def step(self): """ simulates one timestep in the auction :return: True if auction is simulated, False if no auction data is present """ self.time_last = time.time() self.t += 1 auction_happened = False events = [] for a in self.auctions: if a['iter'] != self.t: continue if len(self.hist) == 0: costs_sum = [0.0] * len(self.pols) revenues_sum = [0.0] * len(self.pols) profits_sum = [0.0] * len(self.pols) else: costs_sum = deepcopy(self.hist[-1]['costs_cumulative']) revenues_sum = deepcopy(self.hist[-1]['revenues_cumulative']) profits_sum = deepcopy(self.hist[-1]['profits_cumulative']) auction_happened = True # bids = [float(p.bid(a['attr'])) for p in self.pols] # bids = [] pool = Pool(24) results = pool.starmap( Simulator().bid_from_policy, zip(self.puids, self.pols, [a] * len(self.pols))) pool.close() pool.join() bids = [b for (b, t, p) in results] time_last = [t for (b, t, p) in results] pols = [p for (b, t, p) in results] self.pols = pols dicts = {} for dict in time_last: dicts.update(dict) for n in dicts.keys(): if n in self.time_spent.keys(): self.time_spent[n] += dicts[n] else: self.time_spent[n] = dicts[n] #for puid, p in zip(self.puids, self.pols): # self._time_log('simulator') # this_bid = p.bid(a['attr']) # self._time_log(puid) # bids.append(float(this_bid)) max_bid_pols_ix = sl.max_ix(bids) winning_bid = bids[max_bid_pols_ix[0]] # take top K bids -- because K slots are there reverse_sorted_bids, sorted_pIx = sl.top_K_max( bids, self.num_of_ad_slots, self.prng) sorted_unique_bids = sorted(list(set(bids))) num_clicks, p_click = self.get_num_clicks(winning_bid, a) cost = sl.compute_second_price_cost( bids, size=num_clicks) # second price auction conversion = Auction.get_conversion(a['prob_conversion'], self.prng, size=num_clicks) revenue = Auction.get_revenue_sample(a['avg_revenue'], self.prng, size=num_clicks) winning_pol_ix = [] for ix in range(num_clicks): # winner_ix = int(self.prng.choice(max_bid_pols_ix)) # max-bidder-wins case # if geometric click prob, then winner_ix is one of top-K bids # fill K positions with pIx, in non-decreasing order of bids[pIx] # and choose one of K with custom set probability in geometrically decaying probability # that chosen one is winner_ix of this click. winner_ix = int( self.prng.choice(sorted_pIx, p=self.ad_slot_click_prob_adjuster)) winning_bid = bids[winner_ix] winning_pol_ix.append(winner_ix) # compute actual cost (second price) for each click (as each click may have different winner than max bidder) actual_cost = sl._compute_actual_second_price_cost( bids[winner_ix], sorted_unique_bids) cost[ix] = actual_cost costs_sum[winner_ix] += cost[ix] revenues_sum[winner_ix] += conversion[ix] * revenue[ix] profits_sum[ winner_ix] = revenues_sum[winner_ix] - costs_sum[winner_ix] event = { 'iter': self.t, 'attr': a['attr'], 'auctions_in_iter': a['num_auct'], 'bids': bids, 'winning_pol_id': winner_ix, 'winning_pol_name': self.puids[winner_ix], 'winning_bid': winning_bid, 'num_click': 1, 'cost_per_click': cost[ix], 'num_conversion': conversion[ix], 'revenue_per_conversion': conversion[ix] * revenue[ix], 'costs_cumulative': deepcopy(costs_sum), 'revenues_cumulative': deepcopy(revenues_sum), 'profits_cumulative': deepcopy(profits_sum) } events.append(event) self.events.append(event) if num_clicks == 0: winner_ix = int(self.prng.choice(max_bid_pols_ix)) event = { 'iter': self.t, 'attr': a['attr'], 'auctions_in_iter': a['num_auct'], 'bids': bids, 'winning_pol_id': winner_ix, 'winning_pol_name': self.puids[winner_ix], 'winning_bid': winning_bid, 'num_click': 0, 'cost_per_click': '', 'num_conversion': 0, 'revenue_per_conversion': '', 'costs_cumulative': deepcopy(costs_sum), 'revenues_cumulative': deepcopy(revenues_sum), 'profits_cumulative': deepcopy(profits_sum) } events.append(event) self.events.append(event) # keep aggregate history for output auct_res = deepcopy(a) auct_res['bids'] = bids auct_res['num_click'] = num_clicks auct_res['p_click'] = p_click auct_res['cost_per_click'] = mean(cost) if num_clicks > 0 else '' auct_res['num_conversion'] = sum(conversion) auct_res['revenue_per_conversion'] = sum([ ncr * rpc for (ncr, rpc) in zip(conversion, revenue) ]) / sum(conversion) if sum(conversion) > 0 else '' auct_res['costs_cumulative'] = deepcopy(costs_sum) auct_res['revenues_cumulative'] = deepcopy(revenues_sum) auct_res['profits_cumulative'] = deepcopy(profits_sum) auct_res['time_spent'] = deepcopy(self.time_spent) self.hist.append(auct_res) # end of auction events handling # if nothing happened. this is the way to go if len(events) == 0: return auction_happened # aggregate information over one iteration is assembled, for each policy p_infos = {} for p_ix, p in enumerate(self.pols): p_infos[p_ix] = [] last_a = events[0]['attr'] bunch = [] if len(self.p_infos[p_ix]) == 0: profit_sum = 0.0 else: profit_sum = self.p_infos[p_ix][-1][-1][ 'your_profit_cumulative'] for ev in events + [{'attr': 'guard_dummy'}]: this_a = ev['attr'] if last_a == this_a and ev != 'guard_dummy': bunch.append(ev) else: bids_that_got_clicked = [ev['winning_bid'] for ev in bunch] # aggregate information p_info = { 'iter': bunch[0]['iter'], 'attr': bunch[0]['attr'], 'num_auct': bunch[0]['auctions_in_iter'], 'your_bid': bunch[0]['bids'][p_ix], 'winning_bid': max(bids_that_got_clicked), 'winning_bid_avg': mean(bids_that_got_clicked) } win_count = [] clicks = [] costs = [] conversions = [] revenues = [] for ev_b in bunch: if p_ix == ev_b['winning_pol_id']: win_count.append(1) clicks.append(ev_b['num_click']) costs.append(ev_b['cost_per_click']) conversions.append(ev_b['num_conversion']) revenues.append(ev_b['revenue_per_conversion']) approx_num_impression = int(p_info['num_auct'] * sum(win_count) / len(bunch)) p_add_info = { 'num_impression': approx_num_impression, 'num_click': sum(clicks), 'cost_per_click': mean(costs) if sum(clicks) > 0 else '', 'num_conversion': sum(conversions), 'revenue_per_conversion': sum(revenues) / sum(conversions) if sum(conversions) > 0 else '' } this_cost = sum([c if c != '' else 0 for c in costs]) this_revenue = sum([ ncr * rpc for (ncr, rpc ) in zip(conversions, [r if r != '' else 0 for r in revenues]) ]) profit_sum += this_revenue - this_cost p_info['your_profit_cumulative'] = profit_sum p_info.update(p_add_info) p_infos[p_ix].append(p_info) # refresh rolling aggregator and its guard bunch = [ev] last_a = this_a # post-auction learning session for policies for p_ix, p in enumerate(self.pols): self._time_log('simulator') p.learn(p_infos[p_ix]) self._time_log(self.puids[p_ix]) self.p_infos[p_ix].append(p_infos[p_ix]) # finish up self._time_log('simulator') print(self.time_spent) return auction_happened
def parse(self, html): auction_list = html.xpath("//dl/dd/a/@href") for auction_url in auction_list: try: url = 'http://www.shjiapai.cn' + auction_url auction_res = requests.get(url, headers=self.headers) con = auction_res.text auction_id = re.search('id/(\d+).html', auction_url).group(1) if not check_auction(source=source, auction_id=auction_id): auction = Auction(source=source, auction_type=auction_type) auction.source_html = con auction.auction_id = auction_id auction.auction_name = re.search('楼盘名称.*?">(.*?)</td', con, re.S | re.M).group(1) auction.city = '上海' auction.html_type = '房产' auction.start_auction_price = re.search( '预计售价.*?">(.*?)</td', con, re.S | re.M).group(1) auction.floor = re.search('层.*?">(.*?)楼</td', con, re.S | re.M).group(1) auction.area = re.search('户型面积.*?">(.*?)</td', con, re.S | re.M).group(1) auction.build_type = re.search('物业类型.*?">(.*?)</td', con, re.S | re.M).group(1) auction.info = re.search('其它.*?>(.*?)</div', con, re.S | re.M).group(1) auction.insert_db() else: log.info("数据已存在") except Exception as e: log.error("{}解析失败".format(auction_url))
def get_detail(self, source, auction_id, auction_status): response, url = self.request_url(auction_id) try: tree = etree.HTML(response.text) html = response.text auction = Auction(source=source) auction.url = url # 起拍价(万) startPrice = self.get_startPrice(html=html) startPrice = float(startPrice.replace(' ', '').replace(',', '')) / 10000 auction.startPrice = startPrice # 评估价(万) evalPrice = self.get_evalPrice(tree=tree) evalPrice = float(evalPrice.replace(' ', '').replace('¥', '').replace(',', '')) / 10000 auction.evalPrice = evalPrice # 保证金(万) bond = tree.xpath('//div[@id="content"]/div/div[2]/div[1]/div/div[2]/div[10]/ul[3]/li/span[2]/text()')[0] bond = bond.replace(' ', '').replace('¥', '').split('.')[0] bond = float(bond) / 10000 auction.bond = bond # 拍卖阶段 auctionStage_info = tree.xpath('//div[@id="content"]/div[1]/div[2]/div[1]/div[1]/div[2]/h1/text()')[0] auctionStage_info = auctionStage_info.replace(' ', '').replace('\n', '').replace('\t', '') auctionStage = auctionStage_info.split('】')[0].split('【')[1] auction.auctionStage = auctionStage # 拍卖次数 auctionCount = self.get_auctionCount(auctionStage) auction.auctionCount = auctionCount # 拍卖物品名称 auction_name = tree.xpath('//div[@id="content"]/div[1]/div[2]/div[1]/div[1]/div[2]/h1/text()')[0] auction_name = auction_name.replace(' ', '').replace('\n', '').replace('\t', '').split('】')[1] auction.auction_name = auction_name # todo 调用黄村粮的方法进行切割获取城市,区域,小区名,经纬度等 cut_info = CutMatch.to_match('上海', auction_name) # 切割后匹配库中的城市 auction.matchCity = cut_info['matchCity'] # 切割后匹配库中的区域 auction.matchRegion = cut_info['matchRegion'] # 切割后匹配库中的小区名称 auction.matchName = cut_info['matchName'] # 切割后匹配库中的地址 auction.matchAddress = cut_info['matchAddress'] # 切割后的房号 auction.roomNum = cut_info['cutRoomnum'] # 切割后的楼号 auction.houseNum = cut_info['cutHousenum'] # 切割后的城市 auction.cutCity = cut_info['cutCity'] # 切割后的区域 auction.cutRegion = cut_info['cutRegion'] # 切割后的小区名称 auction.cutName = cut_info['cutName'] # 切割后的地址 auction.cutAddress = cut_info['cutAddress'] # 切割后跑高德接口得到的经纬度 auction.lat = cut_info['mapLat'] auction.lng = cut_info['mapLng'] # 地址 address = tree.xpath('//em[@id="paimaiAddress"]/text()')[0] auction.address = address # 城市 city = address.split(' ')[0] auction.city = city # 区域 region = address.split(' ')[1] auction.region = region skulid = re.search('id="skuId" value="(.*?)"', html, re.S | re.M).group(1) # 竞拍状态 当前价格 成交价格 curPrice, dealPrice = self.get_curPrice_and_dealPrice(skulid, auction_id) auction.biddingState = auction_status auction.curPrice = curPrice auction.dealPrice = dealPrice # 起始时间 startShootingDate = tree.xpath('//input[@id="startTime"]/@value')[0] startShootingDate = datetime.datetime.strptime(startShootingDate, '%Y-%m-%d %H:%M:%S.%f') auction.startShootingDate = startShootingDate # 结束时间 endShootingDate = tree.xpath('//input[@id="endTime"]/@value')[0] endShootingDate = datetime.datetime.strptime(endShootingDate, '%Y-%m-%d %H:%M:%S.%f') auction.endShootingDate = endShootingDate # 图片 houseImgUrls = [] houseImgUrls_info = tree.xpath('//div[@id="spec-list"]/div/ul/li/img/@src') for houseImgUrl in houseImgUrls_info: houseImgUrl = 'http:' + houseImgUrl big_img = houseImgUrl.replace('jfs','s1000x750_jfs') new_houseImgUrl = qiniufetch(big_img, big_img) houseImgUrls.append(new_houseImgUrl) auction.houseImgUrls = houseImgUrls # 参与人数 浏览数量 participantsNumber, visitCount = self.get_participantsNumber_and_visitCount(auction_id) auction.participantsNumber = participantsNumber auction.visitCount = visitCount # 拍卖物品id auction.auctionId = auction_id # 网站来源(jingdong) auction.source = source auction.update() except Exception as e: log.error('解析错误')