def call_auction(self, auction_id):
auction = Auction.one(auction_id)
if auction is None:
return {
"status": "error",
"errors": ["This auction does not exist."],
}
return auction.end()
def query_all_bids_for_auction(self, auction_id):
auction = Auction.one(auction_id)
if auction is None:
return {
"status": "error",
"errors": ["This auction does not exist."],
}
return auction.get_all_submitted_bids()
def submit_bid_for_auction(self, auction_id, bid_price):
auction = Auction.one(auction_id)
if auction is None:
return {
"status": "error",
"errors": ["This auction does not exist."],
}
return auction.process_bid(bid_price, self.id)
def query_all_live_auctions(self, involved_only=False):
def filter_func(auc_record):
return auc_record.get("status") == const.AUCTION_STATUS_IN_PROGRESS
auctions = Auction.all(filter_func)
if involved_only:
bids = Bid.all(lambda x: x["participant_id"] == self.id)
involved_auction_ids = set([b.auction_id for b in bids])
auctions = [a for a in auctions if a.id in involved_auction_ids]
return auctions
def create_auction(self, item_name):
item = Item.one(item_name)
if item is None:
return {
"status": "error",
"errors": ["Item does not exist."],
}
def filter_func(auc_record):
return auc_record.get("status") != const.AUCTION_STATUS_CALLED_FAIL
if Auction.all(filter_func):
return {
"status": "error",
"errors": ["Auction already exists for this item."],
}
auction = Auction(item_name)
auction.save()
return {
"status": "success",
"auction_id": auction.id,
}
def unstage_item_from_auction(self, item_name):
item = Item.one(item_name)
if item is None:
return {
"status": "error",
"errors": ["Item does not exist."],
}
if item.status != const.ITEM_STATUS_STAGED:
return {
"status": "error",
"errors": ["Item is not staged for auction."],
}
def filter_func(auc_record):
item_name_met = auc_record.get("item_name") == item_name
auc_status_met = auc_record.get("status") == const.AUCTION_STATUS_CREATED
return item_name_met and auc_status_met
ret = Auction.all(filter_func)
if ret:
auction = ret[0]
auction.delete()
else:
return {
"status": "error",
"errors": [
"Auction for this item already progressed, and cannot "
"unstage the item at this time."
],
}
item.status = const.ITEM_STATUS_AVAILABLE
item.save()
return {
"status": "success",
"item_name": item.name,
}
def _runAuction(self, gteeOfferPrice, gteeBidPrice, haveQ):
""" Clears an auction to determine the quantity and price for each
offer/bid.
"""
pOffers = [offer for offer in self.offers if not offer.reactive]
pBids = [bid for bid in self.bids if not bid.reactive]
# Clear offer/bid quantities and prices.
auction = Auction(self.case, pOffers, pBids, self.auctionType,
gteeOfferPrice, gteeBidPrice, self.limits)
auction.run()
# Separate auction for reactive power.
if haveQ:
qOffers = [offer for offer in self.offers if offer.reactive]
qBids = [bid for bid in self.bids if bid.reactive]
# Too complicated to scale with mixed bids/offers (only
# auction_types LAO and FIRST_PRICE allowed)
qAuction = Auction(self.case, qOffers, qBids, self.auctionType,
gteeOfferPrice, gteeBidPrice, self.limits)
qAuction.run()
def demo_auction():
"""Run a random auction. Stop after max_bids bids."""
# number of bids to accept
max_bids = 10
auction = Auction("Vacation to Ko Samui")
auction.start()
print("Starting", auction, "with min bidding increment", auction.increment)
#auction.bid("initial", 20)
for n in range(0,max_bids):
bidder = get_bidder(auction)
amount = int(auction.best_bid() + 10*random.randint(-2,5) + 10)
print_and_bid(auction,bidder,amount)
# pause the auction
if n == max_bids/2:
print(">>> stop()")
auction.stop()
elif n == max_bids/2 + 1:
print(">>> start()")
auction.start()
print("The winner is", auction.winner(), "with a bid of", auction.best_bid())
def detail_parse(self, **kwargs):
auction = Auction(source='ali')
auction.auction_name = kwargs['auction_name']
auction.auctionId = kwargs['auction_id']
auction.biddingState = kwargs['status']
auction.curPrice = kwargs['current_price']
auction.evalPrice = kwargs['evalprice']
auction.startShootingDate = kwargs['start_time']
auction.endShootingDate = kwargs['end_time']
auction.participantsNumber = kwargs['partnumber']
auction.visitCount = kwargs['visitCount']
detail_url = kwargs['url']
auction.url = detail_url
try:
# todo 调用黄村粮的方法进行切割获取城市，区域，小区名，经纬度等
cut_info = CutMatch.to_match('上海', kwargs['auction_name'])
# 切割后匹配库中的城市
auction.matchCity = cut_info['matchCity']
# 切割后匹配库中的区域
auction.matchRegion = cut_info['matchRegion']
# 切割后匹配库中的小区名称
auction.matchName = cut_info['matchName']
# 切割后匹配库中的地址
auction.matchAddress = cut_info['matchAddress']
# 切割后的房号
auction.roomNum = cut_info['cutRoomnum']
# 切割后的楼号
auction.houseNum = cut_info['cutHousenum']
# 切割后的城市
auction.cutCity = cut_info['cutCity']
# 切割后的区域
auction.cutRegion = cut_info['cutRegion']
# 切割后的小区名称
auction.cutName = cut_info['cutName']
# 切割后的地址
auction.cutAddress = cut_info['cutAddress']
# 切割后跑高德接口得到的经纬度
auction.lat = cut_info['mapLat']
auction.lng = cut_info['mapLng']
except Exception as e:
log.error(e)
return
try:
detail_res = requests.get(url=detail_url, headers=self.headers)
except Exception as e:
log.error('url={}, e={}'.format(detail_url, e))
return
html = etree.HTML(detail_res.content.decode('gbk'))
try:
title = html.xpath(
'//div[contains(@class,"pm-main clearfix")]/h1/text()'
)[0].strip()
except:
log.error('没有标题 url={}'.format(detail_url))
return
auctionStage = re.search('【(.*?)】', title).group(1)
auction.auctionStage = auctionStage
auction.auctionCount = self.get_auctionCount(auctionStage)
startPrice = re.search('起拍价￥(.*?) ，',
detail_res.content.decode('gbk')).group(1)
bond = re.search('保 证 金.*?J_Price">(.*?)</span',
detail_res.content.decode('gbk'),
re.S | re.M).group(1)
comm_url = 'http://sf.taobao.com/json/getGovItemSummary.htm?itemId={}'.format(
kwargs['auction_id'])
res = requests.get(comm_url, headers=self.headers)
try:
auction.area = float(
int(res.json()['props']['area']['value']) / 100)
except:
pass
images = html.xpath("//div[@class='pm-pic pm-s80 ']/a/img/@src")
image_list = []
for image_url in images:
big_img = image_url.replace('_80x80.jpg', '')
image = qiniufetch(big_img, big_img)
image_list.append(image)
auction.houseImgUrls = image_list
auction.startPrice = float(float(startPrice.replace(',', '')) / 10000)
auction.bond = float(float(bond.replace(',', '').strip()) / 10000)
if kwargs['status'] == '已成交':
if re.search(
'失败|流拍',
html.xpath('//h1[@class="bid-fail"]/text()')[0]) is None:
auction.update()
else:
auction.biddingState = '流拍'
auction.update()
else:
auction.update()
def detail_parse(auction_res, auction_type, html_type, auction_id):
con = auction_res.json()
auction = Auction(source=source, auction_type=auction_type)
auction.source_html = con
auction.html_type = html_type
auction.auction_id = auction_id
auction.auction_name = con['object_title']
auction.start_auction_price = con['start_price']
auction.assess_value = con['appraise_price']
auction.earnest_money = con['bond_price']
auction.court = con['court_name']
auction_time = con['start_time']
location = con['location']
auction.auction_time = datetime.datetime.strptime(
auction_time, "%Y-%m-%d %H:%M:%S")
province, city, region = location.split(' ')
auction.province = province
auction.city = city
auction.region = region
if html_type == '房产':
auction.floor = con['detail']['house_floor']
auction.area = con['detail']['gross_floor_area']
elif html_type == '土地':
auction.area = con['detail']['l_land_area']
auction.insert_db()
from auction import Input
from auction import Auction
import util
auction_parameters = Input()
test = Auction(auction_parameters)
market_prices, buyer_profits, seller_profits = test.run()
market_prices, round_avgs, seller_avgs, avg_buyer_profit_per_round, avg_seller_profit_per_round, avg_market_price = util.process_data(
market_prices, auction_parameters, buyer_profits, seller_profits)
print(f"Market Prices:")
for i, round_outcome in enumerate(market_prices):
print(f"round {i}: {round_outcome}")
print()
print(
f"Buyer profits for all {auction_parameters.num_round} rounds: \n{buyer_profits}\n"
f"AVG Buyer profits per round: \n{avg_buyer_profit_per_round}\n"
f"Seller profits for all {auction_parameters.num_round} rounds: \n{seller_profits}\n"
f"AVG Seller profits per round: {avg_seller_profit_per_round}\n"
f"AVG market price: {avg_market_price}\n")
def query_all_auctions(self, status_code=None):
auctions = Auction.all()
if status_code:
auctions = [a for a in auctions if a.status == status_code]
return auctions
def crawler_detail_page(self, auction_id, province_name, city_name,
type_name, auction_type):
detail_url = 'http://www.chinesesfpm.com/index/index/info/biao_id/' + auction_id
res = requests.get(detail_url)
tree = etree.HTML(res.text)
a = Auction(source=source, auction_type=auction_type)
a.auction_id = auction_id
a.auction_name = tree.xpath(
'/html/body/div/div[6]/div/div[2]/div[1]/div[1]/text()')[0]
a.html_type = type_name
auction_time = tree.xpath(
'/html/body/div/div[6]/div/div[2]/div[1]/div[2]/div[2]/div[2]/text()'
)[0]
auction_time_ = re.search('开始时间: (.*?)$', auction_time,
re.S | re.M).group(1)
a.auction_time = datetime.datetime.strptime(auction_time_,
"%Y年%m月%d日 %H时%M分%S秒")
a.province = province_name
a.city = city_name
a.info = [
tree.xpath('string(//*[@id="f4"])'),
tree.xpath('string(//*[@id="f6"])')
]
start_auction_price = \
tree.xpath('/html/body/div/div[6]/div/div[2]/div[1]/div[2]/div[2]/div[5]/div[1]/em[3]/text()')[0]
s = start_auction_price.encode('utf-8').decode()
a.start_auction_price = float(
re.search('起拍价: ￥(.*)', s, re.S | re.M).group(1))
court = tree.xpath(
'/html/body/div/div[6]/div/div[2]/div[1]/div[2]/div[2]/div[5]/div[2]/em[1]/text()'
)[0]
a.court = re.search('拍卖机构：(.*)', court, re.S | re.M).group(1)
a.source_html = res.text
a.insert_db()
def test_new_bid(self, tell):
product = Auction().new_product(product_id=189,
initial_bid=1000,
step=50,
duration=5)
Auction().product_chat_request(product.vk_product_id, 213)
obj = MessageProcessor().process_object(
'market_comment_new', {
"id": 3,
"date": 1508607991,
"text": "#bid 1000",
"from_id": 213,
"item_id": 189,
"market_owner_id": -155
})
self.assertEqual(obj.amount, 1000)
obj = MessageProcessor().process_object('message_new', {
"id": 47,
"out": 0,
"body": "#bid 2000",
"user_id": 213
})
self.assertEqual(obj.amount, 2000)
obj = MessageProcessor().process_object(
'message_new', {
"id":
6,
"user_id":
213,
"body":
"PRIVET",
"attachments": [{
"type": "link",
"link": {
"url": "https://m.vk.com/landings/moneysend",
"title": "3000 ?4??4??1?.",
"caption": "Денежный перевод",
}
}],
"group_id":
155403696
})
self.assertEqual(obj.amount, 3000)
obj = MessageProcessor().process_object(
'market_comment_new', {
"id": 3,
"date": 1508607991,
"text": "#bid 123",
"from_id": 213,
"item_id": 189,
"market_owner_id": -155
})
self.assertFalse(obj)
obj = MessageProcessor().process_object(
'market_comment_new', {
"id": 3,
"date": 1508607991,
"text": "#bid 123000",
"from_id": 213,
"item_id": 189,
"market_owner_id": -155
})
self.assertTrue(obj)
def get_detail(self, aution_url, aution_id, aution_time, region_name,
city_name, html_type, auction_type):
info = []
aution = Auction(source, auction_type)
response = requests.get(aution_url, headers=self.headers)
try:
html = response.text
tree = etree.HTML(html)
aution.auction_id = aution_id
aution.region = region_name
aution.city = city_name
aution.source_html = html
aution.html_type = html_type
try:
aution.start_auction_price = float(
tree.xpath('//*[@id="Price_Start"]/text()')[0].replace(
',', ''))
except Exception as e:
aution.start_auction_price = None
if 'item2' in aution_url:
aution.auction_name = tree.xpath(
'//div[@class="d-m-title"]/b/text()')[0]
aution.auction_level = tree.xpath(
'//div[@class="d-m-tb"]/table[1]/tr[1]/td[2]/text()')[0]
try:
assess_value = tree.xpath(
'//div[@class="d-m-tb"]/table[1]/tr[4]/td[1]/text()'
)[0]
aution.assess_value = float(
re.search('(\d+),?(\d+)', assess_value,
re.S | re.M).group(1).replace(',', ''))
except Exception as e:
aution.assess_value = None
earnest_money = tree.xpath(
'//div[@class="d-m-tb"]/table[1]/tr[3]/td[2]/text()')[0]
aution.earnest_money = float(
re.search('(\d+),?(\d+)', earnest_money,
re.S | re.M).group(1).replace(',', ''))
court = tree.xpath('//td[@class="pr7"]/text()')[0]
aution.court = re.search('法院：(.*?)$', court,
re.S | re.M).group(1)
aution.contacts = tree.xpath('//td[@valign="top"]/text()')[0]
phone_number = tree.xpath('//td[@colspan="2"]/text()')[0]
try:
aution.phone_number = re.search('联系电话：(.*?)$',
phone_number,
re.S | re.M).group(1)
except Exception as e:
aution.phone_number = None
info.append(
tree.xpath(
'string(//div[@class="panel-con"]/div[@class="d-block"][2])'
))
info.append(
tree.xpath(
'string(//div[@class="panel-con"]/div[@class="d-article d-article2"][3])'
))
aution.info = info
if aution_time:
aution.auction_time = datetime.datetime.strptime(
aution_time, "%Y-%m-%d %H:%M:%S")
else:
aution.auction_name = tree.xpath(
'//div[@class="DivItemName"]/text()')[0]
aution.auction_level = tree.xpath(
'/html/body/div[1]/div[7]/div[2]/div[1]/div[2]/div[4]/li[4]/text()'
)[0]
try:
assess_value = tree.xpath(
'/html/body/div[1]/div[7]/div[2]/div[1]/div[2]/div[4]/li[5]/text()'
)[0]
aution.assess_value = float(
re.search('(\d+),?(\d+)', assess_value,
re.S | re.M).group(1).replace(',', ''))
except Exception as e:
aution.assess_value = None
earnest_money = tree.xpath(
'/html/body/div[1]/div[7]/div[2]/div[1]/div[2]/div[4]/li[6]/text()'
)[0]
aution.earnest_money = float(
re.search('(\d+),?(\d+)', earnest_money,
re.S | re.M).group(1).replace(',', ''))
court = tree.xpath(
'/html/body/div[1]/div[7]/div[2]/div[1]/div[2]/div[4]/li[8]/text()'
)[0]
aution.court = re.search('法院：(.*?)$', court,
re.S | re.M).group(1)
area = tree.xpath(
'/html/body/div[1]/div[7]/div[2]/div[1]/div[2]/div[4]/li[2]/text()'
)[0]
aution.area = float(
re.search('(\d+)\.(\d+)', area,
re.S | re.M).group(1).replace(',', ''))
info.append(tree.xpath('string(//div[@id="Tab1"])'))
info.append(
tree.xpath('string(//div[@class="bootstrap-table"])'))
aution.info = info
if aution_time:
aution.auction_time = datetime.datetime.strptime(
aution_time, "%Y-%m-%d %H:%M:%S")
aution.insert_db()
except Exception as e:
log.error('解析错误，url="{}",e="{}"'.format(aution_url, e))
import numpy as np
import matplotlib.pyplot as plt
from auction import CostFn, Auction
from utils import create_cost_fn_coef
# single bidder
cost_fns = [CostFn(create_cost_fn_coef(2, 3), 3)]
auction = Auction(2, cost_fns).run()
assert (auction.actual_soln == np.array([2])).all()
assert (auction.approx_soln == np.array([2])).all()
assert auction.actual_cost == 17
# two of same bidder
cost_fns = [
CostFn(create_cost_fn_coef(2, 3), 3),
CostFn(create_cost_fn_coef(2, 3), 3)
]
auction = Auction(4, cost_fns).run()
assert (auction.actual_soln == np.array(
[3, 1])).all() or (auction.actual_soln == np.array([1, 3])).all()
assert (auction.approx_soln == np.array(
[3, 1])).all() or (auction.approx_soln == np.array([1, 3])).all()
assert auction.actual_cost == 32
# one bidder with cheaper cost structure
cost_fns = [
CostFn(create_cost_fn_coef(4, 4), 4),
CostFn(create_cost_fn_coef(2, 3), 3)
]
assert (auction.actual_soln == np.array([1, 3])).all()
assert (auction.approx_soln == np.array([1, 3])).all()
def query_latest_summary_for_item(self, item_name):
item = Item.one(item_name)
if item is None:
return {
"status": "error",
"errors": ["Item does not exist."],
}
item_info = {
"name": item.name,
"reserved_price": item.reserved_price,
"status_code": item.status,
"status_name": const.ITEM_STATUS_NAMES[item.status],
"created_at": item.created_at,
"updated_at": item.updated_at,
}
auction = None
auction_info = None
if item.status > const.ITEM_STATUS_AVAILABLE:
# We will exclude any past failed auctions in summary because
# presumably that's not good for auction.
def filter_func(auc_record):
item_name_met = auc_record.get("item_name") == item_name
auc_status_met = auc_record.get("status") != const.AUCTION_STATUS_CALLED_FAIL
return item_name_met and auc_status_met
ret = Auction.all(filter_func)
if ret:
auction = ret[0]
auction_info = {
"id": auction.id,
"status_code": auction.status,
"status_name": const.AUCTION_STATUS_NAMES[auction.status],
"highest_bid_id": auction.highest_bid_id,
"winning_bid_id": auction.winning_bid_id,
"created_at": auction.created_at,
"started_at": auction.started_at,
"closed_at": auction.closed_at,
}
bid_id = None
if auction and auction.winning_bid_id:
bid_id = auction.winning_bid_id
elif auction and auction.highest_bid_id:
bid_id = auction.highest_bid_id
bid_info = None
if bid_id:
prevailing_bid = Bid.one(bid_id)
if prevailing_bid:
bid_info = {
"id": prevailing_bid.id,
"offer_price": prevailing_bid.offer_price,
"participant_id": prevailing_bid.participant_id,
"submitted_at": prevailing_bid.submitted_at,
}
return {
"status": "success",
"item": item_info,
"auction": auction_info,
"prevailing_bid": bid_info,
}
def get_list_info(self, url_page, html_type, auction_type):
response = requests.get(url_page, headers=self.headers)
html = response.text
tree = etree.HTML(html)
div_list = tree.xpath('//div[@class="sflistdiv"]')
for i in div_list:
info = []
auction = Auction(source, auction_type)
auction.province = '上海'
auction.city = '上海'
auction.html_type = html_type
auction.source_html = html
auction_id = i.xpath('div[@class="sflistdivn2"]/div[@class="f20hei"]/a/@href')[0].split('/')[-1]
is_exist = coll.find_one({'auction_id': str(auction_id), 'source': source})
if is_exist:
log.info('id已存在，id="{}"'.format(str(auction_id)))
continue
auction.auction_id = auction_id
try:
auction_name_ = i.xpath('div[@class="sflistdivn2"]/div[@class="f20hei"]/a/text()')[0]
except Exception as e:
auction_name_ = ''
region = i.xpath('div[@class="sflistdivn2"]/div[@class="sflistban"]/text()')[0]
auction.region = re.search(' - (.*?)$', region, re.S | re.M).group(1)
auction_time_ = i.xpath('div[@class="sflistdivn2"]/div[@class="sflisttime"]/text()')[0]
address = i.xpath('div[@class="sflistdivn2"]/div[@class="sflistcan"]/text()')[3].encode().decode()
auction.auction_name = auction_name_ + address
try:
auction_time = re.search('拍卖时间：(.*?)$', auction_time_, re.S | re.M).group(1)
auction.auction_time = datetime.datetime.strptime(auction_time, "%y.%m.%d")
except Exception as e:
auction.auction_time = None
info.append(i.xpath('string(div[@class="sflistdivn2"])'))
area_ = i.xpath('div[@class="sflistdivn2"]/div[@class="sflistcan"]/span[1]/text()')[0]
auction.area = re.search('面积：(.*?)$', area_, re.S | re.M).group(1)
floor = i.xpath('div[@class="sflistdivn2"]/div[@class="sflistcan"]/span[3]/text()')[0]
auction.floor = re.search('楼层：(.*?)$', floor, re.S | re.M).group(1)
start_auction_price = i.xpath('//div[@class="f34hong"]/text()')[0]
auction.start_auction_price = float(
re.search('(\d+),?(\d+)', start_auction_price, re.S | re.M).group(1).replace(',', '')) * 10000
auction.insert_db()
def get_info(self, url):
response = requests.get(url=url, headers=self.headers)
html = etree.HTML(response.text)
print(url)
wrong_list = []
try:
wrong = html.xpath("//div[@class='dialog']/h1/text()")[0]
wrong_list.append(wrong)
except Exception as e:
print(e)
if "We're sorry, but something went wrong." not in wrong_list:
title = html.xpath("//div[@class='title']/text()")[0]
start_price = html.xpath(
"//table[@class='item-attrs']//tr[1]/td[2]/text()")[0]
assess_price = html.xpath(
"//table[@class='item-attrs']//tr[1]/td[4]/text()")[0]
ensure_price = html.xpath(
"//table[@class='item-attrs']//tr[1]/td[6]/text()")[0]
auction_id = re.search(
"http://auction\.qdauction\.com/items/(\d+)", url).group(1)
auction = Auction(source=source, auction_type=auction_type)
auction.auction_name = title
auction.start_auction_price = start_price
auction.assess_value = assess_price
auction.earnest_money = ensure_price
auction.auction_id = auction_id
try:
time = html.xpath("//tr[@class='deal']/td[4]/text()")[0]
Auction.auction_time = datetime.datetime.strptime(
time, "%Y-%m-%d %H:%M:%S")
except Exception as e:
print(e)
auction.source_html = response.text
auction.city = '青岛'
auction.html_type = '其他'
auction.insert_db()
def gen_token():
curve = PriceSupplyCurve(factor=0.000001, base_price=1)
auction = Auction()
beneficiary = Beneficiary(issuance_fraction=.2)
ct = Mint(curve, beneficiary, auction)
return ct
def start_crawler(self):
for type_num in type_list:
page_num = self.get_page(type_num.code)
for page in range(1, int(page_num) + 1):
url = 'http://auction.jd.com/getJudicatureList.html?page=' + str(
page) + '&limit=40&childrenCateId=' + type_num.code
try:
response = s.get(url, headers=self.headers)
html = response.json()
try:
for info in html['ls']:
auction = Auction(
source=source,
auction_type=type_num.auction_type)
auction.html_type = type_num.html_type
auction.auction_name = info['title'] # 商品名
auction.assess_value = info[
'assessmentPrice'] # 评估值
try:
auction.province = info['province'] # 省
auction.city = info['city'] # 城市
except Exception as e:
auction.province = None
auction.city = None
auction.auction_time = datetime.datetime.fromtimestamp(
int(info['startTime']) / 1000) # 评估值
auction.earnest_money = info['currentPrice'] # 保证金
auction.auction_id = str(info['id']) # 商品id
is_exist = coll.find_one({
'auction_id':
str(info['id']),
'source':
source
})
if is_exist:
log.info('id已存在，id="{}"'.format(str(
info['id'])))
continue
self.get_detail(str(info['id']), auction)
except Exception as e:
log.error('解析错误，url="{}"'.format(url))
except Exception as e:
log.error('请求错误，url="{}"'.format(url))
def auction(self):
if not getattr(self, '_auction', None):
self._auction = Auction()
return self._auction
wb.save(fname)
def output_all(self):
self.output_hist_to_xlsx("output_master_aggregate.xlsx")
for ix, puid in enumerate(self.puids):
self.output_policy_info_to_xlsx(
"output_policy_info_{}.xlsx".format(puid), ix)
self.output_time_logged_to_xlsx("output_time_spent_in_seconds.xlsx")
if __name__ == "__main__":
t_start = time.time()
print("{:.2f} sec: start loading simulator".format(time.time() - t_start))
sim = Simulator()
param, attrs = Auction.read_init_xlsx("auction_ini_01.xlsx")
auc = Auction(param, attrs)
aucts = auc.generate_sample()
# aucts = sl.load_auction_p("auction_01.p") # loading from a snapshot example.
sim.read_in_auction(aucts)
print("{:.2f} sec: finished loading simulator".format(time.time() -
t_start))
for t in range(param['max iteration']):
sim_res = sim.step()
print("{:.2f} sec: simulation iter {}, auction happened? {}".format(
time.time() - t_start, t, sim_res))
sim.output_all()
print("{:.2f} sec: created output files.".format(time.time() - t_start))
pass
def get_detail(self, id_, auction_time, html_type, auction_type, province,
city, region):
auction = Auction(source=source, auction_type=auction_type)
auction.html_type = html_type
auction.auction_type = auction_type
auction.province = province
auction.city = city
auction.region = region
detail_url = 'http://www1.rmfysszc.gov.cn/Handle/' + id_ + '.shtml'
try:
response = requests.get(detail_url, headers=self.headers)
html = response.content.decode()
auction.source_html = html
info_list = []
try:
if 'GetRecord()' in html:
tree = etree.HTML(html)
auction.auction_name = tree.xpath(
'//div[@id="Title"]/h1/text()')[0]
start_auction_price = tree.xpath(
'//*[@id="price"]/div[1]/span/text()')[0]
auction.start_auction_price = self.get_float(
start_auction_price)
assess_value = tree.xpath(
'//*[@id="bg1"]/div[1]/table/tr[1]/td/span[2]/text()'
)[0]
try:
auction.assess_value = self.get_float(assess_value)
except Exception as e:
auction.assess_value = None
earnest_money = tree.xpath(
'//*[@id="bg1"]/div[1]/table/tr[2]/td/span[2]/text()'
)[0]
auction.earnest_money = self.get_float(earnest_money)
announcement_date = tree.xpath(
'//*[@id="bg1"]/div[1]/table/tr[3]/td/span/text()')[0]
announcement_date_ = re.search(': (.*?)$',
announcement_date,
re.S | re.M).group(1)
auction.announcement_date = datetime.datetime.strptime(
announcement_date_, "%Y.%m.%d")
auction_level = tree.xpath(
'//*[@id="bg1"]/div[1]/table/tr[4]/td/span/text()')[0]
auction.auction_level = re.search(': (.*?)$',
auction_level,
re.S | re.M).group(1)
court = tree.xpath(
'//*[@id="bg1"]/div[2]/table/tr[1]/td/span/text()')[0]
auction.court = re.search(': (.*?)$', court,
re.S | re.M).group(1)
info_list.append(
tree.xpath(
'string(//*[@id="bdjs11"])').encode().decode())
info_list.append(
tree.xpath(
'string(//*[@id="jjjl"])').encode().decode())
contacts = tree.xpath(
'//*[@id="bg1"]/div[2]/table/tr[2]/td/span/text()')[0]
auction.contacts = re.search(': (.*?)$', contacts,
re.S | re.M).group(1)
phone_number = tree.xpath(
'//*[@id="bg1"]/div[2]/table/tr[3]/td/span/text()')[0]
auction.phone_number = re.search(': (.*?)$', phone_number,
re.S | re.M).group(1)
auction.info = info_list
try:
auction.build_type = tree.xpath(
'//*[@id="bdjs11"]/table[1]/tr[2]/td[4]/text()')[0]
except Exception as e:
auction.build_type = None
auction.auction_id = id_
auction.auction_time = self.get_date(date=auction_time)
auction.insert_db()
elif 'bmnumber()' in html:
tree = etree.HTML(html)
auction.auction_name = tree.xpath(
'//div[@id="Title"]/h1/text()')[0]
start_auction_price = tree.xpath(
'//*[@id="price"]/div[1]/span/text()')[0]
auction.start_auction_price = self.get_float(
start_auction_price)
assess_value = tree.xpath(
'//*[@id="bg1"]/div[1]/table/tr[1]/td/span[2]/text()'
)[0]
auction.assess_value = self.get_float(assess_value)
earnest_money = tree.xpath(
'//*[@id="bg1"]/div[1]/table/tr[2]/td/span[2]/text()'
)[0]
auction.earnest_money = self.get_float(earnest_money)
announcement_date = tree.xpath(
'//*[@id="bg1"]/div[1]/table/tr[3]/td/span/text()')[0]
announcement_date_ = re.search(': (.*?)$',
announcement_date,
re.S | re.M).group(1)
auction.announcement_date = datetime.datetime.strptime(
announcement_date_, "%Y-%m-%d")
auction_level = tree.xpath(
'//*[@id="bg1"]/div[1]/table/tr[4]/td/span/text()')[0]
auction.auction_level = re.search(': (.*?)$',
auction_level,
re.S | re.M).group(1)
court = tree.xpath(
'//*[@id="bg1"]/div[2]/table/tr[1]/td/span/text()')[0]
auction.court = re.search(': (.*?)$', court,
re.S | re.M).group(1)
info_list.append(
tree.xpath(
'string(//*[@id="bdjs"])').encode().decode())
contacts = tree.xpath(
'//*[@id="bg1"]/div[2]/table/tr[2]/td/span/text()')[0]
auction.contacts = re.search(': (.*?)$', contacts,
re.S | re.M).group(1)
phone_number = tree.xpath(
'//*[@id="bg1"]/div[2]/table/tr[3]/td/span/text()')[0]
auction.phone_number = re.search(': (.*?)$', phone_number,
re.S | re.M).group(1)
auction.info = info_list
try:
auction.build_type = tree.xpath(
'//*[@id="bdjs11"]/table[1]/tr[2]/td[4]/text()')[0]
except Exception as e:
auction.build_type = None
auction.auction_id = id_
auction.auction_time = self.get_date(date=auction_time)
auction.insert_db()
else:
tree = etree.HTML(html)
auction.auction_name = tree.xpath(
'//*[@id="xmgg"]/div/div[1]/text()')[0]
assess_value = tree.xpath(
'/html/body/div[6]/table/tr/td/ul/li[3]/span/text()'
)[0]
auction.assess_value = self.get_float(assess_value)
announcement_date = tree.xpath(
'/html/body/div[6]/table/tr/td/ul/li[2]/span/text()'
)[0]
try:
auction.announcement_date = datetime.datetime.strptime(
announcement_date, "%Y-%m-%d")
except Exception as e:
auction.announcement_date = datetime.datetime.strptime(
announcement_date, "%Y/%m/%d")
auction.court = tree.xpath(
'/html/body/div[6]/table/tr/td/ul/li[1]/span/text()'
)[0]
info_list.append(
tree.xpath(
'string(//*[@id="bdxx"]/div)').encode().decode())
info_list.append(
tree.xpath('string(//*[@id="tjzl"]/div/div[2])').
encode().decode())
auction.contacts = tree.xpath(
'/html/body/div[6]/table/tr/td/ul/li[4]/span/text()'
)[0]
auction.phone_number = tree.xpath(
'/html/body/div[6]/table/tr/td/ul/li[5]/span/text()'
)[0]
auction.info = info_list
try:
auction.build_type = tree.xpath(
'//*[@id="bdxx"]/div/div[2]/table/tr[2]/td[3]/text()'
)[0]
except Exception as e:
auction.build_type = None
auction.auction_id = id_
auction.auction_time = self.get_date(date=auction_time)
auction.insert_db()
except Exception as e:
log.error('解析错误,url="{}",e="{}"'.format(detail_url, e))
except Exception as e:
log.error('详情页请求错误,url="{}",e="{}"'.format(detail_url, e))
def step(self):
"""
simulates one timestep in the auction
:return: True if auction is simulated, False if no auction data is present
"""
self.time_last = time.time()
self.t += 1
auction_happened = False
events = []
for a in self.auctions:
if a['iter'] != self.t:
continue
if len(self.hist) == 0:
costs_sum = [0.0] * len(self.pols)
revenues_sum = [0.0] * len(self.pols)
profits_sum = [0.0] * len(self.pols)
else:
costs_sum = deepcopy(self.hist[-1]['costs_cumulative'])
revenues_sum = deepcopy(self.hist[-1]['revenues_cumulative'])
profits_sum = deepcopy(self.hist[-1]['profits_cumulative'])
auction_happened = True
# bids = [float(p.bid(a['attr'])) for p in self.pols]
# bids = []
pool = Pool(24)
results = pool.starmap(
Simulator().bid_from_policy,
zip(self.puids, self.pols, [a] * len(self.pols)))
pool.close()
pool.join()
bids = [b for (b, t, p) in results]
time_last = [t for (b, t, p) in results]
pols = [p for (b, t, p) in results]
self.pols = pols
dicts = {}
for dict in time_last:
dicts.update(dict)
for n in dicts.keys():
if n in self.time_spent.keys():
self.time_spent[n] += dicts[n]
else:
self.time_spent[n] = dicts[n]
#for puid, p in zip(self.puids, self.pols):
# self._time_log('simulator')
# this_bid = p.bid(a['attr'])
# self._time_log(puid)
# bids.append(float(this_bid))
max_bid_pols_ix = sl.max_ix(bids)
winning_bid = bids[max_bid_pols_ix[0]]
# take top K bids -- because K slots are there
reverse_sorted_bids, sorted_pIx = sl.top_K_max(
bids, self.num_of_ad_slots, self.prng)
sorted_unique_bids = sorted(list(set(bids)))
num_clicks, p_click = self.get_num_clicks(winning_bid, a)
cost = sl.compute_second_price_cost(
bids, size=num_clicks) # second price auction
conversion = Auction.get_conversion(a['prob_conversion'],
self.prng,
size=num_clicks)
revenue = Auction.get_revenue_sample(a['avg_revenue'],
self.prng,
size=num_clicks)
winning_pol_ix = []
for ix in range(num_clicks):
# winner_ix = int(self.prng.choice(max_bid_pols_ix)) # max-bidder-wins case
# if geometric click prob, then winner_ix is one of top-K bids
# fill K positions with pIx, in non-decreasing order of bids[pIx]
# and choose one of K with custom set probability in geometrically decaying probability
# that chosen one is winner_ix of this click.
winner_ix = int(
self.prng.choice(sorted_pIx,
p=self.ad_slot_click_prob_adjuster))
winning_bid = bids[winner_ix]
winning_pol_ix.append(winner_ix)
# compute actual cost (second price) for each click (as each click may have different winner than max bidder)
actual_cost = sl._compute_actual_second_price_cost(
bids[winner_ix], sorted_unique_bids)
cost[ix] = actual_cost
costs_sum[winner_ix] += cost[ix]
revenues_sum[winner_ix] += conversion[ix] * revenue[ix]
profits_sum[
winner_ix] = revenues_sum[winner_ix] - costs_sum[winner_ix]
event = {
'iter': self.t,
'attr': a['attr'],
'auctions_in_iter': a['num_auct'],
'bids': bids,
'winning_pol_id': winner_ix,
'winning_pol_name': self.puids[winner_ix],
'winning_bid': winning_bid,
'num_click': 1,
'cost_per_click': cost[ix],
'num_conversion': conversion[ix],
'revenue_per_conversion': conversion[ix] * revenue[ix],
'costs_cumulative': deepcopy(costs_sum),
'revenues_cumulative': deepcopy(revenues_sum),
'profits_cumulative': deepcopy(profits_sum)
}
events.append(event)
self.events.append(event)
if num_clicks == 0:
winner_ix = int(self.prng.choice(max_bid_pols_ix))
event = {
'iter': self.t,
'attr': a['attr'],
'auctions_in_iter': a['num_auct'],
'bids': bids,
'winning_pol_id': winner_ix,
'winning_pol_name': self.puids[winner_ix],
'winning_bid': winning_bid,
'num_click': 0,
'cost_per_click': '',
'num_conversion': 0,
'revenue_per_conversion': '',
'costs_cumulative': deepcopy(costs_sum),
'revenues_cumulative': deepcopy(revenues_sum),
'profits_cumulative': deepcopy(profits_sum)
}
events.append(event)
self.events.append(event)
# keep aggregate history for output
auct_res = deepcopy(a)
auct_res['bids'] = bids
auct_res['num_click'] = num_clicks
auct_res['p_click'] = p_click
auct_res['cost_per_click'] = mean(cost) if num_clicks > 0 else ''
auct_res['num_conversion'] = sum(conversion)
auct_res['revenue_per_conversion'] = sum([
ncr * rpc for (ncr, rpc) in zip(conversion, revenue)
]) / sum(conversion) if sum(conversion) > 0 else ''
auct_res['costs_cumulative'] = deepcopy(costs_sum)
auct_res['revenues_cumulative'] = deepcopy(revenues_sum)
auct_res['profits_cumulative'] = deepcopy(profits_sum)
auct_res['time_spent'] = deepcopy(self.time_spent)
self.hist.append(auct_res)
# end of auction events handling
# if nothing happened. this is the way to go
if len(events) == 0:
return auction_happened
# aggregate information over one iteration is assembled, for each policy
p_infos = {}
for p_ix, p in enumerate(self.pols):
p_infos[p_ix] = []
last_a = events[0]['attr']
bunch = []
if len(self.p_infos[p_ix]) == 0:
profit_sum = 0.0
else:
profit_sum = self.p_infos[p_ix][-1][-1][
'your_profit_cumulative']
for ev in events + [{'attr': 'guard_dummy'}]:
this_a = ev['attr']
if last_a == this_a and ev != 'guard_dummy':
bunch.append(ev)
else:
bids_that_got_clicked = [ev['winning_bid'] for ev in bunch]
# aggregate information
p_info = {
'iter': bunch[0]['iter'],
'attr': bunch[0]['attr'],
'num_auct': bunch[0]['auctions_in_iter'],
'your_bid': bunch[0]['bids'][p_ix],
'winning_bid': max(bids_that_got_clicked),
'winning_bid_avg': mean(bids_that_got_clicked)
}
win_count = []
clicks = []
costs = []
conversions = []
revenues = []
for ev_b in bunch:
if p_ix == ev_b['winning_pol_id']:
win_count.append(1)
clicks.append(ev_b['num_click'])
costs.append(ev_b['cost_per_click'])
conversions.append(ev_b['num_conversion'])
revenues.append(ev_b['revenue_per_conversion'])
approx_num_impression = int(p_info['num_auct'] *
sum(win_count) / len(bunch))
p_add_info = {
'num_impression':
approx_num_impression,
'num_click':
sum(clicks),
'cost_per_click':
mean(costs) if sum(clicks) > 0 else '',
'num_conversion':
sum(conversions),
'revenue_per_conversion':
sum(revenues) /
sum(conversions) if sum(conversions) > 0 else ''
}
this_cost = sum([c if c != '' else 0 for c in costs])
this_revenue = sum([
ncr * rpc
for (ncr, rpc
) in zip(conversions,
[r if r != '' else 0 for r in revenues])
])
profit_sum += this_revenue - this_cost
p_info['your_profit_cumulative'] = profit_sum
p_info.update(p_add_info)
p_infos[p_ix].append(p_info)
# refresh rolling aggregator and its guard
bunch = [ev]
last_a = this_a
# post-auction learning session for policies
for p_ix, p in enumerate(self.pols):
self._time_log('simulator')
p.learn(p_infos[p_ix])
self._time_log(self.puids[p_ix])
self.p_infos[p_ix].append(p_infos[p_ix])
# finish up
self._time_log('simulator')
print(self.time_spent)
return auction_happened
def parse(self, html):
auction_list = html.xpath("//dl/dd/a/@href")
for auction_url in auction_list:
try:
url = 'http://www.shjiapai.cn' + auction_url
auction_res = requests.get(url, headers=self.headers)
con = auction_res.text
auction_id = re.search('id/(\d+).html', auction_url).group(1)
if not check_auction(source=source, auction_id=auction_id):
auction = Auction(source=source, auction_type=auction_type)
auction.source_html = con
auction.auction_id = auction_id
auction.auction_name = re.search('楼盘名称.*?">(.*?)</td', con,
re.S | re.M).group(1)
auction.city = '上海'
auction.html_type = '房产'
auction.start_auction_price = re.search(
'预计售价.*?">(.*?)</td', con, re.S | re.M).group(1)
auction.floor = re.search('层.*?">(.*?)楼</td', con,
re.S | re.M).group(1)
auction.area = re.search('户型面积.*?">(.*?)</td', con,
re.S | re.M).group(1)
auction.build_type = re.search('物业类型.*?">(.*?)</td', con,
re.S | re.M).group(1)
auction.info = re.search('其它.*?>(.*?)</div', con,
re.S | re.M).group(1)
auction.insert_db()
else:
log.info("数据已存在")
except Exception as e:
log.error("{}解析失败".format(auction_url))
def get_detail(self, source, auction_id, auction_status):
response, url = self.request_url(auction_id)
try:
tree = etree.HTML(response.text)
html = response.text
auction = Auction(source=source)
auction.url = url
# 起拍价(万)
startPrice = self.get_startPrice(html=html)
startPrice = float(startPrice.replace(' ', '').replace(',', '')) / 10000
auction.startPrice = startPrice
# 评估价(万)
evalPrice = self.get_evalPrice(tree=tree)
evalPrice = float(evalPrice.replace(' ', '').replace('¥', '').replace(',', '')) / 10000
auction.evalPrice = evalPrice
# 保证金(万)
bond = tree.xpath('//div[@id="content"]/div/div[2]/div[1]/div/div[2]/div[10]/ul[3]/li/span[2]/text()')[0]
bond = bond.replace(' ', '').replace('¥', '').split('.')[0]
bond = float(bond) / 10000
auction.bond = bond
# 拍卖阶段
auctionStage_info = tree.xpath('//div[@id="content"]/div[1]/div[2]/div[1]/div[1]/div[2]/h1/text()')[0]
auctionStage_info = auctionStage_info.replace(' ', '').replace('\n', '').replace('\t', '')
auctionStage = auctionStage_info.split('】')[0].split('【')[1]
auction.auctionStage = auctionStage
# 拍卖次数
auctionCount = self.get_auctionCount(auctionStage)
auction.auctionCount = auctionCount
# 拍卖物品名称
auction_name = tree.xpath('//div[@id="content"]/div[1]/div[2]/div[1]/div[1]/div[2]/h1/text()')[0]
auction_name = auction_name.replace(' ', '').replace('\n', '').replace('\t', '').split('】')[1]
auction.auction_name = auction_name
# todo 调用黄村粮的方法进行切割获取城市，区域，小区名，经纬度等
cut_info = CutMatch.to_match('上海', auction_name)
# 切割后匹配库中的城市
auction.matchCity = cut_info['matchCity']
# 切割后匹配库中的区域
auction.matchRegion = cut_info['matchRegion']
# 切割后匹配库中的小区名称
auction.matchName = cut_info['matchName']
# 切割后匹配库中的地址
auction.matchAddress = cut_info['matchAddress']
# 切割后的房号
auction.roomNum = cut_info['cutRoomnum']
# 切割后的楼号
auction.houseNum = cut_info['cutHousenum']
# 切割后的城市
auction.cutCity = cut_info['cutCity']
# 切割后的区域
auction.cutRegion = cut_info['cutRegion']
# 切割后的小区名称
auction.cutName = cut_info['cutName']
# 切割后的地址
auction.cutAddress = cut_info['cutAddress']
# 切割后跑高德接口得到的经纬度
auction.lat = cut_info['mapLat']
auction.lng = cut_info['mapLng']
# 地址
address = tree.xpath('//em[@id="paimaiAddress"]/text()')[0]
auction.address = address
# 城市
city = address.split(' ')[0]
auction.city = city
# 区域
region = address.split(' ')[1]
auction.region = region
skulid = re.search('id="skuId" value="(.*?)"', html, re.S | re.M).group(1)
# 竞拍状态 当前价格 成交价格
curPrice, dealPrice = self.get_curPrice_and_dealPrice(skulid, auction_id)
auction.biddingState = auction_status
auction.curPrice = curPrice
auction.dealPrice = dealPrice
# 起始时间
startShootingDate = tree.xpath('//input[@id="startTime"]/@value')[0]
startShootingDate = datetime.datetime.strptime(startShootingDate, '%Y-%m-%d %H:%M:%S.%f')
auction.startShootingDate = startShootingDate
# 结束时间
endShootingDate = tree.xpath('//input[@id="endTime"]/@value')[0]
endShootingDate = datetime.datetime.strptime(endShootingDate, '%Y-%m-%d %H:%M:%S.%f')
auction.endShootingDate = endShootingDate
# 图片
houseImgUrls = []
houseImgUrls_info = tree.xpath('//div[@id="spec-list"]/div/ul/li/img/@src')
for houseImgUrl in houseImgUrls_info:
houseImgUrl = 'http:' + houseImgUrl
big_img = houseImgUrl.replace('jfs','s1000x750_jfs')
new_houseImgUrl = qiniufetch(big_img, big_img)
houseImgUrls.append(new_houseImgUrl)
auction.houseImgUrls = houseImgUrls
# 参与人数 浏览数量
participantsNumber, visitCount = self.get_participantsNumber_and_visitCount(auction_id)
auction.participantsNumber = participantsNumber
auction.visitCount = visitCount
# 拍卖物品id
auction.auctionId = auction_id
# 网站来源(jingdong)
auction.source = source
auction.update()
except Exception as e:
log.error('解析错误')
