def test_item():
# do not test this
return
for id in [15779110791, 19667991376, 21825059650, 19536353824, 19536097533, 19521682005, 19536509268]:
data = get_item(id)
assert data['id'] == id
assert isinstance(data['price'], float)
def test_item():
# do not test this
return
for id in [
15779110791, 19667991376, 21825059650, 19536353824, 19536097533,
19521682005, 19536509268
]:
data = get_item(id)
assert data['id'] == id
assert isinstance(data['price'], float)
def blacklist_add():
global blacklist, whitelist
if whitelist is None:
whitelist = []
if blacklist is None:
blacklist = []
type = request.args.get('type')
if type == 'blacklist':
for itemid in request.args.get('itemid', '0').split(','):
try:
itemid = int(itemid)
except:
itemid = 0
if itemid == 0:
# illegal input
pass
elif itemid not in [v[1] for v in blacklist]:
# should get shopid and save to blacklist
i = get_item(itemid)
if not i or 'error' in i:
pass
else:
if i['shopid'] != 0:
shopid = i['shopid']
print 'inserting shopblack, {}, {}'.format(
shopid, itemid)
blacklist.insert(0, [shopid, itemid])
db.execute(
'insert into ataobao2.blacklist (type, args, value) values (:type, :args, :value)',
dict(type='shopblack',
args=str(shopid),
value=str(itemid)))
elif type == 'whitelist':
for shopid in request.args.get('shopid', '0').split(','):
try:
shopid = int(shopid)
except:
shopid = 0
if shopid == 0:
pass
else:
print 'inserting shopwhite, {}'.format(shopid)
whitelist.insert(0, [shopid, None])
db.execute(
'insert into ataobao2.blacklist (type, args) values (:type, :args)',
dict(type='shopwhite', args=str(shopid)))
return redirect('/blacklist/#' + type)
dict(shopid=shopid))
db.execute('''delete from ataobao2.shop_by_date where id=:shopid''',
dict(shopid=shopid))
def update_shop(shop):
d = shop
d['rating'] = json.dumps(d['rating'])
if 'promise' in d:
d['promise'] = json.dumps(d['promise'])
table = tables['shop']
keys = list(set(table['cols'].keys()) & set(d.keys()))
arg1 = '(' + ', '.join(keys) + ')'
arg2 = '(:' + ', :'.join(keys) + ')'
insert_into_shop = '''INSERT INTO ataobao2.shop {} VALUES {}'''.format(
arg1, arg2)
db.execute(insert_into_shop, d)
if __name__ == '__main__':
from crawler.tbitem import get_item
from crawler.tbshopinfo import get_shop
item = get_item(36305605982)
from pprint import pprint
pprint(item)
update_item(item)
#shop = get_shop(63782021)
#pprint(shop)
#update_shop(shop)
def printget():
print time.time(), 'id' in get_item(22183623058)
def bench(itemid):
global count
get_item(itemid)
count += 1
def warmup():
print 'warmup'
get_item(itemid)
print 'warmed'
def bench(itemid):
global count
get_item(itemid)
count += 1
def warmup():
print 'warmup'
get_item(itemid)
print 'warmed'
db.execute('''delete from ataobao2.item where id=:itemid''', dict(itemid=itemid))
db.execute('''delete from ataobao2.item_by_date where id=:itemid''', dict(itemid=itemid))
def delete_shop(shopid):
db.execute('''delete from ataobao2.shop where id=:shopid''', dict(shopid=shopid))
db.execute('''delete from ataobao2.shop_by_date where id=:shopid''', dict(shopid=shopid))
def update_shop(shop):
d = shop
d['rating'] = json.dumps(d['rating'])
if 'promise' in d:
d['promise'] = json.dumps(d['promise'])
table = tables['shop']
keys = list(set(table['cols'].keys()) & set(d.keys()))
arg1 = '('+', '.join(keys)+')'
arg2 = '(:'+', :'.join(keys)+')'
insert_into_shop = '''INSERT INTO ataobao2.shop {} VALUES {}'''.format(arg1, arg2)
db.execute(insert_into_shop, d)
if __name__ == '__main__':
from crawler.tbitem import get_item
from crawler.tbshopinfo import get_shop
item = get_item(36305605982)
from pprint import pprint
pprint(item)
update_item(item)
#shop = get_shop(63782021)
#pprint(shop)
#update_shop(shop)