def test_item(): # do not test this return for id in [15779110791, 19667991376, 21825059650, 19536353824, 19536097533, 19521682005, 19536509268]: data = get_item(id) assert data['id'] == id assert isinstance(data['price'], float)
def test_item(): # do not test this return for id in [ 15779110791, 19667991376, 21825059650, 19536353824, 19536097533, 19521682005, 19536509268 ]: data = get_item(id) assert data['id'] == id assert isinstance(data['price'], float)
def blacklist_add(): global blacklist, whitelist if whitelist is None: whitelist = [] if blacklist is None: blacklist = [] type = request.args.get('type') if type == 'blacklist': for itemid in request.args.get('itemid', '0').split(','): try: itemid = int(itemid) except: itemid = 0 if itemid == 0: # illegal input pass elif itemid not in [v[1] for v in blacklist]: # should get shopid and save to blacklist i = get_item(itemid) if not i or 'error' in i: pass else: if i['shopid'] != 0: shopid = i['shopid'] print 'inserting shopblack, {}, {}'.format( shopid, itemid) blacklist.insert(0, [shopid, itemid]) db.execute( 'insert into ataobao2.blacklist (type, args, value) values (:type, :args, :value)', dict(type='shopblack', args=str(shopid), value=str(itemid))) elif type == 'whitelist': for shopid in request.args.get('shopid', '0').split(','): try: shopid = int(shopid) except: shopid = 0 if shopid == 0: pass else: print 'inserting shopwhite, {}'.format(shopid) whitelist.insert(0, [shopid, None]) db.execute( 'insert into ataobao2.blacklist (type, args) values (:type, :args)', dict(type='shopwhite', args=str(shopid))) return redirect('/blacklist/#' + type)
dict(shopid=shopid)) db.execute('''delete from ataobao2.shop_by_date where id=:shopid''', dict(shopid=shopid)) def update_shop(shop): d = shop d['rating'] = json.dumps(d['rating']) if 'promise' in d: d['promise'] = json.dumps(d['promise']) table = tables['shop'] keys = list(set(table['cols'].keys()) & set(d.keys())) arg1 = '(' + ', '.join(keys) + ')' arg2 = '(:' + ', :'.join(keys) + ')' insert_into_shop = '''INSERT INTO ataobao2.shop {} VALUES {}'''.format( arg1, arg2) db.execute(insert_into_shop, d) if __name__ == '__main__': from crawler.tbitem import get_item from crawler.tbshopinfo import get_shop item = get_item(36305605982) from pprint import pprint pprint(item) update_item(item) #shop = get_shop(63782021) #pprint(shop) #update_shop(shop)
def printget(): print time.time(), 'id' in get_item(22183623058)
def bench(itemid): global count get_item(itemid) count += 1
def warmup(): print 'warmup' get_item(itemid) print 'warmed'
db.execute('''delete from ataobao2.item where id=:itemid''', dict(itemid=itemid)) db.execute('''delete from ataobao2.item_by_date where id=:itemid''', dict(itemid=itemid)) def delete_shop(shopid): db.execute('''delete from ataobao2.shop where id=:shopid''', dict(shopid=shopid)) db.execute('''delete from ataobao2.shop_by_date where id=:shopid''', dict(shopid=shopid)) def update_shop(shop): d = shop d['rating'] = json.dumps(d['rating']) if 'promise' in d: d['promise'] = json.dumps(d['promise']) table = tables['shop'] keys = list(set(table['cols'].keys()) & set(d.keys())) arg1 = '('+', '.join(keys)+')' arg2 = '(:'+', :'.join(keys)+')' insert_into_shop = '''INSERT INTO ataobao2.shop {} VALUES {}'''.format(arg1, arg2) db.execute(insert_into_shop, d) if __name__ == '__main__': from crawler.tbitem import get_item from crawler.tbshopinfo import get_shop item = get_item(36305605982) from pprint import pprint pprint(item) update_item(item) #shop = get_shop(63782021) #pprint(shop) #update_shop(shop)