# place holder for listings and about page # shop['about'] = { } # shop['active_listings'] = [] f.write(json.dumps(shop) + '\n') ##################### ## begin main script ##################### # n: number of shops to load # d: how to treat existing data files n, test_size, d = get_shop_ct() # load n shops shops = Shops() # can only pull 100 shops per query all_shops = [] rem = n while (rem): ns = shops.findAll({ 'limit': min(rem, 100), 'offset': len(all_shops) }) add = [] for s in ns: if not any(s['shop_id'] == asi['shop_id'] for asi in all_shops): add.append(s) all_shops.extend(add) print 'added shop ct: %s' % len(add) rem -= len(add)
# parse args arg_dict = cli_opts(sys.argv[1:], inp, call_conv) return arg_dict['-f'], arg_dict['-o'], arg_dict['-d'] ###################### ## begin script ###################### f, o, d = get_shop_file() temp_f = '{0}-temp.json'.format(f.split('.')[0]) e_shops = Shops() with open(f, 'r') as shops, \ open(temp_f, 'w+') as tp, \ open(o, d) as listings: over_limit = False for s in shops: shop = json.loads(s) # if already over limit or no active listings, don't bother trying if not over_limit: # if listings haven't been assigned, query them try: # the get and setdefault methods would yield # a query call each time and waste my 5k allocation
# define command line calling convetion for script for error reporting call_conv = "fill_shop_about.py -f <shop file> -o <about out> -d <over/append out>" # parse args arg_dict = cli_opts(sys.argv[1:], inp, call_conv) return arg_dict["-f"], arg_dict["-o"], arg_dict["-d"] ###################### ## begin script ###################### f, o, d = get_shop_file() temp_f = "{0}-temp.json".format(f.split(".")[0]) e_shops = Shops() with open(f, "r") as shops, open(temp_f, "w+") as tp, open(o, d) as about_f: over_limit = False for s in shops: shop = json.loads(s) # if already over limit or no active listings, don't bother trying if not over_limit: # if listings haven't been assigned, query them try: # the get and setdefault methods would yield # a query call each time and waste my 5k allocation # better to just throw an error on key lookup shop["about"] except: