def main(_): pp.pprint(flags.FLAGS.__flags) if not os.path.exists(FLAGS.checkpoint_dir): os.makedirs(FLAGS.checkpoint_dir) if not os.path.exists(FLAGS.sample_dir): os.makedirs(FLAGS.sample_dir) with tf.Session() as sess: if FLAGS.dataset == 'mnist': dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size, y_dim=10, dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir) else: dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size, dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir) if FLAGS.is_train: dcgan.train(FLAGS) else: dcgan.load(FLAGS.checkpoint_dir) to_json("./web/js/gen_layers.js", dcgan.h0_w, dcgan.h1_w, dcgan.h2_w, dcgan.h3_w, dcgan.h4_w) z_sample = np.random.uniform(-1, 1, size=(FLAGS.batch_size, dcgan.z_dim)) samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample}) save_images(samples, [8, 8], './samples/test_%s.png' % strftime("%Y-%m-%d %H:%M:%S", gmtime()))
def main(_): pp.pprint(flags.FLAGS.__flags) if not os.path.exists(FLAGS.checkpoint_dir): os.makedirs(FLAGS.checkpoint_dir) if not os.path.exists(FLAGS.sample_dir): os.makedirs(FLAGS.sample_dir) with tf.Session() as sess: if FLAGS.dataset == 'mnist': dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size, y_dim=10, dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir) else: dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size, dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir) if FLAGS.is_train: dcgan.train(FLAGS) else: dcgan.load(FLAGS.checkpoint_dir) to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0], [dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1], [dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2], [dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3], [dcgan.h4_w, dcgan.h4_b, None]) # Below is codes for visualization OPTION = 2 visualize(sess, dcgan, FLAGS, OPTION)
def _send_api_command(self, request): """ Send an API command to Galah. :param request: A properly formed JSON object to send Galah. :returns: A ``requests.Response`` object. """ request = copy.copy(request) # Extract any files file_args = {} for i in (k for k, v in request.items() if isinstance(v, file)): file_args[str(i)] = request.pop(i) try: requester = self._requester() if not file_args: return requester.post( urlparse.urljoin(config.CONFIG["host"], "/api/call"), data = utils.to_json(request), headers = {"Content-Type": "application/json"}, verify = _get_verify() ) else: return requester.post( urlparse.urljoin(config.CONFIG["host"], "/api/call"), data = {"request": utils.to_json(request)}, files = file_args, verify = _get_verify() ) except requests.exceptions.SSLError as e: logger.critical( "There was a problem with communicating via SSL: %s.", str(e), exc_info = True ) sys.exit(1) except requests.exceptions.ConnectionError: logger.critical( "Galah did not respond at %s.", urlparse.urljoin(config.CONFIG["host"], "/api/call"), exc_info = True ) sys.exit(1)
def get(self, slug): match = ndb.Key('Match', slug).get() if not match: self.response.status_int = 404 return self.response.headers['Content-Type'] = 'application/json' self.response.write(utils.to_json(match.to_dict()))
def on_logged_in(self, userid, gadget, wavelet): waveid = wavelet.wave_id if not models.Login.exists(waveid, userid): # Wait, this user has no login. Bail out. gadget.set_status(Gadget.LOGIN) return blogger = utils.Blogger.open(wavelet.wave_id, userid) if not blogger: # There was some error. Bail out. gadget.set_status(Gadget.LOGIN) return blogs = blogger.get_blogs() if len(blogs) == 0: # User has no blogs. Bail out for now. gadget.set_status(Gadget.LOGIN) return blog = blogs[0]["id"] connection = models.Connection.get(waveid) if connection: connection.owner = userid connection.blog = blog connection.post = None else: connection = models.Connection(waveid=waveid, owner=userid, blog=blog) connection.put() gadget.set_owner(userid) gadget.set_status(Gadget.CONNECTED) gadget.set("blogs", utils.to_json(blogs)) gadget.set_blog(blog)
def one_election_drive_tally(request, election): """ JavaScript-based driver for the entire tallying process, now done in JavaScript. """ if election.tally_type != "homomorphic": return HttpResponseRedirect(reverse(one_election_view,args=[election.election_id])) election_pk = election.public_key election_pk_json = utils.to_json(election_pk.toJSONDict()) election_sk = election.private_key if election_sk: election_sk_json = utils.to_json(election_sk.toJSONDict()) else: election_sk_json = None return render_template(request, 'drive_tally', {'election': election, 'election_pk_json' : election_pk_json, 'election_sk_json' : election_sk_json})
def convert_to_json(self, *args, **kwargs): return_val = func(self, *args, **kwargs) try: return render_json(utils.to_json(return_val)) except Exception, e: import logging logging.error("problem with serialization: " + str(return_val) + " / " + str(e)) raise e
def check_user_auth(self): waveid = self.request.get('waveid') userid = self.request.get('userid') if (not waveid) or (not userid): return return to_json({ 'isSignedIn': models.Login.exists(waveid, userid) })
def get_auth_url(self): waveid = self.request.get('waveid') userid = self.request.get('userid') if (not waveid) or (not userid): return return to_json({ 'url': Blogger.get_auth_url(waveid, userid, _UPGRADE_URL) })
def social_cart(request): data = SocialCartResource().to_dict(obj=request.user, request=request) return render(request, 'cart/social_cart.j.html', { 'cart_json': to_json(data), 'is_empty': not data['buys'] and \ not data['pending_shipping_requests'] and \ not data['pickup_requests'] and \ not data['shipping_requests'] })
def one_election_keyshares_tally_manage(request, election): election_pk_json = utils.to_json(election.public_key.toJSONDict()) keyshares = election.get_keyshares() ready_p = True for keyshare in keyshares: ready_p = ready_p and (keyshare.decryption_factors != None) return render_template(request,"keyshares_tally_manage", {'election': election, 'election_pk_json': election_pk_json, 'ready_p' : ready_p})
def post(self): request = to_json(self.request.body) # Verify the token on the Google servers url = "https://www.googleapis.com/oauth2/v3/tokeninfo?id_token={}".format(request.get('token')) result = urlfetch.fetch(url=url, method=urlfetch.GET, ) if result.status_code == 200: content = to_json(result.content) # Need to check the 'aud' property once we have a user to actually test this stuff with. # For now we'll just assume everything worked and return a formatted new user user = User.query(User.email==content.get('email')).get() if user is None: # Need to create a new user user = User() user.email = content.get('email') user.put() self.response.status_int = 200 self.response.headers['Access-Control-Allow-Origin'] = "http://www.myvoyagr.co" self.response.write(user.format()) else: # Get all the trips associated with the user and add it to the response trips = {} trip_qry = Trip.query(ancestor=user.key).fetch() for trip in trip_qry: trips[trip.name] = trip.key.id() self.response.status_int = 200 self.response.headers['Access-Control-Allow-Origin'] = "http://www.myvoyagr.co" self.response.write({'user': user.format(), 'trips' : trips}) return else: self.response.status_int = 400 self.response.headers['Access-Control-Allow-Origin'] = "http://www.myvoyagr.co" self.response.write({'error': 'There was an error authenticating the user'}) return
def post_status(room_id): # print request.get_json() update_status = request.get_json() # update statuses status = statuses.get(room_id) # update given fields if update_status.get("status"): status.status = update_status.get("status") status.timestamp = utils.get_currrent_time_str() if update_status.get("status") == "FIRE": print "fire alert!" emailer.send_email( "[email protected], [email protected], [email protected], [email protected], [email protected]", "FIRE Alert! [%s]" % room_id, "Status Details: \n" + utils.to_json(status)) # send_message("9492664065", "There is FIRE at room [%s]!" % (room_id)) if update_status.get("occupancy"): status.occupancy = update_status.get("occupancy") # if update_status.get("carbon_detected"): if "carbon_detected" in update_status: status.carbon_detected = update_status.get("carbon_detected") return utils.to_json({"Update": True})
def serialize(self, obj, **kw): """ Function for serializing object => string. This can be overwritten for custom uses. The default is to do nothing ('serializer'=None) If the connection is intialized with 'serializer' set to 'json.gz', 'json', 'gz', or 'zip', we'll do the transformations. """ serializer = kw.get('serializer', self._serializer) if serializer == "json.gz": return utils.to_gz(utils.to_json(obj)) elif serializer == "json": return utils.to_json(obj) elif serializer == "gz": assert(isinstance(obj, basestring)) return utils.to_gz(obj) elif serializer == "zip": assert(isinstance(obj, basestring)) return utils.to_zip(obj) elif serializer == "pickle": return utils.to_pickle(obj) elif serializer is not None: raise NotImplementedError( 'Only json, gz, json.gz, zip, and pickle' 'are supported as serializers.') return obj
def post(self, slug): user = users.get_current_user() if not user: self.redirect(users.create_login_url(self.request.uri)) return values = json.loads(self.request.body) event = models.Event(parent=ndb.Key('Match', slug)) event.body = values.get('body') event.meta = values.get('meta') event.user = user event.put() self.response.headers['Content-Type'] = 'application/json' self.response.write(utils.to_json(event.to_dict()))
def post(self): user = users.get_current_user() if not user: self.redirect(users.create_login_url(self.request.uri)) return values = json.loads(self.request.body) match = models.Match() match.title = values.get('title') match.team_a = values.get('team_a') match.team_b = values.get('team_b') match.user = user match.put() self.response.headers['Content-Type'] = 'application/json' self.response.write(utils.to_json(match.to_dict()))
def log(self, action, data): parent = self.get_version() or None #mongo.db.history.remove(); parent = None prev = self.replay() data = self.get_delta(prev, data) if not data: return delta = { 'uri': self.target, 'action': action, 'data': data, 'parent': ObjectId(parent) if parent else None, 'utc': dt.utcnow() } mongo.db.history.insert( delta ) redis.set("history:version:%s" % self.target, str( delta['_id'] )) print delta redis.publish('history', to_json(delta))
def election(request): return HttpResponse(utils.to_json(HELIOS_CLIENT.election_get(ELECTION_ID).toJSONDict()))
def json(self): return to_json(self, self.__class__)
def toJSON(self): import utils return utils.to_json(self.toJSONDict())
def toJSON(self): return utils.to_json(self.toJSONDict())
def _cast_ballot(self, election_id, username, password, need_login=True, check_user_logged_in=False): """ check_user_logged_in looks for the "you're already logged" message """ # vote by preparing a ballot via the server-side encryption response = self.app.post("/helios/elections/%s/encrypt-ballot" % election_id, { 'answers_json': utils.to_json([[1]])}) self.assertContains(response, "answers") # parse it as an encrypted vote, and re-serialize it ballot = datatypes.LDObject.fromDict(utils.from_json(response.testbody), type_hint='legacy/EncryptedVote') encrypted_vote = ballot.serialize() # cast the ballot response = self.app.post("/helios/elections/%s/cast" % election_id, { 'encrypted_vote': encrypted_vote}) self.assertRedirects(response, "%s/helios/elections/%s/cast_confirm" % (settings.SECURE_URL_HOST, election_id)) cast_confirm_page = response.follow() if need_login: if check_user_logged_in: self.assertContains(cast_confirm_page, "You are logged in as") self.assertContains(cast_confirm_page, "requires election-specific credentials") # set the form login_form = cast_confirm_page.form login_form['voter_id'] = username login_form['password'] = password cast_confirm_page = login_form.submit() self.assertRedirects(cast_confirm_page, "/helios/elections/%s/cast_confirm" % election_id) cast_confirm_page = cast_confirm_page.follow() # here we should be at the cast-confirm page and logged in self.assertContains(cast_confirm_page, "I am ") # confirm the vote, now with the actual form cast_form = cast_confirm_page.form if 'status_update' in cast_form.fields.keys(): cast_form['status_update'] = False response = cast_form.submit() self.assertRedirects(response, "%s/helios/elections/%s/cast_done" % (settings.URL_HOST, election_id)) # at this point an email should have gone out to the user # at position num_messages after, since that was the len() before we cast this ballot email_message = mail.outbox[len(mail.outbox) - 1] url = re.search('http://[^/]+(/[^ \n]*)', email_message.body).group(1) # check that we can get at that URL if not need_login: # confusing piece: if need_login is True, that means it was a public election # that required login before casting a ballot. # so if need_login is False, it was a private election, and we do need to re-login here # we need to re-login if it's a private election, because all data, including ballots # is otherwise private login_page = self.app.get("/helios/elections/%s/password_voter_login" % election_id) # if we redirected, that's because we can see the page, I think if login_page.status_int != 302: login_form = login_page.form # try with extra spaces login_form['voter_id'] = ' ' + username + ' ' login_form['password'] = '******' + password + ' ' login_form.submit() response = self.app.get(url) self.assertContains(response, ballot.hash) self.assertContains(response, html_escape(encrypted_vote)) # if we request the redirect to cast_done, the voter should be logged out, but not the user response = self.app.get("/helios/elections/%s/cast_done" % election_id)
def send(self, data): channel.send_message(self.token, utils.to_json(data))
def kfold_lightgbm(train_df, test_df, num_folds): print('Starting LightGBM. Train shape: {}'.format(train_df.shape)) # Cross validation folds = CustomTimeSeriesSplitter(end_train=1941) # Create arrays and dataframes to store results oof_preds = np.zeros(train_df.shape[0]) feature_importance_df = pd.DataFrame() feats = [f for f in train_df.columns if f not in FEATS_EXCLUDED] valid_idxs = [] best_iterations = [] # average of best iteration # k-fold for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df)): # split train/valid train_x, train_y = train_df[feats].iloc[train_idx], train_df[ 'demand'].iloc[train_idx] valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df[ 'demand'].iloc[valid_idx] # save validation indexes valid_idxs += list(valid_idx) # set data structure lgb_train = lgb.Dataset(train_x, label=train_y, free_raw_data=False) lgb_test = lgb.Dataset(valid_x, label=valid_y, free_raw_data=False) params = { # 'device' : 'gpu', # 'gpu_use_dp':True, 'boosting': 'gbdt', 'metric': ['rmse'], 'objective': 'tweedie', 'learning_rate': 0.05, 'tweedie_variance_power': 1.1, 'subsample': 0.5, 'subsample_freq': 1, 'num_leaves': 2**8 - 1, 'min_data_in_leaf': 2**8 - 1, 'feature_fraction': 0.8, 'verbose': -1, 'seed': 326, 'bagging_seed': 326, 'drop_seed': 326, 'num_threads': -1 } # train model reg = lgb.train(params, lgb_train, valid_sets=[lgb_train, lgb_test], valid_names=['train', 'test'], num_boost_round=10000, early_stopping_rounds=200, verbose_eval=10) # save model reg.save_model(f'../output/lgbm_household_{n_fold}.txt') # save predictions oof_preds[valid_idx] = reg.predict(valid_x, num_iteration=reg.best_iteration) # save best iteration best_iterations.append(reg.best_iteration) # save feature importances fold_importance_df = pd.DataFrame() fold_importance_df['feature'] = feats fold_importance_df['importance'] = np.log1p( reg.feature_importance(importance_type='gain', iteration=reg.best_iteration)) fold_importance_df['fold'] = n_fold + 1 feature_importance_df = pd.concat( [feature_importance_df, fold_importance_df], axis=0) print('Fold %2d RMSE : %.6f' % (n_fold + 1, rmse(valid_y, oof_preds[valid_idx]))) del reg, train_x, train_y, valid_x, valid_y gc.collect() # display importances display_importances(feature_importance_df, '../imp/lgbm_importances_cv_household.png', '../imp/feature_importance_lgbm_cv_household.csv') # Full RMSE score and LINE Notify full_rmse = rmse(train_df['demand'][valid_idxs], oof_preds[valid_idxs]) line_notify('Full RMSE score %.6f' % full_rmse) # save out of fold prediction train_df.loc[:, 'demand'] = oof_preds train_df[['id', 'd', 'demand']].to_csv(oof_file_name, index=False) # save number of best iteration configs['num_boost_round'] = best_iterations configs['rmse'] = full_rmse to_json(configs, '../configs/307_train_household.json') # LINE notify line_notify('{} done. best iterations:{}'.format(sys.argv[0], best_iterations))
def main(_): pp.pprint(flags.FLAGS.__flags) if not os.path.exists(FLAGS.checkpoint_dir): os.makedirs(FLAGS.checkpoint_dir) if not os.path.exists(FLAGS.sample_dir): os.makedirs(FLAGS.sample_dir) with tf.Session() as sess: if FLAGS.dataset == 'mnist': dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size, y_dim=10, dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir) else: dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size, dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir) if FLAGS.is_train: dcgan.train(FLAGS) else: dcgan.load(FLAGS.checkpoint_dir) to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0], [dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1], [dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2], [dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3], [dcgan.h4_w, dcgan.h4_b, None]) # Below is codes for visualization OPTION = 2 if OPTION == 0: z_sample = np.random.uniform(-0.5, 0.5, size=(FLAGS.batch_size, dcgan.z_dim)) samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample}) save_images(samples, [8, 8], './samples/test_%s.png' % strftime("%Y-%m-%d %H:%M:%S", gmtime())) elif OPTION == 1: values = np.arange(0, 1, 1./FLAGS.batch_size) for idx in xrange(100): print(" [*] %d" % idx) z_sample = np.zeros([FLAGS.batch_size, dcgan.z_dim]) for kdx, z in enumerate(z_sample): z[idx] = values[kdx] samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample}) save_images(samples, [8, 8], './samples/test_arange_%s.png' % (idx)) elif OPTION == 2: values = np.arange(0, 1, 1./FLAGS.batch_size) for idx in [random.randint(0, 99) for _ in xrange(100)]: print(" [*] %d" % idx) z = np.random.uniform(-0.2, 0.2, size=(dcgan.z_dim)) z_sample = np.tile(z, (FLAGS.batch_size, 1)) #z_sample = np.zeros([FLAGS.batch_size, dcgan.z_dim]) for kdx, z in enumerate(z_sample): z[idx] = values[kdx] samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample}) make_gif(samples, './samples/test_gif_%s.gif' % (idx)) elif OPTION == 3: values = np.arange(0, 1, 1./FLAGS.batch_size) for idx in xrange(100): print(" [*] %d" % idx) z_sample = np.zeros([FLAGS.batch_size, dcgan.z_dim]) for kdx, z in enumerate(z_sample): z[idx] = values[kdx] samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample}) make_gif(samples, './samples/test_gif_%s.gif' % (idx)) elif OPTION == 4: image_set = [] values = np.arange(0, 1, 1./FLAGS.batch_size) for idx in xrange(100): print(" [*] %d" % idx) z_sample = np.zeros([FLAGS.batch_size, dcgan.z_dim]) for kdx, z in enumerate(z_sample): z[idx] = values[kdx] image_set.append(sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})) make_gif(image_set[-1], './samples/test_gif_%s.gif' % (idx)) new_image_set = [merge(np.array([images[idx] for images in image_set]), [10, 10]) for idx in range(64) + range(63, -1, -1)] make_gif(new_image_set, './samples/test_gif_merged.gif', duration=8) elif OPTION == 5: image_set = [] values = np.arange(0, 1, 1./FLAGS.batch_size) z_idx = [[random.randint(0,99) for _ in xrange(5)] for _ in xrange(200)] for idx in xrange(200): print(" [*] %d" % idx) #z_sample = np.zeros([FLAGS.batch_size, dcgan.z_dim]) z = np.random.uniform(-1e-1, 1e-1, size=(dcgan.z_dim)) z_sample = np.tile(z, (FLAGS.batch_size, 1)) for kdx, z in enumerate(z_sample): for jdx in xrange(5): z_sample[kdx][z_idx[idx][jdx]] = values[kdx] image_set.append(sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})) make_gif(image_set[-1], './samples/test_gif_%s.gif' % (idx)) new_image_set = [merge(np.array([images[idx] for images in image_set]), [10, 20]) for idx in range(64) + range(63, -1, -1)] make_gif(new_image_set, './samples/test_gif_random_merged.gif', duration=4) elif OPTION == 6: image_set = [] values = np.arange(0, 1, 1.0/FLAGS.batch_size).tolist() z_idx = [[random.randint(0,99) for _ in xrange(10)] for _ in xrange(100)] for idx in xrange(100): print(" [*] %d" % idx) z = np.random.uniform(-0.2, 0.2, size=(dcgan.z_dim)) z_sample = np.tile(z, (FLAGS.batch_size, 1)) for kdx, z in enumerate(z_sample): for jdx in xrange(10): z_sample[kdx][z_idx[idx][jdx]] = values[kdx] image_set.append(sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})) save_images(image_set[-1], [8, 8], './samples/test_random_arange_%s.png' % (idx)) new_image_set = [merge(np.array([images[idx] for images in image_set]), [10, 10]) for idx in range(64) + range(63, -1, -1)] make_gif(new_image_set, './samples/test_gif_merged_random.gif', duration=4) elif OPTION == 7: for _ in xrange(50): z_idx = [[random.randint(0,99) for _ in xrange(10)] for _ in xrange(8)] zs = [] for idx in xrange(8): z = np.random.uniform(-0.2, 0.2, size=(dcgan.z_dim)) zs.append(np.tile(z, (8, 1))) z_sample = np.concatenate(zs) values = np.arange(0, 1, 1/8.) for idx in xrange(FLAGS.batch_size): for jdx in xrange(8): z_sample[idx][z_idx[idx/8][jdx]] = values[idx%8] samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample}) save_images(samples, [8, 8], './samples/multiple_testt_%s.png' % strftime("%Y-%m-%d %H:%M:%S", gmtime())) elif OPTION == 8: counter = 0 for _ in xrange(50): import scipy.misc z_idx = [[random.randint(0,99) for _ in xrange(10)] for _ in xrange(8)] zs = [] for idx in xrange(8): z = np.random.uniform(-0.2, 0.2, size=(dcgan.z_dim)) zs.append(np.tile(z, (8, 1))) z_sample = np.concatenate(zs) values = np.arange(0, 1, 1/8.) for idx in xrange(FLAGS.batch_size): for jdx in xrange(8): z_sample[idx][z_idx[idx/8][jdx]] = values[idx%8] samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample}) for sample in samples: scipy.misc.imsave('./samples/turing/%s.png' % counter, sample) counter += 1 else: import scipy.misc from glob import glob samples = [] fnames = glob("/Users/carpedm20/Downloads/x/1/*.png") fnames = sorted(fnames, key = lambda x: int(x.split("_")[1]) * 10000 + int(x.split('_')[2].split(".")[0])) for f in fnames: samples.append(scipy.misc.imread(f)) make_gif(samples, './samples/training.gif', duration=8, true_image=True)
def get_hash(self): return utils.hash_b64(utils.to_json(self.toJSONDict()))
def main(): # load pkls df = read_pickles('../feats/sales') df_calendar = loadpkl('../feats/calendar.pkl') df_sell_prices = loadpkl('../feats/sell_prices.pkl') # extract foods df = df[df['cat_id'] == 'FOODS'] # merge df = df.merge(df_calendar, on='d',how='left') df = df.merge(df_sell_prices, on=['store_id','item_id','wm_yr_wk'],how='left') del df_calendar, df_sell_prices gc.collect() # drop pre-release rows df = df[df['wm_yr_wk']>=df['release']] # make lag features df = make_lags(df,28) # add categorical features df['item_id_store_id'] = df['item_id']+'_'+df['store_id'] df['item_id_state_id'] = df['item_id']+'_'+df['state_id'] df['dept_id_store_id'] = df['dept_id']+'_'+df['store_id'] df['dept_id_state_id'] = df['dept_id']+'_'+df['state_id'] # label encoding cols_string = ['item_id','dept_id','cat_id','store_id','state_id', 'item_id_store_id','item_id_state_id','dept_id_store_id', 'dept_id_state_id'] for c in cols_string: df[c], _ = pd.factorize(df[c]) df[c].replace(-1,np.nan,inplace=True) # add price features df_grouped = df[['id','sell_price']].groupby('id')['sell_price'] df['shift_price_t1'] = df_grouped.transform(lambda x: x.shift(1)) df['price_change_t1'] = (df['shift_price_t1'] - df['sell_price']) / (df['shift_price_t1']) df['rolling_price_max_t365'] = df_grouped.transform(lambda x: x.shift(1).rolling(365).max()) df['price_change_t365'] = (df['rolling_price_max_t365'] - df['sell_price']) / (df['rolling_price_max_t365']) df['rolling_price_std_t7'] = df_grouped.transform(lambda x: x.rolling(7).std()) df['rolling_price_std_t30'] = df_grouped.transform(lambda x: x.rolling(30).std()) # features release date df['release'] = df['release'] - df['release'].min() # price momentum by month & year df['price_momentum_m'] = df['sell_price']/df.groupby(['store_id','item_id','month'])['sell_price'].transform('mean') df['price_momentum_y'] = df['sell_price']/df.groupby(['store_id','item_id','year'])['sell_price'].transform('mean') # days for CustomTimeSeriesSplitter df['d_numeric'] = df['d'].apply(lambda x: str(x)[2:]).astype(int) # reduce memory usage df = reduce_mem_usage(df) # save as feather to_feature(df, '../feats/f106') # save feature name list features_json = {'features':df.columns.tolist()} to_json(features_json,'../configs/106_all_features_foods.json') # LINE notify line_notify('{} done.'.format(sys.argv[0]))
def hash(self): s = utils.to_json(self.toJSONDict()) return utils.hash_b64(s)
# get all r = requests.get("http://localhost:5000/statuses") print r.content r = requests.get("http://localhost:5000/status/livingroom") print r.content # update r = requests.post( url="http://localhost:5000/status/livingroom", headers={'Content-type': 'application/json'}, data=utils.to_json({"status": "FIRE" }) ) print r.content # send sms r = requests.post( url="http://localhost:5000/status/livingroom", headers={'Content-type': 'application/json'}, data=utils.to_json({"occupancy": 1 }) ) print r.content r = requests.post( url="http://localhost:5000/status/livingroom", headers={'Content-type': 'application/json'}, data=utils.to_json({"occupancy": 0, "carbon_detected": False })
def _setup_complete_election(self, election_params={}): "do the setup part of a whole election" # a bogus call to set up the session self.client.get("/") # REPLACE with params? self.setup_login() # create the election full_election_params = { "short_name" : "test-complete", "name" : "Test Complete", "description" : "A complete election test", "election_type" : "referendum", "use_voter_aliases": "0", "use_advanced_audit_features": "1", "private_p" : "0"} # override with the given full_election_params.update(election_params) response = self.client.post("/helios/elections/new", full_election_params) # we are redirected to the election, let's extract the ID out of the URL election_id = re.search('/elections/([^/]+)/', str(response['Location'])).group(1) # helios is automatically added as a trustee # check that helios is indeed a trustee response = self.client.get("/helios/elections/%s/trustees/view" % election_id) self.assertContains(response, "Trustee #1") # add a few voters, via file upload FILE = "helios/fixtures/voter-file.csv" voters_file = open(FILE) response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'voters_file': voters_file}) voters_file.close() self.assertContains(response, "first few rows of this file") # now we confirm the upload response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'confirm_p': "1"}) self.assertRedirects(response, "/helios/elections/%s/voters/list" % election_id) # and we want to check that there are now voters response = self.client.get("/helios/elections/%s/voters/" % election_id) NUM_VOTERS = 4 self.assertEquals(len(utils.from_json(response.content)), NUM_VOTERS) # let's get a single voter single_voter = models.Election.objects.get(uuid = election_id).voter_set.all()[0] response = self.client.get("/helios/elections/%s/voters/%s" % (election_id, single_voter.uuid)) self.assertContains(response, '"uuid": "%s"' % single_voter.uuid) response = self.client.get("/helios/elections/%s/voters/foobar" % election_id) self.assertEquals(response.status_code, 404) # add questions response = self.client.post("/helios/elections/%s/save_questions" % election_id, { 'questions_json': utils.to_json([{"answer_urls": [None,None], "answers": ["Alice", "Bob"], "choice_type": "approval", "max": 1, "min": 0, "question": "Who should be president?", "result_type": "absolute", "short_name": "Who should be president?", "tally_type": "homomorphic"}]), 'csrf_token': self.client.session['csrf_token']}) self.assertContains(response, "SUCCESS") # freeze election response = self.client.post("/helios/elections/%s/freeze" % election_id, { "csrf_token" : self.client.session['csrf_token']}) self.assertRedirects(response, "/helios/elections/%s/view" % election_id) # email the voters num_messages_before = len(mail.outbox) response = self.client.post("/helios/elections/%s/voters/email" % election_id, { "csrf_token" : self.client.session['csrf_token'], "subject" : "your password", "body" : "time to vote", "suppress_election_links" : "0", "send_to" : "all" }) self.assertRedirects(response, "/helios/elections/%s/view" % election_id) num_messages_after = len(mail.outbox) self.assertEquals(num_messages_after - num_messages_before, NUM_VOTERS) email_message = mail.outbox[num_messages_before] assert "your password" in email_message.subject, "bad subject in email" # get the username and password username = re.search('voter ID: (.*)', email_message.body).group(1) password = re.search('password: (.*)', email_message.body).group(1) # now log out as administrator self.clear_login() self.assertEquals(self.client.session.has_key('user'), False) # return the voter username and password to vote return election_id, username, password
def convert_to_json(self, *args, **kwargs): return render_json(utils.to_json(func(self, *args, **kwargs)))
def personal_cart(request): data = PersonalCartResource().to_dict(obj=request.user, request=request) return render(request, 'cart/personal_cart.j.html', { 'cart_json': to_json(data), 'is_empty': not data['buys'] })