def main(_):
pp.pprint(flags.FLAGS.__flags)
if not os.path.exists(FLAGS.checkpoint_dir):
os.makedirs(FLAGS.checkpoint_dir)
if not os.path.exists(FLAGS.sample_dir):
os.makedirs(FLAGS.sample_dir)
with tf.Session() as sess:
if FLAGS.dataset == 'mnist':
dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size, y_dim=10,
dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir)
else:
dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size,
dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir)
if FLAGS.is_train:
dcgan.train(FLAGS)
else:
dcgan.load(FLAGS.checkpoint_dir)
to_json("./web/js/gen_layers.js", dcgan.h0_w, dcgan.h1_w, dcgan.h2_w, dcgan.h3_w, dcgan.h4_w)
z_sample = np.random.uniform(-1, 1, size=(FLAGS.batch_size, dcgan.z_dim))
samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})
save_images(samples, [8, 8], './samples/test_%s.png' % strftime("%Y-%m-%d %H:%M:%S", gmtime()))
def main(_):
pp.pprint(flags.FLAGS.__flags)
if not os.path.exists(FLAGS.checkpoint_dir):
os.makedirs(FLAGS.checkpoint_dir)
if not os.path.exists(FLAGS.sample_dir):
os.makedirs(FLAGS.sample_dir)
with tf.Session() as sess:
if FLAGS.dataset == 'mnist':
dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size, y_dim=10,
dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir)
else:
dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size,
dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir)
if FLAGS.is_train:
dcgan.train(FLAGS)
else:
dcgan.load(FLAGS.checkpoint_dir)
to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0],
[dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1],
[dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2],
[dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3],
[dcgan.h4_w, dcgan.h4_b, None])
# Below is codes for visualization
OPTION = 2
visualize(sess, dcgan, FLAGS, OPTION)
def _send_api_command(self, request):
"""
Send an API command to Galah.
:param request: A properly formed JSON object to send Galah.
:returns: A ``requests.Response`` object.
"""
request = copy.copy(request)
# Extract any files
file_args = {}
for i in (k for k, v in request.items() if isinstance(v, file)):
file_args[str(i)] = request.pop(i)
try:
requester = self._requester()
if not file_args:
return requester.post(
urlparse.urljoin(config.CONFIG["host"], "/api/call"),
data = utils.to_json(request),
headers = {"Content-Type": "application/json"},
verify = _get_verify()
)
else:
return requester.post(
urlparse.urljoin(config.CONFIG["host"], "/api/call"),
data = {"request": utils.to_json(request)},
files = file_args,
verify = _get_verify()
)
except requests.exceptions.SSLError as e:
logger.critical(
"There was a problem with communicating via SSL: %s.",
str(e),
exc_info = True
)
sys.exit(1)
except requests.exceptions.ConnectionError:
logger.critical(
"Galah did not respond at %s.",
urlparse.urljoin(config.CONFIG["host"], "/api/call"),
exc_info = True
)
sys.exit(1)
def get(self, slug):
match = ndb.Key('Match', slug).get()
if not match:
self.response.status_int = 404
return
self.response.headers['Content-Type'] = 'application/json'
self.response.write(utils.to_json(match.to_dict()))
def on_logged_in(self, userid, gadget, wavelet):
waveid = wavelet.wave_id
if not models.Login.exists(waveid, userid):
# Wait, this user has no login. Bail out.
gadget.set_status(Gadget.LOGIN)
return
blogger = utils.Blogger.open(wavelet.wave_id, userid)
if not blogger:
# There was some error. Bail out.
gadget.set_status(Gadget.LOGIN)
return
blogs = blogger.get_blogs()
if len(blogs) == 0:
# User has no blogs. Bail out for now.
gadget.set_status(Gadget.LOGIN)
return
blog = blogs[0]["id"]
connection = models.Connection.get(waveid)
if connection:
connection.owner = userid
connection.blog = blog
connection.post = None
else:
connection = models.Connection(waveid=waveid, owner=userid, blog=blog)
connection.put()
gadget.set_owner(userid)
gadget.set_status(Gadget.CONNECTED)
gadget.set("blogs", utils.to_json(blogs))
gadget.set_blog(blog)
def one_election_drive_tally(request, election):
"""
JavaScript-based driver for the entire tallying process, now done in JavaScript.
"""
if election.tally_type != "homomorphic":
return HttpResponseRedirect(reverse(one_election_view,args=[election.election_id]))
election_pk = election.public_key
election_pk_json = utils.to_json(election_pk.toJSONDict())
election_sk = election.private_key
if election_sk:
election_sk_json = utils.to_json(election_sk.toJSONDict())
else:
election_sk_json = None
return render_template(request, 'drive_tally', {'election': election, 'election_pk_json' : election_pk_json, 'election_sk_json' : election_sk_json})
def convert_to_json(self, *args, **kwargs):
return_val = func(self, *args, **kwargs)
try:
return render_json(utils.to_json(return_val))
except Exception, e:
import logging
logging.error("problem with serialization: " + str(return_val) + " / " + str(e))
raise e
def check_user_auth(self):
waveid = self.request.get('waveid')
userid = self.request.get('userid')
if (not waveid) or (not userid):
return
return to_json({
'isSignedIn': models.Login.exists(waveid, userid)
})
def get_auth_url(self):
waveid = self.request.get('waveid')
userid = self.request.get('userid')
if (not waveid) or (not userid):
return
return to_json({
'url': Blogger.get_auth_url(waveid, userid, _UPGRADE_URL)
})
def social_cart(request):
data = SocialCartResource().to_dict(obj=request.user, request=request)
return render(request, 'cart/social_cart.j.html', {
'cart_json': to_json(data),
'is_empty': not data['buys'] and \
not data['pending_shipping_requests'] and \
not data['pickup_requests'] and \
not data['shipping_requests']
})
def one_election_keyshares_tally_manage(request, election):
election_pk_json = utils.to_json(election.public_key.toJSONDict())
keyshares = election.get_keyshares()
ready_p = True
for keyshare in keyshares:
ready_p = ready_p and (keyshare.decryption_factors != None)
return render_template(request,"keyshares_tally_manage", {'election': election, 'election_pk_json': election_pk_json, 'ready_p' : ready_p})
def post(self):
request = to_json(self.request.body)
# Verify the token on the Google servers
url = "https://www.googleapis.com/oauth2/v3/tokeninfo?id_token={}".format(request.get('token'))
result = urlfetch.fetch(url=url,
method=urlfetch.GET,
)
if result.status_code == 200:
content = to_json(result.content)
# Need to check the 'aud' property once we have a user to actually test this stuff with.
# For now we'll just assume everything worked and return a formatted new user
user = User.query(User.email==content.get('email')).get()
if user is None:
# Need to create a new user
user = User()
user.email = content.get('email')
user.put()
self.response.status_int = 200
self.response.headers['Access-Control-Allow-Origin'] = "http://www.myvoyagr.co"
self.response.write(user.format())
else:
# Get all the trips associated with the user and add it to the response
trips = {}
trip_qry = Trip.query(ancestor=user.key).fetch()
for trip in trip_qry:
trips[trip.name] = trip.key.id()
self.response.status_int = 200
self.response.headers['Access-Control-Allow-Origin'] = "http://www.myvoyagr.co"
self.response.write({'user': user.format(), 'trips' : trips})
return
else:
self.response.status_int = 400
self.response.headers['Access-Control-Allow-Origin'] = "http://www.myvoyagr.co"
self.response.write({'error': 'There was an error authenticating the user'})
return
def post_status(room_id):
# print request.get_json()
update_status = request.get_json()
# update statuses
status = statuses.get(room_id)
# update given fields
if update_status.get("status"):
status.status = update_status.get("status")
status.timestamp = utils.get_currrent_time_str()
if update_status.get("status") == "FIRE":
print "fire alert!"
emailer.send_email(
"[email protected], [email protected], [email protected], [email protected], [email protected]",
"FIRE Alert! [%s]" % room_id, "Status Details: \n" + utils.to_json(status))
# send_message("9492664065", "There is FIRE at room [%s]!" % (room_id))
if update_status.get("occupancy"):
status.occupancy = update_status.get("occupancy")
# if update_status.get("carbon_detected"):
if "carbon_detected" in update_status:
status.carbon_detected = update_status.get("carbon_detected")
return utils.to_json({"Update": True})
def serialize(self, obj, **kw):
"""
Function for serializing object => string.
This can be overwritten for custom
uses.
The default is to do nothing ('serializer'=None)
If the connection is intialized with 'serializer' set to
'json.gz', 'json', 'gz', or 'zip', we'll do the
transformations.
"""
serializer = kw.get('serializer', self._serializer)
if serializer == "json.gz":
return utils.to_gz(utils.to_json(obj))
elif serializer == "json":
return utils.to_json(obj)
elif serializer == "gz":
assert(isinstance(obj, basestring))
return utils.to_gz(obj)
elif serializer == "zip":
assert(isinstance(obj, basestring))
return utils.to_zip(obj)
elif serializer == "pickle":
return utils.to_pickle(obj)
elif serializer is not None:
raise NotImplementedError(
'Only json, gz, json.gz, zip, and pickle'
'are supported as serializers.')
return obj
def post(self, slug):
user = users.get_current_user()
if not user:
self.redirect(users.create_login_url(self.request.uri))
return
values = json.loads(self.request.body)
event = models.Event(parent=ndb.Key('Match', slug))
event.body = values.get('body')
event.meta = values.get('meta')
event.user = user
event.put()
self.response.headers['Content-Type'] = 'application/json'
self.response.write(utils.to_json(event.to_dict()))
def post(self):
user = users.get_current_user()
if not user:
self.redirect(users.create_login_url(self.request.uri))
return
values = json.loads(self.request.body)
match = models.Match()
match.title = values.get('title')
match.team_a = values.get('team_a')
match.team_b = values.get('team_b')
match.user = user
match.put()
self.response.headers['Content-Type'] = 'application/json'
self.response.write(utils.to_json(match.to_dict()))
def log(self, action, data):
parent = self.get_version() or None
#mongo.db.history.remove(); parent = None
prev = self.replay()
data = self.get_delta(prev, data)
if not data:
return
delta = {
'uri': self.target,
'action': action,
'data': data,
'parent': ObjectId(parent) if parent else None,
'utc': dt.utcnow()
}
mongo.db.history.insert( delta )
redis.set("history:version:%s" % self.target, str( delta['_id'] ))
print delta
redis.publish('history', to_json(delta))
def election(request): return HttpResponse(utils.to_json(HELIOS_CLIENT.election_get(ELECTION_ID).toJSONDict()))
def json(self):
return to_json(self, self.__class__)
def toJSON(self):
import utils
return utils.to_json(self.toJSONDict())
def toJSON(self):
return utils.to_json(self.toJSONDict())
def _cast_ballot(self, election_id, username, password, need_login=True, check_user_logged_in=False):
"""
check_user_logged_in looks for the "you're already logged" message
"""
# vote by preparing a ballot via the server-side encryption
response = self.app.post("/helios/elections/%s/encrypt-ballot" % election_id, {
'answers_json': utils.to_json([[1]])})
self.assertContains(response, "answers")
# parse it as an encrypted vote, and re-serialize it
ballot = datatypes.LDObject.fromDict(utils.from_json(response.testbody), type_hint='legacy/EncryptedVote')
encrypted_vote = ballot.serialize()
# cast the ballot
response = self.app.post("/helios/elections/%s/cast" % election_id, {
'encrypted_vote': encrypted_vote})
self.assertRedirects(response, "%s/helios/elections/%s/cast_confirm" % (settings.SECURE_URL_HOST, election_id))
cast_confirm_page = response.follow()
if need_login:
if check_user_logged_in:
self.assertContains(cast_confirm_page, "You are logged in as")
self.assertContains(cast_confirm_page, "requires election-specific credentials")
# set the form
login_form = cast_confirm_page.form
login_form['voter_id'] = username
login_form['password'] = password
cast_confirm_page = login_form.submit()
self.assertRedirects(cast_confirm_page, "/helios/elections/%s/cast_confirm" % election_id)
cast_confirm_page = cast_confirm_page.follow()
# here we should be at the cast-confirm page and logged in
self.assertContains(cast_confirm_page, "I am ")
# confirm the vote, now with the actual form
cast_form = cast_confirm_page.form
if 'status_update' in cast_form.fields.keys():
cast_form['status_update'] = False
response = cast_form.submit()
self.assertRedirects(response, "%s/helios/elections/%s/cast_done" % (settings.URL_HOST, election_id))
# at this point an email should have gone out to the user
# at position num_messages after, since that was the len() before we cast this ballot
email_message = mail.outbox[len(mail.outbox) - 1]
url = re.search('http://[^/]+(/[^ \n]*)', email_message.body).group(1)
# check that we can get at that URL
if not need_login:
# confusing piece: if need_login is True, that means it was a public election
# that required login before casting a ballot.
# so if need_login is False, it was a private election, and we do need to re-login here
# we need to re-login if it's a private election, because all data, including ballots
# is otherwise private
login_page = self.app.get("/helios/elections/%s/password_voter_login" % election_id)
# if we redirected, that's because we can see the page, I think
if login_page.status_int != 302:
login_form = login_page.form
# try with extra spaces
login_form['voter_id'] = ' ' + username + ' '
login_form['password'] = '******' + password + ' '
login_form.submit()
response = self.app.get(url)
self.assertContains(response, ballot.hash)
self.assertContains(response, html_escape(encrypted_vote))
# if we request the redirect to cast_done, the voter should be logged out, but not the user
response = self.app.get("/helios/elections/%s/cast_done" % election_id)
def send(self, data):
channel.send_message(self.token, utils.to_json(data))
def toJSON(self):
import utils
return utils.to_json(self.toJSONDict())
def kfold_lightgbm(train_df, test_df, num_folds):
print('Starting LightGBM. Train shape: {}'.format(train_df.shape))
# Cross validation
folds = CustomTimeSeriesSplitter(end_train=1941)
# Create arrays and dataframes to store results
oof_preds = np.zeros(train_df.shape[0])
feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in FEATS_EXCLUDED]
valid_idxs = []
best_iterations = [] # average of best iteration
# k-fold
for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df)):
# split train/valid
train_x, train_y = train_df[feats].iloc[train_idx], train_df[
'demand'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df[
'demand'].iloc[valid_idx]
# save validation indexes
valid_idxs += list(valid_idx)
# set data structure
lgb_train = lgb.Dataset(train_x, label=train_y, free_raw_data=False)
lgb_test = lgb.Dataset(valid_x, label=valid_y, free_raw_data=False)
params = {
# 'device' : 'gpu',
# 'gpu_use_dp':True,
'boosting': 'gbdt',
'metric': ['rmse'],
'objective': 'tweedie',
'learning_rate': 0.05,
'tweedie_variance_power': 1.1,
'subsample': 0.5,
'subsample_freq': 1,
'num_leaves': 2**8 - 1,
'min_data_in_leaf': 2**8 - 1,
'feature_fraction': 0.8,
'verbose': -1,
'seed': 326,
'bagging_seed': 326,
'drop_seed': 326,
'num_threads': -1
}
# train model
reg = lgb.train(params,
lgb_train,
valid_sets=[lgb_train, lgb_test],
valid_names=['train', 'test'],
num_boost_round=10000,
early_stopping_rounds=200,
verbose_eval=10)
# save model
reg.save_model(f'../output/lgbm_household_{n_fold}.txt')
# save predictions
oof_preds[valid_idx] = reg.predict(valid_x,
num_iteration=reg.best_iteration)
# save best iteration
best_iterations.append(reg.best_iteration)
# save feature importances
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = feats
fold_importance_df['importance'] = np.log1p(
reg.feature_importance(importance_type='gain',
iteration=reg.best_iteration))
fold_importance_df['fold'] = n_fold + 1
feature_importance_df = pd.concat(
[feature_importance_df, fold_importance_df], axis=0)
print('Fold %2d RMSE : %.6f' %
(n_fold + 1, rmse(valid_y, oof_preds[valid_idx])))
del reg, train_x, train_y, valid_x, valid_y
gc.collect()
# display importances
display_importances(feature_importance_df,
'../imp/lgbm_importances_cv_household.png',
'../imp/feature_importance_lgbm_cv_household.csv')
# Full RMSE score and LINE Notify
full_rmse = rmse(train_df['demand'][valid_idxs], oof_preds[valid_idxs])
line_notify('Full RMSE score %.6f' % full_rmse)
# save out of fold prediction
train_df.loc[:, 'demand'] = oof_preds
train_df[['id', 'd', 'demand']].to_csv(oof_file_name, index=False)
# save number of best iteration
configs['num_boost_round'] = best_iterations
configs['rmse'] = full_rmse
to_json(configs, '../configs/307_train_household.json')
# LINE notify
line_notify('{} done. best iterations:{}'.format(sys.argv[0],
best_iterations))
def main(_):
pp.pprint(flags.FLAGS.__flags)
if not os.path.exists(FLAGS.checkpoint_dir):
os.makedirs(FLAGS.checkpoint_dir)
if not os.path.exists(FLAGS.sample_dir):
os.makedirs(FLAGS.sample_dir)
with tf.Session() as sess:
if FLAGS.dataset == 'mnist':
dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size, y_dim=10,
dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir)
else:
dcgan = DCGAN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size,
dataset_name=FLAGS.dataset, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir)
if FLAGS.is_train:
dcgan.train(FLAGS)
else:
dcgan.load(FLAGS.checkpoint_dir)
to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0],
[dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1],
[dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2],
[dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3],
[dcgan.h4_w, dcgan.h4_b, None])
# Below is codes for visualization
OPTION = 2
if OPTION == 0:
z_sample = np.random.uniform(-0.5, 0.5, size=(FLAGS.batch_size, dcgan.z_dim))
samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})
save_images(samples, [8, 8], './samples/test_%s.png' % strftime("%Y-%m-%d %H:%M:%S", gmtime()))
elif OPTION == 1:
values = np.arange(0, 1, 1./FLAGS.batch_size)
for idx in xrange(100):
print(" [*] %d" % idx)
z_sample = np.zeros([FLAGS.batch_size, dcgan.z_dim])
for kdx, z in enumerate(z_sample):
z[idx] = values[kdx]
samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})
save_images(samples, [8, 8], './samples/test_arange_%s.png' % (idx))
elif OPTION == 2:
values = np.arange(0, 1, 1./FLAGS.batch_size)
for idx in [random.randint(0, 99) for _ in xrange(100)]:
print(" [*] %d" % idx)
z = np.random.uniform(-0.2, 0.2, size=(dcgan.z_dim))
z_sample = np.tile(z, (FLAGS.batch_size, 1))
#z_sample = np.zeros([FLAGS.batch_size, dcgan.z_dim])
for kdx, z in enumerate(z_sample):
z[idx] = values[kdx]
samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})
make_gif(samples, './samples/test_gif_%s.gif' % (idx))
elif OPTION == 3:
values = np.arange(0, 1, 1./FLAGS.batch_size)
for idx in xrange(100):
print(" [*] %d" % idx)
z_sample = np.zeros([FLAGS.batch_size, dcgan.z_dim])
for kdx, z in enumerate(z_sample):
z[idx] = values[kdx]
samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})
make_gif(samples, './samples/test_gif_%s.gif' % (idx))
elif OPTION == 4:
image_set = []
values = np.arange(0, 1, 1./FLAGS.batch_size)
for idx in xrange(100):
print(" [*] %d" % idx)
z_sample = np.zeros([FLAGS.batch_size, dcgan.z_dim])
for kdx, z in enumerate(z_sample): z[idx] = values[kdx]
image_set.append(sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample}))
make_gif(image_set[-1], './samples/test_gif_%s.gif' % (idx))
new_image_set = [merge(np.array([images[idx] for images in image_set]), [10, 10]) for idx in range(64) + range(63, -1, -1)]
make_gif(new_image_set, './samples/test_gif_merged.gif', duration=8)
elif OPTION == 5:
image_set = []
values = np.arange(0, 1, 1./FLAGS.batch_size)
z_idx = [[random.randint(0,99) for _ in xrange(5)] for _ in xrange(200)]
for idx in xrange(200):
print(" [*] %d" % idx)
#z_sample = np.zeros([FLAGS.batch_size, dcgan.z_dim])
z = np.random.uniform(-1e-1, 1e-1, size=(dcgan.z_dim))
z_sample = np.tile(z, (FLAGS.batch_size, 1))
for kdx, z in enumerate(z_sample):
for jdx in xrange(5):
z_sample[kdx][z_idx[idx][jdx]] = values[kdx]
image_set.append(sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample}))
make_gif(image_set[-1], './samples/test_gif_%s.gif' % (idx))
new_image_set = [merge(np.array([images[idx] for images in image_set]), [10, 20]) for idx in range(64) + range(63, -1, -1)]
make_gif(new_image_set, './samples/test_gif_random_merged.gif', duration=4)
elif OPTION == 6:
image_set = []
values = np.arange(0, 1, 1.0/FLAGS.batch_size).tolist()
z_idx = [[random.randint(0,99) for _ in xrange(10)] for _ in xrange(100)]
for idx in xrange(100):
print(" [*] %d" % idx)
z = np.random.uniform(-0.2, 0.2, size=(dcgan.z_dim))
z_sample = np.tile(z, (FLAGS.batch_size, 1))
for kdx, z in enumerate(z_sample):
for jdx in xrange(10):
z_sample[kdx][z_idx[idx][jdx]] = values[kdx]
image_set.append(sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample}))
save_images(image_set[-1], [8, 8], './samples/test_random_arange_%s.png' % (idx))
new_image_set = [merge(np.array([images[idx] for images in image_set]), [10, 10]) for idx in range(64) + range(63, -1, -1)]
make_gif(new_image_set, './samples/test_gif_merged_random.gif', duration=4)
elif OPTION == 7:
for _ in xrange(50):
z_idx = [[random.randint(0,99) for _ in xrange(10)] for _ in xrange(8)]
zs = []
for idx in xrange(8):
z = np.random.uniform(-0.2, 0.2, size=(dcgan.z_dim))
zs.append(np.tile(z, (8, 1)))
z_sample = np.concatenate(zs)
values = np.arange(0, 1, 1/8.)
for idx in xrange(FLAGS.batch_size):
for jdx in xrange(8):
z_sample[idx][z_idx[idx/8][jdx]] = values[idx%8]
samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})
save_images(samples, [8, 8], './samples/multiple_testt_%s.png' % strftime("%Y-%m-%d %H:%M:%S", gmtime()))
elif OPTION == 8:
counter = 0
for _ in xrange(50):
import scipy.misc
z_idx = [[random.randint(0,99) for _ in xrange(10)] for _ in xrange(8)]
zs = []
for idx in xrange(8):
z = np.random.uniform(-0.2, 0.2, size=(dcgan.z_dim))
zs.append(np.tile(z, (8, 1)))
z_sample = np.concatenate(zs)
values = np.arange(0, 1, 1/8.)
for idx in xrange(FLAGS.batch_size):
for jdx in xrange(8):
z_sample[idx][z_idx[idx/8][jdx]] = values[idx%8]
samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})
for sample in samples:
scipy.misc.imsave('./samples/turing/%s.png' % counter, sample)
counter += 1
else:
import scipy.misc
from glob import glob
samples = []
fnames = glob("/Users/carpedm20/Downloads/x/1/*.png")
fnames = sorted(fnames, key = lambda x: int(x.split("_")[1]) * 10000 + int(x.split('_')[2].split(".")[0]))
for f in fnames:
samples.append(scipy.misc.imread(f))
make_gif(samples, './samples/training.gif', duration=8, true_image=True)
def get_hash(self):
return utils.hash_b64(utils.to_json(self.toJSONDict()))
def main():
# load pkls
df = read_pickles('../feats/sales')
df_calendar = loadpkl('../feats/calendar.pkl')
df_sell_prices = loadpkl('../feats/sell_prices.pkl')
# extract foods
df = df[df['cat_id'] == 'FOODS']
# merge
df = df.merge(df_calendar, on='d',how='left')
df = df.merge(df_sell_prices, on=['store_id','item_id','wm_yr_wk'],how='left')
del df_calendar, df_sell_prices
gc.collect()
# drop pre-release rows
df = df[df['wm_yr_wk']>=df['release']]
# make lag features
df = make_lags(df,28)
# add categorical features
df['item_id_store_id'] = df['item_id']+'_'+df['store_id']
df['item_id_state_id'] = df['item_id']+'_'+df['state_id']
df['dept_id_store_id'] = df['dept_id']+'_'+df['store_id']
df['dept_id_state_id'] = df['dept_id']+'_'+df['state_id']
# label encoding
cols_string = ['item_id','dept_id','cat_id','store_id','state_id',
'item_id_store_id','item_id_state_id','dept_id_store_id',
'dept_id_state_id']
for c in cols_string:
df[c], _ = pd.factorize(df[c])
df[c].replace(-1,np.nan,inplace=True)
# add price features
df_grouped = df[['id','sell_price']].groupby('id')['sell_price']
df['shift_price_t1'] = df_grouped.transform(lambda x: x.shift(1))
df['price_change_t1'] = (df['shift_price_t1'] - df['sell_price']) / (df['shift_price_t1'])
df['rolling_price_max_t365'] = df_grouped.transform(lambda x: x.shift(1).rolling(365).max())
df['price_change_t365'] = (df['rolling_price_max_t365'] - df['sell_price']) / (df['rolling_price_max_t365'])
df['rolling_price_std_t7'] = df_grouped.transform(lambda x: x.rolling(7).std())
df['rolling_price_std_t30'] = df_grouped.transform(lambda x: x.rolling(30).std())
# features release date
df['release'] = df['release'] - df['release'].min()
# price momentum by month & year
df['price_momentum_m'] = df['sell_price']/df.groupby(['store_id','item_id','month'])['sell_price'].transform('mean')
df['price_momentum_y'] = df['sell_price']/df.groupby(['store_id','item_id','year'])['sell_price'].transform('mean')
# days for CustomTimeSeriesSplitter
df['d_numeric'] = df['d'].apply(lambda x: str(x)[2:]).astype(int)
# reduce memory usage
df = reduce_mem_usage(df)
# save as feather
to_feature(df, '../feats/f106')
# save feature name list
features_json = {'features':df.columns.tolist()}
to_json(features_json,'../configs/106_all_features_foods.json')
# LINE notify
line_notify('{} done.'.format(sys.argv[0]))
def get_hash(self):
return utils.hash_b64(utils.to_json(self.toJSONDict()))
def toJSON(self): return utils.to_json(self.toJSONDict())
def hash(self): s = utils.to_json(self.toJSONDict()) return utils.hash_b64(s)
# get all
r = requests.get("http://localhost:5000/statuses")
print r.content
r = requests.get("http://localhost:5000/status/livingroom")
print r.content
# update
r = requests.post(
url="http://localhost:5000/status/livingroom",
headers={'Content-type': 'application/json'},
data=utils.to_json({"status": "FIRE" })
)
print r.content
# send sms
r = requests.post(
url="http://localhost:5000/status/livingroom",
headers={'Content-type': 'application/json'},
data=utils.to_json({"occupancy": 1 })
)
print r.content
r = requests.post(
url="http://localhost:5000/status/livingroom",
headers={'Content-type': 'application/json'},
data=utils.to_json({"occupancy": 0, "carbon_detected": False })
def _setup_complete_election(self, election_params={}):
"do the setup part of a whole election"
# a bogus call to set up the session
self.client.get("/")
# REPLACE with params?
self.setup_login()
# create the election
full_election_params = {
"short_name" : "test-complete",
"name" : "Test Complete",
"description" : "A complete election test",
"election_type" : "referendum",
"use_voter_aliases": "0",
"use_advanced_audit_features": "1",
"private_p" : "0"}
# override with the given
full_election_params.update(election_params)
response = self.client.post("/helios/elections/new", full_election_params)
# we are redirected to the election, let's extract the ID out of the URL
election_id = re.search('/elections/([^/]+)/', str(response['Location'])).group(1)
# helios is automatically added as a trustee
# check that helios is indeed a trustee
response = self.client.get("/helios/elections/%s/trustees/view" % election_id)
self.assertContains(response, "Trustee #1")
# add a few voters, via file upload
FILE = "helios/fixtures/voter-file.csv"
voters_file = open(FILE)
response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'voters_file': voters_file})
voters_file.close()
self.assertContains(response, "first few rows of this file")
# now we confirm the upload
response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'confirm_p': "1"})
self.assertRedirects(response, "/helios/elections/%s/voters/list" % election_id)
# and we want to check that there are now voters
response = self.client.get("/helios/elections/%s/voters/" % election_id)
NUM_VOTERS = 4
self.assertEquals(len(utils.from_json(response.content)), NUM_VOTERS)
# let's get a single voter
single_voter = models.Election.objects.get(uuid = election_id).voter_set.all()[0]
response = self.client.get("/helios/elections/%s/voters/%s" % (election_id, single_voter.uuid))
self.assertContains(response, '"uuid": "%s"' % single_voter.uuid)
response = self.client.get("/helios/elections/%s/voters/foobar" % election_id)
self.assertEquals(response.status_code, 404)
# add questions
response = self.client.post("/helios/elections/%s/save_questions" % election_id, {
'questions_json': utils.to_json([{"answer_urls": [None,None], "answers": ["Alice", "Bob"], "choice_type": "approval", "max": 1, "min": 0, "question": "Who should be president?", "result_type": "absolute", "short_name": "Who should be president?", "tally_type": "homomorphic"}]),
'csrf_token': self.client.session['csrf_token']})
self.assertContains(response, "SUCCESS")
# freeze election
response = self.client.post("/helios/elections/%s/freeze" % election_id, {
"csrf_token" : self.client.session['csrf_token']})
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
# email the voters
num_messages_before = len(mail.outbox)
response = self.client.post("/helios/elections/%s/voters/email" % election_id, {
"csrf_token" : self.client.session['csrf_token'],
"subject" : "your password",
"body" : "time to vote",
"suppress_election_links" : "0",
"send_to" : "all"
})
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
num_messages_after = len(mail.outbox)
self.assertEquals(num_messages_after - num_messages_before, NUM_VOTERS)
email_message = mail.outbox[num_messages_before]
assert "your password" in email_message.subject, "bad subject in email"
# get the username and password
username = re.search('voter ID: (.*)', email_message.body).group(1)
password = re.search('password: (.*)', email_message.body).group(1)
# now log out as administrator
self.clear_login()
self.assertEquals(self.client.session.has_key('user'), False)
# return the voter username and password to vote
return election_id, username, password
def convert_to_json(self, *args, **kwargs): return render_json(utils.to_json(func(self, *args, **kwargs)))
def personal_cart(request):
data = PersonalCartResource().to_dict(obj=request.user, request=request)
return render(request, 'cart/personal_cart.j.html', {
'cart_json': to_json(data),
'is_empty': not data['buys']
})
def hash(self):
s = utils.to_json(self.toJSONDict())
return utils.hash_b64(s)
