def _update_config(self, data):
changed = False
new_interval = False
for key in ("name", "description", "interval_time", "interval_units",
"brightness", "contrast", "saturation"):
val = data.get(key, None)
logit("debug", "key:", key, "; val:", val)
if val is None:
continue
local_val = getattr(self, key)
if local_val != val:
setattr(self, key, val)
monitor_keys = ("brightness", "contrast", "saturation")
section = "monitor" if key in monitor_keys else "frame"
self.parser.set(section, key, str(val))
changed = True
new_interval = new_interval or "interval" in key
if changed:
with open(CONFIG_FILE, "w") as ff:
self.parser.write(ff)
if new_interval:
self.interval = _normalize_interval(self.interval_time,
self.interval_units)
logit("info", "Setting timer to", self.interval)
self.set_image_interval()
def get_freespace():
out, err = runproc("df -BM .")
ret = out.split('\n')[1].split()[3]
# Remove the trailing 'M'
ret = ret.replace("M", "")
ret = int(ret) * ONE_MB
logit("debug", "Free disk space =", ret)
return ret
def get_conn():
try:
access_key = env.account['access_key']
access_secret = env.account['access_secret']
conn = boto.ec2.connect_to_region(env.region, aws_access_key_id=access_key, aws_secret_access_key=access_secret)
env.snsc = boto.sns.connect_to_region(env.region, aws_access_key_id=access_key, aws_secret_access_key=access_secret)
return conn
except Exception as e:
error_msg = 'get_conn(): unable to connect to ec2 region, reason: {}'.format(e)
logit('exception', error_msg)
def run(self):
while True:
raw_ircmsg = self.ircsock.recv(2048)
try:
ircmsg = raw_ircmsg.decode("UTF-8")
except UnicodeDecodeError as e:
logit("Received non-UTF8 encoded message: '%s'" % raw_ircmsg,
force=True)
ircmsg = ircmsg.strip(" \n\r")
self.process_msg(ircmsg)
def _download(self, img):
headers = {"user-agent": "photoviewer"}
url = "%s/%s" % (self.dl_url, img)
logit("debug", url)
logit("info", "downloading", img)
resp = requests.get(url, headers=headers, stream=True)
if resp.status_code == 200:
outfile = os.path.join(DOWNLOAD_PHOTODIR, img)
with open(outfile, "wb") as ff:
resp.raw.decode_content = True
shutil.copyfileobj(resp.raw, ff)
return outfile
def wait_for(self, txt):
"""Waits for the server to send a message containing the requested
text. 'txt' can be a list of strings, or a single string.
"""
if not isinstance(txt, (list, tuple)):
txt = [txt]
while True:
ircmsg = self.ircsock.recv(2048).decode("UTF-8")
ircmsg = ircmsg.strip("\n\r")
if any(word in ircmsg for word in txt):
logit(ircmsg, "FOUND", *txt, force=self.verbose)
return
self.process_msg(ircmsg)
def main():
parser = argparse.ArgumentParser(description="IRC Bot")
parser.add_argument("--channel-file", "-f", help="The path of the file "
"containing the names of the channels to join, one per line.")
parser.add_argument("--verbose", "-v", action="store_true",
help="Enables verbose output.")
args = parser.parse_args()
if args.verbose:
utils.LOG.level = utils.logging.DEBUG
# Create a pool of channels for the bots to pick from
channel_queue = Queue()
if args.channel_file:
with open(args.channel_file) as ff:
chan_lines = ff.read().splitlines()
[channel_queue.put(chan.strip()) for chan in chan_lines]
else:
print("You must specify a channel file")
exit(1)
# Add flags for the bots to know the queue is empty
for i in range(4):
channel_queue.put(None)
# We need four bots to cover all the channels
bots = []
procs = []
for num in range(4):
bot = LogBot(num, args.verbose)
logit("NICK:", bot.nick, force=True)
bots.append(bot)
proc = Process(target=connect_bot, args=(bot,))
procs.append(proc)
proc.start()
for p in procs:
p.join()
procs = []
for bot in bots:
proc = Process(target=join_channels, args=(bot, channel_queue),
daemon=False)
proc.start()
procs.append(proc)
[p.join() for p in procs]
logit("All channels joined for %s bot." % args.nick_base, force=True)
procs = []
for bot in bots:
proc = Process(target=runbot, args=(bot,))
proc.start()
procs.append(proc)
process_pids.append(proc.pid)
[p.join() for p in procs]
def _read_config(self, signum=None, frame=None):
logit("info", "_read_config called!")
try:
self.parser.read(CONFIG_FILE)
except ConfigParser.MissingSectionHeaderError as e:
# The file exists, but doesn't have the correct format.
raise exc.InvalidConfigurationFile(e)
def safe_get(section, option, default=None):
try:
return self.parser.get(section, option)
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
return default
self.log_level = safe_get("frame", "log_level", "INFO")
set_log_level(self.log_level)
self.reg_url = safe_get("host", "reg_url")
if not self.reg_url:
logit("error", "No registration URL in photo.cfg; exiting")
exit()
self.dl_url = safe_get("host", "dl_url")
if not self.dl_url:
logit("error", "No download URL configured in photo.cfg; exiting")
exit()
self.check_url = safe_get("host", "check_url", None)
self.frameset = safe_get("frameset", "name", "")
self.name = safe_get("frame", "name", "undefined")
self.pkid = safe_get("frame", "pkid", "")
self.description = safe_get("frame", "description", "")
self.orientation = safe_get("frame", "orientation", "H")
# When to start the image rotation
self.interval_base = safe_get("frame", "interval_base", "*:*")
# How often to change image
self.interval_time = int(safe_get("frame", "interval_time", 10))
# Units of time for the image change interval
self.interval_units = safe_get("frame", "interval_units", "minutes")
self.interval = _normalize_interval(self.interval_time,
self.interval_units)
logit("info", "Setting image interval to", self.interval)
self.set_image_interval()
check_interval = int(safe_get("frame", "host_check", 120))
check_units = safe_get("frame", "host_check_units", "minutes")
self.check_interval = _normalize_interval(check_interval, check_units)
logit("info", "Setting host check interval to", self.check_interval)
self.brightness = safe_get("monitor", "brightness")
self.contrast = safe_get("monitor", "contrast")
self.saturation = safe_get("monitor", "saturation")
def forward(self, x, log_prior=None):
batch_size = x.size(0)
truncation = self.beta_a.size(0)
beta_a = F.softplus(self.beta_a) + 0.01
beta_b = F.softplus(self.beta_b) + 0.01
# might be passed in for IWAE
if log_prior is None:
log_prior = reparametrize(
beta_a.view(1, truncation).expand(batch_size, truncation),
beta_b.view(1, truncation).expand(batch_size, truncation),
ibp=True, log=True)
logit_x, mu, logvar = self.encode(x)
logit_post = logit_x + logit(log_prior.exp())
logsample = reparametrize_discrete(logit_post, self.temp)
z_discrete = F.sigmoid(logsample) # binary
z_continuous = reparametrize_gaussian(mu, logvar)
# zero-temperature rounding
if not self.training:
z_discrete = torch.round(z_discrete)
inputForDecoder = z_discrete * z_continuous
dec = self.decode(inputForDecoder)
return dec, logsample, logit_post, log_prior, mu, logvar, z_discrete, z_continuous
def plot_boundary(X, y, coefs, title='', save=False):
x_min, x_max = np.min(X[:, 0]), np.max(X[:, 0])
y_min, y_max = np.min(X[:, 1]), np.max(X[:, 1])
offset = 1
boundary_x = np.linspace(x_min - offset, x_max + offset, 1000)
boundary_y = [
(utils.logit(1 / 2) - coefs['b'] - coefs['w1'] * x) / coefs['w2']
for x in boundary_x
]
plt.figure(figsize=(9, 9))
plt.plot(boundary_x, boundary_y, c='g', label='Decision boundary')
plt.scatter(X[y == 0, 0], X[y == 0, 1], label='class 0')
plt.scatter(X[y == 1, 0], X[y == 1, 1], label='class 1')
plt.xlim(x_min - offset, x_max + offset)
plt.ylim(y_min - offset, y_max + offset)
plt.xlabel('x1')
plt.ylabel('x2')
plt.title(title)
plt.legend(loc="upper right", fontsize=16)
if save:
print("Save figure results folder.")
plt.savefig('results/{}'.format(utils.sanitize(title)))
plt.show()
def forward(x):
"""
Applies a logit function robust to x=0
:param x: float value satisfying: 0 <= x < 1
:return: the logit of input x
"""
return logit(x)
def get_theta0(y):
prior = GammaIIDIncr.get_prior()
theta0 = prior.rvs()
theta0['log_c'] = np.log(1.0) + 0.1 * npr.randn()
theta0['log_eta'] = np.log(1.0) + 0.1 * npr.randn()
theta0['logit_sigma'] = sigmoid(logit(0.5) + 0.1 * npr.randn())
return theta0
def log_odds(x, mu0, mu1, sigma0, sigma1, pi):
sigma0_inv = np.linalg.inv(sigma0)
sigma1_inv = np.linalg.inv(sigma1)
return utils.logit(pi) + 0.5 * np.log(
np.linalg.det(sigma0) /
np.linalg.det(sigma1)) - 0.5 * (x - mu1).T @ sigma1_inv @ (
x - mu1) + 0.5 * (x - mu0).T @ sigma0_inv @ (x - mu0)
def start(self):
update_alive()
# If a prior import crashed during image processing, re-process the
# images.
self._process_images()
signal.signal(signal.SIGHUP, self._read_config)
signal.signal(signal.SIGURG, self.check_host)
signal.signal(signal.SIGTSTP, self.pause)
signal.signal(signal.SIGCONT, self.resume)
signal.signal(signal.SIGTRAP, self.navigate)
if not self.check_timer.is_alive():
self.check_timer.start()
if not self.photo_timer.is_alive():
self.photo_timer.start()
logit("debug", "Timers started")
self._started = True
self.show_photo()
def forward(self, z):
"""
generate the mean parameter for the Bernoulli distribution
"""
z = z.view(-1, self.k)
z = z.to(self.device)
p = (torch.mm(logit(z), self.log_W.exp()) + self.b).sigmoid()
return p
def set_timer(self, typ, start=False):
# Clear the current timer first
curr_tmr = self.photo_timer if typ == "photo" else self.check_timer
if curr_tmr:
curr_tmr.cancel()
tmr = None
if typ == "photo":
interval = self.initial_interval or self.interval
self.initial_interval = 0
tmr = self.photo_timer = Timer(interval, self.navigate)
elif typ == "check":
tmr = self.check_timer = Timer(self.check_interval,
self.check_host)
logit("debug", "Timer set for:", typ, tmr.interval, tmr.function)
if tmr and start:
tmr.start()
logit("debug", "Timer started for", typ)
def get_theta0(y):
prior = GBFRYIIDIncr.get_prior()
theta0 = prior.rvs()
halpha = hill_estimate(y, 50)[15:].mean()
theta0['log_c'] = np.log(1.0) + 0.1 * npr.randn()
theta0['log_eta'] = np.log(1.0) + 0.1 * npr.randn()
theta0['logit_sigma'] = sigmoid(logit(0.5) + 0.1 * npr.randn())
theta0['log_tau_minus_one'] = np.log(0.5 * halpha)
return theta0
def __init__(self,
mu=0.,
beta=0.,
log_eta=np.log(0.1),
logit_sigma=logit(0.2),
volumes=1.):
super(NSIIDIncr, self).__init__(mu=mu, beta=beta)
self.log_eta = log_eta
self.logit_sigma = logit_sigma
self.volumes = volumes
def _process_images(self):
images_to_process = glob.glob("%s/*.jpg" % DOWNLOAD_PHOTODIR)
if not images_to_process:
logit("debug", "No images to process")
return
num_imgs = len(images_to_process)
logit("debug", "Processing %s images" % num_imgs)
for pos, img in enumerate(images_to_process):
logit("info", "Processing image %s (%s of %s)" %
(img, pos+1, num_imgs))
image.adjust(img, self.brightness, self.contrast, self.saturation)
shutil.move(img, PHOTODIR)
logit("info", "Moved processed image %s from the download to the "
"image directory." % img)
def process_msg(self, ircmsg):
if not ircmsg:
return
# This can be useful for debugging
is_ping = self.verbose or "PING :" in ircmsg
logit(self.nick, ircmsg, force=is_ping)
if "PING :" in ircmsg:
self.ping()
return
mtch = MSG_PAT.match(ircmsg)
if mtch:
groups = mtch.groups()
nick = groups[0]
channel = groups[3]
remark = groups[4]
if remark.startswith("ACTION "):
remark = remark.replace("ACTION ", "/me ")
if len(nick) >= 17:
logit("Odd nick: %s" % nick, force=True)
return
tm = dt.datetime.utcnow().replace(microsecond=0)
record(nick, channel, remark, tm, force=self.verbose)
def record(nick, channel, remark, tm, force=False):
tmstr = tm.strftime("%Y-%m-%dT%H:%M:%S")
body = {"channel": channel, "nick": nick, "posted": tmstr,
"remark": remark}
# Leave the time out of the hash, as two bots could differ on when they saw
# a posting, and generate different IDs, which would result in duplicate
# postings.
hash_body = {"channel": channel, "nick": nick, "remark": remark}
hashval = utils.gen_key(hash_body)
body["id"] = hashval
attempts = 0
while True:
try:
logit("RECORDING", body, "ID =", hashval, force=True)
es_client.index(index="irclog", doc_type="irc", id=hashval,
body=body)
break
except Exception as e:
attempts += 1
if attempts >= MAX_RETRIES:
logit("Elasticsearch exception: %s" % e, force=True)
break
def __init__(self,
mu=0.,
beta=0.,
log_eta=np.log(0.1),
log_tau_minus_one=np.log(3.0 - 1.),
logit_sigma=logit(0.2),
log_c=np.log(1.),
volumes=1.):
super(GBFRYIIDIncr, self).__init__(mu=mu, beta=beta)
self.log_eta = log_eta
self.log_tau_minus_one = log_tau_minus_one
self.logit_sigma = logit_sigma
self.log_c = log_c
self.volumes = volumes
def curate_snapshots():
''' curates snapshots as defined by their retention time in config.yaml '''
for instance in conf['instances']:
env.region = instance['region']
env.account = conf['accounts'][instance['account']]
conn = get_conn()
reservations = conn.get_all_instances(filters={'tag:Name': instance['name']})
res_instances = [i for r in reservations for i in r.instances]
for i in res_instances:
volumes = conn.get_all_volumes(filters={'attachment.instance_id': i.id})
for volume in volumes:
snapshots = conn.get_all_snapshots(filters={'volume-id': volume.id, 'status': 'completed'})
for snapshot in snapshots:
retention_time = datemath('now-%s' % instance.get('retention', '7d'))
snapshot_create_time = dateutil.parser.parse(snapshot.start_time)
# remove snapshots if they are over their retention period
if snapshot_create_time < retention_time:
try:
conn.delete_snapshot(snapshot.id)
logit('info', 'EBS snapshot %s/%s deleted for %s/%s, because it was older than %s old' % ( snapshot.id, volume.id, instance['region'], instance['name'], retention_time))
except Exception as e:
error_msg = 'Unable to delete EBS snapshot: reson: %s, id: %s, volume: %s, host: %s, region: %s' % ( e, snapshot.id, volume.id, instance['name'], instance['region'] )
logit('error', error_msg)
def _register(self):
headers = {"user-agent": "photoviewer"}
# Get free disk space
freespace = get_freespace()
data = {"pkid": self.pkid, "name": self.name,
"description": self.description,
"interval_time": self.interval_time,
"interval_units": self.interval_units,
"orientation": self.orientation, "freespace": freespace,
"frameset": self.frameset}
resp = requests.post(self.reg_url, data=data, headers=headers)
if 200 <= resp.status_code <= 299:
# Success!
pkid, images = resp.json()
if pkid != self.pkid:
self.parser.set("frame", "pkid", pkid)
with open(CONFIG_FILE, "w") as ff:
self.parser.write(ff)
self.host_images = images
self._update_images()
else:
logit("error", resp.status_code, resp.text)
exit()
def join_chan(self, chan):
logit("-" * 88, force=True)
logit("JOINING", chan, force=True)
self.ircsock.send(bytes("JOIN %s\n" % chan, "UTF-8"))
# See if this makes the IRC server happier about flooding it with
# requests...
self.pause(PAUSE_BETWEEN_JOINS)
logit("!!!!!JOINED", chan, force=True)
heartbeat(self)
def linear_coef(mu0, mu1, sigma0, sigma1, pi):
sigma0_inv = np.linalg.inv(sigma0)
sigma1_inv = np.linalg.inv(sigma1)
alpha = sigma0_inv - sigma1_inv
beta = mu1.T @ sigma1_inv - mu0.T @ sigma0_inv
a = 0.5 * alpha[0, 0]
b = 0.5 * alpha[0, 1]
c = 0.5 * alpha[1, 1]
d = beta[0]
e = beta[1]
f = utils.logit(pi) + 0.5 * np.log(
np.linalg.det(sigma0) / np.linalg.det(sigma1)
) - 0.5 * mu1.T @ sigma1_inv @ mu1 + 0.5 * mu0.T @ sigma0_inv @ mu0
return a, b, c, d, e, f
def nll_and_kl(recon_x, x, log_likelihood, a, b, logsample, z_discrete, logit_post, log_prior, mu, logvar, dataset_size, args=Namespace(), test=False):
batch_size = x.size()[0]
NLL = -1 * log_likelihood(recon_x, x)
KL_zreal = -0.5 * (1. + logvar - mu**2 - logvar.exp())
KL_beta = kl_divergence(a, b, prior_alpha=args.alpha0, log_beta_prior=np.log(1./args.alpha0), args=args).repeat(batch_size, 1) * (1. / dataset_size)
# in test mode, our samples are essentially coming from a Bernoulli
if not test:
KL_discrete = kl_discrete(logit_post, logit(log_prior.exp()), logsample, args.temp, args.temp_prior)
else:
pi_prior = torch.exp(log_prior)
pi_posterior = torch.sigmoid(logit_post)
kl_1 = z_discrete * (pi_posterior + SMALL).log() + (1 - z_discrete) * (1 - pi_posterior + SMALL).log()
kl_2 = z_discrete * (pi_prior + SMALL).log() + (1 - z_discrete) * (1 - pi_prior + SMALL).log()
KL_discrete = kl_1 - kl_2
return NLL, KL_zreal, KL_beta, KL_discrete
def infer(args, unlabeled, ckpt_file):
batch_size = args["batch_size"]
unlabeled_train_data = [train_dset[i] for i in unlabeled]
unlabeled_train_dset = tf.data.Dataset.from_generator(
lambda: unlabeled_train_data,
(tf.float32, tf.float32),
(tf.TensorShape([None]), tf.TensorShape([])),
).padded_batch(batch_size)
model = tf.keras.models.load_model(
os.path.join(args["EXPT_DIR"], ckpt_file))
model.compile(
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=tf.keras.optimizers.Adam(1e-4),
metrics=["accuracy"],
)
outputs_fin = {}
i = 0
for data, labels in tqdm(unlabeled_train_dset, desc="Inferring"):
outputs = model(data).numpy().flatten()
pred = np.copy(outputs)
pred[pred <= 0] = 0
pred[pred > 0] = 1
for j in range(len(pred)):
outputs_fin[i] = {}
outputs_fin[i]["prediction"] = pred[j]
outputs_fin[i]["pre_softmax"] = [
utils.logit(1 - tf.math.sigmoid(outputs[j])).numpy(),
outputs[j],
]
i += 1
return {"outputs": outputs_fin}
def make_snapshots():
''' makes EBS snapshots for instances as defined in config.yaml '''
for instance in conf['instances']:
env.region = instance['region']
env.account = conf['accounts'][instance['account']]
conn = get_conn()
reservations = conn.get_all_instances(filters={'tag:Name': instance['name']})
res_instances = [i for r in reservations for i in r.instances]
if len(res_instances) == 0:
logit('error', 'Unable to find reservations for instance name: %s in region: %s on account: %s' % (instance['name'], instance['region'], instance['account']))
continue
for i in res_instances:
volumes = conn.get_all_volumes(filters={'attachment.instance_id': i.id})
for volume in volumes:
snapshot = conn.create_snapshot(volume.id, '%s - %s - EBS snapshot for %s' % (instance['name'], today, volume.id))
if snapshot:
logit('info', 'Snapshot Created: %s - %s on volume: %s, volume_size: %sGB, snapid: %s' % (today, instance['name'], volume.id, volume.size, snapshot.id))
else:
error_msg = 'Unable to create EBS snapshot for %s on volume: %s on %s' % (instance['name'], volume.id, today)
logit('error', error_msg)
self.parser.set(section, key, str(val))
changed = True
new_interval = new_interval or "interval" in key
if changed:
with open(CONFIG_FILE, "w") as ff:
self.parser.write(ff)
if new_interval:
self.interval = _normalize_interval(self.interval_time,
self.interval_units)
logit("info", "Setting timer to", self.interval)
self.set_image_interval()
def kill_all(self):
"""Kills all timers on receiving a Ctrl-C."""
logit("error", "Killing timers")
self.photo_timer.cancel()
self.check_timer.cancel()
logit("error", "Timers canceled")
if __name__ == "__main__":
with open("photo.pid", "w") as ff:
ff.write("%s" % os.getpid())
img_mgr = ImageManager()
try:
img_mgr.start()
logit("debug", "And we're off!")
except KeyboardInterrupt:
img_mgr.kill_all()
def _set_power_on(self):
# Power on the monitor and HDMI output
logit("debug", "Powering on the monitor")
out, err = runproc(MONITOR_CMD)
logit("debug", "Power result", out, err)
parser.add_argument('--T', type=int, default=500)
parser.add_argument('--mu', type=float, default=0.0)
parser.add_argument('--beta', type=float, default=0.0)
parser.add_argument('--eta', type=float, default=.6)
parser.add_argument('--sigma', type=float, default=.7)
parser.add_argument('--tau', type=float, default=1.8)
parser.add_argument('--c', type=float, default=1.0)
parser.add_argument('--filename', type=str, default=None)
args = parser.parse_args()
data = {}
true_params = {
'mu': args.mu,
'beta': args.beta,
'logit_sigma': logit(args.sigma),
'log_eta': np.log(args.eta),
'log_tau_minus_one': np.log(args.tau - 1),
'log_c': np.log(args.c)
}
model = GBFRYIIDIncr(**true_params)
x, y = model.simulate(args.T)
data['true_params'] = true_params
data['x'] = x
data['y'] = np.array(y).squeeze()
if not os.path.isdir('../data'):
os.makedirs('../data')
if args.filename is None:
def ping(self): # respond to server Pings.
self.ircsock.send(bytes("PONG :pingis\n", "UTF-8"))
logit("PONG!", force=self.verbose)
heartbeat(self)
def plot_indiv_student_activity(self,
sidx,
filename,
detected_cp_loc,
alpha_i,
bic_with_cp,
bic_wo_cp,
xlim=None,
ylim=None,
ylim2=None,
legend_loc='upper left',
legend_loc2=None,
subtract_mean=False):
"""
Plotting individual student's activity. Used inside of the function 'naive_changepoint_detection()'
"""
X_NT = self.rawdata
X1t = X_NT[sidx, :]
alpha_i1, alpha_i2, alpha_i0 = alpha_i
avg_clicks = np.sum(X_NT, axis=0) / float(X_NT.shape[0])
if self.binary:
mu_t_hat = logit(avg_clicks + 0.0000001)
else:
mu_t_hat = np.log(avg_clicks + 0.0000001)
ai1_arr = np.ones(detected_cp_loc) * (alpha_i1 +
mu_t_hat[:detected_cp_loc])
ai2_arr = np.ones(self.n_days - detected_cp_loc) * (
alpha_i2 + mu_t_hat[detected_cp_loc:])
ai0_arr = alpha_i0 + mu_t_hat
logodds_wo_cp = ai0_arr
logodds_w_cp = np.concatenate((ai1_arr, ai2_arr))
fig, axs = plt.subplots(2,
1,
figsize=(9, 8),
gridspec_kw={'height_ratios': [2, 1]})
ax_lam = axs[0]
ax_lam.plot(logodds_wo_cp,
color='gray',
linewidth=3,
alpha=0.6,
label="M1, BIC=" + str(round(bic_wo_cp, 2)))
ax_lam.plot(logodds_w_cp,
color='#0C4EC9',
linewidth=3,
label="M2, BIC=" + str(round(bic_with_cp, 2)))
ax_lam.axvline(x=detected_cp_loc,
ymin=0,
ymax=1,
color='red',
linewidth=3,
alpha=0.8)
ax_lam.grid(alpha=0.3)
ax_lam.legend(loc=legend_loc, fontsize=14)
if ylim is not None:
ax_lam.set_ylim(ylim[0], ylim[1])
if xlim is None:
ax_lam.set_xlim(-1, self.n_days)
else:
ax_lam.set_xlim(xlim[0], xlim[1])
ax_lam.set_ylabel(r"$\hat{\mu}_t + \hat{\alpha_i}$", fontsize=18)
ax_lam.tick_params(labelsize=12)
ax_raw = axs[1]
if self.binary:
X1t_ones = np.where(X1t == 1)[0]
for xx in X1t_ones[:-1]:
ax_raw.axvline(x=xx,
ymin=0,
ymax=1,
color="black",
linewidth=3,
alpha=0.7)
ax_raw.axvline(x=X1t_ones[-1],
ymin=0,
ymax=1,
color="black",
linewidth=3,
alpha=0.7)
ax_raw.set_yticks([])
else:
if subtract_mean:
X_diff = X1t - avg_clicks
ax_raw.bar(range(len(X_diff)),
X_diff,
width=0.6,
color="black",
alpha=0.7,
edgecolor='white',
align='center')
ax_raw.set_ylabel("$x_{it} - \hat{\lambda}_t$", fontsize=19)
else:
ax_raw.bar(range(len(X1t)),
X1t,
width=0.6,
color="black",
alpha=0.7,
edgecolor='white',
align='center')
ax_raw.set_ylabel(r"$x_{it}$", fontsize=19)
ax_raw.grid(alpha=0.3)
if ylim2 is not None:
ax_raw.set_ylim(ylim2[0], ylim2[1])
ax_raw.axvline(x=detected_cp_loc - 0.5,
ymin=0,
ymax=1,
color='red',
linewidth=3,
alpha=0.8,
label="DETECTED CP")
if legend_loc2 is None:
legend_loc2 = legend_loc
ax_raw.legend(loc=legend_loc2, fontsize=14)
if xlim is None:
ax_raw.set_xlim(-1, self.n_days)
else:
ax_raw.set_xlim(xlim[0], xlim[1])
ax_raw.tick_params(labelsize=12)
ax_raw.set_xlabel('DAYS', fontsize=15)
plt.savefig(filename)
plt.close()
# Number of observations
T = 5000
T_test = 50000
# Parameters of the IID GBFRY model
params = {"eta": 1., "c": 1., "tau": 3., "sigma": .6}
# Directory where to save the generated data
save_dir = "../data/simulated/"
# Simulate y from model
ssm_cls = GBFRYIIDIncr
theta = {
'log_eta': np.log(params['eta']),
'log_c': np.log(params['c']),
'log_tau_minus_one': np.log(params['tau'] - 1.),
'logit_sigma': logit(params['sigma'])
}
model = GBFRYIIDIncr(**theta)
# Generate train
x, y = model.simulate(T)
data = pd.DataFrame()
data['x'] = x
data['y'] = np.array(y).squeeze()
data['Volume'] = 500 * np.ones_like(y)
# Generate test
x_test, y_test = model.simulate(T_test)
def _update_images(self):
"""Compares the associated images received from the server, and updates
the local copies if needed.
"""
images = self.host_images
curr = set([os.path.basename(img) for img in self.current_images])
upd = set(images)
if upd == curr:
logit("debug", "No changes in _update_images")
# No changes
return False
logit("debug", "updating images...")
to_remove = curr - upd
logit("debug", "To remove:", *to_remove)
freespace = get_freespace()
logit("debug", "Freespace", freespace)
for img in to_remove:
curr_loc = os.path.join(PHOTODIR, img)
# Remove the frame buffer copy, if any.
clean_fb(curr_loc)
if freespace < ONE_GB:
# Just delete it
logit("info", "deleting", curr_loc)
os.unlink(curr_loc)
else:
new_loc = os.path.join(INACTIVE_PHOTODIR, img)
logit("info", "inactivating", img)
shutil.move(curr_loc, new_loc)
to_get = upd - curr
logit("debug", "To get:", *to_get)
num_to_get = len(to_get)
for pos, img in enumerate(to_get):
logit("info", "adding", img)
logit("debug", "PROCESSING IMAGE #%s of %s" % (pos, num_to_get))
# Check if it's local
inactive_loc = os.path.join(INACTIVE_PHOTODIR, img)
if os.path.exists(inactive_loc):
logit("info", "retrieving from inactive")
active_loc = os.path.join(PHOTODIR, img)
shutil.move(inactive_loc, active_loc)
continue
# Check if it's downloaded but unprocessed
download_loc = os.path.join(DOWNLOAD_PHOTODIR, img)
if os.path.exists(download_loc):
logit("info", "Image already downloaded")
continue
# Not on disk, so download it
self._download(img)
self._process_images()
logit("info", "Image update is done!")
return True
def navigate(self, signum=None, frame=None):
"""Moves to the next image. """
logit("debug", "navigate called; current index", self.image_index)
num_images = len(self.current_images)
if not num_images:
# Currently no images specified for this display, so just return.
return
new_index = self.image_index + 1
# Boundaries
max_index = len(self.current_images) - 1
if new_index > max_index:
new_index = 0
# Shuffle the images
logit("info", "All images shown; shuffling order.")
random.shuffle(self.current_images)
else:
new_index = max(0, min(max_index, new_index))
logit("debug", "new index", new_index)
if new_index != self.image_index:
self.image_index = new_index
self.show_photo()
elif new_index == 0:
# There is only one image to display; make sure that it is the same
# as the one currently displayed.
# First, handle case where current image has been deleted locally
img_exists = os.path.exists(self.displayed_name)
logit("debug", "image exists:", img_exists)
if not img_exists and self.displayed_name in self.current_images:
self.current_images.remove(self.displayed_name)
logit("debug", "refreshing missing images")
self._update_images()
displayed_file = "%s/%s" % (PHOTODIR, curr_display_list[0])
logit("debug", "comparing:", self.displayed_name, displayed_file)
same_files = filecmp.cmp(self.displayed_name, displayed_file)
logit("debug", "Files are", "same" if same_files else "different")
if not same_files:
self.show_photo()
self.set_timer("photo", True)
def show_photo(self):
if not self._started:
return
if not self.current_images:
return
try:
fname = self.current_images[self.image_index]
except IndexError as e:
logit("error", "BAD INDEX", e)
if self.current_images:
fname = self.current_images[-1]
else:
# Something's screwy
logit("error", "No images!")
self.load_images()
return
if fname == self.displayed_name:
return
if os.path.exists(self.displayed_name):
fb_loc = fb_path(self.displayed_name)
if not os.path.exists(fb_loc):
cmd = "cp -f /dev/fb0 '%s'" % fb_loc
runproc(cmd, wait=True)
logit("debug", "Created frame buffer copy:", fb_loc)
runproc("sudo killall fbi")
self.displayed_name = fname
logit("debug", "displayed image path:", self.displayed_name)
logit("info", "Changing photo to", os.path.basename(fname))
# See if there is already a frame buffer version of the image
new_fb_loc = fb_path(fname)
if os.path.exists(new_fb_loc):
# Just copy it to the frame buffer device
cmd = "cp -f '%s' /dev/fb0" % new_fb_loc
runproc(cmd, wait=False)
logit("debug", "Retrieved frame buffer copy:", self.displayed_name)
return
cmd = VIEWER_CMD % fname
logit("debug", "Command:", cmd)
runproc(cmd, wait=False)
def heartbeat(bot):
cmd = "touch %s" % bot.heartbeat_file
os.system(cmd)
logit("HEARTBEAT!!", cmd, force=True)
def check_host(self, signum=None, frame=None):
"""Contact the host to update local status."""
logit("info", "check_host called")
# Update the flag file.
update_alive()
if self.check_url is None:
# Not configured
logit("warning", "No host check URL defined")
return
# If we are already in check_host, skip
if self.in_check_host:
logit("debug", "check_host called; already in the process")
return
self.in_check_host = True
headers = {"user-agent": "photoviewer"}
url = self.check_url.replace("PKID", self.pkid)
try:
resp = requests.get(url, headers=headers)
if resp.status_code >= 300:
logit("error", resp.status_code, resp.content)
self.in_check_host = False
self.set_timer("check", True)
return
except Exception as e:
logit("error", "check_host", resp.status_code, e)
self.in_check_host = False
self.set_timer("check", True)
return
data = resp.json()
logit("debug", "Data:", data)
# See if anything has updated on the server side
self._update_config(data)
# Check for image changes
self.host_images = data["images"]
logit("debug", "Updating images")
# This will return True if the images have changed
if self._update_images():
logit("debug", "_update_images indicated changed images")
self.load_images()
# Setting this to the end of the image list will cause navigate()
# to shuffle the images and start at index 0.
self.image_index = len(self.current_images)
self.navigate()
logit("debug", "check_host done.")
self.set_timer("check", True)
self.in_check_host = False
def adjust(img_file, bright, contrast, saturation):
"""Uses the local profile to adjust the image to look good on the local
monitor.
"""
img_name = os.path.basename(img_file)
logit("info", "Adjusting image '%s'" % img_name)
img = Image.open(img_file)
img.thumbnail(MAXSIZE, Image.ANTIALIAS)
logit("debug", "Image %s size: %s x %s" %
(img_name, img.width, img.height))
if img.height > img.width:
# Rotate 90 degrees for display
img = img.rotate(90, expand=True)
logit("info", "Image '%s' has been rotated" % img_name)
enhancer = ImageEnhance.Brightness(img)
logit("debug", "Adjusting brightness for image '%s'" % img_name)
img = enhancer.enhance(float(bright))
logit("debug", "Finished adjusting brightness for image '%s'" % img_name)
enhancer = ImageEnhance.Contrast(img)
logit("debug", "Adjusting contrast for image '%s'" % img_name)
img = enhancer.enhance(float(contrast))
logit("debug", "Finished adjusting contrast for image '%s'" % img_name)
enhancer = ImageEnhance.Color(img)
logit("debug", "Adjusting saturation for image '%s'" % img_name)
img = enhancer.enhance(float(saturation))
logit("debug", "Finished adjusting saturation for image '%s'" % img_name)
logit("debug", "Saving image '%s'" % img_name)
img.save(img_file)
gc.collect()
logit("debug", "Finished saving image '%s'" % img_name)
logit("info", "Finished corrections for image '%s'" % img_name)
def resume(self, signum=None, frame=None):
self.set_timer("photo", True)
logit("info", "Photo timer started")
def pause(self, signum=None, frame=None):
self.photo_timer.cancel()
logit("info", "Photo timer stopped")
# will take screenshot => output/screenshot.png
from detect_text import parse_screenshot, IMAGE_PATH
from google_search import run_query_all
from utils import logit
import time
START_FULL = time.time()
def main():
q_and_a = parse_screenshot(IMAGE_PATH)
# START_QUERY_ALL = time.time()
(question, results) = run_query_all(q_and_a['question'], q_and_a['answers'])
# END_QUERY_ALL = time.time()
# logit("QUERY ALL", START_QUERY_ALL, END_QUERY_ALL)
for (answer, total) in results:
print("answer: {} === TOTAL: {}".format(answer, total))
if __name__ == '__main__':
main()
END_FULL = time.time()
logit("FULL", START_FULL, time.time())
def _log(self, msg=''):
if self.DEBUG:
logit('[AuC] %s' % msg)
def calc_loss(predictions, mask, labels):
targets = torch.where(labels)[1]
extraction_probabilities = predictions[mask]
extraction_logits = logit(extraction_probabilities)
loss = cross_entropy_loss(input=extraction_logits, target=targets)
return loss
def conditional_gan(src, tgt, z, opt, opt_t=None, is_reuse_generator=None):
if not opt_t: opt_t = opt
logits_real, H_real, _ = discriminator(
src, tgt, opt, prefix=('d_' if not is_reuse_generator else 'd_rev_'))
syn_sent, syn_one_hot, H_dec, sup_loss, sample_loss, sup_loss_all = generator(
z,
src,
tgt,
opt,
opt_t,
is_reuse=is_reuse_generator,
prefix=('g_' if not is_reuse_generator else 'g_rev_'))
logits_fake, H_fake, _ = discriminator(
src,
syn_one_hot,
opt,
is_prob_src=False,
is_prob_tgt=True,
is_reuse=True,
prefix=('d_' if not is_reuse_generator else 'd_rev_'))
gan_cost_d = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels = tf.ones_like(logits_real), logits = logits_real)) + \
tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels = tf.zeros_like(logits_fake), logits = logits_fake))
real_prob = tf.reduce_mean(tf.nn.sigmoid(logits_real))
fake_prob = tf.reduce_mean(tf.nn.sigmoid(logits_fake))
fake_mean = tf.reduce_mean(H_fake, axis=0)
real_mean = tf.reduce_mean(H_real, axis=0)
mean_dist = tf.sqrt(tf.reduce_mean((fake_mean - real_mean)**2))
if opt.feature_matching == 'mean':
gan_cost_g = mean_dist
elif opt.feature_matching == 'pair_diff':
# improved WGAN lipschitz-penalty
if opt.w_gan:
gan_cost_g = tf.reduce_mean(
logit(logits_real / 2.0 + 0.5) -
logit(logits_fake / 2.0 + 0.5))
else:
gan_cost_g = tf.reduce_mean(
logit((logits_real - logits_fake) / 4.1 + 0.51)
) #tf.reduce_mean( logits_real - logits_fake ) #tf.reduce_mean(logit((logits_real - logits_fake)/4.0 + 0.5))
if opt.grad_penalty != None:
alpha = tf.random_uniform(shape=[opt.batch_size, 1, 1],
minval=0.,
maxval=1.)
real_one_hot = tf.one_hot(tgt, opt.n_words, axis=-1)
differences = syn_one_hot - real_one_hot
interpolates = real_one_hot + (alpha * differences)
logits_interp, _, x_int = discriminator(src,
interpolates,
opt,
is_prob_src=False,
is_prob_tgt=True,
is_reuse=True)
gradients = tf.gradients(logit(logits_interp / 2.0 + 0.5),
[x_int])[0]
slopes = tf.sqrt(
tf.reduce_sum(tf.square(gradients), reduction_indices=[1, 2]) +
1e-10)
gradient_penalty = tf.reduce_mean((slopes - 1.)**2)
gan_cost_d = -gan_cost_g + opt.grad_penalty * gradient_penalty
else:
gan_cost_d = -gan_cost_g
real_prob = tf.reduce_mean(logits_real)
fake_prob = tf.reduce_mean(logits_fake)
else:
gan_cost_g = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(logits_fake), logits=logits_fake))
if opt.lambda_sup_G:
if 0 <= opt.lambda_sup_G < 1:
g_cost = opt.lambda_sup_G * sup_loss + (
1 - opt.lambda_sup_G) * gan_cost_g
else:
g_cost = sup_loss
else:
g_cost = 0.0
# maximizing variational mutual information. (VIM)
if opt.lambda_MI:
rev_z = tf.random_uniform(shape=[opt.fake_size, opt.n_z],
minval=0.,
maxval=1.)
rev_tgt = tf.identity(src[:, (opt.filter_shape - 1):]) # B L
if opt.MI_pg:
rev_src = tf.one_hot(tf.cast(syn_sent, tf.int32),
opt.n_words,
on_value=1.0,
off_value=0.0,
axis=-1) # syn_tgt
tf.stop_gradient(rev_src)
else:
rev_src = syn_one_hot # syn_tgt
rev_syn_sent, _, _, rev_sup_loss, _, rev_sup_loss_all = generator(
rev_z,
rev_src,
rev_tgt,
opt,
opt_t,
is_prob_src=True,
is_softargmax=False,
is_reuse=is_reuse_generator,
prefix=('g_rev_' if not is_reuse_generator else 'g_'))
if opt.MI_pg:
g_frwd_vars = [
var for var in tf.trainable_variables()
if 'g_' in var.name and 'rev_' not in var.name
]
g_rev_vars = [
var for var in tf.trainable_variables() if 'g_rev_' in var.name
]
reward = -rev_sup_loss_all
base_line = tf.reduce_mean(reward)
reward_stop = tf.stop_gradient(reward)
base_line_stop = tf.stop_gradient(base_line)
mi_loss = tf.reduce_mean(
(reward_stop - base_line_stop) * sup_loss_all) + rev_sup_loss
else:
mi_loss = rev_sup_loss
tf.summary.scalar('MI', -mi_loss)
g_cost = (1 - opt.lambda_MI) * g_cost + mi_loss * opt.lambda_MI
else:
rev_sup_loss = tf.zeros(shape=1)
rev_sup_loss_all = tf.zeros(shape=1)
res_dict = {
'syn_sent': syn_sent,
'mean_dist': mean_dist,
'prob_r': real_prob,
'prob_f': fake_prob,
#'loss_mi_fwd': (reward_stop - base_line_stop) ,
'loss_mi_rev': sup_loss_all,
'sample_loss': sample_loss,
'sup_loss': sup_loss_all,
'rev_sup_loss': rev_sup_loss_all,
}
if opt.grad_penalty:
res_dict['gp'] = opt.grad_penalty * gradient_penalty
if opt.lambda_MI:
res_dict['mi'] = rev_sup_loss
res_dict['rev_sent'] = rev_syn_sent
return res_dict, gan_cost_d, g_cost
def _logit_transform(x):
"""
Transforms pixel values with logit to be unconstrained.
"""
return utils.logit(CIFAR10.alpha + (1 - 2 * CIFAR10.alpha) * x)
def kill_all(self):
"""Kills all timers on receiving a Ctrl-C."""
logit("error", "Killing timers")
self.photo_timer.cancel()
self.check_timer.cancel()
logit("error", "Timers canceled")
