def _update_config(self, data): changed = False new_interval = False for key in ("name", "description", "interval_time", "interval_units", "brightness", "contrast", "saturation"): val = data.get(key, None) logit("debug", "key:", key, "; val:", val) if val is None: continue local_val = getattr(self, key) if local_val != val: setattr(self, key, val) monitor_keys = ("brightness", "contrast", "saturation") section = "monitor" if key in monitor_keys else "frame" self.parser.set(section, key, str(val)) changed = True new_interval = new_interval or "interval" in key if changed: with open(CONFIG_FILE, "w") as ff: self.parser.write(ff) if new_interval: self.interval = _normalize_interval(self.interval_time, self.interval_units) logit("info", "Setting timer to", self.interval) self.set_image_interval()
def get_freespace(): out, err = runproc("df -BM .") ret = out.split('\n')[1].split()[3] # Remove the trailing 'M' ret = ret.replace("M", "") ret = int(ret) * ONE_MB logit("debug", "Free disk space =", ret) return ret
def get_conn(): try: access_key = env.account['access_key'] access_secret = env.account['access_secret'] conn = boto.ec2.connect_to_region(env.region, aws_access_key_id=access_key, aws_secret_access_key=access_secret) env.snsc = boto.sns.connect_to_region(env.region, aws_access_key_id=access_key, aws_secret_access_key=access_secret) return conn except Exception as e: error_msg = 'get_conn(): unable to connect to ec2 region, reason: {}'.format(e) logit('exception', error_msg)
def run(self): while True: raw_ircmsg = self.ircsock.recv(2048) try: ircmsg = raw_ircmsg.decode("UTF-8") except UnicodeDecodeError as e: logit("Received non-UTF8 encoded message: '%s'" % raw_ircmsg, force=True) ircmsg = ircmsg.strip(" \n\r") self.process_msg(ircmsg)
def _download(self, img): headers = {"user-agent": "photoviewer"} url = "%s/%s" % (self.dl_url, img) logit("debug", url) logit("info", "downloading", img) resp = requests.get(url, headers=headers, stream=True) if resp.status_code == 200: outfile = os.path.join(DOWNLOAD_PHOTODIR, img) with open(outfile, "wb") as ff: resp.raw.decode_content = True shutil.copyfileobj(resp.raw, ff) return outfile
def wait_for(self, txt): """Waits for the server to send a message containing the requested text. 'txt' can be a list of strings, or a single string. """ if not isinstance(txt, (list, tuple)): txt = [txt] while True: ircmsg = self.ircsock.recv(2048).decode("UTF-8") ircmsg = ircmsg.strip("\n\r") if any(word in ircmsg for word in txt): logit(ircmsg, "FOUND", *txt, force=self.verbose) return self.process_msg(ircmsg)
def main(): parser = argparse.ArgumentParser(description="IRC Bot") parser.add_argument("--channel-file", "-f", help="The path of the file " "containing the names of the channels to join, one per line.") parser.add_argument("--verbose", "-v", action="store_true", help="Enables verbose output.") args = parser.parse_args() if args.verbose: utils.LOG.level = utils.logging.DEBUG # Create a pool of channels for the bots to pick from channel_queue = Queue() if args.channel_file: with open(args.channel_file) as ff: chan_lines = ff.read().splitlines() [channel_queue.put(chan.strip()) for chan in chan_lines] else: print("You must specify a channel file") exit(1) # Add flags for the bots to know the queue is empty for i in range(4): channel_queue.put(None) # We need four bots to cover all the channels bots = [] procs = [] for num in range(4): bot = LogBot(num, args.verbose) logit("NICK:", bot.nick, force=True) bots.append(bot) proc = Process(target=connect_bot, args=(bot,)) procs.append(proc) proc.start() for p in procs: p.join() procs = [] for bot in bots: proc = Process(target=join_channels, args=(bot, channel_queue), daemon=False) proc.start() procs.append(proc) [p.join() for p in procs] logit("All channels joined for %s bot." % args.nick_base, force=True) procs = [] for bot in bots: proc = Process(target=runbot, args=(bot,)) proc.start() procs.append(proc) process_pids.append(proc.pid) [p.join() for p in procs]
def _read_config(self, signum=None, frame=None): logit("info", "_read_config called!") try: self.parser.read(CONFIG_FILE) except ConfigParser.MissingSectionHeaderError as e: # The file exists, but doesn't have the correct format. raise exc.InvalidConfigurationFile(e) def safe_get(section, option, default=None): try: return self.parser.get(section, option) except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return default self.log_level = safe_get("frame", "log_level", "INFO") set_log_level(self.log_level) self.reg_url = safe_get("host", "reg_url") if not self.reg_url: logit("error", "No registration URL in photo.cfg; exiting") exit() self.dl_url = safe_get("host", "dl_url") if not self.dl_url: logit("error", "No download URL configured in photo.cfg; exiting") exit() self.check_url = safe_get("host", "check_url", None) self.frameset = safe_get("frameset", "name", "") self.name = safe_get("frame", "name", "undefined") self.pkid = safe_get("frame", "pkid", "") self.description = safe_get("frame", "description", "") self.orientation = safe_get("frame", "orientation", "H") # When to start the image rotation self.interval_base = safe_get("frame", "interval_base", "*:*") # How often to change image self.interval_time = int(safe_get("frame", "interval_time", 10)) # Units of time for the image change interval self.interval_units = safe_get("frame", "interval_units", "minutes") self.interval = _normalize_interval(self.interval_time, self.interval_units) logit("info", "Setting image interval to", self.interval) self.set_image_interval() check_interval = int(safe_get("frame", "host_check", 120)) check_units = safe_get("frame", "host_check_units", "minutes") self.check_interval = _normalize_interval(check_interval, check_units) logit("info", "Setting host check interval to", self.check_interval) self.brightness = safe_get("monitor", "brightness") self.contrast = safe_get("monitor", "contrast") self.saturation = safe_get("monitor", "saturation")
def forward(self, x, log_prior=None): batch_size = x.size(0) truncation = self.beta_a.size(0) beta_a = F.softplus(self.beta_a) + 0.01 beta_b = F.softplus(self.beta_b) + 0.01 # might be passed in for IWAE if log_prior is None: log_prior = reparametrize( beta_a.view(1, truncation).expand(batch_size, truncation), beta_b.view(1, truncation).expand(batch_size, truncation), ibp=True, log=True) logit_x, mu, logvar = self.encode(x) logit_post = logit_x + logit(log_prior.exp()) logsample = reparametrize_discrete(logit_post, self.temp) z_discrete = F.sigmoid(logsample) # binary z_continuous = reparametrize_gaussian(mu, logvar) # zero-temperature rounding if not self.training: z_discrete = torch.round(z_discrete) inputForDecoder = z_discrete * z_continuous dec = self.decode(inputForDecoder) return dec, logsample, logit_post, log_prior, mu, logvar, z_discrete, z_continuous
def plot_boundary(X, y, coefs, title='', save=False): x_min, x_max = np.min(X[:, 0]), np.max(X[:, 0]) y_min, y_max = np.min(X[:, 1]), np.max(X[:, 1]) offset = 1 boundary_x = np.linspace(x_min - offset, x_max + offset, 1000) boundary_y = [ (utils.logit(1 / 2) - coefs['b'] - coefs['w1'] * x) / coefs['w2'] for x in boundary_x ] plt.figure(figsize=(9, 9)) plt.plot(boundary_x, boundary_y, c='g', label='Decision boundary') plt.scatter(X[y == 0, 0], X[y == 0, 1], label='class 0') plt.scatter(X[y == 1, 0], X[y == 1, 1], label='class 1') plt.xlim(x_min - offset, x_max + offset) plt.ylim(y_min - offset, y_max + offset) plt.xlabel('x1') plt.ylabel('x2') plt.title(title) plt.legend(loc="upper right", fontsize=16) if save: print("Save figure results folder.") plt.savefig('results/{}'.format(utils.sanitize(title))) plt.show()
def forward(x): """ Applies a logit function robust to x=0 :param x: float value satisfying: 0 <= x < 1 :return: the logit of input x """ return logit(x)
def get_theta0(y): prior = GammaIIDIncr.get_prior() theta0 = prior.rvs() theta0['log_c'] = np.log(1.0) + 0.1 * npr.randn() theta0['log_eta'] = np.log(1.0) + 0.1 * npr.randn() theta0['logit_sigma'] = sigmoid(logit(0.5) + 0.1 * npr.randn()) return theta0
def log_odds(x, mu0, mu1, sigma0, sigma1, pi): sigma0_inv = np.linalg.inv(sigma0) sigma1_inv = np.linalg.inv(sigma1) return utils.logit(pi) + 0.5 * np.log( np.linalg.det(sigma0) / np.linalg.det(sigma1)) - 0.5 * (x - mu1).T @ sigma1_inv @ ( x - mu1) + 0.5 * (x - mu0).T @ sigma0_inv @ (x - mu0)
def start(self): update_alive() # If a prior import crashed during image processing, re-process the # images. self._process_images() signal.signal(signal.SIGHUP, self._read_config) signal.signal(signal.SIGURG, self.check_host) signal.signal(signal.SIGTSTP, self.pause) signal.signal(signal.SIGCONT, self.resume) signal.signal(signal.SIGTRAP, self.navigate) if not self.check_timer.is_alive(): self.check_timer.start() if not self.photo_timer.is_alive(): self.photo_timer.start() logit("debug", "Timers started") self._started = True self.show_photo()
def forward(self, z): """ generate the mean parameter for the Bernoulli distribution """ z = z.view(-1, self.k) z = z.to(self.device) p = (torch.mm(logit(z), self.log_W.exp()) + self.b).sigmoid() return p
def set_timer(self, typ, start=False): # Clear the current timer first curr_tmr = self.photo_timer if typ == "photo" else self.check_timer if curr_tmr: curr_tmr.cancel() tmr = None if typ == "photo": interval = self.initial_interval or self.interval self.initial_interval = 0 tmr = self.photo_timer = Timer(interval, self.navigate) elif typ == "check": tmr = self.check_timer = Timer(self.check_interval, self.check_host) logit("debug", "Timer set for:", typ, tmr.interval, tmr.function) if tmr and start: tmr.start() logit("debug", "Timer started for", typ)
def get_theta0(y): prior = GBFRYIIDIncr.get_prior() theta0 = prior.rvs() halpha = hill_estimate(y, 50)[15:].mean() theta0['log_c'] = np.log(1.0) + 0.1 * npr.randn() theta0['log_eta'] = np.log(1.0) + 0.1 * npr.randn() theta0['logit_sigma'] = sigmoid(logit(0.5) + 0.1 * npr.randn()) theta0['log_tau_minus_one'] = np.log(0.5 * halpha) return theta0
def __init__(self, mu=0., beta=0., log_eta=np.log(0.1), logit_sigma=logit(0.2), volumes=1.): super(NSIIDIncr, self).__init__(mu=mu, beta=beta) self.log_eta = log_eta self.logit_sigma = logit_sigma self.volumes = volumes
def _process_images(self): images_to_process = glob.glob("%s/*.jpg" % DOWNLOAD_PHOTODIR) if not images_to_process: logit("debug", "No images to process") return num_imgs = len(images_to_process) logit("debug", "Processing %s images" % num_imgs) for pos, img in enumerate(images_to_process): logit("info", "Processing image %s (%s of %s)" % (img, pos+1, num_imgs)) image.adjust(img, self.brightness, self.contrast, self.saturation) shutil.move(img, PHOTODIR) logit("info", "Moved processed image %s from the download to the " "image directory." % img)
def process_msg(self, ircmsg): if not ircmsg: return # This can be useful for debugging is_ping = self.verbose or "PING :" in ircmsg logit(self.nick, ircmsg, force=is_ping) if "PING :" in ircmsg: self.ping() return mtch = MSG_PAT.match(ircmsg) if mtch: groups = mtch.groups() nick = groups[0] channel = groups[3] remark = groups[4] if remark.startswith("ACTION "): remark = remark.replace("ACTION ", "/me ") if len(nick) >= 17: logit("Odd nick: %s" % nick, force=True) return tm = dt.datetime.utcnow().replace(microsecond=0) record(nick, channel, remark, tm, force=self.verbose)
def record(nick, channel, remark, tm, force=False): tmstr = tm.strftime("%Y-%m-%dT%H:%M:%S") body = {"channel": channel, "nick": nick, "posted": tmstr, "remark": remark} # Leave the time out of the hash, as two bots could differ on when they saw # a posting, and generate different IDs, which would result in duplicate # postings. hash_body = {"channel": channel, "nick": nick, "remark": remark} hashval = utils.gen_key(hash_body) body["id"] = hashval attempts = 0 while True: try: logit("RECORDING", body, "ID =", hashval, force=True) es_client.index(index="irclog", doc_type="irc", id=hashval, body=body) break except Exception as e: attempts += 1 if attempts >= MAX_RETRIES: logit("Elasticsearch exception: %s" % e, force=True) break
def __init__(self, mu=0., beta=0., log_eta=np.log(0.1), log_tau_minus_one=np.log(3.0 - 1.), logit_sigma=logit(0.2), log_c=np.log(1.), volumes=1.): super(GBFRYIIDIncr, self).__init__(mu=mu, beta=beta) self.log_eta = log_eta self.log_tau_minus_one = log_tau_minus_one self.logit_sigma = logit_sigma self.log_c = log_c self.volumes = volumes
def curate_snapshots(): ''' curates snapshots as defined by their retention time in config.yaml ''' for instance in conf['instances']: env.region = instance['region'] env.account = conf['accounts'][instance['account']] conn = get_conn() reservations = conn.get_all_instances(filters={'tag:Name': instance['name']}) res_instances = [i for r in reservations for i in r.instances] for i in res_instances: volumes = conn.get_all_volumes(filters={'attachment.instance_id': i.id}) for volume in volumes: snapshots = conn.get_all_snapshots(filters={'volume-id': volume.id, 'status': 'completed'}) for snapshot in snapshots: retention_time = datemath('now-%s' % instance.get('retention', '7d')) snapshot_create_time = dateutil.parser.parse(snapshot.start_time) # remove snapshots if they are over their retention period if snapshot_create_time < retention_time: try: conn.delete_snapshot(snapshot.id) logit('info', 'EBS snapshot %s/%s deleted for %s/%s, because it was older than %s old' % ( snapshot.id, volume.id, instance['region'], instance['name'], retention_time)) except Exception as e: error_msg = 'Unable to delete EBS snapshot: reson: %s, id: %s, volume: %s, host: %s, region: %s' % ( e, snapshot.id, volume.id, instance['name'], instance['region'] ) logit('error', error_msg)
def _register(self): headers = {"user-agent": "photoviewer"} # Get free disk space freespace = get_freespace() data = {"pkid": self.pkid, "name": self.name, "description": self.description, "interval_time": self.interval_time, "interval_units": self.interval_units, "orientation": self.orientation, "freespace": freespace, "frameset": self.frameset} resp = requests.post(self.reg_url, data=data, headers=headers) if 200 <= resp.status_code <= 299: # Success! pkid, images = resp.json() if pkid != self.pkid: self.parser.set("frame", "pkid", pkid) with open(CONFIG_FILE, "w") as ff: self.parser.write(ff) self.host_images = images self._update_images() else: logit("error", resp.status_code, resp.text) exit()
def join_chan(self, chan): logit("-" * 88, force=True) logit("JOINING", chan, force=True) self.ircsock.send(bytes("JOIN %s\n" % chan, "UTF-8")) # See if this makes the IRC server happier about flooding it with # requests... self.pause(PAUSE_BETWEEN_JOINS) logit("!!!!!JOINED", chan, force=True) heartbeat(self)
def linear_coef(mu0, mu1, sigma0, sigma1, pi): sigma0_inv = np.linalg.inv(sigma0) sigma1_inv = np.linalg.inv(sigma1) alpha = sigma0_inv - sigma1_inv beta = mu1.T @ sigma1_inv - mu0.T @ sigma0_inv a = 0.5 * alpha[0, 0] b = 0.5 * alpha[0, 1] c = 0.5 * alpha[1, 1] d = beta[0] e = beta[1] f = utils.logit(pi) + 0.5 * np.log( np.linalg.det(sigma0) / np.linalg.det(sigma1) ) - 0.5 * mu1.T @ sigma1_inv @ mu1 + 0.5 * mu0.T @ sigma0_inv @ mu0 return a, b, c, d, e, f
def nll_and_kl(recon_x, x, log_likelihood, a, b, logsample, z_discrete, logit_post, log_prior, mu, logvar, dataset_size, args=Namespace(), test=False): batch_size = x.size()[0] NLL = -1 * log_likelihood(recon_x, x) KL_zreal = -0.5 * (1. + logvar - mu**2 - logvar.exp()) KL_beta = kl_divergence(a, b, prior_alpha=args.alpha0, log_beta_prior=np.log(1./args.alpha0), args=args).repeat(batch_size, 1) * (1. / dataset_size) # in test mode, our samples are essentially coming from a Bernoulli if not test: KL_discrete = kl_discrete(logit_post, logit(log_prior.exp()), logsample, args.temp, args.temp_prior) else: pi_prior = torch.exp(log_prior) pi_posterior = torch.sigmoid(logit_post) kl_1 = z_discrete * (pi_posterior + SMALL).log() + (1 - z_discrete) * (1 - pi_posterior + SMALL).log() kl_2 = z_discrete * (pi_prior + SMALL).log() + (1 - z_discrete) * (1 - pi_prior + SMALL).log() KL_discrete = kl_1 - kl_2 return NLL, KL_zreal, KL_beta, KL_discrete
def infer(args, unlabeled, ckpt_file): batch_size = args["batch_size"] unlabeled_train_data = [train_dset[i] for i in unlabeled] unlabeled_train_dset = tf.data.Dataset.from_generator( lambda: unlabeled_train_data, (tf.float32, tf.float32), (tf.TensorShape([None]), tf.TensorShape([])), ).padded_batch(batch_size) model = tf.keras.models.load_model( os.path.join(args["EXPT_DIR"], ckpt_file)) model.compile( loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), optimizer=tf.keras.optimizers.Adam(1e-4), metrics=["accuracy"], ) outputs_fin = {} i = 0 for data, labels in tqdm(unlabeled_train_dset, desc="Inferring"): outputs = model(data).numpy().flatten() pred = np.copy(outputs) pred[pred <= 0] = 0 pred[pred > 0] = 1 for j in range(len(pred)): outputs_fin[i] = {} outputs_fin[i]["prediction"] = pred[j] outputs_fin[i]["pre_softmax"] = [ utils.logit(1 - tf.math.sigmoid(outputs[j])).numpy(), outputs[j], ] i += 1 return {"outputs": outputs_fin}
def make_snapshots(): ''' makes EBS snapshots for instances as defined in config.yaml ''' for instance in conf['instances']: env.region = instance['region'] env.account = conf['accounts'][instance['account']] conn = get_conn() reservations = conn.get_all_instances(filters={'tag:Name': instance['name']}) res_instances = [i for r in reservations for i in r.instances] if len(res_instances) == 0: logit('error', 'Unable to find reservations for instance name: %s in region: %s on account: %s' % (instance['name'], instance['region'], instance['account'])) continue for i in res_instances: volumes = conn.get_all_volumes(filters={'attachment.instance_id': i.id}) for volume in volumes: snapshot = conn.create_snapshot(volume.id, '%s - %s - EBS snapshot for %s' % (instance['name'], today, volume.id)) if snapshot: logit('info', 'Snapshot Created: %s - %s on volume: %s, volume_size: %sGB, snapid: %s' % (today, instance['name'], volume.id, volume.size, snapshot.id)) else: error_msg = 'Unable to create EBS snapshot for %s on volume: %s on %s' % (instance['name'], volume.id, today) logit('error', error_msg)
self.parser.set(section, key, str(val)) changed = True new_interval = new_interval or "interval" in key if changed: with open(CONFIG_FILE, "w") as ff: self.parser.write(ff) if new_interval: self.interval = _normalize_interval(self.interval_time, self.interval_units) logit("info", "Setting timer to", self.interval) self.set_image_interval() def kill_all(self): """Kills all timers on receiving a Ctrl-C.""" logit("error", "Killing timers") self.photo_timer.cancel() self.check_timer.cancel() logit("error", "Timers canceled") if __name__ == "__main__": with open("photo.pid", "w") as ff: ff.write("%s" % os.getpid()) img_mgr = ImageManager() try: img_mgr.start() logit("debug", "And we're off!") except KeyboardInterrupt: img_mgr.kill_all()
def _set_power_on(self): # Power on the monitor and HDMI output logit("debug", "Powering on the monitor") out, err = runproc(MONITOR_CMD) logit("debug", "Power result", out, err)
parser.add_argument('--T', type=int, default=500) parser.add_argument('--mu', type=float, default=0.0) parser.add_argument('--beta', type=float, default=0.0) parser.add_argument('--eta', type=float, default=.6) parser.add_argument('--sigma', type=float, default=.7) parser.add_argument('--tau', type=float, default=1.8) parser.add_argument('--c', type=float, default=1.0) parser.add_argument('--filename', type=str, default=None) args = parser.parse_args() data = {} true_params = { 'mu': args.mu, 'beta': args.beta, 'logit_sigma': logit(args.sigma), 'log_eta': np.log(args.eta), 'log_tau_minus_one': np.log(args.tau - 1), 'log_c': np.log(args.c) } model = GBFRYIIDIncr(**true_params) x, y = model.simulate(args.T) data['true_params'] = true_params data['x'] = x data['y'] = np.array(y).squeeze() if not os.path.isdir('../data'): os.makedirs('../data') if args.filename is None:
def ping(self): # respond to server Pings. self.ircsock.send(bytes("PONG :pingis\n", "UTF-8")) logit("PONG!", force=self.verbose) heartbeat(self)
def plot_indiv_student_activity(self, sidx, filename, detected_cp_loc, alpha_i, bic_with_cp, bic_wo_cp, xlim=None, ylim=None, ylim2=None, legend_loc='upper left', legend_loc2=None, subtract_mean=False): """ Plotting individual student's activity. Used inside of the function 'naive_changepoint_detection()' """ X_NT = self.rawdata X1t = X_NT[sidx, :] alpha_i1, alpha_i2, alpha_i0 = alpha_i avg_clicks = np.sum(X_NT, axis=0) / float(X_NT.shape[0]) if self.binary: mu_t_hat = logit(avg_clicks + 0.0000001) else: mu_t_hat = np.log(avg_clicks + 0.0000001) ai1_arr = np.ones(detected_cp_loc) * (alpha_i1 + mu_t_hat[:detected_cp_loc]) ai2_arr = np.ones(self.n_days - detected_cp_loc) * ( alpha_i2 + mu_t_hat[detected_cp_loc:]) ai0_arr = alpha_i0 + mu_t_hat logodds_wo_cp = ai0_arr logodds_w_cp = np.concatenate((ai1_arr, ai2_arr)) fig, axs = plt.subplots(2, 1, figsize=(9, 8), gridspec_kw={'height_ratios': [2, 1]}) ax_lam = axs[0] ax_lam.plot(logodds_wo_cp, color='gray', linewidth=3, alpha=0.6, label="M1, BIC=" + str(round(bic_wo_cp, 2))) ax_lam.plot(logodds_w_cp, color='#0C4EC9', linewidth=3, label="M2, BIC=" + str(round(bic_with_cp, 2))) ax_lam.axvline(x=detected_cp_loc, ymin=0, ymax=1, color='red', linewidth=3, alpha=0.8) ax_lam.grid(alpha=0.3) ax_lam.legend(loc=legend_loc, fontsize=14) if ylim is not None: ax_lam.set_ylim(ylim[0], ylim[1]) if xlim is None: ax_lam.set_xlim(-1, self.n_days) else: ax_lam.set_xlim(xlim[0], xlim[1]) ax_lam.set_ylabel(r"$\hat{\mu}_t + \hat{\alpha_i}$", fontsize=18) ax_lam.tick_params(labelsize=12) ax_raw = axs[1] if self.binary: X1t_ones = np.where(X1t == 1)[0] for xx in X1t_ones[:-1]: ax_raw.axvline(x=xx, ymin=0, ymax=1, color="black", linewidth=3, alpha=0.7) ax_raw.axvline(x=X1t_ones[-1], ymin=0, ymax=1, color="black", linewidth=3, alpha=0.7) ax_raw.set_yticks([]) else: if subtract_mean: X_diff = X1t - avg_clicks ax_raw.bar(range(len(X_diff)), X_diff, width=0.6, color="black", alpha=0.7, edgecolor='white', align='center') ax_raw.set_ylabel("$x_{it} - \hat{\lambda}_t$", fontsize=19) else: ax_raw.bar(range(len(X1t)), X1t, width=0.6, color="black", alpha=0.7, edgecolor='white', align='center') ax_raw.set_ylabel(r"$x_{it}$", fontsize=19) ax_raw.grid(alpha=0.3) if ylim2 is not None: ax_raw.set_ylim(ylim2[0], ylim2[1]) ax_raw.axvline(x=detected_cp_loc - 0.5, ymin=0, ymax=1, color='red', linewidth=3, alpha=0.8, label="DETECTED CP") if legend_loc2 is None: legend_loc2 = legend_loc ax_raw.legend(loc=legend_loc2, fontsize=14) if xlim is None: ax_raw.set_xlim(-1, self.n_days) else: ax_raw.set_xlim(xlim[0], xlim[1]) ax_raw.tick_params(labelsize=12) ax_raw.set_xlabel('DAYS', fontsize=15) plt.savefig(filename) plt.close()
# Number of observations T = 5000 T_test = 50000 # Parameters of the IID GBFRY model params = {"eta": 1., "c": 1., "tau": 3., "sigma": .6} # Directory where to save the generated data save_dir = "../data/simulated/" # Simulate y from model ssm_cls = GBFRYIIDIncr theta = { 'log_eta': np.log(params['eta']), 'log_c': np.log(params['c']), 'log_tau_minus_one': np.log(params['tau'] - 1.), 'logit_sigma': logit(params['sigma']) } model = GBFRYIIDIncr(**theta) # Generate train x, y = model.simulate(T) data = pd.DataFrame() data['x'] = x data['y'] = np.array(y).squeeze() data['Volume'] = 500 * np.ones_like(y) # Generate test x_test, y_test = model.simulate(T_test)
def _update_images(self): """Compares the associated images received from the server, and updates the local copies if needed. """ images = self.host_images curr = set([os.path.basename(img) for img in self.current_images]) upd = set(images) if upd == curr: logit("debug", "No changes in _update_images") # No changes return False logit("debug", "updating images...") to_remove = curr - upd logit("debug", "To remove:", *to_remove) freespace = get_freespace() logit("debug", "Freespace", freespace) for img in to_remove: curr_loc = os.path.join(PHOTODIR, img) # Remove the frame buffer copy, if any. clean_fb(curr_loc) if freespace < ONE_GB: # Just delete it logit("info", "deleting", curr_loc) os.unlink(curr_loc) else: new_loc = os.path.join(INACTIVE_PHOTODIR, img) logit("info", "inactivating", img) shutil.move(curr_loc, new_loc) to_get = upd - curr logit("debug", "To get:", *to_get) num_to_get = len(to_get) for pos, img in enumerate(to_get): logit("info", "adding", img) logit("debug", "PROCESSING IMAGE #%s of %s" % (pos, num_to_get)) # Check if it's local inactive_loc = os.path.join(INACTIVE_PHOTODIR, img) if os.path.exists(inactive_loc): logit("info", "retrieving from inactive") active_loc = os.path.join(PHOTODIR, img) shutil.move(inactive_loc, active_loc) continue # Check if it's downloaded but unprocessed download_loc = os.path.join(DOWNLOAD_PHOTODIR, img) if os.path.exists(download_loc): logit("info", "Image already downloaded") continue # Not on disk, so download it self._download(img) self._process_images() logit("info", "Image update is done!") return True
def navigate(self, signum=None, frame=None): """Moves to the next image. """ logit("debug", "navigate called; current index", self.image_index) num_images = len(self.current_images) if not num_images: # Currently no images specified for this display, so just return. return new_index = self.image_index + 1 # Boundaries max_index = len(self.current_images) - 1 if new_index > max_index: new_index = 0 # Shuffle the images logit("info", "All images shown; shuffling order.") random.shuffle(self.current_images) else: new_index = max(0, min(max_index, new_index)) logit("debug", "new index", new_index) if new_index != self.image_index: self.image_index = new_index self.show_photo() elif new_index == 0: # There is only one image to display; make sure that it is the same # as the one currently displayed. # First, handle case where current image has been deleted locally img_exists = os.path.exists(self.displayed_name) logit("debug", "image exists:", img_exists) if not img_exists and self.displayed_name in self.current_images: self.current_images.remove(self.displayed_name) logit("debug", "refreshing missing images") self._update_images() displayed_file = "%s/%s" % (PHOTODIR, curr_display_list[0]) logit("debug", "comparing:", self.displayed_name, displayed_file) same_files = filecmp.cmp(self.displayed_name, displayed_file) logit("debug", "Files are", "same" if same_files else "different") if not same_files: self.show_photo() self.set_timer("photo", True)
def show_photo(self): if not self._started: return if not self.current_images: return try: fname = self.current_images[self.image_index] except IndexError as e: logit("error", "BAD INDEX", e) if self.current_images: fname = self.current_images[-1] else: # Something's screwy logit("error", "No images!") self.load_images() return if fname == self.displayed_name: return if os.path.exists(self.displayed_name): fb_loc = fb_path(self.displayed_name) if not os.path.exists(fb_loc): cmd = "cp -f /dev/fb0 '%s'" % fb_loc runproc(cmd, wait=True) logit("debug", "Created frame buffer copy:", fb_loc) runproc("sudo killall fbi") self.displayed_name = fname logit("debug", "displayed image path:", self.displayed_name) logit("info", "Changing photo to", os.path.basename(fname)) # See if there is already a frame buffer version of the image new_fb_loc = fb_path(fname) if os.path.exists(new_fb_loc): # Just copy it to the frame buffer device cmd = "cp -f '%s' /dev/fb0" % new_fb_loc runproc(cmd, wait=False) logit("debug", "Retrieved frame buffer copy:", self.displayed_name) return cmd = VIEWER_CMD % fname logit("debug", "Command:", cmd) runproc(cmd, wait=False)
def heartbeat(bot): cmd = "touch %s" % bot.heartbeat_file os.system(cmd) logit("HEARTBEAT!!", cmd, force=True)
def check_host(self, signum=None, frame=None): """Contact the host to update local status.""" logit("info", "check_host called") # Update the flag file. update_alive() if self.check_url is None: # Not configured logit("warning", "No host check URL defined") return # If we are already in check_host, skip if self.in_check_host: logit("debug", "check_host called; already in the process") return self.in_check_host = True headers = {"user-agent": "photoviewer"} url = self.check_url.replace("PKID", self.pkid) try: resp = requests.get(url, headers=headers) if resp.status_code >= 300: logit("error", resp.status_code, resp.content) self.in_check_host = False self.set_timer("check", True) return except Exception as e: logit("error", "check_host", resp.status_code, e) self.in_check_host = False self.set_timer("check", True) return data = resp.json() logit("debug", "Data:", data) # See if anything has updated on the server side self._update_config(data) # Check for image changes self.host_images = data["images"] logit("debug", "Updating images") # This will return True if the images have changed if self._update_images(): logit("debug", "_update_images indicated changed images") self.load_images() # Setting this to the end of the image list will cause navigate() # to shuffle the images and start at index 0. self.image_index = len(self.current_images) self.navigate() logit("debug", "check_host done.") self.set_timer("check", True) self.in_check_host = False
def adjust(img_file, bright, contrast, saturation): """Uses the local profile to adjust the image to look good on the local monitor. """ img_name = os.path.basename(img_file) logit("info", "Adjusting image '%s'" % img_name) img = Image.open(img_file) img.thumbnail(MAXSIZE, Image.ANTIALIAS) logit("debug", "Image %s size: %s x %s" % (img_name, img.width, img.height)) if img.height > img.width: # Rotate 90 degrees for display img = img.rotate(90, expand=True) logit("info", "Image '%s' has been rotated" % img_name) enhancer = ImageEnhance.Brightness(img) logit("debug", "Adjusting brightness for image '%s'" % img_name) img = enhancer.enhance(float(bright)) logit("debug", "Finished adjusting brightness for image '%s'" % img_name) enhancer = ImageEnhance.Contrast(img) logit("debug", "Adjusting contrast for image '%s'" % img_name) img = enhancer.enhance(float(contrast)) logit("debug", "Finished adjusting contrast for image '%s'" % img_name) enhancer = ImageEnhance.Color(img) logit("debug", "Adjusting saturation for image '%s'" % img_name) img = enhancer.enhance(float(saturation)) logit("debug", "Finished adjusting saturation for image '%s'" % img_name) logit("debug", "Saving image '%s'" % img_name) img.save(img_file) gc.collect() logit("debug", "Finished saving image '%s'" % img_name) logit("info", "Finished corrections for image '%s'" % img_name)
def resume(self, signum=None, frame=None): self.set_timer("photo", True) logit("info", "Photo timer started")
def pause(self, signum=None, frame=None): self.photo_timer.cancel() logit("info", "Photo timer stopped")
# will take screenshot => output/screenshot.png from detect_text import parse_screenshot, IMAGE_PATH from google_search import run_query_all from utils import logit import time START_FULL = time.time() def main(): q_and_a = parse_screenshot(IMAGE_PATH) # START_QUERY_ALL = time.time() (question, results) = run_query_all(q_and_a['question'], q_and_a['answers']) # END_QUERY_ALL = time.time() # logit("QUERY ALL", START_QUERY_ALL, END_QUERY_ALL) for (answer, total) in results: print("answer: {} === TOTAL: {}".format(answer, total)) if __name__ == '__main__': main() END_FULL = time.time() logit("FULL", START_FULL, time.time())
def _log(self, msg=''): if self.DEBUG: logit('[AuC] %s' % msg)
def calc_loss(predictions, mask, labels): targets = torch.where(labels)[1] extraction_probabilities = predictions[mask] extraction_logits = logit(extraction_probabilities) loss = cross_entropy_loss(input=extraction_logits, target=targets) return loss
def conditional_gan(src, tgt, z, opt, opt_t=None, is_reuse_generator=None): if not opt_t: opt_t = opt logits_real, H_real, _ = discriminator( src, tgt, opt, prefix=('d_' if not is_reuse_generator else 'd_rev_')) syn_sent, syn_one_hot, H_dec, sup_loss, sample_loss, sup_loss_all = generator( z, src, tgt, opt, opt_t, is_reuse=is_reuse_generator, prefix=('g_' if not is_reuse_generator else 'g_rev_')) logits_fake, H_fake, _ = discriminator( src, syn_one_hot, opt, is_prob_src=False, is_prob_tgt=True, is_reuse=True, prefix=('d_' if not is_reuse_generator else 'd_rev_')) gan_cost_d = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels = tf.ones_like(logits_real), logits = logits_real)) + \ tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels = tf.zeros_like(logits_fake), logits = logits_fake)) real_prob = tf.reduce_mean(tf.nn.sigmoid(logits_real)) fake_prob = tf.reduce_mean(tf.nn.sigmoid(logits_fake)) fake_mean = tf.reduce_mean(H_fake, axis=0) real_mean = tf.reduce_mean(H_real, axis=0) mean_dist = tf.sqrt(tf.reduce_mean((fake_mean - real_mean)**2)) if opt.feature_matching == 'mean': gan_cost_g = mean_dist elif opt.feature_matching == 'pair_diff': # improved WGAN lipschitz-penalty if opt.w_gan: gan_cost_g = tf.reduce_mean( logit(logits_real / 2.0 + 0.5) - logit(logits_fake / 2.0 + 0.5)) else: gan_cost_g = tf.reduce_mean( logit((logits_real - logits_fake) / 4.1 + 0.51) ) #tf.reduce_mean( logits_real - logits_fake ) #tf.reduce_mean(logit((logits_real - logits_fake)/4.0 + 0.5)) if opt.grad_penalty != None: alpha = tf.random_uniform(shape=[opt.batch_size, 1, 1], minval=0., maxval=1.) real_one_hot = tf.one_hot(tgt, opt.n_words, axis=-1) differences = syn_one_hot - real_one_hot interpolates = real_one_hot + (alpha * differences) logits_interp, _, x_int = discriminator(src, interpolates, opt, is_prob_src=False, is_prob_tgt=True, is_reuse=True) gradients = tf.gradients(logit(logits_interp / 2.0 + 0.5), [x_int])[0] slopes = tf.sqrt( tf.reduce_sum(tf.square(gradients), reduction_indices=[1, 2]) + 1e-10) gradient_penalty = tf.reduce_mean((slopes - 1.)**2) gan_cost_d = -gan_cost_g + opt.grad_penalty * gradient_penalty else: gan_cost_d = -gan_cost_g real_prob = tf.reduce_mean(logits_real) fake_prob = tf.reduce_mean(logits_fake) else: gan_cost_g = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( labels=tf.ones_like(logits_fake), logits=logits_fake)) if opt.lambda_sup_G: if 0 <= opt.lambda_sup_G < 1: g_cost = opt.lambda_sup_G * sup_loss + ( 1 - opt.lambda_sup_G) * gan_cost_g else: g_cost = sup_loss else: g_cost = 0.0 # maximizing variational mutual information. (VIM) if opt.lambda_MI: rev_z = tf.random_uniform(shape=[opt.fake_size, opt.n_z], minval=0., maxval=1.) rev_tgt = tf.identity(src[:, (opt.filter_shape - 1):]) # B L if opt.MI_pg: rev_src = tf.one_hot(tf.cast(syn_sent, tf.int32), opt.n_words, on_value=1.0, off_value=0.0, axis=-1) # syn_tgt tf.stop_gradient(rev_src) else: rev_src = syn_one_hot # syn_tgt rev_syn_sent, _, _, rev_sup_loss, _, rev_sup_loss_all = generator( rev_z, rev_src, rev_tgt, opt, opt_t, is_prob_src=True, is_softargmax=False, is_reuse=is_reuse_generator, prefix=('g_rev_' if not is_reuse_generator else 'g_')) if opt.MI_pg: g_frwd_vars = [ var for var in tf.trainable_variables() if 'g_' in var.name and 'rev_' not in var.name ] g_rev_vars = [ var for var in tf.trainable_variables() if 'g_rev_' in var.name ] reward = -rev_sup_loss_all base_line = tf.reduce_mean(reward) reward_stop = tf.stop_gradient(reward) base_line_stop = tf.stop_gradient(base_line) mi_loss = tf.reduce_mean( (reward_stop - base_line_stop) * sup_loss_all) + rev_sup_loss else: mi_loss = rev_sup_loss tf.summary.scalar('MI', -mi_loss) g_cost = (1 - opt.lambda_MI) * g_cost + mi_loss * opt.lambda_MI else: rev_sup_loss = tf.zeros(shape=1) rev_sup_loss_all = tf.zeros(shape=1) res_dict = { 'syn_sent': syn_sent, 'mean_dist': mean_dist, 'prob_r': real_prob, 'prob_f': fake_prob, #'loss_mi_fwd': (reward_stop - base_line_stop) , 'loss_mi_rev': sup_loss_all, 'sample_loss': sample_loss, 'sup_loss': sup_loss_all, 'rev_sup_loss': rev_sup_loss_all, } if opt.grad_penalty: res_dict['gp'] = opt.grad_penalty * gradient_penalty if opt.lambda_MI: res_dict['mi'] = rev_sup_loss res_dict['rev_sent'] = rev_syn_sent return res_dict, gan_cost_d, g_cost
def _logit_transform(x): """ Transforms pixel values with logit to be unconstrained. """ return utils.logit(CIFAR10.alpha + (1 - 2 * CIFAR10.alpha) * x)
def kill_all(self): """Kills all timers on receiving a Ctrl-C.""" logit("error", "Killing timers") self.photo_timer.cancel() self.check_timer.cancel() logit("error", "Timers canceled")