def handle_stream(status_api=False): stream = start_stream() global HANG_TIME try: while True: time.sleep(5) elapsed = (time.time() - HANG_TIME) if elapsed > 600: # TODO: Temp to try and stop crash tweet spam for now if os.path.exists(update['last_crash_file']): if time.time() - os.path.getctime( update['last_crash_file']) > 80000: os.remove(update['last_crash_file']) open(update['last_crash_file'], 'w') msg = """[{0}] Restarting! The bot will catch up on missed messages now!""".format( time.strftime("%Y-%m-%d %H:%M")) if status_api: status_api.update_status(status=msg) else: print(msg) stream.disconnect() threading.Thread(target=read_notifications, args=(API, True, TWEETS_READ)).start() time.sleep(5) stream = start_stream() HANG_TIME = time.time() except (KeyboardInterrupt, SystemExit): sys.exit(0)
def register_user( user, password, displayname, server_location, access_token, admin=False, user_type=None, ): url = "%s/_synapse/admin/v2/users/@%s:%s" % (server_location, user, config_yaml['domain']) headers = {'Authorization': ' '.join(['Bearer', access_token])} data = { "password": password, "displayname": "".join([user, config_yaml["name-suffix"]]), "admin": admin, } r = requests.put(url, json=data, headers=headers, verify=False) if r.status_code != 200 and r.status_code != 201: print("ERROR! Received %d %s" % (r.status_code, r.reason)) if 400 <= r.status_code < 500: try: print(r.json()["error"]) except Exception: pass return False return r
def filter_invalid_abs_states(state_list, pi_seq_list, ci_seq_list, A, init_cons): valid_idx_list = [] for idx, abs_state in enumerate(state_list): ival_cons = A.plant_abs.get_ival_cons_abs_state(abs_state.plant_state) # ##!!##logger.debug('ival_cons: {}'.format(ival_cons)) # find the intersection b/w the cell and the initial cons # print('init_cons', init_cons) ic = ival_cons & init_cons if ic is not None: valid_idx_list.append(idx) #valid_state_list.append(abs_state) # TODO: this should be logged and not printed if valid_idx_list == []: for abs_state in state_list: ival_cons = A.plant_abs.get_ival_cons_abs_state(abs_state.plant_state) print(ival_cons) valid_state_list = [] respective_pi_seq_list = [] respective_ci_seq_list = [] for i in valid_idx_list: valid_state_list.append(state_list[i]) respective_pi_seq_list.append(pi_seq_list[i]) respective_ci_seq_list.append(ci_seq_list[i]) return valid_state_list, respective_pi_seq_list, respective_ci_seq_list
def autojoin_users( invitees, roomId, config, ): for user in invitees: #POST /_matrix/client/r0/rooms/{roomId}/join url = "%s/_matrix/client/r0/rooms/%s/join?user_id=%s" % ( config["homeserver"], roomId, user, ) #_print("Sending registration request...") r = requests.post( url, headers={'Authorization': 'Bearer ' + config["as_token"]}, verify=False) if r.status_code != 200: print("ERROR! Received %d %s" % (r.status_code, r.reason)) if 400 <= r.status_code < 500: try: print(r.json()["error"]) except Exception: pass
def run_secam(sys, prop, opts): MODE = opts.MODE #plot = opts.plot if MODE == 'simulate': if not isinstance( opts.property_checker, properties.PropertyChecker): raise err.Fatal('property checker must be enabled when ' 'random testing!') start_time = time.time() trace_list = simulate(sys, prop, opts) #if plot: if opts.dump_trace: dump_trace(trace_list) opts.plotting.plot_trace_list(trace_list, opts.plots) opts.plotting.show() elif MODE == 'falsify': # ignore time taken to create_abstraction: mainly to ignore parsing # time current_abs, sampler = create_abstraction(sys, prop, opts) start_time = time.time() falsify(sys, prop, opts, current_abs, sampler) else: raise err.Fatal('bad MODE supplied: {}'.format(MODE)) stop_time = time.time() print('*'*20) print('time spent(s) = {}'.format(stop_time - start_time), file=SYS.stderr) return
def feasible(num_dims, prop, pwa_trace, solver=gopts.opt_engine): cons, Vars, vars_grouped_by_state = pwatrace2cons(pwa_trace, num_dims, prop) cons = list(cons) # for c in cons: # print(c) #nvars = num_dims.x + num_dims.pi #A_ub, b_ub = truncate(A_ub, b_ub) #err.warn_severe('faking output of optimizer') #res = True #varval_map = {v: 0 for v in Vars} # TODO: Make choice of opt engine #res, varval_map = z3opt.nlinprog(obj, cons, Vars) #res, varval_map = nlpfun(solver)(obj, cons, Vars) #ret_val = optsoln2x([varval_map[v] for v in Vars], len(pwa_trace)) if res else None obj = 0 res = nlpfun(solver)(obj, cons, Vars) ret_val = optsoln2x(res.x, len(pwa_trace)) if res.success else None print(res.success) #print(cons) print(ret_val) if res.success: U.pause() #embed() return ret_val
def get_rels(self, prop, sim, N): """TODO: EXPLICITLY ignores t, d, pvt, ci, pi """ d, pvt = [np.array([])]*2 pi = [np.array([])] t0 = 0 Yl = [] #plt.figure() x_array_ = self.xcell.sample_UR(N) # remove any sample drawn from the property box x_array = x_array_[~prop.final_cons.sat(x_array_), :] if x_array.size == 0: print(prop.final_cons) print(self.xcell.ival_constraints) #print(t0) #print(x_array) for x in x_array: (t_, x_, d_, pvt_) = sim(t0, x, d, pvt, pi) Yl.append(x_) if settings.debug_plot: # close intermediate plots which can not be switched off; # as these are in the system model *.py #plt.title('ignore') #plt.show() #plt.close() pass # return empty arrays if x_array.size == 0: return x_array, x_array else: return x_array, np.vstack(Yl)
def migrate_messages(fileList, matrix_room, config, tick): global later archive = zipfile.ZipFile(config["zipfile"], 'r') txnId = 1 progress = 0 for file in fileList: try: fileData = archive.open(file) messageData = json.load(fileData) except: print("Warning: Couldn't load data from file " + file + " in archive. Skipping this file.") for message in messageData: txnId = parse_and_send_message(config, message, matrix_room, txnId, False) progress = progress + tick update_progress(progress) # process postponed messages for message in later: txnId = parse_and_send_message(config, message, matrix_room, txnId, True) # clean up postponed messages later = []
def create(self): print(': SETUP\n') data = self.receive_data() with self.open('w') as conf_file: toml.dump(data, conf_file)
def test_model_(abs_state, AA, sp, am, step_sim): test_samples = sp.sampler.sample(abs_state, AA, sp, AA.num_samples*MORE_FACTOR*TEST_FACTOR) X, Y = getxy(abs_state, test_samples, step_sim) e = am.model_error(X, Y) if __debug__: print(e) return e
def on_status(self, status): global HAD_ERROR global HANG_TIME global TWEETS_READ HANG_TIME = time.time() tweet, command = acceptable_tweet(status) if not command: return True try: open(update['is_busy_file'], 'w') except PermissionError: # This wont happen all the time, the file is probably busy pass print("[{0}] Reading: {1} ({2}): {3}".format( time.strftime("%Y-%m-%d %H:%M"), status.user.screen_name, status.user.id, status.text)) tweet_command(API, status, tweet, command) HAD_ERROR = False TWEETS_READ.append(str(status.id)) with open(os.path.join(settings['ignore_loc'], "tweets_read.txt"), 'w') as file: file.write("\n".join(TWEETS_READ)) try: os.remove(update['is_busy_file']) except (PermissionError, FileNotFoundError): # Related to above PermissionError. pass
def handle_stream(sapi, status_api=False): stream_sapi = start_stream(sapi) global HANG_TIME while True: time.sleep(5) elapsed = (time.time() - HANG_TIME) if elapsed > 600: # TODO: Temp to try and stop crash tweet spam for now if os.path.exists(update['last_crash_file']): if time.time() - os.path.getctime(update['last_crash_file']) > 80000: os.remove(update['last_crash_file']) open(update['last_crash_file'], 'w') msg = """[{0}] Restarting! The bot will catch up on missed messages now!""".format( time.strftime("%Y-%m-%d %H:%M")) if status_api: post_tweet(status_api, msg) else: print(msg) stream_sapi.disconnect() time.sleep(3) if not stream_sapi.running: stream_sapi = start_stream(sapi) Thread(target=read_notifications, args=(API, True, TWEETS_READ)).start() HANG_TIME = time.time()
def check_prop_violation(prop, trace): """check_prop_violation Parameters ---------- trace : prop : Returns ------- Notes ------ """ # check using random sims idx = prop.final_cons.sat(trace.x_array + 0.0001) sat_x, sat_t = trace.x_array[idx], trace.t_array[idx] if sat_x.size != 0: print('x0={} -> x={}, t={}'.format( trace.x_array[0, :], sat_x[0, :], # the first violating state sat_t[0], # corresponding time instant )) return True else: return False
def build_pwa_model(AA, abs_obj, sp, tol, include_err, model_type): """build_pwa_model Builds both dft and rel models Parameters ---------- AA : AA abs_obj : Either abs_states (for dft models) or relations [tuple(abs_state_src, abs_state_target)] for rel model sp : system params tol : modeling error tolerance include_err : include error in determining next state x' = x +- errror """ dt = AA.plant_abs.delta_t step_sim = simsys.get_step_simulator(sp.controller_sim, sp.plant_sim, dt) #abs_state_models = {} modelers = { 'dft': (pwa.PWA, abs_state_affine_models), 'rel': (rel.PWARelational, abs_rel_affine_models), } M, model = modelers[model_type] pwa_model = M() for a in abs_obj: print('modeling: {}'.format(a)) for sub_model in model(a, AA, step_sim, tol, sp, include_err): if sub_model is not None: pwa_model.add(sub_model) #abs_state_models[abs_state] = sub_model return pwa_model
def on_success(self, data): global HAD_ERROR global HANG_TIME global TWEETS_READ tweet_datetime = datetime.datetime.strptime( data['created_at'], '%a %b %d %H:%M:%S %z %Y') days_past = datetime.datetime.now(datetime.timezone.utc) - tweet_datetime if days_past.days > 1: return False # Fri Apr 15 16:59:09 +0000 2016 if data['id_str'] in TWEETS_READ: return True TWEETS_READ.append(data['id_str']) HANG_TIME = time.time() tweet, command = acceptable_tweet(data) if not command: return True try: open(update['is_busy_file'], 'w') except PermissionError: # This wont happen all the time, the file is probably busy pass print("[{0}] {1} ({2}): {3}".format( time.strftime("%Y-%m-%d %H:%M"), data['user']['screen_name'], data['user']['id_str'], data['text'])) tweet_command(API, data, tweet, command) HAD_ERROR = False with open(os.path.join(settings['ignore_loc'], "tweets_read.txt"), 'w') as file: file.write("\n".join(TWEETS_READ)) try: os.remove(update['is_busy_file']) except (PermissionError, FileNotFoundError): # Related to above PermissionError. pass
def get_data(self): def parseHM_(s): try: h, m, _ = s.split(':') except: return return parse_time_period(h, m) while True: fine_state = True data = self.load() for field_name, field in self.__dataclass_fields__.items(): if not isinstance(data.get(field_name), field.type): print('! Config file is corrupped') self.update() fine_state = False break if not fine_state: continue data['rest_per_day'] = parseHM_(data['rest_per_day']) if not all(not_none(var) for var in data.values()): print("Corrupted data") self.update() continue break self.rest_per_day = data['rest_per_day']
def obs_operator_ensemble(istage): # assumes that prior ensemble is already linked to advance_temp<i>/wrfout_d01 print('running obs operator on ensemble forecast') os.chdir(cluster.dartrundir) if sat_channel: list_ensemble_truths = [] for iens in range(1, exp.n_ens+1): print('observation operator for ens #'+str(iens)) # ens members are already linked to advance_temp<i>/wrfout_d01 copy(cluster.dartrundir+'/advance_temp'+str(iens)+'/wrfout_d01', cluster.dartrundir+'/wrfout_d01') # DART may need a wrfinput file as well, which serves as a template for dimension sizes symlink(cluster.dartrundir+'/wrfout_d01', cluster.dartrundir+'/wrfinput_d01') # add geodata, if istage>0, wrfout is DART output (has coords) if istage == 0: wrfout_add_geo.run(cluster.dartrundir+'/geo_em.d01.nc', cluster.dartrundir+'/wrfout_d01') # run perfect_model obs (forward operator) os.system('mpirun -np 12 ./perfect_model_obs > /dev/null') # truth values in obs_seq.out are H(x) values true, _ = read_truth_obs_obsseq(cluster.dartrundir+'/obs_seq.out') list_ensemble_truths.append(true) n_obs = len(list_ensemble_truths[0]) np_array = np.full((exp.n_ens, n_obs), np.nan) for i in range(exp.n_ens): np_array[i, :] = list_ensemble_truths[i] return np_array else: raise NotImplementedError()
def read_rss(url, name, pre_msg, find_xml): recent_id = open(os.path.join(settings['ignore_loc'], name), 'r').read() try: rss = urllib.request.urlopen(url).read().decode("utf-8") xml = etree.fromstring(rss) except: # Don't need anymore than this for something like this print("Failed to read/parse {0} ({1}) RSS".format(name, url)) return False if bool(find_xml['sub_listing']): entry = xml[0][find_xml['entries_in']] else: entry = xml[find_xml['entries_in']] current_id = entry.findtext(find_xml['entry_id']) if current_id == recent_id: return False with open(os.path.join(settings['ignore_loc'], name), "w") as f: f.write(current_id) if bool(find_xml['get_href']): msg_url = entry.find(find_xml['link_id']).get('href') else: msg_url = entry.findtext(find_xml['link_id']) msg_msg = re.sub('<[^<]+?>', '', entry.findtext(find_xml['msg_id'])) msg_msg = re.sub( ' +', ' ', os.linesep.join([s for s in msg_msg.splitlines() if s])).lstrip() msg = "{0}{1}\n{2}".format(pre_msg, utils.short_string(msg_msg, 90), msg_url) post_tweet(status_api, msg)
async def on_server_remove(server): """Called when kicked or left the server. Remove the server from server_settings.ini :param server: Discord.Server object. """ func.config_delete_section(server.id, discord_settings['server_settings']) print("$ Left server: {} ({})".format(server, server.id))
def decorator_route(func): @functools.wraps(func) def wrapper_route(*args, **kwargs): if isinstance((res := func(*args, **kwargs)), Level): return lvl + res elif isinstance(res, Route): if transition_message := res.message: print(transition_message, end="\n\n")
async def on_server_remove(server): """Called when kicked or left the server. Remove the server from server_settings.ini :param server: Discord.Server object. """ config_delete_section(server.id, discord_settings['server_settings']) print("$ Left server: {} ({})".format(server, server.id))
def input_val(self, m): print('\033[1m\t [{}] VAL with val {}, round {}\033[0m'.format( self.pid, m, self.clock_round)) self.val = m for pid in self.except_me(): self.send_in_o1(pid, ('ECHO', self.val)) self.valaccepted = True self.echoreceived = 1
def rename_old(downloaded, lectures): for lec_no, path in downloaded.items(): for lecture in lectures: if lec_no == lecture["seqNo"]: new_name = make_filename(lecture) if path.name != new_name: print(f"Renaming '{path.name}' to '{new_name}'") path.rename(path.with_name(new_name)) break
def handle_tcp(c): filter = config.get_filter() for tcp in get_tcpconn(c.infile): if filter.index != None and tcp.index not in filter.index: continue tcp_msg = "\033[31;2m%s [%s:%d] -- -- --> [%s:%d]\033[0m\n" % \ (tcp.index, tcp.con_tuple[0], tcp.con_tuple[1], tcp.con_tuple[2], tcp.con_tuple[3]) utils.print(tcp_msg)
def alter(ls_start, ls_end): print("? You fell asleep early by") new_dt = ls_start.alter_time_by_td('__sub__') print("You fell asleep", new_dt) if get_confirmation(): ls_start.time = new_dt return ls_start, ls_end
def cmd_build(args): if args.group is None: putter.error('Specify a group to build.') print() with putter.group('Available groups'): print(putter.render_list(mainlist.groups.keys())) return apply_define_args(args) build_project('client')
def load_configuration(self): if not os.path.exists(self.config_path): print('Config file not found...', verbosity=1) return config = open(self.config_path, "rb") self.clusters_indices = load(config) self.n_clusters = load(config) self.use_kmeans = load(config) self.train_dataset = load(config) config.close()
def adv_delay(self, T): if self.Dhat + T <= self.delta: self.D += T self.Dhat += T self.f2a.write(('delay-set', )) dump.dump() else: print('Delay failed with T=', T, 'Dhat=', self.Dhat, 'delta=', self.delta) dump.dump()
def run(p): ss = socket.socket(socket.AF_INET, socket.SOCK_STREAM) ss.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) ss.bind(("0.0.0.0", p)) ss.listen(5) utils.print(f"Server started on port {p}...") while (True): cs, a = ss.accept() cs.settimeout(0.1) threading.Thread(target=_handle, args=(cs, a), kwargs={}).start()
def alter(ls_start, ls_end): print("? You underslept by") new_dt = ls_end.alter_time_by_td('__sub__') print("You awoke at", new_dt) if get_confirmation(): ls_end.time = new_dt return ls_start, ls_end
def load_model(self, model_name): model_files = os.listdir('./models/') model_exists = f'forecaster_{model_name}.pkl' in model_files if model_exists: self.trained = True print(f'MODEL forecaster_{model_name} EXISTS, LOADING...') forecaster_file = f'forecaster_{model_name}.pkl' self.forecaster = load_model(os.path.join('./models/', forecaster_file)) return True print(f'FILE `forecaster_{model_name}.pkl` NOT EXIST') return False
def archive_diagnostics(archive_dir, time): print('archive obs space diagnostics') mkdir(archive_dir) fout = archive_dir+time.strftime('/%Y-%m-%d_%H:%M_obs_seq.final') copy(cluster.dartrundir+'/obs_seq.final', fout) print(fout, 'saved.') try: copy(cluster.dartrundir+'/obs_coords.pkl', archive_stage+'/obs_coords.pkl') except Exception as e: warnings.warn(str(e))
def simulate(AA, s, sp, pwa_model, max_path_len, S0): NUM_SIMS = 100 # sample only initial abstract state x0_samples = (sp.sampler.sample_multiple(S0, AA, sp, NUM_SIMS)).x_array #print(x0_samples) # sample the entire given initial set #X0 = sp.init_cons #x0_samples = sample.sample_ival_constraints(X0, n=1000) print('path length: {}'.format(max_path_len)) traces = [i for i in simulate_pwa(pwa_model, x0_samples, N=max_path_len)] return traces
def start_stream(sapi=None): if sapi is None: sapi = func.login(rest=False) try: stream_sapi = tweepy.Stream(sapi, CustomStreamListener()) print("[INFO] Reading Twitter Stream!") stream_sapi.filter( track=[x.lower() for x in settings['twitter_track']], async=True) except (KeyboardInterrupt, SystemExit): stream_sapi.disconnect() sys.exit(0) return stream_sapi
def print_folder(name, folder_data): with putter.group(f'{name}/'): for k, v in folder_data.folders.items(): print_folder(k, v) for f in folder_data.files: print(f'{cf.bold(f.path)}{cond_suffix(f)}') with putter.indent(): for inst in f.instructions: print_inst(inst) # TODO(maximsmol): only print this when there was a lot of output? print(cf.gray(f'--- {name} ---'))
def on_error(self, status_code): global LAST_STATUS_CODE global HANG_TIME HANG_TIME = time.time() if int(status_code) != int(LAST_STATUS_CODE): LAST_STATUS_CODE = status_code msg = ("[{0}] Twitter Returning Status Code: {1}.\n" "More Info: https://dev.twitter.com/overview/api/response-codes").format( time.strftime("%Y-%m-%d %H:%M"), status_code) print(msg) post_tweet(func.login(status=True), msg) return True
def migrate_dms(roomFile, config): roomlist = [] # channels channelData = json.load(roomFile) for channel in channelData: if config["skip-archived"]: if channel["is_archived"] == True: continue # skip dms with slackbot if channel["user"] == "USLACKBOT": continue _mxCreator = userLUT[channel["user"]] _invitees = [] for user in channel["members"]: if user != channel["user"]: _invitees.append(userLUT[user]) roomDetails = { "slack_id": channel["id"], "slack_members": channel["members"], "slack_created": channel["created"], "slack_creator": channel["user"], "matrix_id": '', "matrix_creator": _mxCreator, } if not config["dry-run"]: res = register_room('', roomDetails["matrix_creator"], '', _invitees, "trusted_private_chat", config["homeserver"], config["as_token"]) if res == False: print("ERROR while registering room '" + roomDetails["slack_name"] + "'") continue else: _content = json.loads(res.content) roomDetails["matrix_id"] = _content["room_id"] print("Registered Slack DM channel " + roomDetails["slack_id"] + " -> " + roomDetails["matrix_id"]) #autojoin all members autojoin_users(_invitees, roomDetails["matrix_id"], config) dmLUT[roomDetails["slack_id"]] = roomDetails["matrix_id"] roomlist.append(roomDetails) return roomlist
def on_error(self, status_code): global LAST_STATUS_CODE global HANG_TIME HANG_TIME = time.time() if int(status_code) != int(LAST_STATUS_CODE): LAST_STATUS_CODE = status_code msg = ( "[{0}] Twitter Returning Status Code: {1}.\n" "More Info: https://dev.twitter.com/overview/api/response-codes" ).format(time.strftime("%Y-%m-%d %H:%M"), status_code) print(msg) post_tweet(func.login(status=True), msg) return True
def input_echo(self, m): print('\033[1m\t [{}] ECHO with val {}, round {}\033[0m'.format( self.pid, m, self.clock_round)) if m != self.val: assert False return # TODO remove assert self.echoreceived += 1 n = len(self.parties) if self.echoreceived == (ceil(n + (n / 3)) / 2): self.todo = [] for pid in self.except_me(): self.send_in_o1(pid, ('READY', self.val)) self.readyreceived = 1
def post_tweet(_API, tweet, media="", command=False, rts=False): try: if media: media = media.replace("\\", "\\\\") if rts and command: print("[{0}] Tweeting: {1} ({2}): [{3}] {4}".format( time.strftime("%Y-%m-%d %H:%M"), rts.user.screen_name, rts.user.id, command, tweet)) else: print("[{0}] Tweeting: {1}".format( time.strftime("%Y-%m-%d %H:%M"), tweet)) if rts: if media: print("(Image: {0})".format(media)) _API.update_with_media(media, status=tweet, in_reply_to_status_id=rts.id) else: _API.update_status(status=tweet, in_reply_to_status_id=rts.id) else: if media: print("(Image: {0})".format(media)) _API.update_with_media(media, status=tweet) else: _API.update_status(status=tweet) except: pass
def cell_affine_models(cell, step_sim, ntrain, ntest, tol, include_err): """cell_affine_models Parameters ---------- cell : cell step_sim : 1 time step (delta_t) simulator tol : each abs state is split further into num_splits cells in order to meet: modeling error < tol (module ntests samples) Returns ------- pwa.SubModel() Notes ------ """ # XXX: Generate different samples for each time step or reuse? # Not clear! sub_models = [] X, Y = getxy_ignoramous(cell, ntrain, step_sim) rm = RegressionModel(X, Y) X, Y = getxy_ignoramous(cell, ntest, step_sim) e_pc = rm.error_pc(X, Y) # error % if __debug__: print('error%:', e_pc) #error = np.linalg.norm(e_pc, 2) # error exceeds tol in error_dims error_dims = np.arange(len(e_pc))[np.where(e_pc >= tol)] if len(error_dims) > 0: err.warn('splitting on e%:{}, |e%|:{}'.format( e_pc, np.linalg.norm(e_pc, 2))) for split_cell in cell.split(axes=error_dims): sub_models_ = cell_affine_models( split_cell, step_sim, ntrain, ntest, tol, include_err) sub_models.extend(sub_models_) return sub_models else: #print('error%:', rm.error_pc(X, Y)) A, b = rm.Ab C, d = cell.ival_constraints.poly() e = rm.error(X, Y) if include_err else None dmap = pwa.DiscreteAffineMap(A, b, e) part = pwa.Partition(C, d, cell) sub_model = pwa.SubModel(part, dmap) if __debug__: print('----------------Finalized------------------') return [sub_model]
def train(net, optimizer, num_epochs, batch_size, trn_set, vld_set=None): assert isinstance(net, NeuralNetwork) assert num_epochs > 0 assert batch_size > 0 trn_x, trn_y = trn_set inputs = [(x, y) for x, y in zip(trn_x, trn_y)] for i in range(num_epochs): np.random.shuffle(inputs) # divide input observations into batches batches = [inputs[j:j+batch_size] for j in range(0, len(inputs), batch_size)] inputs_done = 0 for j, batch in enumerate(batches): net.backpropagate(batch, optimizer) inputs_done += len(batch) u.print("Epoch %02d %s [%d/%d]" % (i+1, u.bar(inputs_done, len(inputs)), inputs_done, len(inputs)), override=True) if vld_set: # test the net at the end of each epoch u.print("Epoch %02d %s [%d/%d] > Testing..." % (i+1, u.bar(inputs_done, len(inputs)), inputs_done, len(inputs)), override=True) accuracy = test(net, vld_set) u.print("Epoch %02d %s [%d/%d] > Validation accuracy: %0.2f%%" % (i+1, u.bar(inputs_done, len(inputs)), inputs_done, len(inputs), accuracy*100), override=True) u.print()
def load_accounts(): accounts = sorted( [p for p in pathlib.Path( os.path.join(BASE_DIR, 'accounts')).iterdir() if p.is_file()]) account_count = 0 print(os.path.join(BASE_DIR, 'accounts')) print(accounts) for acc in accounts: bot_name = os.path.basename(str(acc)).split(".")[0] if DEBUG: if bot_name not in DEBUG_ACCS: continue elif not DEBUG: if "example" in bot_name.lower(): continue account_count += 1 account_list[bot_name] = OrderedDict() Config = ConfigObj(str(acc)) has_cred = False has_sett = False for sec in (Config.iteritems()): sec = tuple(sec) if sec[0] == "credentials": has_cred = True elif sec[0] == "settings": has_sett = True if "-thread" in sec[0]: # Start thread import setttings and creds pass account_list[bot_name][sec[0].lower()] = (sec[1].copy()) if not has_cred: print("Credentials not found for bot: {}".format(bot_name)) input("Press ENTER to close.") sys.exit(0) elif not has_sett: print("No settings are set for bot: {}".format(bot_name)) input("Press ENTER to close.") sys.exit(0) temp = OrderedDict() for k, v in account_list[bot_name].items(): for a, b in v.items(): a = a.lower() try: temp[k][a] = b except: temp[k] = {a: b} account_list[bot_name] = temp.copy() del temp print("Running {0} Accounts!\n".format(account_count)) return(account_list)
def printheaders(headers): l = 0 for k in headers.keys(): if l < len(k): l = len(k) for k, v in headers.items(): utils.print(k.ljust(l)) utils.print(': ') utils.print(v) utils.print('\n')
def start_stream(): try: stream = TwitterStream(credentials['consumer_key'], credentials['consumer_secret'], credentials['access_token'], credentials['access_token_secret']) print("[INFO] Reading Twitter Stream!") # TODO: Start Thread here stream_thread = StoppableThread( target=stream.statuses.filter, kwargs={'track': ', '.join( [x.lower() for x in settings['twitter_track']])}) stream_thread.daemon = True stream_thread.start() except (KeyboardInterrupt, SystemExit, RuntimeError): stream_thread.stop() stream.disconnect() sys.exit(0) return stream
def get_reachable_abs_states( self, intermediate_state, A, system_params, ): state = intermediate_state total_num_samples = state.n property_checker = lambda t, Y: Y in system_params.final_cons rchd_concrete_state_array = cp.compute_concrete_plant_output( A, system_params.plant_sim, state, total_num_samples, property_checker) #print(rchd_concrete_state_array) #exit() # ================= DIRECT MANIPULATION =================== # of StateArray object # rchd_cont_state_array = rchd_concrete_state_array.cont_states # for each reached state, get the abstract state pi_ref = system_params.pi_ref ci_ref = system_params.ci_ref abs2rchd_abs_state_ci_pi_list = [] for rchd_concrete_state in rchd_concrete_state_array.iterable(): rchd_abs_state = \ A.get_abs_state_from_concrete_state(rchd_concrete_state) ci = rchd_concrete_state.ci pi = rchd_concrete_state.pi pi_cell = self.cell_id_from_concrete(pi, pi_ref.eps) ci_cell = self.cell_id_from_concrete(ci, ci_ref.eps) if rchd_concrete_state.x in system_params.final_cons: if not system_params.is_final(A, rchd_abs_state): print(rchd_concrete_state) print(self.get_ival_cons_abs_state(rchd_abs_state.ps)) print(rchd_concrete_state.x) print(system_params.final_cons) raise err.Fatal('cant happen!') if rchd_abs_state is not None: abs2rchd_abs_state_ci_pi_list.append((rchd_abs_state, ci_cell, pi_cell)) # ##!!##logger.debug('abs_state obtained {} from concrete_state {}'.format(rchd_abs_state, rchd_concrete_state)) return abs2rchd_abs_state_ci_pi_list
def on_status(self, status): global HAD_ERROR global HANG_TIME global TWEETS_READ HANG_TIME = time.time() tweet, command = acceptable_tweet(status) if not command: return True open(update['is_busy_file'], 'w') print("[{0}] Reading: {1} ({2}): {3}".format( time.strftime("%Y-%m-%d %H:%M"), status.user.screen_name, status.user.id, status.text)) tweet_command(API, status, tweet, command) HAD_ERROR = False TWEETS_READ.append(str(status.id)) with open(os.path.join(settings['ignore_loc'], "tweets_read.txt"), 'w') as file: file.write("\n".join(TWEETS_READ)) os.remove(update['is_busy_file'])
def handle_stream(sapi, status_api=False): stream_sapi = start_stream(sapi) global HANG_TIME while True: time.sleep(5) elapsed = (time.time() - HANG_TIME) if elapsed > 600: msg = """[{0}] Crashed/Hanging! The bot will catch up on missed messages now!""".format( time.strftime("%Y-%m-%d %H:%M")) print(msg) if status_api: post_tweet(status_api, msg) stream_sapi.disconnect() time.sleep(3) if not stream_sapi.running: stream_sapi = start_stream(sapi) Thread(target=read_notifications, args=(API, True, TWEETS_READ)).start() HANG_TIME = time.time()
def sampleOLD( self, ival_cons, controller_states, ignore, ): ival_cons = 0 controller_states = 0 consolidated_samples = Samples() # TODO: remove this loop by making a unique abstract state for each # combination of (x,s,d,pvt,etc) instead of unique x for s in controller_states: test_cases = self.CE.get_test_cases(s, ival_cons) test_cases.d_array = d_array sample = test_case2sample(test_cases) consolidated_samples.append(sample) print(consolidated_samples) return consolidated_samples
def getxy_rel_ignoramous_force_min_samples(cell1, cell2, force, N, sim, t0=0): """getxy_rel_ignoramous Parameters ---------- force : force to return non-zero samples. Will loop for infinitiy if none exists. """ xl = [] yl = [] sat_count = 0 if __debug__: obs_cells = set() while True: x_array, y_array = getxy_ignoramous(cell1, N, sim, t0=0) if __debug__: for i in y_array: obs_cells.add(CM.cell_from_concrete(i, cell1.eps)) print('reachable cells:', obs_cells) # satisfying indexes sat_array = cell2.ival_constraints.sat(y_array) sat_count += np.sum(sat_array) xl.append(x_array[sat_array]) yl.append(y_array[sat_array]) # If no sample is found and force is True, must keep sampling till # satisfying samples are found if (sat_count >= MIN_TRAIN) or (not force): break if __debug__: print('re-sampling, count:', sat_count) print('found samples: ', sat_count) return np.vstack(xl), np.vstack(yl)
def run(self): line_sep_exp = re.compile(b'\r?\n') socketBuffer = b'' Thread(target=self.timeout_channel).start() for channel in func.config_all_sections(twitch_settings['settings_file']): if channel == "#acepicturebot": continue self.join_channel(channel) while True: try: self.connected = True r, _, _ = select.select([self.irc_sock], [], []) if r: socketBuffer += self.irc_sock.recv(1024) msgs = line_sep_exp.split(socketBuffer) socketBuffer = msgs.pop() for msg in msgs: msg = msg.decode('utf-8') Thread(target=self.on_message, args=(msg,)).start() except Exception as e: print(e) raise
def simulate(sys, prop, opts): num_samples = opts.num_sim_samples num_violations = 0 concrete_states = sample.sample_init_UR(sys, prop, num_samples) trace_list = [] sys_sim = simsys.get_system_simulator(sys) for i in tqdm.trange(num_samples): trace = simsys.simulate(sys_sim, concrete_states[i], prop.T) trace_list.append(trace) sat_x, sat_t = check_prop_violation(trace, prop) if sat_x.size != 0: num_violations += 1 print('x0={} -> x={}, t={}...num_vio_counter={}'.format( trace.x_array[0, :], sat_x[0, :], # the first violating state sat_t[0], # corresponding time instant num_violations), file=SYS.stderr) print('number of violations: {}'.format(num_violations)) return trace_list
def read_rss(url, name, pre_msg, find_xml): recent_id = open(os.path.join(settings['ignore_loc'], name), 'r').read() try: rss = urllib.request.urlopen(url).read().decode("utf-8") xml = etree.fromstring(rss) except: # Don't need anymore than this for something like this print("Failed to read/parse {0} ({1}) RSS".format(name, url)) return False if bool(find_xml['sub_listing']): entry = xml[0][find_xml['entries_in']] else: entry = xml[find_xml['entries_in']] current_id = entry.findtext( find_xml['entry_id']) if current_id == recent_id: return False with open(os.path.join(settings['ignore_loc'], name), "w") as f: f.write(current_id) if bool(find_xml['get_href']): msg_url = entry.find(find_xml['link_id']).get('href') else: msg_url = entry.findtext(find_xml['link_id']) msg_msg = re.sub('<[^<]+?>', '', entry.findtext(find_xml['msg_id'])) msg_msg = re.sub(' +', ' ', os.linesep.join( [s for s in msg_msg.splitlines() if s])).lstrip() msg = "{0}{1}\n{2}".format(pre_msg, utils.short_string(msg_msg, 90), msg_url) if not DEBUG: print(msg) status_api.update_status(status=msg)
def post_tweet(API, tweet, media="", command=False, rts=False): try: if rts and command: print("[{0}] Tweeting: {1} ({2}): [{3}] {4}".format( time.strftime("%Y-%m-%d %H:%M"), rts['user']['screen_name'], rts['user']['id_str'], command, tweet)) else: print("[{0}] Tweeting: {1}".format( time.strftime("%Y-%m-%d %H:%M"), tweet)) if rts: if media: media_ids = [] print("(Image: {0})".format(media)) if isinstance(media, list): for img in media: p = open(img, 'rb') if ".mp4" in img: media_ids.append( API.upload_video( media=p, media_type='video/mp4')['media_id']) else: media_ids.append( API.upload_media(media=p)['media_id']) else: p = open(media, 'rb') if ".mp4" in media: media_ids.append( API.upload_video( media=p, media_type='video/mp4')['media_id']) else: media_ids.append( API.upload_media(media=p)['media_id']) API.update_status(status=tweet, media_ids=media_ids, in_reply_to_status_id=rts['id']) else: API.update_status(status=tweet, in_reply_to_status_id=rts['id']) except Exception as e: print(e) pass
def latest_ver(): import urllib.request url = "http://ace3df.github.io/AcePictureBot/it_ver.txt" try: site_ver = urllib.request.urlopen(url).read().strip().decode("utf-8") if site_ver != __version__: print("!WARNING! A new version is out ({0})!".format(site_ver)) print("Download it here: http://bombch.us/BWVH") print("----------------------------------------\n") except: # Just in case pass
def getxy_rel_ignoramous(cell1, cell2, force, N, sim, t0=0): """getxy_rel_ignoramous Parameters ---------- force : force to return non-zero samples. Will loop for infinitiy if none exists. """ xl = [] yl = [] sat_count = 0 iter_count = itertools.count() print(cell1.ival_constraints) print(cell2.ival_constraints) if __debug__: obs_cells = set() while next(iter_count) <= MAX_ITER: x_array, y_array = getxy_ignoramous(cell1, N, sim, t0=0) if __debug__: for i in y_array: obs_cells.add(CM.cell_from_concrete(i, cell1.eps)) print('reachable cells:', obs_cells) # satisfying indexes sat_array = cell2.ival_constraints.sat(y_array) sat_count += np.sum(sat_array) xl.append(x_array[sat_array]) yl.append(y_array[sat_array]) if sat_count >= MIN_TRAIN: break # If no sample is found and force is True, must keep sampling till # satisfying samples are found if __debug__: if sat_count < MIN_TRAIN: err.warn('Fewer than MIN_TRAIN samples found!') print('found samples: ', sat_count) return np.vstack(xl), np.vstack(yl)