def handle_stream(status_api=False):
stream = start_stream()
global HANG_TIME
try:
while True:
time.sleep(5)
elapsed = (time.time() - HANG_TIME)
if elapsed > 600:
# TODO: Temp to try and stop crash tweet spam for now
if os.path.exists(update['last_crash_file']):
if time.time() - os.path.getctime(
update['last_crash_file']) > 80000:
os.remove(update['last_crash_file'])
open(update['last_crash_file'], 'w')
msg = """[{0}] Restarting!
The bot will catch up on missed messages now!""".format(
time.strftime("%Y-%m-%d %H:%M"))
if status_api:
status_api.update_status(status=msg)
else:
print(msg)
stream.disconnect()
threading.Thread(target=read_notifications,
args=(API, True, TWEETS_READ)).start()
time.sleep(5)
stream = start_stream()
HANG_TIME = time.time()
except (KeyboardInterrupt, SystemExit):
sys.exit(0)
def register_user(
user,
password,
displayname,
server_location,
access_token,
admin=False,
user_type=None,
):
url = "%s/_synapse/admin/v2/users/@%s:%s" % (server_location, user,
config_yaml['domain'])
headers = {'Authorization': ' '.join(['Bearer', access_token])}
data = {
"password": password,
"displayname": "".join([user, config_yaml["name-suffix"]]),
"admin": admin,
}
r = requests.put(url, json=data, headers=headers, verify=False)
if r.status_code != 200 and r.status_code != 201:
print("ERROR! Received %d %s" % (r.status_code, r.reason))
if 400 <= r.status_code < 500:
try:
print(r.json()["error"])
except Exception:
pass
return False
return r
def filter_invalid_abs_states(state_list, pi_seq_list, ci_seq_list, A, init_cons):
valid_idx_list = []
for idx, abs_state in enumerate(state_list):
ival_cons = A.plant_abs.get_ival_cons_abs_state(abs_state.plant_state)
# ##!!##logger.debug('ival_cons: {}'.format(ival_cons))
# find the intersection b/w the cell and the initial cons
# print('init_cons', init_cons)
ic = ival_cons & init_cons
if ic is not None:
valid_idx_list.append(idx)
#valid_state_list.append(abs_state)
# TODO: this should be logged and not printed
if valid_idx_list == []:
for abs_state in state_list:
ival_cons = A.plant_abs.get_ival_cons_abs_state(abs_state.plant_state)
print(ival_cons)
valid_state_list = []
respective_pi_seq_list = []
respective_ci_seq_list = []
for i in valid_idx_list:
valid_state_list.append(state_list[i])
respective_pi_seq_list.append(pi_seq_list[i])
respective_ci_seq_list.append(ci_seq_list[i])
return valid_state_list, respective_pi_seq_list, respective_ci_seq_list
def autojoin_users(
invitees,
roomId,
config,
):
for user in invitees:
#POST /_matrix/client/r0/rooms/{roomId}/join
url = "%s/_matrix/client/r0/rooms/%s/join?user_id=%s" % (
config["homeserver"],
roomId,
user,
)
#_print("Sending registration request...")
r = requests.post(
url,
headers={'Authorization': 'Bearer ' + config["as_token"]},
verify=False)
if r.status_code != 200:
print("ERROR! Received %d %s" % (r.status_code, r.reason))
if 400 <= r.status_code < 500:
try:
print(r.json()["error"])
except Exception:
pass
def run_secam(sys, prop, opts):
MODE = opts.MODE
#plot = opts.plot
if MODE == 'simulate':
if not isinstance(
opts.property_checker,
properties.PropertyChecker):
raise err.Fatal('property checker must be enabled when '
'random testing!')
start_time = time.time()
trace_list = simulate(sys, prop, opts)
#if plot:
if opts.dump_trace:
dump_trace(trace_list)
opts.plotting.plot_trace_list(trace_list, opts.plots)
opts.plotting.show()
elif MODE == 'falsify':
# ignore time taken to create_abstraction: mainly to ignore parsing
# time
current_abs, sampler = create_abstraction(sys, prop, opts)
start_time = time.time()
falsify(sys, prop, opts, current_abs, sampler)
else:
raise err.Fatal('bad MODE supplied: {}'.format(MODE))
stop_time = time.time()
print('*'*20)
print('time spent(s) = {}'.format(stop_time - start_time), file=SYS.stderr)
return
def feasible(num_dims, prop, pwa_trace, solver=gopts.opt_engine):
cons, Vars, vars_grouped_by_state = pwatrace2cons(pwa_trace, num_dims,
prop)
cons = list(cons)
# for c in cons:
# print(c)
#nvars = num_dims.x + num_dims.pi
#A_ub, b_ub = truncate(A_ub, b_ub)
#err.warn_severe('faking output of optimizer')
#res = True
#varval_map = {v: 0 for v in Vars}
# TODO: Make choice of opt engine
#res, varval_map = z3opt.nlinprog(obj, cons, Vars)
#res, varval_map = nlpfun(solver)(obj, cons, Vars)
#ret_val = optsoln2x([varval_map[v] for v in Vars], len(pwa_trace)) if res else None
obj = 0
res = nlpfun(solver)(obj, cons, Vars)
ret_val = optsoln2x(res.x, len(pwa_trace)) if res.success else None
print(res.success)
#print(cons)
print(ret_val)
if res.success:
U.pause()
#embed()
return ret_val
def get_rels(self, prop, sim, N):
"""TODO: EXPLICITLY ignores t, d, pvt, ci, pi
"""
d, pvt = [np.array([])]*2
pi = [np.array([])]
t0 = 0
Yl = []
#plt.figure()
x_array_ = self.xcell.sample_UR(N)
# remove any sample drawn from the property box
x_array = x_array_[~prop.final_cons.sat(x_array_), :]
if x_array.size == 0:
print(prop.final_cons)
print(self.xcell.ival_constraints)
#print(t0)
#print(x_array)
for x in x_array:
(t_, x_, d_, pvt_) = sim(t0, x, d, pvt, pi)
Yl.append(x_)
if settings.debug_plot:
# close intermediate plots which can not be switched off;
# as these are in the system model *.py
#plt.title('ignore')
#plt.show()
#plt.close()
pass
# return empty arrays
if x_array.size == 0:
return x_array, x_array
else:
return x_array, np.vstack(Yl)
def migrate_messages(fileList, matrix_room, config, tick):
global later
archive = zipfile.ZipFile(config["zipfile"], 'r')
txnId = 1
progress = 0
for file in fileList:
try:
fileData = archive.open(file)
messageData = json.load(fileData)
except:
print("Warning: Couldn't load data from file " + file +
" in archive. Skipping this file.")
for message in messageData:
txnId = parse_and_send_message(config, message, matrix_room, txnId,
False)
progress = progress + tick
update_progress(progress)
# process postponed messages
for message in later:
txnId = parse_and_send_message(config, message, matrix_room, txnId,
True)
# clean up postponed messages
later = []
def create(self):
print(': SETUP\n')
data = self.receive_data()
with self.open('w') as conf_file:
toml.dump(data, conf_file)
def test_model_(abs_state, AA, sp, am, step_sim):
test_samples = sp.sampler.sample(abs_state, AA, sp, AA.num_samples*MORE_FACTOR*TEST_FACTOR)
X, Y = getxy(abs_state, test_samples, step_sim)
e = am.model_error(X, Y)
if __debug__:
print(e)
return e
def on_status(self, status):
global HAD_ERROR
global HANG_TIME
global TWEETS_READ
HANG_TIME = time.time()
tweet, command = acceptable_tweet(status)
if not command:
return True
try:
open(update['is_busy_file'], 'w')
except PermissionError:
# This wont happen all the time, the file is probably busy
pass
print("[{0}] Reading: {1} ({2}): {3}".format(
time.strftime("%Y-%m-%d %H:%M"),
status.user.screen_name, status.user.id, status.text))
tweet_command(API, status, tweet, command)
HAD_ERROR = False
TWEETS_READ.append(str(status.id))
with open(os.path.join(settings['ignore_loc'],
"tweets_read.txt"),
'w') as file:
file.write("\n".join(TWEETS_READ))
try:
os.remove(update['is_busy_file'])
except (PermissionError, FileNotFoundError):
# Related to above PermissionError.
pass
def handle_stream(sapi, status_api=False):
stream_sapi = start_stream(sapi)
global HANG_TIME
while True:
time.sleep(5)
elapsed = (time.time() - HANG_TIME)
if elapsed > 600:
# TODO: Temp to try and stop crash tweet spam for now
if os.path.exists(update['last_crash_file']):
if time.time() - os.path.getctime(update['last_crash_file']) > 80000:
os.remove(update['last_crash_file'])
open(update['last_crash_file'], 'w')
msg = """[{0}] Restarting!
The bot will catch up on missed messages now!""".format(
time.strftime("%Y-%m-%d %H:%M"))
if status_api:
post_tweet(status_api, msg)
else:
print(msg)
stream_sapi.disconnect()
time.sleep(3)
if not stream_sapi.running:
stream_sapi = start_stream(sapi)
Thread(target=read_notifications,
args=(API, True, TWEETS_READ)).start()
HANG_TIME = time.time()
def check_prop_violation(prop, trace):
"""check_prop_violation
Parameters
----------
trace :
prop :
Returns
-------
Notes
------
"""
# check using random sims
idx = prop.final_cons.sat(trace.x_array + 0.0001)
sat_x, sat_t = trace.x_array[idx], trace.t_array[idx]
if sat_x.size != 0:
print('x0={} -> x={}, t={}'.format(
trace.x_array[0, :],
sat_x[0, :], # the first violating state
sat_t[0], # corresponding time instant
))
return True
else:
return False
def build_pwa_model(AA, abs_obj, sp, tol, include_err, model_type):
"""build_pwa_model
Builds both dft and rel models
Parameters
----------
AA : AA
abs_obj : Either abs_states (for dft models) or relations
[tuple(abs_state_src, abs_state_target)] for rel model
sp : system params
tol : modeling error tolerance
include_err : include error in determining next state
x' = x +- errror
"""
dt = AA.plant_abs.delta_t
step_sim = simsys.get_step_simulator(sp.controller_sim, sp.plant_sim, dt)
#abs_state_models = {}
modelers = {
'dft': (pwa.PWA, abs_state_affine_models),
'rel': (rel.PWARelational, abs_rel_affine_models),
}
M, model = modelers[model_type]
pwa_model = M()
for a in abs_obj:
print('modeling: {}'.format(a))
for sub_model in model(a, AA, step_sim, tol, sp, include_err):
if sub_model is not None:
pwa_model.add(sub_model)
#abs_state_models[abs_state] = sub_model
return pwa_model
def on_success(self, data):
global HAD_ERROR
global HANG_TIME
global TWEETS_READ
tweet_datetime = datetime.datetime.strptime(
data['created_at'], '%a %b %d %H:%M:%S %z %Y')
days_past = datetime.datetime.now(datetime.timezone.utc) - tweet_datetime
if days_past.days > 1:
return False
# Fri Apr 15 16:59:09 +0000 2016
if data['id_str'] in TWEETS_READ:
return True
TWEETS_READ.append(data['id_str'])
HANG_TIME = time.time()
tweet, command = acceptable_tweet(data)
if not command:
return True
try:
open(update['is_busy_file'], 'w')
except PermissionError:
# This wont happen all the time, the file is probably busy
pass
print("[{0}] {1} ({2}): {3}".format(
time.strftime("%Y-%m-%d %H:%M"),
data['user']['screen_name'], data['user']['id_str'], data['text']))
tweet_command(API, data, tweet, command)
HAD_ERROR = False
with open(os.path.join(settings['ignore_loc'],
"tweets_read.txt"), 'w') as file:
file.write("\n".join(TWEETS_READ))
try:
os.remove(update['is_busy_file'])
except (PermissionError, FileNotFoundError):
# Related to above PermissionError.
pass
def get_data(self):
def parseHM_(s):
try:
h, m, _ = s.split(':')
except:
return
return parse_time_period(h, m)
while True:
fine_state = True
data = self.load()
for field_name, field in self.__dataclass_fields__.items():
if not isinstance(data.get(field_name), field.type):
print('! Config file is corrupped')
self.update()
fine_state = False
break
if not fine_state:
continue
data['rest_per_day'] = parseHM_(data['rest_per_day'])
if not all(not_none(var) for var in data.values()):
print("Corrupted data")
self.update()
continue
break
self.rest_per_day = data['rest_per_day']
def obs_operator_ensemble(istage):
# assumes that prior ensemble is already linked to advance_temp<i>/wrfout_d01
print('running obs operator on ensemble forecast')
os.chdir(cluster.dartrundir)
if sat_channel:
list_ensemble_truths = []
for iens in range(1, exp.n_ens+1):
print('observation operator for ens #'+str(iens))
# ens members are already linked to advance_temp<i>/wrfout_d01
copy(cluster.dartrundir+'/advance_temp'+str(iens)+'/wrfout_d01',
cluster.dartrundir+'/wrfout_d01')
# DART may need a wrfinput file as well, which serves as a template for dimension sizes
symlink(cluster.dartrundir+'/wrfout_d01', cluster.dartrundir+'/wrfinput_d01')
# add geodata, if istage>0, wrfout is DART output (has coords)
if istage == 0:
wrfout_add_geo.run(cluster.dartrundir+'/geo_em.d01.nc', cluster.dartrundir+'/wrfout_d01')
# run perfect_model obs (forward operator)
os.system('mpirun -np 12 ./perfect_model_obs > /dev/null')
# truth values in obs_seq.out are H(x) values
true, _ = read_truth_obs_obsseq(cluster.dartrundir+'/obs_seq.out')
list_ensemble_truths.append(true)
n_obs = len(list_ensemble_truths[0])
np_array = np.full((exp.n_ens, n_obs), np.nan)
for i in range(exp.n_ens):
np_array[i, :] = list_ensemble_truths[i]
return np_array
else:
raise NotImplementedError()
def read_rss(url, name, pre_msg, find_xml):
recent_id = open(os.path.join(settings['ignore_loc'], name),
'r').read()
try:
rss = urllib.request.urlopen(url).read().decode("utf-8")
xml = etree.fromstring(rss)
except:
# Don't need anymore than this for something like this
print("Failed to read/parse {0} ({1}) RSS".format(name, url))
return False
if bool(find_xml['sub_listing']):
entry = xml[0][find_xml['entries_in']]
else:
entry = xml[find_xml['entries_in']]
current_id = entry.findtext(find_xml['entry_id'])
if current_id == recent_id:
return False
with open(os.path.join(settings['ignore_loc'], name), "w") as f:
f.write(current_id)
if bool(find_xml['get_href']):
msg_url = entry.find(find_xml['link_id']).get('href')
else:
msg_url = entry.findtext(find_xml['link_id'])
msg_msg = re.sub('<[^<]+?>', '', entry.findtext(find_xml['msg_id']))
msg_msg = re.sub(
' +', ' ',
os.linesep.join([s for s in msg_msg.splitlines() if s])).lstrip()
msg = "{0}{1}\n{2}".format(pre_msg, utils.short_string(msg_msg, 90),
msg_url)
post_tweet(status_api, msg)
async def on_server_remove(server):
"""Called when kicked or left the server.
Remove the server from server_settings.ini
:param server: Discord.Server object.
"""
func.config_delete_section(server.id, discord_settings['server_settings'])
print("$ Left server: {} ({})".format(server, server.id))
def decorator_route(func):
@functools.wraps(func)
def wrapper_route(*args, **kwargs):
if isinstance((res := func(*args, **kwargs)), Level):
return lvl + res
elif isinstance(res, Route):
if transition_message := res.message:
print(transition_message, end="\n\n")
async def on_server_remove(server):
"""Called when kicked or left the server.
Remove the server from server_settings.ini
:param server: Discord.Server object.
"""
config_delete_section(server.id, discord_settings['server_settings'])
print("$ Left server: {} ({})".format(server, server.id))
def input_val(self, m):
print('\033[1m\t [{}] VAL with val {}, round {}\033[0m'.format(
self.pid, m, self.clock_round))
self.val = m
for pid in self.except_me():
self.send_in_o1(pid, ('ECHO', self.val))
self.valaccepted = True
self.echoreceived = 1
def rename_old(downloaded, lectures):
for lec_no, path in downloaded.items():
for lecture in lectures:
if lec_no == lecture["seqNo"]:
new_name = make_filename(lecture)
if path.name != new_name:
print(f"Renaming '{path.name}' to '{new_name}'")
path.rename(path.with_name(new_name))
break
def handle_tcp(c):
filter = config.get_filter()
for tcp in get_tcpconn(c.infile):
if filter.index != None and tcp.index not in filter.index:
continue
tcp_msg = "\033[31;2m%s [%s:%d] -- -- --> [%s:%d]\033[0m\n" % \
(tcp.index, tcp.con_tuple[0], tcp.con_tuple[1],
tcp.con_tuple[2], tcp.con_tuple[3])
utils.print(tcp_msg)
def alter(ls_start, ls_end):
print("? You fell asleep early by")
new_dt = ls_start.alter_time_by_td('__sub__')
print("You fell asleep", new_dt)
if get_confirmation():
ls_start.time = new_dt
return ls_start, ls_end
def cmd_build(args):
if args.group is None:
putter.error('Specify a group to build.')
print()
with putter.group('Available groups'):
print(putter.render_list(mainlist.groups.keys()))
return
apply_define_args(args)
build_project('client')
def load_configuration(self):
if not os.path.exists(self.config_path):
print('Config file not found...', verbosity=1)
return
config = open(self.config_path, "rb")
self.clusters_indices = load(config)
self.n_clusters = load(config)
self.use_kmeans = load(config)
self.train_dataset = load(config)
config.close()
def adv_delay(self, T):
if self.Dhat + T <= self.delta:
self.D += T
self.Dhat += T
self.f2a.write(('delay-set', ))
dump.dump()
else:
print('Delay failed with T=', T, 'Dhat=', self.Dhat, 'delta=',
self.delta)
dump.dump()
def run(p):
ss = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ss.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
ss.bind(("0.0.0.0", p))
ss.listen(5)
utils.print(f"Server started on port {p}...")
while (True):
cs, a = ss.accept()
cs.settimeout(0.1)
threading.Thread(target=_handle, args=(cs, a), kwargs={}).start()
def alter(ls_start, ls_end):
print("? You underslept by")
new_dt = ls_end.alter_time_by_td('__sub__')
print("You awoke at", new_dt)
if get_confirmation():
ls_end.time = new_dt
return ls_start, ls_end
def load_model(self, model_name):
model_files = os.listdir('./models/')
model_exists = f'forecaster_{model_name}.pkl' in model_files
if model_exists:
self.trained = True
print(f'MODEL forecaster_{model_name} EXISTS, LOADING...')
forecaster_file = f'forecaster_{model_name}.pkl'
self.forecaster = load_model(os.path.join('./models/', forecaster_file))
return True
print(f'FILE `forecaster_{model_name}.pkl` NOT EXIST')
return False
def archive_diagnostics(archive_dir, time):
print('archive obs space diagnostics')
mkdir(archive_dir)
fout = archive_dir+time.strftime('/%Y-%m-%d_%H:%M_obs_seq.final')
copy(cluster.dartrundir+'/obs_seq.final', fout)
print(fout, 'saved.')
try:
copy(cluster.dartrundir+'/obs_coords.pkl', archive_stage+'/obs_coords.pkl')
except Exception as e:
warnings.warn(str(e))
def simulate(AA, s, sp, pwa_model, max_path_len, S0):
NUM_SIMS = 100
# sample only initial abstract state
x0_samples = (sp.sampler.sample_multiple(S0, AA, sp, NUM_SIMS)).x_array
#print(x0_samples)
# sample the entire given initial set
#X0 = sp.init_cons
#x0_samples = sample.sample_ival_constraints(X0, n=1000)
print('path length: {}'.format(max_path_len))
traces = [i for i in simulate_pwa(pwa_model, x0_samples, N=max_path_len)]
return traces
def start_stream(sapi=None):
if sapi is None:
sapi = func.login(rest=False)
try:
stream_sapi = tweepy.Stream(sapi, CustomStreamListener())
print("[INFO] Reading Twitter Stream!")
stream_sapi.filter(
track=[x.lower() for x in settings['twitter_track']], async=True)
except (KeyboardInterrupt, SystemExit):
stream_sapi.disconnect()
sys.exit(0)
return stream_sapi
def print_folder(name, folder_data):
with putter.group(f'{name}/'):
for k, v in folder_data.folders.items():
print_folder(k, v)
for f in folder_data.files:
print(f'{cf.bold(f.path)}{cond_suffix(f)}')
with putter.indent():
for inst in f.instructions:
print_inst(inst)
# TODO(maximsmol): only print this when there was a lot of output?
print(cf.gray(f'--- {name} ---'))
def on_error(self, status_code):
global LAST_STATUS_CODE
global HANG_TIME
HANG_TIME = time.time()
if int(status_code) != int(LAST_STATUS_CODE):
LAST_STATUS_CODE = status_code
msg = ("[{0}] Twitter Returning Status Code: {1}.\n"
"More Info: https://dev.twitter.com/overview/api/response-codes").format(
time.strftime("%Y-%m-%d %H:%M"), status_code)
print(msg)
post_tweet(func.login(status=True), msg)
return True
def start_stream(sapi=None):
if sapi is None:
sapi = func.login(rest=False)
try:
stream_sapi = tweepy.Stream(sapi, CustomStreamListener())
print("[INFO] Reading Twitter Stream!")
stream_sapi.filter(
track=[x.lower() for x in settings['twitter_track']], async=True)
except (KeyboardInterrupt, SystemExit):
stream_sapi.disconnect()
sys.exit(0)
return stream_sapi
def migrate_dms(roomFile, config):
roomlist = []
# channels
channelData = json.load(roomFile)
for channel in channelData:
if config["skip-archived"]:
if channel["is_archived"] == True:
continue
# skip dms with slackbot
if channel["user"] == "USLACKBOT":
continue
_mxCreator = userLUT[channel["user"]]
_invitees = []
for user in channel["members"]:
if user != channel["user"]:
_invitees.append(userLUT[user])
roomDetails = {
"slack_id": channel["id"],
"slack_members": channel["members"],
"slack_created": channel["created"],
"slack_creator": channel["user"],
"matrix_id": '',
"matrix_creator": _mxCreator,
}
if not config["dry-run"]:
res = register_room('', roomDetails["matrix_creator"], '',
_invitees, "trusted_private_chat",
config["homeserver"], config["as_token"])
if res == False:
print("ERROR while registering room '" +
roomDetails["slack_name"] + "'")
continue
else:
_content = json.loads(res.content)
roomDetails["matrix_id"] = _content["room_id"]
print("Registered Slack DM channel " + roomDetails["slack_id"] +
" -> " + roomDetails["matrix_id"])
#autojoin all members
autojoin_users(_invitees, roomDetails["matrix_id"], config)
dmLUT[roomDetails["slack_id"]] = roomDetails["matrix_id"]
roomlist.append(roomDetails)
return roomlist
def on_error(self, status_code):
global LAST_STATUS_CODE
global HANG_TIME
HANG_TIME = time.time()
if int(status_code) != int(LAST_STATUS_CODE):
LAST_STATUS_CODE = status_code
msg = (
"[{0}] Twitter Returning Status Code: {1}.\n"
"More Info: https://dev.twitter.com/overview/api/response-codes"
).format(time.strftime("%Y-%m-%d %H:%M"), status_code)
print(msg)
post_tweet(func.login(status=True), msg)
return True
def input_echo(self, m):
print('\033[1m\t [{}] ECHO with val {}, round {}\033[0m'.format(
self.pid, m, self.clock_round))
if m != self.val:
assert False
return # TODO remove assert
self.echoreceived += 1
n = len(self.parties)
if self.echoreceived == (ceil(n + (n / 3)) / 2):
self.todo = []
for pid in self.except_me():
self.send_in_o1(pid, ('READY', self.val))
self.readyreceived = 1
def post_tweet(_API, tweet, media="", command=False, rts=False):
try:
if media:
media = media.replace("\\", "\\\\")
if rts and command:
print("[{0}] Tweeting: {1} ({2}): [{3}] {4}".format(
time.strftime("%Y-%m-%d %H:%M"),
rts.user.screen_name, rts.user.id,
command, tweet))
else:
print("[{0}] Tweeting: {1}".format(
time.strftime("%Y-%m-%d %H:%M"),
tweet))
if rts:
if media:
print("(Image: {0})".format(media))
_API.update_with_media(media, status=tweet,
in_reply_to_status_id=rts.id)
else:
_API.update_status(status=tweet,
in_reply_to_status_id=rts.id)
else:
if media:
print("(Image: {0})".format(media))
_API.update_with_media(media, status=tweet)
else:
_API.update_status(status=tweet)
except:
pass
def cell_affine_models(cell, step_sim, ntrain, ntest, tol, include_err):
"""cell_affine_models
Parameters
----------
cell : cell
step_sim : 1 time step (delta_t) simulator
tol : each abs state is split further into num_splits cells
in order to meet: modeling error < tol (module ntests samples)
Returns
-------
pwa.SubModel()
Notes
------
"""
# XXX: Generate different samples for each time step or reuse?
# Not clear!
sub_models = []
X, Y = getxy_ignoramous(cell, ntrain, step_sim)
rm = RegressionModel(X, Y)
X, Y = getxy_ignoramous(cell, ntest, step_sim)
e_pc = rm.error_pc(X, Y) # error %
if __debug__:
print('error%:', e_pc)
#error = np.linalg.norm(e_pc, 2)
# error exceeds tol in error_dims
error_dims = np.arange(len(e_pc))[np.where(e_pc >= tol)]
if len(error_dims) > 0:
err.warn('splitting on e%:{}, |e%|:{}'.format(
e_pc, np.linalg.norm(e_pc, 2)))
for split_cell in cell.split(axes=error_dims):
sub_models_ = cell_affine_models(
split_cell, step_sim, ntrain, ntest, tol, include_err)
sub_models.extend(sub_models_)
return sub_models
else:
#print('error%:', rm.error_pc(X, Y))
A, b = rm.Ab
C, d = cell.ival_constraints.poly()
e = rm.error(X, Y) if include_err else None
dmap = pwa.DiscreteAffineMap(A, b, e)
part = pwa.Partition(C, d, cell)
sub_model = pwa.SubModel(part, dmap)
if __debug__:
print('----------------Finalized------------------')
return [sub_model]
def train(net, optimizer, num_epochs, batch_size, trn_set, vld_set=None):
assert isinstance(net, NeuralNetwork)
assert num_epochs > 0
assert batch_size > 0
trn_x, trn_y = trn_set
inputs = [(x, y) for x, y in zip(trn_x, trn_y)]
for i in range(num_epochs):
np.random.shuffle(inputs)
# divide input observations into batches
batches = [inputs[j:j+batch_size] for j in range(0, len(inputs), batch_size)]
inputs_done = 0
for j, batch in enumerate(batches):
net.backpropagate(batch, optimizer)
inputs_done += len(batch)
u.print("Epoch %02d %s [%d/%d]" % (i+1, u.bar(inputs_done, len(inputs)), inputs_done, len(inputs)), override=True)
if vld_set:
# test the net at the end of each epoch
u.print("Epoch %02d %s [%d/%d] > Testing..." % (i+1, u.bar(inputs_done, len(inputs)), inputs_done, len(inputs)), override=True)
accuracy = test(net, vld_set)
u.print("Epoch %02d %s [%d/%d] > Validation accuracy: %0.2f%%" % (i+1, u.bar(inputs_done, len(inputs)), inputs_done, len(inputs), accuracy*100), override=True)
u.print()
def load_accounts():
accounts = sorted(
[p for p in pathlib.Path(
os.path.join(BASE_DIR, 'accounts')).iterdir() if p.is_file()])
account_count = 0
print(os.path.join(BASE_DIR, 'accounts'))
print(accounts)
for acc in accounts:
bot_name = os.path.basename(str(acc)).split(".")[0]
if DEBUG:
if bot_name not in DEBUG_ACCS:
continue
elif not DEBUG:
if "example" in bot_name.lower():
continue
account_count += 1
account_list[bot_name] = OrderedDict()
Config = ConfigObj(str(acc))
has_cred = False
has_sett = False
for sec in (Config.iteritems()):
sec = tuple(sec)
if sec[0] == "credentials":
has_cred = True
elif sec[0] == "settings":
has_sett = True
if "-thread" in sec[0]:
# Start thread import setttings and creds
pass
account_list[bot_name][sec[0].lower()] = (sec[1].copy())
if not has_cred:
print("Credentials not found for bot: {}".format(bot_name))
input("Press ENTER to close.")
sys.exit(0)
elif not has_sett:
print("No settings are set for bot: {}".format(bot_name))
input("Press ENTER to close.")
sys.exit(0)
temp = OrderedDict()
for k, v in account_list[bot_name].items():
for a, b in v.items():
a = a.lower()
try:
temp[k][a] = b
except:
temp[k] = {a: b}
account_list[bot_name] = temp.copy()
del temp
print("Running {0} Accounts!\n".format(account_count))
return(account_list)
def printheaders(headers):
l = 0
for k in headers.keys():
if l < len(k):
l = len(k)
for k, v in headers.items():
utils.print(k.ljust(l))
utils.print(': ')
utils.print(v)
utils.print('\n')
def start_stream():
try:
stream = TwitterStream(credentials['consumer_key'],
credentials['consumer_secret'],
credentials['access_token'],
credentials['access_token_secret'])
print("[INFO] Reading Twitter Stream!")
# TODO: Start Thread here
stream_thread = StoppableThread(
target=stream.statuses.filter,
kwargs={'track': ', '.join(
[x.lower() for x in settings['twitter_track']])})
stream_thread.daemon = True
stream_thread.start()
except (KeyboardInterrupt, SystemExit, RuntimeError):
stream_thread.stop()
stream.disconnect()
sys.exit(0)
return stream
def get_reachable_abs_states(
self,
intermediate_state,
A,
system_params,
):
state = intermediate_state
total_num_samples = state.n
property_checker = lambda t, Y: Y in system_params.final_cons
rchd_concrete_state_array = cp.compute_concrete_plant_output(
A,
system_params.plant_sim,
state,
total_num_samples,
property_checker)
#print(rchd_concrete_state_array)
#exit()
# ================= DIRECT MANIPULATION ===================
# of StateArray object
# rchd_cont_state_array = rchd_concrete_state_array.cont_states
# for each reached state, get the abstract state
pi_ref = system_params.pi_ref
ci_ref = system_params.ci_ref
abs2rchd_abs_state_ci_pi_list = []
for rchd_concrete_state in rchd_concrete_state_array.iterable():
rchd_abs_state = \
A.get_abs_state_from_concrete_state(rchd_concrete_state)
ci = rchd_concrete_state.ci
pi = rchd_concrete_state.pi
pi_cell = self.cell_id_from_concrete(pi, pi_ref.eps)
ci_cell = self.cell_id_from_concrete(ci, ci_ref.eps)
if rchd_concrete_state.x in system_params.final_cons:
if not system_params.is_final(A, rchd_abs_state):
print(rchd_concrete_state)
print(self.get_ival_cons_abs_state(rchd_abs_state.ps))
print(rchd_concrete_state.x)
print(system_params.final_cons)
raise err.Fatal('cant happen!')
if rchd_abs_state is not None:
abs2rchd_abs_state_ci_pi_list.append((rchd_abs_state, ci_cell, pi_cell))
# ##!!##logger.debug('abs_state obtained {} from concrete_state {}'.format(rchd_abs_state, rchd_concrete_state))
return abs2rchd_abs_state_ci_pi_list
def on_status(self, status):
global HAD_ERROR
global HANG_TIME
global TWEETS_READ
HANG_TIME = time.time()
tweet, command = acceptable_tweet(status)
if not command:
return True
open(update['is_busy_file'], 'w')
print("[{0}] Reading: {1} ({2}): {3}".format(
time.strftime("%Y-%m-%d %H:%M"),
status.user.screen_name, status.user.id, status.text))
tweet_command(API, status, tweet, command)
HAD_ERROR = False
TWEETS_READ.append(str(status.id))
with open(os.path.join(settings['ignore_loc'],
"tweets_read.txt"),
'w') as file:
file.write("\n".join(TWEETS_READ))
os.remove(update['is_busy_file'])
def handle_stream(sapi, status_api=False):
stream_sapi = start_stream(sapi)
global HANG_TIME
while True:
time.sleep(5)
elapsed = (time.time() - HANG_TIME)
if elapsed > 600:
msg = """[{0}] Crashed/Hanging!
The bot will catch up on missed messages now!""".format(
time.strftime("%Y-%m-%d %H:%M"))
print(msg)
if status_api:
post_tweet(status_api, msg)
stream_sapi.disconnect()
time.sleep(3)
if not stream_sapi.running:
stream_sapi = start_stream(sapi)
Thread(target=read_notifications,
args=(API, True, TWEETS_READ)).start()
HANG_TIME = time.time()
def sampleOLD(
self,
ival_cons,
controller_states,
ignore,
):
ival_cons = 0
controller_states = 0
consolidated_samples = Samples()
# TODO: remove this loop by making a unique abstract state for each
# combination of (x,s,d,pvt,etc) instead of unique x
for s in controller_states:
test_cases = self.CE.get_test_cases(s, ival_cons)
test_cases.d_array = d_array
sample = test_case2sample(test_cases)
consolidated_samples.append(sample)
print(consolidated_samples)
return consolidated_samples
def getxy_rel_ignoramous_force_min_samples(cell1, cell2, force, N, sim, t0=0):
"""getxy_rel_ignoramous
Parameters
----------
force : force to return non-zero samples. Will loop for infinitiy
if none exists.
"""
xl = []
yl = []
sat_count = 0
if __debug__:
obs_cells = set()
while True:
x_array, y_array = getxy_ignoramous(cell1, N, sim, t0=0)
if __debug__:
for i in y_array:
obs_cells.add(CM.cell_from_concrete(i, cell1.eps))
print('reachable cells:', obs_cells)
# satisfying indexes
sat_array = cell2.ival_constraints.sat(y_array)
sat_count += np.sum(sat_array)
xl.append(x_array[sat_array])
yl.append(y_array[sat_array])
# If no sample is found and force is True, must keep sampling till
# satisfying samples are found
if (sat_count >= MIN_TRAIN) or (not force):
break
if __debug__:
print('re-sampling, count:', sat_count)
print('found samples: ', sat_count)
return np.vstack(xl), np.vstack(yl)
def run(self):
line_sep_exp = re.compile(b'\r?\n')
socketBuffer = b''
Thread(target=self.timeout_channel).start()
for channel in func.config_all_sections(twitch_settings['settings_file']):
if channel == "#acepicturebot":
continue
self.join_channel(channel)
while True:
try:
self.connected = True
r, _, _ = select.select([self.irc_sock], [], [])
if r:
socketBuffer += self.irc_sock.recv(1024)
msgs = line_sep_exp.split(socketBuffer)
socketBuffer = msgs.pop()
for msg in msgs:
msg = msg.decode('utf-8')
Thread(target=self.on_message, args=(msg,)).start()
except Exception as e:
print(e)
raise
def simulate(sys, prop, opts):
num_samples = opts.num_sim_samples
num_violations = 0
concrete_states = sample.sample_init_UR(sys, prop, num_samples)
trace_list = []
sys_sim = simsys.get_system_simulator(sys)
for i in tqdm.trange(num_samples):
trace = simsys.simulate(sys_sim, concrete_states[i], prop.T)
trace_list.append(trace)
sat_x, sat_t = check_prop_violation(trace, prop)
if sat_x.size != 0:
num_violations += 1
print('x0={} -> x={}, t={}...num_vio_counter={}'.format(
trace.x_array[0, :],
sat_x[0, :], # the first violating state
sat_t[0], # corresponding time instant
num_violations), file=SYS.stderr)
print('number of violations: {}'.format(num_violations))
return trace_list
def read_rss(url, name, pre_msg, find_xml):
recent_id = open(os.path.join(settings['ignore_loc'],
name), 'r').read()
try:
rss = urllib.request.urlopen(url).read().decode("utf-8")
xml = etree.fromstring(rss)
except:
# Don't need anymore than this for something like this
print("Failed to read/parse {0} ({1}) RSS".format(name, url))
return False
if bool(find_xml['sub_listing']):
entry = xml[0][find_xml['entries_in']]
else:
entry = xml[find_xml['entries_in']]
current_id = entry.findtext(
find_xml['entry_id'])
if current_id == recent_id:
return False
with open(os.path.join(settings['ignore_loc'], name), "w") as f:
f.write(current_id)
if bool(find_xml['get_href']):
msg_url = entry.find(find_xml['link_id']).get('href')
else:
msg_url = entry.findtext(find_xml['link_id'])
msg_msg = re.sub('<[^<]+?>', '', entry.findtext(find_xml['msg_id']))
msg_msg = re.sub(' +', ' ', os.linesep.join(
[s for s in msg_msg.splitlines() if s])).lstrip()
msg = "{0}{1}\n{2}".format(pre_msg,
utils.short_string(msg_msg, 90),
msg_url)
if not DEBUG:
print(msg)
status_api.update_status(status=msg)
def post_tweet(API, tweet, media="", command=False, rts=False):
try:
if rts and command:
print("[{0}] Tweeting: {1} ({2}): [{3}] {4}".format(
time.strftime("%Y-%m-%d %H:%M"),
rts['user']['screen_name'], rts['user']['id_str'],
command, tweet))
else:
print("[{0}] Tweeting: {1}".format(
time.strftime("%Y-%m-%d %H:%M"),
tweet))
if rts:
if media:
media_ids = []
print("(Image: {0})".format(media))
if isinstance(media, list):
for img in media:
p = open(img, 'rb')
if ".mp4" in img:
media_ids.append(
API.upload_video(
media=p,
media_type='video/mp4')['media_id'])
else:
media_ids.append(
API.upload_media(media=p)['media_id'])
else:
p = open(media, 'rb')
if ".mp4" in media:
media_ids.append(
API.upload_video(
media=p,
media_type='video/mp4')['media_id'])
else:
media_ids.append(
API.upload_media(media=p)['media_id'])
API.update_status(status=tweet, media_ids=media_ids,
in_reply_to_status_id=rts['id'])
else:
API.update_status(status=tweet,
in_reply_to_status_id=rts['id'])
except Exception as e:
print(e)
pass
def latest_ver():
import urllib.request
url = "http://ace3df.github.io/AcePictureBot/it_ver.txt"
try:
site_ver = urllib.request.urlopen(url).read().strip().decode("utf-8")
if site_ver != __version__:
print("!WARNING! A new version is out ({0})!".format(site_ver))
print("Download it here: http://bombch.us/BWVH")
print("----------------------------------------\n")
except:
# Just in case
pass
def getxy_rel_ignoramous(cell1, cell2, force, N, sim, t0=0):
"""getxy_rel_ignoramous
Parameters
----------
force : force to return non-zero samples. Will loop for infinitiy
if none exists.
"""
xl = []
yl = []
sat_count = 0
iter_count = itertools.count()
print(cell1.ival_constraints)
print(cell2.ival_constraints)
if __debug__:
obs_cells = set()
while next(iter_count) <= MAX_ITER:
x_array, y_array = getxy_ignoramous(cell1, N, sim, t0=0)
if __debug__:
for i in y_array:
obs_cells.add(CM.cell_from_concrete(i, cell1.eps))
print('reachable cells:', obs_cells)
# satisfying indexes
sat_array = cell2.ival_constraints.sat(y_array)
sat_count += np.sum(sat_array)
xl.append(x_array[sat_array])
yl.append(y_array[sat_array])
if sat_count >= MIN_TRAIN:
break
# If no sample is found and force is True, must keep sampling till
# satisfying samples are found
if __debug__:
if sat_count < MIN_TRAIN:
err.warn('Fewer than MIN_TRAIN samples found!')
print('found samples: ', sat_count)
return np.vstack(xl), np.vstack(yl)
