async def test_zero_downtime():
"""Graceful server termination and socket replacement on restarts"""
from signal import SIGINT
from time import monotonic as current_time
async def client():
for _ in range(40):
async with httpx.AsyncClient(uds=SOCKPATH) as client:
r = await client.get("http://localhost/sleep/0.1")
assert r.status_code == 200
assert r.text == f"Slept 0.1 seconds.\n"
def spawn():
command = [
sys.executable,
"-m",
"sanic",
"--unix",
SOCKPATH,
"examples.delayed_response.app",
]
DN = subprocess.DEVNULL
return subprocess.Popen(command,
stdin=DN,
stdout=DN,
stderr=subprocess.PIPE)
try:
processes = [spawn()]
while not os.path.exists(SOCKPATH):
if processes[0].poll() is not None:
raise Exception("Worker did not start properly")
await asyncio.sleep(0.0001)
ino = os.stat(SOCKPATH).st_ino
task = asyncio.get_event_loop().create_task(client())
start_time = current_time()
while current_time() < start_time + 4:
# Start a new one and wait until the socket is replaced
processes.append(spawn())
while ino == os.stat(SOCKPATH).st_ino:
await asyncio.sleep(0.001)
ino = os.stat(SOCKPATH).st_ino
# Graceful termination of the previous one
processes[-2].send_signal(SIGINT)
# Wait until client has completed all requests
await task
processes[-1].send_signal(SIGINT)
for worker in processes:
try:
worker.wait(1.0)
except subprocess.TimeoutExpired:
raise Exception(
f"Worker would not terminate:\n{worker.stderr}")
finally:
for worker in processes:
worker.kill()
# Test for clean run and termination
assert len(processes) > 5
assert [worker.poll() for worker in processes] == len(processes) * [0]
assert not os.path.exists(SOCKPATH)
def manual_record_start(self):
self.session_file_start("manual mode")
hm = pyHook.HookManager()
hm.MouseWheel = self.OnMouseEvent
hm.MouseAllButtonsDown = self.OnMouseEvent
hm.KeyUp = self.OnKeyboardEvent
hm.HookMouse()
hm.HookKeyboard()
# print("--hook-- manual ")
self.mouse_event_count = 0
self.kb_event_count = 0
self.running = True
time_in = current_time()
while self.running:
pythoncom.PumpWaitingMessages()
tsleep(0.001)
hm.UnhookMouse()
hm.UnhookKeyboard()
self.rem_time_secs = round(current_time() - time_in, 2)
print('---unhook---')
self.session_save_stats("~" + str(self.rem_time_secs))
def login(self, username, passwordd):
if self._attempts < 5:
if self.is_logged_in(username):
self._attempts += 1
raise AlreadyLoggedIn(username)
current_usr = User(username, random_number).password
user = self._users.get(current_usr)
if not user:
self._attempts += 1
raise InvalidUsername(username)
if not user.check_password(passwordd):
self._attempts += 1
raise InvalidPassword(passwordd)
user.is_logged_in = True
else:
if not self._cur_time:
# strftime hardly depends on the system time
# and thus makes it vulnerable, deprecated
self._cur_time = str(current_time()).split('.')[0]
if str(current_time()).split('.')[0] != self._cur_time:
self._cur_time = str()
self._attempts = 1
self.login(username, passwordd)
else:
raise LockDownError("You've been locked down for a minute")
def py_1(n):
t_start = current_time()
x, y = [[rand() for i in range(n)] for j in range(2)]
z = []
for i, j in zip(x, y):
z.append(2 * i**2 + 4 * j)
return current_time() - t_start
def record_for(self, secs_to_run):
self.rem_time_secs = secs_to_run
self.mouse_event_count = 0
self.kb_event_count = 0
self.session_file_start(str(self.rem_time_secs) + " s")
hm = pyHook.HookManager()
hm.MouseWheel = self.OnMouseEvent
hm.MouseAllButtonsDown = self.OnMouseEvent
hm.KeyUp = self.OnKeyboardEvent
hm.HookMouse()
hm.HookKeyboard()
# print("--hook-- for " + str(self.rem_time_secs))
timeout = current_time() + self.rem_time_secs
while current_time() < timeout:
pythoncom.PumpWaitingMessages()
tsleep(0.001)
# print(self.mouse_event_count)
# print(self.kb_event_count)
hm.UnhookMouse()
hm.UnhookKeyboard()
# print('---unhook---')
self.session_save_stats(str(self.rem_time_secs))
def get_latency(UDPClientSocket, UDPServerSocket):
first_time = True
time = 0
bytesAddressPair = UDPServerSocket.recvfrom(bufferSize)
message = "{}".format(bytesAddressPair[0])
old_message = message
while True:
bytesAddressPair = UDPServerSocket.recvfrom(bufferSize)
message = "{}".format(bytesAddressPair[0])
if time > 4:
data = generate_data_random(time)
UDPClientSocket.sendto(str.encode(data), (UDP_IP, UDP_SEND_PORT))
if first_time:
start_time = current_time()
else:
data = generate_data_fixed(time)
UDPClientSocket.sendto(str.encode(data), (UDP_IP, UDP_SEND_PORT))
if message[-230:] != old_message[-230:] and not first_time:
return current_time() - start_time
if time > 4:
first_time = False
old_message = message
time += 0.2
async def todo_timer(self):
await self.bot.wait_until_ready()
await self.bot.wait_until_login()
while self is self.bot.get_cog("Todo"):
for entry in self.todo_list:
if self.todo_list[entry][0] != "none" and self.todo_list[
entry][0] != "done":
if self.todo_list[entry][0] < current_time():
self.todo_list[entry][0] = "done"
if self.todo_list[entry][4] == 'on':
self.todo_list[entry][0] = current_time(
) + self.todo_list[entry][5]
elif self.todo_list[entry][4] != 0:
self.todo_list[entry][0] = current_time(
) + self.todo_list[entry][5]
self.todo_list[entry][
4] = self.todo_list[entry][4] - 1
else:
self.todo_list[entry][0] = "done"
try:
if self.todo_list[entry][2] != 0:
if type(self.todo_list[entry][2]) is list:
for channel in self.todo_list[entry][2]:
chnl = self.bot.get_channel(
channel.strip())
await self.bot.send_message(
chnl, self.todo_list[entry][1])
else:
channel = self.bot.get_channel(
self.todo_list[entry][2])
await self.bot.send_message(
channel, self.todo_list[entry][1])
except:
print(
'Unable to send message for todo list entry: %s'
% entry)
self.save_list()
if self.todo_list[entry][3] is True:
if self.bot.notify['type'] == 'msg':
await self.webhook(entry, '')
elif self.bot.notify['type'] == 'ping':
await self.webhook(entry, 'ping')
else:
location = self.bot.log_conf[
'log_location'].split()
server = self.bot.get_server(location[1])
em = discord.Embed(
title='Timer Alert',
color=0x4e42f4,
description=
'Timer for item: **%s** just ran out.' %
entry)
await self.bot.send_message(server.get_channel(
location[0]),
content=None,
embed=em)
await asyncio.sleep(2)
def simulateCS(model_description, fmu, start_time, stop_time, relative_tolerance, start_values, apply_default_start_values, input_signals, output, output_interval, timeout, step_finished):
if output_interval is None:
output_interval = auto_interval(stop_time - start_time)
sim_start = current_time()
is_fmi1 = model_description.fmiVersion == '1.0'
is_fmi2 = model_description.fmiVersion == '2.0'
if is_fmi2:
fmu.setupExperiment(tolerance=relative_tolerance, startTime=start_time)
input = Input(fmu=fmu, modelDescription=model_description, signals=input_signals)
time = start_time
apply_start_values(fmu, model_description, start_values, apply_default_start_values)
# initialize the model
if is_fmi1:
input.apply(time)
fmu.initialize(tStart=time, stopTime=stop_time)
elif is_fmi2:
fmu.enterInitializationMode()
input.apply(time)
fmu.exitInitializationMode()
else:
fmu.enterInitializationMode(tolerance=relative_tolerance, startTime=start_time)
input.apply(time)
fmu.exitInitializationMode()
recorder = Recorder(fmu=fmu, modelDescription=model_description, variableNames=output, interval=output_interval)
n_steps = 0
# simulation loop
while time < stop_time:
if timeout is not None and (current_time() - sim_start) > timeout:
break
recorder.sample(time)
input.apply(time)
fmu.doStep(currentCommunicationPoint=time, communicationStepSize=output_interval)
if step_finished is not None and not step_finished(time, recorder):
break
n_steps += 1
time = n_steps * output_interval
recorder.sample(time, force=True)
fmu.terminate()
return recorder.result()
def wrapper(*args, **kwargs):
sig = "%s(%s, %s)" % (func.__name__, args, kwargs)
print "→ %s" % sig
then = current_time()
result = func(*args, **kwargs)
now = current_time()
print "← %0.6fs :: %s ⇒ %s" % ((now - then), sig, result)
return result
def wrapper(*args, **kwargs):
sig = "%s(%s, %s)" % (func.__name__, args, kwargs)
print "→ %s" % sig
then = current_time()
result = func(*args, **kwargs)
now = current_time()
print "← %0.6fs :: %s ⇒ %s" % ((now - then), sig, result)
return result
def can_request():
if current_time() - StaticVariables.last_reset > MINUTE_IN_SEC:
StaticVariables.requests = 0
StaticVariables.last_reset = current_time()
if StaticVariables.requests < MAX_REQ_PER_MINUTE:
StaticVariables.requests += 1
return True
return False
def __call__(self, embeddings, *args, **kwargs):
assert embeddings.dtype == torch.float32
start_time = current_time()
output = self._process(embeddings, *args, **kwargs)
duration = current_time() - start_time
self._time_log[embeddings.shape[0]].append(duration)
return output
def stop_write_job(args: argparse.Namespace, job_handler: JobHandler) -> None:
stop_time = float(args.stop_after[1])
timeout = int(current_time()) + args.timeout
stop_time = datetime.now() + timedelta(seconds=stop_time)
stop_handler = job_handler.set_stop_time(stop_time)
while not stop_handler.is_done() and not job_handler.is_done():
if int(current_time()) > timeout:
raise ValueError("Timeout.")
def simulateCS(modelDescription, unzipdir, start_time, stop_time, step_size,
start_values, input_signals, output, output_interval, timeout,
fmi_logging):
sim_start = current_time()
guid = modelDescription.guid
modelIdentifier = modelDescription.coSimulation.modelIdentifier
args = {
'guid': guid,
'modelIdentifier': modelIdentifier,
'unzipDirectory': unzipdir,
'instanceName': None,
'logFMICalls': fmi_logging
}
if modelDescription.fmiVersion == '1.0':
fmu = FMU1Slave(**args)
fmu.instantiate("instance1")
apply_start_values(fmu, modelDescription, start_values)
fmu.initialize()
else:
fmu = FMU2Slave(**args)
fmu.instantiate(loggingOn=False)
fmu.setupExperiment(tolerance=None, startTime=start_time)
apply_start_values(fmu, modelDescription, start_values)
fmu.enterInitializationMode()
fmu.exitInitializationMode()
input = Input(fmu=fmu,
modelDescription=modelDescription,
signals=input_signals)
recorder = Recorder(fmu=fmu,
modelDescription=modelDescription,
variableNames=output,
interval=output_interval)
time = start_time
try:
while time < stop_time:
if timeout is not None and (current_time() - sim_start) > timeout:
break
recorder.sample(time)
input.apply(time)
fmu.doStep(currentCommunicationPoint=time,
communicationStepSize=step_size)
time += step_size
recorder.sample(time, force=True)
fmu.terminate()
except Exception as e:
print("Simulation aborted. " + str(e))
finally:
fmu.freeInstance()
return recorder.result()
def start_write_job(write_job: WriteJob) -> Tuple[datetime, float]:
start_handler = JOB_HANDLER.start_job(write_job)
timeout = int(current_time()) + ACK_TIMEOUT
start_time = datetime.now()
while not start_handler.is_done():
if int(current_time()) > timeout:
raise ValueError("Timeout.")
return start_time, timeout
async def safe_get(url, session, wait_time=DEFAULT_WAIT_TIME):
global LAST_REQUEST_TIME
sleep_time = 0
if wait_time != -1 and abs(current_time() - LAST_REQUEST_TIME) < wait_time:
sleep_time = abs(current_time() - LAST_REQUEST_TIME)
LAST_REQUEST_TIME = current_time() + sleep_time
if sleep_time > 0:
await asyncio.sleep(sleep_time)
return await fetch_html(url, session)
def py_2(n):
t_start = current_time()
x, y = [[[rand() for i in range(n)] for j in range(n)] for k in range(2)]
z = []
for i in range(n):
t1 = []
for j in range(n):
t1.append(sum([x[i][s] * y[s][j] for s in range(n)]))
z.append(t1)
return current_time() - t_start
def simulateCS(modelDescription, fmu_kwargs, start_time, stop_time,
start_values, input_signals, output, output_interval, timeout,
callbacks, step_finished):
sim_start = current_time()
if modelDescription.fmiVersion == '1.0':
fmu = FMU1Slave(**fmu_kwargs)
fmu.instantiate("instance1", functions=callbacks)
apply_start_values(fmu, modelDescription, start_values)
fmu.initialize()
else:
fmu = FMU2Slave(**fmu_kwargs)
fmu.instantiate(callbacks=callbacks)
fmu.setupExperiment(tolerance=None, startTime=start_time)
apply_start_values(fmu, modelDescription, start_values)
fmu.enterInitializationMode()
fmu.exitInitializationMode()
input = Input(fmu=fmu,
modelDescription=modelDescription,
signals=input_signals)
recorder = Recorder(fmu=fmu,
modelDescription=modelDescription,
variableNames=output,
interval=output_interval)
time = start_time
# simulation loop
while time < stop_time:
if timeout is not None and (current_time() - sim_start) > timeout:
break
recorder.sample(time)
input.apply(time)
fmu.doStep(currentCommunicationPoint=time,
communicationStepSize=output_interval)
if step_finished is not None and not step_finished(time, recorder):
break
time += output_interval
recorder.sample(time, force=True)
fmu.terminate()
fmu.freeInstance()
return recorder.result()
def load_sequence_signal_images(sequence_name,
image_dirname,
verbose=False,
debug=False):
logging.getLogger().setLevel(
logging.INFO if verbose else logging.DEBUG if debug else logging.ERROR)
signal_images = {}
sequence_filenames = []
for time in xrange(max_time):
filename = sequence_name + "_t" + str(time).zfill(2)
if os.path.exists(image_dirname + "/" + filename + "/" + filename +
"_signal_data.csv"):
sequence_filenames += [filename]
if len(sequence_filenames) > 0:
logging.info("--> Loading sequence images " + sequence_name + " : " +
str([f[-3:] for f in sequence_filenames]))
for filename in sequence_filenames:
data_filename = image_dirname + "/" + filename + "/" + filename + "_signal_data.csv"
file_df = pd.read_csv(data_filename)
file_signals = [
c for c in file_df.columns if (not "center" in c) and (
not "layer" in c) and (not 'Unnamed' in c)
]
for signal_name in file_signals:
if not signal_name in signal_images:
signal_images[signal_name] = {}
start_time = current_time()
logging.info(" --> Loading : " + filename + " " + signal_name)
signal_image_file = image_dirname + "/" + filename + "/" + filename + "_" + signal_name + ".inr.gz"
if os.path.exists(signal_image_file):
signal_images[signal_name][filename] = imread(
signal_image_file)
else:
signal_image_file = image_dirname + "/" + filename + "/" + filename + "_" + signal_name + ".inr"
if os.path.exists(signal_image_file):
signal_images[signal_name][filename] = imread(
signal_image_file)
else:
logging.warn(" --> Unable to find : " + filename +
" " + signal_name)
logging.info(" <-- Loading : " + filename + " " +
signal_name + " [" +
str(current_time() - start_time) + " s]")
return signal_images
async def safe_get(url, session, wait_time=DEFAULT_WAIT_TIME, **kwargs):
global LAST_REQUEST_TIME, SAFE_GET_LOCK
sleep_time = 0
async with SAFE_GET_LOCK:
if current_time() - LAST_REQUEST_TIME < wait_time:
sleep_time = LAST_REQUEST_TIME + wait_time - current_time()
LAST_REQUEST_TIME = LAST_REQUEST_TIME + wait_time
if sleep_time > 0:
#print(sleep_time)
await asyncio.sleep(sleep_time)
print(str.format("Запрашиваем {0}, время:{1}", url, current_time()))
return await session.request(method="GET", url=url, raise_for_status=True, headers={'User-Agent': UserAgent().chrome}, **kwargs)
async def get(url, session, wait_time=DEFAULT_WAIT_TIME, **kwargs):
global LAST_REQUEST_TIME, SAFE_GET_LOCK
sleep_time = 0
async with SAFE_GET_LOCK:
if current_time() - LAST_REQUEST_TIME < wait_time:
sleep_time = LAST_REQUEST_TIME + wait_time - current_time()
LAST_REQUEST_TIME = LAST_REQUEST_TIME + wait_time
if sleep_time > 0:
await asyncio.sleep(sleep_time)
return await session.request(method="GET", url=url, **kwargs)
def main_loop(self, update_interval=1):
"""
Основной цикл часов, в котором время бесконечно обновляется с некоторым интервалом
:param update_interval: Интервал обновления времени
"""
start_time = current_time()
while True:
while current_time() - start_time < update_interval * 0.9:
sleep(update_interval / 10)
while current_time() - start_time < update_interval:
sleep(update_interval / 100)
self.time += current_time() - start_time
start_time = current_time()
self.update()
async def safe_get(url, session, wait_time=DEFAULT_WAIT_TIME, **kwargs):
global LAST_REQUEST_TIME, SAFE_GET_LOCK
sleep_time = 0
if current_time() - LAST_REQUEST_TIME < wait_time:
sleep_time = LAST_REQUEST_TIME + wait_time - current_time()
LAST_REQUEST_TIME = LAST_REQUEST_TIME + wait_time
if sleep_time > 0:
await asyncio.sleep(sleep_time)
print(f"Запрашиваем {url}, время: {datetime.now().strftime('%H:%M:%S')}")
return await session.request(method="GET", url=url, **kwargs)
def get_targets(
logger: logging.Logger,
cache_timeout: int = 12 * 3600
) -> typing.List[str]:
"""ICMP targets.
"""
exists = []
update = True
now = current_time()
try:
with open(TARGET_CAHCE_FILEPATH) as cache:
data = json.loads(cache.read())
if data:
update = now - data["timestamp"] > cache_timeout
exists.extend(data["targets"])
except Exception:
pass
if update or not exists:
exists.extend(get_nodes_list())
if update and exists:
try:
with open(TARGET_CAHCE_FILEPATH, "w") as cache:
cache.write(
json.dumps({
"timestamp": now,
"targets": exists
})
)
except Exception as err:
logger.error(f"Cahce {TARGET_CAHCE_FILEPATH} error:", err)
return exists
async def send(self, data):
"""Writes data with backpressure control."""
await self._can_write.wait()
if self.transport.is_closing():
raise CancelledError
self.transport.write(data)
self._time = current_time()
def check_timeouts(self):
"""
Runs itself periodically to enforce any expired timeouts.
"""
try:
if not self._task:
return
duration = current_time() - self._time
stage = self._http.stage
if stage is Stage.IDLE and duration > self.keep_alive_timeout:
logger.debug("KeepAlive Timeout. Closing connection.")
elif stage is Stage.REQUEST and duration > self.request_timeout:
logger.debug("Request Timeout. Closing connection.")
self._http.exception = RequestTimeout("Request Timeout")
elif stage is Stage.HANDLER and self._http.upgrade_websocket:
logger.debug("Handling websocket. Timeouts disabled.")
return
elif (stage in (Stage.HANDLER, Stage.RESPONSE, Stage.FAILED)
and duration > self.response_timeout):
logger.debug("Response Timeout. Closing connection.")
self._http.exception = ServiceUnavailable("Response Timeout")
else:
interval = (min(
self.keep_alive_timeout,
self.request_timeout,
self.response_timeout,
) / 2)
self.loop.call_later(max(0.1, interval), self.check_timeouts)
return
self._task.cancel()
except Exception:
error_logger.exception("protocol.check_timeouts")
def next(event, window):
window.states_log += "%s,CongestionAvoidance,%s,%s,%s\n" % ((current_time() - window.start_time), event.name, event.data, window.dup_ack_count)
if event.name == "timeout":
window.ssthresh = ceil(window.cwnd / 2)
window.update_cwnd(window.MSS)
window.dup_ack_count = 0
window.retansmit_missing_segment(event.data)
return State.slow_start
elif event.name == "new_ack":
window.update_cwnd(window.cwnd + (window.MSS * round(window.MSS / window.cwnd, -1)))
window.update_trace(event.data)
window.transmit_as_allowed()
return State.congestion_avoidance
elif event.name == "dup_ack":
return State.congestion_avoidance
elif event.name == "triple_ack":
window.ssthresh = ceil(window.cwnd / 2)
window.update_cwnd(window.ssthresh + 3 * window.MSS)
window.retansmit_missing_segment(event.data)
return State.fast_recovery
raise Exception("Slow start got weird event")
async def is_not_sync(self, text):
"""Returns diff between current time and when the song started"""
text = BeautifulSoup(text, 'lxml')
text = text.find(id="time")
time = current_time() - (int(text.attrs['time'])/1000) # ms in html whereas time() is in s
time -= 25 # don't know why but it always gives about 30 secs before the song started
return time
def reset(self, phase='Train'):
mission_start_time = current_time('%Y%m%d%H%M%S')
if self.traci_env is not None:
try:
self.traci_env.close()
except KeyError:
print("Traci is not running")
if phase is 'Train':
self.sumo_cfg = SumoCfg(self.data_dir, self.task_name,
self.xnumber, self.ynumber,
self.xlength, self.ylength, self.net_type, self.tls_type,
self.rouprob, self.epoch_steps)
print("Sumo_cfg created")
self.sumo_cfg.make()
sumo_cmd, run_env = self.sumo_cfg.get_start_cmd(mission_start_time, gui=self.gui)
print("Try Raise Traci")
self.traci_env = TraciEnv(self.port, label=self.thread_label)
# print("Get here")
try:
self.traci_env.start(sumo_cmd)
except:
self.sumo_cfg.make()
self.traci_env.start(sumo_cmd)
# traci.switch(self.thread_label)
# traci.init(self.port)
# time.sleep(3)
self.tls = self.traci_env.tls
self.current_epoch += 1
s_0 = self.parse_log(self.step(None)[0])
return s_0
def stop_write_job(stop: float, start_time: datetime, timeout: float) -> None:
stop_time = start_time + timedelta(seconds=stop)
stop_handler = JOB_HANDLER.set_stop_time(stop_time)
while not stop_handler.is_done() and not JOB_HANDLER.is_done():
if int(current_time()) > timeout:
raise ValueError("Timeout.")
def add_metric(path, dbname, time, data):
with sqlite3.connect(os.path.join(path, dbname)) as conn:
c = conn.cursor()
c.execute("INSERT INTO metrics VALUES(?, ?)", (time, json.dumps(data)))
# When data are pulled from temboard server, we need to keep 6 hours of
# data history for recovery.
time_limit = current_time() - (60 * 60 * 6)
c.execute("DELETE FROM metrics WHERE time < ?", (time_limit, ))
def set_time_and_stop() -> None:
stop_time = input("Stop time in seconds = ")
try:
stop_time = float(stop_time)
timeout = int(current_time()) + ACK_TIMEOUT
stop_write_job(stop_time, datetime.now(), timeout)
except ValueError:
# The CLI will simply continue.
print("Input should be a float.")
def __exit__(self, exc_type, exc_val, exc_tb):
self.t1 = current_time()
if self.pretty_time:
print("done (in {})".format(
self.format_time(self.t1 - self.t0, self.n, self.pad)))
else:
print("done (in {} seconds).".format(self.t1 - self.t0, self.n,
self.pad))
stdout.flush()
def start(self, message=None):
"""
Start this timer (possibly again) with an optional message
:param message: optional message (overrides message on init)
:return: self, for chaining
"""
if message is not None:
self.message = message
self.start_time = current_time()
return self
def update_estimate(s, key):
if s.sampling and s.sampling[0] == key:
if s.sent[key].ack_count == 1:
sample_RTT = current_time() - s.sampling[1]
if s.estimated_RTT:
s.estimated_RTT = 0.875 * s.estimated_RTT + .125 * sample_RTT
else:
s.estimated_RTT = sample_RTT
s.dev_RTT = 0.75 * s.dev_RTT + .25 * abs(sample_RTT - s.estimated_RTT)
s.timeout_length = s.estimated_RTT + 4 * s.dev_RTT
s.sampling = None
async def todo_timer(self):
await self.bot.wait_until_ready()
while self is self.bot.get_cog("Todo"):
for entry in self.todo_list:
if self.todo_list[entry][0] != "none" and self.todo_list[entry][0] != "done":
if self.todo_list[entry][0] < current_time():
self.todo_list[entry][0] = "done"
if self.todo_list[entry][4] == 'on':
self.todo_list[entry][0] = current_time() + self.todo_list[entry][5]
elif self.todo_list[entry][4] != 0:
self.todo_list[entry][0] = current_time() + self.todo_list[entry][5]
self.todo_list[entry][4] = self.todo_list[entry][4]-1
else:
self.todo_list[entry][0] = "done"
try:
if self.todo_list[entry][2] != 0:
if type(self.todo_list[entry][2]) is list:
for channel in self.todo_list[entry][2]:
chnl = self.bot.get_channel(int(channel.strip()))
await chnl.send(self.todo_list[entry][1])
else:
channel = self.bot.get_channel(int(self.todo_list[entry][2]))
await channel.send(self.todo_list[entry][1])
except:
print('Unable to send message for todo list entry: %s' % entry)
self.save_list()
if self.todo_list[entry][3] is True:
if self.bot.notify['type'] == 'msg':
await self.webhook(entry, '')
elif self.bot.notify['type'] == 'ping':
await self.webhook(entry, 'ping')
else:
location = self.bot.log_conf['log_location'].split()
guild = self.bot.get_guild(int(location[1]))
em = discord.Embed(title='Timer Alert', color=0x4e42f4,
description='Timer for item: **%s** just ran out.' % entry)
await guild.get_channel(int(location[0])).send(content=None, embed=em)
await asyncio.sleep(2)
def clear_stale_downloads(cls):
""" Eliminate stale downloads
This method takes very little time so can be executed willy-nilly.
"""
now = current_time()
for result in list(Tools.stored_files.values()):
if now - result.createtime > MAX_RETRIVE_AGE:
try:
# Let the garbage collector get the temp file
del cls.stored_files[result.filename]
except KeyError: # Could happen due to multithreading.
pass
def end(self, message=None):
"""
End this timer (possibly again) with an optional message
:param message: optional message (overrides message on init)
:return: self, for chaining
"""
if message is not None:
self.message = message
self.end_time = current_time()
self.iterations += 1
self.running_total += self.duration
return self
def update_time(self, time, max_time):
if self.notify_at:
timestamp = current_time() + max_time - time
date = datetime.fromtimestamp(timestamp)
label = _("Notify at %02d:%02d") % (date.hour, date.minute)
else:
diff_min = ceil((max_time - time) / 60.0)
if diff_min > 60:
hours = diff_min / 60
minutes = diff_min % 60
label = _("Rest time: %d:%02d") % (hours, minutes)
else:
label = _("Rest time: %d min") % (diff_min)
self.time_item.set_label(label)
def putval_future(self, identifier, values, interval=None, time=None):
buf = bytearray(b'PUTVAL ')
buf += identifier.encode('ascii')
if interval is not None:
buf += ' interval={0:d}'.format(interval).encode('ascii')
for tup in values:
if time is None:
time = current_time()
lst = [str(int(time))]
for val in tup:
if val is None:
lst.append('U')
else:
lst.append(str(float(val)))
buf += b' '
buf += ':'.join(lst).encode('ascii')
return self.channel().request(buf)
def putnotif(self, message, severity='warning', time=None, host=None,
plugin=None, plugin_instance=None, type=None, type_instance=None):
buf = bytearray(b'PUTNOTIF')
buf += ' message="{}"'.format(message.replace('"',"'")).encode('ascii')
buf += ' severity={}'.format(severity).encode('ascii')
if time is None:
time = int(current_time())
buf += ' time={}'.format(time).encode('ascii')
if host:
' host={}'.format(host).encode('ascii')
if plugin:
' plugin={}'.format(plugin).encode('ascii')
if plugin_instance:
' plugin_instance={}'.format(plugin_instance).encode('ascii')
if type:
' type={}'.format(type).encode('ascii')
if type_instance:
' type_instance={}'.format(type_instance).encode('ascii')
return self.channel().request(buf).get()
def main(argv):
host, port, file_name, loss_file = parse_input_sender(argv)
udp, destination = setup_socket_sender(host, port)
t = TCPStateMachine(file_name, udp, destination)
while not t.done():
rlist, _, _ = select([udp], [], [], t.window.timeout_length)
if len(rlist) == 0:
t.run(Event("timeout", None))
else:
for sock in rlist:
ack, addr = sock.recvfrom(4096)
t.run(Event("ack", ack))
with open("./trace_results/trace-%s%s" % (file_name, loss_file), "w") as f:
f.writelines(t.window.trace_file)
with open("./cwnd_results/cwnd-%s%s" % (file_name, loss_file), "w") as f:
f.writelines(t.window.cwnd_file)
with open("./state_logs/states-%s%s" % (file_name, loss_file), "w") as f:
f.writelines(t.window.states_log)
print t.ack_count * 100 * 8 / (current_time() - t.window.start_time)
def next(event, window):
window.states_log += "%s,FastRecovery,%s,%s,%s\n" % ((current_time() - window.start_time), event.name, event.data, window.dup_ack_count)
if event.name == "timeout":
window.ssthresh = ceil(window.cwnd / 2)
window.update_cwnd(window.MSS)
window.dup_ack_count = 0
window.retansmit_missing_segment(event.data)
return State.slow_start
elif event.name == "new_ack":
window.update_cwnd(window.ssthresh)
window.update_trace(event.data)
window.transmit_as_allowed()
return State.congestion_avoidance
elif event.name == "dup_ack":
window.update_cwnd(window.cwnd + window.MSS)
return State.fast_recovery
elif event.name == "triple_ack":
window.update_cwnd(window.cwnd + window.MSS)
return State.fast_recovery
def _get_cache(ttl, cache_path):
'''
If url contains valid cache, returns it, else returns empty list.
'''
# Check if we have a valid cached version.
try:
cached_time = os.path.getmtime(cache_path)
except OSError:
return []
if current_time() - cached_time < ttl:
log.debug('%s is less than ttl', cache_path)
try:
with open(cache_path) as json_file:
loaded_json = json.load(json_file)
return loaded_json
except IOError:
return []
except ValueError:
log.error('%s was not json formatted', cache_path)
return []
else:
log.debug('%s was older than ttl', cache_path)
return []
def recreate_old_hist_files(run_dict, sim_dict, g4bl, exec_d4_d5, script_name="make_plot_files.py", force=False):
"""
Check if any of the files are older than their generating script, or
don't exist. If this is true for any re-run the script.
"""
# Get lists of file names to check if any need regenerating
run_files = [(r, run_dict[r]['file_name']) for r in run_dict if 'file_name' in run_dict[r]]
sim_files = [(s, sim_dict[s]['file_name']) for s in sim_dict if 'file_name' in sim_dict[s]]
get_age = lambda path: current_time() - path_last_mod_time(path)
script_age = get_age(script_name)
needs_regen = lambda entry: not path_exists(entry[1]) or script_age < get_age(entry[1])
# regen will be a list of bools showing whether the file exists or is too old
sim_files_to_regen=filter(needs_regen, sim_files)
run_files_to_regen=filter(needs_regen, run_files)
for run_id, path in run_files_to_regen:
print "Regenerating run file:", run_id, "path", path
# generate_data_histograms(run_id)
for sim_id, path in sim_files_to_regen:
print "Regenerating sim file:", sim_id, "path", path
def __init__(s, **kwargs):
super(Window, s).__init__()
s.MSS = kwargs["MSS"]
s.ssthresh = kwargs["ssthresh"]
s.max_cwnd = kwargs["max_cwnd"]
s.sequence_counter = SequenceCounter(kwargs["max_sequence_number"])
s.timeout_length = kwargs["timeout_length"]
s.cwnd = s.MSS
s.destination = kwargs["destination"]
s.dup_ack_count = 0
s.no_more_segments = False
s.sent = OrderedDict()
s.state_machine = kwargs["state_machine"]
s.get_segment = None
s.chunks = iter(kwargs["chunks"])
s.udp = kwargs["udp"]
s.cwnd_file = ""
s.trace_file = ""
s.start_time = current_time()
s.estimated_RTT = None
s.dev_RTT = 0.1
s.sampling = None
s.last_ack = None
s.states_log = ""
def flash(self, *jets):
"""
Turns the specified jets on, then turns them off after the number of
seconds specified by the flash_time object on this service. The flash
time can be adjusted by calling set_flash_time or update_flash_time.
I'll probably add a mechanism later for specifying a custom flash time
when calling flash.
"""
# Figure out what time the jets should turn off
off_time = current_time() + self.flash_time
for n in jets:
# Cancel all manual scheduled events for this jet
self.loop.cancel((MANUAL, n))
# Turn the jet on
self.states[n] = True
# Schedule an event to turn this jet off. We assign it the
# categories MANUAL and (MANUAL, n); the former is used to cancel
# all manual events when switching to automatic mode, and the
# latter is used by other calls to flash (and calls to on/off) to
# cancel other flash-related events for the jet.
self.loop.schedule(Partial(self.off, n), off_time,
MANUAL, (MANUAL, n))
# Write the new states to the parallel port
self.flush()
def update_cwnd(s, cwnd):
s.cwnd = cwnd
s.cwnd_file += "%s %s\n" % (current_time() - s.start_time, cwnd)
def update_trace(s, seq):
s.trace_file += "%s %s\n" % (current_time() - s.start_time, seq)
def find_ring_areas(sorted_ring_list, center, svgfile):
# This codeblock creates a one pixel by one pixel square Ring object to
# act as the core - it is recorded in CP. note: perimeter should be found
# as a path and treated at a ring already
csd = centerSquare(center)
csd_path = parse_path(csd)
if not isCCW(csd_path, center):
csd_path = reversePath(csd_path)
# path_string, color, brooke_tag, center
center_square = Ring(csd, colordict['center'], 'not recorded', Radius(center), csd_path)
# Converts the sorted_ring_list into a CP_Boolset of
# complete rings each containing their IRs
completeRing_CPB = CP_BoolSet()
innerRing = center_square
innerRing_index = -1
for ring_index, ring in enumerate(sorted_ring_list):
# when next closed ring found create CompleteRing object,
# then set all inbetween rings to be IRs
if ring.isClosed():
completeRing_CPB.append(CompleteRing(innerRing, ring))
for inc_ring in sorted_ring_list[innerRing_index+1:ring_index]:
ir = IncompleteRing(inc_ring)
ir.set_inner(innerRing)
ir.set_outer(ring)
completeRing_CPB.cpUpdate(CompleteRing(ir.innerCR_ring, ir.outerCR_ring, ir))
innerRing = ring
innerRing_index = ring_index
# Check (once again) that the last sort-suggested
# boundary is closed and correctly colored
bdry_ring = sorted_ring_list[-1]
if bdry_ring.color != colordict['boundary'] or not bdry_ring.isClosed():
###DEBUG Why is this necessary? Isn't this fixed earlier?
if sorted_ring_list[-1] == max(sorted_ring_list, key=lambda r: r.maxR):
sorted_ring_list[-1].color = colordict['boundary']
else:
raise Exception("Last ring in sorted sorted_ring_list was not "
"closed... this should be outer perimeter.")
# identify the center square created earlier as the core
completeRing_CPB[0].isCore = True
basic_output_on.dprint("All complete_ring objects created and all "
"incomple_ring objects created (and stored inside "
"the appropriate complete_ring object).")
# complete the incomplete rings
CP_start_time = start_time_ring_completion = current_time()
for count,cp in enumerate(completeRing_CPB):
if count:
CP_start_time = current_time()
try:
cp.completeIncompleteRings()
except:
if outputTroubledCPs:
paths = ([cp.inner.path, cp.outer.path] +
[ir.ring.path for ir in cp.ir_boolset] +
[sorted_ring_list[-1].path])
path_colors = ([cp.inner.color, cp.outer.color] +
[ir.ring.color for ir in cp.ir_boolset] +
[colordict['boundary']])
center_line = Line(cp.inner.center-1,cp.inner.center+1)
svgname = os_path.join(output_directory_debug,"trouble_"+svgfile)
disvg(paths,path_colors,lines=[center_line],filename=svgname)
print("Simplified SVG created containing troublesome section "
"(troublesome incomplete ring colored {}) and saved "
"to:\n{}".format(colordict['safe1'], svgname))
raise
mes = ("{}/{} complete rings finished. This CP = {} | Total ET = {}"
"".format(count + 1,
len(completeRing_CPB),
format_time(current_time()-CP_start_time),
format_time(current_time()-start_time_ring_completion)))
showCurrentFilesProgress.dprint(mes)
outputFile = os_path.join(output_directory, svgfile + '_completeRing_info.csv')
with open(outputFile, "wt") as out_file:
out_file.write("complete ring index, type, # of IRs contained, minR, "
"maxR, aveR, area, area Ignoring IRs\n")
cp_index = 0
for cp in completeRing_CPB:
cp_index += 1
out_file.write(cp.info(cp_index,colordict) + '\n')
# Create SVG showing areas (i.e. showing completed paths)
if create_SVG_showing_area_paths:
basic_output_on.dprint("Attempting to create SVG showing completed "
"paths used for area computation...", 'nr')
svgpaths = []
svgcolors = []
for cp in completeRing_CPB:
svgpaths.append(cp.inner.path)
svgcolors.append(cp.inner.color)
for ir in cp.ir_boolset:
svgpaths.append(ir.completed_path)
svgcolors.append(ir.ring.color)
if cp.outer.color == colordict['boundary']:
svgpaths.append(cp.outer.path)
svgcolors.append(cp.outer.color)
tmp = svgfile[0:len(svgfile)-4] + "_area_paths" + ".svg"
svgname = os_path.join(output_directory_debug, tmp)
wsvg(svgpaths, svgcolors, filename=svgname)
basic_output_on.dprint("Done.")
def start_sample(s, key):
if not s.sampling:
s.sampling = (key, current_time())
import socket
import serial
import time
from time import time as current_time
import csv
import struct
from time import sleep
from matplotlib import pyplot
startTime = current_time()
############################### Bluetooth and PC comm ###################################
port = "COM4" # bluetooth comm port
baud = 9600 # baud rate of UART
ser = serial.Serial(port, baud, serial.EIGHTBITS, serial.PARITY_NONE, serial.STOPBITS_ONE, timeout=1)
###########################################################################################
############################### PNA and PC comm ###################################
# connect to network analyzer server
server_address = ('165.91.209.113',5025)
pna = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
pna.connect(server_address)
pna.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
print("PNA network connection: ok")
###########################################################################################
############################### PNA setup ###################################
# python 3 requires encoding string for bit encoding
pna.send('CALC:PAR:DEL:ALL\n'.encode()) # delete all measurement
sleep(1) # wait for 1 second
def __init__(self, fileobj, name):
self.fileobj = fileobj
self.filename = name
self.createtime = current_time()
envs = [(SumoEnv(task_cfg_dir, 'async_%d'%i,
xnumber=3, ynumber=3, gui=False), str(i)) for i in range(concurrent)]
print("ENVS Created")
# pool = Pool()
# pool.map(actor_learner_thread_step, envs)
# pool.close()
# pool.join()
executor = Executor(max_workers=cpu_count())
runs = []
for i in range(concurrent):
runs.append(executor.submit(actor_learner_thread_init, envs[i]))
for e in runs:
print(e.step())
if __name__ == '__main__':
mission_start_time = current_time('%Y%m%d%H%M%S')
task_cfg_dir = os.path.join(CFG_DIR, 'a3c')
async_rl(task_cfg_dir, 4)
# s_t = task.get_initial_state()
# s_t1, r_t, terminated, info = task.step(1)
# print(s_t1)
# print(s_t1.shape)
# print("S_t1%s" % str(s_t1))
# print("R_t1%s" % str(r_t))
# print("Terminated:%s" % str(terminated))
# sumo, x_0 = env.reset()
# print(x_0)
# options = get_options()
# # Generate a 3x3 intersections network
# env_1x1_static = SumoCfg(data_dir, '1x1_static', 1, 1, tls_type='static')
def remove_intersections_from_rings(rings):
from options4rings import intersection_removal_progress_output_on
from time import time as current_time
from andysmod import n_choose_k, format_time
[r.record_wasClosed() for r in rings] # record the current closure status
#for output
num_segments_in_ring_list = sum(len(r.path) for r in rings)
num_seg_pairs2check = n_choose_k(num_segments_in_ring_list, 2)
num_seg_pairs_checked = 0
current_percent_complete = 0
start_time = current_time()
count = 0
overlappingClosedRingPairs = []
for i in xrange(len(rings)):
iring = rings[i]
ipath = iring.path
new_ipath = ipath
iclosed = iring.wasClosed
iringupdated = False
num_segs_in_ipath = len(ipath) # for progress output
for j in xrange(i+1, len(rings)):
if rings[j].maxR < rings[i].minR or rings[i].maxR < rings[j].minR:
continue
jring = rings[j]
jpath = jring.path
new_jpath = jpath
jclosed = jring.wasClosed
jringupdated = False
num_segs_in_jpath = len(jpath) #for progress output
# while loop to remove intersections between iring and jring (if any exist)
run_again = True
maxits = 20
its = 0
while run_again and its < maxits:
try:
args = (new_ipath, new_jpath, iclosed, jclosed)
res = remove_intersections(*args, iringupdated=iringupdated, jringupdated=jringupdated)
new_ipath, new_jpath, iringupdated, jringupdated, run_again = res
except ClosedRingsOverlapError:
overlappingClosedRingPairs.append((i, j))
run_again = False
pass
its += 1
# raise Exception if while loop terminateded due to reaching max allowed iteratations
if its >= maxits:
# remove_intersections_from_rings([iring, jring])
# print(iring.xml)
# print(jring.xml)
raise Exception("Max iterations reached while removing intersections. Either the above two rings have over 20 intersections or this is a bug.")
# Output progess
if intersection_removal_progress_output_on.b:
num_seg_pairs_checked += num_segs_in_jpath*num_segs_in_ipath
if 100 * num_seg_pairs_checked / num_seg_pairs2check > int(100 * current_percent_complete):
current_percent_complete = num_seg_pairs_checked / num_seg_pairs2check
time_elapsed = current_time() - start_time
estimated_time_remaining = (1-current_percent_complete) * time_elapsed / current_percent_complete
stuff = (int(100 * current_percent_complete),
format_time(estimated_time_remaining),
format_time(time_elapsed))
mes = ("[%s%% complete || Est. Remaining Time = %s || "
"Elapsed Time = %s]\r" % stuff)
intersection_removal_progress_output_on.dprint(mes)
# update jring if jpath was trimmed
if jringupdated:
jring.updatePath(new_jpath)
count += 1
# update iring if ipath was trimmed
if iringupdated:
iring.updatePath(new_ipath)
count += 1
return rings, count, overlappingClosedRingPairs
def init_report(self):
report = Report()
self.report = report
self.starttime = current_time()
report.title = "Applying {}".format(self.name)
return report
def finalize_report(self, resultfile):
self.report.footer = "Complete in {}".format(humantime(
current_time() - self.starttime))
if self.report.processed_bytes > 0:
self.report.result = resultfile
def fix_svg(ring_list, center, svgfile):
# Discard inappropriately short rings
from options4rings import appropriate_ring_length_minimum
opt.basic_output_on.dprint("\nChecking for inappropriately short "
"rings...",'nr')
tmp_len = len(ring_list)
short_rings = [idx for idx, ring in enumerate(ring_list) if
ring.path.length() < appropriate_ring_length_minimum]
opt.basic_output_on.dprint("Done (%s inappropriately short rings "
"found)."%len(short_rings))
if short_rings:
if opt.create_svg_highlighting_inappropriately_short_rings:
opt.basic_output_on.dprint("\nCreating svg highlighting "
"inappropriately short rings...",'nr')
paths = [parse_path(r.string) for r in ring_list]
colors = [r.color for r in ring_list]
nodes = [ring_list[idx].path.point(0.5) for idx in short_rings]
center_line = [Line(center-1,center+1)]
tmp = svgfile[0:len(svgfile)-4] + "_short-rings.svg"
shortrings_svg_filename = os_path.join(opt.output_directory, tmp)
disvg(paths + [center_line], colors + [opt.colordict['center']],
nodes=nodes, filename=shortrings_svg_filename)
args = appropriate_ring_length_minimum, shortrings_svg_filename
mes = ("Done. SVG created highlighting short rings by placing a "
"node at each short ring's midpoint. Note: since these "
"rings are all under {} pixels in length, they may be hard "
"to see and may even be completely covered by the node. "
"SVG file saved to:\n{}").format(*args)
opt.basic_output_on.dprint(mes)
if opt.dont_remove_closed_inappropriately_short_rings:
shortest_ring_length = min([r.path.length() for r in
[ring_list[k] for k in short_rings if
ring_list[k].isClosed()]])
open_short_rings = [idx for idx in short_rings if
not ring_list[idx].isClosed()]
num_short_and_closed = len(short_rings)-len(open_short_rings)
if num_short_and_closed:
sug_tol = (opt.tol_isNear * shortest_ring_length /
opt.appropriate_ring_length_minimum)
warn("{} inappropriately short closed rings detected (and not "
"removed as "
"dont_remove_closed_inappropriately_short_rings = True). "
" You should probably decrease tol_isNear to something "
"less than {} and restart this file."
"".format(num_short_and_closed, sug_tol))
short_rings = open_short_rings
if opt.remove_inappropriately_short_rings:
opt.basic_output_on.dprint("\nRemoving inappropriately short "
"rings...",'nr')
ring_list = [ring for idx,ring in enumerate(ring_list) if
idx not in short_rings]
opt.basic_output_on.dprint("Done (%s inappropriately short rings "
"removed)."%(tmp_len - len(ring_list)))
else:
warn("{} inappropriately short rings were found, but "
"remove_inappropriately_short_rings is set to False."
"".format(len(ring_list)))
print("")
# Remove very short segments from rings
def _remove_seg(path, _seg_idx, _newjoint):
_new_path = [x for x in path]
pathisclosed = path[-1].end == path[0].start
# stretch next segment
if _seg_idx != len(path) - 1 or pathisclosed:
old_bpoints = _new_path[(_seg_idx + 1) % len(path)].bpoints()
new_bpoints = (_newjoint,) + old_bpoints[1:]
_new_path[(_seg_idx + 1) % len(path)] = bezier_segment(*new_bpoints)
# stretch previous segment
if _seg_idx != 0 or pathisclosed:
old_bpoints = _new_path[(_seg_idx - 1) % len(path)].bpoints()
new_bpoints = old_bpoints[:-1] + (_newjoint,)
_new_path[(_seg_idx - 1) % len(path)] = bezier_segment(*new_bpoints)
# delete the path to be removed
del _new_path[_seg_idx]
return _new_path
if opt.min_relative_segment_length:
for r_idx, r in enumerate(ring_list):
min_seg_length = r.path.length() * opt.min_relative_segment_length
new_path = [s for s in r.path]
its = 0
flag = False
while its < len(r.path):
its += 1
for seg_idx, seg in enumerate(new_path):
if seg.length() < min_seg_length:
flag = True
if seg == new_path[-1] and not r.path.isclosed():
newjoint = seg.end
elif seg == new_path[0].start and not r.path.isclosed():
newjoint = seg.start
else:
newjoint = seg.point(0.5)
new_path = _remove_seg(new_path, seg_idx, newjoint)
break
else:
break
if flag:
ring_list[r_idx].path = Path(*new_path)
# Close approximately closed rings
for r in ring_list:
r.fixClosure()
# Palette check
from svg2rings import palette_check
ring_list = palette_check(ring_list)
# Check for and fix inconsistencies in closedness of rings
from svg2rings import closedness_consistency_check
ring_list = closedness_consistency_check(ring_list)
# Remove self-intersections in open rings
if opt.remove_self_intersections:
rsi_start_time = current_time()
fixable_count = 0
print("Checking for self-intersections...")
bad_rings = []
for r_idx, r in enumerate(ring_list):
if r.path.end == r.path.start:
continue
first_half = r.path.cropped(0, 0.4)
second_half = r.path.cropped(0.6, 1)
middle_peice = r.path.cropped(0.4, 0.6)
inters = first_half.intersect(second_half)
if inters:
if len(inters) > 1:
Ts = [info1[0] for info1, info2 in inters]
bad_rings.append((r_idx, Ts))
continue
else:
fixable_count += 1
T1, seg1, t1 = inters[0][0]
T2, seg2, t2 = inters[0][1]
if not opt.force_remove_self_intersections:
print("Self-intersection detected!")
greenpart = first_half.cropped(0, T1)
redpart = second_half.cropped(T2, 1)
new_path = [seg for seg in first_half.cropped(T1, 1)]
new_path += [seg for seg in middle_peice]
new_path += [seg for seg in second_half.cropped(0, T2)]
new_path = Path(*new_path)
if opt.force_remove_self_intersections:
dec = True
else:
print("Should I remove the red and green sections?")
disvg([greenpart, new_path, redpart],
['green', 'blue', 'red'],
nodes=[seg1.point(t1)])
dec = inputyn()
if dec:
r.path = new_path
print("Path cropped.")
else:
print("OK... I hope things work out for you.")
if bad_rings:
paths = [r.path for r in ring_list]
colors = [r.color for r in ring_list]
center_line = Line(center-1, center+1)
nodes = []
for r_idx, Ts in bad_rings:
for T in Ts:
nodes.append(ring_list[r_idx].path.point(T))
colors[r_idx] = opt.colordict['safe2']
node_colors = [opt.colordict['safe1']] * len(nodes)
tmp = svgfile[0:len(svgfile)-4] + "_SelfIntersections.svg"
fixed_svg_filename = os_path.join(opt.output_directory, tmp)
disvg(paths + [center_line],
colors + [opt.colordict['center']],
nodes=nodes,
node_colors=node_colors,
filename=fixed_svg_filename)
tmp_mes = (
"Some rings contained multiple self-intersections, you better "
"take a look. They must be fixed manually (in Inkscape or "
"Adobe Illustrator). An svg has been output highlighting the "
"rings which must be fixed manually (and the points where the "
"self-intersections occur). Fix the highlighted rings and "
"replace your old svg with the fixed one (the colors/circles "
"used to highlight the intersections will be fixed/removed "
"automatically).\n Output svg saved to:\n"
"{}".format(fixed_svg_filename))
raise Exception(tmp_mes)
et = format_time(current_time()-rsi_start_time)
print("Done fixing self-intersections ({} detected in {})."
"".format(fixable_count, et))
# Check that all rings are smooth (search for kinks and round them)
if opt.smooth_rings:
print("Smoothing paths...")
bad_rings = []
for r_idx, r in enumerate(ring_list):
args = (r.path, opt.maxjointsize, opt.tightness, True)
r.path = smoothed_path(*args)
still_kinky_list = kinks(r.path)
if still_kinky_list:
bad_rings.append((r_idx, still_kinky_list))
# If unremovable kinks exist, tell user to remove them manually
if opt.ignore_unremovable_kinks or not bad_rings:
opt.rings_may_contain_unremoved_kinks = False
else:
paths = [r.path for r in ring_list]
colors = [r.color for r in ring_list]
center_line = Line(center-1, center+1)
nodes = []
for r_idx, kink_indices in bad_rings:
for idx in kink_indices:
kink = ring_list[r_idx].path[idx].start
nodes.append(kink)
colors[r_idx] = opt.colordict['safe2']
node_colors = [opt.colordict['safe1']] * len(nodes)
tmp = svgfile[0:len(svgfile)-4] + "_kinks.svg"
fixed_svg_filename = os_path.join(opt.output_directory, tmp)
disvg(paths + [center_line],
colors + [opt.colordict['center']],
nodes=nodes,
node_colors=node_colors,
filename=fixed_svg_filename)
raise Exception("Some rings contained kinks which could not be "
"removed automatically. "
"They must be fixed manually (in inkscape or "
"adobe illustrator). An svg has been output "
"highlighting the rings which must be fixed "
"manually (and the points where the "
"kinks occur). Fix the highlighted "
"rings and replace your old svg with the fixed "
"one (the colors/circles used to highlight the "
"kinks will be fixed/removed automatically).\n"
"Output svg saved to:\n"
"%s" % fixed_svg_filename)
print("Done smoothing paths.")
# Check for overlapping ends in open rings
if opt.check4overlappingends:
print("Checking for overlapping ends (that do not intersect)...")
bad_rings = []
for r_idx, r in enumerate(ring_list):
if r.path.isclosed():
continue
startpt = r.path.start
endpt = r.path.end
path_wo_start = r.path.cropped(.1, 1)
path_wo_end = r.path.cropped(0, .9)
start_is_outwards = isPointOutwardOfPath(startpt, path_wo_start)
end_is_outwards = isPointOutwardOfPath(endpt, path_wo_end)
if start_is_outwards:
bad_rings.append((r_idx, 0, start_is_outwards))
if end_is_outwards:
bad_rings.append((r_idx, 1, end_is_outwards))
if bad_rings:
paths = [r.path for r in ring_list]
colors = [r.color for r in ring_list]
center_line = Line(center-1, center+1)
for r_idx, endbin, segts in bad_rings:
colors[r_idx] = opt.colordict['safe2']
# indicator lines
indicator_lines = []
for r_idx, endbin, segts in bad_rings:
bad_path = ring_list[r_idx].path
endpt = bad_path.point(endbin)
for bad_seg_idx, bad_t in segts:
bad_pt = bad_path[bad_seg_idx].point(bad_t)
indicator_lines.append(Line(bad_pt, endpt))
indicator_cols = [opt.colordict['safe1']] * len(indicator_lines)
tmp = svgfile[0:len(svgfile)-4] + "_OverlappingEnds.svg"
fixed_svg_filename = os_path.join(opt.output_directory, tmp)
disvg(paths + [center_line] + indicator_lines,
colors + [opt.colordict['center']] + indicator_cols,
filename=fixed_svg_filename)
bad_ring_count = len(set(x[0] for x in bad_rings))
tmp_mes = (
"Detected {} rings with overlapping (but not intersecting) "
"ends. They must be fixed manually (e.g. in Inkscape or "
"Adobe Illustrator). An svg has been output highlighting the "
"rings which must be fixed manually. Fix the highlighted "
"rings, remove the,indicator lines added, and replace your "
"old svg with the fixed one (the colors used to highlight the "
"intersections will be fixed automatically).\nIf the "
"indicator lines do not appear to be normal to the ring, this "
"is possibly caused by a very short path segment. In this "
"case, you may want to try increasing "
"min_relative_segment_length in options and running again.\n"
"Output svg saved to:\n"
"{}".format(bad_ring_count, fixed_svg_filename))
raise Exception(tmp_mes)
print("Done checking for overlapping ends.")
# Trim paths with high curvature (i.e. curly) ends
if opt.remove_curly_ends:
print("Trimming high curvature ends...")
for ring in ring_list:
if ring.isClosed():
continue
# 90 degree turn in distance of opt.tol_isNear
tol_curvature = 2**.5 / opt.tol_isNear #####Tolerance
# Find any points within tol_isNear of start and end that have
# curvature equal to tol_curvature, later we'll crop them off
from svgpathtools import real, imag
from svgpathtools.polytools import polyroots01
def icurvature(seg, kappa):
"""returns a list of t-values such that 0 <= t<= 1 and
seg.curvature(t) = kappa."""
z = seg.poly()
x, y = real(z), imag(z)
dx, dy = x.deriv(), y.deriv()
ddx, ddy = dx.deriv(), dy.deriv()
p = kappa**2*(dx**2 + dy**2)**3 - (dx*ddy - ddx*dy)**2
return polyroots01(p)
# For first segment
startseg = ring.path[0]
ts = icurvature(startseg, tol_curvature)
ts = [t for t in ts if startseg.length(t1=t) < opt.tol_isNear]
if ts:
T0 = ring.path.t2T(0, max(ts))
else:
T0 = 0
# For last segment
endseg = ring.path[-1]
ts = icurvature(endseg, tol_curvature)
ts = [t for t in ts if endseg.length(t0=t) < opt.tol_isNear]
if ts:
T1 = ring.path.t2T(-1, min(ts))
else:
T1 = 1
# crop (if necessary)
if T0 != 0 or T1 != 1:
ring.path = ring.path.cropped(T0, T1)
print("Done trimming.")
# Check that there are no rings end outside the boundary ring (note
# intersection removal in next step makes this sufficient)
print("Checking for rings outside boundary ring...")
boundary_ring = max([r for r in ring_list if r.isClosed()],
key=lambda rgn: rgn.maxR)
outside_mark_indices = []
for idx, r in enumerate(ring_list):
if r is not boundary_ring:
pt_outside_bdry = center + 2*boundary_ring.maxR
if not ptInsideClosedPath(r.path[0].start,
pt_outside_bdry,
boundary_ring.path):
outside_mark_indices.append(idx)
if outside_mark_indices:
ring_list = [r for i,r in enumerate(ring_list)
if i not in outside_mark_indices]
warn("%s paths were found outside the boundary path and will be "
"ignored." % len(outside_mark_indices))
print("Done removing rings outside of boundary ring.")
# Remove intersections (between distinct rings)
if opt.rings_may_contain_intersections:
print("Removing intersections (between distinct rings)...")
from noIntersections4rings import remove_intersections_from_rings
opt.basic_output_on.dprint("Now attempting to find and remove all "
"intersections from rings (this will take a "
"long time)...")
intersection_removal_start_time = current_time()
ring_list, intersection_count, overlappingClosedRingPairs = \
remove_intersections_from_rings(ring_list)
if not overlappingClosedRingPairs:
tot_ov_time = format_time(current_time() - intersection_removal_start_time)
opt.basic_output_on.dprint("Done (in just %s). Found and removed %s "
"intersections." % (tot_ov_time,
intersection_count))
else:
# fixed_paths = [parse_path(r.string) for r in ring_list]
fixed_paths = [r.path for r in ring_list]
fixed_colors = [r.color for r in ring_list]
center_line = Line(center-1, center+1)
nodes = []
for i, j in overlappingClosedRingPairs:
fixed_colors[i] = opt.colordict['safe1']
fixed_colors[j] = opt.colordict['safe2']
inters = pathXpathIntersections(ring_list[i].path,ring_list[j].path)
nodes += [inter[0].point(inter[2]) for inter in inters]
tmp = svgfile[0:len(svgfile)-4] + "_ClosedRingsOverlap.svg"
fixed_svg_filename = os_path.join(opt.output_directory, tmp)
disvg(fixed_paths + [center_line],
fixed_colors + [opt.colordict['center']],
nodes=nodes,
filename=fixed_svg_filename)
raise Exception("Found %s pair(s) over overlapping closed rings. "
"They must be fixed manually (in inkscape or "
"adobe illustrator). An svg has been output "
"highlighting the rings which must be separated "
"manually (and the points where they intersect). "
"Fix the highlighted rings and replace your old "
"svg with the fixed one (the colors/circles used "
"to highlight the intersections will be "
"fixed/removed automatically).\n"
"Output svg saved to:\n"
"%s" % (len(overlappingClosedRingPairs),
fixed_svg_filename))
# Output a fixed SVG that is (hopefully) how this SVG would be if humans
# were perfect
from options4rings import create_fixed_svg
if create_fixed_svg:
opt.basic_output_on.dprint("Now creating a fixed svg file...", 'nr')
fixed_paths = [r.path for r in ring_list]
fixed_colors = [r.color for r in ring_list]
center_line = Line(center - 1, center + 1)
tmp = svgfile[0:len(svgfile)-4] + "_fixed.svg"
fixed_svg_filename = os_path.join(opt.output_directory, tmp)
wsvg(fixed_paths + [center_line],
fixed_colors + [opt.colordict['center']],
filename=fixed_svg_filename)
opt.basic_output_on.dprint("Done. SVG file saved to:\n"
"%s" % fixed_svg_filename)
def svgtree(svgfile, error_list):
file_start_time = current_time()
SVGfileLocation = os_path.join(opt.input_directory, svgfile)
# svgname = nt_path_basename(SVGfileLocation)[0:-4]
svgname = os_path.splitext(os_path.basename(SVGfileLocation))[0]
# Name pickle Files
pickle_file = os_path.join(opt.pickle_dir, svgname + "-ring_list.p")
sorted_pickle_file = os_path.join(opt.pickle_dir, svgname + "-sorted-ring_list.p")
om_pickle_file = os_path.join(opt.pickle_dir, svgname + "-ordering_matrix.p")
# tmp = 'DataFrom-'+ svgname +'_failed_rings.csv'
# outputFile_failed_rings = os_path.join(opt.output_directory, tmp)
# tmp = 'DataFrom-' + svgname + '.csv'
# outputFile = os_path.join(opt.output_directory, tmp)
# determine if pickle file exists, if it does,
# load ring_list and center from it
if opt.ignore_extant_pickle_file:
pickle_file_exists = False
else:
pickle_file_exists = True
try:
ring_list, center = pickle.load(open(pickle_file, "rb"))
except:
pickle_file_exists = False
# determine if sorted pickle file exists, if it does,
# load ring_list and center from it (instead of unsorted pickle)
if opt.ignore_extant_sorted_pickle_file:
sorted_pickle_file_exists = False
else:
sorted_pickle_file_exists = True
try:
ring_list, center = pickle.load(open(sorted_pickle_file, "rb"))
except:
sorted_pickle_file_exists = False
# If pickle file doesn't exist, create one, and
# store ring_list and center in it
if not (pickle_file_exists or sorted_pickle_file_exists):
center, ring_list = svg2rings(SVGfileLocation)
opt.basic_output_on.dprint("Pickling ring_list... ", 'nr')
pickle.dump((ring_list, center), open(pickle_file, "wb"))
opt.basic_output_on.dprint('pickling complete -> ' + pickle_file)
opt.basic_output_on.dprint("Done.")
###############################################################################
###fix to record svg names in rings ###########################################
###############################################################################
for ring in ring_list:
ring.svgname = svgfile[:-4]
if not opt.assume_svg_is_fixed:
fix_svg(ring_list, center, svgfile)
###############################################################################
###Sort #######################################################################
###############################################################################
# sort ring_list from innermost to outermost and record sort index
if opt.sort_rings_on:
if not sorted_pickle_file_exists:
tmp_mes =("Attempting to sort ring_list. This could take a minute "
"(or thirty)...")
opt.basic_output_on.dprint(tmp_mes,'nr')
#find sorting of ring_list
from sorting4rings import sort_rings
# ring_sorting, psorting = sort_rings(ring_list, om_pickle_file)
ring_sorting, sort_lvl_info = sort_rings(ring_list, om_pickle_file)
opt.basic_output_on.dprint("Done sorting ring_list.")
#record sort index
for i, r_index in enumerate(ring_sorting):
ring_list[r_index].sort_index = i
# ring_list[r_index].psort_index = ???
# pickle "sorted" ring_list (not really sorted, but sort_index's
# are recorded)
opt.basic_output_on.dprint("Pickling sorted ring_list... ", 'nr')
pickle.dump((ring_list, center), open(sorted_pickle_file, "wb"))
opt.basic_output_on.dprint('pickling complete -> ' +
sorted_pickle_file)
###############################################################################
###Generate and Output Transects ##############################################
###############################################################################
# generate transects
skipped_angle_indices = []
tmp = "Generating the {} requested transects...".format(opt.N_transects)
opt.basic_output_on.dprint(tmp, 'nr')
if opt.N_transects > 0:
if opt.use_ring_sort_4transects:
if opt.generate_evenly_spaced_transects:
angles = np.linspace(0, 1, opt.N_transects+1)[:-1]
bdry_ring = max(ring_list, key=lambda r: r.maxR)
bdry_length = bdry_ring.path.length()
Tvals = [bdry_ring.path.ilength(s * bdry_length) for s in angles]
tmp = generate_inverse_transects(ring_list, Tvals)
data, data_indices, skipped_angle_indices = tmp
num_suc = len(data)
nums = (num_suc, opt.N_transects, opt.N_transects - num_suc)
trmes = ("%s / %s evenly spaced transects successfully "
"generated (skipped %s)." % nums)
opt.basic_output_on.dprint(trmes)
else:
tmp = generate_sorted_transects(ring_list, center,
angles2use=opt.angles2use)
data, data_indices, angles = tmp
else:
from transects4rings import generate_unsorted_transects
tmp = generate_unsorted_transects(ring_list, center)
data, data_indices, angles = tmp
opt.basic_output_on.dprint("Done generating transects.")
# show them (this creates an svg file in the output folder)
if opt.create_SVG_picture_of_transects:
# svgname = svgfile[0:len(svgfile)-4]
svgname = os_path.splitext(os_path.basename(svgfile))[0]
svg_trans = os_path.join(opt.output_directory,
svgname + "_transects.svg")
displaySVGPaths_transects(ring_list, data, angles,
skipped_angle_indices, fn=svg_trans)
opt.basic_output_on.dprint("\nSVG showing transects generated "
"saved to:\n{}\n".format(svg_trans))
# Save results from transects
from transects4rings import save_transect_data, save_transect_summary
# Name output csv files
tmp = 'TransectDataFrom-' + svgname + '.csv'
outputFile_transects = os_path.join(opt.output_directory, tmp)
tmp = 'TransectSummary-' + svgname + '.csv'
outputFile_transect_summary = os_path.join(opt.output_directory, tmp)
completed_angles = [x for idx, x in enumerate(angles)
if idx not in skipped_angle_indices]
skipped_angles = [angles[idx] for idx in skipped_angle_indices]
save_transect_data(outputFile_transects, ring_list, data,
data_indices, completed_angles, skipped_angles)
save_transect_summary(outputFile_transect_summary, ring_list, data,
data_indices, completed_angles)
###############################################################################
###Compute Ring Areas #########################################################
###############################################################################
if opt.find_areas:
from area4rings import find_ring_areas
sorted_ring_list = sorted(ring_list, key=lambda rg:rg.sort_index)
# this also completes incomplete rings
find_ring_areas(sorted_ring_list, center, svgfile)
###############################################################################
###Other (optional) stuff #####################################################
###############################################################################
# Create SVG showing ring sorting
if opt.create_SVG_showing_ring_sort:
opt.basic_output_on.dprint("Attempting to create SVG showing ring "
"sorting...", 'nr')
from misc4rings import displaySVGPaths_numbered
tmp = svgfile[0:len(svgfile)-4] + "_sort_numbered" + ".svg"
svgname = os_path.join(opt.output_directory_debug, tmp)
displaySVGPaths_numbered([r.path for r in ring_list], svgname,
[r.color for r in ring_list])
opt.basic_output_on.dprint("Done.")
# test complete ring sort after first sort round
if opt.visual_test_of_all_ring_sort_on:
from svg2rings import visual_test_of_ring_sort
visual_test_of_ring_sort(ring_list)
# plot all rings on a plot with x = theta and y = r
if opt.showUnraveledRingPlot:
opt.basic_output_on.dprint("Creating unraveled ring plot... ", 'nr')
plotUnraveledRings(ring_list, center)
opt.basic_output_on.dprint("Done. (It should have opened "
"automatically and now be visible.)")
###############################################################################
###Report Success/Failure of file #############################################
###############################################################################
tmp = (svgfile, format_time(current_time() - file_start_time))
opt.basic_output_on.dprint("Success! Completed {} in {}.".format(*tmp))
opt.basic_output_on.dprint(":)"*50)
opt.basic_output_on.dprint("<>"*50)
opt.basic_output_on.dprint("<>"*50)
opt.basic_output_on.dprint("\n\n")
error_list.append((svgfile, "Completed Successfully."))
return
def set_timer(self, key, time):
time_diff = max(time / 1000 - current_time(), 0)
loop = trollius.get_event_loop()
loop.call_later(time_diff, self.computation.process_timer,
self, key, time) # args
