def test_manual_network_manipulation():
portfile, logfile = prepare_logging_locations()
threading.current_thread().setName("Main__")
eh = Stolas()
open(portfile, "w").write(str(eh.port))
open(logfile, "w")
print("Started on port {0}".format(eh.port))
print("Building network...")
models = build_network(eh.port, random.randrange(15, 20))
print("Ready ~<f:green]~<s:bright]\u2713~<f:reset]~<s:reset_all]")
eh.start()
manual_stolas_prompt(eh)
eh.stop()
eh.join()
print("Collapsing Network...")
collapse_network(models)
print("Network collapsed ~<s:bright]~<f:green]\u2713~<s:reset_all]")
assert (threading.active_count() == 1)
if os.path.isfile(portfile):
os.remove(portfile)
def network_random(port, quantity=None):
ports = []
cport = port
objects = []
if quantity == None:
quantity = random.randrange(5, 7)
for n in range(quantity):
while True:
try:
stols = stolas.stolas.Stolas(port=cport, virtual=True)
stols.start()
if len(ports) > 0:
rport = random.choice(ports)
pid = stols.networker.peer_add(("127.0.0.1", rport))
print("Created model {0}/{1}...".format(n + 1, quantity),
end="\r")
ports.append(cport)
cport += 1
objects.append(stols)
break
except OSError:
cport += 1
continue
# Assert that every single object is connected to another one
assert (len([x for x in objects if len(x.networker.peers) == 0]) == 0)
return objects
def start(self, urls):
self.lessons = urls
try:
self.loop()
except KeyboardInterrupt:
print("Done \u2713")
self.student.save()
def build_network(port, quantity=None):
ports = []
cport = port
objects = []
if quantity == None:
quantity = random.randrange(5, 7)
for n in range(quantity):
while True:
try:
stols = UnisocketModel(port=cport, name=len(ports))
stols.start()
if len(ports) > 0:
rport = random.choice(ports)
pid = stols.peer_add(("127.0.0.1", rport))
print("Created model {0}/{1}...".format(n + 1, quantity),
end="\r")
ports.append(cport)
cport += 1
objects.append(stols)
break
except OSError:
cport += 1
continue
# Assert that every object is connected to at least another one
assert (len([x for x in objects if len(x.peers) == 0]) == 0)
return objects
def stolas_simple():
slobj = stolas.stolas.Stolas(logdown=True)
slobj.start()
print("Port is {0}".format(slobj.port))
manual_stolas_prompt(slobj)
slobj.stop()
slobj.join()
print("Done")
def main():
import sys
if len(sys.argv) < 2:
show_help()
return
if sys.argv[1] == "feed" and len(sys.argv) > 2:
mkvg = MarkovGenerator()
if len(sys.argv) > 3 and sys.argv[3] == "--advanced":
mkvg = AdvancedMarkovGenerator()
now = time.time()
for filepath in filefindergenerator(sys.argv[2]):
try:
mkvg.feed(open(filepath).read())
except UnicodeDecodeError:
continue
except FileNotFoundError:
continue
except PermissionError:
continue
print("Fed {0} to the MKvG".format(filepath))
if time.time() - now >= 60:
mkvg.save()
now = time.time()
mkvg.save()
elif sys.argv[1] == "puke" and len(sys.argv) > 2:
mkvg = MarkovGenerator()
if len(sys.argv) > 3 and sys.argv[3] == "--advanced":
print("Loading Advanced Options")
mkvg = AdvancedMarkovGenerator()
outp = open(sys.argv[2], "wb")
for char in mkvg.puke():
outp.write(char.encode("utf8"))
outp.close()
elif sys.argv[1] == "automatic":
if len(sys.argv) > 2 and sys.argv[2] == "--advanced":
mkvg = AdvancedMarkovGenerator()
else:
mkvg = MarkovGenerator()
teacher = AutomatedNetworkTeacher(mkvg)
starters = [
"https://docs.python.org", "https://atom.io/",
"https://wikipedia.org/", "https://doc.qt.io/",
"https://stackoverflow.com/", "https://www.parcoursup.fr",
"https://freedesktop.org", "https://godotengine.org/",
"http://www.cplusplus.com/", "https://www.reddit.com/"
]
teacher.start(starters)
elif sys.argv[1] == "help":
show_help()
def create_ctrlfile():
controlfile = "{0}_cluster.ctrl".format(
hashlib.sha1(
str(time.time() + random.randrange(-10000, 10000)).encode(
"utf8")).hexdigest()[:8])
print("Creating control file : {0}".format(controlfile))
open(controlfile, "w")
print(
"~<s:bright]Remove the control file at any time to collapse the cluster.~<s:reset_all]"
)
return controlfile
def run_stolas():
port = None
if len(sys.argv) >= 3:
port = int(sys.argv[2])
eh = Stolas(port=port)
else:
eh = Stolas()
eh.start()
print(eh.port)
manual_stolas_prompt(eh)
eh.stop()
eh.join()
def show_help():
from stolas.betterui import pprint as print
from sys import argv
helps = [
"The following help shows you the available commands for markovgenerator.py :",
" - ~<s:bright]{} feed [directory]~<s:reset_all] : Scans the provided directory for files\n\tending in '.html' to train on.",
" - ~<s:bright]{} puke [outputfile]~<s:reset_all] : Generates random text and writes it\n\tinto the provided file (overwritting). Requires training.",
" - ~<s:bright]{} automatic : Learn from crawling on the internet.",
" - ~<s:bright]{} help~<s:reset_all] : The very thing you are reading right now.\n"
]
for hp in helps:
print(hp.format(argv[0]))
def test_encoders():
for e in range(0, 65536):
stupendous_encoding_minimal = random.randrange(2, 50)
encoded = i2b(e, stupendous_encoding_minimal)
decoded = b2i(encoded)
try:
assert (len(encoded) == stupendous_encoding_minimal)
assert (decoded == e)
except AssertionError:
print()
print(decoded)
print(e)
print(
"~<s:bright]~<f:red]Error with encoders : program aborted~<s:reset_all]"
)
raise
#time.sleep(0.1)
print("Encoders and decoders ~<sf:bright,green]OK \u2713~<s:reset_all]")
# ~:22
return True # We're a test unit
def stolas_gigacluster():
rport = random.randrange(1024, 65400)
controlfile = create_ctrlfile()
print("Starting from port ~<s:bright]~<f:red]{0}~<s:reset_all]".format(
rport))
cluster = network_random(rport, random.randrange(15, 20))
print("~<s:bright]Network is ready! ~<f:green]\u2713~<s:reset_all]")
mm = [x.port for x in cluster if x.is_alive()]
while len(mm) > 0 and os.path.isfile(controlfile):
print("AvgInt: {0:.2f}; FiPort: {1}".format(
sum([len(m.networker.peers)
for m in cluster if m.is_alive()]) / len(mm), mm[0]),
end=" \r")
time.sleep(1)
mm = [x.port for x in cluster if x.is_alive()]
#print([m for m in cluster if m.is_alive()][0].networker.peers)
if os.path.isfile(controlfile):
os.remove(controlfile)
print("\n~<s:bright]~<f:blue]Cluster collapsing~<s:reset_all]")
network_collapse(cluster)
print("Done")
def run_test_unit(title, function):
print("~<s:bright]=> ~<f:blue]{}~<s:reset_all] ".format(title), end="")
stdout = swap_out()
stdout.flush()
try:
now = time.time()
assert (function())
delta = time.time() - now
except Exception as err:
traceback = swap_in(stdout)
print("~<f:red]~<s:bright]\u2717~<s:reset_all]")
traceback.seek(0)
name = "stolas_tucrash_{0}".format(int(time.time()))
with open(name, "w") as out:
out.write(traceback.read())
print("\t=> Traceback written in {}. Aborting!".format(name))
sys.exit(1)
swap_in(stdout)
print("~<f:green]~<s:bright]\u2713~<s:reset_all]\n\tTook {0}s.".format(
int(delta)))
def main():
print("~<s:bright]~) Encoders & Decoders (~~<s:reset_all]")
from encoders import test_encoders
run_test_unit("Encoders Test Unit", test_encoders)
print("~<s:bright]~) Network & Transmission (~~<s:reset_all]")
from network import test_network_integration_and_collapsing
run_test_unit("Network Integration",
test_network_integration_and_collapsing)
from transmitters import test_transmission
run_test_unit("Transmission in a Network", test_transmission)
print("~<s:bright]~) Message Compression (~~<s:reset_all]")
from compression import test_compression_advantages
run_test_unit("Message Compression",
(lambda: test_compression_advantages(upperb=2**20)))
print("Tests successful")
def loop(self):
while len(self.lessons) > 0:
url = random.choice(self.lessons)
self.lessons = self.lessons[1:]
if not self.check_allowed(url):
print("Not allowed to fetch {}".format(url))
print("Left: {}".format(len(self.lessons)))
#self.lessons.task_done()
continue
try:
success, data = self.download(url)
assert (success)
data = data.decode("utf8")
except (AssertionError, ConnectionResetError, UnicodeDecodeError,
socket.gaierror) as err:
print("Didn't succeed: {}".format(url))
#self.lessons.task_done()
continue
self.student.feed(data)
print("Fed {} \u2713".format(url))
# Get the scraps
self.gatherer.feed(data)
scraps = [
self.correctify(url, x) for x in self.gatherer.return_scraps()
]
self.gatherer.clean_scraps()
# Filter out from recent_history
#scraps = [scrap for scrap in scraps if not scrap in self.recent_history]
# Filter out from unauthorized URLS
#scraps = [scrap for scrap in scraps if self.check_allowed(scrap)]
random.shuffle(scraps)
for scrap in scraps:
self.lessons.append(scrap)
end=" \r")
mm = [m for m in mm if m.is_alive()]
time.sleep(1)
print()
print("~<f:blue]~<s:bright]Cluster Collapsing~<s:reset_all]")
collapse_network(models)
assert (threading.active_count() == 1)
if __name__ == "__main__":
import sys
if len(sys.argv) > 1:
if sys.argv[1] == "integration":
print(
"~<s:bright]Launching Network Integration and Collapsing Model~<s:reset_all]"
)
for e in range(random.choice(create_ponderation(10))):
print("TEST NUMBER {0}".format(e + 1))
test_network_integration_and_collapsing()
print("<(~)" + "-" * 50 + "(~)>")
elif sys.argv[1] == "simple":
run_stolas()
elif sys.argv[1] == "cluster":
print("~<s:bright][ Lauching Cluster of Unisocket Models")
print("[" + "-" * 50 + "]~<s:reset_all]")
run_cluster()
elif sys.argv[1] == "average":
def manual_stolas_prompt(eh):
while eh.running and eh.networker.running:
try:
print("~<s:bright]", end="")
command = input("[-]> ")
print("~<s:reset_all]", end="")
except Exception as err:
print("<~s:bright]<~f:red]" + str(err) + "<~s:reset_all]")
eh.running = False
continue
if command == "shutdown":
eh.running = False
csplit = command.split(" ")
if csplit[0] == "connect" and len(
csplit) == 3 and csplit[2].isdecimal():
csplit[1] = socket.gethostbyname(csplit[1])
pid = eh.networker.peer_add((csplit[1], int(csplit[2])))
if type(pid) != type(0) and pid == False:
print("En error occured...")
continue
else:
print("Added peer {0} => PID {1}".format(csplit[1:3], pid))
elif csplit[0] == 'ppeers':
print(eh.networker.possible_peers)
elif csplit[0] == "broadcast" and len(csplit) > 1:
msgobj = Message()
msgobj.set_payload(" ".join(csplit[1:]).encode("utf8"))
msgobj.set_timestamp(time.time())
msgobj.set_ttl(70)
msgobj.set_channel("")
eh.message_broadcast(msgobj)
elif csplit[0] == "haddaway":
eh.networker.peerlock.acquire()
for peer in eh.networker.peers:
eh.networker.raw_peer_send(peer, eh.networker.death_sequence)
eh.networker.peerlock.release()
time.sleep(1)
elif csplit[0] == "peers":
eh.networker.peerlock.acquire()
colors = ["red", "yellow", "green", "cyan", "blue", "magenta"]
col = random.randrange(0, len(colors))
for peer in sorted(list(eh.networker.peers.keys())):
print(
"~<s:bright]{0}~<s:reset_all] => ~<f:{1}]{2}~<s:reset_all]"
.format(
peer, colors[col], eh.networker.peers[peer].listen
or eh.networker.peers[peer].verbinfo))
col = (col + 1) % len(colors)
eh.networker.peerlock.release()
elif csplit[0] == "fpeers":
eh.networker.peerlock.acquire()
for peer in [
eh.networker.peers[peer] for peer in eh.networker.peers
if eh.networker.peers[peer].listen != None
]:
print(
"~<s:bright]{0}~<s:reset_all] => ~<s:bright]{1}~<s:reset_all]"
.format(peer.pid, peer.listen))
eh.networker.peerlock.release()
elif csplit[0] == "port":
print("Port is : {0}".format(eh.port))
elif csplit[0] == "messages":
print("Inbox:")
msgs = eh.mpile.list()
for mid in sorted(msgs):
msg = msgs[mid]
print("[{0}] {1}".format(
time.strftime("%Y/%m/%d %H:%M:%S",
time.localtime(msg.timestamp)),
msg.get_payload()))
print("Prompt terminated")
print("<[~]" + "-" * 40 + "[~]>")
def test_transmission():
print("~<s:bright]Starting Message Transmission Test~<s:reset_all]")
controlfile = create_ctrlfile()
sender = stolas.stolas.Stolas()
receiver = stolas.stolas.Stolas()
print("\t=> Ends created ~<sf:bright,green]\u2713~<s:reset_all]")
sender.start()
receiver.start()
print("\t=> Ends started ~<sf:bright,green]\u2713~<s:reset_all]")
cluster = network_random(sender.port + 1, random.randrange(10, 12))
print(
"\n\t=> Done creating the cluster ~<sf:bright,green]\u2713~<s:reset_all]"
)
sender.networker.peer_add(("localhost", random.choice(cluster).port))
receiver.networker.peer_add(("localhost", random.choice(cluster).port))
print("\t=> Connected the Receiver and Sender")
i, sint, cint, mint, rint = 0, 0, 0, 0, 0
while not sender.networker.integrated or not receiver.networker.integrated or len(
[x for x in cluster if x.networker.integrated]) < len(cluster):
print("[{0}|{1:.2f}|{2}] Integrating{3}{4}".format(
sint, mint, rint, '.' * i, ' ' * (5 - i)),
end="\r")
sint = len(sender.networker.peers)
mint = min([
len(x.networker.peers) for x in cluster + [sender] if x.is_alive()
])
rint = len(receiver.networker.peers)
i = (i + 1) % 6
time.sleep(0.5)
print("=> Integrated ~<s:bright]~<f:green]\u2713~<s:reset_all]" + 17 * ' ')
assert (receiver.networker.integrated)
assert (sender.networker.integrated)
assert (len([True for x in cluster
if x.networker.integrated]) == len(cluster))
ttlt = 120
globalpayload = generate_random_payload()
msgobj = stolas.protocol.Message(ttl=ttlt, channel="")
msgobj.set_payload(globalpayload)
print("Sending out: {0}".format(msgobj))
sender.message_broadcast(msgobj)
then = time.time()
i = 1
worked_out_fine = False
while True:
if len(receiver.mpile) > 0:
print(" " * 10, end="\r")
print(
"\t=> Message Received ~<f:green]~<s:bright]\u2713~<s:reset_all] "
)
worked_out_fine = True
break
print("[{0}|{1:.2f}|{2}>{3}] Waiting{4}{5}".format(
sint, cint, rint,
len([x for x in cluster + [receiver] if len(x.mpile) > 0]),
'.' * i, ' ' * (5 - i)),
end="\r")
time.sleep(0.5)
sint = len(sender.networker.peers)
cint = mean([
len(x.networker.peers) for x in cluster + [sender] if x.is_alive()
])
rint = len(receiver.networker.peers)
i = (i + 1) % 5
if time.time() - then >= ttlt or not os.path.isfile(controlfile):
print(
"~<s:bright]~<f:red]Failed sending the message \u2717~<s:reset_all]"
)
print(
"\t=> Leaving anyways ~<s:bright]~<f:red]\u2717~<s:reset_all]")
break
network_collapse(cluster)
sender.stop()
sender.join()
receiver.stop()
receiver.join()
if os.path.isfile(controlfile):
os.remove(controlfile)
print("Done")
return worked_out_fine # We're a unit test
def run_cluster():
threading.current_thread().setName("Main__")
port = random.randrange(1024, 65400)
print("Main Stolas object started")
controlfile = "{0}_cluster.ctrl".format(
hashlib.sha1(
str(time.time() + random.randrange(-10000, 10000)).encode(
"utf8")).hexdigest()[:8])
print("Creating control file : {0}".format(controlfile))
open(controlfile, "w")
print(
"~<s:bright]Remove the control file at any time to collapse the cluster.~<s:reset_all]"
)
print("Creating network cluster from port {0}...".format(port))
models = build_network(port, random.randrange(5, 20))
print("Running ~<f:green]~<s:bright]\u2713~<s:reset_all]")
mm = [m for m in models if m.is_alive()]
while len(mm) > 0 and os.path.isfile(controlfile):
print("Current count: {0} (0port: {1})".format(
len(mm), min([m.port for m in mm])),
end=" \r")
mm = [m for m in mm if m.is_alive()]
if os.path.isfile(controlfile):
os.remove(controlfile)
print("\n~<f:blue]~<s:bright]Cluster Collapsing~<s:reset_all]")
collapse_network(models)
assert (threading.active_count() == 1)
sender.stop()
sender.join()
receiver.stop()
receiver.join()
if os.path.isfile(controlfile):
os.remove(controlfile)
print("Done")
return worked_out_fine # We're a unit test
if __name__ == '__main__':
if len(argv) == 1:
print("Tell me?")
pass
if argv[1] == "cluster":
stolas_cluster()
elif argv[1] == "gigacluster":
stolas_gigacluster()
elif argv[1] == "simple":
stolas_simple()
elif argv[1] == "transmission":
test_transmission()
else:
def average_cluster_integration():
threading.current_thread().setName("Main__")
port = random.randrange(1024, 65400)
controlfile = "{0}_cluster.ctrl".format(
hashlib.sha1(
str(time.time() + random.randrange(-10000, 10000)).encode(
"utf8")).hexdigest()[:8])
print("Creating control file : {0}".format(controlfile))
open(controlfile, "w")
print("Creating network cluster from port {0}...".format(port))
models = build_network(port, random.randrange(10, 20))
print("Running ~<f:green]~<s:bright]\u2713~<s:reset_all]")
mm = [m for m in models if m.is_alive()]
while len(mm) > 0 and os.path.isfile(controlfile):
print("Avg. integration is : {0:.2f} (0port: {1}) ".format(
sum([len(m.peers) for m in mm]) / len(mm),
min([m.port for m in mm])),
end=" \r")
mm = [m for m in mm if m.is_alive()]
time.sleep(1)
print()
print("~<f:blue]~<s:bright]Cluster Collapsing~<s:reset_all]")
collapse_network(models)
assert (threading.active_count() == 1)
def test_network_integration_and_collapsing():
import time
threading.current_thread().setName("Main__")
port = random.randrange(1024, 65500)
print("Started on port {0}".format(port))
models = build_network(port, random.randrange(10, 20))
print("Readying...")
print("Asserting that the network is correct...", end="")
assert (len([x for x in models if len(x.peers) == 0]) == 0)
print("~<s:bright]~<f:green]\u2713~<s:reset_all]")
print("Ready! ~<f:green]~<s:bright]\u2713~<s:reset_all]")
now = time.time()
print("Now is : {0}".format(
time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(now))))
maxtime = now + len(models) * 5 # Maximum time allowed
# We shall wait for all peers to be integrated
i = 0
while False in [m.integrated for m in models]:
print("Awaiting integration" + (i * ".") + (5 - i) * " ", end="\r")
time.sleep(1)
i = (i + 1) % 6
total_time = time.time() - now
print(
"Network has fully integrated ~<s:bright]~<f:green]\u2713~<s:reset_all]"
)
print("It took : {0:.2f}s ({1}s per peer)".format(total_time, total_time /
len(models)))
worked_out_fine = total_time <= maxtime - now
if not worked_out_fine:
print("~<s:bright]~<f:red]Network took too long. \u2717~<s:reset_all]")
print("Now killing all models")
network_collapse(models)
print("Stopped")
assert (threading.active_count() == 1)
return worked_out_fine # We're a test unit