def __init__(self, server_instance):
selfserv = server_instance
content_length = int(selfserv.headers['Content-Length'])
boundary = '--' + selfserv.headers['Content-Type'].split(
'='
)[1] #get boundary. in headers, boundary is shorter by 2 "-" than in request body
boundary = boundary.encode()
#https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST
postb = selfserv.rfile.read(
content_length) #read entire request body. result is bytes.
self.resdict = dict()
postblist = postb.split(boundary)
delimiter = (linesep + linesep).encode()
for eachpart in postblist[1:-1]:
contDisp = eachpart[0:eachpart.find(delimiter)]
contData = eachpart[eachpart.find(delimiter) +
len(delimiter):-len(linesep)]
if contDisp.find(
b'; filename=') != -1: # in case a file was loaded
if contDisp.find(linesep.encode() + b'Content-Type:') != -1:
filename = contDisp[
contDisp.find(b'; filename=') + 12:-len(linesep) + 1 +
abs(contDisp.find(linesep.encode() +
b'Content-Type:'))]
else:
filename = contDisp[contDisp.find(b'; filename=') + 12:-1]
#
filename = filename.decode()
paramname = contDisp[contDisp.find(b'; name=') +
8:contDisp.find(b'; filename=') - 1]
paramname = paramname.decode()
else:
filename = ''
paramname = contDisp[contDisp.find(b'; name=') + 8:-1]
paramname = paramname.decode()
contData = contData.decode()
#
self.resdict[paramname] = (filename, contData)
def _add_seeds(self):
stream = BytesIO()
for seed in self.graph_manager.seeds:
stream.write(seed.url.encode('utf8'))
stream.write(linesep.encode('utf8'))
stream.seek(0)
self.frontier.add_seeds(stream)
class GameConsoleProtocol(GameReceiver):
delimiter = linesep.encode("ascii")
def __init__(self, factory):
self.factory = factory
self.name = 'GameConsoleProtocol'
self.state = "CONSOLE"
def connectionMade(self):
self.console('Console connected. Type "help" for help.')
def lineReceived(self, line):
self.factory.log.debug('Console command received: ' + line)
console_command = line.split()
if console_command[0].lower() == 'help':
self.console('TW2200 Console Help:')
self.console('There is no help for you... yet.')
elif console_command[0].lower() == 'shutdown':
self.console(
'Sending termination signal for TW2200 Server by console request.'
)
self.factory.log.info(
'Console requested server shutdown in 5 seconds.')
self.factory.shutdown = reactor.callLater(
5, self.factory.GameShutdown, 'console request')
else:
self.console(
'Unknown console command. Please type "help" for help.')
def console(self, message):
self.transport.write(b'[CON] ' + message + '\n')
def write_to_file(target_file, articles, mode):
f = open(target_file, mode)
f.write(
linesep.join([str(article) for article in articles]).encode("utf-8"))
f.write(linesep.encode("utf-8"))
f.flush()
f.close()
def _add_seeds(self):
stream = BytesIO()
for seed in self.graph_manager.seeds:
stream.write(seed.url.encode('utf8'))
stream.write(linesep.encode('utf8'))
stream.seek(0)
self.frontier.add_seeds(stream)
def write(self, msg):
if self.current_file_records_count[0] >= self.settings.get_max_records_per_file():
try:
self.current_file = self.create_datafile()
log.debug("FileStorage_writer -- Created new data file: %s", self.current_file)
except IOError as e:
log.error("Failed to create a new file! %s", e)
self.files.get_data_files().append(self.current_file)
self.current_file_records_count[0] = 0
try:
if self.buffered_writer is not None and self.buffered_writer.closed is False:
self.buffered_writer.close()
except IOError as e:
log.warning("Failed to close buffered writer! %s", e)
self.buffered_writer = None
try:
encoded = b64encode(msg.encode("utf-8"))
self.buffered_writer = self.get_or_init_buffered_writer(self.current_file)
self.buffered_writer.write(encoded + linesep.encode('utf-8'))
# self.buffered_writer.write(linesep.encode('utf-8'))
self.current_file_records_count[0] += 1
if self.current_file_records_count[0] - self.previous_file_records_count[0] >= self.settings.get_max_records_between_fsync():
self.previous_file_records_count = self.current_file_records_count[:]
self.buffered_writer.flush()
except IOError as e:
log.warning("Failed to update data file![%s]\n%s", self.current_file, e)
def evaluate(self, result_file, seed_words, n_bags):
f = None
if result_file is not None:
f = open(result_file, 'wb')
for n_bag in n_bags:
bags = self.build_bags(seed_words, n_bag)
for bag in bags:
most_similar_words = self.model.most_similar(positive=bag, topn=100)
info("Most similar words to: " + str(bag) + linesep)
info(linesep.join([x[0] + " - distance: " + str(x[1]) for x in most_similar_words]))
if f is not None:
f.write(b"Most similar words to: " + str(bag).encode("utf-8") + linesep.encode("utf-8"))
f.write(linesep.join([x[0] + " - distance: " + str(x[1]) for x in most_similar_words])
.encode("utf-8"))
f.write(linesep.encode("utf-8"))
if f is not None:
f.flush()
f.close()
class Echo(basic.LineReceiver):
delimiter = linesep.encode("ascii")
def connectionMade(self):
self.transport.write(b">>> ")
def lineReceived(self, line):
self.sendLine(b"Echo: " + line)
self.transport.write(b">>> ")
class UserInputProtocol(basic.LineReceiver):
"""Handle input bytestream"""
delimiter = linesep.encode("utf8")
def __init__(self, callback):
self.callback = callback
def lineReceived(self, line):
self.callback(line)
class UserInterface(basic.LineReceiver):
'''Creates a protocol with stdin for in-line commands.'''
delimiter = linesep.encode("ascii")
def __init__(self, cmd):
self.cmd = cmd
def lineReceived(self, line):
'''Read line and send it to CmdInterface for processing.'''
self.cmd.onecmd(line.decode('utf-8'))
class UserInputProtocol(basic.LineReceiver):
from os import linesep as delimiter # @UnusedImport
delimiter = delimiter.encode()
def __init__(self, callback):
self.callback = callback
def lineReceived(self, line):
self.callback(line)
def _write_data_to_file(filename, data, index_file=False):
# TODO - Should go in the IndexWriter?
data_as_bytes = io.BytesIO()
pickle.dump(data, data_as_bytes)
compressed_bytes = zlib.compress(data_as_bytes.getbuffer())
with open(filename, "ab") as open_document_file:
open_document_file.write(compressed_bytes)
if not index_file:
open_document_file.write(linesep.encode())
def _add_all(self):
stream = BytesIO()
for page in self.graph_manager.pages:
stream.write(page.url.encode('utf8'))
if not page.has_errors:
for link in page.links:
stream.write(link.url.encode('utf8'))
stream.write(linesep.encode('utf8'))
stream.seek(0)
self.frontier.add_seeds(stream)
def _add_all(self):
stream = BytesIO()
for page in self.graph_manager.pages:
stream.write(page.url.encode('utf8'))
if not page.has_errors:
for link in page.links:
stream.write(link.url.encode('utf8'))
stream.write(linesep.encode('utf8'))
stream.seek(0)
self.frontier.add_seeds(stream)
def format_newlines(input_name, len_linesep, output_name):
"""
Makes a copy of a file and ensure Unix newline characters.
:param str input_name: name of the file to process
:param str output_name: name of the file copy with Unix newlines
"""
with open(input_name, "rb") as infile:
lines = infile.readlines()
with open(output_name, "wb") as outfile:
for line in lines:
line = line[:-len_linesep] + linesep.encode()
outfile.write(line)
class PrintConnCountAndDie(basic.LineReceiver):
delimiter = linesep.encode("ascii")
def __init__(self, conn_factory):
self.conn_factory = conn_factory
# @defer.inlineCallbacks
def lineReceived(self, line):
if line in [b'end', b'die']:
# yield self.sendLine('{}\n'.format(self.conn_factory.conn_count).encode('utf-8'))
print(self.conn_factory.conn_count)
reactor.stop()
else:
self.sendLine(b':-(')
def write(self, msg):
if len(self.files.data_files) <= self.settings.get_max_files_count():
if self.current_file_records_count[
0] >= self.settings.get_max_records_per_file(
) or not exists(self.settings.get_data_folder_path() +
self.current_file):
try:
self.current_file = self.create_datafile()
log.debug(
"FileStorage_writer -- Created new data file: %s",
self.current_file)
except IOError as e:
log.error("Failed to create a new file! %s", e)
self.files.get_data_files().append(self.current_file)
self.current_file_records_count[0] = 0
try:
if self.buffered_writer is not None and self.buffered_writer.closed is False:
self.buffered_writer.close()
except IOError as e:
log.warning("Failed to close buffered writer! %s", e)
self.buffered_writer = None
try:
encoded = b64encode(msg.encode("utf-8"))
if not exists(self.settings.get_data_folder_path() +
self.current_file):
self.current_file = self.create_datafile()
self.buffered_writer = self.get_or_init_buffered_writer(
self.current_file)
self.buffered_writer.write(encoded + linesep.encode('utf-8'))
self.current_file_records_count[0] += 1
if self.current_file_records_count[
0] - self.previous_file_records_count[
0] >= self.settings.get_max_records_between_fsync(
):
self.previous_file_records_count = self.current_file_records_count[:]
self.buffered_writer.flush()
try:
if self.buffered_writer is not None and self.buffered_writer.closed is False:
self.buffered_writer.close()
except IOError as e:
log.warning("Failed to close buffered writer! %s", e)
except IOError as e:
log.warning("Failed to update data file![%s]\n%s",
self.current_file, e)
else:
raise DataFileCountError(
"The number of data files has been exceeded - change the settings or check the connection. New data will be lost."
)
class CLI(basic.LineReceiver):
delimiter = linesep.encode("ascii")
def __init__(self):
self.cli = SubgradientCLI()
def connectionMade(self):
self.transport.write(b'>>> ')
def lineReceived(self, line):
# 1.step parse line content (cmd ....)
key_terms = line.split(' ')
cmd = key_terms[0]
params = key_terms[1:]
try:
kv_args, args = getopt.getopt(params, "", [
"cpu_num=",
"cpu_mem=",
"gpu_num=",
"gpu_mem=",
"disk_size=",
"fee=",
"min_rental_time=",
"max_rental_time=",
"image_name=",
"os_platform=",
"os_version=",
"software_platform=",
"support_gpu=",
"dockerfile=",
"stock_id=",
"stock_status=",
])
kwargs = {}
for kv in kv_args:
kwargs[kv[0].replace('--', '')] = kv[1]
# 2.step process command
result = self.cli.process(cmd, **kwargs)
# 3.step return result
self.sendLine(result.encode('utf-8'))
self.transport.write(b'>>> ')
except:
# 3.step return result
self.sendLine(b'BAD PARAMETER')
self.transport.write(b'>>> ')
class TrivialPrompter(basic.LineReceiver):
from os import linesep as delimiter
delimiter = delimiter.encode('utf-8')
promptDeferred = None
def prompt(self, msg):
assert self.promptDeferred is None
self.display(msg)
self.promptDeferred = defer.Deferred()
return self.promptDeferred
def display(self, msg):
self.transport.write(msg.encode('utf-8'))
def lineReceived(self, line):
if self.promptDeferred is None:
return
d, self.promptDeferred = self.promptDeferred, None
d.callback(line.decode('utf-8'))
class UrsulaCommandProtocol(LineReceiver):
delimiter = linesep.encode("ascii")
encoding = 'utf-8'
width = 80
height = 24
commands = (
'status',
'stop',
)
def __init__(self, ursula):
self.ursula = ursula
self.start_time = maya.now()
super().__init__()
def _paint_known_nodes(self):
# Gather Data
known_nodes = self.ursula.known_nodes
known_certificate_files = os.listdir(
self.ursula.known_certificates_dir)
number_of_known_nodes = len(known_nodes)
seen_nodes = len(known_certificate_files)
# Operating Mode
federated_only = self.ursula.federated_only
if federated_only:
click.secho("Configured in Federated Only mode", fg='green')
# Heading
label = "Known Nodes (connected {} / seen {})".format(
number_of_known_nodes, seen_nodes)
heading = '\n' + label + " " * (45 - len(label)) + "Last Seen "
click.secho(heading, bold=True, nl=False)
# Legend
color_index = {'self': 'yellow', 'known': 'white', 'seednode': 'blue'}
for node_type, color in color_index.items():
click.secho('{0:<6} | '.format(node_type), fg=color, nl=False)
click.echo('\n')
seednode_addresses = list(bn.checksum_address for bn in SEEDNODES)
for address, node in known_nodes.items():
row_template = "{} | {} | {} | {} | {}"
node_type = 'known'
if node.checksum_public_address == self.ursula.checksum_public_address:
node_type = 'self'
row_template += ' ({})'.format(node_type)
elif node.checksum_public_address in seednode_addresses:
node_type = 'seednode'
row_template += ' ({})'.format(node_type)
click.secho(
row_template.format(
node.checksum_public_address,
node.rest_url().ljust(
20), # TODO: Maybe put this 20 somewhere
node.nickname.ljust(50),
node.timestamp,
node.last_seen,
),
fg=color_index[node_type])
def paintStatus(self):
stats = [
'Ursula {}'.format(self.ursula.checksum_public_address),
'-' * 50,
'Uptime: {}'.format(maya.now() - self.start_time), # TODO
'Learning Round: {}'.format(self.ursula._learning_round),
'Operating Mode: {}'.format(
'Federated' if self.ursula.federated_only else 'Decentralized'
), # TODO
'Rest Interface {}'.format(self.ursula.rest_url()),
'Node Storage Type {}:'.format(
self.ursula.node_storage._name.capitalize()),
'Known Nodes: {}'.format(len(self.ursula.known_nodes)),
'Work Orders: {}'.format(len(self.ursula._work_orders))
]
if self.ursula._current_teacher_node:
teacher = 'Current Teacher: {}: ({})'.format(
self.ursula._current_teacher_node,
self.ursula._current_teacher_node.rest_url())
stats.append(teacher)
click.echo('\n' + '\n'.join(stats))
def connectionMade(self):
message = '\nConnected to node console {}@{}'.format(
self.ursula.checksum_public_address, self.ursula.rest_url())
click.secho(message, fg='yellow')
click.secho("Type 'help' or '?' for help")
self.transport.write(b'Ursula >>> ')
def lineReceived(self, line):
line = line.decode(encoding=self.encoding).strip().lower()
commands = {
'known_nodes': self._paint_known_nodes,
'status': self.paintStatus,
'stop': reactor.stop,
'cycle_teacher': self.ursula.cycle_teacher_node
}
try:
commands[line]()
except KeyError:
click.secho("Invalid input. Options are {}".format(
', '.join(commands)))
self.transport.write(b'Ursula >>> ')
def terminalSize(self, width, height):
self.width = width
self.height = height
self.terminal.eraseDisplay()
self._redraw()
#!/usr/bin/env python3
# Get necessary file information using os.stat()
import os, sys, re
from os import linesep
# import stat
import time
import csv
import pandas as pd
import numpy
delimiter = linesep.encode("ascii")
file_details = {}
# Get file info: name, size, row count,
file_name = sys.argv[1]
# print(file_name)
file_details["file_name"] = file_name
file_size = os.path.getsize(file_name)
# print(file_size)
file_details["file_size_bytes"] = file_size
file_row_count = len(open(file_name).readlines())
# print(file_row_count)
file_details["total_rows"] = file_row_count
suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
def norm(bytestr):
"""Normalise line endings."""
return bytestr if linesep == "\n" else bytestr.replace(
linesep.encode(), b"\n")
import os, sys, re
from os import linesep
import time
import csv
import pandas as pd
import numpy
import logging
logger = logging.getLogger(__name__)
"""Usage:
compary.py file_1 file_2
"""
logger.info(f"Running a comparison between {sys.argv[1]} amd {sys.argv[2]}")
delimiter = linesep.encode("ascii")
file_details = {}
# Get file info: name, size, row count,
file_name = sys.argv[1]
file_details["file_name"] = file_name
file_size = os.path.getsize(file_name)
file_details["file_size_bytes"] = file_size
file_row_count = len(open(file_name).readlines())
file_details["total_rows"] = file_row_count
suffixes = ["B", "KB", "MB", "GB", "TB", "PB"]
class UI(basic.LineReceiver):
"""
A basic command line utility to interact with Cohen 3.
The available commands are::
$ help
$ list
$ extract <uuid>
$ send <uuid>
$ quit <uuid>
Example::
from twisted.internet import reactor
from twisted.internet import stdio
from coherence.base import Coherence
c = Coherence({'logmode': 'none'})
ui = UI()
ui.coherence = c
stdio.StandardIO(ui)
reactor.run()
"""
delimiter = linesep.encode("ascii")
def connectionMade(self):
self.print_prompt()
def rawDataReceived(self, data):
pass
def lineReceived(self, line):
# print('lineReceived', line)
self.sendLine(b'UI: ' + line)
args = line.strip().split()
if args:
cmd = args[0].lower().decode('ascii')
print(cmd)
if hasattr(self, 'cmd_%s' % cmd):
getattr(self, 'cmd_%s' % (cmd))(args[1:])
elif cmd == b"?":
self.cmd_help(args[1:])
else:
self.transport.write(b"""Unknown command '%r'\n""" % (cmd))
self.print_prompt()
def cmd_help(self, args):
"help -- show help"
methods = set([getattr(self, x) for x in dir(self) if x[:4] == "cmd_"])
self.transport.write(b"Commands:\n")
for method in methods:
if hasattr(method, '__doc__'):
self.transport.write(b"%r\n" % method.__doc__)
def cmd_list(self, args):
"list -- list devices"
self.transport.write(b"Devices:\n")
for d in self.coherence.get_devices():
self.transport.write(
("%s %s [%s/%s/%s]\n" %
(d.friendly_name, ':'.join(d.device_type.split(':')[3:5]),
d.st, d.usn.split(':')[1], d.host)).encode('ascii'))
def cmd_extract(self, args):
"extract <uuid> -- download xml files from device"
device = self.coherence.get_device_with_id(args[0])
if device is None:
self.transport.write(b"device %s not found - aborting\n" % args[0])
else:
self.transport.write(b"extracting from %r @ %r\n" %
(device.friendly_name, device.host))
try:
l = []
def device_extract(workdevice, path):
tmp_dir = os.path.join(path, workdevice.get_uuid())
os.mkdir(tmp_dir)
d = client.downloadPage(
workdevice.get_location(),
os.path.join(tmp_dir, 'device-description.xml'))
l.append(d)
for service in workdevice.services:
d = client.downloadPage(
service.get_scpd_url(),
os.path.join(
tmp_dir, '%s-description.xml' %
service.service_type.split(':', 3)[3]))
l.append(d)
for ed in workdevice.devices:
device_extract(ed, tmp_dir)
def finished(result):
self.transport.write(
b"\nextraction of device %sr finished\n"
b"files have been saved to /tmp/%r\n" %
(args[0], args[0]))
self.print_prompt()
device_extract(device, '/tmp')
dl = defer.DeferredList(l)
dl.addCallback(finished)
except Exception as msg:
self.transport.write(
str("problem creating download directory %s\n" % msg))
def cmd_send(self, args):
"""
send <uuid> -- send before extracted xml files to the Coherence
home base
"""
if os.path.isdir(os.path.join('/tmp', args[0].decode('utf-8'))) == 1:
cwd = os.getcwd()
os.chdir('/tmp')
import tarfile
tar = tarfile.open(os.path.join('/tmp', args[0] + '.tgz'), "w:gz")
for file in os.listdir(os.path.join('/tmp', args[0])):
tar.add(os.path.join(args[0], file))
tar.close()
os.chdir(cwd)
def got_mx(result):
mx_list = result[0]
# mx_list.sort(
# lambda x, y: cmp(
# x.payload.preference, y.payload.preference))
mx_list = sorted(
mx_list,
key=lambda x, y: cmp_to_key(x.payload.preference, y.payload
.preference))
if len(mx_list) > 0:
import posix
import pwd
import socket
reactor.connectTCP(
mx_list[0].payload.name, 25,
SMTPClientFactory(
'@'.join((pwd.getpwuid(posix.getuid())[0],
socket.gethostname())),
'*****@*****.**', 'xml-files',
os.path.join('/tmp', args[0] + '.tgz')))
mx = namesclient.lookupMailExchange('googlemail.com')
mx.addCallback(got_mx)
def cmd_quit(self, args):
"quit -- quits this program"
reactor.stop()
cmd_exit = cmd_quit
def print_prompt(self):
self.transport.write(b'>>> ')
