def __init__(self, server_instance): selfserv = server_instance content_length = int(selfserv.headers['Content-Length']) boundary = '--' + selfserv.headers['Content-Type'].split( '=' )[1] #get boundary. in headers, boundary is shorter by 2 "-" than in request body boundary = boundary.encode() #https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST postb = selfserv.rfile.read( content_length) #read entire request body. result is bytes. self.resdict = dict() postblist = postb.split(boundary) delimiter = (linesep + linesep).encode() for eachpart in postblist[1:-1]: contDisp = eachpart[0:eachpart.find(delimiter)] contData = eachpart[eachpart.find(delimiter) + len(delimiter):-len(linesep)] if contDisp.find( b'; filename=') != -1: # in case a file was loaded if contDisp.find(linesep.encode() + b'Content-Type:') != -1: filename = contDisp[ contDisp.find(b'; filename=') + 12:-len(linesep) + 1 + abs(contDisp.find(linesep.encode() + b'Content-Type:'))] else: filename = contDisp[contDisp.find(b'; filename=') + 12:-1] # filename = filename.decode() paramname = contDisp[contDisp.find(b'; name=') + 8:contDisp.find(b'; filename=') - 1] paramname = paramname.decode() else: filename = '' paramname = contDisp[contDisp.find(b'; name=') + 8:-1] paramname = paramname.decode() contData = contData.decode() # self.resdict[paramname] = (filename, contData)
def _add_seeds(self): stream = BytesIO() for seed in self.graph_manager.seeds: stream.write(seed.url.encode('utf8')) stream.write(linesep.encode('utf8')) stream.seek(0) self.frontier.add_seeds(stream)
class GameConsoleProtocol(GameReceiver): delimiter = linesep.encode("ascii") def __init__(self, factory): self.factory = factory self.name = 'GameConsoleProtocol' self.state = "CONSOLE" def connectionMade(self): self.console('Console connected. Type "help" for help.') def lineReceived(self, line): self.factory.log.debug('Console command received: ' + line) console_command = line.split() if console_command[0].lower() == 'help': self.console('TW2200 Console Help:') self.console('There is no help for you... yet.') elif console_command[0].lower() == 'shutdown': self.console( 'Sending termination signal for TW2200 Server by console request.' ) self.factory.log.info( 'Console requested server shutdown in 5 seconds.') self.factory.shutdown = reactor.callLater( 5, self.factory.GameShutdown, 'console request') else: self.console( 'Unknown console command. Please type "help" for help.') def console(self, message): self.transport.write(b'[CON] ' + message + '\n')
def write_to_file(target_file, articles, mode): f = open(target_file, mode) f.write( linesep.join([str(article) for article in articles]).encode("utf-8")) f.write(linesep.encode("utf-8")) f.flush() f.close()
def _add_seeds(self): stream = BytesIO() for seed in self.graph_manager.seeds: stream.write(seed.url.encode('utf8')) stream.write(linesep.encode('utf8')) stream.seek(0) self.frontier.add_seeds(stream)
def write(self, msg): if self.current_file_records_count[0] >= self.settings.get_max_records_per_file(): try: self.current_file = self.create_datafile() log.debug("FileStorage_writer -- Created new data file: %s", self.current_file) except IOError as e: log.error("Failed to create a new file! %s", e) self.files.get_data_files().append(self.current_file) self.current_file_records_count[0] = 0 try: if self.buffered_writer is not None and self.buffered_writer.closed is False: self.buffered_writer.close() except IOError as e: log.warning("Failed to close buffered writer! %s", e) self.buffered_writer = None try: encoded = b64encode(msg.encode("utf-8")) self.buffered_writer = self.get_or_init_buffered_writer(self.current_file) self.buffered_writer.write(encoded + linesep.encode('utf-8')) # self.buffered_writer.write(linesep.encode('utf-8')) self.current_file_records_count[0] += 1 if self.current_file_records_count[0] - self.previous_file_records_count[0] >= self.settings.get_max_records_between_fsync(): self.previous_file_records_count = self.current_file_records_count[:] self.buffered_writer.flush() except IOError as e: log.warning("Failed to update data file![%s]\n%s", self.current_file, e)
def evaluate(self, result_file, seed_words, n_bags): f = None if result_file is not None: f = open(result_file, 'wb') for n_bag in n_bags: bags = self.build_bags(seed_words, n_bag) for bag in bags: most_similar_words = self.model.most_similar(positive=bag, topn=100) info("Most similar words to: " + str(bag) + linesep) info(linesep.join([x[0] + " - distance: " + str(x[1]) for x in most_similar_words])) if f is not None: f.write(b"Most similar words to: " + str(bag).encode("utf-8") + linesep.encode("utf-8")) f.write(linesep.join([x[0] + " - distance: " + str(x[1]) for x in most_similar_words]) .encode("utf-8")) f.write(linesep.encode("utf-8")) if f is not None: f.flush() f.close()
class Echo(basic.LineReceiver): delimiter = linesep.encode("ascii") def connectionMade(self): self.transport.write(b">>> ") def lineReceived(self, line): self.sendLine(b"Echo: " + line) self.transport.write(b">>> ")
class UserInputProtocol(basic.LineReceiver): """Handle input bytestream""" delimiter = linesep.encode("utf8") def __init__(self, callback): self.callback = callback def lineReceived(self, line): self.callback(line)
class UserInterface(basic.LineReceiver): '''Creates a protocol with stdin for in-line commands.''' delimiter = linesep.encode("ascii") def __init__(self, cmd): self.cmd = cmd def lineReceived(self, line): '''Read line and send it to CmdInterface for processing.''' self.cmd.onecmd(line.decode('utf-8'))
class UserInputProtocol(basic.LineReceiver): from os import linesep as delimiter # @UnusedImport delimiter = delimiter.encode() def __init__(self, callback): self.callback = callback def lineReceived(self, line): self.callback(line)
def _write_data_to_file(filename, data, index_file=False): # TODO - Should go in the IndexWriter? data_as_bytes = io.BytesIO() pickle.dump(data, data_as_bytes) compressed_bytes = zlib.compress(data_as_bytes.getbuffer()) with open(filename, "ab") as open_document_file: open_document_file.write(compressed_bytes) if not index_file: open_document_file.write(linesep.encode())
def _add_all(self): stream = BytesIO() for page in self.graph_manager.pages: stream.write(page.url.encode('utf8')) if not page.has_errors: for link in page.links: stream.write(link.url.encode('utf8')) stream.write(linesep.encode('utf8')) stream.seek(0) self.frontier.add_seeds(stream)
def _add_all(self): stream = BytesIO() for page in self.graph_manager.pages: stream.write(page.url.encode('utf8')) if not page.has_errors: for link in page.links: stream.write(link.url.encode('utf8')) stream.write(linesep.encode('utf8')) stream.seek(0) self.frontier.add_seeds(stream)
def format_newlines(input_name, len_linesep, output_name): """ Makes a copy of a file and ensure Unix newline characters. :param str input_name: name of the file to process :param str output_name: name of the file copy with Unix newlines """ with open(input_name, "rb") as infile: lines = infile.readlines() with open(output_name, "wb") as outfile: for line in lines: line = line[:-len_linesep] + linesep.encode() outfile.write(line)
class PrintConnCountAndDie(basic.LineReceiver): delimiter = linesep.encode("ascii") def __init__(self, conn_factory): self.conn_factory = conn_factory # @defer.inlineCallbacks def lineReceived(self, line): if line in [b'end', b'die']: # yield self.sendLine('{}\n'.format(self.conn_factory.conn_count).encode('utf-8')) print(self.conn_factory.conn_count) reactor.stop() else: self.sendLine(b':-(')
def write(self, msg): if len(self.files.data_files) <= self.settings.get_max_files_count(): if self.current_file_records_count[ 0] >= self.settings.get_max_records_per_file( ) or not exists(self.settings.get_data_folder_path() + self.current_file): try: self.current_file = self.create_datafile() log.debug( "FileStorage_writer -- Created new data file: %s", self.current_file) except IOError as e: log.error("Failed to create a new file! %s", e) self.files.get_data_files().append(self.current_file) self.current_file_records_count[0] = 0 try: if self.buffered_writer is not None and self.buffered_writer.closed is False: self.buffered_writer.close() except IOError as e: log.warning("Failed to close buffered writer! %s", e) self.buffered_writer = None try: encoded = b64encode(msg.encode("utf-8")) if not exists(self.settings.get_data_folder_path() + self.current_file): self.current_file = self.create_datafile() self.buffered_writer = self.get_or_init_buffered_writer( self.current_file) self.buffered_writer.write(encoded + linesep.encode('utf-8')) self.current_file_records_count[0] += 1 if self.current_file_records_count[ 0] - self.previous_file_records_count[ 0] >= self.settings.get_max_records_between_fsync( ): self.previous_file_records_count = self.current_file_records_count[:] self.buffered_writer.flush() try: if self.buffered_writer is not None and self.buffered_writer.closed is False: self.buffered_writer.close() except IOError as e: log.warning("Failed to close buffered writer! %s", e) except IOError as e: log.warning("Failed to update data file![%s]\n%s", self.current_file, e) else: raise DataFileCountError( "The number of data files has been exceeded - change the settings or check the connection. New data will be lost." )
class CLI(basic.LineReceiver): delimiter = linesep.encode("ascii") def __init__(self): self.cli = SubgradientCLI() def connectionMade(self): self.transport.write(b'>>> ') def lineReceived(self, line): # 1.step parse line content (cmd ....) key_terms = line.split(' ') cmd = key_terms[0] params = key_terms[1:] try: kv_args, args = getopt.getopt(params, "", [ "cpu_num=", "cpu_mem=", "gpu_num=", "gpu_mem=", "disk_size=", "fee=", "min_rental_time=", "max_rental_time=", "image_name=", "os_platform=", "os_version=", "software_platform=", "support_gpu=", "dockerfile=", "stock_id=", "stock_status=", ]) kwargs = {} for kv in kv_args: kwargs[kv[0].replace('--', '')] = kv[1] # 2.step process command result = self.cli.process(cmd, **kwargs) # 3.step return result self.sendLine(result.encode('utf-8')) self.transport.write(b'>>> ') except: # 3.step return result self.sendLine(b'BAD PARAMETER') self.transport.write(b'>>> ')
class TrivialPrompter(basic.LineReceiver): from os import linesep as delimiter delimiter = delimiter.encode('utf-8') promptDeferred = None def prompt(self, msg): assert self.promptDeferred is None self.display(msg) self.promptDeferred = defer.Deferred() return self.promptDeferred def display(self, msg): self.transport.write(msg.encode('utf-8')) def lineReceived(self, line): if self.promptDeferred is None: return d, self.promptDeferred = self.promptDeferred, None d.callback(line.decode('utf-8'))
class UrsulaCommandProtocol(LineReceiver): delimiter = linesep.encode("ascii") encoding = 'utf-8' width = 80 height = 24 commands = ( 'status', 'stop', ) def __init__(self, ursula): self.ursula = ursula self.start_time = maya.now() super().__init__() def _paint_known_nodes(self): # Gather Data known_nodes = self.ursula.known_nodes known_certificate_files = os.listdir( self.ursula.known_certificates_dir) number_of_known_nodes = len(known_nodes) seen_nodes = len(known_certificate_files) # Operating Mode federated_only = self.ursula.federated_only if federated_only: click.secho("Configured in Federated Only mode", fg='green') # Heading label = "Known Nodes (connected {} / seen {})".format( number_of_known_nodes, seen_nodes) heading = '\n' + label + " " * (45 - len(label)) + "Last Seen " click.secho(heading, bold=True, nl=False) # Legend color_index = {'self': 'yellow', 'known': 'white', 'seednode': 'blue'} for node_type, color in color_index.items(): click.secho('{0:<6} | '.format(node_type), fg=color, nl=False) click.echo('\n') seednode_addresses = list(bn.checksum_address for bn in SEEDNODES) for address, node in known_nodes.items(): row_template = "{} | {} | {} | {} | {}" node_type = 'known' if node.checksum_public_address == self.ursula.checksum_public_address: node_type = 'self' row_template += ' ({})'.format(node_type) elif node.checksum_public_address in seednode_addresses: node_type = 'seednode' row_template += ' ({})'.format(node_type) click.secho( row_template.format( node.checksum_public_address, node.rest_url().ljust( 20), # TODO: Maybe put this 20 somewhere node.nickname.ljust(50), node.timestamp, node.last_seen, ), fg=color_index[node_type]) def paintStatus(self): stats = [ 'Ursula {}'.format(self.ursula.checksum_public_address), '-' * 50, 'Uptime: {}'.format(maya.now() - self.start_time), # TODO 'Learning Round: {}'.format(self.ursula._learning_round), 'Operating Mode: {}'.format( 'Federated' if self.ursula.federated_only else 'Decentralized' ), # TODO 'Rest Interface {}'.format(self.ursula.rest_url()), 'Node Storage Type {}:'.format( self.ursula.node_storage._name.capitalize()), 'Known Nodes: {}'.format(len(self.ursula.known_nodes)), 'Work Orders: {}'.format(len(self.ursula._work_orders)) ] if self.ursula._current_teacher_node: teacher = 'Current Teacher: {}: ({})'.format( self.ursula._current_teacher_node, self.ursula._current_teacher_node.rest_url()) stats.append(teacher) click.echo('\n' + '\n'.join(stats)) def connectionMade(self): message = '\nConnected to node console {}@{}'.format( self.ursula.checksum_public_address, self.ursula.rest_url()) click.secho(message, fg='yellow') click.secho("Type 'help' or '?' for help") self.transport.write(b'Ursula >>> ') def lineReceived(self, line): line = line.decode(encoding=self.encoding).strip().lower() commands = { 'known_nodes': self._paint_known_nodes, 'status': self.paintStatus, 'stop': reactor.stop, 'cycle_teacher': self.ursula.cycle_teacher_node } try: commands[line]() except KeyError: click.secho("Invalid input. Options are {}".format( ', '.join(commands))) self.transport.write(b'Ursula >>> ') def terminalSize(self, width, height): self.width = width self.height = height self.terminal.eraseDisplay() self._redraw()
#!/usr/bin/env python3 # Get necessary file information using os.stat() import os, sys, re from os import linesep # import stat import time import csv import pandas as pd import numpy delimiter = linesep.encode("ascii") file_details = {} # Get file info: name, size, row count, file_name = sys.argv[1] # print(file_name) file_details["file_name"] = file_name file_size = os.path.getsize(file_name) # print(file_size) file_details["file_size_bytes"] = file_size file_row_count = len(open(file_name).readlines()) # print(file_row_count) file_details["total_rows"] = file_row_count suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
def norm(bytestr): """Normalise line endings.""" return bytestr if linesep == "\n" else bytestr.replace( linesep.encode(), b"\n")
import os, sys, re from os import linesep import time import csv import pandas as pd import numpy import logging logger = logging.getLogger(__name__) """Usage: compary.py file_1 file_2 """ logger.info(f"Running a comparison between {sys.argv[1]} amd {sys.argv[2]}") delimiter = linesep.encode("ascii") file_details = {} # Get file info: name, size, row count, file_name = sys.argv[1] file_details["file_name"] = file_name file_size = os.path.getsize(file_name) file_details["file_size_bytes"] = file_size file_row_count = len(open(file_name).readlines()) file_details["total_rows"] = file_row_count suffixes = ["B", "KB", "MB", "GB", "TB", "PB"]
class UI(basic.LineReceiver): """ A basic command line utility to interact with Cohen 3. The available commands are:: $ help $ list $ extract <uuid> $ send <uuid> $ quit <uuid> Example:: from twisted.internet import reactor from twisted.internet import stdio from coherence.base import Coherence c = Coherence({'logmode': 'none'}) ui = UI() ui.coherence = c stdio.StandardIO(ui) reactor.run() """ delimiter = linesep.encode("ascii") def connectionMade(self): self.print_prompt() def rawDataReceived(self, data): pass def lineReceived(self, line): # print('lineReceived', line) self.sendLine(b'UI: ' + line) args = line.strip().split() if args: cmd = args[0].lower().decode('ascii') print(cmd) if hasattr(self, 'cmd_%s' % cmd): getattr(self, 'cmd_%s' % (cmd))(args[1:]) elif cmd == b"?": self.cmd_help(args[1:]) else: self.transport.write(b"""Unknown command '%r'\n""" % (cmd)) self.print_prompt() def cmd_help(self, args): "help -- show help" methods = set([getattr(self, x) for x in dir(self) if x[:4] == "cmd_"]) self.transport.write(b"Commands:\n") for method in methods: if hasattr(method, '__doc__'): self.transport.write(b"%r\n" % method.__doc__) def cmd_list(self, args): "list -- list devices" self.transport.write(b"Devices:\n") for d in self.coherence.get_devices(): self.transport.write( ("%s %s [%s/%s/%s]\n" % (d.friendly_name, ':'.join(d.device_type.split(':')[3:5]), d.st, d.usn.split(':')[1], d.host)).encode('ascii')) def cmd_extract(self, args): "extract <uuid> -- download xml files from device" device = self.coherence.get_device_with_id(args[0]) if device is None: self.transport.write(b"device %s not found - aborting\n" % args[0]) else: self.transport.write(b"extracting from %r @ %r\n" % (device.friendly_name, device.host)) try: l = [] def device_extract(workdevice, path): tmp_dir = os.path.join(path, workdevice.get_uuid()) os.mkdir(tmp_dir) d = client.downloadPage( workdevice.get_location(), os.path.join(tmp_dir, 'device-description.xml')) l.append(d) for service in workdevice.services: d = client.downloadPage( service.get_scpd_url(), os.path.join( tmp_dir, '%s-description.xml' % service.service_type.split(':', 3)[3])) l.append(d) for ed in workdevice.devices: device_extract(ed, tmp_dir) def finished(result): self.transport.write( b"\nextraction of device %sr finished\n" b"files have been saved to /tmp/%r\n" % (args[0], args[0])) self.print_prompt() device_extract(device, '/tmp') dl = defer.DeferredList(l) dl.addCallback(finished) except Exception as msg: self.transport.write( str("problem creating download directory %s\n" % msg)) def cmd_send(self, args): """ send <uuid> -- send before extracted xml files to the Coherence home base """ if os.path.isdir(os.path.join('/tmp', args[0].decode('utf-8'))) == 1: cwd = os.getcwd() os.chdir('/tmp') import tarfile tar = tarfile.open(os.path.join('/tmp', args[0] + '.tgz'), "w:gz") for file in os.listdir(os.path.join('/tmp', args[0])): tar.add(os.path.join(args[0], file)) tar.close() os.chdir(cwd) def got_mx(result): mx_list = result[0] # mx_list.sort( # lambda x, y: cmp( # x.payload.preference, y.payload.preference)) mx_list = sorted( mx_list, key=lambda x, y: cmp_to_key(x.payload.preference, y.payload .preference)) if len(mx_list) > 0: import posix import pwd import socket reactor.connectTCP( mx_list[0].payload.name, 25, SMTPClientFactory( '@'.join((pwd.getpwuid(posix.getuid())[0], socket.gethostname())), '*****@*****.**', 'xml-files', os.path.join('/tmp', args[0] + '.tgz'))) mx = namesclient.lookupMailExchange('googlemail.com') mx.addCallback(got_mx) def cmd_quit(self, args): "quit -- quits this program" reactor.stop() cmd_exit = cmd_quit def print_prompt(self): self.transport.write(b'>>> ')