class MockProcess(object): def __init__(self, stdout='MOCK STDOUT\n', stderr='', returncode=0): self.pid = 42 self.stdout = StringIO(stdout) self.stderr = StringIO(stderr) self.stdin = StringIO() self.returncode = returncode def wait(self): return def poll(self): # Consider the process completed when all the stdout and stderr has been read. if (len(self.stdout.getvalue()) != self.stdout.tell() or len(self.stderr.getvalue()) != self.stderr.tell()): return None return self.returncode def communicate(self, *_): return (self.stdout.getvalue(), self.stderr.getvalue()) def kill(self): return def terminate(self): return
def sitemap_generator(request, maps, page, current_site): output = StringIO() protocol = request.is_secure() and 'https' or 'http' xml = SimplerXMLGenerator(output, settings.DEFAULT_CHARSET) xml.startDocument() xml.startElement( 'urlset', {'xmlns': 'http://www.sitemaps.org/schemas/sitemap/0.9'}) yield output.getvalue() pos = output.tell() for site in maps: if callable(site): if issubclass(site, RequestSitemap): site = site(request=request) else: site = site() elif hasattr(site, 'request'): site.request = request try: urls = site.get_urls(page=page, site=current_site, protocol=protocol) except InvalidPage: raise Http404('Page not found') for url in urls: xml.startElement('url', {}) xml.addQuickElement('loc', url['location']) try: if url['lastmod']: xml.addQuickElement( 'lastmod', url['lastmod'].strftime('%Y-%m-%d')) except (KeyError, AttributeError): pass try: if url['changefreq']: xml.addQuickElement('changefreq', url['changefreq']) except KeyError: pass try: if url['priority']: xml.addQuickElement('priority', url['priority']) except KeyError: pass xml.endElement('url') output.seek(pos) yield output.read() pos = output.tell() xml.endElement('urlset') xml.endDocument() output.seek(pos) last = output.read() output.close() yield last
def do_POST(self): """Serve a POST request.""" r, info = self.deal_post_data() print(r, info, "by: ", self.client_address) f = StringIO() f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write("<html>\n<title>Upload Result Page</title>\n") f.write("<body>\n<h2>Upload Result Page</h2>\n") f.write("<hr>\n") if r: f.write("<strong>Success:</strong>") else: f.write("<strong>Failed:</strong>") f.write(info) f.write("<br><a href=\"%s\">back</a>" % self.headers['referer']) f.write("<hr><small>Powerd By: bones7456, check new version at ") f.write("<a href=\"http://li2z.cn/?s=SimpleHTTPServerWithUpload\">") f.write("here</a>.</small></body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() if f: self.copyfile(f, self.wfile) f.close()
def fetch_data(self): # create a data frame directly from the full text of # the response from the returned file-descriptor. data = self.fetch_url(self.url) fd = StringIO() if isinstance(data, str): fd.write(data) else: for chunk in data: fd.write(chunk) self.fetch_size = fd.tell() fd.seek(0) try: # see if pandas can parse csv data frames = read_csv(fd, **self.pandas_kwargs) frames_hash = hashlib.md5(str(fd.getvalue()).encode('utf-8')) self.fetch_hash = frames_hash.hexdigest() except pd.parser.CParserError: # could not parse the data, raise exception raise Exception('Error parsing remote CSV data.') finally: fd.close() return frames
def _send_command(command, host, port, timeout): sock = socket.socket() sock.settimeout(timeout) buf = StringIO() chunk_size = 1024 # try-finally and try-except to stay compatible with python 2.4 try: try: # Connect to the zk client port and send the stat command sock.connect((host, port)) sock.sendall(ensure_bytes(command)) # Read the response into a StringIO buffer chunk = ensure_unicode(sock.recv(chunk_size)) buf.write(chunk) num_reads = 1 max_reads = 10000 while chunk: if num_reads > max_reads: # Safeguard against an infinite loop raise Exception( "Read %s bytes before exceeding max reads of %s. " % (buf.tell(), max_reads)) chunk = ensure_unicode(sock.recv(chunk_size)) buf.write(chunk) num_reads += 1 except (socket.timeout, socket.error): raise ZKConnectionFailure() finally: sock.close() return buf
def render_tasks(self, iterable, **kw): fd = StringIO() err = StringIO(u'') tasks = tasks_from_iterable(iterable) render_tasks(write=fd.write, write_err=err.write, tasks=tasks, **kw) if err.tell(): return fd.getvalue(), err.getvalue() return fd.getvalue()
def sitemap_generator(request, maps, page, current_site): output = StringIO() protocol = request.is_secure() and 'https' or 'http' xml = SimplerXMLGenerator(output, settings.DEFAULT_CHARSET) xml.startDocument() xml.startElement('urlset', {'xmlns':'http://www.sitemaps.org/schemas/sitemap/0.9'}) yield output.getvalue() pos = output.tell() for site in maps: if callable(site): if issubclass(site, RequestSitemap): site = site(request=request) else: site = site() elif hasattr(site, 'request'): site.request = request for url in site.get_urls(page=page, site=current_site, protocol=protocol): xml.startElement('url', {}) xml.addQuickElement('loc', url['location']) try: if url['lastmod']: xml.addQuickElement('lastmod', url['lastmod'].strftime('%Y-%m-%d')) except (KeyError, AttributeError): pass try: if url['changefreq']: xml.addQuickElement('changefreq', url['changefreq']) except KeyError: pass try: if url['priority']: xml.addQuickElement('priority', url['priority']) except KeyError: pass xml.endElement('url') output.seek(pos) yield output.read() pos = output.tell() xml.endElement('urlset') xml.endDocument() output.seek(pos) last = output.read() output.close() yield last
def AddModule(self, m): f = StringIO() m.AppendJSContentsToFile(f, False, None) attrs = {'label': '%s (%i)' % (m.name, f.tell())} f.close() attr_items = ['%s="%s"' % (x, y) for x, y in six.iteritems(attrs)] node = 'M%i [%s];' % (m.id, ','.join(attr_items)) self.nodes.append(node)
def build_csv_response(self, wb, title="report"): """ Take a workbook and return a csv file response """ title = generate_filename(title, ".csv") myfile = StringIO() sh = wb.get_active_sheet() c = csv.writer(myfile) for r in sh.rows: c.writerow([cell.value for cell in r]) response = HttpResponse(myfile.getvalue(), content_type="text/csv") response["Content-Disposition"] = "attachment; filename=%s" % title response["Content-Length"] = myfile.tell() return response
def build_csv_response(self, wb, title="report"): """ Take a workbook and return a csv file response """ title = generate_filename(title, '.csv') myfile = StringIO() sh = wb.get_active_sheet() c = csv.writer(myfile) for r in sh.rows: c.writerow([cell.value for cell in r]) response = HttpResponse(myfile.getvalue(), content_type='text/csv') response['Content-Disposition'] = 'attachment; filename=%s' % title response['Content-Length'] = myfile.tell() return response
def getSize(obj): """Calculate the size as cheap as possible """ # Try the cheap variants first. # Actually the checks ensure the code never fails but beeing sure # is better. try: # check if to return zero (length is zero) if len(obj) == 0: return 0 except: pass try: # check if ``IStreamableReference`` if IStreamableReference.providedBy(obj): size = obj.getSize() if size is not None: return size except: pass try: # string if isinstance(obj, types.StringTypes): return len(obj) except: pass try: # file like object methods = dir(obj) if "seek" in methods and "tell" in methods: currentPos = obj.tell() obj.seek(0, 2) size = obj.tell() obj.seek(currentPos) return size except: pass try: # fallback: pickling the object stream = StringIO() p = Pickler(stream, 1) p.dump(obj) size = stream.tell() except: size = None return size
def build_csv_response(self, wb, title="report"): """ Take a workbook and return a csv file response """ title = generate_filename(title, '.csv') myfile = StringIO() sh = wb.active c = csv.writer(myfile) for r in sh.rows: c.writerow([cell.value for cell in r]) response = HttpResponse( myfile.getvalue(), content_type='text/csv') response['Content-Disposition'] = 'attachment; filename=%s' % title response['Content-Length'] = myfile.tell() return response
def build_csv_response(self, wb, title="report"): """ Take a workbook and return a csv file response """ title = generate_filename(title, '.csv') myfile = StringIO() sh = wb.active c = csv.writer(myfile) for r in sh.rows: c.writerow([cell.value for cell in r]) response = HttpResponse( myfile.getvalue(), content_type='text/csv') self.add_content_disposition_header(response, title) response['Content-Length'] = myfile.tell() return response
def check(self, instance): host = instance.get('host', 'localhost') port = int(instance.get('port', 2181)) timeout = float(instance.get('timeout', 3.0)) dimensions = self._set_dimensions( {'component': 'zookeeper', 'service': 'zookeeper'}, instance) sock = socket.socket() sock.settimeout(timeout) buf = StringIO() chunk_size = 1024 # try-finally and try-except to stay compatible with python 2.4 try: try: # Connect to the zk client port and send the stat command sock.connect((host, port)) sock.sendall(b'stat') # Read the response into a StringIO buffer chunk = encodeutils.safe_decode(sock.recv(chunk_size), 'utf-8') buf.write(chunk) num_reads = 1 max_reads = 10000 while chunk: if num_reads > max_reads: # Safeguard against an infinite loop raise Exception( "Read %s bytes before exceeding max reads of %s. " % (buf.tell(), max_reads)) chunk = encodeutils.safe_decode(sock.recv(chunk_size), 'utf-8') buf.write(chunk) num_reads += 1 except socket.timeout: buf = None finally: sock.close() if buf is not None: # Parse the response metrics, new_dimensions = self.parse_stat(buf) if new_dimensions is not None: dimensions.update(new_dimensions.copy()) # Write the data for metric, value in metrics: self.gauge(metric, value, dimensions=dimensions) else: # Reading from the client port timed out, track it as a metric self.increment('zookeeper.timeouts', dimensions=dimensions)
def parse_oldraw(f): """Read raw emission potentials from rawfile""" buf = StringIO() re_identifier = re.compile("^#\s*(\d+)\s+(\d+)\s*$") x_single = None x_pair = None i, j = None, None meta = None for line_idx, line in enumerate(f): if line.startswith(META_PREFIX): meta = json.loads(line[len(META_PREFIX):].strip()) elif line.startswith("#"): buf.seek(0) if x_single is not None: x_pair[i, j, :, :] = np.loadtxt(buf) x_pair[j, i, :, :] = x_pair[i, j, :, :].T else: x_single = np.loadtxt(buf) ncol = x_single.shape[0] x_pair = np.zeros((ncol, ncol, 21, 21)) buf = StringIO() m = re_identifier.match(line) if m: i, j = int(m.group(1)), int(m.group(2)) else: raise Exception( "Line {0} starts with # but doesn't match regex!".format( line_idx + 1)) else: buf.write(line) if x_single is not None and buf.tell(): buf.seek(0) x_pair[i, j, :, :] = np.loadtxt(buf) x_pair[j, i, :, :] = x_pair[i, j, :, :].T return CCMRaw(ncol, x_single, x_pair, meta)
def test_unseekable_file(self): def tell_fails(): raise IOError() ticket = get_ticket(urls=[get_http_ticket(EXAMPLE_URL)]) for num_retries in range(10): temp_file = StringIO() temp_file.tell = tell_fails with mock.patch("time.sleep") as mock_sleep, \ mock.patch("logging.warning") as mock_warning: dm = RetryCountDownloadManager( ticket, temp_file, max_retries=num_retries) self.assertEqual(dm.max_retries, num_retries) self.assertRaises(exceptions.RetryableError, dm.run) self.assertEqual(dm.attempt_counts[EXAMPLE_URL], 1) self.assertEqual(mock_sleep.call_count, 0) self.assertEqual(mock_warning.call_count, 0)
def list_directory(self, path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: list = os.listdir(path) except os.error: self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) f = StringIO() displaypath = cgi.escape(unquote(self.path)) f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write("<html>\n<title>Directory listing for %s</title>\n" % displaypath) f.write("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath) f.write("<hr>\n") f.write("<form ENCTYPE=\"multipart/form-data\" method=\"post\">") f.write("<input name=\"file\" type=\"file\"/>") f.write("<input type=\"submit\" value=\"upload\"/></form>\n") f.write("<hr>\n<ul>\n") for name in list: fullname = os.path.join(path, name) displayname = linkname = name # Append / for directories or @ for symbolic links if os.path.isdir(fullname): displayname = name + "/" linkname = name + "/" if os.path.islink(fullname): displayname = name + "@" # Note: a link to a directory displays with @ and links with / f.write('<li><a href="%s">%s</a>\n' % (quote(linkname), cgi.escape(displayname))) f.write("</ul>\n<hr>\n</body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() return f
def _get_data(sock, command): chunk_size = 1024 max_reads = 10000 buf = StringIO() sock.sendall(ensure_bytes(command)) # Read the response into a StringIO buffer chunk = ensure_unicode(sock.recv(chunk_size)) buf.write(chunk) num_reads = 1 while chunk: if num_reads > max_reads: # Safeguard against an infinite loop raise Exception( "Read %s bytes before exceeding max reads of %s. " % (buf.tell(), max_reads)) chunk = ensure_unicode(sock.recv(chunk_size)) buf.write(chunk) num_reads += 1 return buf
class FormatterTest(unittest.TestCase): def setUp(self): self.position = 0 self.logger = structuredlog.StructuredLogger("test_%s" % type(self).__name__) self.output_file = StringIO() self.handler = handlers.StreamHandler(self.output_file, self.get_formatter()) self.logger.add_handler(self.handler) def set_position(self, pos=None): if pos is None: pos = self.output_file.tell() self.position = pos def get_formatter(self): raise NotImplementedError( "FormatterTest subclasses must implement get_formatter") @property def loglines(self): self.output_file.seek(self.position) return [line.rstrip() for line in self.output_file.readlines()]
class FormatterTest(unittest.TestCase): def setUp(self): self.position = 0 self.logger = structuredlog.StructuredLogger( "test_%s" % type(self).__name__) self.output_file = StringIO() self.handler = handlers.StreamHandler( self.output_file, self.get_formatter()) self.logger.add_handler(self.handler) def set_position(self, pos=None): if pos is None: pos = self.output_file.tell() self.position = pos def get_formatter(self): raise NotImplementedError( "FormatterTest subclasses must implement get_formatter") @property def loglines(self): self.output_file.seek(self.position) return [line.rstrip() for line in self.output_file.readlines()]
def _flushHeaderMessage(self, buf, wout, wsz): """Write a message for self.HEADERS_CLIENT_TYPE This method writes a message using the same logic as THeaderTransport._flushHeaderMessage but mutates fields included in self._fuzz_fields """ cls = self.__class__ transform_data = StringIO() num_transforms = len(self._THeaderTransport__write_transforms) for trans_id in self._THeaderTransport__write_transforms: trans_id = self._get_fuzzy_field('transform_id', trans_id, 'i32') transform_data.write(getVarint(trans_id)) if self._THeaderTransport__hmac_func: num_transforms += 1 transform_data.write(getVarint(self.HMAC_TRANSFORM)) transform_data.write(b'\0') # size of hmac, fixup later. # Add in special flags. if self._THeaderTransport__identity: id_version = self._get_fuzzy_field( 'version', self.ID_VERSION, 'str') self._THeaderTransport__write_headers[self.ID_VERSION_HEADER] = ( id_version) identity = self._get_fuzzy_field( 'identity', self._THeaderTransport__identity, 'str') self._THeaderTransport__write_headers[self.IDENTITY_HEADER] = ( identity) info_data = StringIO() # Write persistent kv-headers cls._flush_info_headers( info_data, self._THeaderTransport__write_persistent_headers, self.INFO_PKEYVALUE) # Write non-persistent kv-headers cls._flush_info_headers( info_data, self._THeaderTransport__write_headers, self.INFO_KEYVALUE) header_data = StringIO() proto_id = self._get_fuzzy_field( 'proto_id', self._THeaderTransport__proto_id, 'i32') header_data.write(getVarint(proto_id)) num_transforms = self._get_fuzzy_field( 'num_transforms', num_transforms, 'i32') header_data.write(getVarint(num_transforms)) header_size = (transform_data.tell() + header_data.tell() + info_data.tell()) padding_size = 4 - (header_size % 4) # Fuzz padding size, but do not let total header size exceed 2**16 - 1 padding_size = min((2 ** 16 - 1) - header_size, self._get_fuzzy_field('padding', padding_size, 'i16')) header_size = header_size + padding_size wsz += header_size + 10 self._write_fuzzy_field(buf, 'length', wsz, 'i32') self._write_fuzzy_field(buf, 'magic', self.HEADER_MAGIC, 'i16') self._write_fuzzy_field( buf, 'flags', self._THeaderTransport__flags, 'i16') self._write_fuzzy_field( buf, 'seq_id', self._THeaderTransport__seq_id, 'i32') self._write_fuzzy_field(buf, 'header_size', header_size // 4, 'i16') buf.write(header_data.getvalue()) buf.write(transform_data.getvalue()) hmac_loc = buf.tell() - 1 # Fixup hmac size later buf.write(info_data.getvalue()) # Pad out the header with 0x00 if 'padding_bytes' in self._fuzz_fields: # Print that padding bytes are being fuzzed in favor of printing # the value of each individual padding byte self._print("Fuzzing %d padding bytes" % padding_size) old_verbose, self._verbose = self._verbose, False for _ in sm.xrange(padding_size): self._write_fuzzy_field( buf, 'padding_bytes', six.int2byte(0), 'char') self._verbose = old_verbose self._write_fuzzy_field(buf, 'payload', wout, 'str') # HMAC calculation should always be last. if self._THeaderTransport__hmac_func: hmac_data = buf.getvalue()[4:] hmac = self._THeaderTransport__hmac_func(hmac_data) # Fill in hmac size. buf.seek(hmac_loc) self._write_hmac_size(buf, hmac) buf.seek(0, os.SEEK_END) self._write_hmac(buf, hmac) # Fix packet size since we appended data. new_sz = buf.tell() - 4 buf.seek(0) self._write_wsz(buf, new_sz)
class File(object): def __init__(self, name, data=None, type_=None): self._name = name self.id = None self._url = None self._acl = None self.current_user = None # TODO self._metadata = { 'owner': 'unknown' } if self.current_user and self.current_user is not None: self._metadata['owner'] = self.current_user.id pattern = re.compile('\.([^.]*)$') extension = pattern.findall(name) if extension: self.extension = extension[0].lower() else: self.extension = '' if type_: self._type = type_ else: self._type = mime_types.get(self.extension, 'text/plain') if data is None: self._source = None elif isinstance(data, StringIO): self._source = data elif isinstance(data, file): data.seek(0, os.SEEK_SET) self._source = StringIO(data.read()) elif isinstance(data, buffer): self._source = StringIO(data) elif six.PY2: import cStringIO if isinstance(data, cStringIO.OutputType): data.seek(0, os.SEEK_SET) self._source = StringIO(data.getvalue()) else: raise TypeError('data must be a StringIO / buffer / file instance') else: raise TypeError('data must be a StringIO / buffer / file instance') if self._source: self._source.seek(0, os.SEEK_END) self._metadata['size'] = self._source.tell() self._source.seek(0, os.SEEK_SET) @classmethod def create_with_url(cls, name, url, meta_data=None, type_=None): f = File(name, None, type_) if meta_data: f._metadata.update(meta_data) f._url = url f._metadata['__source'] = 'external' return f @classmethod def create_without_data(cls, object_id): f = File('') f.id = object_id return f def get_acl(self): return self._acl def set_acl(self, acl): if not isinstance(acl, leancloud.ACL): raise TypeError('acl must be a leancloud.ACL instance') self._acl = acl @property def name(self): return self._name @property def url(self): return self._url @property def size(self): return self._metadata['size'] @property def owner_id(self): return self._metadata['owner'] @property def metadata(self): return self._metadata def get_thumbnail_url(self, width, height, quality=100, scale_to_fit=True, fmt='png'): if not self._url: raise ValueError('invalid url') if width < 0 or height < 0: raise ValueError('invalid height or width params') if quality > 100 or quality <= 0: raise ValueError('quality must between 0 and 100') mode = 2 if scale_to_fit else 1 return self.url + '?imageView/{0}/w/{1}/h/{2}/q/{3}/format/{4}'.format(mode, width, height, quality, fmt) def destroy(self): if not self.id: return False response = client.delete('/files/{0}'.format(self.id)) if response.status_code != 200: raise LeanCloudError(1, "the file is not sucessfully destroyed") def _save_to_qiniu(self): self._source.seek(0) hex_octet = lambda: hex(int(0x10000 * (1 + random.random())))[-4:] key = ''.join(hex_octet() for _ in xrange(4)) key = '{0}.{1}'.format(key, self.extension) data = { 'name': self._name, 'key': key, 'ACL': self._acl, 'mime_type': self._type, 'metaData': self._metadata, } response = client.post('/qiniu', data) content = utils.response_to_json(response) self.id = content['objectId'] self._url = content['url'] uptoken = content['token'] ret, info = qiniu.put_data(uptoken, key, self._source) if info.status_code != 200: raise LeanCloudError(1, 'the file is not saved, qiniu status code: {0}'.format(info.status_code)) def _save_to_leancloud(self): self._source.seek(0) encoded = codecs.encode(self._source.read(), 'base64') data = { 'base64': encoded, '_ContentType': self._type, 'ACL': self._acl, 'mime_type': self._type, 'metaData': self._metadata, } response = client.post('/files/{}'.format(self._name), data) response.raise_for_status() content = response.json() self.id = content['objectId'] self._url = content['url'] self._name = content['name'] def save(self): if self._source: if client.REGION == 'US': self._save_to_leancloud() else: self._save_to_qiniu() elif self._url and self.metadata.get('__source') == 'external': data = { 'name': self._name, 'ACL': self._acl, 'metaData': self._metadata, 'mime_type': self._type, 'url': self._url, } response = client.post('/files/{0}'.format(self._name), data) content = utils.response_to_json(response) self._name = content['name'] self._url = content['url'] self.id = content['objectId'] if 'size' in content: self._metadata['size'] = content['size'] else: raise ValueError return self def fetch(self): response = client.get('/files/{0}'.format(self.id)) content = utils.response_to_json(response) self._name = content.get('name') self.id = content.get('objectId') self._url = content.get('url') self._type = content.get('mime_type') self._metadata = content.get('metaData')
class SSHSession(Session): "Implements a :rfc:`4742` NETCONF session over SSH." def __init__(self, device_handler): capabilities = Capabilities(device_handler.get_capabilities()) Session.__init__(self, capabilities) self._host_keys = paramiko.HostKeys() self._transport = None self._connected = False self._channel = None self._channel_id = None self._channel_name = None self._buffer = StringIO() # parsing-related, see _parse() self._device_handler = device_handler self._parsing_state10 = 0 self._parsing_pos10 = 0 self._parsing_pos11 = 0 self._parsing_state11 = 0 self._expchunksize = 0 self._curchunksize = 0 self._inendpos = 0 self._size_num_list = [] self._message_list = [] def _parse(self): "Messages ae delimited by MSG_DELIM. The buffer could have grown by a maximum of BUF_SIZE bytes everytime this method is called. Retains state across method calls and if a byte has been read it will not be considered again." return self._parse10() def _parse10(self): """Messages are delimited by MSG_DELIM. The buffer could have grown by a maximum of BUF_SIZE bytes everytime this method is called. Retains state across method calls and if a chunk has been read it will not be considered again.""" logger.debug("parsing netconf v1.0") buf = self._buffer buf.seek(self._parsing_pos10) if MSG_DELIM in buf.read().decode('UTF-8'): buf.seek(0) msg, _, remaining = buf.read().decode('UTF-8').partition(MSG_DELIM) msg = msg.strip() if sys.version < '3': self._dispatch_message(msg.encode()) else: self._dispatch_message(msg) # create new buffer which contains remaining of old buffer self._buffer = StringIO() self._buffer.write(remaining.encode()) self._parsing_pos10 = 0 if len(remaining) > 0: # There could be another entire message in the # buffer, so we should try to parse again. logger.debug('Trying another round of parsing since there is still data') self._parse10() else: # handle case that MSG_DELIM is split over two chunks self._parsing_pos10 = buf.tell() - MSG_DELIM_LEN if self._parsing_pos10 < 0: self._parsing_pos10 = 0 def _parse11(self): logger.debug("parsing netconf v1.1") expchunksize = self._expchunksize curchunksize = self._curchunksize idle, instart, inmsg, inbetween, inend = range(5) state = self._parsing_state11 inendpos = self._inendpos num_list = self._size_num_list MAX_STARTCHUNK_SIZE = 12 # \#+4294967295+\n pre = 'invalid base:1:1 frame' buf = self._buffer buf.seek(self._parsing_pos11) message_list = self._message_list # a message is a list of chunks chunk_list = [] # a chunk is a list of characters should_recurse = False while True: x = buf.read(1) if not x: logger.debug('No more data to read') # Store the current chunk to the message list chunk = b''.join(chunk_list) message_list.append(textify(chunk)) break # done reading logger.debug('x: %s', x) if state == idle: if x == b'\n': state = instart inendpos = 1 else: logger.debug('%s (%s: expect newline)'%(pre, state)) raise Exception elif state == instart: if inendpos == 1: if x == b'#': inendpos += 1 else: logger.debug('%s (%s: expect "#")'%(pre, state)) raise Exception elif inendpos == 2: if x.isdigit(): inendpos += 1 # == 3 now # num_list.append(x) else: logger.debug('%s (%s: expect digit)'%(pre, state)) raise Exception else: if inendpos == MAX_STARTCHUNK_SIZE: logger.debug('%s (%s: no. too long)'%(pre, state)) raise Exception elif x == b'\n': num = b''.join(num_list) num_list = [] # Reset num_list try: num = int(num) except: logger.debug('%s (%s: invalid no.)'%(pre, state)) raise Exception else: state = inmsg expchunksize = num logger.debug('response length: %d'%expchunksize) curchunksize = 0 inendpos += 1 elif x.isdigit(): inendpos += 1 # > 3 now # num_list.append(x) else: logger.debug('%s (%s: expect digit)'%(pre, state)) raise Exception elif state == inmsg: chunk_list.append(x) curchunksize += 1 chunkleft = expchunksize - curchunksize if chunkleft == 0: inendpos = 0 state = inbetween chunk = b''.join(chunk_list) message_list.append(textify(chunk)) chunk_list = [] # Reset chunk_list logger.debug('parsed new chunk: %s'%(chunk)) elif state == inbetween: if inendpos == 0: if x == b'\n': inendpos += 1 else: logger.debug('%s (%s: expect newline)'%(pre, state)) raise Exception elif inendpos == 1: if x == b'#': inendpos += 1 else: logger.debug('%s (%s: expect "#")'%(pre, state)) raise Exception else: inendpos += 1 # == 3 now # if x == b'#': state = inend elif x.isdigit(): # More trunks state = instart num_list = [] num_list.append(x) else: logger.debug('%s (%s: expect "#")'%(pre, state)) raise Exception elif state == inend: if inendpos == 3: if x == b'\n': inendpos = 0 state = idle logger.debug('dispatching message') self._dispatch_message(''.join(message_list)) # reset rest = buf.read() buf = BytesIO() buf.write(rest) buf.seek(0) message_list = [] self._message_list = message_list chunk_list = [] expchunksize = chunksize = 0 parsing_state11 = idle inendpos = parsing_pos11 = 0 # There could be another entire message in the # buffer, so we should try to parse again. should_recurse = True break else: logger.debug('%s (%s: expect newline)'%(pre, state)) raise Exception else: logger.debug('%s (%s invalid state)'%(pre, state)) raise Exception self._expchunksize = expchunksize self._curchunksize = curchunksize self._parsing_state11 = state self._inendpos = inendpos self._size_num_list = num_list self._buffer = buf self._parsing_pos11 = self._buffer.tell() logger.debug('parse11 ending ...') if should_recurse: logger.debug('Trying another round of parsing since there is still data') self._parse11() def load_known_hosts(self, filename=None): """Load host keys from an openssh :file:`known_hosts`-style file. Can be called multiple times. If *filename* is not specified, looks in the default locations i.e. :file:`~/.ssh/known_hosts` and :file:`~/ssh/known_hosts` for Windows. """ if filename is None: filename = os.path.expanduser('~/.ssh/known_hosts') try: self._host_keys.load(filename) except IOError: # for windows filename = os.path.expanduser('~/ssh/known_hosts') try: self._host_keys.load(filename) except IOError: pass else: self._host_keys.load(filename) def close(self): if self._transport.is_active(): self._transport.close() self._channel = None self._connected = False # REMEMBER to update transport.rst if sig. changes, since it is hardcoded there def connect(self, host, port=830, timeout=None, unknown_host_cb=default_unknown_host_cb, username=None, password=None, key_filename=None, allow_agent=True, hostkey_verify=True, look_for_keys=True, ssh_config=None): """Connect via SSH and initialize the NETCONF session. First attempts the publickey authentication method and then password authentication. To disable attempting publickey authentication altogether, call with *allow_agent* and *look_for_keys* as `False`. *host* is the hostname or IP address to connect to *port* is by default 830, but some devices use the default SSH port of 22 so this may need to be specified *timeout* is an optional timeout for socket connect *unknown_host_cb* is called when the server host key is not recognized. It takes two arguments, the hostname and the fingerprint (see the signature of :func:`default_unknown_host_cb`) *username* is the username to use for SSH authentication *password* is the password used if using password authentication, or the passphrase to use for unlocking keys that require it *key_filename* is a filename where a the private key to be used can be found *allow_agent* enables querying SSH agent (if found) for keys *hostkey_verify* enables hostkey verification from ~/.ssh/known_hosts *look_for_keys* enables looking in the usual locations for ssh keys (e.g. :file:`~/.ssh/id_*`) *ssh_config* enables parsing of an OpenSSH configuration file, if set to its path, e.g. :file:`~/.ssh/config` or to True (in this case, use :file:`~/.ssh/config`). """ # Optionaly, parse .ssh/config config = {} if ssh_config is True: ssh_config = "~/.ssh/config" if sys.platform != "win32" else "~/ssh/config" if ssh_config is not None: config = paramiko.SSHConfig() config.parse(open(os.path.expanduser(ssh_config))) config = config.lookup(host) host = config.get("hostname", host) if username is None: username = config.get("user") if key_filename is None: key_filename = config.get("identityfile") if username is None: username = getpass.getuser() sock = None if config.get("proxycommand"): sock = paramiko.proxy.ProxyCommand(config.get("proxycommand")) else: for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: sock = socket.socket(af, socktype, proto) sock.settimeout(timeout) except socket.error: continue try: sock.connect(sa) except socket.error: sock.close() continue break else: raise SSHError("Could not open socket to %s:%s" % (host, port)) t = self._transport = paramiko.Transport(sock) t.set_log_channel(logger.name) if config.get("compression") == 'yes': t.use_compression() try: t.start_client() except paramiko.SSHException: raise SSHError('Negotiation failed') # host key verification server_key = t.get_remote_server_key() fingerprint = _colonify(hexlify(server_key.get_fingerprint())) if hostkey_verify: known_host = self._host_keys.check(host, server_key) if not known_host and not unknown_host_cb(host, fingerprint): raise SSHUnknownHostError(host, fingerprint) if key_filename is None: key_filenames = [] elif isinstance(key_filename, (str, bytes)): key_filenames = [ key_filename ] else: key_filenames = key_filename self._auth(username, password, key_filenames, allow_agent, look_for_keys) self._connected = True # there was no error authenticating # TODO: leopoul: Review, test, and if needed rewrite this part subsystem_names = self._device_handler.get_ssh_subsystem_names() for subname in subsystem_names: c = self._channel = self._transport.open_session() self._channel_id = c.get_id() channel_name = "%s-subsystem-%s" % (subname, str(self._channel_id)) c.set_name(channel_name) try: c.invoke_subsystem(subname) except paramiko.SSHException as e: logger.info("%s (subsystem request rejected)", e) handle_exception = self._device_handler.handle_connection_exceptions(self) # Ignore the exception, since we continue to try the different # subsystem names until we find one that can connect. #have to handle exception for each vendor here if not handle_exception: continue self._channel_name = c.get_name() self._post_connect() return raise SSHError("Could not open connection, possibly due to unacceptable" " SSH subsystem name.") def _auth(self, username, password, key_filenames, allow_agent, look_for_keys): saved_exception = None for key_filename in key_filenames: for cls in (paramiko.RSAKey, paramiko.DSSKey, paramiko.ECDSAKey): try: key = cls.from_private_key_file(key_filename, password) logger.debug("Trying key %s from %s" % (hexlify(key.get_fingerprint()), key_filename)) self._transport.auth_publickey(username, key) return except Exception as e: saved_exception = e logger.debug(e) if allow_agent: for key in paramiko.Agent().get_keys(): try: logger.debug("Trying SSH agent key %s" % hexlify(key.get_fingerprint())) self._transport.auth_publickey(username, key) return except Exception as e: saved_exception = e logger.debug(e) keyfiles = [] if look_for_keys: rsa_key = os.path.expanduser("~/.ssh/id_rsa") dsa_key = os.path.expanduser("~/.ssh/id_dsa") ecdsa_key = os.path.expanduser("~/.ssh/id_ecdsa") if os.path.isfile(rsa_key): keyfiles.append((paramiko.RSAKey, rsa_key)) if os.path.isfile(dsa_key): keyfiles.append((paramiko.DSSKey, dsa_key)) if os.path.isfile(ecdsa_key): keyfiles.append((paramiko.ECDSAKey, ecdsa_key)) # look in ~/ssh/ for windows users: rsa_key = os.path.expanduser("~/ssh/id_rsa") dsa_key = os.path.expanduser("~/ssh/id_dsa") ecdsa_key = os.path.expanduser("~/ssh/id_ecdsa") if os.path.isfile(rsa_key): keyfiles.append((paramiko.RSAKey, rsa_key)) if os.path.isfile(dsa_key): keyfiles.append((paramiko.DSSKey, dsa_key)) if os.path.isfile(ecdsa_key): keyfiles.append((paramiko.ECDSAKey, ecdsa_key)) for cls, filename in keyfiles: try: key = cls.from_private_key_file(filename, password) logger.debug("Trying discovered key %s in %s" % (hexlify(key.get_fingerprint()), filename)) self._transport.auth_publickey(username, key) return except Exception as e: saved_exception = e logger.debug(e) if password is not None: try: self._transport.auth_password(username, password) return except Exception as e: saved_exception = e logger.debug(e) if saved_exception is not None: # need pep-3134 to do this right raise AuthenticationError(repr(saved_exception)) raise AuthenticationError("No authentication methods available") def run(self): chan = self._channel q = self._q def start_delim(data_len): return '\n#%s\n'%(data_len) try: while True: # select on a paramiko ssh channel object does not ever return it in the writable list, so channels don't exactly emulate the socket api r, w, e = select([chan], [], [], TICK) # will wakeup evey TICK seconds to check if something to send, more if something to read (due to select returning chan in readable list) if r: data = chan.recv(BUF_SIZE) if data: self._buffer.write(data) if self._server_capabilities: if 'urn:ietf:params:netconf:base:1.1' in self._server_capabilities and 'urn:ietf:params:netconf:base:1.1' in self._client_capabilities: logger.debug("Selecting netconf:base:1.1 for encoding") self._parse11() elif 'urn:ietf:params:netconf:base:1.0' in self._server_capabilities or 'urn:ietf:params:xml:ns:netconf:base:1.0' in self._server_capabilities or 'urn:ietf:params:netconf:base:1.0' in self._client_capabilities: logger.debug("Selecting netconf:base:1.0 for encoding") self._parse10() else: raise Exception else: self._parse10() # HELLO msg uses EOM markers. else: raise SessionCloseError(self._buffer.getvalue()) if not q.empty() and chan.send_ready(): logger.debug("Sending message") data = q.get() try: # send a HELLO msg using v1.0 EOM markers. validated_element(data, tags='{urn:ietf:params:xml:ns:netconf:base:1.0}hello') data = "%s%s"%(data, MSG_DELIM) except XMLError: # this is not a HELLO msg # we publish v1.1 support if 'urn:ietf:params:netconf:base:1.1' in self._client_capabilities: if self._server_capabilities: if 'urn:ietf:params:netconf:base:1.1' in self._server_capabilities: # send using v1.1 chunked framing data = "%s%s%s"%(start_delim(len(data)), data, END_DELIM) elif 'urn:ietf:params:netconf:base:1.0' in self._server_capabilities or 'urn:ietf:params:xml:ns:netconf:base:1.0' in self._server_capabilities: # send using v1.0 EOM markers data = "%s%s"%(data, MSG_DELIM) else: raise Exception else: logger.debug('HELLO msg was sent, but server capabilities are still not known') raise Exception # we publish only v1.0 support else: # send using v1.0 EOM markers data = "%s%s"%(data, MSG_DELIM) finally: logger.debug("Sending: %s", data) while data: n = chan.send(data) if n <= 0: raise SessionCloseError(self._buffer.getvalue(), data) data = data[n:] except Exception as e: logger.debug("Broke out of main loop, error=%r", e) self._dispatch_error(e) self.close() @property def transport(self): "Underlying `paramiko.Transport <http://www.lag.net/paramiko/docs/paramiko.Transport-class.html>`_ object. This makes it possible to call methods like :meth:`~paramiko.Transport.set_keepalive` on it." return self._transport
class DocStreamReceiver(ReadBodyProtocol): """ A protocol implementation that can parse incoming data from server based on a line format specified on u1db implementation. Except that we split doc attributes from content to ease parsing and increment throughput for larger documents. [\r\n {metadata},\r\n {doc_info},\r\n {content},\r\n ... {doc_info},\r\n {content},\r\n ] """ def __init__(self, response, deferred, doc_reader): self.deferred = deferred self.status = response.code if response else None self.message = response.phrase if response else None self.headers = response.headers if response else {} self.delimiter = '\r\n' self.metadata = '' self._doc_reader = doc_reader self.reset() def reset(self): self._line = 0 self._buffer = StringIO() self._properly_finished = False def connectionLost(self, reason): """ Deliver the accumulated response bytes to the waiting L{Deferred}, if the response body has been completely received without error. """ if self.deferred.called: return try: if reason.check(ResponseDone): self.dataBuffer = self.metadata else: self.dataBuffer = self.finish() except errors.BrokenSyncStream as e: return self.deferred.errback(e) return ReadBodyProtocol.connectionLost(self, reason) def consumeBufferLines(self): """ Consumes lines from buffer and rewind it, writing remaining data that didn't formed a line back into buffer. """ content = self._buffer.getvalue()[0:self._buffer.tell()] self._buffer.seek(0) lines = content.split(self.delimiter) self._buffer.write(lines.pop(-1)) return lines def dataReceived(self, data): """ Buffer incoming data until a line breaks comes in. We check only the incoming data for efficiency. """ self._buffer.write(data) if '\n' not in data: return lines = self.consumeBufferLines() while lines: line, _ = utils.check_and_strip_comma(lines.pop(0)) self.lineReceived(line) self._line += 1 def lineReceived(self, line): """ Protocol implementation. 0: [\r\n 1: {metadata},\r\n (even): {doc_info},\r\n (odd): {data},\r\n (last): ] """ if self._properly_finished: raise errors.BrokenSyncStream("Reading a finished stream") if ']' == line: self._properly_finished = True elif self._line == 0: if line is not '[': raise errors.BrokenSyncStream("Invalid start") elif self._line == 1: self.metadata = line if 'error' in self.metadata: raise errors.BrokenSyncStream("Error from server: %s" % line) self.total = json.loads(line).get('number_of_changes', -1) elif (self._line % 2) == 0: self.current_doc = json.loads(line) if 'error' in self.current_doc: raise errors.BrokenSyncStream("Error from server: %s" % line) else: d = self._doc_reader(self.current_doc, line.strip() or None, self.total) d.addErrback(self.deferred.errback) def finish(self): """ Checks that ']' came and stream was properly closed. """ if not self._properly_finished: raise errors.BrokenSyncStream('Stream not properly closed') content = self._buffer.getvalue()[0:self._buffer.tell()] self._buffer.close() return content
def create_file_reader(input_files, topology, featurizer, chunksize=None, **kw): r""" Creates a (possibly featured) file reader by a number of input files and either a topology file or a featurizer. Parameters ---------- :param input_files: A single input file or a list of input files. :param topology: A topology file. If given, the featurizer argument can be None. :param featurizer: A featurizer. If given, the topology file can be None. :param chunksize: The chunk size with which the corresponding reader gets initialized. :return: Returns the reader. """ from pyerna.coordinates.data.numpy_filereader import NumPyFileReader from pyerna.coordinates.data.py_csv_reader import PyCSVReader from pyerna.coordinates.data import FeatureReader from pyerna.coordinates.data.fragmented_trajectory_reader import FragmentedTrajectoryReader # fragmented trajectories if (isinstance(input_files, (list, tuple)) and len(input_files) > 0 and any(isinstance(item, (list, tuple)) for item in input_files)): return FragmentedTrajectoryReader(input_files, topology, chunksize, featurizer) # normal trajectories if (isinstance(input_files, str) or (isinstance(input_files, (list, tuple)) and (any(isinstance(item, str) for item in input_files) or len(input_files) is 0))): reader = None # check: if single string create a one-element list if isinstance(input_files, str): input_list = [input_files] elif len(input_files) > 0 and all( isinstance(item, str) for item in input_files): input_list = input_files else: if len(input_files) is 0: raise ValueError("The passed input list should not be empty.") else: raise ValueError( "The passed list did not exclusively contain strings or was a list of lists " "(fragmented trajectory).") # TODO: this does not handle suffixes like .xyz.gz (rare) _, suffix = os.path.splitext(input_list[0]) suffix = str(suffix) # check: do all files have the same file type? If not: raise ValueError. if all(item.endswith(suffix) for item in input_list): # do all the files exist? If not: Raise value error all_exist = True from six import StringIO err_msg = StringIO() for item in input_list: if not os.path.isfile(item): err_msg.write('\n' if err_msg.tell() > 0 else "") err_msg.write('File %s did not exist or was no file' % item) all_exist = False if not all_exist: raise ValueError( 'Some of the given input files were directories' ' or did not exist:\n%s' % err_msg.getvalue()) featurizer_or_top_provided = featurizer is not None or topology is not None # we need to check for h5 first, because of mdtraj custom HDF5 traj format (which is deprecated). if suffix in ('.h5', '.hdf5') and not featurizer_or_top_provided: # This check is potentially expensive for lots of files, we also re-open the file twice (causing atime updates etc.) # So we simply require that no featurizer option is given. # and not all((_is_mdtraj_hdf5_file(f) for f in input_files)): from pyerna.coordinates.data.h5_reader import H5Reader reader = H5Reader(filenames=input_files, chunk_size=chunksize, **kw) # CASE 1.1: file types are MD files elif FeatureReader.supports_format(suffix): # check: do we either have a featurizer or a topology file name? If not: raise ValueError. # create a MD reader with file names and topology if not featurizer_or_top_provided: raise ValueError( 'The input files were MD files which makes it mandatory to have either a ' 'Featurizer or a topology file.') if suffix in ('.pdb', '.pdb.gz'): raise ValueError( 'PyEMMA can not read PDB-fake-trajectories. ' 'Please consider using a sane trajectory format (e.g. xtc, dcd).' ) reader = FeatureReader(input_list, featurizer=featurizer, topologyfile=topology, chunksize=chunksize) elif suffix in ('.npy', '.npz'): reader = NumPyFileReader(input_list, chunksize=chunksize) # otherwise we assume that given files are ascii tabulated data else: reader = PyCSVReader(input_list, chunksize=chunksize, **kw) else: raise ValueError( 'Not all elements in the input list were of the type %s!' % suffix) else: raise ValueError( 'Input "{}" was no string or list of strings.'.format(input_files)) return reader
class NormalHTTPFile(HTTPFile): def __init__(self, path, devid, backup_dests=None, mg=None, fid=None, cls=None, key=None, create_close_arg=None, **kwds): super(NormalHTTPFile, self).__init__(mg, fid, key, cls, create_close_arg) if backup_dests is None: backup_dests = [] self._fp = StringIO() self._paths = [(devid, path)] + list(backup_dests) self._is_closed = 0 def paths(self): return self._paths def read(self, n=-1): return self._fp.read(n) def readline(self, *args, **kwds): return self._fp.readline(*args, **kwds) def readlines(self, *args, **kwds): return self._fp.readlines(*args, **kwds) def write(self, content): self._fp.write(content) def close(self): if not self._is_closed: self._is_closed = True # content = self._fp.getvalue() # self._fp.close() for tried_devid, tried_path in self._paths: try: # self._request(tried_path, "PUT", content) self._fp.seek(0) put.putfile(self._fp, tried_path) devid = tried_devid path = tried_path break except HTTPError as e: continue else: devid = None path = None self._fp.seek(0, 2) size = self._fp.tell() self._fp.close() if devid: params = { 'fid': self.fid, 'domain': self.mg.domain, 'key': self.key, 'path': path, 'devid': devid, 'size': size } if self.create_close_arg: params.update(self.create_close_arg) try: self.mg.backend.do_request('create_close', params) except MogileFSError as e: if e.err != 'empty_file': raise def seek(self, pos, mode=0): return self._fp.seek(pos, mode) def tell(self): return self._fp.tell()
def _flushHeaderMessage(self, buf, wout, wsz): """Write a message for CLIENT_TYPE.HEADER This method writes a message using the same logic as THeaderTransport._flushHeaderMessage but mutates fields included in self._fuzz_fields """ transform_data = StringIO() num_transforms = len(self._THeaderTransport__write_transforms) for trans_id in self._THeaderTransport__write_transforms: trans_id = self._get_fuzzy_field('transform_id', trans_id, 'i32') transform_data.write(getVarint(trans_id)) # Add in special flags. if self._THeaderTransport__identity: id_version = self._get_fuzzy_field( 'version', self.ID_VERSION, 'str') self._THeaderTransport__write_headers[self.ID_VERSION_HEADER] = ( id_version) identity = self._get_fuzzy_field( 'identity', self._THeaderTransport__identity, 'str') self._THeaderTransport__write_headers[self.IDENTITY_HEADER] = ( identity) info_data = StringIO() # Write persistent kv-headers _flush_info_headers( info_data, self._THeaderTransport__write_persistent_headers, INFO.PERSISTENT) # Write non-persistent kv-headers _flush_info_headers( info_data, self._THeaderTransport__write_headers, INFO.NORMAL) header_data = StringIO() proto_id = self._get_fuzzy_field( 'proto_id', self._THeaderTransport__proto_id, 'i32') header_data.write(getVarint(proto_id)) num_transforms = self._get_fuzzy_field( 'num_transforms', num_transforms, 'i32') header_data.write(getVarint(num_transforms)) header_size = (transform_data.tell() + header_data.tell() + info_data.tell()) padding_size = 4 - (header_size % 4) # Fuzz padding size, but do not let total header size exceed 2**16 - 1 padding_size = min((2 ** 16 - 1) - header_size, self._get_fuzzy_field('padding', padding_size, 'i16')) header_size = header_size + padding_size wsz += header_size + 10 self._write_fuzzy_field(buf, 'length', wsz, 'i32') self._write_fuzzy_field(buf, 'magic', HEADER_MAGIC >> 16, 'i16') self._write_fuzzy_field( buf, 'flags', self.header_flags(), 'i16') self._write_fuzzy_field( buf, 'seq_id', self.seq_id, 'i32') self._write_fuzzy_field(buf, 'header_size', header_size // 4, 'i16') buf.write(header_data.getvalue()) buf.write(transform_data.getvalue()) buf.write(info_data.getvalue()) # Pad out the header with 0x00 if 'padding_bytes' in self._fuzz_fields: # Print that padding bytes are being fuzzed in favor of printing # the value of each individual padding byte self._print("Fuzzing %d padding bytes" % padding_size) old_verbose, self._verbose = self._verbose, False for _ in sm.xrange(padding_size): self._write_fuzzy_field( buf, 'padding_bytes', six.int2byte(0), 'char') self._verbose = old_verbose self._write_fuzzy_field(buf, 'payload', wout, 'str')
class NormalHTTPFile(HTTPFile): def __init__(self, path, devid, backup_dests=None, mg=None, fid=None, cls=None, key=None, create_close_arg=None, **kwds): super(NormalHTTPFile, self).__init__(mg, fid, key, cls, create_close_arg) if backup_dests is None: backup_dests = [] self._fp = StringIO() self._paths = [(devid, path)] + list(backup_dests) self._is_closed = 0 def paths(self): return self._paths def read(self, n= -1): return self._fp.read(n) def readline(self, *args, **kwds): return self._fp.readline(*args, **kwds) def readlines(self, *args, **kwds): return self._fp.readlines(*args, **kwds) def write(self, content): self._fp.write(content) def close(self): if not self._is_closed: self._is_closed = True # content = self._fp.getvalue() # self._fp.close() for tried_devid, tried_path in self._paths: try: # self._request(tried_path, "PUT", content) self._fp.seek(0) put.putfile(self._fp, tried_path) devid = tried_devid path = tried_path break except HTTPError as e: continue else: devid = None path = None self._fp.seek(0, 2) size = self._fp.tell() self._fp.close() if devid: params = { 'fid' : self.fid, 'domain': self.mg.domain, 'key' : self.key, 'path' : path, 'devid' : devid, 'size' : size } if self.create_close_arg: params.update(self.create_close_arg) try: self.mg.backend.do_request('create_close', params) except MogileFSError as e: if e.err != 'empty_file': raise def seek(self, pos, mode=0): return self._fp.seek(pos, mode) def tell(self): return self._fp.tell()
def _flushHeaderMessage(self, buf, wout, wsz): """Write a message for CLIENT_TYPE.HEADER This method writes a message using the same logic as THeaderTransport._flushHeaderMessage but mutates fields included in self._fuzz_fields """ cls = self.__class__ transform_data = StringIO() num_transforms = len(self._THeaderTransport__write_transforms) for trans_id in self._THeaderTransport__write_transforms: trans_id = self._get_fuzzy_field('transform_id', trans_id, 'i32') transform_data.write(getVarint(trans_id)) # Add in special flags. if self._THeaderTransport__identity: id_version = self._get_fuzzy_field( 'version', self.ID_VERSION, 'str') self._THeaderTransport__write_headers[self.ID_VERSION_HEADER] = ( id_version) identity = self._get_fuzzy_field( 'identity', self._THeaderTransport__identity, 'str') self._THeaderTransport__write_headers[self.IDENTITY_HEADER] = ( identity) info_data = StringIO() # Write persistent kv-headers _flush_info_headers( info_data, self._THeaderTransport__write_persistent_headers, INFO.PERSISTENT) # Write non-persistent kv-headers _flush_info_headers( info_data, self._THeaderTransport__write_headers, INFO.NORMAL) header_data = StringIO() proto_id = self._get_fuzzy_field( 'proto_id', self._THeaderTransport__proto_id, 'i32') header_data.write(getVarint(proto_id)) num_transforms = self._get_fuzzy_field( 'num_transforms', num_transforms, 'i32') header_data.write(getVarint(num_transforms)) header_size = (transform_data.tell() + header_data.tell() + info_data.tell()) padding_size = 4 - (header_size % 4) # Fuzz padding size, but do not let total header size exceed 2**16 - 1 padding_size = min((2 ** 16 - 1) - header_size, self._get_fuzzy_field('padding', padding_size, 'i16')) header_size = header_size + padding_size wsz += header_size + 10 self._write_fuzzy_field(buf, 'length', wsz, 'i32') self._write_fuzzy_field(buf, 'magic', HEADER_MAGIC >> 16, 'i16') self._write_fuzzy_field( buf, 'flags', self.header_flags(), 'i16') self._write_fuzzy_field( buf, 'seq_id', self.seq_id, 'i32') self._write_fuzzy_field(buf, 'header_size', header_size // 4, 'i16') buf.write(header_data.getvalue()) buf.write(transform_data.getvalue()) buf.write(info_data.getvalue()) # Pad out the header with 0x00 if 'padding_bytes' in self._fuzz_fields: # Print that padding bytes are being fuzzed in favor of printing # the value of each individual padding byte self._print("Fuzzing %d padding bytes" % padding_size) old_verbose, self._verbose = self._verbose, False for _ in sm.xrange(padding_size): self._write_fuzzy_field( buf, 'padding_bytes', six.int2byte(0), 'char') self._verbose = old_verbose self._write_fuzzy_field(buf, 'payload', wout, 'str')
def _flushHeaderMessage(self, buf, wout, wsz): """Write a message for self.HEADERS_CLIENT_TYPE This method writes a message using the same logic as THeaderTransport._flushHeaderMessage but mutates fields included in self._fuzz_fields """ cls = self.__class__ transform_data = StringIO() num_transforms = len(self._THeaderTransport__write_transforms) for trans_id in self._THeaderTransport__write_transforms: trans_id = self._get_fuzzy_field('transform_id', trans_id, 'i32') transform_data.write(getVarint(trans_id)) if self._THeaderTransport__hmac_func: num_transforms += 1 transform_data.write(getVarint(self.HMAC_TRANSFORM)) transform_data.write(b'\0') # size of hmac, fixup later. # Add in special flags. if self._THeaderTransport__identity: id_version = self._get_fuzzy_field('version', self.ID_VERSION, 'str') self._THeaderTransport__write_headers[self.ID_VERSION_HEADER] = ( id_version) identity = self._get_fuzzy_field('identity', self._THeaderTransport__identity, 'str') self._THeaderTransport__write_headers[self.IDENTITY_HEADER] = ( identity) info_data = StringIO() # Write persistent kv-headers cls._flush_info_headers( info_data, self._THeaderTransport__write_persistent_headers, self.INFO_PKEYVALUE) # Write non-persistent kv-headers cls._flush_info_headers(info_data, self._THeaderTransport__write_headers, self.INFO_KEYVALUE) header_data = StringIO() proto_id = self._get_fuzzy_field('proto_id', self._THeaderTransport__proto_id, 'i32') header_data.write(getVarint(proto_id)) num_transforms = self._get_fuzzy_field('num_transforms', num_transforms, 'i32') header_data.write(getVarint(num_transforms)) header_size = (transform_data.tell() + header_data.tell() + info_data.tell()) padding_size = 4 - (header_size % 4) # Fuzz padding size, but do not let total header size exceed 2**16 - 1 padding_size = min((2**16 - 1) - header_size, self._get_fuzzy_field('padding', padding_size, 'i16')) header_size = header_size + padding_size wsz += header_size + 10 self._write_fuzzy_field(buf, 'length', wsz, 'i32') self._write_fuzzy_field(buf, 'magic', self.HEADER_MAGIC, 'i16') self._write_fuzzy_field(buf, 'flags', self._THeaderTransport__flags, 'i16') self._write_fuzzy_field(buf, 'seq_id', self._THeaderTransport__seq_id, 'i32') self._write_fuzzy_field(buf, 'header_size', header_size // 4, 'i16') buf.write(header_data.getvalue()) buf.write(transform_data.getvalue()) hmac_loc = buf.tell() - 1 # Fixup hmac size later buf.write(info_data.getvalue()) # Pad out the header with 0x00 if 'padding_bytes' in self._fuzz_fields: # Print that padding bytes are being fuzzed in favor of printing # the value of each individual padding byte self._print("Fuzzing %d padding bytes" % padding_size) old_verbose, self._verbose = self._verbose, False for _ in sm.xrange(padding_size): self._write_fuzzy_field(buf, 'padding_bytes', six.int2byte(0), 'char') self._verbose = old_verbose self._write_fuzzy_field(buf, 'payload', wout, 'str') # HMAC calculation should always be last. if self._THeaderTransport__hmac_func: hmac_data = buf.getvalue()[4:] hmac = self._THeaderTransport__hmac_func(hmac_data) # Fill in hmac size. buf.seek(hmac_loc) self._write_hmac_size(buf, hmac) buf.seek(0, os.SEEK_END) self._write_hmac(buf, hmac) # Fix packet size since we appended data. new_sz = buf.tell() - 4 buf.seek(0) self._write_wsz(buf, new_sz)
def sitemap_generator(request, maps, page, current_site): output = StringIO() xml = SimplerXMLGenerator(output, settings.DEFAULT_CHARSET) xml.startDocument() ns = { 'xmlns':'http://www.sitemaps.org/schemas/sitemap/0.9', 'xmlns:image':"http://www.google.com/schemas/sitemap-image/1.1", } xml.startElement('urlset', ns) yield output.getvalue() pos = output.tell() for site in maps: if callable(site): if issubclass(site, RequestSitemap): site = site(request=request) else: site = site() elif hasattr(site, 'request'): site.request = request for url in site.get_urls(page=page, site=current_site): xml.startElement('url', {}) xml.addQuickElement('loc', url['location']) try: if url['lastmod']: xml.addQuickElement('lastmod', url['lastmod'].strftime('%Y-%m-%d')) except (KeyError, AttributeError): pass try: if url['changefreq']: xml.addQuickElement('changefreq', url['changefreq']) except KeyError: pass try: if url['priority']: xml.addQuickElement('priority', url['priority']) except KeyError: pass try: # This will generate image links, if the item has an 'image' attribute img = url['item'].image xml.startElement('image:image', {}) xml.addQuickElement('image:loc', request.build_absolute_uri(img.url)) try: # if it also has name and description attributes, it will add those # to the image sitemaps xml.addQuickElement('image:title', url['item'].name) xml.addQuickElement('image:caption', url['item'].description) except: pass xml.endElement('image:image') except: pass xml.endElement('url') output.seek(pos) yield output.read() pos = output.tell() xml.endElement('urlset') xml.endDocument() output.seek(pos) last = output.read() output.close() yield last
class BufferedFile (object): """ Reusable base class to implement python-style file buffering around a simpler stream. """ _DEFAULT_BUFSIZE = 8192 SEEK_SET = 0 SEEK_CUR = 1 SEEK_END = 2 FLAG_READ = 0x1 FLAG_WRITE = 0x2 FLAG_APPEND = 0x4 FLAG_BINARY = 0x10 FLAG_BUFFERED = 0x20 FLAG_LINE_BUFFERED = 0x40 FLAG_UNIVERSAL_NEWLINE = 0x80 def __init__(self): self.newlines = None self._flags = 0 self._bufsize = self._DEFAULT_BUFSIZE self._wbuffer = StringIO() self._rbuffer = '' self._at_trailing_cr = False self._closed = False # pos - position within the file, according to the user # realpos - position according the OS # (these may be different because we buffer for line reading) self._pos = self._realpos = 0 # size only matters for seekable files self._size = 0 def __del__(self): self.close() def __iter__(self): """ Returns an iterator that can be used to iterate over the lines in this file. This iterator happens to return the file itself, since a file is its own iterator. @raise ValueError: if the file is closed. @return: an interator. @rtype: iterator """ if self._closed: raise ValueError('I/O operation on closed file') return self def close(self): """ Close the file. Future read and write operations will fail. """ self.flush() self._closed = True def flush(self): """ Write out any data in the write buffer. This may do nothing if write buffering is not turned on. """ self._write_all(self._wbuffer.getvalue()) self._wbuffer = StringIO() return def next(self): """ Returns the next line from the input, or raises L{StopIteration} when EOF is hit. Unlike python file objects, it's okay to mix calls to C{next} and L{readline}. @raise StopIteration: when the end of the file is reached. @return: a line read from the file. @rtype: str """ line = self.readline() if not line: raise StopIteration return line def read(self, size=None): """ Read at most C{size} bytes from the file (less if we hit the end of the file first). If the C{size} argument is negative or omitted, read all the remaining data in the file. @param size: maximum number of bytes to read @type size: int @return: data read from the file, or an empty string if EOF was encountered immediately @rtype: str """ if self._closed: raise IOError('File is closed') if not (self._flags & self.FLAG_READ): raise IOError('File is not open for reading') if (size is None) or (size < 0): # go for broke result = self._rbuffer self._rbuffer = '' self._pos += len(result) while True: try: new_data = self._read(self._DEFAULT_BUFSIZE) except EOFError: new_data = None if (new_data is None) or (len(new_data) == 0): break result += new_data self._realpos += len(new_data) self._pos += len(new_data) return result if size <= len(self._rbuffer): result = self._rbuffer[:size] self._rbuffer = self._rbuffer[size:] self._pos += len(result) return result while len(self._rbuffer) < size: read_size = size - len(self._rbuffer) if self._flags & self.FLAG_BUFFERED: read_size = max(self._bufsize, read_size) try: new_data = self._read(read_size) except EOFError: new_data = None if (new_data is None) or (len(new_data) == 0): break self._rbuffer += new_data self._realpos += len(new_data) result = self._rbuffer[:size] self._rbuffer = self._rbuffer[size:] self._pos += len(result) return result def readline(self, size=None): """ Read one entire line from the file. A trailing newline character is kept in the string (but may be absent when a file ends with an incomplete line). If the size argument is present and non-negative, it is a maximum byte count (including the trailing newline) and an incomplete line may be returned. An empty string is returned only when EOF is encountered immediately. @note: Unlike stdio's C{fgets()}, the returned string contains null characters (C{'\\0'}) if they occurred in the input. @param size: maximum length of returned string. @type size: int @return: next line of the file, or an empty string if the end of the file has been reached. @rtype: str """ # it's almost silly how complex this function is. if self._closed: raise IOError('File is closed') if not (self._flags & self.FLAG_READ): raise IOError('File not open for reading') line = self._rbuffer while True: if self._at_trailing_cr and (self._flags & self.FLAG_UNIVERSAL_NEWLINE) and (len(line) > 0): # edge case: the newline may be '\r\n' and we may have read # only the first '\r' last time. if line[0] == '\n': line = line[1:] self._record_newline('\r\n') else: self._record_newline('\r') self._at_trailing_cr = False # check size before looking for a linefeed, in case we already have # enough. if (size is not None) and (size >= 0): if len(line) >= size: # truncate line and return self._rbuffer = line[size:] line = line[:size] self._pos += len(line) return line n = size - len(line) else: n = self._bufsize if ('\n' in line) or ((self._flags & self.FLAG_UNIVERSAL_NEWLINE) and ('\r' in line)): break try: new_data = self._read(n) except EOFError: new_data = None if (new_data is None) or (len(new_data) == 0): self._rbuffer = '' self._pos += len(line) return line line += new_data self._realpos += len(new_data) # find the newline pos = line.find('\n') if self._flags & self.FLAG_UNIVERSAL_NEWLINE: rpos = line.find('\r') if (rpos >= 0) and ((rpos < pos) or (pos < 0)): pos = rpos xpos = pos + 1 if (line[pos] == '\r') and (xpos < len(line)) and (line[xpos] == '\n'): xpos += 1 self._rbuffer = line[xpos:] lf = line[pos:xpos] line = line[:pos] + '\n' if (len(self._rbuffer) == 0) and (lf == '\r'): # we could read the line up to a '\r' and there could still be a # '\n' following that we read next time. note that and eat it. self._at_trailing_cr = True else: self._record_newline(lf) self._pos += len(line) return line def readlines(self, sizehint=None): """ Read all remaining lines using L{readline} and return them as a list. If the optional C{sizehint} argument is present, instead of reading up to EOF, whole lines totalling approximately sizehint bytes (possibly after rounding up to an internal buffer size) are read. @param sizehint: desired maximum number of bytes to read. @type sizehint: int @return: list of lines read from the file. @rtype: list """ lines = [] bytes = 0 while True: line = self.readline() if len(line) == 0: break lines.append(line) bytes += len(line) if (sizehint is not None) and (bytes >= sizehint): break return lines def seek(self, offset, whence=0): """ Set the file's current position, like stdio's C{fseek}. Not all file objects support seeking. @note: If a file is opened in append mode (C{'a'} or C{'a+'}), any seek operations will be undone at the next write (as the file position will move back to the end of the file). @param offset: position to move to within the file, relative to C{whence}. @type offset: int @param whence: type of movement: 0 = absolute; 1 = relative to the current position; 2 = relative to the end of the file. @type whence: int @raise IOError: if the file doesn't support random access. """ raise IOError('File does not support seeking.') def tell(self): """ Return the file's current position. This may not be accurate or useful if the underlying file doesn't support random access, or was opened in append mode. @return: file position (in bytes). @rtype: int """ return self._pos def write(self, data): """ Write data to the file. If write buffering is on (C{bufsize} was specified and non-zero), some or all of the data may not actually be written yet. (Use L{flush} or L{close} to force buffered data to be written out.) @param data: data to write. @type data: str """ if self._closed: raise IOError('File is closed') if not (self._flags & self.FLAG_WRITE): raise IOError('File not open for writing') if not (self._flags & self.FLAG_BUFFERED): self._write_all(data) return self._wbuffer.write(data) if self._flags & self.FLAG_LINE_BUFFERED: # only scan the new data for linefeed, to avoid wasting time. last_newline_pos = data.rfind('\n') if last_newline_pos >= 0: wbuf = self._wbuffer.getvalue() last_newline_pos += len(wbuf) - len(data) self._write_all(wbuf[:last_newline_pos + 1]) self._wbuffer = StringIO() self._wbuffer.write(wbuf[last_newline_pos + 1:]) return # even if we're line buffering, if the buffer has grown past the # buffer size, force a flush. if self._wbuffer.tell() >= self._bufsize: self.flush() return def writelines(self, sequence): """ Write a sequence of strings to the file. The sequence can be any iterable object producing strings, typically a list of strings. (The name is intended to match L{readlines}; C{writelines} does not add line separators.) @param sequence: an iterable sequence of strings. @type sequence: sequence """ for line in sequence: self.write(line) return def xreadlines(self): """ Identical to C{iter(f)}. This is a deprecated file interface that predates python iterator support. @return: an iterator. @rtype: iterator """ return self @property def closed(self): return self._closed ### overrides... def _read(self, size): """ I{(subclass override)} Read data from the stream. Return C{None} or raise C{EOFError} to indicate EOF. """ raise EOFError() def _write(self, data): """ I{(subclass override)} Write data into the stream. """ raise IOError('write not implemented') def _get_size(self): """ I{(subclass override)} Return the size of the file. This is called from within L{_set_mode} if the file is opened in append mode, so the file position can be tracked and L{seek} and L{tell} will work correctly. If the file is a stream that can't be randomly accessed, you don't need to override this method, """ return 0 ### internals... def _set_mode(self, mode='r', bufsize=-1): """ Subclasses call this method to initialize the BufferedFile. """ # set bufsize in any event, because it's used for readline(). self._bufsize = self._DEFAULT_BUFSIZE if bufsize < 0: # do no buffering by default, because otherwise writes will get # buffered in a way that will probably confuse people. bufsize = 0 if bufsize == 1: # apparently, line buffering only affects writes. reads are only # buffered if you call readline (directly or indirectly: iterating # over a file will indirectly call readline). self._flags |= self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED elif bufsize > 1: self._bufsize = bufsize self._flags |= self.FLAG_BUFFERED self._flags &= ~self.FLAG_LINE_BUFFERED elif bufsize == 0: # unbuffered self._flags &= ~(self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED) if ('r' in mode) or ('+' in mode): self._flags |= self.FLAG_READ if ('w' in mode) or ('+' in mode): self._flags |= self.FLAG_WRITE if ('a' in mode): self._flags |= self.FLAG_WRITE | self.FLAG_APPEND self._size = self._get_size() self._pos = self._realpos = self._size if ('b' in mode): self._flags |= self.FLAG_BINARY if ('U' in mode): self._flags |= self.FLAG_UNIVERSAL_NEWLINE # built-in file objects have this attribute to store which kinds of # line terminations they've seen: # <http://www.python.org/doc/current/lib/built-in-funcs.html> self.newlines = None def _write_all(self, data): # the underlying stream may be something that does partial writes (like # a socket). while len(data) > 0: count = self._write(data) data = data[count:] if self._flags & self.FLAG_APPEND: self._size += count self._pos = self._realpos = self._size else: self._pos += count self._realpos += count return None def _record_newline(self, newline): # silliness about tracking what kinds of newlines we've seen. # i don't understand why it can be None, a string, or a tuple, instead # of just always being a tuple, but we'll emulate that behavior anyway. if not (self._flags & self.FLAG_UNIVERSAL_NEWLINE): return if self.newlines is None: self.newlines = newline elif (type(self.newlines) is str) and (self.newlines != newline): self.newlines = (self.newlines, newline) elif newline not in self.newlines: self.newlines += (newline,)
class DocStreamReceiver(ReadBodyProtocol): """ A protocol implementation that can parse incoming data from server based on a line format specified on u1db implementation. Except that we split doc attributes from content to ease parsing and increment throughput for larger documents. [\r\n {metadata},\r\n {doc_info},\r\n {content},\r\n ... {doc_info},\r\n {content},\r\n ] """ def __init__(self, response, deferred, doc_reader): self.deferred = deferred self.status = response.code if response else None self.message = response.phrase if response else None self.headers = response.headers if response else {} self.delimiter = '\r\n' self.metadata = '' self._doc_reader = doc_reader self.reset() def reset(self): self._line = 0 self._buffer = StringIO() self._properly_finished = False def connectionLost(self, reason): """ Deliver the accumulated response bytes to the waiting L{Deferred}, if the response body has been completely received without error. """ if self.deferred.called: return try: if reason.check(ResponseDone): self.dataBuffer = self.metadata else: self.dataBuffer = self.finish() except errors.BrokenSyncStream as e: return self.deferred.errback(e) return ReadBodyProtocol.connectionLost(self, reason) def consumeBufferLines(self): """ Consumes lines from buffer and rewind it, writing remaining data that didn't formed a line back into buffer. """ content = self._buffer.getvalue()[0:self._buffer.tell()] self._buffer.seek(0) lines = content.split(self.delimiter) self._buffer.write(lines.pop(-1)) return lines def dataReceived(self, data): """ Buffer incoming data until a line breaks comes in. We check only the incoming data for efficiency. """ self._buffer.write(data) if '\n' not in data: return lines = self.consumeBufferLines() while lines: line, _ = utils.check_and_strip_comma(lines.pop(0)) self.lineReceived(line) self._line += 1 def lineReceived(self, line): """ Protocol implementation. 0: [\r\n 1: {metadata},\r\n (even): {doc_info},\r\n (odd): {data},\r\n (last): ] """ if self._properly_finished: raise errors.BrokenSyncStream("Reading a finished stream") if ']' == line: self._properly_finished = True elif self._line == 0: if line is not '[': raise errors.BrokenSyncStream("Invalid start") elif self._line == 1: self.metadata = line if 'error' in self.metadata: raise errors.BrokenSyncStream("Error from server: %s" % line) self.total = json.loads(line).get('number_of_changes', -1) elif (self._line % 2) == 0: self.current_doc = json.loads(line) if 'error' in self.current_doc: raise errors.BrokenSyncStream("Error from server: %s" % line) else: d = self._doc_reader( self.current_doc, line.strip() or None, self.total) d.addErrback(self.deferred.errback) def finish(self): """ Checks that ']' came and stream was properly closed. """ if not self._properly_finished: raise errors.BrokenSyncStream('Stream not properly closed') content = self._buffer.getvalue()[0:self._buffer.tell()] self._buffer.close() return content
class Message(object): """ An SSH2 I{Message} is a stream of bytes that encodes some combination of strings, integers, bools, and infinite-precision integers (known in python as I{long}s). This class builds or breaks down such a byte stream. Normally you don't need to deal with anything this low-level, but it's exposed for people implementing custom extensions, or features that paramiko doesn't support yet. """ def __init__(self, content=None): """ Create a new SSH2 Message. @param content: the byte stream to use as the Message content (passed in only when decomposing a Message). @type content: string """ if content != None: self.packet = StringIO(content) else: self.packet = StringIO() def __str__(self): """ Return the byte stream content of this Message, as a string. @return: the contents of this Message. @rtype: string """ return self.packet.getvalue() def __repr__(self): """ Returns a string representation of this object, for debugging. @rtype: string """ return "paramiko.Message(" + repr(self.packet.getvalue()) + ")" def rewind(self): """ Rewind the message to the beginning as if no items had been parsed out of it yet. """ self.packet.seek(0) def get_remainder(self): """ Return the bytes of this Message that haven't already been parsed and returned. @return: a string of the bytes not parsed yet. @rtype: string """ position = self.packet.tell() remainder = self.packet.read() self.packet.seek(position) return remainder def get_so_far(self): """ Returns the bytes of this Message that have been parsed and returned. The string passed into a Message's constructor can be regenerated by concatenating C{get_so_far} and L{get_remainder}. @return: a string of the bytes parsed so far. @rtype: string """ position = self.packet.tell() self.rewind() return self.packet.read(position) def get_bytes(self, n): """ Return the next C{n} bytes of the Message, without decomposing into an int, string, etc. Just the raw bytes are returned. @return: a string of the next C{n} bytes of the Message, or a string of C{n} zero bytes, if there aren't C{n} bytes remaining. @rtype: string """ b = self.packet.read(n) if len(b) < n: return b + "\x00" * (n - len(b)) return b def get_byte(self): """ Return the next byte of the Message, without decomposing it. This is equivalent to L{get_bytes(1)<get_bytes>}. @return: the next byte of the Message, or C{'\000'} if there aren't any bytes remaining. @rtype: string """ return self.get_bytes(1) def get_boolean(self): """ Fetch a boolean from the stream. @return: C{True} or C{False} (from the Message). @rtype: bool """ b = self.get_bytes(1) return b != "\x00" def get_int(self): """ Fetch an int from the stream. @return: a 32-bit unsigned integer. @rtype: int """ return struct.unpack(">I", self.get_bytes(4))[0] def get_int64(self): """ Fetch a 64-bit int from the stream. @return: a 64-bit unsigned integer. @rtype: long """ return struct.unpack(">Q", self.get_bytes(8))[0] def get_mpint(self): """ Fetch a long int (mpint) from the stream. @return: an arbitrary-length integer. @rtype: long """ return util.inflate_long(self.get_string()) def get_string(self): """ Fetch a string from the stream. This could be a byte string and may contain unprintable characters. (It's not unheard of for a string to contain another byte-stream Message.) @return: a string. @rtype: string """ return self.get_bytes(self.get_int()) def get_list(self): """ Fetch a list of strings from the stream. These are trivially encoded as comma-separated values in a string. @return: a list of strings. @rtype: list of strings """ return self.get_string().split(",") def add_bytes(self, b): """ Write bytes to the stream, without any formatting. @param b: bytes to add @type b: str """ self.packet.write(b) return self def add_byte(self, b): """ Write a single byte to the stream, without any formatting. @param b: byte to add @type b: str """ self.packet.write(b) return self def add_boolean(self, b): """ Add a boolean value to the stream. @param b: boolean value to add @type b: bool """ if b: self.add_byte("\x01") else: self.add_byte("\x00") return self def add_int(self, n): """ Add an integer to the stream. @param n: integer to add @type n: int """ self.packet.write(struct.pack(">I", n)) return self def add_int64(self, n): """ Add a 64-bit int to the stream. @param n: long int to add @type n: long """ self.packet.write(struct.pack(">Q", n)) return self def add_mpint(self, z): """ Add a long int to the stream, encoded as an infinite-precision integer. This method only works on positive numbers. @param z: long int to add @type z: long """ self.add_string(util.deflate_long(z)) return self def add_string(self, s): """ Add a string to the stream. @param s: string to add @type s: str """ self.add_int(len(s)) self.packet.write(s) return self def add_list(self, l): """ Add a list of strings to the stream. They are encoded identically to a single string of values separated by commas. (Yes, really, that's how SSH2 does it.) @param l: list of strings to add @type l: list(str) """ self.add_string(",".join(l)) return self def _add(self, i): if type(i) is str: return self.add_string(i) elif type(i) is int: return self.add_int(i) elif type(i) is long: if i > long(0xFFFFFFFF): return self.add_mpint(i) else: return self.add_int(i) elif type(i) is bool: return self.add_boolean(i) elif type(i) is list: return self.add_list(i) else: raise Exception("Unknown type") def add(self, *seq): """ Add a sequence of items to the stream. The values are encoded based on their type: str, int, bool, list, or long. @param seq: the sequence of items @type seq: sequence @bug: longs are encoded non-deterministically. Don't use this method. """ for item in seq: self._add(item)
class SSHSession(Session): "Implements a :rfc:`4742` NETCONF session over SSH." def __init__(self, device_handler): capabilities = Capabilities(device_handler.get_capabilities()) Session.__init__(self, capabilities) self._host_keys = paramiko.HostKeys() self._transport = None self._connected = False self._channel = None self._channel_id = None self._channel_name = None self._buffer = StringIO() # parsing-related, see _parse() self._device_handler = device_handler self._parsing_state10 = 0 self._parsing_pos10 = 0 self._parsing_pos11 = 0 self._parsing_state11 = 0 self._expchunksize = 0 self._curchunksize = 0 self._inendpos = 0 self._size_num_list = [] self._message_list = [] def _parse(self): "Messages ae delimited by MSG_DELIM. The buffer could have grown by a maximum of BUF_SIZE bytes everytime this method is called. Retains state across method calls and if a byte has been read it will not be considered again." return self._parse10() def _parse10(self): """Messages are delimited by MSG_DELIM. The buffer could have grown by a maximum of BUF_SIZE bytes everytime this method is called. Retains state across method calls and if a chunk has been read it will not be considered again.""" logger.debug("parsing netconf v1.0") buf = self._buffer buf.seek(self._parsing_pos10) if MSG_DELIM in buf.read().decode('UTF-8'): buf.seek(0) msg, _, remaining = buf.read().decode('UTF-8').partition(MSG_DELIM) msg = msg.strip() if sys.version < '3': self._dispatch_message(msg.encode()) else: self._dispatch_message(msg) # create new buffer which contains remaining of old buffer self._buffer = StringIO() self._buffer.write(remaining.encode()) self._parsing_pos10 = 0 else: # handle case that MSG_DELIM is split over two chunks self._parsing_pos10 = buf.tell() - MSG_DELIM_LEN if self._parsing_pos10 < 0: self._parsing_pos10 = 0 def _parse11(self): logger.debug("parsing netconf v1.1") expchunksize = self._expchunksize curchunksize = self._curchunksize idle, instart, inmsg, inbetween, inend = range(5) state = self._parsing_state11 inendpos = self._inendpos num_list = self._size_num_list MAX_STARTCHUNK_SIZE = 12 # \#+4294967295+\n pre = 'invalid base:1:1 frame' buf = self._buffer buf.seek(self._parsing_pos11) message_list = self._message_list # a message is a list of chunks chunk_list = [] # a chunk is a list of characters while True: x = buf.read(1) if not x: logger.debug('No more data to read') # Store the current chunk to the message list chunk = b''.join(chunk_list) message_list.append(textify(chunk)) break # done reading logger.debug('x: %s', x) if state == idle: if x == b'\n': state = instart inendpos = 1 else: logger.debug('%s (%s: expect newline)' % (pre, state)) raise Exception elif state == instart: if inendpos == 1: if x == b'#': inendpos += 1 else: logger.debug('%s (%s: expect "#")' % (pre, state)) raise Exception elif inendpos == 2: if x.isdigit(): inendpos += 1 # == 3 now # num_list.append(x) else: logger.debug('%s (%s: expect digit)' % (pre, state)) raise Exception else: if inendpos == MAX_STARTCHUNK_SIZE: logger.debug('%s (%s: no. too long)' % (pre, state)) raise Exception elif x == b'\n': num = b''.join(num_list) num_list = [] # Reset num_list try: num = int(num) except: logger.debug('%s (%s: invalid no.)' % (pre, state)) raise Exception else: state = inmsg expchunksize = num logger.debug('response length: %d' % expchunksize) curchunksize = 0 inendpos += 1 elif x.isdigit(): inendpos += 1 # > 3 now # num_list.append(x) else: logger.debug('%s (%s: expect digit)' % (pre, state)) raise Exception elif state == inmsg: chunk_list.append(x) curchunksize += 1 chunkleft = expchunksize - curchunksize if chunkleft == 0: inendpos = 0 state = inbetween chunk = b''.join(chunk_list) message_list.append(textify(chunk)) chunk_list = [] # Reset chunk_list logger.debug('parsed new chunk: %s' % (chunk)) elif state == inbetween: if inendpos == 0: if x == b'\n': inendpos += 1 else: logger.debug('%s (%s: expect newline)' % (pre, state)) raise Exception elif inendpos == 1: if x == b'#': inendpos += 1 else: logger.debug('%s (%s: expect "#")' % (pre, state)) raise Exception else: inendpos += 1 # == 3 now # if x == b'#': state = inend elif x.isdigit(): # More trunks state = instart num_list = [] num_list.append(x) else: logger.debug('%s (%s: expect "#")' % (pre, state)) raise Exception elif state == inend: if inendpos == 3: if x == b'\n': inendpos = 0 state = idle logger.debug('dispatching message') self._dispatch_message(''.join(message_list)) # reset rest = buf.read() buf = BytesIO() buf.write(rest) buf.seek(0) message_list = [] self._message_list = message_list chunk_list = [] expchunksize = chunksize = 0 parsing_state11 = idle inendpos = parsing_pos11 = 0 break else: logger.debug('%s (%s: expect newline)' % (pre, state)) raise Exception else: logger.debug('%s (%s invalid state)' % (pre, state)) raise Exception self._expchunksize = expchunksize self._curchunksize = curchunksize self._parsing_state11 = state self._inendpos = inendpos self._size_num_list = num_list self._buffer = buf self._parsing_pos11 = self._buffer.tell() logger.debug('parse11 ending ...') def load_known_hosts(self, filename=None): """Load host keys from an openssh :file:`known_hosts`-style file. Can be called multiple times. If *filename* is not specified, looks in the default locations i.e. :file:`~/.ssh/known_hosts` and :file:`~/ssh/known_hosts` for Windows. """ if filename is None: filename = os.path.expanduser('~/.ssh/known_hosts') try: self._host_keys.load(filename) except IOError: # for windows filename = os.path.expanduser('~/ssh/known_hosts') try: self._host_keys.load(filename) except IOError: pass else: self._host_keys.load(filename) def close(self): if self._transport.is_active(): self._transport.close() self._channel = None self._connected = False # REMEMBER to update transport.rst if sig. changes, since it is hardcoded there def connect(self, host, port=830, timeout=None, unknown_host_cb=default_unknown_host_cb, username=None, password=None, key_filename=None, allow_agent=True, hostkey_verify=True, look_for_keys=True, ssh_config=None): """Connect via SSH and initialize the NETCONF session. First attempts the publickey authentication method and then password authentication. To disable attempting publickey authentication altogether, call with *allow_agent* and *look_for_keys* as `False`. *host* is the hostname or IP address to connect to *port* is by default 830, but some devices use the default SSH port of 22 so this may need to be specified *timeout* is an optional timeout for socket connect *unknown_host_cb* is called when the server host key is not recognized. It takes two arguments, the hostname and the fingerprint (see the signature of :func:`default_unknown_host_cb`) *username* is the username to use for SSH authentication *password* is the password used if using password authentication, or the passphrase to use for unlocking keys that require it *key_filename* is a filename where a the private key to be used can be found *allow_agent* enables querying SSH agent (if found) for keys *hostkey_verify* enables hostkey verification from ~/.ssh/known_hosts *look_for_keys* enables looking in the usual locations for ssh keys (e.g. :file:`~/.ssh/id_*`) *ssh_config* enables parsing of an OpenSSH configuration file, if set to its path, e.g. :file:`~/.ssh/config` or to True (in this case, use :file:`~/.ssh/config`). """ # Optionaly, parse .ssh/config config = {} if ssh_config is True: ssh_config = "~/.ssh/config" if sys.platform != "win32" else "~/ssh/config" if ssh_config is not None: config = paramiko.SSHConfig() config.parse(open(os.path.expanduser(ssh_config))) config = config.lookup(host) host = config.get("hostname", host) if username is None: username = config.get("user") if key_filename is None: key_filename = config.get("identityfile") if username is None: username = getpass.getuser() sock = None if config.get("proxycommand"): sock = paramiko.proxy.ProxyCommand(config.get("proxycommand")) else: for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: sock = socket.socket(af, socktype, proto) sock.settimeout(timeout) except socket.error: continue try: sock.connect(sa) except socket.error: sock.close() continue break else: raise SSHError("Could not open socket to %s:%s" % (host, port)) t = self._transport = paramiko.Transport(sock) t.set_log_channel(logger.name) if config.get("compression") == 'yes': t.use_compression() try: t.start_client() except paramiko.SSHException: raise SSHError('Negotiation failed') # host key verification server_key = t.get_remote_server_key() fingerprint = _colonify(hexlify(server_key.get_fingerprint())) if hostkey_verify: known_host = self._host_keys.check(host, server_key) if not known_host and not unknown_host_cb(host, fingerprint): raise SSHUnknownHostError(host, fingerprint) if key_filename is None: key_filenames = [] elif isinstance(key_filename, (str, bytes)): key_filenames = [key_filename] else: key_filenames = key_filename self._auth(username, password, key_filenames, allow_agent, look_for_keys) self._connected = True # there was no error authenticating # TODO: leopoul: Review, test, and if needed rewrite this part subsystem_names = self._device_handler.get_ssh_subsystem_names() for subname in subsystem_names: c = self._channel = self._transport.open_session() self._channel_id = c.get_id() channel_name = "%s-subsystem-%s" % (subname, str(self._channel_id)) c.set_name(channel_name) try: c.invoke_subsystem(subname) except paramiko.SSHException as e: logger.info("%s (subsystem request rejected)", e) handle_exception = self._device_handler.handle_connection_exceptions( self) # Ignore the exception, since we continue to try the different # subsystem names until we find one that can connect. #have to handle exception for each vendor here if not handle_exception: continue self._channel_name = c.get_name() self._post_connect() return raise SSHError( "Could not open connection, possibly due to unacceptable" " SSH subsystem name.") def _auth(self, username, password, key_filenames, allow_agent, look_for_keys): saved_exception = None for key_filename in key_filenames: for cls in (paramiko.RSAKey, paramiko.DSSKey): try: key = cls.from_private_key_file(key_filename, password) logger.debug( "Trying key %s from %s" % (hexlify(key.get_fingerprint()), key_filename)) self._transport.auth_publickey(username, key) return except Exception as e: saved_exception = e logger.debug(e) if allow_agent: for key in paramiko.Agent().get_keys(): try: logger.debug("Trying SSH agent key %s" % hexlify(key.get_fingerprint())) self._transport.auth_publickey(username, key) return except Exception as e: saved_exception = e logger.debug(e) keyfiles = [] if look_for_keys: rsa_key = os.path.expanduser("~/.ssh/id_rsa") dsa_key = os.path.expanduser("~/.ssh/id_dsa") if os.path.isfile(rsa_key): keyfiles.append((paramiko.RSAKey, rsa_key)) if os.path.isfile(dsa_key): keyfiles.append((paramiko.DSSKey, dsa_key)) # look in ~/ssh/ for windows users: rsa_key = os.path.expanduser("~/ssh/id_rsa") dsa_key = os.path.expanduser("~/ssh/id_dsa") if os.path.isfile(rsa_key): keyfiles.append((paramiko.RSAKey, rsa_key)) if os.path.isfile(dsa_key): keyfiles.append((paramiko.DSSKey, dsa_key)) for cls, filename in keyfiles: try: key = cls.from_private_key_file(filename, password) logger.debug("Trying discovered key %s in %s" % (hexlify(key.get_fingerprint()), filename)) self._transport.auth_publickey(username, key) return except Exception as e: saved_exception = e logger.debug(e) if password is not None: try: self._transport.auth_password(username, password) return except Exception as e: saved_exception = e logger.debug(e) if saved_exception is not None: # need pep-3134 to do this right raise AuthenticationError(repr(saved_exception)) raise AuthenticationError("No authentication methods available") def run(self): chan = self._channel q = self._q def start_delim(data_len): return '\n#%s\n' % (data_len) try: while True: # select on a paramiko ssh channel object does not ever return it in the writable list, so channels don't exactly emulate the socket api r, w, e = select([chan], [], [], TICK) # will wakeup evey TICK seconds to check if something to send, more if something to read (due to select returning chan in readable list) if r: data = chan.recv(BUF_SIZE) if data: self._buffer.write(data) if self._server_capabilities: if 'urn:ietf:params:netconf:base:1.1' in self._server_capabilities and 'urn:ietf:params:netconf:base:1.1' in self._client_capabilities: logger.debug( "Selecting netconf:base:1.1 for encoding") self._parse11() elif 'urn:ietf:params:netconf:base:1.0' in self._server_capabilities or 'urn:ietf:params:xml:ns:netconf:base:1.0' in self._server_capabilities or 'urn:ietf:params:netconf:base:1.0' in self._client_capabilities: logger.debug( "Selecting netconf:base:1.0 for encoding") self._parse10() else: raise Exception else: self._parse10() # HELLO msg uses EOM markers. else: raise SessionCloseError(self._buffer.getvalue()) if not q.empty() and chan.send_ready(): logger.debug("Sending message") data = q.get() try: # send a HELLO msg using v1.0 EOM markers. validated_element( data, tags= '{urn:ietf:params:xml:ns:netconf:base:1.0}hello') data = "%s%s" % (data, MSG_DELIM) except XMLError: # this is not a HELLO msg # we publish v1.1 support if 'urn:ietf:params:netconf:base:1.1' in self._client_capabilities: if self._server_capabilities: if 'urn:ietf:params:netconf:base:1.1' in self._server_capabilities: # send using v1.1 chunked framing data = "%s%s%s" % (start_delim( len(data)), data, END_DELIM) elif 'urn:ietf:params:netconf:base:1.0' in self._server_capabilities or 'urn:ietf:params:xml:ns:netconf:base:1.0' in self._server_capabilities: # send using v1.0 EOM markers data = "%s%s" % (data, MSG_DELIM) else: raise Exception else: logger.debug( 'HELLO msg was sent, but server capabilities are still not known' ) raise Exception # we publish only v1.0 support else: # send using v1.0 EOM markers data = "%s%s" % (data, MSG_DELIM) finally: logger.debug("Sending: %s", data) while data: n = chan.send(data) if n <= 0: raise SessionCloseError( self._buffer.getvalue(), data) data = data[n:] except Exception as e: logger.debug("Broke out of main loop, error=%r", e) self._dispatch_error(e) self.close() @property def transport(self): "Underlying `paramiko.Transport <http://www.lag.net/paramiko/docs/paramiko.Transport-class.html>`_ object. This makes it possible to call methods like :meth:`~paramiko.Transport.set_keepalive` on it." return self._transport