def __init__(self): assert isinstance(self.chunkMark, str), 'Invalid chunk mark %s' % self.chunkMark assert isinstance(self.chunkEnding, str), 'Invalid chunk ending %s' % self.chunkEnding assert isinstance(self.maximumChunkSize, int), 'Invalid stream chunck size %s' % self.maximumChunkSize super().__init__() self._chunkEndingBytes = ascii_encode(self.chunkEnding)[0]
def startElement(self, name, attrs): name = codecs.ascii_encode(name)[0] #parent = self._parents[-1] #print name, "under", parent._name #print attrs._attrs self._parents.append(Child(name, attrs)) self._parents[-2]._children.append(self._parents[-1])
def startElement(self, name, attrs): name = codecs.ascii_encode(name)[0] #parent = self._parents[-1] #print name, "under", parent._name #print attrs._attrs self._parents.append(Child(name,attrs)) self._parents[-2]._children.append(self._parents[-1])
def build_stub_packet(ip): pad = make_random_string(10) cmd = pack_ndr_string("A" * 2000) try: stub = pack_ndr_string(codecs.ascii_encode(get_hostname(ip))[0]) stub += pack_ndr_string("..\\..\\..\\..\\..\\..\\..\\..\\..\\Windows\\system32\\cmd /c \"""\"""") + cmd stub += pack_ndr_string(pad) + pack_ndr_long(2) + pack_ndr_long(2) stub += pack_ndr_string(make_random_string(random.randint(0,4) + 1).upper()) + pack_ndr_long(0) + pack_ndr_long(4) except Exception, e: raise e
def parse(class_, lines): r'''Parse a list of lines into a Part, popping all parsed lines off the list, leaving those that were not parsed. >>> lines = ['+\n', 'a 1\n', 'a 2\n', 'a 3\n', '-\n'] >>> p = Part.parse(lines) >>> lines ['-\n'] >>> len(p) 3 >>> list(p.keys()) ['a'] >>> p['a'] [('1', None), ('2', None), ('3', None)] >>> p.delim '+' ''' assert len(lines) != 0 line = lines[0] loc = None if hasattr(line, 'loc') and isinstance(line.loc, collections.Callable): loc = line.loc() delim = None if (len(line) == 2 and not (line[0].isalpha() or line[0].isspace()) and line[1] == '\n'): delim = line[0] lines.pop(0) part = class_(delim=delim, loc=loc) dataset._parse(part, lines) for key in part.keys(): try: codecs.ascii_encode(key) except UnicodeError: raise InputError('invalid key - not ASCII', char=key) if not key[0].isalpha(): raise InputError('invalid key - must start with letter', char=key) return part
def chuncks(self, source): ''' Provides the chuncks. @param source: Iterable(bytes) The source to stream chuncks for. @return: Iterable(bytes) The transfer chuncks. ''' assert isinstance(source, Iterable), 'Invalid source %s' % source buffer, size = deque(), 0 source = iter(source) while True: try: chunk = next(source) except StopIteration: if not buffer: break chunk = b''.join(buffer) buffer.clear() size = 0 else: if size < self.maximumChunkSize and len(chunk) < self.maximumChunkSize: buffer.append(chunk) size += len(chunk) continue if buffer: buffer.append(chunk) chunk = b''.join(buffer) buffer.clear() size = 0 yield ascii_encode(self.chunkMark % hex(len(chunk))[2:])[0] yield chunk yield self._chunkEndingBytes yield ascii_encode(self.chunkMark % '0')[0] yield self._chunkEndingBytes
def getInitialSet(self, victor_corpus): substitutions_initial = {} lex = open(victor_corpus) for line in lex: data = line.strip().split('\t') target = data[1].strip() url = 'http://www.dictionaryapi.com/api/v1/references/thesaurus/xml/' + target + '?key=' + self.thesaurus_key conn = urllib.urlopen(url) root = ET.fromstring(conn.read()) root = root.findall('entry') cands = {} if len(root)>0: for root_node in root: node_pos = root_node.find('fl') if node_pos != None: node_pos = node_pos.text.strip()[0].lower() if node_pos not in cands.keys(): cands[node_pos] = set([]) for sense in root_node.iter('sens'): syn = sense.findall('syn')[0] res = '' for snip in syn.itertext(): res += snip + ' ' finds = re.findall('\([^\)]+\)', res) for find in finds: res = res.replace(find, '') synonyms = [s.strip() for s in res.split(',')] for synonym in synonyms: if len(synonym.split(' '))==1: try: test = codecs.ascii_encode(synonym) cands[node_pos].add(synonym) except UnicodeEncodeError: cands = cands for pos in cands.keys(): if target in cands[pos]: cands[pos].remove(target) if len(cands.keys())>0: substitutions_initial[target] = cands lex.close() return substitutions_initial
def test_codecs_builtins(self): s = "abc" encoded = codecs.utf_8_encode(s) self.assertEqual(s, codecs.utf_8_decode(encoded[0])[0]) encoded = codecs.utf_7_encode(s) self.assertEqual(s, codecs.utf_7_decode(encoded[0])[0]) encoded = codecs.utf_16_encode(s) self.assertEqual(s, codecs.utf_16_decode(encoded[0])[0]) encoded = codecs.utf_16_le_encode(s) self.assertEqual(s, codecs.utf_16_le_decode(encoded[0])[0]) encoded = codecs.utf_16_be_encode(s) self.assertEqual(s, codecs.utf_16_be_decode(encoded[0])[0]) encoded = codecs.utf_32_encode(s) self.assertEqual(s, codecs.utf_32_decode(encoded[0])[0]) encoded = codecs.utf_32_le_encode(s) self.assertEqual(s, codecs.utf_32_le_decode(encoded[0])[0]) encoded = codecs.utf_32_be_encode(s) self.assertEqual(s, codecs.utf_32_be_decode(encoded[0])[0]) encoded = codecs.utf_32_be_encode(s) self.assertEqual(s, codecs.utf_32_be_decode(encoded[0])[0]) encoded = codecs.raw_unicode_escape_encode(s) self.assertEqual(s, codecs.raw_unicode_escape_decode(encoded[0])[0]) encoded = codecs.unicode_escape_encode(s) self.assertEqual(s, codecs.unicode_escape_decode(encoded[0])[0]) encoded = codecs.latin_1_encode(s) self.assertEqual(s, codecs.latin_1_decode(encoded[0])[0]) encoded = codecs.ascii_encode(s) self.assertEqual(s, codecs.ascii_decode(encoded[0])[0])
def encode(self, input, final=False): if _allowed_coercion(input): return codecs.ascii_encode(input, self.errors)[0] raise UnicodeError("encoding coercion blocked")
def encode(self, input, errors='string'): if _allowed_coercion(input): return codecs.ascii_encode(input, errors) raise UnicodeError("encoding coercion blocked")
def utf2ascii(text): return ascii_encode(utf_16_le_decode(text, "ignore")[0], "ignore")[0]
else: status = ' \x0305[check failed]\x0f' if buffer[-1] == '\x01': buffer = buffer[:-1] + status + '\x01' else: buffer += status del status del length, timestamp, check except Exception as e: pass del parts buffer = re_BUFFER_CTCP_DCC('',buffer) + '\x01' if '\x01ACTION ' in buffer.upper() else buffer.replace('\x01','') if not COLOUR: buffer = re_BUFFER_COLOUR('',buffer) if not UNICODE: buffer = codecs.ascii_encode(unicodedata.normalize('NFKD',unicode(buffer,'utf-8','replace')),'ignore')[0] buffer = ''.join(byte for byte in buffer if 127 > ord(byte) > 31 or byte in ['\x01','\x02','\x03','\x0f','\x1d','\x1f']) buffer += '\n' #os.write(2, 'socket-done: ' + buffer) #os.write(2, '[sock_read] ' + buffer) if re_URC_PRIVMSG_NOTICE_TOPIC(buffer) or re_URC_PRIVMSG_PRIVATE(buffer): dst = re_SPLIT(buffer,3)[2].lower() if re_URC_PRIVMSG_PRIVATE(buffer) and dst.lower() not in localnicks: continue nick = buffer.split('!', 1)[0][1:] while nick in localnicks: nick = nick + "_" host = buffer.split(' ', 1)[0].split('@', 1)[1] if nick.lower() not in nicks: try_write(wr, 'NICK ' + nick + ' :1\n') try_write(wr, ':' + nick + ' USER remote ' + host + ' noIdea :real name\n') nicks[nick.lower()] = list()
reader = list(csv.reader( open("changes.csv", newline=''),delimiter=',')) # row 1 : original filename, strings to find l0=reader[1] items= len(l0) #read original file as binary with open(l0[0],"rb") as f: s=bytes(f.read()) #generate output files c=3 for row in reader[2:]: l= row print(l) s1=s print (l) # do as many replaces as columns -1 for i in range(1,items): #stop if replace is not same size as search if len(l0[i])!=len(l[i]): print('Line ',c, l[i],' not same length as ', l0[i]) sys.exit(0) s1=s1.replace(ascii_encode(l0[i])[0],ascii_encode(l[i])[0]) #after replacements, write file with open(l[0],"wb") as f1: f1.write(s1) c+=1
def test_ascii_encode(self): #sanity new_str, num_processed = codecs.ascii_encode("abc") self.assertEqual(new_str, b'abc') self.assertEqual(num_processed, 3)
def b(x): return codecs.ascii_encode(x)[0]
def utf2ascii(text): return ascii_encode(utf_16_le_decode(text, 'ignore')[0], 'ignore')[0]
def test_ascii_encode(self): #sanity new_str, size = codecs.ascii_encode("abc") self.assertEqual(new_str, 'abc') self.assertEqual(size, 3)
def encode(input, errors='strict'): if _deferred_codec.other or max(input) > u'\x7F': _deferred_codec.other = _deferred_codec.other or codecs.lookup(other) return _deferred_codec.other.encode(input, errors) else: return codecs.ascii_encode(input, errors)
def update_event(self, inp=-1): self.set_output_val(0, codecs.ascii_encode(self.input(0), self.input(1)))
def b( x ): return codecs.ascii_encode( x )[ 0 ];
def encode(input, errors='strict'): warnings.warn("Implicit conversion of unicode to str", UnicodeWarning, 2) return codecs.ascii_encode(input, errors)
def encode(self, input, final=False): return codecs.ascii_encode(input, self.errors)[0]
buffer = buffer[:-1] + status + '\x01' else: buffer += status del status del length, timestamp, check except Exception as e: pass del parts buffer = re_BUFFER_CTCP_DCC( '', buffer) + '\x01' if '\x01ACTION ' in buffer.upper( ) else buffer.replace('\x01', '') if not COLOUR: buffer = re_BUFFER_COLOUR('', buffer) if not UNICODE: buffer = codecs.ascii_encode( unicodedata.normalize('NFKD', unicode(buffer, 'utf-8', 'replace')), 'ignore')[0] buffer = ''.join( byte for byte in buffer if 127 > ord(byte) > 31 or byte in ['\x01', '\x02', '\x03', '\x0f', '\x1d', '\x1f']) buffer += '\n' #os.write(2, 'socket-done: ' + buffer) #os.write(2, '[sock_read] ' + buffer) if re_URC_PRIVMSG_NOTICE_TOPIC(buffer) or re_URC_PRIVMSG_PRIVATE( buffer): dst = re_SPLIT(buffer, 3)[2].lower() if re_URC_PRIVMSG_PRIVATE( buffer) and dst.lower() not in localnicks: continue nick = buffer.split('!', 1)[0][1:]
def check(content): try: codecs.ascii_encode(content, 'strict') except UnicodeEncodeError as exc: self.fail("Unable to read '%s' as ascii: %s" % (content, exc))
def encode(txt, err): """Hack to avoid generating erroneous XML""" #TODO(nikita): backslash (\) is not encoded. This causes parsing problems res = codecs.ascii_encode(txt, err) res = res[0].replace(r'unicode="\"', r'unicode="\"') return res, len(res)
def encrypt(text): return codecs.ascii_encode(text)
def warning_encode(input, errors='strict'): warnings.warn(get_warning(sys._getframe(1), target=str), stacklevel=2) return codecs.ascii_encode(input, errors)
else: buffer += status del status del length, timestamp, check except Exception as e: pass del parts buffer = ( re_BUFFER_CTCP_DCC("", buffer) + "\x01" if "\x01ACTION " in buffer.upper() else buffer.replace("\x01", "") ) if not COLOUR: buffer = re_BUFFER_COLOUR("", buffer) if not UNICODE: buffer = codecs.ascii_encode(unicodedata.normalize("NFKD", unicode(buffer, "utf-8", "replace")), "ignore")[ 0 ] buffer = "".join( byte for byte in buffer if 127 > ord(byte) > 31 or byte in ["\x01", "\x02", "\x03", "\x0f", "\x1d", "\x1f"] ) buffer += "\n" # os.write(2, 'socket-done: ' + buffer) # os.write(2, '[sock_read] ' + buffer) if re_URC_PRIVMSG_NOTICE_TOPIC(buffer) or re_URC_PRIVMSG_PRIVATE(buffer): dst = re_SPLIT(buffer, 3)[2].lower() if re_URC_PRIVMSG_PRIVATE(buffer) and dst.lower() not in localnicks: continue nick = buffer.split("!", 1)[0][1:]