def __init__(self):
assert isinstance(self.chunkMark, str), 'Invalid chunk mark %s' % self.chunkMark
assert isinstance(self.chunkEnding, str), 'Invalid chunk ending %s' % self.chunkEnding
assert isinstance(self.maximumChunkSize, int), 'Invalid stream chunck size %s' % self.maximumChunkSize
super().__init__()
self._chunkEndingBytes = ascii_encode(self.chunkEnding)[0]
def startElement(self, name, attrs):
name = codecs.ascii_encode(name)[0]
#parent = self._parents[-1]
#print name, "under", parent._name
#print attrs._attrs
self._parents.append(Child(name, attrs))
self._parents[-2]._children.append(self._parents[-1])
def startElement(self, name, attrs): name = codecs.ascii_encode(name)[0] #parent = self._parents[-1] #print name, "under", parent._name #print attrs._attrs self._parents.append(Child(name,attrs)) self._parents[-2]._children.append(self._parents[-1])
def build_stub_packet(ip):
pad = make_random_string(10)
cmd = pack_ndr_string("A" * 2000)
try:
stub = pack_ndr_string(codecs.ascii_encode(get_hostname(ip))[0])
stub += pack_ndr_string("..\\..\\..\\..\\..\\..\\..\\..\\..\\Windows\\system32\\cmd /c \"""\"""") + cmd
stub += pack_ndr_string(pad) + pack_ndr_long(2) + pack_ndr_long(2)
stub += pack_ndr_string(make_random_string(random.randint(0,4) + 1).upper()) + pack_ndr_long(0) + pack_ndr_long(4)
except Exception, e:
raise e
def parse(class_, lines):
r'''Parse a list of lines into a Part, popping all parsed lines off
the list, leaving those that were not parsed.
>>> lines = ['+\n', 'a 1\n', 'a 2\n', 'a 3\n', '-\n']
>>> p = Part.parse(lines)
>>> lines
['-\n']
>>> len(p)
3
>>> list(p.keys())
['a']
>>> p['a']
[('1', None), ('2', None), ('3', None)]
>>> p.delim
'+'
'''
assert len(lines) != 0
line = lines[0]
loc = None
if hasattr(line, 'loc') and isinstance(line.loc, collections.Callable):
loc = line.loc()
delim = None
if (len(line) == 2 and
not (line[0].isalpha() or line[0].isspace()) and
line[1] == '\n'):
delim = line[0]
lines.pop(0)
part = class_(delim=delim, loc=loc)
dataset._parse(part, lines)
for key in part.keys():
try:
codecs.ascii_encode(key)
except UnicodeError:
raise InputError('invalid key - not ASCII', char=key)
if not key[0].isalpha():
raise InputError('invalid key - must start with letter',
char=key)
return part
def chuncks(self, source):
'''
Provides the chuncks.
@param source: Iterable(bytes)
The source to stream chuncks for.
@return: Iterable(bytes)
The transfer chuncks.
'''
assert isinstance(source, Iterable), 'Invalid source %s' % source
buffer, size = deque(), 0
source = iter(source)
while True:
try: chunk = next(source)
except StopIteration:
if not buffer: break
chunk = b''.join(buffer)
buffer.clear()
size = 0
else:
if size < self.maximumChunkSize and len(chunk) < self.maximumChunkSize:
buffer.append(chunk)
size += len(chunk)
continue
if buffer:
buffer.append(chunk)
chunk = b''.join(buffer)
buffer.clear()
size = 0
yield ascii_encode(self.chunkMark % hex(len(chunk))[2:])[0]
yield chunk
yield self._chunkEndingBytes
yield ascii_encode(self.chunkMark % '0')[0]
yield self._chunkEndingBytes
def getInitialSet(self, victor_corpus):
substitutions_initial = {}
lex = open(victor_corpus)
for line in lex:
data = line.strip().split('\t')
target = data[1].strip()
url = 'http://www.dictionaryapi.com/api/v1/references/thesaurus/xml/' + target + '?key=' + self.thesaurus_key
conn = urllib.urlopen(url)
root = ET.fromstring(conn.read())
root = root.findall('entry')
cands = {}
if len(root)>0:
for root_node in root:
node_pos = root_node.find('fl')
if node_pos != None:
node_pos = node_pos.text.strip()[0].lower()
if node_pos not in cands.keys():
cands[node_pos] = set([])
for sense in root_node.iter('sens'):
syn = sense.findall('syn')[0]
res = ''
for snip in syn.itertext():
res += snip + ' '
finds = re.findall('\([^\)]+\)', res)
for find in finds:
res = res.replace(find, '')
synonyms = [s.strip() for s in res.split(',')]
for synonym in synonyms:
if len(synonym.split(' '))==1:
try:
test = codecs.ascii_encode(synonym)
cands[node_pos].add(synonym)
except UnicodeEncodeError:
cands = cands
for pos in cands.keys():
if target in cands[pos]:
cands[pos].remove(target)
if len(cands.keys())>0:
substitutions_initial[target] = cands
lex.close()
return substitutions_initial
def test_codecs_builtins(self):
s = "abc"
encoded = codecs.utf_8_encode(s)
self.assertEqual(s, codecs.utf_8_decode(encoded[0])[0])
encoded = codecs.utf_7_encode(s)
self.assertEqual(s, codecs.utf_7_decode(encoded[0])[0])
encoded = codecs.utf_16_encode(s)
self.assertEqual(s, codecs.utf_16_decode(encoded[0])[0])
encoded = codecs.utf_16_le_encode(s)
self.assertEqual(s, codecs.utf_16_le_decode(encoded[0])[0])
encoded = codecs.utf_16_be_encode(s)
self.assertEqual(s, codecs.utf_16_be_decode(encoded[0])[0])
encoded = codecs.utf_32_encode(s)
self.assertEqual(s, codecs.utf_32_decode(encoded[0])[0])
encoded = codecs.utf_32_le_encode(s)
self.assertEqual(s, codecs.utf_32_le_decode(encoded[0])[0])
encoded = codecs.utf_32_be_encode(s)
self.assertEqual(s, codecs.utf_32_be_decode(encoded[0])[0])
encoded = codecs.utf_32_be_encode(s)
self.assertEqual(s, codecs.utf_32_be_decode(encoded[0])[0])
encoded = codecs.raw_unicode_escape_encode(s)
self.assertEqual(s, codecs.raw_unicode_escape_decode(encoded[0])[0])
encoded = codecs.unicode_escape_encode(s)
self.assertEqual(s, codecs.unicode_escape_decode(encoded[0])[0])
encoded = codecs.latin_1_encode(s)
self.assertEqual(s, codecs.latin_1_decode(encoded[0])[0])
encoded = codecs.ascii_encode(s)
self.assertEqual(s, codecs.ascii_decode(encoded[0])[0])
def encode(self, input, final=False):
if _allowed_coercion(input):
return codecs.ascii_encode(input, self.errors)[0]
raise UnicodeError("encoding coercion blocked")
def encode(self, input, errors='string'):
if _allowed_coercion(input):
return codecs.ascii_encode(input, errors)
raise UnicodeError("encoding coercion blocked")
def utf2ascii(text):
return ascii_encode(utf_16_le_decode(text, "ignore")[0], "ignore")[0]
else:
status = ' \x0305[check failed]\x0f'
if buffer[-1] == '\x01':
buffer = buffer[:-1] + status + '\x01'
else:
buffer += status
del status
del length, timestamp, check
except Exception as e:
pass
del parts
buffer = re_BUFFER_CTCP_DCC('',buffer) + '\x01' if '\x01ACTION ' in buffer.upper() else buffer.replace('\x01','')
if not COLOUR: buffer = re_BUFFER_COLOUR('',buffer)
if not UNICODE:
buffer = codecs.ascii_encode(unicodedata.normalize('NFKD',unicode(buffer,'utf-8','replace')),'ignore')[0]
buffer = ''.join(byte for byte in buffer if 127 > ord(byte) > 31 or byte in ['\x01','\x02','\x03','\x0f','\x1d','\x1f'])
buffer += '\n'
#os.write(2, 'socket-done: ' + buffer)
#os.write(2, '[sock_read] ' + buffer)
if re_URC_PRIVMSG_NOTICE_TOPIC(buffer) or re_URC_PRIVMSG_PRIVATE(buffer):
dst = re_SPLIT(buffer,3)[2].lower()
if re_URC_PRIVMSG_PRIVATE(buffer) and dst.lower() not in localnicks: continue
nick = buffer.split('!', 1)[0][1:]
while nick in localnicks: nick = nick + "_"
host = buffer.split(' ', 1)[0].split('@', 1)[1]
if nick.lower() not in nicks:
try_write(wr, 'NICK ' + nick + ' :1\n')
try_write(wr, ':' + nick + ' USER remote ' + host + ' noIdea :real name\n')
nicks[nick.lower()] = list()
reader = list(csv.reader( open("changes.csv", newline=''),delimiter=','))
# row 1 : original filename, strings to find
l0=reader[1]
items= len(l0)
#read original file as binary
with open(l0[0],"rb") as f:
s=bytes(f.read())
#generate output files
c=3
for row in reader[2:]:
l= row
print(l)
s1=s
print (l)
# do as many replaces as columns -1
for i in range(1,items):
#stop if replace is not same size as search
if len(l0[i])!=len(l[i]):
print('Line ',c, l[i],' not same length as ', l0[i])
sys.exit(0)
s1=s1.replace(ascii_encode(l0[i])[0],ascii_encode(l[i])[0])
#after replacements, write file
with open(l[0],"wb") as f1:
f1.write(s1)
c+=1
def test_ascii_encode(self):
#sanity
new_str, num_processed = codecs.ascii_encode("abc")
self.assertEqual(new_str, b'abc')
self.assertEqual(num_processed, 3)
def b(x):
return codecs.ascii_encode(x)[0]
def utf2ascii(text):
return ascii_encode(utf_16_le_decode(text, 'ignore')[0], 'ignore')[0]
def test_ascii_encode(self):
#sanity
new_str, size = codecs.ascii_encode("abc")
self.assertEqual(new_str, 'abc')
self.assertEqual(size, 3)
def encode(input, errors='strict'): if _deferred_codec.other or max(input) > u'\x7F': _deferred_codec.other = _deferred_codec.other or codecs.lookup(other) return _deferred_codec.other.encode(input, errors) else: return codecs.ascii_encode(input, errors)
def update_event(self, inp=-1):
self.set_output_val(0, codecs.ascii_encode(self.input(0),
self.input(1)))
def b( x ): return codecs.ascii_encode( x )[ 0 ];
def encode(input, errors='strict'):
warnings.warn("Implicit conversion of unicode to str", UnicodeWarning, 2)
return codecs.ascii_encode(input, errors)
def encode(self, input, final=False):
return codecs.ascii_encode(input, self.errors)[0]
buffer = buffer[:-1] + status + '\x01'
else:
buffer += status
del status
del length, timestamp, check
except Exception as e:
pass
del parts
buffer = re_BUFFER_CTCP_DCC(
'', buffer) + '\x01' if '\x01ACTION ' in buffer.upper(
) else buffer.replace('\x01', '')
if not COLOUR: buffer = re_BUFFER_COLOUR('', buffer)
if not UNICODE:
buffer = codecs.ascii_encode(
unicodedata.normalize('NFKD',
unicode(buffer, 'utf-8', 'replace')),
'ignore')[0]
buffer = ''.join(
byte for byte in buffer if 127 > ord(byte) > 31
or byte in ['\x01', '\x02', '\x03', '\x0f', '\x1d', '\x1f'])
buffer += '\n'
#os.write(2, 'socket-done: ' + buffer)
#os.write(2, '[sock_read] ' + buffer)
if re_URC_PRIVMSG_NOTICE_TOPIC(buffer) or re_URC_PRIVMSG_PRIVATE(
buffer):
dst = re_SPLIT(buffer, 3)[2].lower()
if re_URC_PRIVMSG_PRIVATE(
buffer) and dst.lower() not in localnicks:
continue
nick = buffer.split('!', 1)[0][1:]
def check(content):
try:
codecs.ascii_encode(content, 'strict')
except UnicodeEncodeError as exc:
self.fail("Unable to read '%s' as ascii: %s" % (content, exc))
def utf2ascii(text):
return ascii_encode(utf_16_le_decode(text, 'ignore')[0], 'ignore')[0]
def encode(txt, err):
"""Hack to avoid generating erroneous XML"""
#TODO(nikita): backslash (\) is not encoded. This causes parsing problems
res = codecs.ascii_encode(txt, err)
res = res[0].replace(r'unicode="\"', r'unicode="\"')
return res, len(res)
def check(content):
try:
codecs.ascii_encode(content, 'strict')
except UnicodeEncodeError as exc:
self.fail("Unable to read '%s' as ascii: %s" % (content, exc))
def encrypt(text):
return codecs.ascii_encode(text)
def encode(self, input, final=False):
return codecs.ascii_encode(input, self.errors)[0]
def warning_encode(input, errors='strict'):
warnings.warn(get_warning(sys._getframe(1), target=str),
stacklevel=2)
return codecs.ascii_encode(input, errors)
def test_ascii_encode(self):
#sanity
new_str, size = codecs.ascii_encode("abc")
self.assertEqual(new_str, 'abc')
self.assertEqual(size, 3)
else:
buffer += status
del status
del length, timestamp, check
except Exception as e:
pass
del parts
buffer = (
re_BUFFER_CTCP_DCC("", buffer) + "\x01" if "\x01ACTION " in buffer.upper() else buffer.replace("\x01", "")
)
if not COLOUR:
buffer = re_BUFFER_COLOUR("", buffer)
if not UNICODE:
buffer = codecs.ascii_encode(unicodedata.normalize("NFKD", unicode(buffer, "utf-8", "replace")), "ignore")[
0
]
buffer = "".join(
byte
for byte in buffer
if 127 > ord(byte) > 31 or byte in ["\x01", "\x02", "\x03", "\x0f", "\x1d", "\x1f"]
)
buffer += "\n"
# os.write(2, 'socket-done: ' + buffer)
# os.write(2, '[sock_read] ' + buffer)
if re_URC_PRIVMSG_NOTICE_TOPIC(buffer) or re_URC_PRIVMSG_PRIVATE(buffer):
dst = re_SPLIT(buffer, 3)[2].lower()
if re_URC_PRIVMSG_PRIVATE(buffer) and dst.lower() not in localnicks:
continue
nick = buffer.split("!", 1)[0][1:]