def __init__(self, filename, line, msg, charset, fuzzy, fmt, noqa): # pylint: disable=too-many-arguments
"""Build a PO message."""
self.filename = filename
self.line = line
# unescape strings
if sys.version_info < (3,):
# python 2.x
msg = {k: escape_decode(v)[0] for k, v in msg.items()}
else:
# python 3.x
msg = {k: escape_decode(v)[0]. decode(charset)
for k, v in msg.items()}
# build messages as a list of tuples: (string, translation)
self.messages = []
if 'msgid_plural' in msg:
i = 0
while True:
key = 'msgstr[{0}]'.format(i)
if key not in msg:
break
self.messages.append((msg['msgid_plural'], msg[key]))
i += 1
else:
self.messages.append((msg.get('msgid', ''), msg.get('msgstr', '')))
self.fuzzy = fuzzy
self.fmt = fmt
self.noqa = noqa
def esc(el, l):
"""Apply an escape match to a line annotated with '(esc)'"""
ann = b(' (esc)\n')
if el.endswith(ann):
el = codecs.escape_decode(el[:-len(ann)])[0] + b('\n')
if l.endswith(ann):
l = codecs.escape_decode(l[:-len(ann)])[0] + b('\n')
return el == l
def files_in_archive(self, force_refresh=False):
if self._files_in_archive and not force_refresh:
return self._files_in_archive
cmd = [self.cmd_path, '--list', '-C', self.dest]
if self.zipflag:
cmd.append(self.zipflag)
if self.opts:
cmd.extend(['--show-transformed-names'] + self.opts)
if self.excludes:
cmd.extend(['--exclude=' + quote(f) for f in self.excludes])
cmd.extend(['-f', self.src])
rc, out, err = self.module.run_command(cmd, cwd=self.dest, environ_update=dict(LANG='C', LC_ALL='C', LC_MESSAGES='C'))
if rc != 0:
raise UnarchiveError('Unable to list files in the archive')
for filename in out.splitlines():
# Compensate for locale-related problems in gtar output (octal unicode representation) #11348
# filename = filename.decode('string_escape')
filename = to_native(codecs.escape_decode(filename)[0])
if filename and filename not in self.excludes:
# We don't allow absolute filenames. If the user wants to unarchive rooted in "/"
# they need to use "dest: '/'". This follows the defaults for gtar, pax, etc.
# Allowing absolute filenames here also causes bugs: https://github.com/ansible/ansible/issues/21397
if filename.startswith('/'):
filename = filename[1:]
self._files_in_archive.append(filename)
return self._files_in_archive
def raw_bytes(raw):
"""Make a bytes object out of a raw string.
This is analogous to raw_unicode, but processes byte escape characters
to produce a bytes object.
"""
return codecs.escape_decode(raw)[0]
def str_to_re(self, string):
"""Convert plain string to escaped regexp that can be compiled"""
if isinstance(string, str):
string = string.encode('utf-8')
string = codecs.escape_decode(string)[0]
string = string.decode('utf-8')
return "^%s$" % re_escape(string)
def AssocInfoAdHoc():
# Get encrypted password and decrypt it
pw_encrypted = open(r'D:\webapps\_server\pyodbc\pw', 'r').read()
key = codecs.escape_decode(open(r'D:\webapps\_server\pyodbc\key', 'r').read())[0]
pw = crypto.decrypt_with_key(key, pw_encrypted)
userid = 'ma17151'
querystr = request.POST.get('sqlString', '').strip()
cnxn_string = 'DSN=DSNOGW01;UID=' + userid + ';PWD=' + pw
cnxn = pyodbc.connect(cnxn_string)
cursor = cnxn.cursor()
cursor.execute(querystr)
cnxn.commit()
resultset = cursor.fetchall()
# Get column name of the resultset
column_name_list = [tuple[0] for tuple in cursor.description]
cursor.close()
cnxn.close()
# Send the response/result as Excel output
response.content_type = 'application/vnd.ms-excel'
output = template(r'D:\webapps\_templates\make_table.tpl', rows=resultset, column_names=column_name_list)
return output
def insn__dot_ASCII_N(self, insn, mode, *args):
"""Insert the given text as bytes."""
result = []
for am, value, m in args:
if am != 'STRING':
raise AssemblerError('Bad argument (.ASCII only allows strings): %r' % value)
result.extend([u'%s 8bit' % (ord(x),) for x in codecs.escape_decode(m.group('STR'))[0]])
return u' '.join(result)
def _parse_entry(line, quoted, escaped, get_bytes):
"""
Parses the next entry from the given space separated content.
:param str line: content to be parsed
:param bool quoted: parses the next entry as a quoted value, removing the quotes
:param bool escaped: unescapes the string
:returns: **tuple** of the form (entry, remainder)
:raises:
* **ValueError** if quoted is True without the next value being quoted
* **IndexError** if there's nothing to parse from the line
"""
if line == '':
raise IndexError('no remaining content to parse')
next_entry, remainder = '', line
if quoted:
# validate and parse the quoted value
start_quote, end_quote = _get_quote_indices(remainder, escaped)
if start_quote != 0 or end_quote == -1:
raise ValueError("the next entry isn't a quoted value: " + line)
next_entry, remainder = remainder[1:end_quote], remainder[end_quote + 1:]
else:
# non-quoted value, just need to check if there's more data afterward
if ' ' in remainder:
next_entry, remainder = remainder.split(' ', 1)
else:
next_entry, remainder = remainder, ''
if escaped:
# Tor does escaping in its 'esc_for_log' function of 'common/util.c'. It's
# hard to tell what controller functions use this in practice, but direct
# users are...
#
# * 'COOKIEFILE' field of PROTOCOLINFO responses
# * logged messages about bugs
# * the 'getinfo_helper_listeners' function of control.c
#
# Ideally we'd use "next_entry.decode('string_escape')" but it was removed
# in python 3.x and 'unicode_escape' isn't quite the same...
#
# https://stackoverflow.com/questions/14820429/how-do-i-decodestring-escape-in-python3
next_entry = codecs.escape_decode(next_entry)[0]
if stem.prereq.is_python_3() and not get_bytes:
next_entry = stem.util.str_tools._to_unicode(next_entry) # normalize back to str
if get_bytes:
next_entry = stem.util.str_tools._to_bytes(next_entry)
return (next_entry, remainder.lstrip())
def load_string(self):
"""Copied from python 3.4 pickle.Unpickler.load_string"""
data = self.readline()[:-1]
# Strip outermost quotes
if len(data) >= 2 and data[0] == data[-1] and data[0] in b"\"'":
data = data[1:-1]
else:
raise pickle.UnpicklingError("the STRING opcode argument must be quoted")
self.append(self._decode_string(codecs.escape_decode(data)[0]))
def load_string(self):
data = self.readline()[:-1]
# Strip outermost quotes
if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
data = data[1:-1]
else:
raise UnpicklingError("the STRING opcode argument must be quoted")
self.append(codecs.escape_decode(data)[0]
.decode(self.encoding, self.errors))
def _unscramble_name(nameparts):
name = ' '.join(nameparts)
# no special chars
if '\\' not in name:
return name
if name[0] == '"':
name = name.strip('"')
return codecs.escape_decode(name)[0].decode()
def str_to_re(self, string):
"""Convert plain string to escaped regexp that can be compiled"""
if isinstance(string, text_type):
string = string.encode("utf-8")
if PY3:
string = codecs.escape_decode(string)[0]
else:
string = string.decode("string_escape")
string = string.decode("utf-8")
return "^%s$" % re_escape(string)
def __init__(self, name, command, response_size, byte_filter, bit_filter, response_type):
self.name = name
# Decode any escaped hex chars that need to be sent to the device.
self.command = codecs.escape_decode(command)[0]
self.response_size = int(response_size)
self.byte_filter = byte_filter
self.bit_filter = bit_filter
self.response_type = response_type
# Initialize the response list
self.responses = []
def emailCurrentWeather():
json = request.urlopen('http://api.wunderground.com/api/your_key/conditions/q/OH/Dublin.json')
parsed_json = simplejson.load(json)
recipient = ['*****@*****.**']
key = codecs.escape_decode(open('/home/pi/tmp/key','r').read().strip())[0]
pw = open('/home/pi/tmp/pw','r').read().strip()
pwd = crypto.decrypt_with_key(key, pw)
sender = 'your_email'
subject = 'Current Weather Conditions'
top_border = "Current weather conditions from the Wunderground\n\n"+"Brought to you by the Raspberry Pi!\n"+\
"*" * 45+"\n"
bottom_border = "*" * 45+"\n\n"
if parsed_json['current_observation']:
city = parsed_json['current_observation']['display_location']['city']
state = parsed_json['current_observation']['display_location']['state']
current_temp = parsed_json['current_observation']['temperature_string']
feels_like = parsed_json['current_observation']['feelslike_string']
as_of = parsed_json['current_observation']['observation_time']
rel_humidity = parsed_json['current_observation']['relative_humidity']
weather = parsed_json['current_observation']['weather']
forecast_url = parsed_json['current_observation']['forecast_url']
email = top_border+as_of+"\n"+"City/State: "+city+", "+state+"\n"+bottom_border+ \
"Current temp: "+current_temp+"\n"+"Feels like: "+feels_like+"\n"+"Rel. humidity: "+rel_humidity+"\n"+ \
"Weather condition: "+weather+"\n"+"Forecast URL: "+forecast_url
subject = subject+": "+current_temp
msg = MIMEText(email)
msg['Subject'] = subject
msg['From'] = sender
msg['To'] = COMMASPACE.join(recipient) # COMMASPACE still works ok with just one recipient
#msg['CC'] = COMMASPACE.join(recipient)
#msg['BCC'] = COMMASPACE.join(recipient)
server = smtplib.SMTP('smtp.gmail.com:587')
server.starttls()
try:
server.login(sender,pwd)
except smtplib.SMTPAuthenticationError: # Check for authentication error
return "ERROR"
try:
server.sendmail(sender, recipient, msg.as_string())
except smtplib.SMTPRecipientsRefused: # Check if recipient's email was accepted by the server
return "ERROR"
server.quit()
else:
now = datetime.now()
print("Current weather conditions are not available at this time:",now.strftime("%Y-%m-%d %I:%M%p"))
def load_string(self):
orig = self.readline()
rep = orig[:-1]
for q in (b'"', b"'"): # double or single quote
if rep.startswith(q):
if not rep.endswith(q):
raise ValueError("insecure string pickle")
rep = rep[len(q) : -len(q)]
break
else:
raise ValueError("insecure string pickle: %r" % orig)
self.append(codecs.escape_decode(rep)[0].decode(self.encoding, self.errors))
def get_real_path(name):
name = name.strip()
if os.name == "nt":
# Git returns unix-style paths on Windows
name = os.path.normpath(name)
# Unicode file names and file names containing quotes are
# returned by git as quoted strings
if name[0] == '"':
name = name.encode("latin1")
name = codecs.escape_decode(name[1:-1])[0].decode("utf-8")
return os.path.abspath(os.path.join(self.location, name))
def escaped_str_to_bytes(data):
"""
Take an escaped string and return the unescaped bytes equivalent.
"""
if not isinstance(data, str):
raise ValueError("data must be str")
if six.PY2:
return data.decode("string-escape")
# This one is difficult - we use an undocumented Python API here
# as per http://stackoverflow.com/a/23151714/934719
return codecs.escape_decode(data)[0]
def escaped_str_to_bytes(data):
"""
Take an escaped string and return the unescaped bytes equivalent.
Raises:
ValueError, if the escape sequence is invalid.
"""
if not isinstance(data, str):
raise ValueError("data must be str, but is {}".format(data.__class__.__name__))
# This one is difficult - we use an undocumented Python API here
# as per http://stackoverflow.com/a/23151714/934719
return codecs.escape_decode(data)[0]
def encrypt_with_key(key, privateInfo):
""" Method to encrypt your message using AES encryption """
BLOCK_SIZE = 16
PADDING = '{'
pad = lambda s: s + (BLOCK_SIZE - len(s) % BLOCK_SIZE) * PADDING
EncodeAES = lambda c, s: base64.b64encode(c.encrypt(pad(s)))
cipher = AES.new(codecs.escape_decode(key)[0]) # Need to use codecs to prevent key from being parsed with double slashes
encoded = EncodeAES(cipher, privateInfo)
return encoded.decode('utf-8')
def binaryize(arg):
if arg is None:
return None
if six.PY3:
# arg is unicode. It might have been originally unicode (e.g. on
# windows), or originally binary (e.g. on POSIX). If it was originally
# unicode then we can definitely convert it to a utf-8 bytestring. If
# it was originally bytes, then this is the magic incantation that
# gives us back the original bytestring, no matter what it
# was, assuming that the user is in a utf-8 locale.
# (See http://legacy.python.org/dev/peps/pep-0383/)
arg = arg.encode("utf-8", "surrogateescape")
# now arg is a bytestring. we interpret \ escapes in a py2-and-py3
# compatible way.
return codecs.escape_decode(arg)[0]
def generate_suggestions(self):
offset = 0
data = self.obj_vm.read(offset, constants.SCAN_BLOCKSIZE)
sig = codecs.escape_decode(bytes(str(self.obj_parent.DTBSignature), sys.getdefaultencoding()))[0]
while data:
found = data.find(sig, 0)
while found >= 0:
proc = obj.Object("_EPROCESS", offset = offset + found,
vm = self.obj_vm)
if b'Idle' in proc.ImageFileName.v():
yield proc.Pcb.DirectoryTableBase.v()
found = data.find(sig, found + 1)
offset += len(data)
data = self.obj_vm.read(offset, constants.SCAN_BLOCKSIZE)
def initialize(self, randgen, max_termlength):
# pre: validTermLengths(self, randgen, max_termlength)
for i in self.initials:
if self.verbose:
print 'Generating random %s for %s' % (i.type, i.label)
i.value = randgen.generate(i.type, max_termlength)
while True:
try:
f = open(i.label, 'w')
break
except IOError as e:
i.iter = i.iter + 1
i.label = i.base + '_' + str(i.iter)
f.write(codecs.escape_decode(i.value)[0])
f.close()
def decode(self, lines):
result = []
for line in lines:
if not line:
pass
elif line[0] == 'i':
result.append(int(line[1:], 0))
elif line[0] == 'h':
result.append(line[1:].decode('hex'))
elif line[0] == 's':
result.append(codecs.escape_decode(line[1:])[0])
elif line[0] == 'o':
sys.stdout.write(line[1:])
else:
raise ValueError('unknown line type: %r' % (line,))
return result
def word_string_literal(self, stack):
"""Put a string on the stack."""
words = []
while True:
word = self.next_word()
if word.endswith('"'):
# emulate character wise reading
if word != '"':
words.append(word[:-1])
break
words.append(word)
text = codecs.escape_decode(u' '.join(words))[0]
if self.compiling:
self.frame.append(self.instruction_literal)
self.frame.append(text)
else:
self.push(text)
def word_copy_words(self, stack):
"""Output a string."""
words = []
while True:
word = self.next_word()
if word.endswith('"'):
# emulate character wise reading
if word != '"':
words.append(word[:-1])
break
words.append(word)
text = codecs.escape_decode(u' '.join(words))[0]
if self.compiling:
self.frame.append(self.instruction_output_text)
self.frame.append(text)
else:
self.doctree.write(text)
def delete(self, item):
if self.type == "TXT":
# creating a _new_rdata will re-esacpe escaped characters,
# so we have to 'unescape' them here (so that \\n doesn't become
# \\\\n).
item = codecs.escape_decode(item)[0]
# If a TXT record, strip off '"' if present. Dns module will add this
# automatically, and it breaks if we have it here.
if item.startswith('"') and item.endswith('"'):
item = item[1:-1]
rd = _new_rdata(self.type, item)
try:
self._rdataset.remove(rd)
except ValueError:
raise RecordsError("No such item in record: %s" % item)
def scan_wifi(iface):
while True:
try:
with open('/dev/null', 'w') as null:
scan = subprocess.check_output(['iwlist', iface, 'scanning'],
stderr=null).decode()
except subprocess.CalledProcessError:
subprocess.call(['ip', 'link', 'set', iface, 'up'])
else:
break
networks = SSID_REG.findall(scan)
encryptions = PASSWORD_REG.findall(scan)
for index, network in enumerate(networks[:]):
if '\\' in network:
networks[index] = codecs.escape_decode(network)[0].decode()
return networks, encryptions
def escaped_str_to_bytes(data):
"""
Take an escaped string and return the unescaped bytes equivalent.
"""
if not isinstance(data, six.string_types):
if six.PY2:
raise ValueError("data must be str or unicode, but is {}".format(data.__class__.__name__))
raise ValueError("data must be str, but is {}".format(data.__class__.__name__))
if six.PY2:
if isinstance(data, unicode):
data = data.encode("utf8")
return data.decode("string-escape")
# This one is difficult - we use an undocumented Python API here
# as per http://stackoverflow.com/a/23151714/934719
return codecs.escape_decode(data)[0]
def value_from_fsnative(arg, escape):
"""Takes an item from argv and returns a text_type value without
surrogate escapes or raises ValueError.
"""
assert isinstance(arg, fsnative)
if escape:
bytes_ = fsn2bytes(arg, "utf-8")
if PY2:
bytes_ = bytes_.decode("string_escape")
else:
bytes_ = codecs.escape_decode(bytes_)[0]
arg = bytes2fsn(bytes_, "utf-8")
text = fsn2text(arg, strict=True)
return text
def _ReadSpecificationFile(self, path):
"""Reads the format specification file.
Args:
path (str): path of the format specification file.
Returns:
FormatSpecificationStore: format specification store.
"""
specification_store = specification.FormatSpecificationStore()
with io.open(
path, 'rt', encoding=self._SPECIFICATION_FILE_ENCODING) as file_object:
for line in file_object.readlines():
line = line.strip()
if not line or line.startswith('#'):
continue
try:
identifier, offset, pattern = line.split()
except ValueError:
logger.error('[skipping] invalid line: {0:s}'.format(line))
continue
try:
offset = int(offset, 10)
except ValueError:
logger.error('[skipping] invalid offset in line: {0:s}'.format(line))
continue
try:
# TODO: find another way to do this that doesn't use an undocumented
# API.
pattern = codecs.escape_decode(pattern)[0]
# ValueError is raised e.g. when the patterns contains "\xg1".
except ValueError:
logger.error(
'[skipping] invalid pattern in line: {0:s}'.format(line))
continue
format_specification = specification.FormatSpecification(identifier)
format_specification.AddNewSignature(pattern, offset=offset)
specification_store.AddSpecification(format_specification)
return specification_store
def get(self):
data = self.get_urldata()
match = re.search("n.reduxState=(.*);", data)
if not match:
match = re.search(r"stateData = JSON.parse\(\"(.*)\"\)\<\/script",
data)
if not match:
yield ServiceError("Cant find video info.")
return
janson = json.loads(
codecs.escape_decode(match.group(1))[0].decode("utf-8"))
if janson["recipe"]["content"]["data"]["videoClips"]:
vid = janson["recipe"]["content"]["data"]["videoClips"][0][
"id"]
else:
vid = janson["recipe"]["content"]["data"]["videoEpisodes"][0][
"id"]
res = self.http.get(
"https://api.svt.se/videoplayer-api/video/{}".format(vid))
else:
janson = json.loads(match.group(1))
vid = janson["areaData"]["articles"][list(
janson["areaData"]
["articles"].keys())[0]]["media"][0]["image"]["svtId"]
res = self.http.get("https://api.svt.se/video/{}".format(vid))
janson = res.json()
if "subtitleReferences" in janson:
for i in janson["subtitleReferences"]:
if i["format"] == "websrt" and "url" in i:
yield subtitle(copy.copy(self.config),
"wrst",
i["url"],
output=self.output)
videos = self._get_video(janson)
yield from videos
def OnFind(self, evt):
et = evt.GetEventType()
findstring = codecs.escape_decode(evt.GetFindString())[0]
case = bool(evt.GetFlags() & 4)
def abortcheck():
wx.Yield()
return self._abort_find
def find_next():
self._searching = True
self._abort_find = False
try:
offset = self.hexEditor.bin.find(
self.hexView.offset + 1,
findstring,
casesensitive=case,
wrap = True,
abortcheck=abortcheck
)
except StopIteration:
self.statusBar.SetStatusText("Search aborted", 0)
self._find_start = None
else:
if offset is not None:
self.hexView.setSelection(offset, offset+len(findstring)-1, offset=offset)
self.statusBar.SetStatusText("String found at offset 0x%08x"% offset, 0)
self._find_start = offset
else:
wx.Bell()
self.statusBar.SetStatusText("String not found", 0)
self._find_start = None
self._searching = False
if et == wx.wxEVT_COMMAND_FIND or et == wx.wxEVT_COMMAND_FIND_NEXT:
self.statusBar.SetStatusText("Searching...", 0)
find_next()
def parse_kconfig(file: TextIO, allow_include: bool,
config: Dict[str, str]) -> None:
for lineno, line in enumerate(file, 1):
line = _line_re.match(line.rstrip()).group()
if not line:
continue
match = _config_re.fullmatch(line)
if match:
config[match.group(1)] = match.group(2)
continue
if allow_include:
match = _include_re.fullmatch(line)
if match:
include_path = os.path.join(
os.fsencode(os.path.dirname(file.name)),
codecs.escape_decode(match.group(1))[0],
)
with open(include_path, "r") as include_file:
parse_kconfig(include_file, True, config)
continue
print(f"{file.name}:{lineno}:invalid syntax", file=sys.stderr)
def AssocInfo():
# Get encrypted password and decrypt it
pw_encrypted = open(r'D:\webapps\_server\pyodbc\pw', 'r').read()
key = codecs.escape_decode(open(r'D:\webapps\_server\pyodbc\key', 'r').read())[0]
pw = crypto.decrypt_with_key(key, pw_encrypted)
userid = 'ma17151'
querystr = request.POST.get('qryParameter', '').strip()
cnxn_string = 'DSN=DSNOGW01;UID=' + userid + ';PWD=' + pw
cnxn = pyodbc.connect(cnxn_string)
cursor = cnxn.cursor()
# Get the resultset
resultset = AssocLookUp.getResultset(querystr, cursor)
# Get column name of the resultset
column_name_list = [tuple[0] for tuple in cursor.description]
cursor.close()
cnxn.close()
output = template(r'D:\webapps\_templates\make_table.tpl', rows=resultset, column_names=column_name_list)
return output
def get(self):
par=serialparm.parse_args()
payload=par.get('payload','')
type=par.get('type','')
try:
# 错误示例-1,直接使用pick反序列化不可信数据
if type=='pick_infected':
#请求url：http://0.0.0.0:8888/api/deserialization/?payload=\x80\x03cposix\nsystem\nq\x00X\x06\x00\x00\x00whoamiq\x01\x85q\x02Rq\x03.'&type=pick_infected
payload_byte=bytes(payload, encoding = "utf-8")
import codecs
payload_tuple = codecs.escape_decode(payload_byte, "hex-escape")
return {"code": 200, "message": "{}".format(pickle.loads(payload_tuple[0]))}
# 正确示例-1,直接使用json替代pick，json库为安全库
if type=='pick_safe':
return {"code": 200, "message": "{}".format(json.loads(payload))}
# 错误示例-2,yaml[>5.1]设置加载模式为UnsafeLoader，或者使用5.1版本以下的都存在问题
if type=='yaml_infected':
#请求url：http://0.0.0.0:8888/api/deserialization/?payload=%21%21python%2Fobject%2Fnew%3Aos.system+%5B%22whoami%22%5D&type=yaml_infected
return {"code": 200, "message": "{}".format(yaml.load(payload,Loader=yaml.UnsafeLoader))}
# 正确示例-2,yaml[>5.1]设置加载模式为SafeLoader来序列化不可信数据
if type=='yaml_safe':
return {"code": 200, "message": "{}".format(yaml.load(payload,Loader=yaml.SafeLoader))}
except Exception as e:
return jsonify({"code": "异常", "message": "{}".format(e)})
def files_in_archive(self, force_refresh=False):
if self._files_in_archive and not force_refresh:
return self._files_in_archive
cmd = [self.cmd_path, '--list', '-C', self.dest]
if self.zipflag:
cmd.append(self.zipflag)
if self.opts:
cmd.extend(['--show-transformed-names'] + self.opts)
if self.excludes:
cmd.extend(['--exclude=' + quote(f) for f in self.excludes])
cmd.extend(['-f', self.src])
rc, out, err = self.module.run_command(cmd,
cwd=self.dest,
environ_update=dict(
LANG='C',
LC_ALL='C',
LC_MESSAGES='C'))
if rc != 0:
raise UnarchiveError('Unable to list files in the archive')
for filename in out.splitlines():
# Compensate for locale-related problems in gtar output (octal unicode representation) #11348
# filename = filename.decode('string_escape')
filename = to_native(codecs.escape_decode(filename)[0])
if filename and filename not in self.excludes:
# We don't allow absolute filenames. If the user wants to unarchive rooted in "/"
# they need to use "dest: '/'". This follows the defaults for gtar, pax, etc.
# Allowing absolute filenames here also causes bugs: https://github.com/ansible/ansible/issues/21397
if filename.startswith('/'):
filename = filename[1:]
self._files_in_archive.append(filename)
return self._files_in_archive
import numpy as np import _pickle as cPickle import codecs a2 = np.arange(24).reshape(3, 4, 2) a = [[[2.45, 243], [3.45, 128]], [[1, 67], [8.6, 255]]] b = cPickle.dumps(a) print(b) print(type(b)) s = str(b)[2:-1] print(s) print(type(s)) b1 = bytes(s, encoding="gbk") print(b1) print(type(b1)) b2 = codecs.escape_decode(b1, 'hex-escape')[0] print(b2) print(type(b2)) a1 = cPickle.loads(b2) print(a1) print(type(a1))
def parsed_separator(self):
return codecs.escape_decode(bytes(self.separator,
"utf-8"))[0].decode("utf-8")
def _parse_entry(line: str, quoted: bool, escaped: bool,
get_bytes: bool) -> Tuple[Union[str, bytes], str]:
"""
Parses the next entry from the given space separated content.
:param line: content to be parsed
:param quoted: parses the next entry as a quoted value, removing the quotes
:param escaped: unescapes the string
:param get_bytes: provides **bytes** for the entry rather than a **str**
:returns: **tuple** of the form (entry, remainder)
:raises:
* **ValueError** if quoted is True without the next value being quoted
* **IndexError** if there's nothing to parse from the line
"""
if line == '':
raise IndexError('no remaining content to parse')
next_entry, remainder = '', line
if quoted:
# validate and parse the quoted value
start_quote, end_quote = _get_quote_indices(remainder, escaped)
if start_quote != 0 or end_quote == -1:
raise ValueError("the next entry isn't a quoted value: " + line)
next_entry, remainder = remainder[1:end_quote], remainder[end_quote +
1:]
else:
# non-quoted value, just need to check if there's more data afterward
if ' ' in remainder:
next_entry, remainder = remainder.split(' ', 1)
else:
next_entry, remainder = remainder, ''
if escaped:
# Tor does escaping in its 'esc_for_log' function of 'common/util.c'. It's
# hard to tell what controller functions use this in practice, but direct
# users are...
#
# * 'COOKIEFILE' field of PROTOCOLINFO responses
# * logged messages about bugs
# * the 'getinfo_helper_listeners' function of control.c
#
# Ideally we'd use "next_entry.decode('string_escape')" but it was removed
# in python 3.x and 'unicode_escape' isn't quite the same...
#
# https://stackoverflow.com/questions/14820429/how-do-i-decodestring-escape-in-python3
next_entry = codecs.escape_decode(next_entry)[0] # type: ignore
if not get_bytes:
next_entry = stem.util.str_tools._to_unicode(
next_entry) # normalize back to str
if get_bytes:
return (stem.util.str_tools._to_bytes(next_entry), remainder.lstrip())
else:
return (next_entry, remainder.lstrip())
def test_empty(self):
self.assertEqual(codecs.escape_decode(b""), (b"", 0))
self.assertEqual(codecs.escape_decode(bytearray()), (b"", 0))
def load_model_from_pyml_exp_db(args_dict, only_done=True, logs_dir="."):
# merge database from log files
logs_dir = os.path.abspath(logs_dir)
dfs = []
for pyml_exp_db_path in glob.glob(os.path.join(logs_dir, "log*.db*")):
logging.info("Loading experiments db %s" % pyml_exp_db_path)
try:
df = pyml_db_to_pandas(
pyml_exp_db_path,
args_dict=args_dict,
exclude_args=[
"num_bootstraps",
"scores_file",
"inference",
"alpha", # XXX rm
"num_threads",
"disable_eval",
])
except (sqlite3.DatabaseError, pd.io.sql.DatabaseError) as e:
# database is probably empy
logging.error("DatabaseError: %s" % e)
continue
dfs.append(df)
if not dfs:
return
df = pd.concat(dfs)
df.fillna(value=np.nan, inplace=True)
# maybe run was aborted
if only_done:
df = df.query("_state == 'done'")
# get model binary string
if len(df) == 0:
return
if not "model" in df.columns:
logging.warn("Database %s doesn't have a 'model' column" %
(pyml_exp_db_path))
return
df = df[~df.model.isnull()] # only consider experiments which with chckpts
if len(df) == 0:
return
model_token = df.loc[df._end_date == df._end_date.max()].model.tolist()[0]
try:
# XXX hack to remove quotes from string
model_token = model_token[1:-1]
logging.info("Loading saved model state from %s" % model_token)
state_dict = torch.load(model_token)
print(state_dict)
return state_dict, model_token
except:
# load model from binary string
model_str = codecs.escape_decode(model_token[1:-1])[0][1:]
copy = {}
for param, val in args_dict.items():
if isinstance(val, list):
val = tuple(val)
copy[param] = val
key = hash(frozenset(copy.items()))
ofile = "model%s.pkl" % key
return unserialize_model(model_str), ofile
def parse_args():
"""
Argument parsing
The client works by giving general options, a subcommand
and then options specific to the subcommand.
For example:
clikraken ticker # just a subcommand
clikraken depth --pair XETHZEUR # subcommand option
clikraken ohlc -i 15 -s 1508513700
clikraken --raw olist # global option
clikraken place buy 0.1337 10.42 # subcommand argument
"""
# some help strings that are repeated many times
pairs_help = "comma delimited list of asset pairs"
pair_help = "asset pair"
epilog_str = textwrap.dedent("""\
To get help about a subcommand use: clikraken SUBCOMMAND --help
For example:
clikraken place --help
Current default currency pair: {dp}.
Create or edit the setting file {usp} to change it.
If the setting file doesn't exist yet, you can create one by doing:
clikraken generate_settings > {usp}
You can also set the CLIKRAKEN_DEFAULT_PAIR environment variable
which has precedence over the settings from the settings file.
""".format(dp=gv.DEFAULT_PAIR, usp=gv.USER_SETTINGS_PATH))
parser = argparse.ArgumentParser(
description='clikraken - Command line client for the Kraken exchange',
epilog=epilog_str,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-V',
'--version',
action='store_const',
const=ck_utils.version,
dest='main_func',
help='show program version')
parser.add_argument('--debug', action='store_true', help='debug mode')
parser.add_argument('--raw',
action='store_true',
help='output raw json results from the API')
parser.add_argument('--csv',
action='store_true',
help='output results from the API as CSV')
parser.add_argument('--csvseparator',
default=';',
help='separator character to use with CSV output')
parser.add_argument(
'--cron',
action='store_true',
help=
'activate cron mode (tone down errors due to timeouts or unavailable Kraken service)'
)
parser.set_defaults(main_func=None)
subparsers = parser.add_subparsers(dest='subparser_name',
help='available subcommands')
# Generate setting.ini
parser_gen_settings = subparsers.add_parser(
'generate_settings',
help='[clikraken] Print default settings.ini to stdout',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_gen_settings.set_defaults(
sub_func=ck_utils.output_default_settings_ini)
# ----------
# Public API
# ----------
# Asset Pairs
parser_asset_pairs = subparsers.add_parser(
'asset_pairs',
aliases=['ap'],
help='[public] Get the list of available asset pairs',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_asset_pairs.set_defaults(sub_func=asset_pairs)
# Ticker
parser_ticker = subparsers.add_parser(
'ticker',
aliases=['t'],
help='[public] Get the ticker',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_ticker.add_argument('-p',
'--pair',
default=gv.TICKER_PAIRS,
help=pairs_help + " to get info on. ")
parser_ticker.set_defaults(sub_func=ticker)
# Market depth (Order book)
parser_depth = subparsers.add_parser(
'depth',
aliases=['d'],
help='[public] Get the current market depth data',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_depth.add_argument('-p',
'--pair',
default=gv.DEFAULT_PAIR,
help=pair_help)
parser_depth.add_argument('-c',
'--count',
type=int,
default=7,
help="maximum number of asks/bids.")
parser_depth.set_defaults(sub_func=depth)
# List of last trades
parser_last_trades = subparsers.add_parser(
'last_trades',
aliases=['lt'],
help='[public] Get the last trades',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_last_trades.add_argument('-p',
'--pair',
default=gv.DEFAULT_PAIR,
help=pair_help)
parser_last_trades.add_argument('-s',
'--since',
default=None,
help="return trade data since given id")
parser_last_trades.add_argument('-c',
'--count',
type=int,
default=15,
help="maximum number of trades.")
parser_last_trades.set_defaults(sub_func=last_trades)
# Open High Low Close data
parser_ohlc = subparsers.add_parser(
'ohlc',
aliases=['oh'],
help='[public] Get the ohlc data',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_ohlc.add_argument('-p',
'--pair',
default=gv.DEFAULT_PAIR,
help=pair_help)
parser_ohlc.add_argument(
'-i',
'--interval',
default=1,
help=
"return ohlc data for interval in minutes; 1, 5, 15, 30, 60, 240, 1440, 10800, 21600."
)
parser_ohlc.add_argument('-s',
'--since',
default=None,
help="return ohlc data since given id")
parser_ohlc.add_argument('-c',
'--count',
type=int,
default=50,
help="maximum number of intervals.")
parser_ohlc.set_defaults(sub_func=ohlc)
# -----------
# Private API
# -----------
# User balance
parser_balance = subparsers.add_parser(
'balance',
aliases=['bal'],
help='[private] Get your current balance',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_balance.set_defaults(sub_func=get_balance)
# User trade balance
parser_trade_balance = subparsers.add_parser(
'trade_balance',
aliases=['tbal'],
help='[private] Get your current trade balance',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_trade_balance.set_defaults(sub_func=get_trade_balance)
# Place an order
parser_place = subparsers.add_parser(
'place',
aliases=['p'],
help='[private] Place an order',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_place.add_argument('type', choices=['sell', 'buy'])
parser_place.add_argument('volume', type=Decimal)
parser_place.add_argument('price', default=None, nargs='?')
parser_place.add_argument('-l',
'--leverage',
default="none",
help='leverage for margin trading')
parser_place.add_argument('-p',
'--pair',
default=gv.DEFAULT_PAIR,
help=pair_help)
parser_place.add_argument(
'-t',
'--ordertype',
choices=['market', 'limit'],
default='limit',
help="order type. Currently implemented: [limit, market].")
parser_place.add_argument('-s',
'--starttm',
default=0,
help="scheduled start time")
parser_place.add_argument('-e',
'--expiretm',
default=0,
help="expiration time")
parser_place.add_argument('-q',
'--viqc',
action='store_true',
help="volume in quote currency")
parser_place.add_argument('-v',
'--validate',
action='store_true',
help="validate inputs only. do not submit order")
parser_place.set_defaults(sub_func=place_order)
# cancel an order
parser_cancel = subparsers.add_parser(
'cancel',
aliases=['x'],
help='[private] Cancel orders',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_cancel.add_argument('order_ids',
type=str,
nargs='+',
help="transaction ids")
parser_cancel.set_defaults(sub_func=cancel_order)
# List of open orders
parser_olist = subparsers.add_parser(
'olist',
aliases=['ol'],
help='[private] Get a list of your open orders',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_olist.add_argument('-p', '--pair', default=None, help=pair_help)
parser_olist.add_argument(
'-i',
'--txid',
default=None,
help=
'comma delimited list of transaction ids to query info about (20 maximum)'
)
parser_olist.set_defaults(sub_func=list_open_orders)
# List of open positions
parser_oplist = subparsers.add_parser(
'positions',
aliases=['pos'],
help='[private] Get a list of your open positions',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_oplist.set_defaults(sub_func=list_open_positions)
# List of closed orders
parser_clist = subparsers.add_parser(
'clist',
aliases=['cl'],
help='[private] Get a list of your closed orders',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_clist.add_argument('-p', '--pair', default=None, help=pair_help)
parser_clist.add_argument(
'-i',
'--txid',
default=None,
help=
'comma delimited list of transaction ids to query info about (20 maximum)'
)
parser_clist.set_defaults(sub_func=list_closed_orders)
# Get ledgers info
parser_ledgers = subparsers.add_parser(
'ledgers',
aliases=['lg'],
help='[private] Get ledgers info',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_ledgers.add_argument(
'-a',
'--asset',
default='all',
help='comma delimited list of assets to restrict output to')
parser_ledgers.add_argument(
'-t',
'--type',
default='all',
help=
'type of ledger to retrieve. Possible values: all|deposit|withdrawal|trade|margin'
)
parser_ledgers.add_argument(
'-s',
'--start',
default=None,
help='starting unix timestamp or ledger id of results (exclusive)')
parser_ledgers.add_argument(
'-e',
'--end',
default=None,
help='ending unix timestamp or ledger id of results (exclusive)')
parser_ledgers.add_argument('-o',
'--ofs',
default=None,
help='result offset')
parser_ledgers.add_argument(
'-i',
'--id',
default=None,
help=
'comma delimited list of ledger ids to query info about (20 maximum)')
parser_ledgers.set_defaults(sub_func=get_ledgers)
# Get trades info
parser_trades = subparsers.add_parser(
'trades',
aliases=['tr'],
help='[private] Get trades history',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_trades.add_argument(
'-t',
'--type',
default='all',
help=
'type of trade. Values: all|any position|closed position|closing position|no position'
)
# TODO: trades parameter
parser_trades.add_argument(
'-s',
'--start',
default=None,
help='starting unix timestamp or trade tx id of results (exclusive)')
parser_trades.add_argument(
'-e',
'--end',
default=None,
help='ending unix timestamp or trade tx id of results (exclusive)')
parser_trades.add_argument('-o',
'--ofs',
default=None,
help='result offset')
parser_trades.add_argument(
'-i',
'--id',
default=None,
help=
'comma delimited list of transaction ids to query info about (20 maximum)'
)
parser_trades.add_argument('-p', '--pair', default=None, help=pair_help)
parser_trades.set_defaults(sub_func=trades)
# User Funding
# Deposit Methods
parser_deposit_methods = subparsers.add_parser(
'deposit_methods',
aliases=['dm'],
help='[private] Get deposit methods',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_deposit_methods.add_argument('-a',
'--asset',
default=gv.DEFAULT_ASSET,
help='asset being deposited')
parser_deposit_methods.set_defaults(sub_func=get_deposit_methods)
# Deposit Addresses
parser_deposit_addresses = subparsers.add_parser(
'deposit_addresses',
aliases=['da'],
help='[private] Get deposit addresses',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_deposit_addresses.add_argument('-a',
'--asset',
default=gv.DEFAULT_ASSET,
help='asset being deposited')
parser_deposit_addresses.add_argument('-m',
'--method',
default=None,
help='name of the deposit method')
parser_deposit_addresses.add_argument(
'-n',
'--new',
action='store_true',
help="whether or not to generate a new address")
parser_deposit_addresses.add_argument('-1',
'--one',
action='store_true',
help="return just one address")
parser_deposit_addresses.set_defaults(sub_func=get_deposit_addresses)
args = parser.parse_args()
# make sure that either sub_func or main_func is defined
# otherwise just print usage and exit
# (this weird construction is a hack to work around Python bug #9351 https://bugs.python.org/issue9351)
if all(
[vars(args).get(f, None) is None for f in ['sub_func', 'main_func']]):
parser.print_usage()
sys.exit(0)
gv.CRON = args.cron
# Trick from https://stackoverflow.com/a/37059682/862188
# in order to be able to parse things like "\t" or "\\" for example
separator = codecs.escape_decode(bytes(args.csvseparator,
"utf-8"))[0].decode("utf-8")
gv.CSV_SEPARATOR = separator
return args
"--extrusion",
metavar=('EXTRUSION'),
dest='extrusion',
type=float,
default=0.,
help="EXTRUSION = (height of patch - box height) / 2, default 0.")
parser.add_argument("-d",
"--delimiter",
metavar=('DELIMITER'),
dest='inputdelimiter',
default=',',
help="delimiter to seperate coordinates, default ','.")
args = parser.parse_args()
delimiter = codecs.escape_decode(bytes(args.inputdelimiter,
"utf-8"))[0].decode("utf-8")
# http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python#answer-37059682
inputfileobj = open(args.MASKFILENAME, 'r')
filereader = csv.reader(inputfileobj, delimiter=delimiter)
linenum = 0
for coordquad in filereader:
linenum += 1
if (len(coordquad) < 4):
raise (Exception("Error: incomplete quadruple detected at line %d." %
linenum))
else:
rtx = min(float(coordquad[0]), float(coordquad[2]))
rty = min(float(coordquad[1]), float(coordquad[3]))
rtw = abs(float(coordquad[0]) - float(coordquad[2]))
rth = abs(float(coordquad[1]) - float(coordquad[3]))
# rotate 180: (900,900) - All cood
def unescapestr(s):
return codecs.escape_decode(s)[0]
# #define PROT_GROWSDOWN 0x01000000
# #define PROT_GROWSUP 0x02000000
PROT_NONE = 0x0
PROT_READ = 0x1
PROT_WRITE = 0x2
PROT_EXEC = 0x4
# Machine code of an empty C function, generated with gcc
# Disassembly:
# 55 push %ebp
# 89 e5 mov %esp,%ebp
# 5d pop %ebp
# c3 ret
#code = "\x48\x31\xc0\x50\x48\xbf\x2f\x62\x69\x6e\x2f\x2f\x73\x68\x57\xb0\x3b\x48\x89\xe7\x48\x31\xf6\x48\x31\xd2\x0f\x05"
code = codecs.escape_decode(sys.argv[1])[0]
# Get the address of the code
addr = addressof(cast(c_char_p(code), POINTER(c_char)).contents)
# Get the start of the page containing the code and set the permissions
pagesize = 0x1000
pagestart = addr & ~(pagesize - 1)
if mprotect(pagestart, pagesize, PROT_READ | PROT_WRITE | PROT_EXEC):
raise RuntimeError("Failed to set permissions using mprotect()")
# Generate ctypes function object from code
functype = CFUNCTYPE(None)
f = functype(addr)
# Call the function
def test_empty_escape_decode(self):
self.assertEquals(codecs.escape_decode(""), ("", 0))
def OnToolSearchUpClick(self, evt):
findString = codecs.escape_decode(self.ctrlSearch.GetSearchString())[0]
if len(findString) == 0:
wx.MessageBox("Please enter string to search", "Error", wx.ICON_ERROR)
return
self.DoSearch(findString, False)
def unescape(value):
return codecs.escape_decode(value)[0].decode('utf-8')
def print(self, data):
data = codecs.escape_decode(bytes(data, "utf-8"))[0]
self._print(data)
def decoded(string):
return codecs.escape_decode(bytes(string.replace(r'\"', r'\\"'), "utf-8"))[0].decode("utf-8")
def decode_escaped_string(text, encoding='utf-8'):
return codecs.escape_decode(text)[0].decode(encoding)
# to a string. The function must return a value that is compatible with
# FieldDescriptor.default_value and therefore a unicode string.
_DEFAULT_TO_STRING_MAP = {
messages.IntegerField: unicode,
messages.FloatField: unicode,
messages.BooleanField: lambda value: value and u'true' or u'false',
messages.BytesField: lambda value: codecs.escape_encode(value)[0],
messages.StringField: lambda value: value,
messages.EnumField: lambda value: unicode(value.number),
}
_DEFAULT_FROM_STRING_MAP = {
messages.IntegerField: int,
messages.FloatField: float,
messages.BooleanField: lambda value: value == u'true',
messages.BytesField: lambda value: codecs.escape_decode(value)[0],
messages.StringField: lambda value: value,
messages.EnumField: int,
}
class EnumValueDescriptor(messages.Message):
"""Enum value descriptor.
Fields:
name: Name of enumeration value.
number: Number of enumeration value.
"""
# TODO(rafek): Why are these listed as optional in descriptor.proto.
# Harmonize?
def string_to_bytes(string):
string = string.encode().hex()
string = '\\x' + '\\x'.join(a + b
for a, b in zip(string[::2], string[1::2]))
return codecs.escape_decode(string, 'hex')[0]
def generate(
self,
prompt: str = "",
temperature: float = 0.7,
max_tokens: int = 32,
stop: str = "",
model: str = "davinci",
bg: tuple = (31, 36, 40),
accent: tuple = (0, 64, 0),
pngquant: bool = False,
output_txt: str = None,
output_img: str = None,
include_prompt: bool = True,
include_coloring: bool = True,
watermark: str = "Generated using GPT-3 via OpenAI's API",
):
assert isinstance(stop, str), "stop is not a str."
data = {
"prompt": prompt,
"max_tokens": max_tokens,
"temperature": temperature,
"stop": stop,
"stream": True,
"logprobs": 1,
}
console = Console(record=True)
console.clear()
if include_prompt:
prompt_text = Text(prompt, style="bold", end="")
console.print(prompt_text, end="")
with httpx.stream(
"POST",
f"https://api.openai.com/v1/engines/{model}/completions",
headers=self.headers,
data=json.dumps(data),
timeout=None,
) as r:
for chunk in r.iter_text():
text = chunk[6:] # JSON chunks are prepended with "data: "
if len(text) < 10 and "[DONE]" in text:
break
temp_token = None
logprobs = json.loads(text)["choices"][0]["logprobs"]
tokens = logprobs["tokens"]
token_logprobs = logprobs["token_logprobs"]
for i in range(len(tokens)):
token = tokens[i]
log_prob = token_logprobs[i]
if token == stop or token == "<|endoftext|>":
break
if token.startswith("bytes:") and not temp_token:
# We need to hold the 2-byte token to the next 1-byte token
# to get the full bytestring to decode
#
# The API-returned tokens are in the form:
# "bytes:\xe2\x80" and "bytes:\x9d"
temp_token = token[6:]
temp_prob = log_prob
else:
if temp_token:
bytestring = temp_token + token[6:]
# https://stackoverflow.com/a/37059682/9314418
token = codecs.escape_decode(bytestring, "utf-8")[0].decode(
"utf-8"
)
temp_token = None
log_prob = temp_prob # the true prob is the first one
text = Text(
token,
style=f"on {self.derive_token_bg(log_prob, bg, accent, include_coloring,)}",
end="",
)
console.print(text, end="")
# Export the generated text as HTML.
raw_html = self.replace_hex_colors(
console.export_html(inline_styles=True, code_format="{code}", clear=False)
)
# Render the HTML as an image
prompt_hash = hashlib.sha256(bytes(prompt, "utf-8")).hexdigest()[0:8]
temp_string = str(temperature).replace(".", "_")
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
if output_img:
img_file_name = output_img
else:
if not os.path.exists("img_output"):
os.makedirs("img_output")
img_file_name = f"img_output/{timestamp}__{prompt_hash}__{temp_string}.png"
if self.imgmaker:
self.imgmaker.generate(
"dark.html",
{
"html": raw_html.replace("\n", "</br>"),
"accent": f"rgb({accent[0]},{accent[1]},{accent[2]})",
"watermark": watermark,
},
width=450,
height=600,
downsample=False,
output_file=img_file_name,
use_pngquant=pngquant,
)
# Save the generated text to a plain-text file
if output_txt:
txt_file_name = output_txt
else:
if not os.path.exists("txt_output"):
os.makedirs("txt_output")
txt_file_name = f"txt_output/{prompt_hash}__{temp_string}.txt"
with open(txt_file_name, "a", encoding="utf-8") as f:
f.write(console.export_text() + "\n" + "=" * 20 + "\n")
console.line()
def decode(self, input, final=False):
return codecs.escape_decode(input, self.errors)[0]
def parse(self):
isdir = os.path.isdir(self._path)
if isdir:
props = os.path.join(self._path, "font.props")
if os.path.isfile(props):
with open(props) as fd:
data = fd.readlines()
else:
raise Exception("Not an SFD directory")
else:
with open(self._path) as fd:
data = fd.readlines()
font = self._font
info = font.info
charData = None
offsetMetrics = []
i = 0
while i < len(data):
line = data[i]
i += 1
if ":" in line:
key, value = [v.strip() for v in line.split(":", 1)]
else:
key = line.strip()
value = None
if i == 1:
if key != "SplineFontDB":
raise Exception("Not an SFD file.")
version = float(value)
if version != 3.0:
raise Exception("Unsupported SFD version: %f" % version)
elif key == "FontName":
info.postscriptFontName = value
elif key == "FullName":
info.postscriptFullName = value
elif key == "FamilyName":
info.familyName = value
elif key == "DefaultBaseFilename":
pass # info.XXX = value
elif key == "Weight":
info.postscriptWeightName = value
elif key == "Copyright":
# Decode escape sequences.
info.copyright = codecs.escape_decode(value)[0].decode("utf-8")
elif key == "Comments":
info.note = value
elif key == "UComments":
old = info.note
info.note = SFDReadUTF7(value)
if old:
info.note += "\n" + old
elif key == "FontLog":
if not info.note:
info.note = ""
else:
info.note = "\n"
info.note += "Font log:\n" + SFDReadUTF7(value)
elif key == "Version":
info.versionMajor, info.versionMinor = parseVersion(value)
elif key == "ItalicAngle":
info.italicAngle = info.postscriptSlantAngle = float(value)
elif key == "UnderlinePosition":
info.postscriptUnderlinePosition = float(value)
elif key == "UnderlineWidth":
info.postscriptUnderlineThickness = float(value)
elif key in "Ascent":
info.ascender = int(value)
elif key in "Descent":
info.descender = -int(value)
elif key == "sfntRevision":
pass # info.XXX = int(value, 16)
elif key == "WidthSeparation":
pass # XXX = float(value) # auto spacing
elif key == "LayerCount":
self._layers = int(value) * [None]
self._layerType = int(value) * [None]
elif key == "Layer":
m = LAYER_RE.match(value)
idx, quadratic, name, _ = m.groups()
idx = int(idx)
quadratic = bool(int(quadratic))
name = SFDReadUTF7(name)
if idx == 1:
self._layers[idx] = font.layers.defaultLayer
else:
self._layers[idx] = name
self._layerType[idx] = quadratic
elif key == "DisplayLayer":
pass # XXX default layer
elif key == "DisplaySize":
pass # GUI
elif key == "AntiAlias":
pass # GUI
elif key == "FitToEm":
pass # GUI
elif key == "WinInfo":
pass # GUI
elif key == "Encoding":
pass # XXX encoding = value
elif key == "CreationTime":
v = datetime.utcfromtimestamp(int(value))
info.openTypeHeadCreated = v.strftime("%Y/%m/%d %H:%M:%S")
elif key == "ModificationTime":
pass # XXX
elif key == "FSType":
v = int(value)
v = [bit for bit in range(16) if v & (1 << bit)]
info.openTypeOS2Type = v
elif key == "PfmFamily":
pass # info.XXX = value
elif key in ("TTFWeight", "PfmWeight"):
info.openTypeOS2WeightClass = int(value)
elif key == "TTFWidth":
info.openTypeOS2WidthClass = int(value)
elif key == "Panose":
v = value.split()
info.openTypeOS2Panose = [int(n) for n in v]
elif key == "LineGap":
info.openTypeHheaLineGap = int(value)
elif key == "VLineGap":
info.openTypeVheaVertTypoLineGap = int(value)
elif key == "HheadAscent":
info.openTypeHheaAscender = int(value)
elif key == "HheadDescent":
info.openTypeHheaDescender = int(value)
elif key == "OS2TypoLinegap":
info.openTypeOS2TypoLineGap = int(value)
elif key == "OS2Vendor":
info.openTypeOS2VendorID = value.strip("'")
elif key == "OS2FamilyClass":
v = int(value)
info.openTypeOS2FamilyClass = (v >> 8, v & 0xff)
elif key == "OS2Version":
pass # XXX
elif key == "OS2_WeightWidthSlopeOnly":
if int(value):
if not info.openTypeOS2Selection:
info.openTypeOS2Selection = []
info.openTypeOS2Selection += [8]
elif key == "OS2_UseTypoMetrics":
if not info.openTypeOS2Selection:
info.openTypeOS2Selection = []
info.openTypeOS2Selection += [7]
elif key == "OS2CodePages":
pass # XXX
elif key == "OS2UnicodeRanges":
pass # XXX
elif key == "OS2TypoAscent":
info.openTypeOS2TypoAscender = int(value)
elif key == "OS2TypoDescent":
info.openTypeOS2TypoDescender = int(value)
elif key == "OS2WinAscent":
info.openTypeOS2WinAscent = int(value)
elif key == "OS2WinDescent":
info.openTypeOS2WinDescent = int(value)
elif key in self._OFFSET_METRICS:
if int(value):
offsetMetrics.append(self._OFFSET_METRICS[key])
elif key == "OS2SubXSize":
info.openTypeOS2SubscriptXSize = int(value)
elif key == "OS2SubYSize":
info.openTypeOS2SubscriptYSize = int(value)
elif key == "OS2SubXOff":
info.openTypeOS2SubscriptXOffset = int(value)
elif key == "OS2SubYOff":
info.openTypeOS2SubscriptYOffset = int(value)
elif key == "OS2SupXSize":
info.openTypeOS2SuperscriptXSize = int(value)
elif key == "OS2SupYSize":
info.openTypeOS2SuperscriptYSize = int(value)
elif key == "OS2SupXOff":
info.openTypeOS2SuperscriptXOffset = int(value)
elif key == "OS2SupYOff":
info.openTypeOS2SuperscriptYOffset = int(value)
elif key == "OS2StrikeYSize":
info.openTypeOS2StrikeoutSize = int(value)
elif key == "OS2StrikeYPos":
info.openTypeOS2StrikeoutPosition = int(value)
elif key == "OS2CapHeight":
info.capHeight = int(value)
elif key == "OS2XHeight":
info.xHeight = int(value)
elif key == "UniqueID":
info.postscriptUniqueID = int(value)
elif key == "LangName":
self._parseNames(value)
elif key == "GaspTable":
self._parseGaspTable(value)
elif key == "BeginPrivate":
section, i = self._getSection(data, i, "EndPrivate", value)
self._parsePrivateDict(section)
elif key == "BeginChars":
charData, i = self._getSection(data, i, "EndChars")
elif key == "Grid":
grid, i = self._getSection(data, i, "EndSplineSet")
self._parseGrid(grid)
elif key == "KernClass2":
i = self._parseKernClass(data, i, value)
elif key == "Lookup":
self._parseLookup(value)
elif key == "AnchorClass2":
self._parseAnchorClass(value)
elif key == "XUID":
pass # XXX
elif key == "UnicodeInterp":
pass # XXX
elif key == "NameList":
pass # XXX
elif key == "DEI":
pass
elif key == "EndSplineFont":
break
#else:
# print(key, value)
for idx, name in enumerate(self._layers):
if not isinstance(name, (str, unicode)):
continue
if idx not in (0, 1) and self._layers.count(name) != 1:
# FontForge layer names are not unique, make sure ours are.
name += "_%d" % idx
self._layers[idx] = font.newLayer(name)
if isdir:
assert charData is None
import glob
charData = []
for filename in glob.iglob(os.path.join(self._path, '*.glyph')):
with open(filename) as fp:
charData += fp.readlines()
self._parseChars(charData)
# We can’t insert the references while parsing the glyphs since
# FontForge uses glyph indices so we need to know the glyph order
# first.
self._processReferences()
# Same for kerning.
self._processKerns()
# We process all kern classes together so we can detect UFO group
# overlap issue and act accordingly.
subtables = []
for lookup in self._gposLookups:
for subtable in self._gposLookups[lookup]:
if subtable in self._kernClasses:
subtables.append(self._kernClasses[subtable])
processKernClasses(self._font, subtables)
# Need to run after parsing glyphs so that we can calculate font
# bounding box.
self._fixOffsetMetrics(offsetMetrics)
self._writeGSUBGPOS(isgpos=False)
self._writeGSUBGPOS(isgpos=True)
self._writeGDEF()
# FontForge does not have an explicit UPEM setting, it is the sum of its
# ascender and descender.
info.unitsPerEm = info.ascender - info.descender
# Fallback for missing styleName.
# FontForge does more magic in its _GetModifiers functions, but this is
# a stripped down version.
if info.styleName is None:
value = "Regular"
if info.postscriptFontName and "-" in info.postscriptFontName:
value = info.postscriptFontName.split("-", 1)[1]
elif info.postscriptWeightName:
value = info.postscriptWeightName
info.styleName = value
def get_song(self, videoId: str) -> Dict:
"""
Returns metadata about a song or video.
:param videoId: Video id
:return: Dictionary with song metadata.
Example::
{
"videoId": "ZrOKjDZOtkA",
"title": "Wonderwall (Remastered)",
"lengthSeconds": "259",
"keywords": [
"Oasis",
"(What's",
"..."
],
"channelId": "UCmMUZbaYdNH0bEd1PAlAqsA",
"isOwnerViewing": false,
"shortDescription": "Provided to YouTube by Ignition...",
"isCrawlable": true,
"thumbnail": {
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/ZrOKjDZOtkA/maxresdefault.jpg",
"width": 1920,
"height": 1080
}
]
},
"averageRating": 4.5673099,
"allowRatings": true,
"viewCount": "18136380",
"author": "Oasis - Topic",
"isPrivate": false,
"isUnpluggedCorpus": false,
"isLiveContent": false,
"provider": "Ignition",
"artists": [
"Oasis"
],
"copyright": "℗ 2014 Big Brother Recordings ...",
"production": [
"Composer: Noel Gallagher",
"Lyricist: Noel Gallagher",
"Producer: Owen Morris & Noel Gallagher"
],
"release": "2014-09-29",
"streamingData": {
"expiresInSeconds": "21540",
"formats": [
{
"itag": 18,
"mimeType": "video/mp4; codecs=\"avc1.42001E, mp4a.40.2\"",
"bitrate": 306477,
"width": 360,
"height": 360,
"lastModified": "1574970034520502",
"contentLength": "9913027",
"quality": "medium",
"fps": 25,
"qualityLabel": "360p",
"projectionType": "RECTANGULAR",
"averageBitrate": 306419,
"audioQuality": "AUDIO_QUALITY_LOW",
"approxDurationMs": "258809",
"audioSampleRate": "44100",
"audioChannels": 2,
"signatureCipher": "..."
}
],
"adaptiveFormats": []
}
}
"""
endpoint = "https://www.youtube.com/get_video_info"
params = {"video_id": videoId, "hl": self.language, "el": "detailpage"}
response = requests.get(endpoint, params, headers=self.headers, proxies=self.proxies)
text = parse_qs(response.text)
if 'player_response' not in text:
return text
player_response = json.loads(text['player_response'][0])
song_meta = player_response['videoDetails']
if song_meta['shortDescription'].endswith("Auto-generated by YouTube."):
try:
description = song_meta['shortDescription'].split('\n\n')
for i, detail in enumerate(description):
description[i] = codecs.escape_decode(detail)[0].decode('utf-8')
song_meta['provider'] = description[0].replace('Provided to YouTube by ', '')
song_meta['artists'] = [artist for artist in description[1].split(' · ')[1:]]
song_meta['copyright'] = description[3]
song_meta['release'] = None if len(description) < 5 else description[4].replace(
'Released on: ', '')
song_meta['production'] = None if len(description) < 6 else [
pub for pub in description[5].split('\n')
]
except (KeyError, IndexError):
pass
song_meta['streamingData'] = player_response['streamingData']
return song_meta
def parse_txt(mnemo, attrib, txt, loc_db):
"""Parse an assembly listing. Returns an AsmCfg instance
@mnemo: architecture used
@attrib: architecture attribute
@txt: assembly listing
@loc_db: the LocationDB instance used to handle labels of the listing
"""
C_NEXT = asmblock.AsmConstraint.c_next
C_TO = asmblock.AsmConstraint.c_to
lines = []
# parse each line
for line in txt.split('\n'):
# empty
if EMPTY_RE.match(line):
continue
# comment
if COMMENT_RE.match(line):
continue
# labels to forget
if FORGET_LABEL_RE.match(line):
continue
# label beginning with .L
match_re = LABEL_RE.match(line)
if match_re:
label_name = match_re.group(1)
label = loc_db.get_or_create_name_location(label_name)
lines.append(label)
continue
# directive
if DIRECTIVE_START_RE.match(line):
match_re = DIRECTIVE_RE.match(line)
directive = match_re.group(1)
if directive in ['text', 'data', 'bss']:
continue
if directive in ['string', 'ascii']:
# XXX HACK
line = line.replace(r'\n', '\n').replace(r'\r', '\r')
raw = line[line.find(r'"') + 1:line.rfind(r'"')]
raw = codecs.escape_decode(raw)[0]
if directive == 'string':
raw += b"\x00"
lines.append(asmblock.AsmRaw(raw))
continue
if directive == 'ustring':
# XXX HACK
line = line.replace(r'\n', '\n').replace(r'\r', '\r')
raw = line[line.find(r'"') + 1:line.rfind(r'"')] + "\x00"
raw = codecs.escape_decode(raw)[0]
out = b''
for i in range(len(raw)):
out += raw[i:i + 1] + b'\x00'
lines.append(asmblock.AsmRaw(out))
continue
if directive in declarator:
data_raw = line[match_re.end():].split(' ', 1)[1]
data_raw = data_raw.split(',')
size = declarator[directive]
expr_list = []
# parser
for element in data_raw:
element = element.strip()
element_parsed = base_expr.parseString(element)[0]
element_expr = asm_ast_to_expr_with_size(
element_parsed, loc_db, size)
expr_list.append(element_expr)
raw_data = asmblock.AsmRaw(expr_list)
raw_data.element_size = size
lines.append(raw_data)
continue
if directive == 'comm':
# TODO
continue
if directive == 'split': # custom command
lines.append(DirectiveSplit())
continue
if directive == 'dontsplit': # custom command
lines.append(DirectiveDontSplit())
continue
if directive == "align":
align_value = int(line[match_re.end():], 0)
lines.append(DirectiveAlign(align_value))
continue
if directive in [
'file', 'intel_syntax', 'globl', 'local', 'type', 'size',
'align', 'ident', 'section'
]:
continue
if directive[0:4] == 'cfi_':
continue
raise ValueError("unknown directive %s" % directive)
# label
match_re = LABEL_RE.match(line)
if match_re:
label_name = match_re.group(1).encode()
label = loc_db.get_or_create_name_location(label_name)
lines.append(label)
continue
# code
if ';' in line:
line = line[:line.find(';')]
line = line.strip(' ').strip('\t')
instr = mnemo.fromstring(line, loc_db, attrib)
lines.append(instr)
asmblock.log_asmblock.info("___pre asm oki___")
# make asmcfg
cur_block = None
state = STATE_NO_BLOC
i = 0
asmcfg = asmblock.AsmCFG(loc_db)
block_to_nlink = None
delayslot = 0
while i < len(lines):
if delayslot:
delayslot -= 1
if delayslot == 0:
state = STATE_NO_BLOC
line = lines[i]
# no current block
if state == STATE_NO_BLOC:
if isinstance(line, DirectiveDontSplit):
block_to_nlink = cur_block
i += 1
continue
elif isinstance(line, DirectiveSplit):
block_to_nlink = None
i += 1
continue
elif not isinstance(line, LocKey):
# First line must be a label. If it's not the case, generate
# it.
loc = loc_db.add_location()
cur_block = asmblock.AsmBlock(loc_db,
loc,
alignment=mnemo.alignment)
else:
cur_block = asmblock.AsmBlock(loc_db,
line,
alignment=mnemo.alignment)
i += 1
# Generate the current block
asmcfg.add_block(cur_block)
state = STATE_IN_BLOC
if block_to_nlink:
block_to_nlink.addto(
asmblock.AsmConstraint(cur_block.loc_key, C_NEXT))
block_to_nlink = None
continue
# in block
elif state == STATE_IN_BLOC:
if isinstance(line, DirectiveSplit):
state = STATE_NO_BLOC
block_to_nlink = None
elif isinstance(line, DirectiveDontSplit):
state = STATE_NO_BLOC
block_to_nlink = cur_block
elif isinstance(line, DirectiveAlign):
cur_block.alignment = line.alignment
elif isinstance(line, asmblock.AsmRaw):
cur_block.addline(line)
block_to_nlink = cur_block
elif isinstance(line, LocKey):
if block_to_nlink:
cur_block.addto(asmblock.AsmConstraint(line, C_NEXT))
block_to_nlink = None
state = STATE_NO_BLOC
continue
# instruction
elif isinstance(line, instruction):
cur_block.addline(line)
block_to_nlink = cur_block
if not line.breakflow():
i += 1
continue
if delayslot:
raise RuntimeError("Cannot have breakflow in delayslot")
if line.dstflow():
for dst in line.getdstflow(loc_db):
if not isinstance(dst, ExprId):
continue
if dst in mnemo.regs.all_regs_ids:
continue
cur_block.addto(asmblock.AsmConstraint(dst.name, C_TO))
if not line.splitflow():
block_to_nlink = None
delayslot = line.delayslot + 1
else:
raise RuntimeError("unknown class %s" % line.__class__)
i += 1
for block in asmcfg.blocks:
# Fix multiple constraints
block.fix_constraints()
# Log block
asmblock.log_asmblock.info(block)
return asmcfg
def test_escape_decode(self):
#sanity checks
value, length = codecs.escape_decode("ab\a\b\t\n\r\f\vba")
self.assertEqual(value, 'ab\x07\x08\t\n\r\x0c\x0bba')
self.assertEqual(length, 11)
value, length = codecs.escape_decode("\\a")
self.assertEqual(value, '\x07')
self.assertEqual(length, 2)
value, length = codecs.escape_decode("ab\a\b\t\n\r\f\vbaab\\a\\b\\t\\n\\r\\f\\vbaab\\\a\\\b\\\t\\\n\\\r\\\f\\\vba")
self.assertEqual(value, 'ab\x07\x08\t\n\r\x0c\x0bbaab\x07\x08\t\n\r\x0c\x0bbaab\\\x07\\\x08\\\t\\\r\\\x0c\\\x0bba')
self.assertEqual(length, 47)
value, length = codecs.escape_decode("\\\a")
self.assertEqual(value, '\\\x07')
self.assertEqual(length, 2)
self.assertEqual("abc", codecs.escape_decode("abc", None)[0])
self.assertEqual("?\\", codecs.escape_decode("\\x", 'replace')[0])
self.assertEqual("?\\x", codecs.escape_decode("\\x2", 'replace')[0])
self.assertEqual("?\\x", codecs.escape_decode("\\xI", 'replace')[0])
self.assertEqual("?\\xI", codecs.escape_decode("\\xII", 'replace')[0])
self.assertEqual("?\\x1", codecs.escape_decode("\\x1I", 'replace')[0])
self.assertEqual("?\\xI", codecs.escape_decode("\\xI1", 'replace')[0])
def decode_string(text):
return codecs.escape_decode(text.replace('=', '\\x'))[0].decode()
