def __change_provider(expression, locals_, globals_, provider,
return_value_queue, iterate_literal_arrays, operator):
'''
guyadini - TODO: I don't really understand this function.
Itsik - can you help explain this?
The only documentation so far is "thread or process?"
TODO 2: Why do we need to pass iterate_literal_arrays and operator? This is implicit and strange
(since they aren't used anywhere in the code, but rather inside the evaluation).
'''
future_object = python.eval(expression[1][1], globals_, locals_)
(ignored_value, new_provider,
new_return_value_queue) = python.eval(future_object.get_attributes())
if new_provider != provider:
changed_provider = True
return_value_queue = new_return_value_queue()
provider = new_provider
# Unpack __builtins.__attributedcode()
expression[1] = python.eval(future_object.get_expression())
return (provider, changed_provider, return_value_queue)
def __change_provider(expression, locals_, globals_, provider, return_value_queue,
iterate_literal_arrays, operator):
'''
guyadini - TODO: I don't really understand this function.
Itsik - can you help explain this?
The only documentation so far is "thread or process?"
TODO 2: Why do we need to pass iterate_literal_arrays and operator? This is implicit and strange
(since they aren't used anywhere in the code, but rather inside the evaluation).
'''
future_object = python.eval(expression[1][1], globals_, locals_)
(ignored_value, new_provider, new_return_value_queue) = python.eval(future_object.get_attributes())
if new_provider != provider:
changed_provider = True
return_value_queue = new_return_value_queue()
provider = new_provider
# Unpack __builtins.__attributedcode()
expression[1] = python.eval(future_object.get_expression())
return (provider, changed_provider, return_value_queue)
def __change_provider_in_multi(item, provider, operator, locals_,
return_value_queue, expression,
iterate_literal_arrays):
(ignored_value, new_provider,
new_return_value_queue) = python.eval(item.get_attributes())
if new_provider != provider:
changed_provider = True
return_value_queue = new_return_value_queue()
provider = new_provider
# Unpack __builtins.__attributedcode()
expression = [(None, None),
python.eval(item.get_expression())] + expression[1:]
# Overwrite `item` with current input
item = __fix_str_form(locals_.get('_', None))
return (item, expression, return_value_queue, changed_provider)
def a01(t, token_error=False):
i = 0
for e in range(0, len(t)):
if token_error == False:
i += ord(t[e]) + e
else:
try:
i += eval('ord(t[%s]) %s' % (e, TOKEN_OPER[0]))
except:
i += eval('ord(t[%s]) %s' % (e, TOKEN_OPER[1]))
return i
def getAllQuals(url, online=None):
try:
if online == None:
if check(url) == False:
raise Exception('Video not available')
video_url_a = []
page_data_string = client.request(url, httpsskip=True)
srcs = re.findall(r'{.*mp4.*}', page_data_string)[0]
srcs = srcs.replace("file", "'file'").replace("label", "'label'")
links = eval("{'file':'','label':''}," + srcs)
#print links
for link in links:
try:
#print link
page_link = link['file']
if page_link != None and len(page_link) > 0:
qs = link['label'].replace(' ', '').lower()
if 'auto' in qs:
qs = '720p'
if 'hls' in qs:
pass
else:
f_i = {'label': qs, 'page': page_link}
video_url_a.append(f_i)
except:
pass
video_urlf = video_url_a
return (video_urlf, '', None)
except Exception as e:
e = '{}'.format(e)
return (None, e, None)
def eval(source, scope=None, origin="__main"):
name = uuid.uuid4().hex
mod = module(name)
mod.__name__ = name
mod._m_outerScope = scope
pysrc, _, lastline = ecompile(source, scope, origin).rpartition('\n')
pysrc = '\n'.join(["from monte.runtime import compiler_helpers as _monte",
pysrc,
"_m_evalResult = " + lastline])
mod.__loader__ = GeneratedCodeLoader(pysrc)
code = compile(pysrc, name, "exec")
import __builtin__
__builtin__.eval(code, mod.__dict__)
sys.modules[name] = mod
linecache.getlines(name, mod.__dict__)
return mod._m_evalResult
def custom_parse_json(data):
output = []
i = 0
while i < len(data):
match = JSON_TOKEN_RE.match(data, i)
if not match:
raise ValueError('JSON syntax error: %r' % (data[i : i + 16]))
i = match.end()
if match.group(1): # Whitespace.
pass
elif match.group(2): # String literal.
output.append('u')
if '\\/' not in match.group(2):
output.append(match.group(2)) # Fortunately same as Python syntax.
else:
for match2 in JSON_STRING_PART_RE.finditer(match.group(2)):
if match2.group() == '\\/':
output.append('/')
else:
output.append(match2.group())
elif match.group(3):
output.append(JSON_NAMED_VALUES[match.group(3)])
else:
# Fortunately punctuation and number literals are also the same as
# Python syntax.
output.append(match.group())
data = ''.join(output)
if ast:
return ast.literal_eval(data)
else:
# This is still safe because of the regexp scanning above.
return eval(data, {})
def nexec(statement, globals=None, locals=None, **kwargs):
"""Execute *statement* using *globals* and *locals* dictionaries as
*global* and *local* namespace. *statement* is transformed using
:class:`.NapiTransformer`."""
try:
import __builtin__ as builtins
except ImportError:
import builtins
from ast import parse
from napi.transformers import NapiTransformer
from ast import fix_missing_locations as fml
try:
node = parse(statement, '<string>', 'exec')
except ImportError:#KeyError:
exec(statement)
else:
if globals is None:
globals = builtins.globals()
if locals is None:
locals = {}
trans = NapiTransformer(globals=globals, locals=locals, **kwargs)
trans.visit(node)
code = compile(fml(node), '<string>', 'exec')
return builtins.eval(code, globals, locals)
def parseJSString(s):
try:
offset=1 if s[0]=='+' else 0
val = int(eval(s.replace('!+[]','1').replace('!![]','1').replace('[]','0').replace('(','str(')[offset:]))
return val
except:
pass
def neval(expression, globals=None, locals=None, **kwargs):
"""Evaluate *expression* using *globals* and *locals* dictionaries as
*global* and *local* namespace. *expression* is transformed using
:class:`.NapiTransformer`."""
try:
import __builtin__ as builtins
except ImportError:
import builtins
from ast import parse
from ast import fix_missing_locations as fml
try:
transformer = kwargs['transformer']
except KeyError:
from napi.transformers import NapiTransformer as transformer
#try:
node = parse(expression, '<string>', 'eval')
#except ImportError:
# builtins.eval(expression)
#else:
if globals is None:
globals = builtins.globals()
if locals is None:
locals = {}
trans = transformer(globals=globals, locals=locals, **kwargs)
trans.visit(node)
code = compile(fml(node), '<string>', 'eval')
return builtins.eval(code, globals, locals)
def notice_thread():
while True:
notice = get_redis().rpop('notice')
if (notice):
try:
n_dict = simplejson.loads(notice)
if (n_dict['cmd'] in CMD_CNF):
user_exists = user_request.has_key(str(n_dict['to']))
if (user_exists):
eval(n_dict['cmd'].lower())(n_dict)
else:
if (n_dict['t'] == 2 and time.time() < n_dict['ex']):
get_redis().lpush('notice60', notice)
except Exception as e:
print(e)
else:
time.sleep(5)
def notice_thread():
while True:
notice = get_redis().rpop('notice')
if(notice):
try:
n_dict = simplejson.loads(notice)
if(n_dict['cmd'] in CMD_CNF):
user_exists = user_request.has_key(str(n_dict['to']))
if(user_exists):
eval(n_dict['cmd'].lower())(n_dict)
else:
if(n_dict['t'] == 2 and time.time() < n_dict['ex']):
get_redis().lpush('notice60' , notice)
except Exception as e:
print(e)
else:
time.sleep(5)
def eval(value, expression):
'''
>>> eval(10, 'value / 10') == 1
True
'''
if six.PY2:
import __builtin__ as builtins
else:
import builtins
return builtins.eval(expression, {}, {'value': value})
def write(self, globals, locals):
result = __builtin__.eval(self.code, globals, locals)
if result!=None:
write = locals['_RF_write']
encode = self.encode
if encode:
result = self.encode(result)
else:
result = str(result)
write(result)
def eval(context):
'''.eval <command>'''
if not utils.isadmin(context.line['prefix'], bot):
return
if context.args:
try:
return str(__builtin__.eval(context.args))
except:
return repr(sys.exc_info()[1])
else:
return eval.__doc__
def a01(t, token_error=False, is9Anime=False, use_code=True):
i = 0
if use_code == True:
for e in range(0, len(t)):
if token_error == False:
if is9Anime == False:
i += ord(t[e]) + e
else:
i += ord(t[e]) + e
else:
try:
i += eval('ord(t[%s]) %s' % (e, TOKEN_OPER[0]))
except:
i += eval('ord(t[%s]) %s' % (e, TOKEN_OPER[1]))
else:
if is9Anime == True:
i = int(FLAGS[0]["no_code_val_anime"])
else:
i = int(FLAGS[0]["no_code_val"])
return i
def get_cookie(self, netloc, ua, timeout):
try:
headers = {'User-Agent': ua}
request = urllib2.Request(netloc, headers=headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
result = response.read(5242880)
jschl = re.findall('name="jschl_vc" value="(.+?)"/>', result)[0]
init = re.findall('setTimeout\(function\(\){\s*.*?.*:(.*?)};', result)[-1]
builder = re.findall(r"challenge-form\'\);\s*(.*)a.v", result)[0]
decryptVal = self.parseJSString(init)
lines = builder.split(';')
for line in lines:
if len(line) > 0 and '=' in line:
sections=line.split('=')
line_val = self.parseJSString(sections[1])
decryptVal = int(eval(str(decryptVal)+sections[0][-1]+str(line_val)))
answer = decryptVal + len(urlparse.urlparse(netloc).netloc)
query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % (netloc, jschl, answer)
if 'type="hidden" name="pass"' in result:
passval = re.findall('name="pass" value="(.*?)"', result)[0]
query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % (netloc, urllib.quote_plus(passval), jschl, answer)
time.sleep(6)
cookies = cookielib.LWPCookieJar()
handlers = [urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.HTTPCookieProcessor(cookies)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
try:
request = urllib2.Request(query, headers=headers)
response = urllib2.urlopen(request, timeout=int(timeout))
except:
pass
cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
if 'cf_clearance' in cookie: self.cookie = cookie
except:
pass
def eval(source, scope=None, origin="__main"):
name = origin.encode('ascii')
mod = module(name)
mod.__name__ = name
mod._m_outerScope = scope
if source in COMPILE_CACHE:
pysrc = COMPILE_CACHE[source].encode('ascii')
else:
pysrc, _, lastline = ecompile(source, scope, origin).rpartition('\n')
pysrc = '\n'.join(["from monte.runtime import compiler_helpers as _monte",
pysrc.encode('utf-8'),
"_m_evalResult = " + lastline.encode('utf-8')])
mod.__loader__ = GeneratedCodeLoader(pysrc)
COMPILE_CACHE[source] = pysrc
code = compile(pysrc, name, "exec")
import __builtin__
__builtin__.eval(code, mod.__dict__)
sys.modules[name] = mod
linecache.getlines(name, mod.__dict__)
return mod._m_evalResult
def eval(context):
'''.eval <command>'''
if context.args:
try:
bot.log(context, ('EVAL'), context.args)
return str(__builtin__.eval(context.args))
except:
return repr(sys.exc_info()[1])
else:
return eval.__doc__
def __change_provider_in_multi(item, provider, operator, locals_, return_value_queue, expression, iterate_literal_arrays):
(ignored_value, new_provider, new_return_value_queue) = python.eval(item.get_attributes())
if new_provider != provider:
changed_provider = True
return_value_queue = new_return_value_queue()
provider = new_provider
# Unpack __builtins.__attributedcode()
expression = [(None, None), python.eval(item.get_expression())] + expression[1:]
# Overwrite `item` with current input
item = __fix_str_form(locals_.get('_', None))
return (item, expression, return_value_queue, changed_provider)
def __init__(self, parser, parent, atts={}):
HandlerBase.__init__(self, parser, parent)
self.locals = self.parent.locals
self.globals = self.parent.globals
self.codestr = ""
encode = atts.get("encode", None)
if encode:
encode = __builtin__.eval(encode, self.globals, self.locals)
else:
encode = encode_character_data
# TODO: add supoprt for EvalNode without Encoding?
self.element = EncodedEvalNode(None, encode)
def __eval_current_atom(atom, locals_, globals_):
# TODO: This is a hack to support instances, should be re-written. If atom is `literal` or `instance`, it should appear as a metadata/attribute
try:
object_or_objects = python.eval(atom, globals_, locals_)
except NameError as e:
try:
import importlib
# NameError: name 'os' is not defined
# - OR -
# NameError: global name 'os' is not defined
mod_name = e.message[e.message.index("'") + 1:e.message.rindex("'")]
globals_.update({mod_name: importlib.import_module(mod_name)})
object_or_objects = python.eval(atom, globals_, locals_)
except Exception as e1:
raise e1
except TypeError as e:
# Due to eval()?
if (e.message == 'eval() arg 1 must be a string or code object'):
object_or_objects = atom
else:
raise e
return object_or_objects
def eval(*args):
## Note: shadowing built-in functions is considered herecy
cond = [isinstance(a, Raster) for a in args]
if len(args) and not all(cond):
if not any(cond):
return __builtin__.eval(*args) ## calls built-in eval
assert (0) ## a partial number of Rasters is not accepted
# if all arguments are a Raster, forward them to _lib.ma_
num = len(args)
NodeArr = Node * num # C array of Nodes
nodearr = NodeArr(*[r._node for r in args])
nodeptr = ct.cast(nodearr, ct.POINTER(Node))
_lib.ma_eval(nodeptr, num)
def __eval_current_atom(atom, locals_, globals_):
# TODO: This is a hack to support instances, should be re-written. If atom is `literal` or `instance`, it should appear as a metadata/attribute
try:
object_or_objects = python.eval(atom, globals_, locals_)
except NameError as e:
try:
import importlib
# NameError: name 'os' is not defined
mod_name = e.message.split()[1][1:-1]
globals_.update({mod_name: importlib.import_module(mod_name)})
object_or_objects = python.eval(atom, globals_, locals_)
except Exception as e1:
raise e1
except TypeError as e:
# Due to eval()?
if (e.message == 'eval() arg 1 must be a string or code object'):
object_or_objects = atom
else:
raise e
return object_or_objects
def __init__(self, linkBaseDict = None) : # object base resource dict can be passed in
if linkBaseDict is None :
try:
import __builtin__
self.linkBaseDict = __builtin__.eval('SmartObjectServiceBaseDict')
except AttributeError:
print 'SmartObjectServiceBaseDict not found'
else:
self.linkBaseDict = linkBaseDict
self._propertyLinks = {}
self._linkCache = {}
def parseJSString(self, s):
try:
offset=1 if s[0]=='+' else 0
chain = s.replace('!+[]','1').replace('!![]','1').replace('[]','0').replace('(','str(')[offset:]
if '/' in chain:
#print('division ok ')
#print('avant ' + chain)
val = chain.split('/')
gauche,sizeg = checkpart(val[0],-1)
droite,sized = checkpart(val[1],1)
sign = ''
chain = droite.replace(droite,'')
if droite.startswith('+') or droite.startswith('-'):
sign = droite[0]
droite = droite[1:]
#print('debug1 ' + str(gauche))
#print('debug2 ' + str(droite))
gg = eval(gauche)
dd = eval(droite)
chain = val[0][:-sizeg] + str(gg) + '/' + str(dd) + val[1][sized:]
#print('apres ' + chain)
val = float( eval(chain))
return val
except:
pass
def __init__(self, linkBaseDict=None): # object base resource dict can be passed in
if linkBaseDict is None:
try:
import __builtin__
self.linkBaseDict = __builtin__.eval("SmartObjectServiceBaseDict")
except AttributeError:
print "SmartObjectServiceBaseDict not found"
else:
self.linkBaseDict = linkBaseDict
self._propertyLinks = {}
self._linkCache = {}
def eval(expression, _dict={}, **kw):
# build execution namespace
args = ops.copy()
args.update(_dict)
args.update(kw)
for k, v in args.items():
if hasattr(v, "im"):
args[k] = _Operand(v)
import __builtin__
out = __builtin__.eval(expression, args)
try:
return out.im
except AttributeError:
return out
def eval(expression, _dict={}, **kw):
# build execution namespace
args = ops.copy()
args.update(_dict)
args.update(kw)
for k, v in args.items():
if hasattr(v, "im"):
args[k] = _Operand(v)
import __builtin__
out = __builtin__.eval(expression, args)
try:
return out.im
except AttributeError:
return out
def child(self, name, atts):
codestr = self.codestr
if codestr!="":
exec codestr+"\n" in self.globals, self.locals
self.codestr = ""
if name==RF_CLASS:
if atts.has_key('bases'):
import string
bases = "(" + string.join(string.split(atts['bases']),",") + ",)"
base_classes = __builtin__.eval(bases, self.globals, self.locals)
else:
base_classes = ()
eh = RF_CLASS_Handler(self.parser, self, atts['name'], base_classes)
classobj = eh.classobj
module = self.module
classobj.__module__ = module.__name__
module.__dict__[classobj.__name__] = classobj
else:
sys.stderr.write("Ignoring '%s'\n" % name)
sys.stderr.flush()
def write(self, globals, locals):
result = __builtin__.eval(self.code, globals, locals)
if result!=None:
locals['response'].write(str(result))
def write(self, globals, locals):
for item in __builtin__.eval(self.list, globals, locals):
locals[self.item] = item
self.children.write(globals, locals)
def write(self, globals, locals):
for clause in self.clauses:
if __builtin__.eval(clause[0], globals, locals):
clause[1].write(globals, locals)
break
def really_build_fake_type(cpy_type):
"NOT_RPYTHON (not remotely so!)."
debug_print('faking %r'%(cpy_type,))
kw = {}
if cpy_type.__name__ == 'SRE_Pattern':
import re
import __builtin__
p = re.compile("foo")
for meth_name in p.__methods__:
kw[meth_name] = EvenMoreObscureWrapping(__builtin__.eval("lambda p,*args,**kwds: p.%s(*args,**kwds)" % meth_name))
elif cpy_type.__name__ == 'SRE_Match':
import re
import __builtin__
m = re.compile("foo").match('foo')
for meth_name in m.__methods__:
kw[meth_name] = EvenMoreObscureWrapping(__builtin__.eval("lambda m,*args,**kwds: m.%s(*args,**kwds)" % meth_name))
else:
for s, v in cpy_type.__dict__.items():
if not (cpy_type is unicode and s in ['__add__', '__contains__']):
if s != '__getattribute__' or cpy_type is type(sys) or cpy_type is type(Exception):
kw[s] = v
kw['__module__'] = cpy_type.__module__
def fake__new__(space, w_type, __args__):
args_w, kwds_w = __args__.unpack()
args = [space.unwrap(w_arg) for w_arg in args_w]
kwds = {}
for (key, w_value) in kwds_w.items():
kwds[key] = space.unwrap(w_value)
try:
r = cpy_type.__new__(*[cpy_type]+args, **kwds)
except:
wrap_exception(space)
raise
w_obj = space.allocate_instance(W_Fake, w_type)
W_Fake.__init__(w_obj, space, r)
return w_obj
fake__new__.func_name = "fake__new__" + cpy_type.__name__
kw['__new__'] = gateway.interp2app(fake__new__)
if cpy_type.__base__ is not object and not issubclass(cpy_type, Exception):
assert cpy_type.__base__ is basestring, cpy_type
from pypy.objspace.std.basestringtype import basestring_typedef
base = basestring_typedef
else:
base = None
class W_Fake(W_Object):
typedef = StdTypeDef(
cpy_type.__name__, base, **kw)
def __init__(w_self, space, val):
w_self.val = val
w_self.space = space
def getdict(w_self, space):
try:
d = w_self.val.__dict__
except AttributeError:
return W_Object.getdict(w_self, space)
return space.wrap(d)
def unwrap(w_self, space):
return w_self.val
if cpy_type is types.FunctionType:
def __get__(self, obj, owner):
return fake_object(self.space, self.val.__get__(obj, owner))
W_Fake.__name__ = 'W_Fake%s'%(cpy_type.__name__.capitalize())
W_Fake.typedef.fakedcpytype = cpy_type
return W_Fake
def __node_main(current_value, last_value, globals_, locals_):
logging.info('In __node_main, current_value = %s, last_value = %s' % (current_value, last_value))
return_value = None
try:
###################################
# Try to eval()uate current_value #
###################################
try:
return_value = python.eval(current_value, globals_, locals_)
# Autoloader Try & Catch
except NameError as e:
try:
__import_module_from_exception(e, globals_)
return_value = python.eval(current_value, globals_, locals_)
except Exception as e1:
raise e1
# Current value is already Python Object
except TypeError as e:
# Due to eval()?
if (e.message == 'eval() arg 1 must be a string or code object'):
return_value = current_value
else:
raise e
##########################
# eval() Post Processing #
##########################
if isinstance(return_value, lang.remotefunction):
return_value.evaluate_host(globals_, locals_)
if isinstance(return_value, dict):
if last_value is not None:
return_value = return_value.get(last_value, False)
if return_value is None or (isinstance(return_value, bool) and return_value is True):
return_value = last_value
if callable(return_value):
# Ignore "copyright", "credits", "license", and "help"
if isinstance(return_value, (site._Printer, site._Helper)):
return_value = return_value()
else:
try:
return_value = return_value(last_value)
except TypeError as e:
if e.args[0].find('takes no arguments') != -1:
return_value = return_value()
else:
raise e
if return_value is None:
return_value = last_value
except SyntaxError as e:
##################################
# Try to exec()ute current_value #
##################################
try:
exec current_value in globals_, locals_
# Autoloader Try & Catch
except NameError as e:
try:
__import_module_from_exception(e, globals_)
exec current_value in globals_, locals_
except Exception as e1:
raise e1
return_value = last_value
return return_value
def ws_dispatch(line, request):
msg = simplejson.loads(line)
if (msg):
if (msg['cmd'] in CMD_CNF):
eval(msg['cmd'].lower())(request, msg)
def __node_main(current_value, last_value, globals_, locals_):
logging.info('In __node_main, current_value = %s, last_value = %s' %
(current_value, last_value))
return_value = None
try:
###################################
# Try to eval()uate current_value #
###################################
try:
return_value = python.eval(current_value, globals_, locals_)
# Autoloader Try & Catch
except NameError as e:
try:
__import_module_from_exception(e, globals_)
return_value = python.eval(current_value, globals_, locals_)
except Exception as e1:
raise e1
# Current value is already Python Object
except TypeError as e:
# Due to eval()?
if (e.message == 'eval() arg 1 must be a string or code object'):
return_value = current_value
else:
raise e
##########################
# eval() Post Processing #
##########################
if isinstance(return_value, lang.remotefunction):
return_value.evaluate_host(globals_, locals_)
if isinstance(return_value, dict):
if last_value is not None:
return_value = return_value.get(last_value, False)
if return_value is None or (isinstance(return_value, bool)
and return_value is True):
return_value = last_value
if callable(return_value):
# Ignore "copyright", "credits", "license", and "help"
if isinstance(return_value, (site._Printer, site._Helper)):
return_value = return_value()
else:
try:
return_value = return_value(last_value)
except TypeError as e:
if e.args[0].find('takes no arguments') != -1:
return_value = return_value()
else:
raise e
if return_value is None:
return_value = last_value
except SyntaxError as e:
##################################
# Try to exec()ute current_value #
##################################
try:
exec current_value in globals_, locals_
# Autoloader Try & Catch
except NameError as e:
try:
__import_module_from_exception(e, globals_)
exec current_value in globals_, locals_
except Exception as e1:
raise e1
return_value = last_value
return return_value
def really_build_fake_type(cpy_type):
"NOT_RPYTHON (not remotely so!)."
debug_print('faking %r' % (cpy_type, ))
kw = {}
if cpy_type.__name__ == 'SRE_Pattern':
import re
import __builtin__
p = re.compile("foo")
for meth_name in p.__methods__:
kw[meth_name] = EvenMoreObscureWrapping(
__builtin__.eval("lambda p,*args,**kwds: p.%s(*args,**kwds)" %
meth_name))
elif cpy_type.__name__ == 'SRE_Match':
import re
import __builtin__
m = re.compile("foo").match('foo')
for meth_name in m.__methods__:
kw[meth_name] = EvenMoreObscureWrapping(
__builtin__.eval("lambda m,*args,**kwds: m.%s(*args,**kwds)" %
meth_name))
else:
for s, v in cpy_type.__dict__.items():
if not (cpy_type is unicode and s in ['__add__', '__contains__']):
if s != '__getattribute__' or cpy_type is type(
sys) or cpy_type is type(Exception):
kw[s] = v
kw['__module__'] = cpy_type.__module__
def fake__new__(space, w_type, __args__):
args_w, kwds_w = __args__.unpack()
args = [space.unwrap(w_arg) for w_arg in args_w]
kwds = {}
for (key, w_value) in kwds_w.items():
kwds[key] = space.unwrap(w_value)
try:
r = cpy_type.__new__(*[cpy_type] + args, **kwds)
except:
wrap_exception(space)
raise
w_obj = space.allocate_instance(W_Fake, w_type)
W_Fake.__init__(w_obj, space, r)
return w_obj
fake__new__.func_name = "fake__new__" + cpy_type.__name__
kw['__new__'] = gateway.interp2app(fake__new__)
if cpy_type.__base__ is not object and not issubclass(cpy_type, Exception):
assert cpy_type.__base__ is basestring, cpy_type
from pypy.objspace.std.basestringtype import basestring_typedef
base = basestring_typedef
else:
base = None
class W_Fake(W_Object):
typedef = StdTypeDef(cpy_type.__name__, base, **kw)
def __init__(w_self, space, val):
w_self.val = val
w_self.space = space
def getdict(w_self, space):
try:
d = w_self.val.__dict__
except AttributeError:
return W_Object.getdict(w_self, space)
return space.wrap(d)
def unwrap(w_self, space):
return w_self.val
if cpy_type is types.FunctionType:
def __get__(self, obj, owner):
return fake_object(self.space, self.val.__get__(obj, owner))
W_Fake.__name__ = 'W_Fake%s' % (cpy_type.__name__.capitalize())
W_Fake.typedef.fakedcpytype = cpy_type
return W_Fake
def treetransforms():
"""Treebank binarization and conversion.
Usage: discodop treetransforms [input [output]] [options]
where input and output are treebanks; standard in/output is used if not given.
"""
flags = ('binarize optimalbinarize unbinarize splitdisc mergedisc '
'introducepreterminals renumber sentid removeempty '
'help markorigin markhead leftunary rightunary '
'tailmarker direction').split()
options = ('inputfmt= outputfmt= inputenc= outputenc= slice= ensureroot= '
'punct= headrules= functions= morphology= lemmas= factor= fmt= '
'markorigin= maxlen= enc= transforms= markovthreshold= labelfun= '
'transforms= reversetransforms= ').split()
try:
origopts, args = gnu_getopt(sys.argv[2:], 'h:v:H:', flags + options)
if len(args) > 2:
raise GetoptError('expected 0, 1, or 2 positional arguments')
except GetoptError as err:
print('error:', err, file=sys.stderr)
print(treetransforms.__doc__)
sys.exit(2)
opts = dict(origopts)
if '--fmt' in opts:
opts['--inputfmt'] = opts['--outputfmt'] = opts['--fmt']
if '--enc' in opts:
opts['--inputenc'] = opts['--outputenc'] = opts['--enc']
if opts.get('--outputfmt', treebank.WRITERS[0]) not in treebank.WRITERS:
print('error: unrecognized output format: %r\navailable formats: %s'
% (opts.get('--outputfmt'), ' '.join(treebank.WRITERS)),
file=sys.stderr)
sys.exit(2)
infilename = (args[0] if len(args) >= 1 else '-')
outfilename = (args[1] if len(args) == 2 and args[1] != '-'
else sys.stdout.fileno())
# open corpus
corpus = treebank.READERS[opts.get('--inputfmt', 'export')](
infilename,
encoding=opts.get('--inputenc', 'utf8'),
headrules=opts.get('--headrules'),
ensureroot=opts.get('--ensureroot'),
removeempty='--removeempty' in opts,
punct=opts.get('--punct'),
functions=opts.get('--functions'),
morphology=opts.get('--morphology'),
lemmas=opts.get('--lemmas'))
start, end = opts.get('--slice', ':').split(':')
start, end = (int(start) if start else None), (int(end) if end else None)
trees = corpus.itertrees(start, end)
if '--maxlen' in opts:
maxlen = int(opts['--maxlen'])
trees = ((key, item) for key, item in trees
if len(item.sent) <= maxlen)
if '--renumber' in opts:
trees = (('%8d' % n, item) for n, (_key, item) in enumerate(trees, 1))
# select transformations
actions = []
for key, value in origopts: # pylint: disable=unused-variable
if key == '--introducepreterminals':
actions.append(lambda tree, sent:
(introducepreterminals(tree, sent), sent))
if key == '--transforms':
actions.append(lambda tree, sent, value=value:
(treebanktransforms.transform(tree, sent,
treebanktransforms.expandpresets(value.split(','))),
sent))
if key in ('--binarize', '--optimalbinarize'):
if key == '--binarize':
actions.append(lambda tree, sent:
(binarize(
tree,
opts.get('--factor', 'right'),
int(opts.get('-h', 999)),
int(opts.get('-v', 1)),
revhorzmarkov=int(opts.get('-H', 0)),
leftmostunary='--leftunary' in opts,
rightmostunary='--rightunary' in opts,
tailmarker='$' if '--tailmarker' in opts else '',
direction='--direction' in opts,
headoutward='--headrules' in opts,
markhead='--markhead' in opts,
labelfun=builtins.eval( # pylint: disable=eval-used
opts['--labelfun'])
if '--labelfun' in opts else None),
sent))
elif key == '--optimalbinarize':
actions.append(lambda tree, sent:
(optimalbinarize(
tree, '|',
'--headrules' in opts,
int(opts.get('-h', 999)),
int(opts.get('-v', 1))),
sent))
if key == '--splitdisc':
actions.append(lambda tree, sent:
(splitdiscnodes(tree, '--markorigin' in opts), sent))
if key == '--mergediscnodes':
actions.append(lambda tree, sent: (mergediscnodes(tree), sent))
if key == '--unbinarize':
actions.append(lambda tree, sent: (unbinarize(tree, sent), sent))
if key == '--reversetransforms':
actions.append(lambda tree, sent, value=value:
(treebanktransforms.reversetransform(tree,
treebanktransforms.expandpresets(value.split(','))),
sent))
# read, transform, & write trees
if actions:
def applytransforms(trees):
"""Apply transforms and yield modified items."""
for key, item in trees:
for action in actions:
item.tree, item.sent = action(item.tree, item.sent)
yield key, item
trees = applytransforms(trees)
if 'binarize' in opts and '--markovthreshold' in opts:
trees = list(trees)
h, v = int(opts.get('-h', 999)), int(opts.get('-v', 1))
revh = int(opts.get('-H', 0))
markovthreshold([item.tree for _, item in trees],
int(opts['--markovthreshold']),
revh + h - 1,
v - 1 if v > 1 else 1)
if opts.get('--outputfmt') in ('mst', 'conll'):
if not opts.get('--headrules'):
raise ValueError('need head rules for dependency conversion')
cnt = 0
if opts.get('--outputfmt') == 'dact':
import alpinocorpus
outfile = alpinocorpus.CorpusWriter(outfilename)
if (not actions and opts.get('--inputfmt') in ('alpino', 'dact')
and set(opts) <= {'--slice', '--inputfmt', '--outputfmt',
'--renumber'}):
for n, (key, block) in islice(enumerate(
corpus.blocks().items(), 1), start, end):
outfile.write((('%8d' % n) if '--renumber' in opts
else key).encode('utf8'), block)
cnt += 1
else:
for key, item in trees:
outfile.write(key.encode('utf8'), treebank.writetree(
item.tree, item.sent, key, 'alpino',
comment=item.comment).encode('utf8'))
cnt += 1
else:
encoding = opts.get('outputenc', 'utf8')
with io.open(outfilename, 'w', encoding=encoding) as outfile:
# copy trees verbatim when only taking slice or converting encoding
if (not actions and opts.get('--inputfmt') == opts.get(
'--outputfmt') and set(opts) <= {'--slice', '--inputenc',
'--outputenc', '--inputfmt', '--outputfmt'}):
for block in islice(corpus.blocks().values(), start, end):
outfile.write(block)
cnt += 1
else:
if opts.get('--outputfmt', 'export') == 'bracket':
trees = ((key, canonicalize(item.tree) and item)
for key, item in trees)
if opts.get('--outputfmt', 'export') == 'export':
outfile.write(treebank.EXPORTHEADER)
fmt = opts.get('--outputfmt', 'export')
sentid = '--sentid' in opts
for key, item in trees:
outfile.write(treebank.writetree(item.tree, item.sent, key,
fmt, comment=item.comment, sentid=sentid))
cnt += 1
print('%s: transformed %d trees' % (args[0] if args else 'stdin', cnt),
file=sys.stderr)
def really_build_fake_type(cpy_type):
"NOT_RPYTHON (not remotely so!)."
#assert not issubclass(cpy_type, file), cpy_type
debug_print('faking %r'%(cpy_type,))
kw = {}
if cpy_type.__name__ == 'SRE_Pattern':
import re
import __builtin__
p = re.compile("foo")
for meth_name in p.__methods__:
kw[meth_name] = EvenMoreObscureWrapping(__builtin__.eval("lambda p,*args,**kwds: p.%s(*args,**kwds)" % meth_name))
elif cpy_type.__name__ == 'SRE_Match':
import re
import __builtin__
m = re.compile("foo").match('foo')
for meth_name in m.__methods__:
kw[meth_name] = EvenMoreObscureWrapping(__builtin__.eval("lambda m,*args,**kwds: m.%s(*args,**kwds)" % meth_name))
else:
for s, v in cpy_type.__dict__.items():
if not (cpy_type is unicode and s in ['__add__', '__contains__']):
if s != '__getattribute__' or cpy_type is type(sys) or cpy_type is type(Exception):
kw[s] = v
kw['__module__'] = cpy_type.__module__
def fake__new__(space, w_type, __args__):
args_w, kwds_w = __args__.unpack()
args = [space.unwrap(w_arg) for w_arg in args_w]
kwds = {}
for (key, w_value) in kwds_w.items():
kwds[key] = space.unwrap(w_value)
try:
r = apply(cpy_type.__new__, [cpy_type]+args, kwds)
except:
wrap_exception(space)
raise
w_obj = space.allocate_instance(W_Fake, w_type)
W_Fake.__init__(w_obj, space, r)
return w_obj
kw['__new__'] = gateway.interp2app(fake__new__,
unwrap_spec=[baseobjspace.ObjSpace,
baseobjspace.W_Root,
argument.Arguments])
if cpy_type.__base__ is not object:
assert cpy_type.__base__ is basestring
from pypy.objspace.std.basestringtype import basestring_typedef
base = basestring_typedef
else:
base = None
class W_Fake(W_Object):
typedef = StdTypeDef(
cpy_type.__name__, base, **kw)
def __init__(w_self, space, val):
w_self.val = val
def unwrap(w_self, space):
return w_self.val
# cannot write to W_Fake.__name__ in Python 2.2!
W_Fake = type(W_Object)('W_Fake%s'%(cpy_type.__name__.capitalize()),
(W_Object,),
dict(W_Fake.__dict__.items()))
W_Fake.typedef.fakedcpytype = cpy_type
return W_Fake
def really_build_fake_type(cpy_type):
"NOT_RPYTHON (not remotely so!)."
#assert not issubclass(cpy_type, file), cpy_type
debug_print('faking %r' % (cpy_type, ))
kw = {}
if cpy_type.__name__ == 'SRE_Pattern':
import re
import __builtin__
p = re.compile("foo")
for meth_name in p.__methods__:
kw[meth_name] = EvenMoreObscureWrapping(
__builtin__.eval("lambda p,*args,**kwds: p.%s(*args,**kwds)" %
meth_name))
elif cpy_type.__name__ == 'SRE_Match':
import re
import __builtin__
m = re.compile("foo").match('foo')
for meth_name in m.__methods__:
kw[meth_name] = EvenMoreObscureWrapping(
__builtin__.eval("lambda m,*args,**kwds: m.%s(*args,**kwds)" %
meth_name))
else:
for s, v in cpy_type.__dict__.items():
if not (cpy_type is unicode and s in ['__add__', '__contains__']):
if s != '__getattribute__' or cpy_type is type(
sys) or cpy_type is type(Exception):
kw[s] = v
kw['__module__'] = cpy_type.__module__
def fake__new__(space, w_type, __args__):
args_w, kwds_w = __args__.unpack()
args = [space.unwrap(w_arg) for w_arg in args_w]
kwds = {}
for (key, w_value) in kwds_w.items():
kwds[key] = space.unwrap(w_value)
try:
r = apply(cpy_type.__new__, [cpy_type] + args, kwds)
except:
wrap_exception(space)
raise
w_obj = space.allocate_instance(W_Fake, w_type)
W_Fake.__init__(w_obj, space, r)
return w_obj
kw['__new__'] = gateway.interp2app(fake__new__,
unwrap_spec=[
baseobjspace.ObjSpace,
baseobjspace.W_Root,
argument.Arguments
])
if cpy_type.__base__ is not object:
assert cpy_type.__base__ is basestring
from pypy.objspace.std.basestringtype import basestring_typedef
base = basestring_typedef
else:
base = None
class W_Fake(W_Object):
typedef = StdTypeDef(cpy_type.__name__, base, **kw)
def __init__(w_self, space, val):
w_self.val = val
def unwrap(w_self, space):
return w_self.val
# cannot write to W_Fake.__name__ in Python 2.2!
W_Fake = type(W_Object)('W_Fake%s' % (cpy_type.__name__.capitalize()),
(W_Object, ), dict(W_Fake.__dict__.items()))
W_Fake.typedef.fakedcpytype = cpy_type
return W_Fake
def write(self, globals, locals):
locals['response'].write(encode_attribute(str(__builtin__.eval(self.code, globals, locals))))
def write(self, globals, locals):
from new import instancemethod
locals['_RF_callback'] = instancemethod(self.callback, locals['self'], Node)
__builtin__.eval(self.code, globals, locals)
def ws_dispatch(line, request):
msg = simplejson.loads(line)
if(msg):
if(msg['cmd'] in CMD_CNF):
eval(msg['cmd'].lower())(request, msg)
def write(self, globals, locals):
result = __builtin__.eval(self.code, globals, locals)
if result!=None:
write = locals['_RF_write']
write(str(result))
