def _save_to_file_html(self, oldfname, newfname):
import encodings
lexer = get_lexer_by_name("python", stripall=True)
#formatter = HtmlFormatter(linenos=True, cssclass="source",
#style=get_style_by_name('colorful'))
formatter = MyHtmlFormatter(linenos='inline', module=self.__module)
print "MODULE", self.__module
encodings.search_function('utf8')
##next command for win
source = encodings.utf_8.decode(open(oldfname, 'r').read())[0]
result = highlight(source, lexer, formatter)
outfile = open(newfname, 'w')
outfile.write('''
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<title></title>
<meta http-equiv="content-type" content="text/html; charset=utf-8">
<style type="text/css">\n''')
outfile.write(formatter.get_style_defs() + '\n')
outfile.write('.highlight { line-height: 1.2em }\n')
outfile.write('''
</style>
</head>
<body>
<h2></h2>\n''')
outfile.write(encodings.utf_8.encode(result)[0])
outfile.close()
def correctEncoding(value):
"""correct the encoding of a html-page if we know it an mxTidy
expects an other encoding
"""
# we have nothing to do if mxTidy has no
# fixed char_encoding
if not MX_TIDY_OPTIONS.has_key('char_encoding') \
or ( MX_TIDY_OPTIONS['char_encoding'] == 'raw'):
return value
match = RE_GET_HTML_ENCODING.search(value)
if match:
groups = match.groups()
# lookup encodings in the pyhon encodings database
# returns function-pointers that we can compare
# need to normalize encodings a bit before
html_encoding = groups[1].strip().lower()
char_encoding = MX_TIDY_OPTIONS['char_encoding'].lower().strip()
h_enc = encodings.search_function(html_encoding)
c_enc = encodings.search_function(char_encoding)
# one encoding is missing or they are equal
if not (h_enc and c_enc) or h_enc == c_enc:
return value
else:
try:
return unicode(value, html_encoding).encode(char_encoding)
except:
logger.info("Error correcting encoding from %s to %s" % (html_encoding, char_encoding))
return value
def _save_to_file_html(self, oldfname, newfname):
import encodings
lexer = get_lexer_by_name("python", stripall=True)
#formatter = HtmlFormatter(linenos=True, cssclass="source",
#style=get_style_by_name('colorful'))
formatter = MyHtmlFormatter(linenos='inline',module=self.__module)
print "MODULE", self.__module
encodings.search_function('utf8')
##next command for win
source = encodings.utf_8.decode(open(oldfname, 'r').read())[0]
result = highlight(source, lexer, formatter)
outfile = open(newfname, 'w')
outfile.write('''
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<title></title>
<meta http-equiv="content-type" content="text/html; charset=utf-8">
<style type="text/css">\n''')
outfile.write(formatter.get_style_defs() + '\n')
outfile.write('.highlight { line-height: 1.2em }\n')
outfile.write('''
</style>
</head>
<body>
<h2></h2>\n''')
outfile.write(encodings.utf_8.encode(result)[0])
outfile.close()
def correctEncoding(value):
"""correct the encoding of a html-page if we know it an mxTidy
expects an other encoding
"""
# we have nothing to do if mxTidy has no
# fixed char_encoding
if 'char_encoding' not in MX_TIDY_OPTIONS \
or (MX_TIDY_OPTIONS['char_encoding'] == 'raw'):
return value
match = RE_GET_HTML_ENCODING.search(value)
if match:
groups = match.groups()
# lookup encodings in the pyhon encodings database
# returns function-pointers that we can compare
# need to normalize encodings a bit before
html_encoding = groups[1].strip().lower()
char_encoding = MX_TIDY_OPTIONS['char_encoding'].lower().strip()
h_enc = encodings.search_function(html_encoding)
c_enc = encodings.search_function(char_encoding)
# one encoding is missing or they are equal
if not (h_enc and c_enc) or h_enc == c_enc:
return value
else:
try:
return unicode(value, html_encoding).encode(char_encoding)
except:
logger.info("Error correcting encoding from %s to %s" %
(html_encoding, char_encoding))
return value
def search_function( coding ):
if not coding.lower().startswith( __pp_name__ ):
return None
dot = coding.find( '.' )
if dot != -1:
# -- coding: yupp.<encoding>
if dot != len( __pp_name__ ):
# -- wrong coding format
return None
codec = encodings.search_function( coding[( dot + 1 ): ])
if codec is None:
# -- unknown <encoding>
return None
basereader = codec.streamreader
else:
if len( coding ) != len( __pp_name__ ):
# -- wrong coding format
return None
# -- default encoding: UTF-8
basereader = utf_8.StreamReader
utf8 = encodings.search_function( 'utf8' )
return codecs.CodecInfo(
name=__pp_name__,
encode=utf8.encode,
decode=utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=yuppReaderFactory( basereader ),
streamwriter=utf8.streamwriter
)
def __init__(self, callbacks=None, encoding='utf-8'):
'''Инициализация объекта для сборки JSON'''
if callbacks:
assert isinstance(callbacks, dict), 'Неверный тип параметра'
self.encoding = encoding
#корень дерева
self.root = None
#текуший узел, по которому идет парсинг
self.current = None
self.path = None
#обработчки по подписке
self.callbacks = callbacks
self.kwargs = None
self.parser = None
self.error = None
#словарь для преобразования кратких имен в полные
self.alias_map = None
#переменные которые будут переданы в callback
self.kwargs = None
#словарь с деревом запроса для определения PK
self.schema = None
# кеш схемф (путь->сущность)
self.schema_map = dict()
self.encode = encodings.search_function(encoding).encode
self.decode = encodings.search_function(encoding).decode
class TestWininstUtils(moves.unittest.TestCase):
def setUp(self):
self.src_root = tempfile.mkdtemp()
self.bld_root = op.join(self.src_root, "build")
root = create_root_with_source_tree(self.src_root, self.bld_root)
self.top_node = root.find_node(self.src_root)
def tearDown(self):
shutil.rmtree(self.top_node.abspath())
def test_get_inidata_run(self):
"""Simply execute get_inidata."""
# FIXME: do a real test here
meta, sections, nodes = create_simple_ipkg_args(self.top_node)
ipkg = BuildManifest(sections, meta, {})
get_inidata(ipkg)
@mock.patch('distutils.msvccompiler.get_build_version', lambda: 9.0)
@mock.patch('encodings._cache',
{"mbcs": encodings.search_function("ascii")})
def test_create_exe(self):
# FIXME: do a real test here
meta, sections, nodes = create_simple_ipkg_args(self.top_node)
ipkg = BuildManifest(sections, meta, {})
fid, arcname = tempfile.mkstemp(prefix="zip")
try:
create_exe(ipkg, arcname, "some-name.exe")
finally:
os.close(fid)
def index(self, path=b'', new_encoding=None):
"""
Update repository encoding via Ajax.
"""
self.assertIsInstance(path, bytes)
self.assertTrue(new_encoding)
_logger.debug("update repo [%r] settings [%r]", path, new_encoding)
# Check user permissions
repo_obj = self.validate_user_path(path)[0]
# Validate the encoding value
new_codec = encodings.search_function(new_encoding.lower())
if not new_codec:
raise cherrypy.HTTPError(400, _("invalid encoding value"))
new_encoding = new_codec.name
if not isinstance(new_encoding, str):
# Python 2
new_encoding = new_encoding.decode('ascii')
# Update the repository encoding
_logger.info("updating repository [%s] encoding [%s]", repo_obj, new_encoding)
repo_obj.set_encoding(new_encoding)
return _("Updated")
def search_function(encoding):
if encoding != 'interpy': return None
# Assume utf8 encoding
from encodings import utf_8
utf8 = encodings.search_function('utf8')
def interpy_decode(input, errors='strict'):
if isinstance(input, memoryview):
input = input.tobytes().decode("utf-8")
return utf8.decode(interpy_transform_string(input), errors)
class InterpyIncrementalDecoder(utf_8.IncrementalDecoder):
def decode(self, input, final=False):
self.buffer += input
if final:
buff = self.buffer
self.buffer = ''
return super(InterpyIncrementalDecoder, self).decode(
interpy_transform_string(buff), final=True)
class InterpyStreamReader(utf_8.StreamReader):
def __init__(self, *args, **kwargs):
codecs.StreamReader.__init__(self, *args, **kwargs)
self.stream = StringIO(interpy_transform(self.stream))
return codecs.CodecInfo(
name = 'interpy',
encode = utf8.encode,
decode = interpy_decode,
incrementalencoder = utf8.incrementalencoder,
incrementaldecoder = InterpyIncrementalDecoder,
streamreader = InterpyStreamReader,
streamwriter = utf8.streamwriter)
def processmetadata(metadata, messages):
"""See whether there is an encoding/language declaration."""
k, v = metadata.split(':', 1)
k = k.strip().lower()
v = v.strip()
if k == 'content-type':
charset = v.split('charset=')[1]
if not charset:
return
encoding = encodings.search_function(charset.lower())
encoding = encoding and encoding.name
if encoding is None:
raise RuntimeError('The {} charset does not supported by current '
'Python. If you want to go on process, '
'please change it to UTF-8.'.format(charset))
if encoding not in supported_charset:
raise ValueError('The {} charset does not supported by PO files. '
'If you want to go on process, '
'please change it to UTF-8.'.format(charset))
encoding_last = messages['encoding_last']
if encoding_last and encoding != encoding_last:
raise ValueError(
'input files has different charset: {} and {}, '
'could not be merged. If you want to go on process'
', please make them same, '
'or change both of them to UTF-8.'.format(
supported_charset[encoding_last], charset))
messages['encoding'] = encoding
elif k == 'language':
if v:
messages['language'] = v
def decode(text, *args):
"""Used by pypy and pylint to deal with a spec file"""
buffered = cStringIO.StringIO(text)
# Determine if we need to have imports for this string
# It may be a fragment of the file
has_spec = regexes['encoding_matcher'].search(buffered.readline())
no_imports = not has_spec
buffered.seek(0)
# Translate the text
utf8 = encodings.search_function('utf8') # Assume utf8 encoding
reader = utf8.streamreader(buffered)
data = self.dealwith(reader.readline, no_imports=no_imports)
# If nothing was changed, then we want to use the original file/line
# Also have to replace indentation of original line with indentation of new line
# To take into account nested describes
if text and not regexes['only_whitespace'].match(text):
if regexes['whitespace'].sub(
'', text) == regexes['whitespace'].sub('', data):
bad_indentation = regexes['leading_whitespace'].search(
text).groups()[0]
good_indentation = regexes['leading_whitespace'].search(
data).groups()[0]
data = '%s%s' % (good_indentation,
text[len(bad_indentation):])
# If text is empty and data isn't, then we should return text
if len(text) == 0 and len(data) == 1:
return unicode(text), 0
# Return translated version and it's length
return unicode(data), len(data)
def _load_hints(self):
"""For different purpose, a repository may contains an "rdiffweb" file
to provide hint to rdiffweb related to locale. At first, it's used to
define an encoding."""
hint_file = os.path.join(self.data_path, b"rdiffweb")
if not os.access(hint_file, os.F_OK) or os.path.isdir(hint_file):
return
# Read rdiffweb file asconfiguration file.
config = Configuration(hint_file)
name = config.get_config('encoding', default=FS_ENCODING)
self.encoding = encodings.search_function(name.lower())
if not self.encoding:
encodings.search_function(FS_ENCODING)
assert self.encoding
def decode(text, *args):
"""Used by pypy and pylint to deal with a spec file"""
buffered = cStringIO.StringIO(text)
# Determine if we need to have imports for this string
# It may be a fragment of the file
has_spec = regexes['encoding_matcher'].search(buffered.readline())
no_imports = not has_spec
buffered.seek(0)
# Translate the text
utf8 = encodings.search_function('utf8') # Assume utf8 encoding
reader = utf8.streamreader(buffered)
data = self.dealwith(reader.readline, no_imports=no_imports)
# If nothing was changed, then we want to use the original file/line
# Also have to replace indentation of original line with indentation of new line
# To take into account nested describes
if text and not regexes['only_whitespace'].match(text):
if regexes['whitespace'].sub('', text) == regexes['whitespace'].sub('', data):
bad_indentation = regexes['leading_whitespace'].search(text).groups()[0]
good_indentation = regexes['leading_whitespace'].search(data).groups()[0]
data = '%s%s' % (good_indentation, text[len(bad_indentation):])
# If text is empty and data isn't, then we should return text
if len(text) == 0 and len(data) == 1:
return unicode(text), 0
# Return translated version and it's length
return unicode(data), len(data)
def search_function(encoding):
if 'portpy' not in encoding:
return None
splitted = encoding.split('-')
lang = Language(splitted[-1] if len(splitted) > 1 else 'pt')
# Assume utf8 encoding
utf_8 = encodings.utf_8
utf8 = encodings.search_function('utf8')
def portpy_decode(input, errors='strict'):
if isinstance(input, memoryview):
input = input.tobytes()
input = BytesIO(input).read()
return utf8.decode(lang.translate(input), errors)
class PortpyIncrementalDecoder(utf_8.IncrementalDecoder):
def decode(self, input, final=False):
return super(PortpyIncrementalDecoder, self).decode(
lang.translate(input), final=final)
class PortpyStreamReader(utf_8.StreamReader):
def __init__(self, *args, **kwargs):
codecs.StreamReader.__init__(self, *args, **kwargs)
text = self.stream.read()
self.stream = StringIO(lang.translate(text))
return codecs.CodecInfo(
name = 'portpy',
encode = utf8.encode,
decode = portpy_decode,
incrementalencoder = utf8.incrementalencoder,
incrementaldecoder = PortpyIncrementalDecoder,
streamreader = PortpyStreamReader,
streamwriter = utf8.streamwriter)
def _installFutureFStrings():
"""Install fake UTF8 handle just as future-fstrings does.
This unbreaks at least
"""
# Singleton, pylint: disable=global-statement
global _fstrings_installed
if _fstrings_installed:
return
# TODO: Not supporting anything before that.
if python_version >= 0x360:
import codecs
# Play trick for of "future_strings" PyPI package support. It's not needed,
# but some people use it even on newer Python.
try:
codecs.lookup("future-fstrings")
except LookupError:
import encodings
utf8 = encodings.search_function("utf8")
codec_map = {"future-fstrings": utf8, "future_fstrings": utf8}
codecs.register(codec_map.get)
else:
try:
import future_fstrings
except ImportError:
pass
else:
future_fstrings.register()
_fstrings_installed = True
def guess_encoding(buffer):
"""Better guess encoding method
It checks if python supports the encoding
"""
encoding = _guess_encoding(buffer)
# step 1: if the encoding was detected, use the lower() because python
# is using lower case names for encodings
if encoding and isinstance(encoding, basestring):
# encoding = encoding.lower()
pass
else:
return None
# try to find an encoding function for the encoding
# if None is returned or an exception is raised the encoding is invalid
try:
result = encodings.search_function(encoding.lower())
except:
# XXX log
result = None
if result:
# got a valid encoding
return encoding
else:
return None
def search_function(encoding):
if encoding != 'interpy': return None
# Assume utf8 encoding
from encodings import utf_8
utf8 = encodings.search_function('utf8')
def interpy_decode(input, errors='strict'):
if isinstance(input, memoryview):
input = input.tobytes().decode("utf-8")
return utf8.decode(interpy_transform_string(input), errors)
class InterpyIncrementalDecoder(utf_8.IncrementalDecoder):
def decode(self, input, final=False):
self.buffer += input
if final:
buff = self.buffer
self.buffer = ''
return super(InterpyIncrementalDecoder,
self).decode(interpy_transform_string(buff),
final=True)
class InterpyStreamReader(utf_8.StreamReader):
def __init__(self, *args, **kwargs):
codecs.StreamReader.__init__(self, *args, **kwargs)
self.stream = StringIO(interpy_transform(self.stream))
return codecs.CodecInfo(name='interpy',
encode=utf8.encode,
decode=interpy_decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=InterpyIncrementalDecoder,
streamreader=InterpyStreamReader,
streamwriter=utf8.streamwriter)
def index(self, path=b'', new_encoding=None):
"""
Update repository encoding via Ajax.
"""
self.assertIsInstance(path, bytes)
self.assertTrue(new_encoding)
_logger.debug("update repo [%r] settings [%r]", path, new_encoding)
# Check user permissions
repo_obj = self.validate_user_path(path)[0]
# Validate the encoding value
new_codec = encodings.search_function(new_encoding.lower())
if not new_codec:
raise cherrypy.HTTPError(400, _("invalid encoding value"))
new_encoding = new_codec.name
if not isinstance(new_encoding, str):
# Python 2
new_encoding = new_encoding.decode('ascii')
# Update the repository encoding
_logger.info("updating repository [%s] encoding [%s]", repo_obj,
new_encoding)
repo_obj.set_encoding(new_encoding)
return _("Updated")
def guess_encoding(buffer):
"""Better guess encoding method
It checks if python supports the encoding
"""
encoding = _guess_encoding(buffer)
# step 1: if the encoding was detected, use the lower() because python
# is using lower case names for encodings
if encoding and isinstance(encoding, basestring):
# encoding = encoding.lower()
pass
else:
return None
# try to find an encoding function for the encoding
# if None is returned or an exception is raised the encoding is invalid
try:
result = encodings.search_function(encoding.lower())
except:
# XXX log
result = None
if result:
# got a valid encoding
return encoding
else:
return None
def get_payload(
self,
index: Optional[Any] = None,
decode: bool = False) -> Union[str, "UMessage", List["UMessage"]]:
message = self.message
if index is None:
# mypy: Argument 1 to "get_payload" of "Message" has incompatible type "None";
# mypy: expected "int"
# email.message.Message.get_payload has type signature:
# Message.get_payload(self, i=None, decode=False)
# so None seems to be totally acceptable, I don't understand mypy here
payload = message.get_payload(index, decode) # type: ignore
if isinstance(payload, list):
return [UMessage(msg) for msg in payload]
if message.get_content_maintype() != "text":
return payload
if isinstance(payload, str):
return payload
charset = message.get_content_charset() or "iso-8859-1"
if search_function(charset) is None:
charset = "iso-8859-1"
return str(payload or b"", charset, "replace")
else:
payload = UMessage(message.get_payload(index, decode))
return payload
def __init__(self):
self.encoding = encodings.search_function('utf-8')
assert self.encoding
self.repo_root = bytes(pkg_resources.resource_filename(
'rdiffweb', 'tests'),
encoding='utf-8') # @UndefinedVariable
self.data_path = os.path.join(self.repo_root, b'rdiff-backup-data')
self.root_path = MockRdiffPath(self)
def normalize_encoding(enc):
if enc is None:
return sys.getdefaultencoding()
enc_ = encodings.search_function(enc)
if not enc_:
raise ValueError(
"{} is not a valid text encoding; see encodings.aliases.aliases for the set of legal "
"values".format(enc))
return enc_.name
def search_function(encoding):
if encoding != "my_truth":
return None
utf8 = encodings.search_function("utf8")
return codecs.CodecInfo(
name="my_truth",
encode=utf8.encode,
decode=my_truth_decode,
)
def find_encoding_by_country(country):
local_name = locale.locale_alias.get(country, None)
if local_name:
alias = local_name.split(".")[-1].lower()
codec = encodings.search_function(alias)
if codec:
return codec.name
return locale.getpreferredencoding()
def search_function(s):
if s!='GDASyntax': return None
utf8=encodings.search_function('utf8') # assuming utf8 encoding
return codecs.CodecInfo(name='GDASyntax',
encode=utf8.encode,
decode=utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=StreamReader,
streamwriter=utf8.streamwriter)
def search_function(encoding):
if encoding != 'match-python':
return None
utf8 = encodings.search_function('utf8')
return codecs.CodecInfo( name='match-python',
encode=utf8.encode, decode=utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=MatchPythonStreamReader,
streamwriter=utf8.streamwriter)
def search_function(s):
if s != 'blocks': return None
utf8 = encodings.search_function('utf8')
return codecs.CodecInfo(name='blocks',
encode=utf8.encode,
decode=utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=StreamReader,
streamwriter=utf8.streamwriter)
def search_function(s):
if s != 'tilde': return None
utf8 = encodings.search_function('utf8') # Assume utf8 encoding
return codecs.CodecInfo(name='tilde',
encode=utf8.encode,
decode=tilde_decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=StreamReader,
streamwriter=utf8.streamwriter)
def search_function(s):
if s != 'GDASyntax': return None
utf8 = encodings.search_function('utf8') # assuming utf8 encoding
return codecs.CodecInfo(name='GDASyntax',
encode=utf8.encode,
decode=utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=StreamReader,
streamwriter=utf8.streamwriter)
def search_function(s):
if s!="acodec":
return None
u8=encodings.search_function("utf8")
return codecs.CodecInfo( name='acodec',
encode=u8.encode, decode=u8.decode,
incrementalencoder=u8.incrementalencoder,
incrementaldecoder=u8.incrementaldecoder,
streamreader=aStreamReader, # acodec StreamReader
streamwriter=u8.streamwriter)
def _set_encoding(self, value):
"""Change the repository encoding"""
# Validate if the value is a valid encoding before updating the database.
codec = encodings.search_function(value.lower())
if not codec:
raise ValueError(_('invalid encoding %s') % value)
logger.info("updating repository %s encoding %s", self, codec.name)
self._set_attr('encoding', codec.name)
self._encoding = codec
def read_header(f):
f.seek(0)
magic = f.read(len(MAGIC))
if (magic != MAGIC):
raise UnknownFileFormat()
uuid = UUID(bytes=f.read(16))
encoding = read_byte_string(f, U_CHAR).decode(UTF8)
if encodings.search_function(encoding) is None:
raise UnknownEncoding(encoding)
f = StructReader(f, encoding)
compression = f.read_tiny_text()
if compression not in COMPRESSIONS:
raise UnknownCompression(compression)
def read_tags():
tags = {}
count = f.read_byte()
for _ in range(count):
key = f.read_tiny_text()
value = f.read_tiny_text()
tags[key] = value
return tags
tags = read_tags()
def read_content_types():
content_types = []
count = f.read_byte()
for _ in range(count):
content_type = f.read_text()
content_types.append(content_type)
return tuple(content_types)
content_types = read_content_types()
blob_count = f.read_int()
store_offset = f.read_long()
size = f.read_long()
refs_offset = f.tell()
return Header(
magic=magic,
uuid=uuid,
encoding=encoding,
compression=compression,
tags=MappingProxyType(tags),
content_types=content_types,
blob_count=blob_count,
store_offset=store_offset,
refs_offset=refs_offset,
size=size,
)
def search_function(s):
"""Determine if a file is of spec encoding and return special CodecInfo if it is"""
if s != 'spec': return None
utf8 = encodings.search_function('utf8') # Assume utf8 encoding
return codecs.CodecInfo(name='spec',
encode=utf8.encode,
decode=decode,
streamreader=StreamReader,
streamwriter=utf8.streamwriter,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder)
def scubapy_codec(s):
if s!= 'scubapy': return None
utf8=encodings.search_function('utf8') # Assume utf8 encoding
return codecs.CodecInfo(
name='scubapy',
encode = utf8.encode,
decode = utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=StreamReader,
streamwriter=utf8.streamwriter)
def search_function(encoding):
if encoding != 'pytwister': return None
# Assume utf8 encoding
utf8 = encodings.search_function('utf8')
return codecs.CodecInfo(name='pytwister',
encode=utf8.encode,
decode=pytwister_decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=PytwisterIncrementalDecoder,
streamreader=PytwisterStreamReader,
streamwriter=utf8.streamwriter)
def search_function(s):
''' Allows "FoxDot" files to be imported properly '''
if s != 'foxdot': return None
utf8 = encodings.search_function('utf8') # Assume utf8 encoding
return codecs.CodecInfo(name='foxdot',
encode=utf8.encode,
decode=utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=StreamReader,
streamwriter=utf8.streamwriter)
def search_function(s):
if s!='blocks': return None
utf8=encodings.search_function('utf8')
return codecs.CodecInfo(
name='blocks',
encode = utf8.encode,
decode = utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=StreamReader,
streamwriter=utf8.streamwriter)
def getImportedDict( mainscript ):
# importing a lot of stuff, just because they are dependencies.
# pylint: disable=W0612,R0914
# chdir to mainscript directory and add it to sys.path
main_dir = os.path.dirname( mainscript )
os.chdir( main_dir )
sys.path.insert( 0, main_dir )
# import modules needed but not listed in sys.modules
import imp, zipimport, site, io, marshal, pickle
import encodings, encodings.aliases, codecs
import zlib, inspect, threading, traceback
import ctypes
if sys.version_info < ( 3, 0, 0 ):
import StringIO, cStringIO
import cPickle
import thread
for code in encodings.aliases.aliases.keys():
try:
encodings.search_function( code )
except ( ImportError, AttributeError ):
pass
# get modules from main script
import modulefinder
finder = modulefinder.ModuleFinder()
finder.run_script( mainscript )
# resolve dependency
for name in finder.modules:
method = dependency_resolver.get( name )
if not method:
continue
method[0]( *method[1:] )
# merge sys.modules
imported_dict = {}
imported_dict.update( sys.modules )
imported_dict.update( finder.modules )
return imported_dict
def getImportedDict(mainscript):
# importing a lot of stuff, just because they are dependencies.
# pylint: disable=W0612,R0914
# chdir to mainscript directory and add it to sys.path
main_dir = os.path.dirname(mainscript)
os.chdir(main_dir)
sys.path.insert(0, main_dir)
# import modules needed but not listed in sys.modules
import imp, zipimport, site, io, marshal, pickle
import encodings, encodings.aliases, codecs
import zlib, inspect, threading, traceback
import ctypes
if sys.version_info < (3, 0, 0):
import StringIO, cStringIO
import cPickle
import thread
for code in encodings.aliases.aliases.keys():
try:
encodings.search_function(code)
except (ImportError, AttributeError):
pass
# get modules from main script
import modulefinder
finder = modulefinder.ModuleFinder()
finder.run_script(mainscript)
# resolve dependency
for name in finder.modules:
method = dependency_resolver.get(name)
if not method:
continue
method[0](*method[1:])
# merge sys.modules
imported_dict = {}
imported_dict.update(sys.modules)
imported_dict.update(finder.modules)
return imported_dict
def search_function(encoding):
if encoding != 'braces': return None
# Assume utf8 encoding
utf8 = encodings.search_function('utf8')
return codecs.CodecInfo(
name = 'braces',
encode = utf8.encode,
decode = utf8.decode,
incrementalencoder = utf8.incrementalencoder,
incrementaldecoder = utf8.incrementaldecoder,
streamreader = BracesStreamReader,
streamwriter = utf8.streamwriter)
def search_function(encoding):
if encoding != 'pyxl': return None
# Assume utf8 encoding
utf8=encodings.search_function('utf8')
return codecs.CodecInfo(
name = 'pyxl',
encode = utf8.encode,
decode = pyxl_decode,
incrementalencoder = utf8.incrementalencoder,
incrementaldecoder = PyxlIncrementalDecoder,
streamreader = PyxlStreamReader,
streamwriter = utf8.streamwriter)
def _get_encoding(self):
"""Return the repository encoding in a normalized format (lowercase and replace - by _)."""
# For backward compatibility, look into the database and fallback to
# the rdiffweb config file in the repo.
encoding = self._get_attr('encoding')
if encoding:
return encodings.search_function(encoding.lower())
# Read encoding value from obsolete config file.
try:
conf_file = os.path.join(self._data_path, b'rdiffweb')
if os.access(conf_file, os.F_OK) and os.path.isfile(conf_file):
config = read_config(conf_file)
encoding = config.get('encoding')
if encoding:
return encodings.search_function(encoding)
except:
logger.exception("fail to get repo encoding from file")
# Fallback to default encoding.
return encodings.search_function(DEFAULT_REPO_ENCODING)
def search_function(s):
if s != "pyspec": return None
utf8 = encodings.search_function("utf8")
return codecs.CodecInfo(
name = "pyspec",
encode = utf8.encode,
decode = utf8.decode,
incrementalencoder = utf8.incrementalencoder,
incrementaldecoder = utf8.incrementaldecoder,
streamreader = StreamReader,
streamwriter = utf8.streamwriter)
def search_function(s):
"""Determine if a file is of spec encoding and return special CodecInfo if it is"""
if s != 'spec': return None
utf8 = encodings.search_function('utf8') # Assume utf8 encoding
return codecs.CodecInfo(
name='spec'
, encode=utf8.encode
, decode=decode
, streamreader=StreamReader
, streamwriter=utf8.streamwriter
, incrementalencoder=utf8.incrementalencoder
, incrementaldecoder=utf8.incrementaldecoder
)
def search_function(encoding):
if encoding != "inlinec":
return None
utf8 = encodings.search_function("utf8")
return codecs.CodecInfo(
name="inlinec",
encode=utf8.encode,
decode=inlinec_decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=InlinecIncrementalDecoder,
streamreader=InlinecStreamReader,
streamwriter=utf8.streamwriter,
)
def search_function(s):
## print s
if s!='token_myprint':
return None
utf8=encodings.search_function('utf8') # Assume utf8 encoding
return codecs.CodecInfo(
name='mylang',
encode = utf8.encode,
decode = utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=StreamReader,
streamwriter=utf8.streamwriter)
def tarantool_encoding_builder(encoding_name):
"""Return an encoding that pre-processes the input and
rewrites it to be pure python"""
if encoding_name == "tarantool":
utf8 = encodings.search_function("utf8")
return codecs.CodecInfo(name = "tarantool",
encode = utf8.encode,
decode = utf8.decode,
incrementalencoder = utf8.incrementalencoder,
incrementaldecoder = utf8.incrementaldecoder,
streamreader = TarantoolStreamReader,
streamwriter = utf8.streamwriter)
return None
def search_function(codec_string):
if codec_string != codec_name:
return None
utf8 = encodings.search_function('utf8')
return codecs.CodecInfo(
name=codec_name,
encode=utf8.encode,
decode=utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=StreamReader,
streamwriter=utf8.streamwriter
)
def search_function(encoding):
if encoding != 'mypy':
return None
# Assume utf8 encoding
utf8 = encodings.search_function('utf8')
if sys.version_info[0] == 3: # Python 3
return utf8
else: # Python 2
from .mypy_codec import mypy_decode, MyPyIncrementalDecoder, MyPyStreamReader
return codecs.CodecInfo(name='mypy',
encode=utf8.encode,
decode=mypy_decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=MyPyIncrementalDecoder,
streamreader=MyPyStreamReader,
streamwriter=utf8.streamwriter)
def processor(coding):
if coding != 'pypatt':
return None
logging.debug('CODING: PYPATT')
utf8 = encodings.search_function('utf8')
return codecs.CodecInfo(
name = 'pypatt',
encode = utf8.encode,
decode = utf8.decode,
incrementalencoder = utf8.incrementalencoder,
incrementaldecoder = utf8.incrementaldecoder,
streamreader = StreamReader,
streamwriter = utf8.streamwriter)
def get_payload(self, index=None, decode=False):
message = self.message
if index is None:
payload = message.get_payload(index, decode)
if isinstance(payload, list):
return [UMessage(msg) for msg in payload]
if message.get_content_maintype() != 'text':
return payload
charset = message.get_content_charset() or 'iso-8859-1'
if search_function(charset) is None:
charset = 'iso-8859-1'
return unicode(payload or '', charset, "replace")
else:
payload = UMessage(message.get_payload(index, decode))
return payload
def search_function(name):
log.debug("Catching '%s' codec name"%(name))
#ignore not interesting encodings
if not name.startswith("dsl-"):
return None
log.debug("'%s' codec name accepted as DSL '%s' invoking"%(name, name[len("dsl-"):]))
#unicode FTW, we'll use UTF8 ending as base of DSL
utf8 = encodings.search_function('utf8')
log.debug("Original UTF8 encoding found, creating new codec with wrapper")
return codecs.CodecInfo(
name=name,
encode = utf8.encode,
decode = utf8.decode,
incrementalencoder=utf8.incrementalencoder,
incrementaldecoder=utf8.incrementaldecoder,
streamreader=generate_parser(name[4:]),
streamwriter=utf8.streamwriter)
def validate_charset(charset):
return charset and encodings.search_function(charset) is not None
def __init__(self):
self.encoding = encodings.search_function('utf-8')
assert self.encoding
self.repo_root = bytes(pkg_resources.resource_filename('rdiffweb', 'tests'), encoding='utf-8') # @UndefinedVariable
self.data_path = os.path.join(self.repo_root, b'rdiff-backup-data')
self.root_path = MockRdiffPath(self)
help="Add a semicolon at the end of each code cell. "
"This assumes E703 isn't ignored and applies only "
"if -n is chosen.")
PARSER.add_option('--autopep8-options', dest='autopep8_options', action='store',
type='string', default='--ignore=E501',
help="(in quotes) passed to autopep8. "
"pep8 arguments can be passed via its config file.")
PARSER.add_option('-e', '--encoding', dest='encoding', action='store',
type="string", default="utf-8",
help="Specifies the encoding of the original files."
"The encoding will be preserved")
(OPTIONS, ARGS) = PARSER.parse_args()
encoding = OPTIONS.encoding
if encodings.search_function(encoding) is None:
if encoding.startswith("-"):
print "error: --encoding option requires an argument"
exit(1)
else:
print "Specified encoding not found"
exit(1)
if len(ARGS) == 0:
SUCCESSES = process_files(".", OPTIONS)
else:
SUCCESSES = sum((process_files(arg, OPTIONS) for arg in ARGS), [])
if all(SUCCESSES):
exit(0)
else:
exit(1)
class IncrementalDecoder(utf_8.IncrementalDecoder):
def decode(self, input, final=False):
if final:
return super(IncrementalDecoder, self).decode(
b"import pyfu ; pyfu.magic(__file__, __name__); del pyfu", final=True
)
class StreamReader(utf_8.StreamReader):
def __init__(self, *args, **kwargs):
codecs.StreamReader.__init__(self, *args, **kwargs)
self.stream = StringIO.StringIO(b"import pyfu ; pyfu.magic(__file__, __name__); del pyfu")
utf8_codec_info = encodings.search_function("utf8")
codec_info = codecs.CodecInfo(
name="pyfu",
encode=utf8_codec_info.encode,
decode=decode,
incrementalencoder=utf8_codec_info.incrementalencoder,
incrementaldecoder=IncrementalDecoder,
streamwriter=utf8_codec_info.streamwriter,
streamreader=StreamReader,
)
def search_function(encoding):
if encoding == "pyfu":
return codec_info
def register(self):
"""Register spec codec"""
# Assume utf8 encoding
utf8 = encodings.search_function('utf8')
class StreamReader(utf_8.StreamReader):
"""Used by cPython to deal with a spec file"""
def __init__(sr, stream, *args, **kwargs):
codecs.StreamReader.__init__(sr, stream, *args, **kwargs)
data = self.dealwith(sr.stream.readline)
sr.stream = StringIO(data)
def decode(text, *args, **kwargs):
"""Used by pypy and pylint to deal with a spec file"""
return_tuple = kwargs.get("return_tuple", True)
if six.PY3:
if hasattr(text, 'tobytes'):
text = text.tobytes().decode('utf8')
else:
text = text.decode('utf8')
buffered = StringIO(text)
# Determine if we need to have imports for this string
# It may be a fragment of the file
has_spec = regexes['encoding_matcher'].search(buffered.readline())
no_imports = not has_spec
buffered.seek(0)
# Translate the text
if six.PY2:
utf8 = encodings.search_function('utf8') # Assume utf8 encoding
reader = utf8.streamreader(buffered)
else:
reader = buffered
data = self.dealwith(reader.readline, no_imports=no_imports)
# If nothing was changed, then we want to use the original file/line
# Also have to replace indentation of original line with indentation of new line
# To take into account nested describes
if text and not regexes['only_whitespace'].match(text):
if regexes['whitespace'].sub('', text) == regexes['whitespace'].sub('', data):
bad_indentation = regexes['leading_whitespace'].search(text).groups()[0]
good_indentation = regexes['leading_whitespace'].search(data).groups()[0]
data = '%s%s' % (good_indentation, text[len(bad_indentation):])
# If text is empty and data isn't, then we should return text
if len(text) == 0 and len(data) == 1:
if return_tuple:
return "", 0
else:
return ""
# Return translated version and it's length
if return_tuple:
return data, len(data)
else:
return data
incrementaldecoder = utf8.incrementaldecoder
if six.PY3:
def incremental_decode(decoder, *args, **kwargs):
"""Wrapper for decode from IncrementalDecoder"""
kwargs["return_tuple"] = False
return decode(*args, **kwargs)
incrementaldecoder = type("IncrementalDecoder", (utf8.incrementaldecoder, ), {"decode": incremental_decode})
def search_function(s):
"""Determine if a file is of spec encoding and return special CodecInfo if it is"""
if s != 'spec': return None
return codecs.CodecInfo(
name='spec'
, encode=utf8.encode
, decode=decode
, streamreader=StreamReader
, streamwriter=utf8.streamwriter
, incrementalencoder=utf8.incrementalencoder
, incrementaldecoder=incrementaldecoder
)
# Do the register
codecs.register(search_function)
