def expression2js(expr):
"""Convert a short expression (e.g. a Component Parameter) Python to JS"""
# if the code contains a tuple (anywhere), convert parenths to be list.
# This now works for compounds like `(2*(4, 5))` where the inner
# parenths becomes a list and the outer parens indicate priority.
# This works by running an ast transformer class to swap the contents of the tuple
# into a list for the number of tuples in the expression.
try:
syntaxTree = ast.parse(expr)
except Exception as err:
logging.error(err)
syntaxTree = ast.parse(unicode(expr))
for node in ast.walk(syntaxTree):
TupleTransformer().visit(node) # Transform tuples to list
# for py2 using 'unicode_literals' we don't want
if isinstance(node, ast.Str) and type(node.s)==bytes:
node.s = unicode(node.s, 'utf-8')
elif isinstance(node, ast.Str) and node.s.startswith("u'"):
node.s = node.s[1:]
if isinstance(node, ast.Name):
if node.id == 'undefined':
continue
node.id = namesJS[node.id]
jsStr = unparse(syntaxTree).strip()
return jsStr
def filter_by(self, condition='and', **kwargs):
if kwargs.get('condition'):
condition = kwargs.get('condition')
del kwargs['condition']
condition = self._translate_bool_condition(condition)
for k, v in iteritems(kwargs):
if isinstance(v, list):
v = [self._check_datetime(vi) for vi in v]
else:
v = self._check_datetime(v)
id_filter = []
if k == 'id' or k == 'ids':
id_filter = v
if not isinstance(id_filter, list):
id_filter = [id_filter]
id_filter = [_id for _id in id_filter if _id != None]
if len(id_filter) < 1:
raise ValueError('%s keyword must not be empty' % k)
if len(id_filter) > 0:
self._querybody.chain(qdsl.ids(id_filter), condition=condition)
else:
try:
analyzed = is_analyzed(getattr(self.base_obj, k))
except AttributeError:
analyzed = is_analyzed(v)
q_type = 'filter'
if analyzed:
q_type = 'query'
if isinstance(v, list):
# lists are treat as like "OR" (using terms() on not_analyzed, bool/matched on analyzed)
if analyzed:
match_queries = []
for item in v:
match_queries.append(qdsl.match(k, item))
self._querybody.chain(qdsl.bool_(qdsl.should(match_queries)), condition=condition, type=q_type)
else:
self._querybody.chain(qdsl.terms(k, v), condition=condition,
type=q_type)
else:
# search_value = unicode(v)
if analyzed:
self._querybody.chain(qdsl.match(unicode(k), v), condition=condition, type=q_type)
else:
self._querybody.chain(qdsl.term(unicode(k), v), condition=condition, type=q_type)
return self
def _handle_attr(self, layer, feature, props):
for k, v in props.items():
if self._can_handle_attr(k, v):
if not PY3 and isinstance(k, str):
k = k.decode('utf-8')
if k not in self.seen_keys_idx:
layer.keys.append(k)
self.seen_keys_idx[k] = self.key_idx
self.key_idx += 1
feature.tags.append(self.seen_keys_idx[k])
if v not in self.seen_values_idx:
self.seen_values_idx[v] = self.val_idx
self.val_idx += 1
val = layer.values.add()
if isinstance(v, bool):
val.bool_value = v
elif isinstance(v, str):
if PY3:
val.string_value = v
else:
val.string_value = unicode(v, 'utf-8')
elif isinstance(v, unicode):
val.string_value = v
elif isinstance(v, (int, long)):
val.int_value = v
elif isinstance(v, float):
val.double_value = v
feature.tags.append(self.seen_values_idx[v])
def setter(self, entry):
if isinstance(entry, Entry) or isinstance(entry, dict):
setattr(
self, name, unicode(json.dumps(only_builtins(dict(entry)), encode_datetime=True))
)
else:
raise TypeError('%r is not of type Entry or dict.' % type(entry))
def char(self):
"""The unicode value of the keyboard event, if available. This field is
only populated when the keyboard event results in a character that
could be printable.
:return: unicode, '' if no char value is available for the event.
"""
return unicode(self._char, 'utf-8')
def __getitem__(self, key):
if isinstance(key, int):
return self._get_ith_value(self._validate_index(key))
try:
return self._get_ith_value(self.variable_names.index(key))
except ValueError:
raise KeyError(("Illegal key '%s' passed to binding set." +
"\n Legal keys are %s") % (key, unicode(self.variable_names)))
def _toDict(self, strings_dict=False):
"""
Return a Python dictionary representation of this binding set.
"""
d = {}
for i in range(len(self.variable_names)):
v = self[i]
if strings_dict: v = unicode(v)
d[self.variable_names[i]] = v
return d
def encode_ntriple_string(string):
"""
Return a unicode string escaped according to N-Triples
canonical encoding rules.
"""
if not isinstance(string, unicode):
string = unicode(string, 'utf-8')
for char, replacement in ESCAPES:
string = string.replace(char, replacement)
return string
def make_entity(self, content):
entity = entity_pb2.Entity()
if self._namespace is not None:
entity.key.partition_id.namespace_id = self._namespace
# All entities created will have the same ancestor
datastore_helper.add_key_path(entity.key, self._kind, self._ancestor,
self._kind, str(uuid.uuid4()))
datastore_helper.add_properties(entity, {"content": unicode(content)})
return entity
def mangle_docstrings(app,
what,
name,
obj,
options,
lines,
reference_offset=[0]):
cfg = dict()
if what == 'module':
# Strip top title
title_re = re.compile(r'^\s*[#*=]{4,}\n[a-z0-9 -]+\n[#*=]{4,}\s*',
re.I | re.S)
lines[:] = title_re.sub(u'', u"\n".join(lines)).split(u"\n")
exported_members = getattr(obj, '__all__', None)
if exported_members:
lines.append('*Exported members:* ')
# do not print more than 25 members
lines.append(', '.join(
['`%s`' % member for member in exported_members[:25]]))
if len(exported_members) > 25:
lines.append('... (%d more members)' %
(len(exported_members) - 25))
lines.append('')
else:
doc = get_doc_object(obj,
what,
u"\n".join(lines),
name=name,
config=cfg)
lines[:] = unicode(doc).split(u"\n")
# replace reference numbers so that there are no duplicates
references = []
for line in lines:
line = line.strip()
m = re.match(r'^.. \[([a-z0-9_.-])\]', line, re.I)
if m:
references.append(m.group(1))
# start renaming from the longest string, to avoid overwriting parts
references.sort(key=lambda x: -len(x))
if references:
for i, line in enumerate(lines):
for r in references:
if re.match(r'^\d+$', r):
new_r = u"R%d" % (reference_offset[0] + int(r))
else:
new_r = u"%s%d" % (r, reference_offset[0])
lines[i] = lines[i].replace(u'[%s]_' % r, u'[%s]_' % new_r)
lines[i] = lines[i].replace(u'.. [%s]' % r, u'.. [%s]' % new_r)
reference_offset[0] += len(references)
def __unicode__(self):
if 'multi_value' in self.attrs:
res_fields = []
for entry in self.attrs['multi_value']:
res_fields.append(u'<field{0}>{1}</field>'.format(
u' {0}'.format(self.get_attr()), unicode(entry)))
return u'\n'.join(res_fields)
else:
return u'<field{0}>{1}</field>'.format(
u' {0}'.format(self.get_attr()), self.attrs['value'])
def test_input_and_input_required_optional_error(self):
class SimpleIO:
input = 'qwerty'
input_required = '123', '456'
input_optional = 'aaa', 'bbb'
# Cannot have both input and input_required on input
with self.assertRaises(ValueError) as ctx:
self.get_sio(SimpleIO, test_class_name)
elem1 = repr(unicode('123'))
elem2 = repr(unicode('456'))
elem3 = repr(unicode('aaa'))
elem4 = repr(unicode('bbb'))
expected = "Cannot provide input_required/input_optional if input is given, input:`qwerty`, " \
"input_required:`({}, {})`, input_optional:`({}, {})`".format(elem1, elem2, elem3, elem4)
self.assertEquals(ctx.exception.args[0], expected)
def levenshtein_distance(statement , other_statement): """ Compare two statements based on the Levenshtein distance of each statement's text. For example, there is a 65% similarity between the statements "where is the post office?" and "looking for the post office" based on the Levenshtein distance algorithm. :return: The percent of similarity between the text of the statements. :rtype: float """ import sys # Use python-Levenshtein if available try: from Levenshtein.StringMatcher import StringMatcher as SequenceMatcher except ImportError: from difflib import SequenceMatcher PYTHON = sys.version_info[ 0 ] # Return 0 if either statement has a falsy text value if not statement.text or not other_statement.text: return 0 # Get the lowercase version of both strings if PYTHON < 3: statement_text = unicode(statement.text.lower()) other_statement_text = unicode(other_statement.text.lower()) else: statement_text = str(statement.text.lower()) other_statement_text = str(other_statement.text.lower()) similarity = SequenceMatcher(None , statement_text , other_statement_text) # Calculate a decimal percent of the similarity percent = round(similarity.ratio() , 2) return percent
def to_str(value, encoding='utf-8'):
"""Convert value to a str type
Returns a bytes object in Python 2, and a unicode string in Python 3. Encodes or decodes the
input value as necessary.
"""
if isinstance(value, str):
return value
if PY2:
return unicode(value).encode(encoding=encoding)
else:
return bytes(value).decode(encoding=encoding)
def hour_minute_int(time_str, splitter=u':'):
# return datetime.datetime.strptime(str, "%H:%M:%S")
if not time_str:
return 0
time_str = unicode(time_str)
if splitter not in time_str:
return 0
#
if len(time_str) >= 8:
hour, minute = time_str[-8:][:5].split(splitter)
else:
hour, minute = time_str[:5].split(splitter)
return int('{:02d}{:02d}'.format(int(hour), int(minute)))
def test_builder_url_should_be_same(self):
expected_url = 'http://test.com/path/to/resource'
url = UrlBuilder.from_url(expected_url)
self.assertEqual(expected_url, unicode(url))
expected_url = 'http://*****:*****@test.com:8080/some/path?test=hi&test=hi2&other=bye#my-frag/path/to/frag'
url = UrlBuilder.from_url(expected_url)
actual_url = urlparse(unicode(url))
actual_query = parse_qs(actual_url.query)
self.assertEqual(actual_url.scheme, 'http')
self.assertEqual(actual_url.username, 'my-user')
self.assertEqual(actual_url.password, 'my-pass')
self.assertEqual(actual_url.hostname, 'test.com')
self.assertEqual(actual_url.path, '/some/path')
self.assertEqual(actual_url.fragment, 'my-frag/path/to/frag')
self.assertIn('hi', actual_query['test'])
self.assertIn('hi2', actual_query['test'])
self.assertIn('bye', actual_query['other'])
def test_str():
"""
Test that __str__ translates query to string.
"""
expected = canonical(u"INSERT { <http://a> <http://b> <http://c> }")
statement = URIRef("http://a"), URIRef("http://b"), URIRef("http://c")
query = insert().template(statement)
# test str()
assert expected == canonical(str(unicode(query)))
# test unicode()
assert expected == canonical(str(query))
def _convert_object_to_json(obj):
# Since there will be parts of this that may have unicode, either as
# measurement or in the logs, we have to be careful and convert everything
# to unicode, merge, then encode to UTF-8 to put it into the proto.
json_encoder = TestRecordEncoder(sort_keys=True, indent=2, ensure_ascii=False)
pieces = []
for piece in json_encoder.iterencode(obj):
if isinstance(piece, bytes):
pieces.append(unicode(piece, errors='replace'))
else:
pieces.append(piece)
return (u''.join(pieces)).encode('utf8', errors='replace')
def _append_uri(self, sb, uri):
try:
uriString = uri.uri
except AttributeError:
uriString = unicode(uri)
if len(uriString) > 50:
sb.append("<" + uriString[:19] + "..")
sb.append(uriString[len(uriString) - 29:] + ">/n")
else:
sb.append("<")
sb.append(uriString)
sb.append(">")
def _append_uri(self, sb, uri):
try:
uriString = uri.uri
except AttributeError:
uriString = unicode(uri)
if len(uriString) > 50:
sb.append("<" + uriString[:19] + "..")
sb.append(uriString[len(uriString) - 29:] + ">/n")
else:
sb.append("<")
sb.append(uriString)
sb.append(">")
def expression2js(expr):
"""Convert a short expression (e.g. a Component Parameter) Python to JS"""
# if the code contains a tuple (anywhere), convert parenths to be list.
# This now works for compounds like `(2*(4, 5))` where the inner
# parenths becomes a list and the outer parens indicate priority.
# This works by running an ast transformer class to swap the contents of the tuple
# into a list for the number of tuples in the expression.
try:
syntaxTree = ast.parse(expr)
except Exception:
try:
syntaxTree = ast.parse(unicode(expr))
except Exception as err:
logging.error(err)
return
for node in ast.walk(syntaxTree):
TupleTransformer().visit(node) # Transform tuples to list
# for py2 using 'unicode_literals' we don't want
if isinstance(node, ast.Str) and type(node.s)==bytes:
node.s = unicode(node.s, 'utf-8')
elif isinstance(node, ast.Str) and node.s.startswith("u'"):
node.s = node.s[1:]
if isinstance(node, ast.Name):
if node.id == 'undefined':
continue
node.id = namesJS[node.id]
jsStr = unparse(syntaxTree).strip()
if not any(ch in jsStr for ch in ("=",";","\n")):
try:
jsStr = translatePythonToJavaScript(jsStr)
if jsStr.endswith(';\n'):
jsStr = jsStr[:-2]
except:
# If translation fails, just use old translation
pass
return jsStr
def testEchoService(self):
"""Tests client-side of the Echo service."""
self.StartEchoServer()
try:
(server_host, server_port) = self._server.server_address
transceiver = ipc.HTTPTransceiver(host=server_host,
port=server_port)
requestor = ipc.Requestor(
local_protocol=ECHO_PROTOCOL,
transceiver=transceiver,
)
response = requestor.Request(
message_name='ping',
request_datum={
'ping': {
'timestamp': long(31415),
'text': unicode('hello ping')
}
},
)
logging.info('Received echo response: %s', response)
response = requestor.Request(
message_name='ping',
request_datum={
'ping': {
'timestamp': long(123456),
'text': unicode('hello again')
}
},
)
logging.info('Received echo response: %s', response)
transceiver.Close()
finally:
self.StopEchoServer()
def dict_to_json_str(d):
"""Function that converts a dictionary into a JSON string.
Supports types: basestring, bool, int and nested dicts.
Does not support lists.
If the value is None, it sets it to False."""
json_entry = u'"{0}":{1}'
json_entry_str = u'"{0}":"{1}"'
entries = []
for entry in d:
key = entry
value = d[entry]
if value is None:
value = False
if isinstance(value, list):
pass
if isinstance(value, basestring):
value = value.replace(u'"', u'\\"')
entries.append(json_entry_str.format(unicode(key), unicode(value)))
elif isinstance(value, unicode):
entries.append(json_entry.format(unicode(key), unicode(value)))
elif isinstance(value, bool):
value = 'true' if value else 'false'
entries.append(json_entry.format(key, value))
elif isinstance(value, int):
entries.append(json_entry.format(unicode(key), value))
elif isinstance(value, dict):
entries.append(json_entry.format(key, dict_to_json_str(value)))
return u'{0} {1} {2}'.format(u'{', ','.join(entries), u'}')
def state(self):
"""
Returns all currently pressed keys as a dictionary of key : time
values. The key is taken from the originating press event .key field.
The time value is time of the key press event.
Note that any pressed, or active, modifier keys are included in the
return value.
:return: dict
"""
self._syncDeviceState()
self._pressed_keys = {unicode(keys, 'utf-8'): vals for keys, vals in self._pressed_keys.items()}
return self._pressed_keys
def add_usr(self, message):
user_attrs = set(dir(message)) - reserved_attributes
if self.has_debug:
logger.debug('user_attrs:`%s`' % user_attrs)
if user_attrs:
usr = etree.Element('usr')
for user_attr in user_attrs:
user_attr_value = getattr(message, user_attr)
# Some values are integers, e.g. delivery_mode
if isinstance(user_attr_value, basestring):
user_attr_value = escape(user_attr_value)
# Create a JMS attribute and set its value.
user_attr = etree.Element(unicode(user_attr))
user_attr.text = unicode(user_attr_value)
usr.append(user_attr)
self.folders['usr'] = usr
def argv_to_unicode(s):
"""
Decode given argv element to unicode. If this fails, raise a UsageError.
"""
if isinstance(s, unicode):
return s
precondition(isinstance(s, bytes), s)
try:
return unicode(s, io_encoding)
except UnicodeDecodeError:
raise usage.UsageError("Argument %s cannot be decoded as %s." %
(quote_output(s), io_encoding))
def setDatatype(self, datatype):
"""Sets the datatype of the value"""
if isinstance(datatype, bytes):
datatype = unicode(datatype, "utf-8")
if isinstance(datatype, unicode):
if datatype[0] == '<':
datatype = datatype[1:-1]
datatype = URI(datatype)
elif datatype is not None:
if not isinstance(datatype, URI):
datatype = URI(datatype)
elif datatype.uri is None:
datatype = None
self._datatype = datatype # pylint: disable-msg=W0201
def get_parameter_display_large(parameter):
u"""
If a numerical value is found, round and return it. Otherwise return the full string.
Used together with get_parameter_display_small to draw the number part of a
parameter value larger than the unit part.
There are some exceptions, sometimes the unit looks better in the large part.
See the tests for intended usage and special cases.
"""
parameter_string = unicode(parameter)
if note_pattern.search(parameter_string) is not None:
return parameter_string
if hybrid_rate_pattern.search(parameter_string) is not None:
return parameter_string[0:-2]
large_string = u''.join(large_pattern.findall(parameter_string))
if large_string in (u'inf', u'-inf'):
return large_string
large_number = _try_to_round_number(large_string)
if large_number is None:
return parameter_string
if large_string.startswith(u'+'):
return u'+' + unicode(large_number)
return unicode(large_number)
def setDatatype(self, datatype):
"""Sets the datatype of the value"""
if isinstance(datatype, bytes):
datatype = unicode(datatype, "utf-8")
if isinstance(datatype, unicode):
if datatype[0] == '<':
datatype = datatype[1:-1]
datatype = XMLSchema.uristr2obj.get(datatype, None) or URI(datatype)
elif datatype is not None:
if not isinstance(datatype, URI):
datatype = URI(datatype)
elif datatype.uri is None:
datatype = None
self._datatype = datatype # pylint: disable-msg=W0201
def dump(db, f, **options): # type: (canmatrix.CanMatrix, typing.IO, **typing.Any) -> None
__init_yaml()
new_db = copy.deepcopy(db)
for i, frame in enumerate(new_db.frames):
for j, signal in enumerate(frame.signals):
if not signal.is_little_endian:
signal.start_bit = signal.get_startbit(bit_numbering=1, start_little=True)
# new_db.frames[i].signals[j].start_bit = signal.start_bit
# f = open(filename, "w")
if representers:
f.write(unicode(yaml.dump(new_db)))
else:
f.write(yaml.dump(new_db).encode('utf8'))
def _validate(f):
if code is not None:
self.failUnlessEqual(f.value.status, b"%d" % code, which)
if substring:
code_string = unicode(f)
self.failUnless(
substring in code_string,
"%s: substring '%s' not in '%s'" %
(which, substring, code_string))
response_body = f.value.response
if response_substring:
self.failUnless(
response_substring in response_body,
"%s: response substring '%s' not in '%s'" %
(which, response_substring, response_body))
return response_body
def datatype_from_python(value, datatype):
"""
If 'value' is not a string, convert it into one, and infer its
datatype, unless 'datatype' is set (i.e., overrides it).
"""
if isinstance(value, unicode):
return value, datatype
if isinstance(value, bytes):
return unicode(value, "utf-8"), datatype
## careful: test for 'bool' must precede test for 'int':
if isinstance(value, bool):
return unicode(value).lower(), datatype or XMLSchema.BOOLEAN
if isinstance(value, int):
return unicode(value), datatype or XMLSchema.INTEGER
if isinstance(value, long):
return unicode(value), datatype or XMLSchema.LONG
if isinstance(value, float):
return unicode(value), datatype or XMLSchema.DOUBLE
if isinstance(value, datetime.datetime):
# There is an ambiguity for times that occur twice due to
# DST switches, but that is a problem with Python's standard
# library and there is nothing we can do about it here.
if value.utcoffset() is not None:
value = value.replace(tzinfo=None) - value.utcoffset()
str_value = value.isoformat() + 'Z'
return str_value, datatype or XMLSchema.DATETIME
if isinstance(value, datetime.time):
if value.utcoffset() is not None:
dt = datetime.datetime.combine(RANDOM_DAY, value)
dt -= value.utcoffset()
# Note: this will strip TZ
value = dt.time()
str_value = value.isoformat() + 'Z'
return str_value, datatype or XMLSchema.TIME
if isinstance(value, datetime.date):
return value.isoformat(), datatype or XMLSchema.DATE
if isinstance(value, Decimal):
return unicode(value), datatype or XMLSchema.DECIMAL
return unicode(value), datatype
def datatype_from_python(value, datatype):
"""
If 'value' is not a string, convert it into one, and infer its
datatype, unless 'datatype' is set (i.e., overrides it).
"""
if isinstance(value, unicode):
return value, datatype
if isinstance(value, bytes):
return unicode(value, "utf-8"), datatype
## careful: test for 'bool' must precede test for 'int':
if isinstance(value, bool):
return unicode(value).lower(), datatype or XMLSchema.BOOLEAN
if isinstance(value, int):
return unicode(value), datatype or XMLSchema.INTEGER
if isinstance(value, long):
return unicode(value), datatype or XMLSchema.LONG
if isinstance(value, float):
return unicode(value), datatype or XMLSchema.DOUBLE
if isinstance(value, datetime.datetime):
# There is an ambiguity for times that occur twice due to
# DST switches, but that is a problem with Python's standard
# library and there is nothing we can do about it here.
if value.utcoffset() is not None:
value = value.replace(tzinfo=None) - value.utcoffset()
str_value = value.isoformat() + 'Z'
return str_value, datatype or XMLSchema.DATETIME
if isinstance(value, datetime.time):
if value.utcoffset() is not None:
dt = datetime.datetime.combine(RANDOM_DAY, value)
dt -= value.utcoffset()
# Note: this will strip TZ
value = dt.time()
str_value = value.isoformat() + 'Z'
return str_value, datatype or XMLSchema.TIME
if isinstance(value, datetime.date):
return value.isoformat(), datatype or XMLSchema.DATE
if isinstance(value, Decimal):
return unicode(value), datatype or XMLSchema.DECIMAL
return unicode(value), datatype
def get_server_information_from_email(self):
"""
Returns the parsed server information from email
"""
email_id = None
for email in self.get_emails():
e_id = email['id']
title = email['title']
if 'ready' in title.lower():
email_id = e_id
break
self._browser.open(self.email_url + '?id=' + email_id)
soup = self._browser.get_current_page()
server_info = {
'ip_address': None,
'server_user': None,
'server_password': None,
'vmuser': None,
'vmuser_password': None,
'control_panel_url': None
}
ps = soup.findAll('p')
# map of server_info fields to the labels in the e-mail
server_keyword = 'Hostname'
server_fields = {
'ip_address': 'Main IP',
'server_user': '******',
'server_password': '******'
}
vm_keyword = 'Manager Details'
vm_fields = {'vmuser': '******', 'vmuser_password': '******'}
for p in ps:
for line in p:
self._parse_email_section(p, line, server_keyword,
server_fields, server_info)
self._parse_email_section(p, line, vm_keyword, vm_fields,
server_info)
if isinstance(line, Tag) and line.name == 'a':
server_info['control_panel_url'] = unicode(line.next)
return server_info
def dump(db, f, **options
): # type: (canmatrix.CanMatrix, typing.IO, **typing.Any) -> None
__init_yaml()
new_db = copy.deepcopy(db)
for i, frame in enumerate(new_db.frames):
for j, signal in enumerate(frame.signals):
if not signal.is_little_endian:
signal.start_bit = signal.get_startbit(bit_numbering=1,
start_little=True)
# new_db.frames[i].signals[j].start_bit = signal.start_bit
# f = open(filename, "w")
if representers:
f.write(unicode(yaml.dump(new_db)))
else:
f.write(yaml.dump(new_db).encode('utf8'))
def convert_children_json(nodemaker, children_json):
"""I convert the JSON output of GET?t=json into the dict-of-nodes input
to both dirnode.create_subdirectory() and
client.create_directory(initial_children=). This is used by
t=mkdir-with-children and t=mkdir-immutable"""
children = {}
if children_json:
data = json.loads(children_json)
for (namex, (ctype, propdict)) in data.iteritems():
namex = unicode(namex)
writecap = to_bytes(propdict.get("rw_uri"))
readcap = to_bytes(propdict.get("ro_uri"))
metadata = propdict.get("metadata", {})
# name= argument is just for error reporting
childnode = nodemaker.create_from_cap(writecap, readcap, name=namex)
children[namex] = (childnode, metadata)
return children
def deserialize(string):
def posInteger(chars):
result = shift = 0
# Set value to get into the loop the first time
value = 0x80
while value & 0x80:
value = next(chars)
result += ((value & 0x7f) << shift)
shift += 7
return result
if isinstance(string, old_str):
string = bytes(string)
chars = iter(string)
value = next(chars)
if value == SerialConstants.SO_BYTEVECTOR:
length = posInteger(chars)
import array
return array.array(b'b', [ord(next(chars)) for i in range(length)])
if (value == SerialConstants.SO_VECTOR
or value == SerialConstants.SO_LIST):
length = posInteger(chars)
return [deserialize(chars) for i in range(length)]
if value == SerialConstants.SO_STRING:
length = posInteger(chars)
return unicode(ibytes(islice(chars, 0, length)), 'utf-8')
if value == SerialConstants.SO_POS_INTEGER:
return posInteger(chars)
if value == SerialConstants.SO_NEG_INTEGER:
return -posInteger(chars)
if value == SerialConstants.SO_NULL:
return None
if value == SerialConstants.SO_END_OF_ITEMS:
return None
raise ValueError("bad code found by deserializer: %d" % value)
def deserialize(string):
def posInteger(chars):
result = shift = 0
# Set value to get into the loop the first time
value = 0x80
while value & 0x80:
value = next(chars)
result += ((value & 0x7f) << shift)
shift += 7
return result
if isinstance(string, old_str):
string = bytes(string)
chars = iter(string)
value = next(chars)
if value == SerialConstants.SO_BYTEVECTOR:
length = posInteger(chars)
import array
return array.array(b'b', [ord(next(chars)) for i in range(length)])
if (value == SerialConstants.SO_VECTOR or
value == SerialConstants.SO_LIST):
length = posInteger(chars)
return [deserialize(chars) for i in range(length)]
if value == SerialConstants.SO_STRING:
length = posInteger(chars)
return unicode(ibytes(islice(chars, 0, length)), 'utf-8')
if value == SerialConstants.SO_POS_INTEGER:
return posInteger(chars)
if value == SerialConstants.SO_NEG_INTEGER:
return - posInteger(chars)
if value == SerialConstants.SO_NULL:
return None
if value == SerialConstants.SO_END_OF_ITEMS:
return None
raise ValueError("bad code found by deserializer: %d" % value)
def test_simple_df_with_beam_row(self):
expected = pd.DataFrame(
{
'name': list(unicode(i) for i in range(5)),
'id': list(range(5)),
'height': list(float(i) for i in range(5))
},
columns=['name', 'id', 'height'])
with TestPipeline() as p:
res = (p
| beam.Create([(str(i), i, float(i)) for i in range(5)])
| beam.Select(name=lambda r: str(r[0]),
id=lambda r: int(r[1]),
height=lambda r: float(r[2]))
| schemas.BatchRowsAsDataFrame(min_batch_size=10,
max_batch_size=10))
assert_that(res, matches_df(expected))
def render(self, req):
"""
Dispatch to a renderer for a particular format, as selected by a query
argument.
A renderer for the format given by the query argument matching
``formatArgument`` will be selected and invoked. render_HTML will be
used as a default if no format is selected (either by query arguments
or by ``formatDefault``).
:return: The result of the selected renderer.
"""
t = get_arg(req, self.formatArgument, self.formatDefault)
# It's either bytes or None.
if isinstance(t, bytes):
t = unicode(t, "ascii")
renderer = self._get_renderer(t)
return renderer(req)
def listdir_unicode_fallback(path):
"""
This function emulates a fallback Unicode API similar to one available
under Windows or MacOS X.
If badly encoded filenames are encountered, an exception is raised.
"""
precondition(isinstance(path, unicode), path)
try:
byte_path = path.encode(filesystem_encoding)
except (UnicodeEncodeError, UnicodeDecodeError):
raise FilenameEncodingError(path)
try:
return [unicode(fn, filesystem_encoding) for fn in os.listdir(byte_path)]
except UnicodeDecodeError as e:
raise FilenameEncodingError(e.object)
def make_kind_stats_query(namespace, kind, latest_timestamp):
"""Make a Query to fetch the latest kind statistics."""
kind_stat_query = query_pb2.Query()
if namespace is None:
kind_stat_query.kind.add().name = '__Stat_Kind__'
else:
kind_stat_query.kind.add().name = '__Stat_Ns_Kind__'
kind_filter = datastore_helper.set_property_filter(
query_pb2.Filter(), 'kind_name', PropertyFilter.EQUAL,
unicode(kind))
timestamp_filter = datastore_helper.set_property_filter(
query_pb2.Filter(), 'timestamp', PropertyFilter.EQUAL,
latest_timestamp)
datastore_helper.set_composite_filter(kind_stat_query.filter,
CompositeFilter.AND, kind_filter,
timestamp_filter)
return kind_stat_query
def make_kind_stats_query(namespace, kind, latest_timestamp):
"""Make a Query to fetch the latest kind statistics."""
kind_stat_query = query_pb2.Query()
if namespace is None:
kind_stat_query.kind.add().name = '__Stat_Kind__'
else:
kind_stat_query.kind.add().name = '__Stat_Ns_Kind__'
kind_filter = datastore_helper.set_property_filter(
query_pb2.Filter(), 'kind_name', PropertyFilter.EQUAL, unicode(kind))
timestamp_filter = datastore_helper.set_property_filter(
query_pb2.Filter(), 'timestamp', PropertyFilter.EQUAL, latest_timestamp)
datastore_helper.set_composite_filter(
kind_stat_query.filter,
CompositeFilter.AND,
kind_filter,
timestamp_filter)
return kind_stat_query
def get_service_table_line(self, idx, name, docs, sio):
name_fs_safe = 'service_{}'.format(fs_safe_name(name))
file_name = '{}.rst'.format(name_fs_safe)
summary = docs.summary
if summary:
summary = self._make_sphinx_safe(summary)
return bunchify({
'ns': unicode(idx),
'orig_name': name,
'sphinx_name': name.replace('_', '\_'), # Needed for Sphinx to ignore undescores # noqa: W605
'name': name_fs_safe,
'name_link': """:doc:`{} <./{}>`""".format(name, name_fs_safe),
'file_name': file_name,
'description': summary or no_value,
'docs': docs,
'sio': sio
})
def read_config(basedir, portnumfile, generated_files=[], _valid_config=None):
"""
Read and validate configuration.
:param unicode basedir: directory where configuration data begins
:param unicode portnumfile: filename fragment for "port number" files
:param list generated_files: a list of automatically-generated
configuration files.
:param ValidConfiguration _valid_config: (internal use, optional) a
structure defining valid configuration sections and keys
:returns: :class:`allmydata.node._Config` instance
"""
basedir = abspath_expanduser_unicode(unicode(basedir))
if _valid_config is None:
_valid_config = _common_valid_config()
# complain if there's bad stuff in the config dir
_error_about_old_config_files(basedir, generated_files)
# canonicalize the portnum file
portnumfile = os.path.join(basedir, portnumfile)
# (try to) read the main config file
config_fname = os.path.join(basedir, "tahoe.cfg")
try:
parser = configutil.get_config(config_fname)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
# The file is missing, just create empty ConfigParser.
parser = configutil.get_config_from_string(u"")
configutil.validate_config(config_fname, parser, _valid_config)
# make sure we have a private configuration area
fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
return _Config(parser, portnumfile, basedir, config_fname)
def _parquet_compatible(df):
df = df.reset_index(drop=True)
to_join = {}
if not df.empty:
for c in df:
if df[c].dtype == object:
dropped = df[c].dropna()
if not dropped.empty and isinstance(dropped.iloc[0], tuple):
to_join[c] = []
# Assume all tuples are of same length
for i in range(len(dropped.iloc[0])):
new_name = str(uuid.uuid1())
df[new_name] = np.nan
df.loc[dropped.index, new_name] = dropped.apply(lambda x: x[i])
to_join[c].append(new_name)
del df[c]
if sys.version_info <= (3, 0):
from past.builtins import unicode
df.columns = [unicode(c) for c in df.columns]
return df, to_join
def exact(self, field, value, **kwargs):
kwargs = self._check_datetime_dict(kwargs)
try:
field_template = getattr(self.base_obj, field)
if type(field_template) != ESType:
field_template = create_es_type(field_template)
for estype in [String, IP, Attachment]:
if isinstance(field_template, estype) and field_template.analyzed == True:
logger.warn('%s types may not exact match correctly if they are analyzed' % unicode(
estype.__class__.__name__))
except AttributeError:
logger.warn('%s is not in the base model.' % unicode(field))
kwargs['type'] = 'filter'
if isinstance(value, list):
self._querybody.chain(qdsl.terms(field, value), **kwargs)
else:
self._querybody.chain(qdsl.term(field, value), **kwargs)
return self
def random_string(length):
return unicode(''.join(random.choice(
string.ascii_letters + string.digits) for _ in range(length)))
def mk_unicode(text):
if not isinstance(text, unicode):
return unicode(text, 'utf-8')
return text
def decode_shortstrings(input_buffer, errors='strict'): """Decoder (to Unicode) that suppresses long base64 strings.""" shortened, length = encode_shortstrings(input_buffer, errors) return unicode(shortened), length
def received_message(self, message):
if message.is_text:
self.runtime.events.put((self.handler.onWSMessage, (self.ws, unicode(message)), {}))
else:
self.runtime.events.put((self.handler.onWSBinary, (self.ws, Buffer(message.data)), {}))
def display_data(self):
return {'unicode_string': unicode('my string'),
'unicode_literal_string': u'my literal string'}
def key(self):
return unicode(self._key, 'utf-8')
def to_entity(content):
entity = entity_pb2.Entity()
googledatastore.helper.add_key_path(entity.key, kind, str(uuid.uuid4()))
googledatastore.helper.add_properties(entity,
{'content': unicode(content)})
return entity
def __str__(self):
return '%s, key: %s char: %s, modifiers: %s' % (
ioEvent.__str__(self), unicode(self.key, 'utf-8'),
unicode(self.char, 'utf-8'),
unicode(self.modifiers, 'utf-8'))
def handle(self, request, replyTo):
if self._running is False:
return False
self.feed(request)
request = self.unpack()
# print2err(">> Rx Packet: {}, {}".format(request, replyTo))
request_type = unicode(request.pop(0), 'utf-8') # convert bytes to string for compatibility
if request_type == 'SYNC_REQ':
self.sendResponse(['SYNC_REPLY', getTime()], replyTo)
return True
elif request_type == 'PING':
_ = request.pop(0) #client time
msg_id = request.pop(0)
payload = request.pop(0)
ctime = getTime()
self.sendResponse(['PING_BACK', ctime, msg_id,
payload, replyTo], replyTo)
return True
elif request_type == 'GET_EVENTS':
return self.handleGetEvents(replyTo)
elif request_type == 'EXP_DEVICE':
return self.handleExperimentDeviceRequest(request, replyTo)
elif request_type == 'CUSTOM_TASK':
return self.handleCustomTaskRequest(request, replyTo)
elif request_type == 'RPC':
callable_name = request.pop(0)
args = None
kwargs = None
if len(request) == 1:
args = request.pop(0)
if len(request) == 1:
kwargs = request.pop(0)
result = None
try:
result = getattr(self, unicode(callable_name, 'utf-8'))
except Exception:
print2err('RPC_ATTRIBUTE_ERROR')
printExceptionDetailsToStdErr()
self.sendResponse('RPC_ATTRIBUTE_ERROR', replyTo)
return False
if result and callable(result):
funcPtr = result
try:
if args is None and kwargs is None:
result = funcPtr()
elif args and kwargs:
result = funcPtr(*args, **convertByteKeysToStr(kwargs))
elif args and not kwargs:
result = funcPtr(*args)
elif not args and kwargs:
result = funcPtr(**convertByteKeysToStr(kwargs))
edata = ('RPC_RESULT', callable_name, result)
self.sendResponse(edata, replyTo)
return True
except Exception:
print2err('RPC_RUNTIME_ERROR')
printExceptionDetailsToStdErr()
self.sendResponse('RPC_RUNTIME_ERROR', replyTo)
return False
else:
print2err('RPC_NOT_CALLABLE_ERROR')
printExceptionDetailsToStdErr()
self.sendResponse('RPC_NOT_CALLABLE_ERROR', replyTo)
return False
elif request_type == 'GET_IOHUB_STATUS':
self.sendResponse((request_type, self.iohub.getStatus()), replyTo)
return True
elif request_type == 'STOP_IOHUB_SERVER':
self.shutDown()
else:
print2err('RPC_NOT_CALLABLE_ERROR')
printExceptionDetailsToStdErr()
self.sendResponse('RPC_NOT_CALLABLE_ERROR', replyTo)
return False
def handleExperimentDeviceRequest(self, request, replyTo):
request_type = unicode(request.pop(0), 'utf-8') # convert bytes to string for compatibility
io_dev_dict = ioServer.deviceDict
if request_type == 'EVENT_TX':
exp_events = request.pop(0)
exp_dev_cb = io_dev_dict['Experiment']._nativeEventCallback
for eventAsTuple in exp_events:
exp_dev_cb(eventAsTuple)
self.sendResponse(('EVENT_TX_RESULT', len(exp_events)), replyTo)
return True
elif request_type == 'DEV_RPC':
dclass = unicode(request.pop(0), 'utf-8')
dmethod = unicode(request.pop(0), 'utf-8')
args = None
kwargs = None
if len(request) == 1:
args = request[0]
elif len(request) == 2:
args = request[0]
kwargs = request[1]
if len(kwargs) == 0:
kwargs = None
dev = None
if dclass.find('.') > 0:
for dname, dev in ioServer.deviceDict.items():
if dname.endswith(dclass):
dev = ioServer.deviceDict.get(dname, None)
break
else:
dev = ioServer.deviceDict.get(dclass, None)
if dev is None:
print2err('IOHUB_DEVICE_ERROR')
printExceptionDetailsToStdErr()
self.sendResponse('IOHUB_DEVICE_ERROR', replyTo)
return False
try:
method = getattr(dev, dmethod)
except Exception:
print2err('IOHUB_DEVICE_METHOD_ERROR')
printExceptionDetailsToStdErr()
self.sendResponse('IOHUB_DEVICE_METHOD_ERROR', replyTo)
return False
result = []
try:
if args and kwargs:
result = method(*args, **convertByteKeysToStr(kwargs))
elif args:
result = method(*args)
elif kwargs:
result = method(**convertByteKeysToStr(kwargs))
else:
result = method()
self.sendResponse(('DEV_RPC_RESULT', result), replyTo)
return True
except Exception:
print2err('RPC_DEVICE_RUNTIME_ERROR')
printExceptionDetailsToStdErr()
self.sendResponse('RPC_DEVICE_RUNTIME_ERROR', replyTo)
return False
elif request_type == 'GET_DEVICE_LIST':
try:
dev_list = []
for d in self.iohub.devices:
dev_list.append((d.name, d.__class__.__name__))
self.sendResponse(
('GET_DEV_LIST_RESULT', len(dev_list), dev_list), replyTo)
return True
except Exception:
print2err('RPC_DEVICE_RUNTIME_ERROR')
printExceptionDetailsToStdErr()
self.sendResponse('RPC_DEVICE_RUNTIME_ERROR', replyTo)
return False
elif request_type == 'GET_DEV_INTERFACE':
dclass = unicode(request.pop(0), 'utf-8')
data = None
if dclass in ['EyeTracker', 'DAQ']:
for dname, hdevice in ioServer.deviceDict.items():
if dname.endswith(dclass):
data = hdevice._getRPCInterface()
break
else:
dev = ioServer.deviceDict.get(dclass, None)
if dev:
data = dev._getRPCInterface()
if data:
self.sendResponse(('GET_DEV_INTERFACE', data), replyTo)
return True
else:
print2err('GET_DEV_INTERFACE_ERROR: ',
'_getRPCInterface returned: ', data)
self.sendResponse('GET_DEV_INTERFACE_ERROR', replyTo)
return False
elif request_type == 'ADD_DEVICE':
cls_name = request.pop(0)
dev_cfg = request.pop(1)
data = self.iohub.createNewMonitoredDevice(cls_name, dev_cfg)
if data:
self.sendResponse(('ADD_DEVICE', data), replyTo)
return True
else:
print2err('ADD_DEVICE_ERROR: createNewMonitoredDevice ',
'returned: ', data)
self.sendResponse('ADD_DEVICE_ERROR', replyTo)
return False
else:
print2err('DEVICE_RPC_TYPE_NOT_SUPPORTED_ERROR: ',
'Unknown Request Type: ', request_type)
self.sendResponse('DEVICE_RPC_TYPE_NOT_SUPPORTED_ERROR', replyTo)
return False
def normalize_quint(stringTuple):
st = stringTuple
return (st[1], st[2], st[3], None if len(st) == 4 else st[4], unicode(st[0]))
def setter(self, entry):
setattr(self, name, unicode(json.dumps(entry, encode_datetime=True)))
