def convert_to_pandas_timedelta(x):
r"""Convert variable with time units to a pandas.Timedelta instance.
Args:
x (object): Scalar/array with units to convert to a pandas.Timedelta
instance.
Returns:
pandas.Timedelta: Equivalent Timedelta variable.
"""
assert (has_units(x))
t_data = get_data(x)
t_unit = get_units(x)
unit_map = {
'ns': 'ns',
(tools.bytes2str(b'\xc2\xb5') + 's'): 'us',
(tools.bytes2str(b'\xce\xbcs') + 's'): 'us',
'ms': 'ms',
's': 's',
'min': 'm',
'hr': 'h',
'day': 'D'
}
return pd.Timedelta(t_data, unit=unit_map[t_unit])
def convert_unit_string(orig_str, replacements=None):
r"""Convert unit string to string that the Python package can
understand.
Args:
orig_str (str): Original units string to convert.
replacements (dict, optional): Mapping from unit to another.
Defaults to empty dict.
Returns:
str: Converted string.
"""
if not orig_str.strip():
return ''
out = []
if replacements is None:
replacements = {
'h': 'hr',
'hrs': 'hr',
'days': 'day',
'100%': 'percent'
}
regex_mu = [
tools.bytes2str(b'\xc2\xb5'),
tools.bytes2str(b'\xce\xbcs'),
tools.bytes2str(b'\xc2\xb0'), r'(?:100\%)'
]
regex = (r'(?P<paren>\()?(?P<name>[A-Za-z%s]+)'
r'(?:(?:(?:\^)|(?:\*\*))?(?P<exp_paren>\()?(?P<exp>-?[0-9]+)'
r'(?(exp_paren)\)))?'
r'(?(paren)\)|)(?P<op> |(?:\*)|(?:\/))?' % ''.join(regex_mu))
out = ''
if re.fullmatch(r'(?:%s)+' % regex, orig_str.strip()):
for x in re.finditer(regex, orig_str.strip()):
xdict = x.groupdict()
if xdict['name'] in replacements:
xdict['name'] = replacements[xdict['name']]
if xdict['exp']:
out += '({name}**{exp})'.format(**xdict)
else:
out += xdict['name']
if xdict['op']:
if xdict['op'].isspace():
xdict['op'] = '*'
out += xdict['op']
else: # pragma: debug
print(repr(orig_str), type(orig_str))
m = re.search(r'(?:%s)+' % regex, orig_str.strip())
if m:
print(repr(m.group(0)), m.groupdict())
else:
print('no match')
for m in re.finditer(regex, orig_str.strip()):
print(m.group(0), m.groupdict())
raise Exception("Could not standardize units: %s" % repr(orig_str))
return out
def test_convert_R_unit_string(self):
r"""Test convert_R_unit_string."""
pairs = [('g', 'g'), ('g2', '(g**2)'),
('g2 km s-2', '(g**2)*km*(s**-2)'), ('degC d', 'degC*d'),
(tools.bytes2str(b'\xc2\xb0C d'),
tools.bytes2str(b'\xc2\xb0C*d')), ('h', 'hr'),
('hrs/kg', 'hr/kg'), ('', ''), ('cm**(-2)', '(cm**-2)')]
for x, y in pairs:
self.assert_equal(units.convert_R_unit_string(x), y)
self.assert_equal(units.convert_R_unit_string(y), y)
units.add_units(1.0, x)
def get_field_units(self, as_bytes=False):
r"""Get the field units for an array of fields.
Args:
as_bytes (bool, optional): If True, the field units will be returned
as bytes. If False the field units will be returned as unicode.
Defaults to False.
Returns:
list: Units for each field in the data type.
"""
if self.typedef['type'] != 'array':
return None
if getattr(self, 'field_units', None) is not None:
out = copy.deepcopy(self.field_units)
elif isinstance(self.typedef['items'], dict): # pragma: debug
raise Exception("Variable number of items not yet supported.")
elif isinstance(self.typedef['items'], list):
out = []
any_units = False
for i, x in enumerate(self.typedef['items']):
out.append(x.get('units', ''))
if len(x.get('units', '')) > 0:
any_units = True
# Don't use field units if they are all defaults
if not any_units:
out = None
if (out is not None):
if as_bytes:
out = tools.str2bytes(out, recurse=True)
else:
out = tools.bytes2str(out, recurse=True)
return out
def add_units(arr, unit_str, dtype=None):
r"""Add units to an array or scalar.
Args:
arr (np.ndarray, float, int): Scalar or array of data to add units to.
unit_str (str): Unit string.
dtype (np.dtype, optional): Numpy data type that should be maintained for
array/qunatity with units. If not provided, this is determined from the
array.
Returns:
unyt.unyt_array: Array with units.
"""
ureg = get_ureg()
unit_str = tools.bytes2str(unit_str)
if is_null_unit(unit_str):
return arr
unit_str = convert_unit_string(unit_str)
if has_units(arr):
return convert_to(arr, unit_str)
if dtype is None:
if isinstance(arr, np.ndarray):
dtype = arr.dtype
else:
dtype = np.array([arr]).dtype
try:
if isinstance(arr, np.ndarray) and (arr.ndim > 0):
out = unyt.unyt_array(arr, unit_str, dtype=dtype, registry=ureg)
else:
out = unyt.unyt_quantity(arr, unit_str, dtype=dtype, registry=ureg)
except BaseException:
raise ValueError("Error parsing unit: %s, type(%s)." %
(repr(unit_str), type(unit_str)))
return out
def func_deserialize(self, msg):
r"""Deserialize a message.
Args:
msg (bytes): Message to be deserialized.
Returns:
dict: Deserialized Python dictionary.
"""
out = dict()
lines = tools.bytes2str(msg.split(self.newline), recurse=True)
for line in lines:
kv = [x for x in line.split(self.delimiter) if x]
if len(kv) <= 1:
# TODO: Allow empty?
continue
elif len(kv) == 2:
if kv[1].startswith("'") and kv[1].endswith("'"):
out[kv[0]] = kv[1].strip("'")
else:
try:
out[kv[0]] = json.loads(kv[1])
except BaseException:
out[kv[0]] = kv[1]
else:
raise ValueError("Line has more than one delimiter: " +
str(line))
return out
def func_serialize(self, args):
r"""Serialize a message.
Args:
args (dict): Python dictionary to be serialized.
Returns:
bytes, str: Serialized message.
"""
lines = []
for k in args.keys():
v = args[k]
if not isinstance(k, (str, bytes)): # pragma: debug
raise ValueError(
"Serialization of non-string keys not supported.")
iline = tools.bytes2str(k) + self.delimiter
if isinstance(v, list):
indent = ' ' * len(iline)
arr_lines = []
assert (len(v) == 2)
assert (v[0].shape == v[1].shape)
for i in range(len(v[0])):
arr_lines.append(self._array_fmt % (v[0][i], v[1][i]))
v_units = [str(getattr(vv, 'units', '-')) for vv in v]
arr_lines[0] += f"\t! [{'; '.join(v_units)}]"
iline += (',\n' + indent).join(arr_lines)
elif isinstance(v, str):
iline += "\'%s\'" % v
else:
iline += json.dumps(v, cls=JSONReadableEncoder)
if hasattr(v, 'units'):
iline += f'\t! [{v.units}]'
lines.append(iline)
return tools.str2bytes('\n'.join(lines))
def add_units(arr, unit_str, dtype=None):
r"""Add units to an array or scalar.
Args:
arr (np.ndarray, float, int): Scalar or array of data to add units to.
unit_str (str): Unit string.
dtype (np.dtype, optional): Numpy data type that should be maintained for
array/qunatity with units. If not provided, this is determined from the
array.
Returns:
unyt.unyt_array: Array with units.
"""
unit_str = tools.bytes2str(unit_str)
if is_null_unit(unit_str):
return arr
if has_units(arr):
return convert_to(arr, unit_str)
if dtype is None:
if isinstance(arr, np.ndarray):
dtype = arr.dtype
else:
dtype = np.array([arr]).dtype
if isinstance(arr, np.ndarray) and (arr.ndim > 0):
out = unyt.unyt_array(arr, unit_str, dtype=dtype, registry=_ureg_unyt)
else:
out = unyt.unyt_quantity(arr,
unit_str,
dtype=dtype,
registry=_ureg_unyt)
return out
def input_kwargs(self):
r"""dict: Get the input keyword arguments used to create this class."""
out = {}
for k in self._schema_properties.keys():
v = getattr(self, k, None)
if v is not None:
out[k] = copy.deepcopy(v)
for k in self._attr_conv:
if k in out:
out[k] = tools.bytes2str(out[k])
return out
def convert_matlab_unit_string(m_str): # pragma: matlab
r"""Convert Matlab unit string to string that the Python package
can understand.
Args:
m_str (str): Matlab units string to convert.
Returns:
str: Converted string.
"""
out = m_str
replacements = {'h': 'hr'}
regex_mu = [tools.bytes2str(b'\xc2\xb5'), tools.bytes2str(b'\xce\xbcs')]
regex = r'(?P<name>[A-Za-z%s]+)' % ''.join(regex_mu)
for x in re.finditer(regex, m_str):
xdict = x.groupdict()
if xdict['name'] in replacements:
xdict['name'] = replacements[xdict['name']]
out = out[:(x.start())] + xdict['name'] + out[(x.end()):]
return out
def test_popen_nobuffer():
r"""Test open of process without buffer."""
ans = os.getcwd() # + '\n'
# Test w/o shell
if platform._is_win: # pragma: windows
args = ['cmd', '/c', 'cd']
else:
args = ['pwd']
p = tools.popen_nobuffer(args)
out, err = p.communicate()
res = tools.bytes2str(out).splitlines()[0]
assert_equal(res, ans)
# Test w/ shell
if platform._is_win: # pragma: windows
args = 'cd'
else:
args = 'pwd'
p = tools.popen_nobuffer(args, shell=True)
out, err = p.communicate()
res = tools.bytes2str(out).splitlines()[0]
assert_equal(res, ans)
def set_reply_socket_recv(self, address):
r"""Set the recv reply socket if the address dosn't exist."""
address = tools.bytes2str(address)
if address not in self.reply_socket_recv:
s = self.context.socket(zmq.REQ)
s.setsockopt(zmq.LINGER, 0)
s.connect(address)
self.register_comm('REPLY_RECV_' + address, s)
with self.reply_socket_lock:
self._n_reply_recv[address] = 0
self._n_zmq_recv[address] = 0
self.reply_socket_recv[address] = s
self.debug("new recv address: %s", address)
return address
def test_bytes2str():
r"""Test bytes2str."""
vals = [(b'hello', 'hello'), ('hello', 'hello'),
((b'a', b'b'), ('a', 'b')),
({
'a': b'a',
'b': b'b'
}, {
'a': 'a',
'b': 'b'
}), ([b'a', b'b'], ['a', 'b']),
([b'a', [b'b', b'c']], ['a', ['b', 'c']])]
for x, exp in vals:
assert_equal(tools.bytes2str(x, recurse=True), exp)
def func_serialize(self, args):
r"""Serialize a message.
Args:
args (dict): Python dictionary to be serialized.
Returns:
bytes, str: Serialized message.
"""
out = ''
order = sorted([k for k in args.keys()])
newline_str = tools.bytes2str(self.newline)
for k in order:
v = args[k]
if not isinstance(k, (str, bytes)):
raise ValueError(
"Serialization of non-string keys not supported.")
out += tools.bytes2str(k)
out += self.delimiter
out += json.dumps(v, cls=JSONReadableEncoder)
out += newline_str
return tools.str2bytes(out)
def serializer_info(self):
r"""dict: Serializer info."""
out = copy.deepcopy(self.extra_kwargs)
for k in self._schema_properties.keys():
if k in ['datatype']:
continue
v = getattr(self, k, None)
if v is not None:
out[k] = copy.deepcopy(v)
for k in out.keys():
v = out[k]
if isinstance(v, (bytes, list, tuple)):
out[k] = tools.bytes2str(v, recurse=True)
else:
out[k] = v
return out
def serializer_info(self):
r"""dict: Serializer info."""
out = copy.deepcopy(self.extra_kwargs)
for k in self._schema_properties.keys():
if (k != 'seritype') and (k in ['datatype'] + self._defaults_set):
continue
v = getattr(self, k, None)
if v is not None:
out[k] = copy.deepcopy(v)
for k in out.keys():
v = out[k]
try:
out[k] = tools.bytes2str(v, recurse=True)
except TypeError:
out[k] = v
return out
def normalize(cls, obj):
r"""Normalize an object, if possible, to conform to this type.
Args:
obj (object): Object to normalize.
Returns:
object: Normalized object.
"""
if isinstance(obj, (str, bytes)):
try:
obj_str = tools.bytes2str(obj)
obj = cls.decode_data(obj_str, {'type': cls.name})
except (ValueError, AttributeError):
pass
return obj
def array_to_table(arrs, fmt_str, use_astropy=False):
r"""Serialize an array as an ASCII table.
Args:
arrs (np.ndarray, list, tuple): Structured array or list/tuple of
arrays that contain table information.
fmt_str (str, bytes): Format string that should be used to structure
the ASCII array.
use_astropy (bool, optional): If True, astropy will be used to format
the table if it is installed. Defaults to False.
Returns:
bytes: ASCII table.
"""
if not _use_astropy:
use_astropy = False
dtype = cformat2nptype(fmt_str)
if len(dtype) == 0:
dtype = np.dtype([('f0', dtype)])
info = format2table(fmt_str)
comment = info.get('comment', None)
if comment is not None:
fmt_str = fmt_str.split(comment, 1)[-1]
arr1 = consolidate_array(arrs, dtype=dtype)
if use_astropy:
fd = sio.StringIO()
table = apy_Table(arr1)
delimiter = tools.bytes2str(info['delimiter'])
apy_ascii.write(table, fd, delimiter=delimiter,
format='no_header')
out = tools.str2bytes(fd.getvalue())
else:
fd = sio.BytesIO()
fmt_str = tools.str2bytes(fmt_str)
for ele in arr1:
line = format_message(ele.tolist(), fmt_str)
fd.write(line)
# fmt = fmt_str.split(info['newline'])[0]
# np.savetxt(fd, arr1,
# fmt=fmt, delimiter=info['delimiter'],
# newline=info['newline'], header='')
out = fd.getvalue()
fd.close()
return out
def is_unit(ustr):
r"""Determine if a string is a valid unit.
Args:
ustr (str): String representation to test.
Returns:
bool: True if the string is a valid unit. False otherwise.
"""
ustr = tools.bytes2str(ustr)
if is_null_unit(ustr):
return True
try:
as_unit(ustr)
except ValueError:
return False
return True
def decode_json(msg, **kwargs):
r"""Decode a Python object from a JSON serialization.
Args:
msg (str): JSON serialization to decode.
**kwargs: Additional keyword arguments are passed to json.loads.
Returns:
object: Deserialized Python object.
"""
if isinstance(msg, (str, bytes)):
msg_decode = tools.bytes2str(msg)
func_decode = json.loads
else:
msg_decode = msg
func_decode = json.load
func_decode = JSONDecoder()
return func_decode(msg_decode, **kwargs)
def coerce_type(cls, obj, typedef=None, **kwargs):
r"""Coerce objects of specific types to match the data type.
Args:
obj (object): Object to be coerced.
typedef (dict, optional): Type defintion that object should be
coerced to. Defaults to None.
**kwargs: Additional keyword arguments are metadata entries that may
aid in coercing the type.
Returns:
object: Coerced object.
"""
if isinstance(obj, dict) and ('material' in obj):
obj['material'] = tools.bytes2str(obj['material'])
return super(ObjMetaschemaType, cls).coerce_type(obj,
typedef=typedef,
**kwargs)
def normalize(cls, obj, working_dir=None):
r"""Normalize an object, if possible, to conform to this type.
Args:
obj (object): Object to normalize.
working_dir (str, optional): Working directory that should
be used to make relative paths absolute. Defaults to None.
Returns:
object: Normalized object.
"""
if isinstance(obj, (str, bytes)):
try:
obj_str = tools.bytes2str(obj)
obj = cls.decode_data(obj_str, {'type': cls.name},
working_dir=working_dir)
except (ValueError, AttributeError):
pass
return obj
def convert_R_unit_string(r_str):
r"""Convert R unit string to string that the Python package can
understand.
Args:
r_str (str): R units string to convert.
Returns:
str: Converted string.
"""
out = []
regex_mu = tools.bytes2str(b'\xc2\xb5')
regex = r'(?P<name>[A-Za-z%s]+)(?P<exp>-?[0-9]*)(?: |$)' % regex_mu
for x in re.finditer(regex, r_str):
xdict = x.groupdict()
if xdict['exp']:
out.append('({name}**{exp})'.format(**xdict))
else:
out.append(xdict['name'])
return '*'.join(out)
def decode_data(cls, msg, typedef):
r"""Decode an object.
Args:
msg (string): Encoded object to decode.
typedef (dict): Type definition that should be used to decode the
object.
Returns:
object: Decoded object.
"""
msg = tools.bytes2str(msg)
lines = msg.splitlines()
metadata = {'comments': []}
out = {}
# Parse
for line_count, line in enumerate(lines):
if line.startswith('#'):
metadata['comments'].append(line)
continue
values = line.split()
if not values:
continue
if values[0] not in _map_code2element:
raise ValueError("Type code '%s' on line %d not understood" %
(values[0], line_count))
e = _map_code2element[values[0]]
if e not in out:
out[e] = []
if e in ['material']:
out[e] = values[1]
continue
else:
out[e].append(
cls._decode_object_property(values[1:],
_default_property_order[e]))
# Return
# out.update(**metadata)
return ObjDict(out)
def decode_data(cls, msg, typedef):
r"""Decode an object.
Args:
msg (string): Encoded object to decode.
typedef (dict): Type definition that should be used to decode the
object.
Returns:
object: Decoded object.
"""
msg = tools.bytes2str(msg)
lines = msg.splitlines()
metadata = {'comments': [], 'element_order': [], 'property_order': {}}
if lines[0] != 'ply':
raise ValueError("The first line must be 'ply'")
# Parse header
e = None
p = None
type_map = {}
size_map = {}
obj = {}
for i, line in enumerate(lines):
if line.startswith('format'):
metadata['plyformat'] = line.split(None, 1)[-1]
elif line.startswith('comment'):
out = line.split(None, 1)[-1]
if out.startswith('material:'):
metadata['element_order'].append('material')
obj['material'] = out.split(None, 1)[-1]
metadata['comments'].append(out)
elif line.startswith('element'):
vars = line.split()
e_sing = vars[1]
e = singular2plural(e_sing)
size_map[e] = int(float(vars[2]))
type_map[e] = {}
metadata['element_order'].append(e)
metadata['property_order'][e] = []
obj[e] = []
elif line.startswith('property'):
vars = line.split()
p = vars[-1]
type_map[e][p] = ' '.join(vars[1:-1])
metadata['property_order'][e].append(p)
elif 'end_header' in line:
headline = i + 1
break
# Parse body
i = headline
for e in metadata['element_order']:
if e == 'material':
continue
for ie in range(size_map[e]):
vars = lines[i].split()
iv = 0
new = {}
for p in metadata['property_order'][e]:
if type_map[e][p].startswith('list'):
type_vars = type_map[e][p].split()
count_type = translate_ply2py(type_vars[1])
plist_type = translate_ply2py(type_vars[2])
count = count_type(vars[iv])
plist = []
iv += 1
for ip in range(count):
plist.append(plist_type(vars[iv]))
iv += 1
new[p] = plist
else:
prop_type = translate_ply2py(type_map[e][p])
new[p] = prop_type(vars[iv])
iv += 1
assert(iv == len(vars))
obj[e].append(new)
i += 1
# Check that all properties filled in
for e in metadata['element_order']:
if e not in metadata['property_order']:
continue
for p in metadata['property_order'][e]:
assert(len(obj[e]) == size_map[e])
# Return
return PlyDict(obj)
def discover_header(fd,
serializer,
newline=_default_newline,
comment=_default_comment,
delimiter=None,
lineno_format=None,
lineno_names=None,
lineno_units=None,
use_astropy=False):
r"""Discover ASCII table header info from a file.
Args:
fd (file): File object containing the table.
serializer (DefaultSerialize): Serializer that should be updated with
header information.
newline (str, optional): Newline character that should be used to split
header if it is not already a list. Defaults to _default_newline.
comment (bytes, optional): String that should be used to mark the
header lines. If not provided and not in format_str, defaults to
_default_comment.
delimiter (bytes, optional): String that should be used to separate
columns. If not provided and not in format_str, defaults to
_default_delimiter.
lineno_format (int, optional): Line number where formats are located.
If not provided, an attempt will be made to locate one.
lineno_names (int, optional): Line number where field names are located.
If not provided, an attempt will be made to locate one.
lineno_units (int, optional): Line number where field units are located.
If not provided, an attempt will be made to locate one.
use_astropy (bool, optional): If True, astropy will be used to parse
the table if it is installed. Defaults to False.
"""
header_lines = []
header_size = 0
prev_pos = fd.tell()
for line in fd:
sline = line.replace(platform._newline, newline)
if not sline.startswith(comment):
break
header_size += len(line)
header_lines.append(sline)
# Parse header & set serializer attributes
header = parse_header(header_lines,
newline=newline,
lineno_format=lineno_format,
lineno_names=lineno_names,
lineno_units=lineno_units)
# Override header with information set explicitly in serializer
for k in serializer._oldstyle_kws:
v = getattr(serializer, k, None)
if v is not None:
header[k] = v
header.setdefault('format_str', None)
if (delimiter is None) or ('format_str' in header):
delimiter = header['delimiter']
# Try to determine format from array without header
str_fmt = b'%s'
if ((header['format_str'] is None) or (str_fmt in header['format_str'])):
fd.seek(prev_pos + header_size)
all_contents = fd.read()
if len(all_contents) == 0: # pragma: debug
return # In case the file has not been written
arr = table_to_array(all_contents,
names=header.get('field_names', None),
comment=comment,
delimiter=delimiter,
use_astropy=use_astropy)
header['field_names'] = arr.dtype.names
# Get format from array
if header['format_str'] is None:
header['format_str'] = table2format(arr.dtype,
delimiter=delimiter,
comment=b'',
newline=header['newline'])
# Determine maximum size of string field
while str_fmt in header['format_str']:
field_formats = extract_formats(header['format_str'])
ifld = tools.bytes2str(
header['field_names'][field_formats.index(str_fmt)])
max_len = len(max(arr[ifld], key=len))
new_str_fmt = b'%%%ds' % max_len
header['format_str'] = header['format_str'].replace(
str_fmt, new_str_fmt, 1)
# Update serializer
serializer.initialize_serializer(header)
# Seek to just after the header
fd.seek(prev_pos + header_size)
def table_to_array(msg,
fmt_str=None,
use_astropy=False,
names=None,
delimiter=None,
comment=None,
encoding='utf-8'):
r"""Extract information from an ASCII table as an array.
Args:
msg (bytes): ASCII table as bytes string.
fmt_str (bytes): Format string that should be used to parse the table.
If not provided, this will attempt to determine the types of columns
based on their contents.
use_astropy (bool, optional): If True, astropy will be used to parse
the table if it is installed. Defaults to False.
names (list, optional): Field names that should be used for the
structured data type of the output array. If not provided, names
are generated based on the order of the fields in the table.
delimiter (str, optional): String used to separate columns. Defaults to
None and is not used. This is only used if fmt_str is not provided.
comment (str, optional): String used to denote comments. Defaults to
None and is not used. This is only used if fmt_str is not provided.
encoding (str, optional): Encoding that should be used in Python 3 or
higher to extract information from the message. Defaults to 'utf-8'.
Returns:
np.ndarray: Table contents as an array.
"""
if not _use_astropy:
use_astropy = False
if fmt_str is None:
dtype = None
info = dict(delimiter=delimiter, comment=comment)
else:
dtype = cformat2nptype(fmt_str, names=names)
info = format2table(fmt_str)
names = dtype.names
fd = sio.BytesIO(msg)
if names is not None:
names = tools.bytes2str(names, recurse=True)
np_kws = dict()
if info.get('delimiter', None) is not None:
np_kws['delimiter'] = info['delimiter']
if info.get('comment', None) is not None:
np_kws['comments'] = info['comment']
np_kws = tools.bytes2str(np_kws, recurse=True)
if use_astropy:
if 'comments' in np_kws:
np_kws['comment'] = np_kws.pop('comments')
tab = apy_ascii.read(fd,
names=names,
guess=True,
encoding=encoding,
format='no_header',
**np_kws)
arr = tab.as_array()
typs = [arr.dtype[i].str for i in range(len(arr.dtype))]
cols = [c for c in tab.columns]
# Convert type bytes if python 3
new_typs = copy.copy(typs)
convert = []
for i in range(len(arr.dtype)):
if np.issubdtype(arr.dtype[i], np.dtype('U')):
new_typs[i] = 'S' + typs[i].split('U')[-1]
convert.append(i)
if convert:
old_arr = arr
new_dtyp = np.dtype([(c, t) for c, t in zip(cols, new_typs)])
new_arr = np.zeros(arr.shape, new_dtyp)
for i in range(len(arr.dtype)):
if i in convert:
x = np.char.encode(old_arr[cols[i]], encoding='utf-8')
new_arr[cols[i]] = x
else:
new_arr[cols[i]] = old_arr[cols[i]]
arr = new_arr
typs = new_typs
# Convert complex type
for i in range(len(arr.dtype)):
if np.issubdtype(arr.dtype[i], np.dtype('S')):
new_typs = copy.copy(typs)
new_typs[i] = 'complex'
new_dtyp = np.dtype([(c, t) for c, t in zip(cols, new_typs)])
try:
arr = arr.astype(new_dtyp)
except ValueError:
pass
if dtype is not None:
arr = arr.astype(dtype)
else:
np_ver = tuple([float(x) for x in (np.__version__).split('.')])
np_kws.update(autostrip=True, dtype=None, names=names)
if (np_ver >= (1.0, 14.0, 0.0)):
np_kws['encoding'] = 'bytes'
arr = np.genfromtxt(fd, **np_kws)
if dtype is not None:
arr = arr.astype(dtype)
fd.close()
return arr
def cformat2nptype(cfmt, names=None):
r"""Convert a c format string to a numpy data type.
Args:
cfmt (str, bytes): c format that should be translated.
names (list, optional): Names that should be assigned to fields in the
format string if there is more than one. If not provided, names
are generated based on the order of the format codes.
Returns:
np.dtype: Corresponding numpy data type.
Raises:
TypeError: if cfmt is not a string.
ValueError: If the c format does not begin with '%'.
ValueError: If the c format does not contain type info.
ValueError: If the c format cannot be translated to a numpy datatype.
"""
# TODO: this may fail on 32bit systems where C long types are 32 bit
if not (isinstance(cfmt, list) or isinstance(cfmt, (str, bytes))):
raise TypeError(
"Input must be a string, bytes string, or list, not %s" %
type(cfmt))
if isinstance(cfmt, (str, bytes)):
cfmt = tools.bytes2str(cfmt)
fmt_list = extract_formats(cfmt)
if len(fmt_list) == 0:
raise ValueError("Could not locate any format codes in the " +
"provided format string (%s)." % cfmt)
else:
fmt_list = cfmt
nfmt = len(fmt_list)
if nfmt == 1:
cfmt_str = fmt_list[0]
else:
dtype_list = [cformat2nptype(f) for f in fmt_list]
if names is None:
names = ['f%d' % i for i in range(nfmt)]
elif len(names) != nfmt:
raise ValueError(
"Number of names does not match the number of fields.")
else:
names = tools.bytes2str(names, recurse=True)
out = np.dtype(dict(names=names, formats=dtype_list))
# out = np.dtype([(n, d) for n, d in zip(names, dtype_list)])
return out
out = None
if cfmt_str[-1] in ['j']:
out = 'complex128'
elif cfmt_str[-1] in ['f', 'F', 'e', 'E', 'g', 'G']:
# if 'hh' in cfmt_str:
# out = 'float8'
# elif cfmt_str[-2] == 'h':
# out = 'float16'
# elif 'll' in cfmt_str:
# out = 'longfloat'
# elif cfmt_str[-2] == 'l':
# out = 'double'
# else:
# out = 'single'
out = 'float64'
elif cfmt_str[-1] in ['d', 'i']:
if 'hh' in cfmt_str: # short short, single char
out = 'int8'
elif cfmt_str[-2] == 'h': # short
out = 'short'
elif ('ll' in cfmt_str) or ('l64' in cfmt_str):
out = 'longlong' # long long
elif cfmt_str[-2] == 'l':
out = 'int_' # long (broken in python)
else:
out = 'intc' # int, platform dependent
elif cfmt_str[-1] in ['u', 'o', 'x', 'X']:
if 'hh' in cfmt_str: # short short, single char
out = 'uint8'
elif cfmt_str[-2] == 'h': # short
out = 'ushort'
elif ('ll' in cfmt_str) or ('l64' in cfmt_str):
out = 'ulonglong' # long long
elif cfmt_str[-2] == 'l':
out = 'uint64' # long (broken in python)
else:
out = 'uintc' # int, platform dependent
elif cfmt_str[-1] in ['c', 's']:
lstr = cfmt_str[1:-1]
if lstr:
lint = int(lstr)
else:
lint = 0
lsiz = lint * np.dtype('S1').itemsize
out = 'S%d' % lsiz
else:
raise ValueError("Could not find match for format str %s" % cfmt)
return np.dtype(out)
def update_typedef_from_oldstyle(self, typedef):
r"""Update a given typedef using an old, table-style serialization spec.
Existing typedef values are not overwritten and warnings are raised if the
provided serialization spec is not compatible with the type definition.
Args:
typedef (dict): Type definition to update.
Returns:
dict: Updated typedef.
"""
for k in self._oldstyle_kws:
used = []
updated = []
v = self.extra_kwargs.get(k, getattr(self, k, None))
if v is None:
continue
# Check status
if ((k != 'format_str')
and (typedef.get('type', None) != 'array')):
continue
# Key specific changes to type
if k == 'format_str':
v = tools.bytes2str(v)
fmts = serialize.extract_formats(v)
if 'type' in typedef:
if (typedef.get('type', None) == 'array'):
assert (len(typedef.get('items', [])) == len(fmts))
# This continue is covered, but the optimization
# causes it to be skipped at runtime
# https://bitbucket.org/ned/coveragepy/issues/198/
# continue-marked-as-not-covered
continue # pragma: no cover
as_array = self.extra_kwargs.get(
'as_array', getattr(self, 'as_array', False))
typedef.update(type='array', items=[])
for i, fmt in enumerate(fmts):
nptype = self.cformat2nptype(fmt)
itype = OneDArrayMetaschemaType.encode_type(
np.ones(1, nptype))
itype = OneDArrayMetaschemaType.extract_typedef(itype)
if (fmt == '%s') and ('precision' in itype):
del itype['precision']
if as_array:
itype['type'] = '1darray'
else:
itype['type'] = itype.pop('subtype')
if (((itype['type'] in _flexible_types)
and ('precision' in itype))):
del itype['precision']
typedef['items'].append(itype)
used.append('as_array')
updated.append('format_str')
elif k == 'as_array':
# Can only be used in conjunction with format_str
pass
elif k in ['field_names', 'field_units']:
v = tools.bytes2str(v, recurse=True)
if k == 'field_names':
tk = 'title'
else:
tk = 'units'
if isinstance(typedef['items'], dict):
typedef['items'] = [
copy.deepcopy(typedef['items']) for _ in range(len(v))
]
assert (len(v) == len(typedef.get('items', [])))
# if len(v) != len(typedef.get('items', [])):
# warnings.warn('%d %ss provided, but only %d items in typedef.'
# % (len(v), k, len(typedef.get('items', []))))
# continue
all_updated = True
for iv, itype in zip(v, typedef.get('items', [])):
if tk in itype:
all_updated = False
itype.setdefault(tk, iv)
if all_updated:
used.append(k)
updated.append(
k) # Won't change anything unless its an attribute
else: # pragma: debug
raise ValueError(
"Unrecognized table-style specification keyword: '%s'." %
k)
for rk in used:
if rk in self.extra_kwargs:
del self.extra_kwargs[rk]
for rk in updated:
if rk in self.extra_kwargs:
self.extra_kwargs[rk] = v
elif hasattr(self, rk):
setattr(self, rk, v)
return typedef
def ygginfo():
r"""Print information about yggdrasil installation."""
from yggdrasil import __version__, tools, config, platform
from yggdrasil.components import import_component
lang_list = tools.get_installed_lang()
prefix = ' '
curr_prefix = ''
vardict = [
('Location', os.path.dirname(__file__)), ('Version', __version__),
('Languages', ', '.join(lang_list)),
('Communication Mechanisms', ', '.join(tools.get_installed_comm())),
('Default Comm Mechanism', tools.get_default_comm()),
('Config File', config.usr_config_file)
]
parser = argparse.ArgumentParser(
description=
'Display information about the current yggdrasil installation.')
parser.add_argument(
'--no-languages',
action='store_true',
dest='no_languages',
help='Don\'t print information about individual languages.')
parser.add_argument(
'--verbose',
action='store_true',
help='Increase the verbosity of the printed information.')
args = parser.parse_args()
try:
# Add language information
if not args.no_languages:
# Install languages
vardict.append(('Installed Languages:', ''))
curr_prefix += prefix
for lang in sorted(lang_list):
drv = import_component('model', lang)
vardict.append((curr_prefix + '%s:' % lang.upper(), ''))
curr_prefix += prefix
if lang == 'executable':
vardict.append((curr_prefix + 'Location', ''))
else:
exec_name = drv.language_executable()
if not os.path.isabs(exec_name):
exec_name = tools.which(exec_name)
vardict.append((curr_prefix + 'Location', exec_name))
vardict.append(
(curr_prefix + 'Version', drv.language_version()))
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
# Not installed languages
vardict.append(("Languages Not Installed:", ''))
curr_prefix += prefix
for lang in tools.get_supported_lang():
if lang in lang_list:
continue
drv = import_component('model', lang)
vardict.append((curr_prefix + '%s:' % lang.upper(), ''))
curr_prefix += prefix
vardict.append((curr_prefix + "Language Installed",
drv.is_language_installed()))
vardict.append((curr_prefix + "Base Languages Installed",
drv.are_base_languages_installed()))
if not drv.are_base_languages_installed():
vardict.append(
(curr_prefix + "Base Languages Not Installed", [
b for b in drv.base_languages if
(not import_component('model', b).is_installed())
]))
vardict.append((curr_prefix + "Dependencies Installed",
drv.are_dependencies_installed()))
vardict.append((curr_prefix + "Interface Installed",
drv.is_interface_installed()))
vardict.append(
(curr_prefix + "Comm Installed", drv.is_comm_installed()))
vardict.append(
(curr_prefix + "Configured", drv.is_configured()))
vardict.append((curr_prefix + "Disabled", drv.is_disabled()))
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
# Add verbose information
if args.verbose:
# Conda info
if os.environ.get('CONDA_PREFIX', ''):
out = tools.bytes2str(
subprocess.check_output(['conda', 'info'])).strip()
curr_prefix += prefix
vardict.append((curr_prefix + 'Conda Info:',
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
# R and reticulate info
Rdrv = import_component("model", "R")
if Rdrv.is_installed():
env_reticulate = copy.deepcopy(os.environ)
env_reticulate['RETICULATE_PYTHON'] = sys.executable
# Stack size
out = Rdrv.run_executable(["-e", "Cstack_info()"]).strip()
vardict.append((curr_prefix + "R Cstack_info:",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
# Compilation tools
interp = 'R'.join(Rdrv.get_interpreter().rsplit('Rscript', 1))
vardict.append((curr_prefix + "R C Compiler:", ""))
curr_prefix += prefix
for x in ['CC', 'CFLAGS', 'CXX', 'CXXFLAGS']:
out = tools.bytes2str(
subprocess.check_output([interp, 'CMD', 'config',
x])).strip()
vardict.append((curr_prefix + x,
"%s" % ("\n" + curr_prefix + prefix).join(
out.splitlines(False))))
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
# Session info
out = Rdrv.run_executable(["-e", "sessionInfo()"]).strip()
vardict.append((curr_prefix + "R sessionInfo:",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
# Reticulate conda_list
if os.environ.get('CONDA_PREFIX', ''):
out = Rdrv.run_executable([
"-e",
("library(reticulate); "
"reticulate::conda_list()")
],
env=env_reticulate).strip()
vardict.append(
(curr_prefix + "R reticulate::conda_list():",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
# Windows python versions
if platform._is_win: # pragma: windows
out = Rdrv.run_executable([
"-e",
("library(reticulate); "
"reticulate::py_versions_windows()")
],
env=env_reticulate).strip()
vardict.append(
(curr_prefix + "R reticulate::py_versions_windows():",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
# conda_binary
if platform._is_win: # pragma: windows
out = Rdrv.run_executable([
"-e",
("library(reticulate); "
"conda <- reticulate:::conda_binary(\"auto\"); "
"system(paste(conda, \"info --json\"))")
],
env=env_reticulate).strip()
vardict.append(
(curr_prefix + "R reticulate::py_versions_windows():",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
# Reticulate py_config
out = Rdrv.run_executable([
"-e", ("library(reticulate); "
"reticulate::py_config()")
],
env=env_reticulate).strip()
vardict.append((curr_prefix + "R reticulate::py_config():",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
finally:
# Print things
max_len = max(len(x[0]) for x in vardict)
lines = []
line_format = '%-' + str(max_len) + 's' + prefix + '%s'
for k, v in vardict:
lines.append(line_format % (k, v))
logger.info("yggdrasil info:\n%s" % '\n'.join(lines))
