def numberToPhonebookTuple(self, nstring):
"""
Returns a phonebook tuple.
"""
if type(nstring) != types.StringType():
# even though we set +CSCS="UCS2" (modem charset), the name is always encoded in text format, not PDU.
nstring = nstring.encode("iso-8859-1")
if nstring[0] == '+':
return nstring[1:], 145
else:
return nstring, 129
def unicodeToString(uni):
#=========================================================================#
"""
Returns an iso-8859-1 string for a text given to the modem.
"""
if type(uni) == types.StringType():
return uni
else:
try:
result = uni.encode("iso-8859-1") # as set via +CSCS
except UnicodeEncodeError:
result = "<??? unencodable ???>"
# log warning
return result
def _get_terminal_size_unixlike():
import fcntl
import termios
# this try/except block detects and works around a Py2.7.5 bug with
# passing a unicode value as the first arg to struct methods
try:
struct.pack('HH', 0, 0)
except TypeError:
fmt = types.StringType('HHHH')
else:
fmt = 'HHHH'
winsize_struct = struct.pack(fmt, 0, 0, 0, 0)
packed_winsize = fcntl.ioctl(0, termios.TIOCGWINSZ, winsize_struct)
height, width, _, _ = struct.unpack(fmt, packed_winsize)
return width, height
def get_terminal_size(default_width=100, default_height=50):
# this try/except block detects and works around a Py2.7.5 bug with
# passing a unicode value as the first arg to struct methods
try:
struct.pack('HH', 0, 0)
except TypeError:
fmt = types.StringType('HHHH')
else:
fmt = 'HHHH'
winsize_struct = struct.pack(fmt, 0, 0, 0, 0)
try:
packed_winsize = fcntl.ioctl(0, termios.TIOCGWINSZ, winsize_struct)
except IOError:
height, width = (default_height, default_width)
else:
height, width, _, _ = struct.unpack(fmt, packed_winsize)
return width, height
def test_DatetimeField(self):
# TODO: test timezone-aware datetime.date
serialized = types.StringType('2015-05-27T01:02:03')
self.assertIs(fields.DatetimeField.TYPE, datetime.datetime)
deserialized = fields.DatetimeField.deserialize(serialized)
self.assertIs(type(deserialized), fields.DatetimeField.TYPE)
self.assertEqual(fields.DatetimeField.serialize(None), '')
self.assertIs(type(fields.DatetimeField.serialize(None)),
types.UnicodeType)
value = datetime.datetime(2015, 5, 27, 1, 2, 3)
self.assertEqual(fields.DatetimeField.deserialize(serialized), value)
self.assertEqual(fields.DatetimeField.deserialize(deserialized),
deserialized)
self.assertEqual(fields.DatetimeField.deserialize(None), None)
self.assertEqual(fields.DatetimeField.serialize(value), serialized)
self.assertIs(type(fields.DatetimeField.serialize(value)),
types.UnicodeType)
with self.assertRaises(ValueError):
fields.DatetimeField.deserialize(42)
fields.DatetimeField.deserialize('2015-01-01')
def test_DateField(self):
# TODO: test timezone-aware datetime.date
serialized = types.StringType('2015-05-27')
deserialized = datetime.date(2015, 5, 27)
self.assertIs(fields.DateField.TYPE, datetime.date)
self.assertEqual(fields.DateField.serialize(None), '')
self.assertIs(type(fields.DateField.serialize(None)),
types.UnicodeType)
self.assertIs(type(fields.DateField.deserialize(serialized)),
fields.DateField.TYPE)
self.assertEqual(fields.DateField.deserialize(serialized),
deserialized)
self.assertEqual(fields.DateField.deserialize(deserialized),
deserialized)
self.assertEqual(fields.DateField.deserialize(None), None)
self.assertEqual(fields.DateField.serialize(deserialized), serialized)
self.assertIs(type(fields.DateField.serialize(deserialized)),
types.UnicodeType)
with self.assertRaises(ValueError):
fields.DateField.deserialize(42)
with self.assertRaises(ValueError):
fields.DateField.deserialize(serialized + 'T00:00:00')
def get_default_instances():
return [
types.StringType(), types.DecimalType(), types.IntegerType(),
types.DateType('%Y-%m-%d %H:%M:%S'), types.BooleanType()
]
def type_guess(rows, types=types.DEFAULT_TYPES, strict=False):
""" The type guesser aggregates the number of successful
conversions of each column to each type, weights them by a
fixed type priority and select the most probable type for
each column based on that figure. It returns a list of
``CellType``. Empty cells are ignored.
Strict means that a type will not be guessed
if parsing fails for a single cell in the column."""
guesses = []
type_instances = [i for t in types for i in t.instances()]
if strict:
at_least_one_value = []
for ri, row in enumerate(rows):
diff = len(row) - len(guesses)
for _ in range(diff):
typesdict = {}
for type in type_instances:
typesdict[type] = 0
guesses.append(typesdict)
at_least_one_value.append(False)
for ci, cell in enumerate(row):
if not cell.value:
continue
at_least_one_value[ci] = True
for type in list(guesses[ci].keys()):
if not type.test(cell.value):
guesses[ci].pop(type)
# no need to set guessing weights before this
# because we only accept a type if it never fails
for i, guess in enumerate(guesses):
for type in guess:
guesses[i][type] = type.guessing_weight
# in case there were no values at all in the column,
# we just set the guessed type to string
for i, v in enumerate(at_least_one_value):
if not v:
guesses[i] = {types.StringType(): 0}
else:
for i, row in enumerate(rows):
diff = len(row) - len(guesses)
for _ in range(diff):
guesses.append(defaultdict(int))
for i, cell in enumerate(row):
# add string guess so that we have at least one guess
guesses[i][types.StringType()] = guesses[i].get(types.StringType(), 0)
if not cell.value:
continue
for type in type_instances:
if type.test(cell.value):
guesses[i][type] += type.guessing_weight
_columns = []
_columns = []
for guess in guesses:
# this first creates an array of tuples because we want the types to be
# sorted. Even though it is not specified, python chooses the first
# element in case of a tie
# See: http://stackoverflow.com/a/6783101/214950
guesses_tuples = [(t, guess[t]) for t in type_instances if t in guess]
_columns.append(max(guesses_tuples, key=lambda t_n: t_n[1])[0])
return _columns
def as_string(value):
if isinstance(value, types.StringTypes):
return value
else:
return types.StringType(value)
import torch.nn as nn
import torch.nn.functional as F
import sys
import abc
import types
if sys.version_info >= (3, 4):
ABC = abc.ABC
else:
ABC = abc.ABCMeta(types.StringType('ABC'), (), {})
class BaseModel(ABC, nn.Module):
"""
Abstract base class for models.
Args:
encoder_module (baseRNN or baseConv) : module that encodes inputs
decoder_module (baseRNN or baseConv, optional): module to decode encoded inputs
decode_function (callable, optional): function to generate symbols from output hidden states (default: F.log_softmax)
"""
def __init__(self,
encoder_module,
decoder_module=None,
decode_function=F.log_softmax):
super(BaseModel, self).__init__()
self.encoder_module = encoder_module
self.decoder_module = decoder_module
self.decode_function = decode_function
# def flatten_parameters(self):
def load_data(self, directory, file_name):
file_ = types.StringType(os.path.join(directory, file_name))
f = open(file_, "r")
next_ = f.readline().split()
# store the initial information about the coordinates and values listed in the first few lines of the file
coordinates = []
values = []
while next_[0] == "#":
if next_[1] == "Column":
colname = f.readline().split()[2]
coltype = f.readline().split()[2]
if coltype == "coordinate":
coordinates.append({'name': colname, 'type': coltype})
elif coltype == "value":
values.append({'name': colname, 'type': coltype})
else:
raise ValueError(
"Inputs should only be of type coordinate or value")
next_ = f.readline().split()
if len(coordinates) < 1:
f.close()
del coordinates
del values
raise RuntimeError('Data with less than 1 coordinate is currently '
'not supported')
file_lines = f.readlines()
numLinesOfData = len(file_lines)
for i in range(0, numLinesOfData):
# check if this really needs to be there
if len(file_lines[i].split()) == (len(coordinates) + len(values)):
file_lines[i] = file_lines[i].split()
else:
file_lines[i] = [np.nan] * (len(coordinates) + len(values))
file_lines = [next_] + file_lines
data = pd.DataFrame(file_lines)
numLinesOfData += 1
# this is not meaningful for NaN data e.g. labels
# labels should be stored and replaced with indices in the data!
for i in range(0, len(coordinates)):
# NOTE potential precision issues if you evaluate an int as a float
coordinates[i]['start'] = float(data[i].iloc[0])
coordinates[i]['end'] = float(data[i].iloc[-1])
n = 0L
coordinateValues = []
# O(N)
for j in range(0, numLinesOfData):
if data[i].iloc[j] in coordinateValues:
pass
else:
n += 1
coordinateValues.append(data[i].iloc[j])
### is there a better way to do this? Is it worth soring the data
coordinates[i]['size'] = n
# return to the start of the file
# is there a better way to do this?
f.close()
f = open(file_, "r")
# create array of arbitrary shape
size = len(coordinates) + 1
shape = [0] * (size)
for i in range(0, size - 1):
shape[i] = long(coordinates[i]['size'])
shape[size - 1] = long(len(coordinates) + len(values))
shape = tuple(shape)
nd_data = data.as_matrix()
shaped_data = np.reshape(nd_data, shape).astype(float)
#skip through the first couple of lines where they talk about columns
next_ = f.readline().split()
while next_[0] == "#":
next_ = f.readline().split()
# populated shaped_data with the data values
self.coordinates = coordinates
self.shaped_data = shaped_data
self.values = values
if len(coordinates) == 1:
self.coordinates.append(
dict(type='coordinate', name='None', size=1))
self.shaped_data.shape = (self.shaped_data.shape[0], 1,
self.shaped_data.shape[1])
f.close()
del coordinates
del values
del shaped_data
del shape
del next_
self._set_axis_cb_choices()
self._set_value_choices()
self._set_coordinate_choices()
self.data_selection = [0 for __ in self.shaped_data.shape]
if self.value_func is None:
self.draw(self.x_idx, self.y_idx, self.value_idx, recenter=True)
else:
self.draw_function(self.value_func["function"],
self.value_func["values"],
recenter=True)
def setParameter(self,name,field,value):
"""setParameter(name,field,value) -> Set the 'field' of parameter 'name' with 'value'
Returns True if the assigned or False if not"""
if name not in self.parameters:
if self.verbose > 0: print "Parameter '"+name+"' is not valid for learner '"+self.learner
return False
if field not in self.paramFields:
if self.verbose > 0: print "Field '"+field+"' is not valid"
return False
if field == "default":
realValue = value
try:
if type(realValue) in types.StringTypes:
if realValue in self.originalParametersDict[name][self.paramFields["alias"]]:
idx = self.originalParametersDict[name][self.paramFields["alias"]].index(realValue)
realValue = eval(self.originalParametersDict[name][self.paramFields["range"]])[idx]
except:
if self.verbose > 0: print "Field '"+field+"' was not able to be set up via alias!"
realValue = value
try:
if type(eval(self.parametersDict[name][self.paramFields["type"]])) == types.ListType:
self.parametersDict[name][self.paramFields[field]] = eval(self.parametersDict[name][self.paramFields["type"][0]])(realValue)
else:
self.parametersDict[name][self.paramFields[field]] = eval(self.parametersDict[name][self.paramFields["type"]])(realValue)
# special cases where the parameter is defined as string, it cannot have spaces
try:
if eval(self.parametersDict[name][self.paramFields["type"]]) == types.StringType:
self.parametersDict[name][self.paramFields[field]] = self.parametersDict[name][self.paramFields[field]].replace(" ","")
except:
pass
except:
if self.verbose > 0: print "Field '"+field+"' was not able to be set up"
return False
elif (field == "editableRange"):
if self.verbose > 0: print "The field '"+field+"' is read only"
return False
elif (field == "optimize"):
if type(value) == types.BooleanType:
self.parametersDict[name][self.paramFields[field]] = value
elif type(value) == types.IntType and (value == 0 or value == 1):
self.parametersDict[name][self.paramFields[field]] = types.BooleanType(value)
else:
if self.verbose > 0: print "Field '"+field+"' was not able to be set up. Wrong type"
return False
elif (field == "description"):
self.parametersDict[name][self.paramFields[field]] = types.StringType(value)
elif (field == "alias"):
if type(value) == types.ListType:
self.parametersDict[name][self.paramFields[field]] = [str(alias) for alias in values]
else:
if self.verbose > 0: print "Field '"+field+"' was not able to be set up. Wrong type"
return False
elif (field == "type"):
if self.verbose > 0: print "The field '"+field+"' is read only"
return False
elif (field == "rangeType"):
if type(value) in types.StringTypes:
if value.lower() == "values" or value.lower() == "interval":
self.parametersDict[name][self.paramFields[field]] = value.lower()
else:
if self.verbose > 0: print "Field '"+field+"' was not able to be set up. Wrong keyword. Use 'values' or 'interval'"
return False
else:
if self.verbose > 0: print "Field '"+field+"' was not able to be set up. Wrong type"
return False
else: # field == "range"
if type(value) == types.ListType:
self.parametersDict[name][self.paramFields[field]] = str(value).replace(", "," , ")
elif type(value) in types.StringTypes:
self.parametersDict[name][self.paramFields[field]] = value.strip()
else:
if self.verbose > 0: print "Field '"+field+"' was not able to be set up. Wrong type"
return False
return True
