def _classify_db(self):
# create output channels
self.gdb.delete_channel(self.outClass)
self.gdb.new_channel(self.outClass, np.int32)
self.gdb.delete_channel(self.outDist)
self.gdb.new_channel(self.outDist, np.float64)
# if filter, add filter to the data
if self.ch_filter[0]:
self.fields.append(self.ch_filter[0])
# put results back in the database
lines = self.gdb.list_lines(select=True)
nl = len(lines)
n = 0
for l in lines:
ln, lsymb = self.gdb.line_name_symb(l)
data, ch, fid = self.gdb.read_line(l, channels=self.fields)
dummy = gxu.gx_dummy(data.dtype)
# mask will hold True for data to be removed from output
mask = np.apply_along_axis(lambda a: dummy in a, 1, data)
if self.ch_filter[0]:
filt = data[:, -1] != self.ch_filter[1]
mask += filt
data = data[:, :-1] # remove filter channel
data[mask] = 0.0
self._apply_norms(data)
classes = self.som.som.reshape(-1, data.shape[1])
clss = classify_data(classes, data, similarity=self._sim)
err = euclidean_distance(classes[clss], data)
clss[mask] = gxu.gx_dummy(clss.dtype)
err[mask] = gxu.gx_dummy(err.dtype)
self.gdb.write_channel(lsymb, self.outClass, clss, fid=fid)
self.gdb.write_channel(lsymb, self.outDist, err, fid=fid)
n += 1
self.progress('Writing line {}'.format(ln), (n * 100.0) / nl)
if self.stop():
raise MvarException("Stop requested")
if self.ch_filter[0]:
self.fields = self.fields[:-1]
def test_misc(self):
self.start(gsys.func_name())
self.assertEqual(gxu.__version__, geosoft.__version__)
self.assertEqual(gxu.gx_dtype('float'), gxapi.GS_DOUBLE)
self.assertEqual(gxu.gx_dtype('int'), gxapi.GS_LONG)
self.assertEqual(gxu.gx_dtype("<U18"), -18)
self.assertEqual(gxu.gx_dtype('uint64'), gxapi.GS_ULONG64)
self.assertEqual(gxu.dtype_gx(gxapi.GS_DOUBLE), np.float)
self.assertEqual(gxu.dtype_gx(gxapi.GS_FLOAT), np.float32)
self.assertEqual(gxu.dtype_gx(gxapi.GS_LONG), np.int32)
self.assertEqual(gxu.dtype_gx(-2000).str, "<U2000")
self.assertEqual(gxu.dtype_gx(gxapi.GS_TYPE_DEFAULT), None)
self.assertEqual(gxu.dtype_gx(gxapi.GS_ULONG64), np.uint64)
self.assertEqual(gxu.gx_dummy(np.float), gxapi.rDUMMY)
self.assertEqual(gxu.gx_dummy(np.int32), gxapi.iDUMMY)
self.assertEqual(gxu.gx_dummy(np.int64), gxapi.GS_S8DM)
self.assertEqual(gxu.gx_dummy(np.str_), '')
self.assertEqual(gxu.gx_dummy('U48'), '')
self.assertTrue(gxu.is_float(gxu.gx_dtype('float')))
self.assertFalse(gxu.is_int(gxu.gx_dtype('float')))
self.assertTrue(gxu.is_int(gxu.gx_dtype('uint64')))
self.assertFalse(gxu.is_float(gxu.gx_dtype('uint64')))
self.assertTrue(gxu.is_string(gxu.gx_dtype('U18')))
self.assertFalse(gxu.is_int(gxu.gx_dtype('U18')))
self.assertFalse(gxu.is_float(gxu.gx_dtype('U18')))
self.assertEqual(gxu.is_string(gxu.gx_dtype('U18')), 18)
npd = np.array([[1, 1], [2, 2], [-127, 1], [3, 3]],
dtype=gxu.dtype_gx(gxapi.GS_BYTE))
self.assertEqual(list(gxu.dummy_mask(npd)),
[False, False, True, False])
npd = np.array([1, 2, 3, 4], dtype=gxu.dtype_gx(gxapi.GS_BYTE))
try:
gxu.dummy_mask(npd)
self.assertTrue(False)
except:
pass
def test_dummy_GDB(self):
self.start(gsys.func_name())
with gxgdb.GXdb.open(self.gdb_name) as gdb:
gdb.delete_channel('test')
gdb.new_channel('test', dtype=np.int)
dummy = gxu.gx_dummy(np.int)
gdb.write_channel('D590875', 'test', np.array([1, 2, dummy, 4]))
npd, ch, fid = gdb.read_line('D590875',
channels=['test'],
dtype=np.int)
self.assertEqual(npd.shape, (4, 1))
self.assertEqual(npd[:, 0].tolist(), [1, 2, dummy, 4])
dm = gxu.dummy_mask(npd)
self.assertEqual(dm.tolist(), [False, False, True, False])
gdb.discard()
def test_misc(self):
self.start()
self.assertEqual(gxu.__version__, geosoft.__version__)
self.assertEqual(gxu.gx_dtype('float'), gxapi.GS_DOUBLE)
self.assertEqual(gxu.gx_dtype('int'), gxapi.GS_LONG)
self.assertEqual(gxu.gx_dtype("<U18"),
-72) # x4 for full range of UTF-8 characters
self.assertEqual(gxu.gx_dtype('uint64'), gxapi.GS_ULONG64)
self.assertEqual(gxu.dtype_gx(gxapi.GS_DOUBLE), np.float)
self.assertEqual(gxu.dtype_gx(gxapi.GS_FLOAT), np.float32)
self.assertEqual(gxu.dtype_gx(gxapi.GS_LONG), np.int32)
self.assertEqual(gxu.dtype_gx(-2000).str, "<U2000")
self.assertEqual(gxu.dtype_gx(gxapi.GS_ULONG64), np.uint64)
self.assertEqual(gxu.gx_dummy(np.float), gxapi.rDUMMY)
self.assertEqual(gxu.gx_dummy(np.int32), gxapi.iDUMMY)
self.assertEqual(gxu.gx_dummy(np.int64), gxapi.GS_S8DM)
self.assertEqual(gxu.gx_dummy(np.str_), '')
self.assertEqual(gxu.gx_dummy('U48'), '')
self.assertEqual(gxu.gx_dummy(np.uint), gxapi.GS_U4DM)
self.assertEqual(gxu.gx_dummy(np.uint8), gxapi.GS_U1DM)
self.assertEqual(gxu.gx_dummy(np.uint16), gxapi.GS_U2DM)
self.assertEqual(gxu.gx_dummy(np.uint32), gxapi.GS_U4DM)
self.assertEqual(gxu.gx_dummy(np.uint64), gxapi.GS_U8DM)
self.assertEqual(gxu.gx_dummy(1.5), gxapi.rDUMMY)
self.assertEqual(gxu.gx_dummy(type(1.5)), gxapi.rDUMMY)
self.assertEqual(gxu.gx_dummy(3), gxapi.iDUMMY)
self.assertEqual(gxu.gx_dummy(type(3)), gxapi.iDUMMY)
self.assertEqual(gxu.gx_dummy(0xff), gxapi.iDUMMY)
self.assertEqual(gxu.gx_dummy('string'), '')
self.assertEqual(gxu.gx_dummy('U48'), '')
self.assertRaises(KeyError, gxu.gx_dummy, 1j)
self.assertRaises(KeyError, gxu.gx_dummy, type(1j))
self.assertEqual(gxu.dummy_none(0), 0)
self.assertEqual(gxu.dummy_none(1.), 1.)
self.assertEqual(gxu.dummy_none(gxapi.iDUMMY), None)
self.assertEqual(gxu.dummy_none(gxapi.rDUMMY), None)
self.assertTrue(gxu.is_float(gxu.gx_dtype('float')))
self.assertFalse(gxu.is_int(gxu.gx_dtype('float')))
self.assertTrue(gxu.is_int(gxu.gx_dtype('uint64')))
self.assertFalse(gxu.is_float(gxu.gx_dtype('uint64')))
self.assertTrue(gxu.is_string(gxu.gx_dtype('U18')))
self.assertFalse(gxu.is_int(gxu.gx_dtype('U18')))
self.assertFalse(gxu.is_float(gxu.gx_dtype('U18')))
self.assertEqual(gxu.is_string(gxu.gx_dtype('U18')), 72)
self.assertEqual(gxu.dtype_gx_dimension(gxapi.GS_FLOAT),
(np.float32, 1))
self.assertEqual(gxu.dtype_gx_dimension(gxapi.GS_FLOAT2D),
(np.float32, 2))
self.assertEqual(gxu.dtype_gx_dimension(gxapi.GS_FLOAT3D),
(np.float32, 3))
self.assertEqual(gxu.dtype_gx_dimension(gxapi.GS_DOUBLE),
(np.float64, 1))
self.assertEqual(gxu.dtype_gx_dimension(gxapi.GS_DOUBLE2D),
(np.float64, 2))
self.assertEqual(gxu.dtype_gx_dimension(gxapi.GS_DOUBLE3D),
(np.float64, 3))
self.assertEqual(gxu.gx_dtype_dimension(np.float32), gxapi.GS_FLOAT)
self.assertEqual(gxu.gx_dtype_dimension(np.float32, 1), gxapi.GS_FLOAT)
self.assertEqual(gxu.gx_dtype_dimension(np.float32, 2),
gxapi.GS_FLOAT2D)
self.assertEqual(gxu.gx_dtype_dimension(np.float32, 3),
gxapi.GS_FLOAT3D)
self.assertEqual(gxu.gx_dtype_dimension(np.float64), gxapi.GS_DOUBLE)
self.assertEqual(gxu.gx_dtype_dimension(np.float64, 1),
gxapi.GS_DOUBLE)
self.assertEqual(gxu.gx_dtype_dimension(np.float64, 2),
gxapi.GS_DOUBLE2D)
self.assertEqual(gxu.gx_dtype_dimension(np.float64, 3),
gxapi.GS_DOUBLE3D)
self.assertRaises(gxu.UtilityException, gxu.gx_dtype_dimension, np.int,
2)
self.assertRaises(gxu.UtilityException, gxu.gx_dtype_dimension, np.int,
3)
self.assertRaises(gxu.UtilityException, gxu.gx_dtype_dimension,
np.float32, 4)
npd = np.array([[1, 1], [2, 2], [-127, 1], [3, 3]],
dtype=gxu.dtype_gx(gxapi.GS_BYTE))
self.assertEqual(list(gxu.dummy_mask(npd)),
[False, False, True, False])
npd = np.array([1, 2, 3, 4], dtype=gxu.dtype_gx(gxapi.GS_BYTE))
try:
gxu.dummy_mask(npd)
self.assertTrue(False)
except:
pass
def test_np(self):
self.start()
fid = (99, 0.1)
npdata = np.array(range(45)).reshape((9, 5))
with gxva.GXva(npdata, fid=fid) as va:
self.assertEqual(va.fid, fid)
self.assertEqual(va.length, npdata.shape[0])
self.assertEqual(va.width, npdata.shape[1])
np2 = va.get_data(va.dtype)
self.assertEqual(np2[0].shape, npdata.shape)
np2, fid2 = va.get_data(dtype=va.dtype, start=1)
self.assertEqual(fid2, (99.1, .1))
self.assertEqual(np2.shape, (8, 5))
self.assertEqual(va.get_data(start=6)[0].shape, (3, 5))
try:
self.assertEqual(
va.get_data(dtype=va.dtype, start=50)[0].shape, (0, ))
self.assertTrue(False)
except gxva.VAException:
pass
np3, fid3 = va.get_data(np.int)
self.assertEqual(fid3, fid)
self.assertEqual(np3[0, 0], 0)
self.assertEqual(np3[1, 4], 9)
np3, fid3 = va.get_data(np.float64)
self.assertEqual(fid3, fid)
self.assertEqual(np3[0, 0], 0.0)
self.assertEqual(np3[1, 4], 9.0)
np3, fid3 = va.get_data(np.float64, n=2)
self.assertEqual(fid3, fid)
self.assertEqual(np3.shape[0], 2)
self.assertEqual(np3[0, 0], 0.0)
self.assertEqual(np3[1, 4], 9.0)
np3, fid3 = va.get_data(np.float64, n=99)
self.assertEqual(fid3, fid)
self.assertEqual(np3.shape[0], va.length)
np3, fid3 = va.get_data(np.float64, n_col=3)
self.assertEqual(fid3, fid)
self.assertEqual(np3.shape[1], 3)
np3, fid3 = va.get_data(np.float64, n_col=99)
self.assertEqual(fid3, fid)
self.assertEqual(np3.shape[1], va.width)
npdata = np.array(range(64), dtype=np.int).reshape(4, 16)
npdata[1, 2] = gxapi.iDUMMY
with gxva.GXva(npdata, fid=fid) as va:
np3, fid = va.get_data(dtype=np.int64)
self.assertEqual(np3[0, 0], 0.)
self.assertEqual(np3[2, 11], 43)
self.assertEqual(np3[1, 2], gxapi.GS_S8DM)
np3, fid = va.get_data(dtype=np.int32)
self.assertEqual(np3[0, 0], 0.)
self.assertEqual(np3[2, 11], 43)
self.assertEqual(np3[1, 2], gxapi.GS_S4DM)
self.assertEqual(np3[1, 2], gxapi.iDUMMY)
np3, fid = va.get_data(np.float)
self.assertEqual(np3[0, 0], 0.)
self.assertEqual(np3[2, 11], 43.)
self.assertTrue(np.isnan(np3[1, 2]))
d = np.array(range(32), dtype=np.int).reshape(-1, va.width)
d[0, 3] = gxu.gx_dummy(d.dtype)
va.set_data(d)
np3, fid = va.get_data(dtype=np.int32)
self.assertEqual(np3.shape[0], 2)
self.assertEqual(np3[0, 0], 0)
self.assertEqual(np3[1, 15], 31)
self.assertEqual(np3[0, 3], gxapi.GS_S4DM)
def rungx():
# api version
gxpy.utility.check_version('9.2')
# get the current database
db = gxpy.gdb.Geosoft_gdb.open()
# project parameters
group = 'CHANADD'
p_chan = 'CHANNEL'
p_addval = 'ADDVAL'
# get previous parameters from the parameter block, initializing to start-up defaults '' and 0.0
parms = gxu.get_parameters(group, {p_chan: '', p_addval: 0.0})
# if interactive, get user input
if not gxprj.running_script():
try:
# get channel to process from list of database channels
chan = gxprj.get_user_input(
'Channel to process',
'Channel:',
kind='list',
default=parms.get(p_chan),
items=sorted([k for k in db.list_channels().keys()]))
# value to add to the channel
addval = gxprj.get_user_input(
'Value to add to the data',
'value to add:',
kind='float',
default=parms.get(p_addval))
except gxprj.ProjectException:
exit()
# save parameters to new user settings
parms[p_chan] = chan
parms[p_addval] = addval
gxu.save_parameters(group, parms)
# work through the data a line at a time - get a list of selected lines
lines = db.list_lines()
# for each line, get the data, add a value, return the data to the line
for l in lines:
# print to the console to reflect progress
print('line {}...'.format(str(l)))
# get the data and determine the dummy to the data type
data, ch, fid = db.read_line(l, channels=chan)
dummy = gxu.gx_dummy(data.dtype)
# make a dummy mask so we can replace dummies after processing
dMask = gxu.dummy_mask(data)
# process - add the value, then replace the dummies
sum = data + addval
sum[dMask] = dummy
# write the data back to the database
db.write_channel(l, chan, sum, fid)
# pause the console so user can review
input("Press return to continue...")