def __wiggle(self):
dprint("__wiggle")
skipt = int(self._params.get("skipt"))
maxval = float(self._params.get("gain"))
maxval = self._norm_amp / maxval
lwidth = float(self._params.get("lwidth"))
import numpy
import copy
data = numpy.transpose(self._data)
data = copy.copy(data)
t = [self._t_start + i for i in range(self._ns)]
for i in range(0, self._ntraces, skipt):
trace = data[i]
trace[0] = 0
trace[self._ns - 1] = 0
for a in range(len(trace)):
if (trace[a] > maxval):
trace[a] = maxval
if (trace[a] < -maxval):
trace[a] = -maxval
self._ax.plot(i + trace / maxval,
t,
color='black',
linewidth=lwidth)
for a in range(len(trace)):
if (trace[a] < 0):
trace[a] = 0
self._ax.fill(i + trace / maxval, t, 'k', linewidth=0)
def do(self, rdd):
dprint("SetKeyByHeader")
rdd = RDD_backToFlat(rdd)
rdd = rdd.map(self._ha.getHeaderKV)
if seisspark_config.debug:
RDD_test("End SetKeyByHeader", rdd)
return rdd
def getSelectedModule(self):
selected_item_key = self.listView.get_key()
dprint('getSelectedModule', selected_item_key)
module = None
if selected_item_key != None:
module_item = self.listView.children[selected_item_key]
module = module_item._module
return module
def do(self, rdd):
if seisspark_config.debug:
dprint("FilterByHeader")
rdd = RDD_backToFlat(rdd)
rdd = rdd.filter(self._hf.filtKV)
if seisspark_config.debug:
RDD_test("End FilterByHeader", rdd)
return rdd
def pipe(self, kv):
import subprocess
assert (type(kv) is tuple)
assert (type(kv[1]) is list) # should be gather
assert (type(kv[1][0]) is bytearray) # should be trace
# if self._p == None:
p = subprocess.Popen(self._args,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
# simplest communication
in_data = concatenateByteArray(kv[1])
out, err = p.communicate(input=in_data)
out_data_array = bytearray(out)
dprint('ERROR STREAM', err)
ns = KV_HeaderAccess('ns').getHeaderV(out_data_array)
bps = 4
trace_len = 240 + bps * ns
trace_count = int(len(out_data_array) / trace_len)
assert (trace_len * trace_count == len(out_data_array))
out_data = []
for i in range(trace_count):
trace = out_data_array[i * trace_len:(i + 1) * trace_len]
out_data.append(trace)
# in_data = kv[1]
# for d in in_data:
# self._p.stdin.write(d) # write one by one
# self._p.stdin.close()
#
# out_data = []
# while True:
# head = bytearray(self._p.stdout.read(240))
# if not head:
# break
#
# head = bytearray(head)
# ns = KV_HeaderAccess('ns').getHeaderV(head)
# bps = 4
#
# body = self._p.stdout.read(ns * bps)
# if not body:
# print('cannot read trace body')
# exit(1)
# body = bytearray(body)
#
# data = head
# head.extend(body)
#
# out_data.append(data)
# TODO optimization sometimes i can use kv[0] as new key
return (None, out_data)
def do(self, rdd):
dprint("RDD_Processing")
##
## TODO why we do not use spark pipe?!!!!
##
rdd = rdd.map(self.pipe)
if seisspark_config.debug:
RDD_test("End RDD_Processing", rdd)
return rdd
def do(self, rdd):
dprint("GroupByHeader")
rdd = RDD_backToFlat(rdd)
rdd = self._sk.do(rdd) # set key
rdd = rdd.groupByKey().mapValues(list)
#rdd = rdd.sortByKey()
if seisspark_config.debug:
RDD_test("End GroupByHeader", rdd)
return rdd
def onattribute_changed(self, widget, widgetAttributeMember, attributeName,
newValue):
dprint(
"setting attribute name: %s value: %s attributeMember: %s" %
(attributeName, newValue, widgetAttributeMember))
self._params.put(attributeName, newValue)
return self.eventManager.propagate(
self.EVENT_ATTRIB_ONCHANGE,
(widgetAttributeMember, attributeName, newValue))
def init(self, appInstance, uName, name, params=Params([])):
import copy
self._appInstance = appInstance
self._uName = uName
self._name = name
self._id = random.random()
self._params = copy.deepcopy(params)
self._outRddCache = None
self._inRddCache = None
dprint('module.init', self._name, 'id', self._id, 'params',
self._params)
def calcCached(self, m, rdd):
if rdd is not m._inRddCache: # input was changed
m._outRddCache = None # drop output
if m._outRddCache != None:
dprint("module", m._name, "used cache")
return m._outRddCache # return out
dprint("module", m._name, "didnt used cache")
m._inRddCache = rdd
m._outRddCache = m.calc(rdd)
return m._outRddCache
def __dense(self):
dprint("__dense")
maxval = float(self._params.get("gain"))
maxval = self._norm_amp / maxval
palette = self._params.get("palette")
self._ax.imshow(self._data,
palette,
aspect='auto',
vmin=-maxval,
vmax=maxval)
def KV_flatList(kv):
assert (type(kv) is tuple)
out = list()
values = kv[1]
if type(values) is list:
for value in values:
out.append((None, value))
else:
if type(values) is not bytearray:
dprint("KV_flatList error: ", type(values))
assert (type(values) is bytearray)
out.append((None, values))
return out
def __createFigure(self):
dprint('plot_w', seisspark_config.plot_w, 'plot_h',
seisspark_config.plot_h)
dprint('image_w', seisspark_config.image_w, 'image_h',
seisspark_config.image_h)
self._fig = Figure(figsize=(seisspark_config.image_w,
seisspark_config.image_h))
self._ax = self._fig.add_subplot(111)
self._fig.gca().invert_yaxis()
self._ax.tick_params(axis='both',
which='major',
labelsize=seisspark_config.plot_label_size)
self._ax.tick_params(axis='both',
which='minor',
labelsize=seisspark_config.plot_label_size)
if seisspark_config.axis_label:
self._ax.set_ylabel('Time')
self._ax.set_xlabel('Trace')
def updateNorm__(self):
array = self._data.reshape(self._ntraces * self._ns)
array = np.absolute(array)
array.sort(axis=0)
percent = float(self._params.get("percent"))
dprint('percent', percent)
norm = int((len(array) - 1) * percent)
dprint('len', norm)
self._norm_amp = array[norm]
dprint('array', array)
dprint('_norm_amp', self._norm_amp)
def setData(self, data, header):
dprint("matplot setData")
self._data = data
self._header = header
self._ntraces = data.shape[1]
self._t_start = 0
first_trace = self._header[0]
self._dt = seisspark.KV_HeaderAccess('dt').getHeaderV(first_trace)
self._ns = seisspark.KV_HeaderAccess('ns').getHeaderV(first_trace)
self.updateNorm__()
dprint("ntraces", self._ntraces)
dprint("t_start", self._t_start)
dprint("dt", self._dt)
dprint("ns", self._ns)
# dprint ('Data type', type(self._data), 'shape', self._data.shape)
# dprint ('Header type', type(self._header), 'len', len(self._header))
self.__redraw()
def prepareRDDtoDraw(rdd, count=None):
dprint("prepareRDDtoDraw")
from numpy import transpose
from numpy import reshape
import struct
rdd = RDD_backToFlat(rdd)
if count == None:
DataFromRDD = rdd.collect()
else:
DataFromRDD = rdd.take(count)
assert (type(DataFromRDD) is list)
assert (type(DataFromRDD[0]) is tuple)
assert (type(DataFromRDD[0][1]) is bytearray)
first_trace = DataFromRDD[0][1]
ns = KV_HeaderAccess('ns').getHeaderV(first_trace)
dt = KV_HeaderAccess('dt').getHeaderV(first_trace)
ntraces = len(DataFromRDD)
bps = 4
ndummy_samples = 240 / bps
number = ntraces * (ns + ndummy_samples)
# concatenate to single bytearray
Data = KV_concatenateByteArray(DataFromRDD)
header = []
for d in DataFromRDD:
header.append(d[1][0:240])
# convert to matrix
Data = struct.unpack(segypy.endian + 'f' * number, Data)
Data = reshape(Data, (ntraces, ns + ndummy_samples))
Data = Data[:, ndummy_samples:(ns + ndummy_samples)]
Data = transpose(Data)
dprint("End prepareRDDtoDraw")
return Data, header
def __createEditor(self):
attributeType = self._param._type
additionalInfo = self._param._additionalInfo
attributeValue = self._param._v
attributeName = self._param._k
attributeDesc = self._param._description
if additionalInfo == None:
additionalInfo = {}
dprint('name', attributeName, 'type', attributeType, 'value',
attributeValue, 'info', additionalInfo)
self.inputWidget = None
#'background-repeat':{'type':str, 'description':'The repeat behaviour of an optional background image', ,'additional_data':{'affected_widget_attribute':'style', 'possible_values':'repeat | repeat-x | repeat-y | no-repeat | inherit'}},
if attributeType == bool or attributeType == 'bool':
if attributeValue == 'true':
attributeValue = True
if attributeValue == 'false':
attributeValue = False
self.inputWidget = gui.CheckBox('checked')
elif attributeType == int or attributeType == float or attributeType == 'int' or attributeType == 'float':
min_val = -1000000
if "min" in additionalInfo:
min_val = additionalInfo['min']
max_val = 1000000
if "max" in additionalInfo:
max_val = additionalInfo['max']
step_val = 1
if "step" in additionalInfo:
step_val = additionalInfo['step']
self.inputWidget = gui.SpinBox(attributeValue, min_val, max_val,
step_val)
elif attributeType == gui.ColorPicker:
self.inputWidget = gui.ColorPicker()
elif attributeType == 'dropdown':
self.inputWidget = gui.DropDown()
for value in additionalInfo['possible_values']:
self.inputWidget.append(gui.DropDownItem(value), value)
# elif attributeType == 'url_editor':
# self.inputWidget = UrlPathInput(self._appInstance)
# elif attributeType == 'css_size':
# self.inputWidget = CssSizeInput(self._appInstance)
else: # default editor is string
self.inputWidget = StringEditor()
self.inputWidget.set_on_change_listener(self.on_attribute_changed)
self.inputWidget.set_size('50%', '22px')
self.inputWidget.attributes['title'] = attributeDesc
self.inputWidget.style['float'] = 'right'
self.inputWidget.set_value(attributeValue)
dprint('setValue', attributeValue)
dprint('getValue', self.inputWidget.get_value())
self.append(self.inputWidget)
def valid(self):
error = None
meta = None
if seisspark_config.handle_exceptions:
for m in self._modules:
dprint("valid", m._name)
try:
meta = m.valid(meta)
except:
error = "Validation of module", m._name, "\nUnexpected error:" + \
str(sys.exc_info()[0])
print(error)
if self._stopOnError:
raise
dprint("valid", m._name, 'key', meta._sort)
else:
for m in self._modules:
dprint("valid", m._name)
meta = m.valid(meta)
dprint("valid", m._name, 'key', meta._sort)
return error
def onattribute_changed(self, widget, widgetAttributeMember, attributeName, newValue):
module = self.getSelectedModule()
dprint('module onattribute_changed', module._name)
module.onParamsChanged()
def moduleSelected(self, widget, selected_item_key):
module_item = self.listView.children[selected_item_key]
dprint('moduleSelected', selected_item_key)
module = module_item._module
if self.paramsWidget != None:
self.paramsWidget.setParams(module._params)
def __init__(self, args):
self._args = args
self._p = None
dprint('RDD_Processing', args)
return
def saveData(rdd, filename):
dprint("saveData")
rdd = RDD_backToFlat(rdd)
rdd.saveAsSequenceFile(filename)
def show(self, *args):
dprint ("EditorFileSelectionDialog.show")
super(EditorFileSaveDialog, self).show(self.baseAppInstance)
def on_saveas_dialog_confirm(self, widget, path):
dprint ('on_saveas_dialog_confirm', path)
if len(path):
filename = path + '/' + self.fileSaveAsDialog.get_fileinput_value()
dprint ('on_saveas_dialog_confirm', filename)
self.save_job(filename)
def setMeta(self, meta):
dprint('meta.sort', self._name, meta._sort)
self._meta = meta
return self._meta
def put(self, k, v):
for p in self._par:
if p._k == k:
dprint('Params::put', p._k, v)
p._v = v
