def __wiggle(self): dprint("__wiggle") skipt = int(self._params.get("skipt")) maxval = float(self._params.get("gain")) maxval = self._norm_amp / maxval lwidth = float(self._params.get("lwidth")) import numpy import copy data = numpy.transpose(self._data) data = copy.copy(data) t = [self._t_start + i for i in range(self._ns)] for i in range(0, self._ntraces, skipt): trace = data[i] trace[0] = 0 trace[self._ns - 1] = 0 for a in range(len(trace)): if (trace[a] > maxval): trace[a] = maxval if (trace[a] < -maxval): trace[a] = -maxval self._ax.plot(i + trace / maxval, t, color='black', linewidth=lwidth) for a in range(len(trace)): if (trace[a] < 0): trace[a] = 0 self._ax.fill(i + trace / maxval, t, 'k', linewidth=0)
def do(self, rdd): dprint("SetKeyByHeader") rdd = RDD_backToFlat(rdd) rdd = rdd.map(self._ha.getHeaderKV) if seisspark_config.debug: RDD_test("End SetKeyByHeader", rdd) return rdd
def getSelectedModule(self): selected_item_key = self.listView.get_key() dprint('getSelectedModule', selected_item_key) module = None if selected_item_key != None: module_item = self.listView.children[selected_item_key] module = module_item._module return module
def do(self, rdd): if seisspark_config.debug: dprint("FilterByHeader") rdd = RDD_backToFlat(rdd) rdd = rdd.filter(self._hf.filtKV) if seisspark_config.debug: RDD_test("End FilterByHeader", rdd) return rdd
def pipe(self, kv): import subprocess assert (type(kv) is tuple) assert (type(kv[1]) is list) # should be gather assert (type(kv[1][0]) is bytearray) # should be trace # if self._p == None: p = subprocess.Popen(self._args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) # simplest communication in_data = concatenateByteArray(kv[1]) out, err = p.communicate(input=in_data) out_data_array = bytearray(out) dprint('ERROR STREAM', err) ns = KV_HeaderAccess('ns').getHeaderV(out_data_array) bps = 4 trace_len = 240 + bps * ns trace_count = int(len(out_data_array) / trace_len) assert (trace_len * trace_count == len(out_data_array)) out_data = [] for i in range(trace_count): trace = out_data_array[i * trace_len:(i + 1) * trace_len] out_data.append(trace) # in_data = kv[1] # for d in in_data: # self._p.stdin.write(d) # write one by one # self._p.stdin.close() # # out_data = [] # while True: # head = bytearray(self._p.stdout.read(240)) # if not head: # break # # head = bytearray(head) # ns = KV_HeaderAccess('ns').getHeaderV(head) # bps = 4 # # body = self._p.stdout.read(ns * bps) # if not body: # print('cannot read trace body') # exit(1) # body = bytearray(body) # # data = head # head.extend(body) # # out_data.append(data) # TODO optimization sometimes i can use kv[0] as new key return (None, out_data)
def do(self, rdd): dprint("RDD_Processing") ## ## TODO why we do not use spark pipe?!!!! ## rdd = rdd.map(self.pipe) if seisspark_config.debug: RDD_test("End RDD_Processing", rdd) return rdd
def do(self, rdd): dprint("GroupByHeader") rdd = RDD_backToFlat(rdd) rdd = self._sk.do(rdd) # set key rdd = rdd.groupByKey().mapValues(list) #rdd = rdd.sortByKey() if seisspark_config.debug: RDD_test("End GroupByHeader", rdd) return rdd
def onattribute_changed(self, widget, widgetAttributeMember, attributeName, newValue): dprint( "setting attribute name: %s value: %s attributeMember: %s" % (attributeName, newValue, widgetAttributeMember)) self._params.put(attributeName, newValue) return self.eventManager.propagate( self.EVENT_ATTRIB_ONCHANGE, (widgetAttributeMember, attributeName, newValue))
def init(self, appInstance, uName, name, params=Params([])): import copy self._appInstance = appInstance self._uName = uName self._name = name self._id = random.random() self._params = copy.deepcopy(params) self._outRddCache = None self._inRddCache = None dprint('module.init', self._name, 'id', self._id, 'params', self._params)
def calcCached(self, m, rdd): if rdd is not m._inRddCache: # input was changed m._outRddCache = None # drop output if m._outRddCache != None: dprint("module", m._name, "used cache") return m._outRddCache # return out dprint("module", m._name, "didnt used cache") m._inRddCache = rdd m._outRddCache = m.calc(rdd) return m._outRddCache
def __dense(self): dprint("__dense") maxval = float(self._params.get("gain")) maxval = self._norm_amp / maxval palette = self._params.get("palette") self._ax.imshow(self._data, palette, aspect='auto', vmin=-maxval, vmax=maxval)
def KV_flatList(kv): assert (type(kv) is tuple) out = list() values = kv[1] if type(values) is list: for value in values: out.append((None, value)) else: if type(values) is not bytearray: dprint("KV_flatList error: ", type(values)) assert (type(values) is bytearray) out.append((None, values)) return out
def __createFigure(self): dprint('plot_w', seisspark_config.plot_w, 'plot_h', seisspark_config.plot_h) dprint('image_w', seisspark_config.image_w, 'image_h', seisspark_config.image_h) self._fig = Figure(figsize=(seisspark_config.image_w, seisspark_config.image_h)) self._ax = self._fig.add_subplot(111) self._fig.gca().invert_yaxis() self._ax.tick_params(axis='both', which='major', labelsize=seisspark_config.plot_label_size) self._ax.tick_params(axis='both', which='minor', labelsize=seisspark_config.plot_label_size) if seisspark_config.axis_label: self._ax.set_ylabel('Time') self._ax.set_xlabel('Trace')
def updateNorm__(self): array = self._data.reshape(self._ntraces * self._ns) array = np.absolute(array) array.sort(axis=0) percent = float(self._params.get("percent")) dprint('percent', percent) norm = int((len(array) - 1) * percent) dprint('len', norm) self._norm_amp = array[norm] dprint('array', array) dprint('_norm_amp', self._norm_amp)
def setData(self, data, header): dprint("matplot setData") self._data = data self._header = header self._ntraces = data.shape[1] self._t_start = 0 first_trace = self._header[0] self._dt = seisspark.KV_HeaderAccess('dt').getHeaderV(first_trace) self._ns = seisspark.KV_HeaderAccess('ns').getHeaderV(first_trace) self.updateNorm__() dprint("ntraces", self._ntraces) dprint("t_start", self._t_start) dprint("dt", self._dt) dprint("ns", self._ns) # dprint ('Data type', type(self._data), 'shape', self._data.shape) # dprint ('Header type', type(self._header), 'len', len(self._header)) self.__redraw()
def prepareRDDtoDraw(rdd, count=None): dprint("prepareRDDtoDraw") from numpy import transpose from numpy import reshape import struct rdd = RDD_backToFlat(rdd) if count == None: DataFromRDD = rdd.collect() else: DataFromRDD = rdd.take(count) assert (type(DataFromRDD) is list) assert (type(DataFromRDD[0]) is tuple) assert (type(DataFromRDD[0][1]) is bytearray) first_trace = DataFromRDD[0][1] ns = KV_HeaderAccess('ns').getHeaderV(first_trace) dt = KV_HeaderAccess('dt').getHeaderV(first_trace) ntraces = len(DataFromRDD) bps = 4 ndummy_samples = 240 / bps number = ntraces * (ns + ndummy_samples) # concatenate to single bytearray Data = KV_concatenateByteArray(DataFromRDD) header = [] for d in DataFromRDD: header.append(d[1][0:240]) # convert to matrix Data = struct.unpack(segypy.endian + 'f' * number, Data) Data = reshape(Data, (ntraces, ns + ndummy_samples)) Data = Data[:, ndummy_samples:(ns + ndummy_samples)] Data = transpose(Data) dprint("End prepareRDDtoDraw") return Data, header
def __createEditor(self): attributeType = self._param._type additionalInfo = self._param._additionalInfo attributeValue = self._param._v attributeName = self._param._k attributeDesc = self._param._description if additionalInfo == None: additionalInfo = {} dprint('name', attributeName, 'type', attributeType, 'value', attributeValue, 'info', additionalInfo) self.inputWidget = None #'background-repeat':{'type':str, 'description':'The repeat behaviour of an optional background image', ,'additional_data':{'affected_widget_attribute':'style', 'possible_values':'repeat | repeat-x | repeat-y | no-repeat | inherit'}}, if attributeType == bool or attributeType == 'bool': if attributeValue == 'true': attributeValue = True if attributeValue == 'false': attributeValue = False self.inputWidget = gui.CheckBox('checked') elif attributeType == int or attributeType == float or attributeType == 'int' or attributeType == 'float': min_val = -1000000 if "min" in additionalInfo: min_val = additionalInfo['min'] max_val = 1000000 if "max" in additionalInfo: max_val = additionalInfo['max'] step_val = 1 if "step" in additionalInfo: step_val = additionalInfo['step'] self.inputWidget = gui.SpinBox(attributeValue, min_val, max_val, step_val) elif attributeType == gui.ColorPicker: self.inputWidget = gui.ColorPicker() elif attributeType == 'dropdown': self.inputWidget = gui.DropDown() for value in additionalInfo['possible_values']: self.inputWidget.append(gui.DropDownItem(value), value) # elif attributeType == 'url_editor': # self.inputWidget = UrlPathInput(self._appInstance) # elif attributeType == 'css_size': # self.inputWidget = CssSizeInput(self._appInstance) else: # default editor is string self.inputWidget = StringEditor() self.inputWidget.set_on_change_listener(self.on_attribute_changed) self.inputWidget.set_size('50%', '22px') self.inputWidget.attributes['title'] = attributeDesc self.inputWidget.style['float'] = 'right' self.inputWidget.set_value(attributeValue) dprint('setValue', attributeValue) dprint('getValue', self.inputWidget.get_value()) self.append(self.inputWidget)
def valid(self): error = None meta = None if seisspark_config.handle_exceptions: for m in self._modules: dprint("valid", m._name) try: meta = m.valid(meta) except: error = "Validation of module", m._name, "\nUnexpected error:" + \ str(sys.exc_info()[0]) print(error) if self._stopOnError: raise dprint("valid", m._name, 'key', meta._sort) else: for m in self._modules: dprint("valid", m._name) meta = m.valid(meta) dprint("valid", m._name, 'key', meta._sort) return error
def onattribute_changed(self, widget, widgetAttributeMember, attributeName, newValue): module = self.getSelectedModule() dprint('module onattribute_changed', module._name) module.onParamsChanged()
def moduleSelected(self, widget, selected_item_key): module_item = self.listView.children[selected_item_key] dprint('moduleSelected', selected_item_key) module = module_item._module if self.paramsWidget != None: self.paramsWidget.setParams(module._params)
def __init__(self, args): self._args = args self._p = None dprint('RDD_Processing', args) return
def saveData(rdd, filename): dprint("saveData") rdd = RDD_backToFlat(rdd) rdd.saveAsSequenceFile(filename)
def show(self, *args): dprint ("EditorFileSelectionDialog.show") super(EditorFileSaveDialog, self).show(self.baseAppInstance)
def on_saveas_dialog_confirm(self, widget, path): dprint ('on_saveas_dialog_confirm', path) if len(path): filename = path + '/' + self.fileSaveAsDialog.get_fileinput_value() dprint ('on_saveas_dialog_confirm', filename) self.save_job(filename)
def setMeta(self, meta): dprint('meta.sort', self._name, meta._sort) self._meta = meta return self._meta
def put(self, k, v): for p in self._par: if p._k == k: dprint('Params::put', p._k, v) p._v = v