def full_linear_regression(ds): myList = regression_massage(ds) myList = remove_nans(myList) myList = sorted(myList, key=lambda tup: tup[0]) time, value = zip(*myList) value = [int(x) for x in value] value = np.array(value) value.astype(int) time = np.array(time) time.astype(int) return list(zip(time, value))
def date_to_integer(date): """Converts support date types to milliseconds since epoch Attempts highest precision conversion of different datetime formats to milliseconds since the epoch (1970-01-01 00:00:00). If datetime is a cftime with a non-standard calendar the caveats described in hv.core.util.cftime_to_timestamp apply. Args: date: Date- or datetime-like object Returns: Milliseconds since 1970-01-01 00:00:00 """ if pd and isinstance(date, pd.Timestamp): try: date = date.to_datetime64() except: date = date.to_datetime() if isinstance(date, np.datetime64): return time.astype('datetime64[ms]').astype(float) elif isinstance(date, cftime_types): return cftime_to_timestamp(date, 'ms') if hasattr(date, 'timetuple'): dt_int = calendar.timegm(date.timetuple()) * 1000 else: raise ValueError('Datetime type not recognized') return dt_int
def process_chunk(self, t0, nt, efield, mask): ninteg = self._nframe_integrate out_ntime = efield.shape[-1] // ninteg time0 = float(t0) / self._nframe_integrate + 1. / 2 time0 = time0 * self.delta_t # output is data = time * pol * freq pol = np.arange(1, 3, dtype=np.float32) time = time0 + np.arange(out_ntime) * self.delta_t intensity = (self.freq.astype(np.float32)[:,None,None] * pol[:,None] * time.astype(np.float32) ) weight = np.empty(intensity.shape, dtype=np.uint8) weight[:] = 255 mask_inds_time = (time * 100) % 20 < 1 intensity[:,:,mask_inds_time] = 0 weight[:,:,mask_inds_time] = 0 mask_inds_freq = (self.freq / 1e6) % 40 < 1 intensity[mask_inds_freq,:,:] = 0 weight[mask_inds_freq,:,:] = 0 self.post_process_intensity(time0, intensity, weight)
def get_history_ann(self, arg): group = arg.getString("entity") if arg.hasField("entity") else "all" msg = arg.getString("message") if arg.hasField("message") else "" svr = arg.getString("severity") if arg.hasField("severity") else "" try: start, end = self._get_time_from_arg(arg) except (pva.FieldNotFound, pva.InvalidRequest, ValueError): print("Error: Invalid argumets") msg = "Arguments Error: starttime or endtime are invalid" msg += ". args = " + str(arg) ret = self._make_error_res(msg) return ret try: if group == "all": df = self._rdb.history_alarm_all(msg, start, end) else: df = self._rdb.history_alarm_group(group, msg, start, end) except psycopg2.Error: temp = ("RDB Error: entity = {}, msg = {}," "starttime = {}, endtime={}") msg = temp.format(entity, msg, starttime, endtime) ret = self._make_error_res(msg) return ret # Drop lines if it has NaN value df = df.dropna() df = df[df["severity"].str.match(svr)] vals = { "column0": [pva.ULONG], "column1": [pva.STRING], "column2": [pva.STRING], "column3": [pva.STRING] } table = pva.PvObject({ "labels": [pva.STRING], "value": vals }, 'epics:nt/NTTable:1.0') table.setScalarArray("labels", ["time", "title", "tags", "text"]) df["eventtime"] = pd.to_datetime(df["eventtime"]) if df["eventtime"].empty: time = df["eventtime"] else: time = df["eventtime"].dt.strftime("%s%f").str[:-3] value = { "column0": time.astype(int).tolist(), "column1": df["message"].tolist(), "column2": df["group"].tolist(), "column3": df["severity"].tolist() } table.setStructure("value", value) return table
def expand_row(time,od,params=None): ret = pd.DataFrame() ret['time'] = time.astype(float) ret['od'] = od.values if not params is None: for c,v in params.iteritems(): ret[c] = v return ret
def expand_row(time, od, params=None): ret = pd.DataFrame() ret['time'] = time.astype(float) ret['od'] = od.values if not params is None: for c, v in six.iteritems(params): ret[c] = v return ret
def load_data(wanted_param): time = primer_load_data("time") time = numpy.array(numpy.reshape(time, len(time))) if wanted_param != "time": param = primer_load_data(wanted_param) if param == "": return ["ignore"] param = numpy.array(numpy.reshape(param, len(time))) return param.astype(numpy.float) else: return time.astype(numpy.float)
def tdmsTransform(directory, filePath): reqFiletdms = glob.glob('*.tdms') tdms_file = td.read(directory + reqFiletdms[0]) num_groups = len(tdms_file.groups()) if num_groups == 1: group = tdms_file['FTA_CTA_Train'] time = tdms_file['FTA_CTA_Train']["TriggerSys1"].properties[ "wf_start_time"] t = time.astype(datetime.datetime) df = group.as_dataframe(time_index=False, absolute_time=True) with open(filePath + t.strftime("%Y%m%d%H%M%S") + '.txt', 'a+') as f: f.write(df.to_string(header=True, index=False)) else: return
def get_current_ann(self, arg): entity = arg.getString("entity") if arg.hasField("entity") else ".*" msg = arg.getString("message") if arg.hasField("message") else "" try: df = self._get_current_alarm(entity, msg) except (ValueError, DatabaseError): msg = "RDB Error: entity = {}, msg = {}".format(entity, msg) ret = self._make_error_res(msg) return ret except re.error as e: msg = "regex error ({}) entity = {}, msg = {}" msg = msg.format(e, entity, msg) ret = self._make_error_res(msg) return ret vals = { "column0": [pva.ULONG], "column1": [pva.STRING], "column2": [pva.STRING], "column3": [pva.STRING] } table = pva.PvObject({ "labels": [pva.STRING], "value": vals }, 'epics:nt/NTTable:1.0') table.setScalarArray("labels", ["time", "title", "tags", "text"]) if df["alarm_time"].empty: time = df["alarm_time"] else: time = df["alarm_time"].dt.strftime("%s%f").str[:-3] value = { "column0": time.astype(int).tolist(), "column1": df["descr"].tolist(), "column2": df["groups"].tolist(), "column3": df["pv_name"].tolist() } table.setStructure("value", value) return table
def get_dims(self, file_name): """ Args: file_name: str: name and path to file. Returns: """ nc = Dataset(file_name) time = nc.variables['time'][:] lat = nc.variables['lat'][:] lon = nc.variables['lon'][:] todate = np.vectorize(dt.strptime) newtime = todate(time.astype('int').astype('str'), '%Y%m%d') nc.close() return time, lat, lon
def compute_velocity(self, track_coords: np.ndarray, time: np.ndarray): """Compute velocities for the given track. Args: track_df (pandas Dataframe): Data for the track Returns: vel (list of float): Velocity at each timestep """ x = track_coords[:, 0].astype('float') y = track_coords[:, 1].astype('float') time = time.astype('float') velocity = np.empty((len(time))) dt = time[1:] - time[:-1] vx = (x[1:] - x[:-1]) / dt vy = (y[1:] - y[:-1]) / dt velocity[1:] = np.sqrt(np.square(vx) + np.square(vy)) velocity[0] = velocity[1] return velocity.reshape((len(time), 1))
def numsats(snr, rxid=None, hk_hours=None, endtime=None, minelev=0, pos='topbot', suffix='', **unused): """Plot number of tracked satellites vs. time from a array of SNR records. If rxid is given, an image is written out in the current directory, named NS-RX##-DOY.png. Otherwise the figure is returned. """ if hk_hours is not None: thresh, endtime = _thresh(hk_hours, endtime) snr = snr[findfirstgt(snr['time'], thresh ):findfirstgt(snr['time'], endtime)] if minelev: snr = snr[snr['el'] > minelev] if not len(snr): info('No records', 'for RX{:02}'.format(rxid) if rxid else '', 'above {}° from'.format(minelev), thresh, 'to', endtime) return time, idx = np.unique(snr['time'], return_index=True) idx.sort() # if the times aren't in order for some reason numsats = [ len(set(snr[a:b]['prn'])) for a, b in zip(idx, np.append(idx[1:], len(snr))) ] # time, idx, numsats = np.unique(snr['time'], return_index=True, return_counts=True) # for n, a, b in zip(numsats, idx, np.append(idx[1:], len(snr))): # if len(set(snr[a:b]['prn'])) != n: # info('Same PRN in multiple records?') # debug(snr[a:b]) doy = mode(time.astype('M8[D]')).tolist() # most common day time = snr['time'][idx].tolist( ) # get datetime.datetime, which matplotlib can handle if rxid: title = 'Rx{:02} {:%Y-%m-%d}'.format(rxid, doy) else: title = 'VAPR {:%Y-%m-%d}'.format(doy) if pos != 'top': title += ': Number of satellites over {}°'.format(minelev) if 'top' in pos: ylabl = 'Tracked satellites' else: ylabl = 'No. satellites over {}°'.format(minelev) fig, ax = _gethouraxes([10, 2.7], pos, title=title, ylabel=ylabl) ax.plot(time, numsats, '-o', ms=2) if hk_hours is not None: ax.set_xlim(thresh.tolist(), endtime.tolist()) else: ax.set_xlim(min(time), max(time)) ax.yaxis.set_major_locator(mp.ticker.MaxNLocator(integer=True)) ax.set_ylim(min(numsats) - 1, max(numsats) + 1) ax.tick_params(axis='y', labelleft='on', labelright='on') ax.grid(True) if rxid: fname = 'NS-RX{:02}{}.png'.format(rxid, suffix) fig.savefig(fname) plt.close(fig) return fname return fig
def convert_datetime(time): return time.astype('datetime64[s]').astype(float) * 1000
def meansnr(snr, rxid=None, hk_hours=None, endtime=None, minelev=0, pos='topbot', suffix='', **unused): """Plot mean snr vs. time from a array of SNR records. If rxid is given, an image is written out in the current directory, named AVG-RX##-DOY.png. Otherwise the figure is returned. """ if hk_hours is not None: thresh, endtime = _thresh(hk_hours, endtime) snr = snr[findfirstgt(snr['time'], thresh ):findfirstgt(snr['time'], endtime)] if minelev: snr = snr[snr['el'] > minelev] if not len(snr): info('No records', 'for RX{:02}'.format(rxid) if rxid else '', 'in the given time period', thresh, 'to', endtime) return time, idx = np.unique(snr['time'], return_index=True) idx.sort() # if the times aren't in order for some reason time = snr['time'][idx] doy = mode(time.astype('M8[D]')).tolist() # most common day idx = np.append(idx, len(snr)) means = [] pmeans = [] for a, b in zip(idx, idx[1:]): tsnr = snr[a:b]['snr'] / 10 means += [np.mean(tsnr)] if np.count_nonzero(tsnr): pmeans += [np.mean(tsnr[tsnr > 0])] else: pmeans += [0] if rxid: title = 'Rx{:02} {:%Y-%m-%d}: Mean SNR'.format(rxid, doy) else: title = 'VAPR {:%Y-%m-%d}: Mean SNR'.format(doy) if minelev: title += ' over {}°'.format(minelev) ylabl = 'Mean SNR (dB-Hz)' if 'top' not in pos and minelev: ylabl += ' over {}°'.format(minelev) fig, ax = _gethouraxes([10, 3], pos, title=title, ylabel=ylabl) pmeans = np.array(pmeans) ax.plot(time.tolist(), means, 'b.', ms=2, label='All values') ax.plot(time[pmeans > 0].tolist(), pmeans[pmeans > 0], 'r.', ms=2, label='Positive only') if hk_hours is not None: ax.set_xlim(thresh.tolist(), endtime.tolist()) else: ax.set_xlim(min(time.tolist()), max(time.tolist())) _setsnrlim( ax, pmeans) # add argument True to restrict y-range to observed values ax.tick_params(axis='y', labelleft='on', labelright='on') ax.legend(loc='best', fontsize='small', numpoints=1, markerscale=4, handletextpad=0.4, handlelength=0.8) if rxid: fname = 'AVG-RX{:02}{}.png'.format(rxid, suffix) fig.savefig(fname) plt.close(fig) return fname return fig
def convert_time(time): return datetime.datetime.utcfromtimestamp(time.astype('O') / 1e9)
def preprocess_image(img_rgb_data, file_name, conf, debug=False): if img_rgb_data is None: logger.error("图像数据为空") return None output_img(file_name + "原图", img_rgb_data, debug) #实在代码是太ugly了,由于邮储太特殊,只认红色的,所以hardcode一下吧 if conf.name == "psbc": logger.debug("特殊处理邮储bank图片,只识别红色字体") img_rgb_data = process_psbc_red(img_rgb_data, debug) #这个方法不靠谱 #image_gray = color.rgb2gray(img) image_gray = cv2.cvtColor(img_rgb_data, cv2.COLOR_BGR2GRAY) logger.debug("灰度图像的shape:%r", image_gray.shape) output_img(file_name + "灰度图", image_gray, debug) #1.根据阈值进行分割 if conf.img_binary_threshold == -1: thresh = filters.threshold_otsu(image_gray) #自动探测一个阈值 else: logger.debug("使用自定义的阈值来二值化:%d", conf.img_binary_threshold) thresh = conf.img_binary_threshold #从配置中读取阈值 t = (image_gray <= thresh) #如果配置中需要黑白颠倒,就把颜色最后颠倒一下 if conf.inverse_color: t = ~t #2.删除掉小的区块,面积是minsize=25,25是个拍脑袋的经验值,可以在配置文件中修改 output_img(file_name + "二值化", t * 255, debug) #*255是为了变成白色的用于显示 min_size = 25 #默认要删除掉的噪点大小 if conf.img_remove_area != -1: min_size = conf.img_remove_area t = morphology.remove_small_objects(t, min_size=min_size, connectivity=1) output_img(file_name + "删除小块", t * 255, debug) #*255是为了变成白色的用于显示 t = t.astype(np.float32) #3.有些图像带黑框,用一个框的mask去掉 pad = conf.mask_pad if pad != 0: logger.debug("去掉黑框,pad=%d", conf.mask_pad) mask = np.zeros(t.shape, dtype="uint8") #画一个需要的地方是1,不需要的地方是0的矩形框,框的厚度是pad cv2.rectangle(mask, (pad, pad), (t.shape[1] - 2 * pad, t.shape[0] - 2 * pad), 1, -1) # output_img(file_name+"黑框mask",mask,debug)#*255是为了变成白色的用于显示 t = cv2.bitwise_and(t, t, mask=mask) output_img(file_name + "去掉黑框", t * 255, debug) #*255是为了变成白色的用于显示 #防止有的图像不是规定的conf.widthxheight,有必要都规范化一下 if t.shape[1] < conf.width: logger.debug("图像宽度%d不够,补齐到%d", t.shape[1], conf.width) t = np.concatenate( (t, np.zeros( (t.shape[0], conf.width - t.shape[1]), dtype='uint8')), axis=1) if t.shape[0] < conf.height: logger.debug("图像高度%d不够,补齐到%d", t.shape[0], conf.height) t = np.concatenate( (t, np.zeros( (conf.height - t.shape[0], t.shape[1]), dtype='uint8')), axis=0) output_img(file_name + "调整大小", t * 255, debug) #*255是为了变成白色的用于显示 #如果图像大了,要剪切成规定大小(经过上面的处理,宽和高至少都是比规定尺寸大了) if t.shape[1] > conf.width or t.shape[0] > conf.height: logger.debug("图像从%r Resize到 %r", t.shape, (conf.width, conf.height)) t = cv2.resize(t, (conf.width, conf.height)) output_img(file_name + "Resize", t * 255, debug) #*255是为了变成白色的用于显示 #同时把原来白色的地方变成1,黑的地方为0,这个是训练要求,有数字显示的地方是1,没有的是0 t = t > 0 #变成一个四维numpy.ndarray,为了和[图片个数?,image channel,conf.height,conf.width] #tensorflow和thenano的格式要求不一样,图像通道的位置反着,烦人,做一下处理 if K.image_data_format() == 'channels_first': I = t.astype(np.float32).reshape((1, 1, conf.height, conf.width)) else: I = t.astype(np.float32).reshape((1, conf.height, conf.width, 1)) logger.debug("图像数据:%r", I.shape) return I #返回一个图像的矩阵
def plotComparisonSingle(fileName, datasetName, attribute1, mOa, UsePercentageTrainingSet): #mOa=0 => max || mOa=1 => avg || mOa=-1 => min #ATT1 SU ASSE X, ATT2 SU CUI EFFETTO COMPARAZIONE -> PRENDO DIVERSE ACCURACY AL VARIARE DEL VALORE DI TALE ATTRIBUTO dfResults = readCsvAsDf(fileName) print(dfResults) dftest = dfResults[dfResults["useValidationSet"] == "False"] dfLocal = dftest # SCELGO LA CONFIGURAZIONE MIGLIORE #dfLocal = dfLocal[(dftest['Candidates'] == 'Discords')] dfLocal = dfLocal[(dfLocal['MaxDepth'] == '3')] dfLocal = dfLocal[(dfLocal['MinSamples'] == '20')] dfLocal = dfLocal[(dfLocal['WindowSize'] == '20')] dfLocal = dfLocal[(dfLocal['RemoveCandidates'] == '1')] dfLocal = dfLocal[(dfLocal['k'] == '2')] dfLocal = dfLocal[(dfLocal['NumClusterMedoid'] == '20')] print(dfLocal) if (UsePercentageTrainingSet): #su asse x metto Percentage attribute1 = 'PercentageTrainingSet' else: dfLocal = dfLocal[dfLocal['PercentageTrainingSet'] == '1'] #prendo training set interi #prendo differenti valori dell'attributo su asse x valuesAtt1 = np.unique(dfLocal[attribute1].values) if (attribute1 == "NumClusterMedoid" or attribute1 == "MaxDepth" or attribute1 == "MinSamples" or attribute1 == "WindowSize" or attribute1 == "k"): print('dentro') valuesAtt1 = valuesAtt1.astype(int) valuesAtt1.sort() valuesAtt1 = valuesAtt1.astype(str) if (attribute1 == "PercentageTrainingSet"): valuesAtt1 = valuesAtt1.astype(float) valuesAtt1 = sorted(valuesAtt1) for i in range(len(valuesAtt1)): valuesAtt1[i] = str(valuesAtt1[i]) if ( UsePercentageTrainingSet ): #setto 1 cosi riesco a fare la query correttamente, sul file è memorizzato come 1 valuesAtt1[-1] = '1' print('ATT') print(valuesAtt1) colori = 0 colors = 'rbgcmyk' accuracyList = [] timeList = [] fig, ax1 = plt.subplots(1, 1, sharex=True, figsize=(8, 4)) ax2 = ax1.twinx() ax1.set_title(datasetName, fontsize=25) for i in range(len(valuesAtt1)): valueAtt1 = valuesAtt1[i] c = colors[colori % len(colors)] accuracy = dfLocal[( dfLocal[attribute1]) == valueAtt1]['Accuracy'].values time = dfLocal[(dfLocal[attribute1]) == valueAtt1]['Time'].values time = time.astype(float) if (len(accuracy) > 0): if (mOa == 0): choosenAccuracy = max(accuracy) choosenTime = min(time) elif (mOa == -1): choosenTime = min(time) choosenAccuracy = min(accuracy) else: choosenTime = sum(time) / len(time) choosenAccuracy = sum(accuracy) / len(accuracy) else: choosenTime = 0 choosenAccuracy = 0 timeList.append(choosenTime) accuracyList.append(choosenAccuracy) print(timeList) print(accuracyList) ax1.plot(valuesAtt1, accuracyList, color='r', marker='o', label='Accuracy') ax2.plot(valuesAtt1, timeList, color='b', marker='^', label='Time') ax1.set_xlabel("Type of candidate", fontsize=25) ax1.set_ylabel('Accuracy', fontsize=25) ax2.set_ylabel('Time (sec)', fontsize=25) ax1.tick_params(axis='both', which='both', labelsize=35) ax2.tick_params(axis='both', which='both', labelsize=35) #plt.savefig('stampeAlgoritmo\ ' + fileName + 'Comparison' + attribute1 + '.png',bbox_inches='tight') plt.show()
def convert_datetime(time): return time.astype('datetime64[s]').astype(float)*1000