def full_linear_regression(ds):
myList = regression_massage(ds)
myList = remove_nans(myList)
myList = sorted(myList, key=lambda tup: tup[0])
time, value = zip(*myList)
value = [int(x) for x in value]
value = np.array(value)
value.astype(int)
time = np.array(time)
time.astype(int)
return list(zip(time, value))
def date_to_integer(date):
"""Converts support date types to milliseconds since epoch
Attempts highest precision conversion of different datetime
formats to milliseconds since the epoch (1970-01-01 00:00:00).
If datetime is a cftime with a non-standard calendar the
caveats described in hv.core.util.cftime_to_timestamp apply.
Args:
date: Date- or datetime-like object
Returns:
Milliseconds since 1970-01-01 00:00:00
"""
if pd and isinstance(date, pd.Timestamp):
try:
date = date.to_datetime64()
except:
date = date.to_datetime()
if isinstance(date, np.datetime64):
return time.astype('datetime64[ms]').astype(float)
elif isinstance(date, cftime_types):
return cftime_to_timestamp(date, 'ms')
if hasattr(date, 'timetuple'):
dt_int = calendar.timegm(date.timetuple()) * 1000
else:
raise ValueError('Datetime type not recognized')
return dt_int
def process_chunk(self, t0, nt, efield, mask):
ninteg = self._nframe_integrate
out_ntime = efield.shape[-1] // ninteg
time0 = float(t0) / self._nframe_integrate + 1. / 2
time0 = time0 * self.delta_t
# output is data = time * pol * freq
pol = np.arange(1, 3, dtype=np.float32)
time = time0 + np.arange(out_ntime) * self.delta_t
intensity = (self.freq.astype(np.float32)[:,None,None]
* pol[:,None]
* time.astype(np.float32)
)
weight = np.empty(intensity.shape, dtype=np.uint8)
weight[:] = 255
mask_inds_time = (time * 100) % 20 < 1
intensity[:,:,mask_inds_time] = 0
weight[:,:,mask_inds_time] = 0
mask_inds_freq = (self.freq / 1e6) % 40 < 1
intensity[mask_inds_freq,:,:] = 0
weight[mask_inds_freq,:,:] = 0
self.post_process_intensity(time0, intensity, weight)
def get_history_ann(self, arg):
group = arg.getString("entity") if arg.hasField("entity") else "all"
msg = arg.getString("message") if arg.hasField("message") else ""
svr = arg.getString("severity") if arg.hasField("severity") else ""
try:
start, end = self._get_time_from_arg(arg)
except (pva.FieldNotFound, pva.InvalidRequest, ValueError):
print("Error: Invalid argumets")
msg = "Arguments Error: starttime or endtime are invalid"
msg += ". args = " + str(arg)
ret = self._make_error_res(msg)
return ret
try:
if group == "all":
df = self._rdb.history_alarm_all(msg, start, end)
else:
df = self._rdb.history_alarm_group(group, msg, start, end)
except psycopg2.Error:
temp = ("RDB Error: entity = {}, msg = {},"
"starttime = {}, endtime={}")
msg = temp.format(entity, msg, starttime, endtime)
ret = self._make_error_res(msg)
return ret
# Drop lines if it has NaN value
df = df.dropna()
df = df[df["severity"].str.match(svr)]
vals = {
"column0": [pva.ULONG],
"column1": [pva.STRING],
"column2": [pva.STRING],
"column3": [pva.STRING]
}
table = pva.PvObject({
"labels": [pva.STRING],
"value": vals
}, 'epics:nt/NTTable:1.0')
table.setScalarArray("labels", ["time", "title", "tags", "text"])
df["eventtime"] = pd.to_datetime(df["eventtime"])
if df["eventtime"].empty:
time = df["eventtime"]
else:
time = df["eventtime"].dt.strftime("%s%f").str[:-3]
value = {
"column0": time.astype(int).tolist(),
"column1": df["message"].tolist(),
"column2": df["group"].tolist(),
"column3": df["severity"].tolist()
}
table.setStructure("value", value)
return table
def expand_row(time,od,params=None): ret = pd.DataFrame() ret['time'] = time.astype(float) ret['od'] = od.values if not params is None: for c,v in params.iteritems(): ret[c] = v return ret
def expand_row(time, od, params=None):
ret = pd.DataFrame()
ret['time'] = time.astype(float)
ret['od'] = od.values
if not params is None:
for c, v in six.iteritems(params):
ret[c] = v
return ret
def load_data(wanted_param):
time = primer_load_data("time")
time = numpy.array(numpy.reshape(time, len(time)))
if wanted_param != "time":
param = primer_load_data(wanted_param)
if param == "":
return ["ignore"]
param = numpy.array(numpy.reshape(param, len(time)))
return param.astype(numpy.float)
else:
return time.astype(numpy.float)
def tdmsTransform(directory, filePath):
reqFiletdms = glob.glob('*.tdms')
tdms_file = td.read(directory + reqFiletdms[0])
num_groups = len(tdms_file.groups())
if num_groups == 1:
group = tdms_file['FTA_CTA_Train']
time = tdms_file['FTA_CTA_Train']["TriggerSys1"].properties[
"wf_start_time"]
t = time.astype(datetime.datetime)
df = group.as_dataframe(time_index=False, absolute_time=True)
with open(filePath + t.strftime("%Y%m%d%H%M%S") + '.txt', 'a+') as f:
f.write(df.to_string(header=True, index=False))
else:
return
def get_current_ann(self, arg):
entity = arg.getString("entity") if arg.hasField("entity") else ".*"
msg = arg.getString("message") if arg.hasField("message") else ""
try:
df = self._get_current_alarm(entity, msg)
except (ValueError, DatabaseError):
msg = "RDB Error: entity = {}, msg = {}".format(entity, msg)
ret = self._make_error_res(msg)
return ret
except re.error as e:
msg = "regex error ({}) entity = {}, msg = {}"
msg = msg.format(e, entity, msg)
ret = self._make_error_res(msg)
return ret
vals = {
"column0": [pva.ULONG],
"column1": [pva.STRING],
"column2": [pva.STRING],
"column3": [pva.STRING]
}
table = pva.PvObject({
"labels": [pva.STRING],
"value": vals
}, 'epics:nt/NTTable:1.0')
table.setScalarArray("labels", ["time", "title", "tags", "text"])
if df["alarm_time"].empty:
time = df["alarm_time"]
else:
time = df["alarm_time"].dt.strftime("%s%f").str[:-3]
value = {
"column0": time.astype(int).tolist(),
"column1": df["descr"].tolist(),
"column2": df["groups"].tolist(),
"column3": df["pv_name"].tolist()
}
table.setStructure("value", value)
return table
def get_dims(self, file_name):
"""
Args:
file_name: str: name and path to file.
Returns:
"""
nc = Dataset(file_name)
time = nc.variables['time'][:]
lat = nc.variables['lat'][:]
lon = nc.variables['lon'][:]
todate = np.vectorize(dt.strptime)
newtime = todate(time.astype('int').astype('str'), '%Y%m%d')
nc.close()
return time, lat, lon
def compute_velocity(self, track_coords: np.ndarray, time: np.ndarray):
"""Compute velocities for the given track.
Args:
track_df (pandas Dataframe): Data for the track
Returns:
vel (list of float): Velocity at each timestep
"""
x = track_coords[:, 0].astype('float')
y = track_coords[:, 1].astype('float')
time = time.astype('float')
velocity = np.empty((len(time)))
dt = time[1:] - time[:-1]
vx = (x[1:] - x[:-1]) / dt
vy = (y[1:] - y[:-1]) / dt
velocity[1:] = np.sqrt(np.square(vx) + np.square(vy))
velocity[0] = velocity[1]
return velocity.reshape((len(time), 1))
def numsats(snr,
rxid=None,
hk_hours=None,
endtime=None,
minelev=0,
pos='topbot',
suffix='',
**unused):
"""Plot number of tracked satellites vs. time from a array of SNR records.
If rxid is given, an image is written out in the current directory, named
NS-RX##-DOY.png. Otherwise the figure is returned.
"""
if hk_hours is not None:
thresh, endtime = _thresh(hk_hours, endtime)
snr = snr[findfirstgt(snr['time'], thresh
):findfirstgt(snr['time'], endtime)]
if minelev:
snr = snr[snr['el'] > minelev]
if not len(snr):
info('No records', 'for RX{:02}'.format(rxid) if rxid else '',
'above {}° from'.format(minelev), thresh, 'to', endtime)
return
time, idx = np.unique(snr['time'], return_index=True)
idx.sort() # if the times aren't in order for some reason
numsats = [
len(set(snr[a:b]['prn']))
for a, b in zip(idx, np.append(idx[1:], len(snr)))
]
# time, idx, numsats = np.unique(snr['time'], return_index=True, return_counts=True)
# for n, a, b in zip(numsats, idx, np.append(idx[1:], len(snr))):
# if len(set(snr[a:b]['prn'])) != n:
# info('Same PRN in multiple records?')
# debug(snr[a:b])
doy = mode(time.astype('M8[D]')).tolist() # most common day
time = snr['time'][idx].tolist(
) # get datetime.datetime, which matplotlib can handle
if rxid:
title = 'Rx{:02} {:%Y-%m-%d}'.format(rxid, doy)
else:
title = 'VAPR {:%Y-%m-%d}'.format(doy)
if pos != 'top':
title += ': Number of satellites over {}°'.format(minelev)
if 'top' in pos:
ylabl = 'Tracked satellites'
else:
ylabl = 'No. satellites over {}°'.format(minelev)
fig, ax = _gethouraxes([10, 2.7], pos, title=title, ylabel=ylabl)
ax.plot(time, numsats, '-o', ms=2)
if hk_hours is not None:
ax.set_xlim(thresh.tolist(), endtime.tolist())
else:
ax.set_xlim(min(time), max(time))
ax.yaxis.set_major_locator(mp.ticker.MaxNLocator(integer=True))
ax.set_ylim(min(numsats) - 1, max(numsats) + 1)
ax.tick_params(axis='y', labelleft='on', labelright='on')
ax.grid(True)
if rxid:
fname = 'NS-RX{:02}{}.png'.format(rxid, suffix)
fig.savefig(fname)
plt.close(fig)
return fname
return fig
def convert_datetime(time):
return time.astype('datetime64[s]').astype(float) * 1000
def meansnr(snr,
rxid=None,
hk_hours=None,
endtime=None,
minelev=0,
pos='topbot',
suffix='',
**unused):
"""Plot mean snr vs. time from a array of SNR records.
If rxid is given, an image is written out in the current directory, named
AVG-RX##-DOY.png. Otherwise the figure is returned.
"""
if hk_hours is not None:
thresh, endtime = _thresh(hk_hours, endtime)
snr = snr[findfirstgt(snr['time'], thresh
):findfirstgt(snr['time'], endtime)]
if minelev:
snr = snr[snr['el'] > minelev]
if not len(snr):
info('No records', 'for RX{:02}'.format(rxid) if rxid else '',
'in the given time period', thresh, 'to', endtime)
return
time, idx = np.unique(snr['time'], return_index=True)
idx.sort() # if the times aren't in order for some reason
time = snr['time'][idx]
doy = mode(time.astype('M8[D]')).tolist() # most common day
idx = np.append(idx, len(snr))
means = []
pmeans = []
for a, b in zip(idx, idx[1:]):
tsnr = snr[a:b]['snr'] / 10
means += [np.mean(tsnr)]
if np.count_nonzero(tsnr):
pmeans += [np.mean(tsnr[tsnr > 0])]
else:
pmeans += [0]
if rxid:
title = 'Rx{:02} {:%Y-%m-%d}: Mean SNR'.format(rxid, doy)
else:
title = 'VAPR {:%Y-%m-%d}: Mean SNR'.format(doy)
if minelev:
title += ' over {}°'.format(minelev)
ylabl = 'Mean SNR (dB-Hz)'
if 'top' not in pos and minelev:
ylabl += ' over {}°'.format(minelev)
fig, ax = _gethouraxes([10, 3], pos, title=title, ylabel=ylabl)
pmeans = np.array(pmeans)
ax.plot(time.tolist(), means, 'b.', ms=2, label='All values')
ax.plot(time[pmeans > 0].tolist(),
pmeans[pmeans > 0],
'r.',
ms=2,
label='Positive only')
if hk_hours is not None:
ax.set_xlim(thresh.tolist(), endtime.tolist())
else:
ax.set_xlim(min(time.tolist()), max(time.tolist()))
_setsnrlim(
ax, pmeans) # add argument True to restrict y-range to observed values
ax.tick_params(axis='y', labelleft='on', labelright='on')
ax.legend(loc='best',
fontsize='small',
numpoints=1,
markerscale=4,
handletextpad=0.4,
handlelength=0.8)
if rxid:
fname = 'AVG-RX{:02}{}.png'.format(rxid, suffix)
fig.savefig(fname)
plt.close(fig)
return fname
return fig
def convert_time(time):
return datetime.datetime.utcfromtimestamp(time.astype('O') / 1e9)
def preprocess_image(img_rgb_data, file_name, conf, debug=False):
if img_rgb_data is None:
logger.error("图像数据为空")
return None
output_img(file_name + "原图", img_rgb_data, debug)
#实在代码是太ugly了，由于邮储太特殊，只认红色的，所以hardcode一下吧
if conf.name == "psbc":
logger.debug("特殊处理邮储bank图片，只识别红色字体")
img_rgb_data = process_psbc_red(img_rgb_data, debug)
#这个方法不靠谱
#image_gray = color.rgb2gray(img)
image_gray = cv2.cvtColor(img_rgb_data, cv2.COLOR_BGR2GRAY)
logger.debug("灰度图像的shape:%r", image_gray.shape)
output_img(file_name + "灰度图", image_gray, debug)
#1.根据阈值进行分割
if conf.img_binary_threshold == -1:
thresh = filters.threshold_otsu(image_gray) #自动探测一个阈值
else:
logger.debug("使用自定义的阈值来二值化：%d", conf.img_binary_threshold)
thresh = conf.img_binary_threshold #从配置中读取阈值
t = (image_gray <= thresh)
#如果配置中需要黑白颠倒，就把颜色最后颠倒一下
if conf.inverse_color: t = ~t
#2.删除掉小的区块，面积是minsize=25,25是个拍脑袋的经验值，可以在配置文件中修改
output_img(file_name + "二值化", t * 255, debug) #*255是为了变成白色的用于显示
min_size = 25 #默认要删除掉的噪点大小
if conf.img_remove_area != -1: min_size = conf.img_remove_area
t = morphology.remove_small_objects(t, min_size=min_size, connectivity=1)
output_img(file_name + "删除小块", t * 255, debug) #*255是为了变成白色的用于显示
t = t.astype(np.float32)
#3.有些图像带黑框，用一个框的mask去掉
pad = conf.mask_pad
if pad != 0:
logger.debug("去掉黑框，pad=%d", conf.mask_pad)
mask = np.zeros(t.shape, dtype="uint8")
#画一个需要的地方是1，不需要的地方是0的矩形框，框的厚度是pad
cv2.rectangle(mask, (pad, pad),
(t.shape[1] - 2 * pad, t.shape[0] - 2 * pad), 1, -1)
# output_img(file_name+"黑框mask",mask,debug)#*255是为了变成白色的用于显示
t = cv2.bitwise_and(t, t, mask=mask)
output_img(file_name + "去掉黑框", t * 255, debug) #*255是为了变成白色的用于显示
#防止有的图像不是规定的conf.widthxheight，有必要都规范化一下
if t.shape[1] < conf.width:
logger.debug("图像宽度%d不够，补齐到%d", t.shape[1], conf.width)
t = np.concatenate(
(t, np.zeros(
(t.shape[0], conf.width - t.shape[1]), dtype='uint8')),
axis=1)
if t.shape[0] < conf.height:
logger.debug("图像高度%d不够，补齐到%d", t.shape[0], conf.height)
t = np.concatenate(
(t, np.zeros(
(conf.height - t.shape[0], t.shape[1]), dtype='uint8')),
axis=0)
output_img(file_name + "调整大小", t * 255, debug) #*255是为了变成白色的用于显示
#如果图像大了，要剪切成规定大小（经过上面的处理，宽和高至少都是比规定尺寸大了）
if t.shape[1] > conf.width or t.shape[0] > conf.height:
logger.debug("图像从%r Resize到 %r", t.shape, (conf.width, conf.height))
t = cv2.resize(t, (conf.width, conf.height))
output_img(file_name + "Resize", t * 255, debug) #*255是为了变成白色的用于显示
#同时把原来白色的地方变成1，黑的地方为0，这个是训练要求，有数字显示的地方是1，没有的是0
t = t > 0
#变成一个四维numpy.ndarray,为了和[图片个数？,image channel,conf.height,conf.width]
#tensorflow和thenano的格式要求不一样，图像通道的位置反着，烦人，做一下处理
if K.image_data_format() == 'channels_first':
I = t.astype(np.float32).reshape((1, 1, conf.height, conf.width))
else:
I = t.astype(np.float32).reshape((1, conf.height, conf.width, 1))
logger.debug("图像数据:%r", I.shape)
return I #返回一个图像的矩阵
def plotComparisonSingle(fileName, datasetName, attribute1, mOa,
UsePercentageTrainingSet):
#mOa=0 => max || mOa=1 => avg || mOa=-1 => min
#ATT1 SU ASSE X, ATT2 SU CUI EFFETTO COMPARAZIONE -> PRENDO DIVERSE ACCURACY AL VARIARE DEL VALORE DI TALE ATTRIBUTO
dfResults = readCsvAsDf(fileName)
print(dfResults)
dftest = dfResults[dfResults["useValidationSet"] == "False"]
dfLocal = dftest
# SCELGO LA CONFIGURAZIONE MIGLIORE
#dfLocal = dfLocal[(dftest['Candidates'] == 'Discords')]
dfLocal = dfLocal[(dfLocal['MaxDepth'] == '3')]
dfLocal = dfLocal[(dfLocal['MinSamples'] == '20')]
dfLocal = dfLocal[(dfLocal['WindowSize'] == '20')]
dfLocal = dfLocal[(dfLocal['RemoveCandidates'] == '1')]
dfLocal = dfLocal[(dfLocal['k'] == '2')]
dfLocal = dfLocal[(dfLocal['NumClusterMedoid'] == '20')]
print(dfLocal)
if (UsePercentageTrainingSet): #su asse x metto Percentage
attribute1 = 'PercentageTrainingSet'
else:
dfLocal = dfLocal[dfLocal['PercentageTrainingSet'] ==
'1'] #prendo training set interi
#prendo differenti valori dell'attributo su asse x
valuesAtt1 = np.unique(dfLocal[attribute1].values)
if (attribute1 == "NumClusterMedoid" or attribute1 == "MaxDepth"
or attribute1 == "MinSamples" or attribute1 == "WindowSize"
or attribute1 == "k"):
print('dentro')
valuesAtt1 = valuesAtt1.astype(int)
valuesAtt1.sort()
valuesAtt1 = valuesAtt1.astype(str)
if (attribute1 == "PercentageTrainingSet"):
valuesAtt1 = valuesAtt1.astype(float)
valuesAtt1 = sorted(valuesAtt1)
for i in range(len(valuesAtt1)):
valuesAtt1[i] = str(valuesAtt1[i])
if (
UsePercentageTrainingSet
): #setto 1 cosi riesco a fare la query correttamente, sul file è memorizzato come 1
valuesAtt1[-1] = '1'
print('ATT')
print(valuesAtt1)
colori = 0
colors = 'rbgcmyk'
accuracyList = []
timeList = []
fig, ax1 = plt.subplots(1, 1, sharex=True, figsize=(8, 4))
ax2 = ax1.twinx()
ax1.set_title(datasetName, fontsize=25)
for i in range(len(valuesAtt1)):
valueAtt1 = valuesAtt1[i]
c = colors[colori % len(colors)]
accuracy = dfLocal[(
dfLocal[attribute1]) == valueAtt1]['Accuracy'].values
time = dfLocal[(dfLocal[attribute1]) == valueAtt1]['Time'].values
time = time.astype(float)
if (len(accuracy) > 0):
if (mOa == 0):
choosenAccuracy = max(accuracy)
choosenTime = min(time)
elif (mOa == -1):
choosenTime = min(time)
choosenAccuracy = min(accuracy)
else:
choosenTime = sum(time) / len(time)
choosenAccuracy = sum(accuracy) / len(accuracy)
else:
choosenTime = 0
choosenAccuracy = 0
timeList.append(choosenTime)
accuracyList.append(choosenAccuracy)
print(timeList)
print(accuracyList)
ax1.plot(valuesAtt1, accuracyList, color='r', marker='o', label='Accuracy')
ax2.plot(valuesAtt1, timeList, color='b', marker='^', label='Time')
ax1.set_xlabel("Type of candidate", fontsize=25)
ax1.set_ylabel('Accuracy', fontsize=25)
ax2.set_ylabel('Time (sec)', fontsize=25)
ax1.tick_params(axis='both', which='both', labelsize=35)
ax2.tick_params(axis='both', which='both', labelsize=35)
#plt.savefig('stampeAlgoritmo\ ' + fileName + 'Comparison' + attribute1 + '.png',bbox_inches='tight')
plt.show()
def convert_datetime(time):
return time.astype('datetime64[s]').astype(float)*1000
