def run_simulation(self):
Start_Time, Stop_Time, Number_Of_Steps, Step_Size, Initial_Orbit_Elements = self.Initial_Value_Get(
)
joblib.dump(Start_Time, 'Start_Time.pkl')
joblib.dump(Step_Size, 'Step_Size.pkl')
joblib.dump(Number_Of_Steps, 'Number_Of_Steps.pkl')
HPOP(Start_Time, Number_Of_Steps, Step_Size, Initial_Orbit_Elements)
HPOP_Results = joblib.load('HPOP_Results.pkl')
Start_time = "%s-%02d-%02dT%02d:%02d:%02dZ" % (
Start_Time[5], Start_Time[4], Start_Time[3], Start_Time[2],
Start_Time[1], Start_Time[0])
end_time = "%s-%02d-%02dT%02d:%02d:%02dZ" % (
Stop_Time[5], Stop_Time[4], Stop_Time[3], Stop_Time[2],
Stop_Time[1], Stop_Time[0])
# Start_time = "2012-03-15T10:00:00Z"
# end_time = "2012-03-16T10:00:00Z"
x = HPOP_Results[1, :]
y = HPOP_Results[2, :]
z = HPOP_Results[3, :]
time = HPOP_Results[0, :]
time = time.tolist()
cartesian_file = []
for i in range(0, len(time)):
cartesian_file.append(time[i])
cartesian_file.append(x[i])
cartesian_file.append(y[i])
cartesian_file.append(z[i])
CZML_Generate(Start_time, end_time, cartesian_file)
def getTodayCsv(a):
table = pd.read_csv("catStock/" + a + 'today.csv')
columns = [
'time', 'open', 'high', 'low', 'close', 'upanddown', 'transaction'
]
table.columns = columns
table = table.sort_index(ascending=False)
time = np.array(table.time)
timelist = time.tolist()
opena = np.array(table.open)
openlist = opena.tolist()
high = np.array(table.high)
highlist = high.tolist()
low = np.array(table.low)
lowlist = low.tolist()
close = np.array(table.close)
closelist = close.tolist()
transaction = np.array(table.transaction)
translist = transaction.tolist()
re = []
re.append(openlist[-1])
z = max(highlist)
re.append(str(z))
z = min(lowlist)
re.append(str(z))
re.append(closelist[1])
z = sum(translist)
re.append(str(z))
return timelist, closelist, translist, re
def get_xy(self, CH: str):
"""
:param CH: channel
:return: data, time, time_unit, data will be in volts, time in time units, everything will be in lists
"""
data, time, t_unit = self.get_data_points_from_channel(CH)
return data.tolist(), time.tolist(), t_unit
def saveJointData(path):
global t, JSposition
test_time = datetime.datetime.now().strftime("%y%m%d%H%M%S")
JSPosition_PATH = path['robotCalibration'] + 'test/' + test_time + '_jsposition.yaml'
TIME_PATH = path['robotCalibration'] + 'test/' + test_time + '_js_time.yaml'
ret = np.array(JSposition)
time = np.array([t[i] - t[0] for i in range(len(t))])
with open(JSPosition_PATH, 'w') as f:
yaml.dump(ret.tolist(), f, default_flow_style=False)
with open(TIME_PATH, 'w') as f:
yaml.dump(time.tolist(), f, default_flow_style=False)
t = []
JSposition = []
print('Data grabbed with series number: %s' % (test_time))
def poly_scale(self, p, ind=None, groupby=None):
"""Scale growth data by a polynomial of degree p, using the first ind datapoints, grouping by groupby."""
if ind == None:
ind = 5
if groupby is None:
group = {(None, self.key.index)}
else:
group = self.key.groupby(groupby)
time = self.data.time.iloc[:ind]
for k, index in six.iteritems(group.groups):
temp = self.data.loc[:, index]
od = temp.values[:ind, :].ravel()
coeff = np.polyfit(time.tolist() * temp.shape[1], od, p)
temp = temp - np.polyval(coeff, self.data.time.values[0])
self.data.loc[:, index] = temp
def poly_scale(self,p,ind=None,groupby=None):
"""Scale growth data by a polynomial of degree p, using the first ind datapoints, grouping by groupby."""
if ind == None:
ind = 5
if groupby is None:
group = {(None,self.key.index)}
else:
group = self.key.groupby(groupby)
time = self.data.time.iloc[:ind]
for k,index in group.groups.iteritems():
temp = self.data.loc[:,index]
od = temp.values[:ind,:].ravel()
coeff = np.polyfit(time.tolist()*temp.shape[1],od,p)
temp = temp - np.polyval(coeff,self.data.time.values[0])
self.data.loc[:,index] = temp
def process_file(self, path, an_object):
#[START function1] a sub function for extfit
def function1(x):
k = 1.0433
c = 3.7
w = 2.5
value = (pow(k,-1*x)*x - pow(k, -1*c)*c)/w
return value
#[END function1]
#[START extfit] Extreme Fit Function
def extfit(x):
y = -0.9745
a = 3.5
value = y + a*math.exp(-1*math.exp(-1*function1(x)) - function1(x) + 1 )
return value
#[END extfit]
#[START Highpass Filter]
def hipassfilter(x):
f0 = 0.75
a = 0.975
b = 0.025
k = 10
value = -1*b + a/(1 + math.exp( -1*k*(x + -1*f0)) )
return value
#[END Highpass Filter]
# Creating Datastack class for saving roughness data
class Datastack(ndb.Model):
material = ndb.StringProperty()
roughness = ndb.FloatProperty()
create_date = ndb.DateTimeProperty(auto_now_add=True)
""" Prepare empty array to hold data """
sampleArray = np.zeros((990,3))
""" Open storage file and read data into lines as list """
gcs_file = gcs.open(path)
logging.info('file %s was read.', an_object)
lines = gcs_file.read().split('\n')
num_data = len(lines)
gcs_file.close()
#self.tmp_filenames_to_clean_up.append(path)
""" Split each lines by comma and fill sampleArray """
for index in range(1,num_data-2): # Skip first and last incomplete data
#logging.info(index)
fragmented_line = lines[index].split(',')
for i in range(3):
sampleArray[index][i] = float(fragmented_line[i]) # index for line #, i = 0 for time interval, i = 1 for fingerprint sensor, i = 2 for graphene sponge
#logging.info('%s %s %s', str(index), str(i), fragmented_line[i])
""" Put time data into sampleArray[i][0] and time domain signal into timeDomain[i] """
time = np.zeros(num_data-2)
timeDomain = np.zeros(num_data-2)
for i in range(1,num_data-2): # Skip 0th line
time[i] = sampleArray[i][0]/1000000 + time[i-1] # Accumulation of sampling interval
timeDomain[i] = sampleArray[i][1] # Copying to timeDomain array
""" Preparation of json data to send to compute engine for linear regression """
timeList = time.tolist()
timeDomainList = timeDomain.tolist()
# For checking
#for i in range(0,10):
# logging.info('%s %s',str(timeList[i]), str(timeDomainList[i]))
form_fields = {
'time' : timeList,
'timeDomain' : timeDomainList
}
form_data = json.dumps(form_fields)
headers = {'Content-Type': 'application/json'}
""" Sending json to compute engine """
result = urlfetch.fetch(
url='http://104.196.190.227/',
payload=form_data,
method=urlfetch.POST,
headers=headers)
""" Receving linear regression results for leveling timeDomain data """
jsonreturn = json.loads(result.content) # Converting to dic type
slope = str(jsonreturn['slope'])
intercept = str(jsonreturn['intercept'])
logging.info('Regression result, Slope: %s', slope)
logging.info('Regression result, Intercept: %s', intercept)
""" Leveling timeDomain data """
for i in range(1,num_data-2):
timeDomain[i] = timeDomain[i] - time[i]*float(slope)
length = num_data-2 # Length of data = number of data-2
aveSamInt = time[num_data-3]/length # Average sample internal
logging.info('Number of data : %s', str(length))
logging.info('Average Sampling Interval : %s sec', str(aveSamInt))
logging.info('Sample Frequency : %s Hz', str(1/aveSamInt))
logging.info('Total Duration of Sampling : %s sec', str(length*aveSamInt))
"""[START] Plotting and saving """
plt.subplot(2,3,1) # Plot for time-domain signal
plt.plot(time, timeDomain, 'k-')
plt.xlabel('time (s)')
plt.ylabel('amplitude')
plt.subplot(2,3,2) # Plot for FFT in dB in frequency axis
pad = np.zeros(length*15.5) # zero padding by 32 times in total length
pad1 = np.append(pad,timeDomain)
pad2 = np.append(pad1,pad) # zero-padded timeDomain
padlen = len(pad2)
Y = np.fft.rfft(pad2) # rfft calculates only positive frequency.
logging.info('length of padded data : %s', str(padlen))
logging.info('length of FFT result : %s', str(len(Y)))
freq = np.linspace(0, np.around(1/aveSamInt/2), str(len(Y)))
logging.info('Length of freq window : %s', str(len(freq))) # length of freq = length of Y
""" Applying highpass filter to remove low freq noise """
for i in range(0, len(freq)):
Y[i] = Y[i]*hipassfilter(freq[i])
Y2 = np.zeros(len(freq)) # For displaying Y in dB
for i in range(0,len(freq)):
Y2[i] = 10*math.log(abs(Y[i]),10)
#plt.ylim(-1,1)
plt.xlim(0,100)
plt.plot(freq, Y2, 'r-')
plt.xlabel('freq (Hz)')
plt.ylabel('|Y(freq) dB|')
lamda = np.zeros(len(freq)) # Lamda as Period length
lamda[0]=20
ExtremeFit = np.zeros(len(freq))
for i in range(1, len(freq)):
lamda[i] = 10/freq[i] # 1cm/sec = 10 mm/sec, lamda = 10 mm/sec divided by freq
ExtremeFit[i] = extfit(lamda[i])
plt.subplot(2,3,3) # Plot for FFT in amplitude in period axis
plt.xlim(0,20)
#plt.ylim(-30,30)
plt.plot(lamda, Y2, 'r-')
plt.xlabel('Period (mm)')
plt.ylabel('Y(mm) in Amp')
plt.subplot(2,3,4) # Plot for Extreme Fit function in period
plt.xlim(0,20)
plt.plot(lamda, ExtremeFit, 'k-')
plt.xlabel('Period (mm)')
plt.ylabel('Extreme Fit Function')
plt.subplot(2,3,5) # Plot for Abs(FFT) in period
plt.plot(lamda, abs(Y2), 'r-')
plt.xlim(0,20)
#plt.ylim(0,30)
plt.xlabel('Period (mm)')
plt.ylabel('|Y(mm) in Amp|')
plt.subplot(2,3,6) # Plot for Abs(FFT)*Extfit in period
Z = np.zeros(len(freq))
for i in range(1, len(freq)):
Z[i] = aveSamInt*math.fabs(Y2[i])*extfit(lamda[i])
plt.plot(lamda, Z, 'r-')
plt.xlim(0,20)
plt.xlabel('Period (mm)')
plt.ylabel('|Z(Period)|')
""" Saving plots as an image in a bucket """
rv = StringIO.StringIO()
plt.savefig(rv, format = 'png')
image_name = list(an_object)
image_name = image_name[0:-4]
an_object = "".join(image_name)
image_file = gcs.open('/testcloudstorage-1470232940384.appspot.com/' + an_object + '.png', 'w', content_type = 'image/png')
image_file.write(rv.getvalue())
image_file.close()
plt.clf()
rv.close()
""" [END] Plotting and saving """
logging.info('image was saved')
""" summation of weighted fourier transform """
sum = 0
for i in range (0, length/2):
sum = sum + Z[i]
""" Storing roughness factor in datastore """
root = ndb.Key('datastack', 'rootdata')
aData = Datastack(parent = root)
aData.material = an_object
aData.roughness = sum
aData.put()
""" Add a task to post the calculation result """
q = taskqueue.Queue('pull-queue')
tasks = []
payload_str = str(sum)
tasks.append(taskqueue.Task(payload=payload_str, method='PULL'))
q.add(tasks)
logging.info('Calculation result, ' + payload_str + ' was saved in queue.')
def findTimeList(data):
time = data[:,1]
time = time.tolist()
return time
def meansnr(snr,
rxid=None,
hk_hours=None,
endtime=None,
minelev=0,
pos='topbot',
suffix='',
**unused):
"""Plot mean snr vs. time from a array of SNR records.
If rxid is given, an image is written out in the current directory, named
AVG-RX##-DOY.png. Otherwise the figure is returned.
"""
if hk_hours is not None:
thresh, endtime = _thresh(hk_hours, endtime)
snr = snr[findfirstgt(snr['time'], thresh
):findfirstgt(snr['time'], endtime)]
if minelev:
snr = snr[snr['el'] > minelev]
if not len(snr):
info('No records', 'for RX{:02}'.format(rxid) if rxid else '',
'in the given time period', thresh, 'to', endtime)
return
time, idx = np.unique(snr['time'], return_index=True)
idx.sort() # if the times aren't in order for some reason
time = snr['time'][idx]
doy = mode(time.astype('M8[D]')).tolist() # most common day
idx = np.append(idx, len(snr))
means = []
pmeans = []
for a, b in zip(idx, idx[1:]):
tsnr = snr[a:b]['snr'] / 10
means += [np.mean(tsnr)]
if np.count_nonzero(tsnr):
pmeans += [np.mean(tsnr[tsnr > 0])]
else:
pmeans += [0]
if rxid:
title = 'Rx{:02} {:%Y-%m-%d}: Mean SNR'.format(rxid, doy)
else:
title = 'VAPR {:%Y-%m-%d}: Mean SNR'.format(doy)
if minelev:
title += ' over {}°'.format(minelev)
ylabl = 'Mean SNR (dB-Hz)'
if 'top' not in pos and minelev:
ylabl += ' over {}°'.format(minelev)
fig, ax = _gethouraxes([10, 3], pos, title=title, ylabel=ylabl)
pmeans = np.array(pmeans)
ax.plot(time.tolist(), means, 'b.', ms=2, label='All values')
ax.plot(time[pmeans > 0].tolist(),
pmeans[pmeans > 0],
'r.',
ms=2,
label='Positive only')
if hk_hours is not None:
ax.set_xlim(thresh.tolist(), endtime.tolist())
else:
ax.set_xlim(min(time.tolist()), max(time.tolist()))
_setsnrlim(
ax, pmeans) # add argument True to restrict y-range to observed values
ax.tick_params(axis='y', labelleft='on', labelright='on')
ax.legend(loc='best',
fontsize='small',
numpoints=1,
markerscale=4,
handletextpad=0.4,
handlelength=0.8)
if rxid:
fname = 'AVG-RX{:02}{}.png'.format(rxid, suffix)
fig.savefig(fname)
plt.close(fig)
return fname
return fig
