def get_train_data(df, train_size=1):
# df = pd.read_csv('data.csv')
col = [
'AWS/EC2 NetworkIn', 'AWS/EC2 NetworkOut',
'System/Linux MemoryUtilization', 'AWS/EC2 CPUUtilization'
]
# 0 = Timestamp, 1 = Net in, 2 = Net Out, 3 = Memory, 4 = CPU
# Feature generation
df_features = pd.DataFrame()
# df_features['NetInDiff'] = df[col[1]] - df[col[1]].shift(1)
# df_features['NetOutDiff'] = df[col[2]] - df[col[2]].shift(1)
# df_features['MemDiff'] = df[col[3]] - df[col[3]].shift(1)
# df_features['CPUDiff'] = df[col[4]] - df[col[4]].shift(1)
df_features[col] = df[col]
# del df_features['Timestamps']
df_features['wavelet'] = wavelet(df)
df_features = df_features.fillna(method='bfill')
col = df_features.columns
for column in col:
df_features[column] = smooth(df_features[column].values).tolist()
plt.plot(df_features[column])
plt.show()
df_features -= df_features.min()
df_features /= df_features.max()
# plt.imshow(df_features.corr())
# plt.show()
df_labels = pd.DataFrame()
for val in range(PREDICT_LEN):
df_labels['lag{0}'.format(val)] = df['AWS/EC2 CPUUtilization'].shift(
-val)
df_labels = df_labels.fillna(method='ffill')
df_labels -= df_labels.min()
df_labels /= df_labels.max()
values = df_features.values
labels = (df_labels.values[FEED_LEN:] + 1) / 2
del df_labels
del df_features
del df
features = []
for i in range(FEED_LEN, values.shape[0]):
features.append(list(values[i - FEED_LEN:i]))
del values
features = np.asarray(features)
x_ts, y_ts = np.array([None]), np.array([None])
if train_size < 1:
features, labels, x_ts, y_ts = \
train_test_split(features, labels, train_size=train_size, shuffle=False)
return features, labels, x_ts, y_ts
def generate_thread(name, label, queue):
global count, MAX_FILE_NUMBER
output_file = open("thread_%s_%s"%(name,label), "w")
while (count <= MAX_FILE_NUMBER):
print 'job ', name
f = queue.get()
dimension = wavelet.wavelet(f)
line = format_output(enumerate(dimension), label)
output_file.write(line)
with output_lock:
count += 1
print count, f
count = 0
output_file.close()
def generate_thread(name, label, queue):
global count, MAX_FILE_NUMBER
output_file = open("thread_%s_%s" % (name, label), "w")
while (count <= MAX_FILE_NUMBER):
print 'job ', name
f = queue.get()
dimension = wavelet.wavelet(f)
line = format_output(enumerate(dimension), label)
output_file.write(line)
with output_lock:
count += 1
print count, f
count = 0
output_file.close()
def __init__(self, pic_dir='src/img.jpg', data=None):
super().__init__()
if data is None:
self.config = json.load(open('config.json'))
self.PIC = np.array(Image.open(pic_dir)).transpose(2, 0, 1)
else:
# Only for debug
self.PIC = np.expand_dims(data, axis=0)
# Define some parameters
self.BLOCK_SIZE = 5
self.BLK_STRIDE = 3
self.WINDOW_SIZE = 25
self.TH = 2500
self.MAX_COUNT = 400
self.WAVELET = wavelet()
self.ASSEMBLE_DICT = {}
data = (spio.loadmat(HomePath+FlickDir+'3_ScalColor256'))['data']
elif FeaType == 4:
data = (spio.loadmat(HomePath+FlickDir+'4_HomoText43'))['data']
elif FeaType == 5:
data = (spio.loadmat(HomePath+FlickDir+'5_EdgeHist150'))['data']
OutFile = OutPath+str(FeaType)+'_0605.csv'
fout = open(OutFile,'wb')
headers = 'qid NumMachine NumForEach k LevelRs Pivots RepeatTime NaiveCost Cost QCost'.split()
dw = csv.DictWriter(fout,headers,restval='NULL');
dw.writeheader()
fout.close()
seed(302)
FeaLen = data.shape[1]
Total = data.shape[0]
W = wavelet(FeaLen).T
RepeatTime = 30;
QList = sample(xrange(Total), RepeatTime)
#################################################################################
# Paramaters to be tuned.
kList = [1,5,10,15,20]
NumForEachList = [100,200,300,400,500]
NumMachList = [100,500,1000,1500,2000]
"""
kList = [2]
NumMachList = [2]
NumForEachList = [2]
"""
#################################################################################
name, ext = os.path.splitext(filename)
img = tv(image)
os.makedirs('denoise/tv', exist_ok=True)
imsave('denoise/tv/%s.png' % name, img)
if (sys.argv[2] == 'wavelet'):
for image in sys.argv[3:]:
filename = os.path.basename(image)
name, ext = os.path.splitext(filename)
img = wavelet(image)
os.makedirs('denoise/wavelet', exist_ok=True)
imsave('denoise/wavelet/%s.png' % name, img)
if (sys.argv[2] == 'wiener'):
for image in sys.argv[3:]:
filename = os.path.basename(image)
name, ext = os.path.splitext(filename)
img = wiener(image)
os.makedirs('denoise/wiener', exist_ok=True)
imsave('denoise/wiener/%s.png' % name, img)
