def spatial_reduction(image_name, neighbourhood):
pix = mu.get_image_as_cv2(image_name)
pix = pix.astype(np.uint16)
row, col = neighbourhood
h, w, c = pix.shape
print(pix.shape)
h2 = int(np.floor(h / row))
w2 = int(np.floor(w / col))
new_img = np.zeros((h2, w2, 3))
print(new_img.shape)
for i in range(0, w, col):
for j in range(0, h, row):
for k in range(0, c):
c1 = j
c2 = i
c3 = min(j + col + 1, h)
c4 = min(i + row + 1, w)
temp = pix[c1:c3, c2:c4, k]
j2 = min(int(np.floor(j / row)), new_img.shape[0] - 1)
i2 = min(int(np.floor(i / col)), new_img.shape[1] - 1)
new_img[j2, i2, k] = int(np.floor(np.mean(temp)))
dest_name = mu.save_img(new_img, image_name,
"_space" + str(neighbourhood).replace(',', '-'))
print(neighbourhood, pix.shape, new_img.shape, mu.get_file_size(dest_name),
mu.get_file_size(image_name))
def TestRead(order, episode_len=6):
if order == "omniglot":
if not os.path.exists("train.npz") or not os.path.exists("test.npz"):
MyUtils.read_omniglot()
input_path = "train.npz"
valid_path = "test.npz"
else:
raise NotImplementedError
if order == "omniglot":
input_size = (episode_len, 28, 28, 1)
else:
raise NotImplementedError
with open(input_path, "rb") as f:
input_npz = np.load(f)
inputs = {}
print('train', np.shape(input_npz.files))
for filename in input_npz.files:
inputs[filename] = input_npz[filename]
with open(valid_path, "rb") as f:
valid_npz = np.load(f)
valid_inputs = {}
print('test', np.shape(valid_npz.files))
for filename in valid_npz.files:
valid_inputs[filename] = valid_npz[
filename] # filename is the class label ,each class has 20 samples
def label(self, task_id, img_file_list, boundingbox):
LOG.debug('label function called! labeling following files:')
LOG.debug(img_file_list)
cache_manager = cachemanagement.cacheManager()
cur_dir =os.path.dirname(os.path.realpath(__file__))
# task_id is needed for separating labeling multiple objects
cache_dir = os.path.dirname(os.path.realpath(__file__)) + '/tmp/s3/'
cache_dir, img_file_list = cache_manager.cacheImageFromFileList(img_file_list, cache_dir)
img_file_list = map(lambda x: cache_dir+'/'+x, img_file_list)
input_img_file = cache_dir+'/'+ str(task_id) +'labeler_input_img_list.txt'
MyUtils.writeListToFile(img_file_list, input_img_file)
output_bx_file = cache_dir +'/'+ str(task_id) + 'labeler_output_bx_list.txt'
cmd = cur_dir+'/TrackingApp '
cmd += str(input_img_file) + ' '
cmd += str(output_bx_file) + ' '
for bx_vec in boundingbox:
cmd += str(bx_vec) + ' '
cmd += 'BOOSTING'
# call tracker
os.system(cmd)
# get input
output_bxes = []
with open(output_bx_file, 'r') as bx_file:
bxes = bx_file.read().splitlines()
for bx in bxes:
LOG.debug(bx)
output_bx = ast.literal_eval(bx)
output_bxes.append(output_bx)
return output_bxes
def main():
global klistener
global mlistener
MyUtils.create_file("EviacamTest/")
while not done:
continue
sys.exit()
def on_click(x, y, button, pressed):
if pressed:
if str(button) == "Button.left":
MyUtils.time_convert(
time.time() - (start_time if start_time > 0 else on_time),
"Left Click")
else:
MyUtils.time_convert(
time.time() - (start_time if start_time > 0 else on_time),
"Right Click")
def main():
global klistener
global done
MyUtils.create_file("LeapMotionTest/")
listener = LeapMotionListener()
controller = Leap.Controller()
controller.add_listener(listener)
while not done:
continue
controller.remove_listener(listener)
def label_old(self, task_id, img_file_list, boundingbox):
LOG.debug('camshift labeler label function called! labeling following files:')
LOG.debug(img_file_list)
cache_manager = cachemanagement.cacheManager()
cache_dir = os.path.dirname(os.path.realpath(__file__)) + '/tmp/s3'
cache_dir, img_file_list = cache_manager.cacheImageFromFileList(img_file_list, cache_dir)
img_file_list = map(lambda x: cache_dir+'/'+x, img_file_list)
MyUtils.writeListToFile(img_file_list, cache_dir+'/'+'labeler_input_img_list.txt')
initialized = False
result_bounding_boxes=[boundingbox]
for each_img_file in img_file_list:
LOG.debug('read in image file' + each_img_file)
self.frame = cv2.imread(each_img_file)
vis = self.frame.copy()
hsv = cv2.cvtColor(self.frame, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(hsv, np.array((0., 60., 32.)), np.array((180., 255., 255.)))
if not initialized:
# set up initial labeled img
x0, y0, width, height = boundingbox
x1 = x0 + width
y1 = y0 + height
self.track_window = (x0, y0, width, height)
hsv_roi = hsv[y0:y1, x0:x1]
mask_roi = mask[y0:y1, x0:x1]
hist = cv2.calcHist( [hsv_roi], [0], mask_roi, [16], [0, 180] )
cv2.normalize(hist, hist, 0, 255, cv2.NORM_MINMAX);
self.hist = hist.reshape(-1)
initialized = True
continue
# self.show_hist()
prob = cv2.calcBackProject([hsv], [0], self.hist, [0, 180], 1)
prob &= mask
term_crit = ( cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 1 )
track_box, self.track_window = cv2.CamShift(prob, self.track_window, term_crit)
# pdb.set_trace()
# get rectangle
track_rect = cv2.boundingRect(cv2.boxPoints(track_box))
# try:
# pt1 = (track_rect[0],track_rect[1])
# pt2 = (track_rect[0] + track_rect[2], track_rect[1] + track_rect[3])
# cv2.rectangle(vis, pt1, pt2, (0, 255, 0), 2)
# cv2.ellipse(vis, track_box, (0, 0, 255), 2)
# except:
print track_rect
result_bounding_boxes.append(list(track_rect))
return result_bounding_boxes
def on_press(key):
global start_time
global spaceFlag
global done
if str(key) == "Key.space" and not spaceFlag:
if start_time == 0:
start_time = time.time()
MyUtils.time_convert(0, "----------Test start!----------")
else:
MyUtils.time_convert(time.time() - start_time, "---------- Test end! ----------")
spaceFlag = True
MyUtils.f.close()
done = True
def read_message(self, sc, addr):
message = sc.recv(1024)
split_message = MyUtils.MessageHandler(message).message_loads()
if split_message[0] == 'need_page':
pass
elif split_message[0] == 'change_page':
pass
else:
sc.send(
MyUtils.MessageBuilder(
['No exist function: ' + split_message[0]],
'error').get_message())
sc.close()
def humomentDetect(VideoName):
myFrames = []
result_path = "humomentDetectResult"
MyUtils.makeDir(result_path)
cap = cv2.VideoCapture(VideoName) # 提取视频
index = -1
list_hu_dist = []
while (cap.isOpened()):
ret, frame = cap.read()
if ret == False:
break
index += 1
# if index < 600:
# continue
if index >= 3563:
break
img = frame
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
moments = cv2.moments(img_gray)
humoments = cv2.HuMoments(moments)
humoments = np.log(np.abs(humoments)) # 7个不变矩
temp_frame = Frame()
temp_frame.hu = humoments
temp_frame.index = index
temp_frame.img = img.copy()
myFrames.append(temp_frame)
if len(myFrames) >= 2:
last_Frame = myFrames[len(myFrames) - 2]
tempdist = (last_Frame.hu[0] - temp_frame.hu[0])**2 + (
last_Frame.hu[1] - temp_frame.hu[1])**2 + (last_Frame.hu[2] -
temp_frame.hu[2])**2
last_Frame.hu_dist = tempdist
print(tempdist)
list_hu_dist.append(tempdist)
max_value = max(list_hu_dist)
print("max_value", max_value)
threshold = 10
num = 0
for i in range(0, len(myFrames) - 1):
if myFrames[i].hu_dist >= threshold:
num += 1
print(myFrames[i].index)
cv2.imwrite(result_path + "/" + str(myFrames[i].index) + ".png",
myFrames[i].img)
print(num)
def check_guess(guess):
with MySQL.UseDatabase(config) as cursor:
cursor.execute("""SELECT * from dict where word_sn = %s;""", (guess, ))
if len(cursor.fetchall()) == 0:
return False
else:
return True
def create_db_dict():
with MySQL.UseDatabase(config) as cursor:
cursor.execute("""drop table if exists dict;""")
cursor.execute(
"""create table dict(dict_id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, word_sn VARCHAR(24));"""
)
print("Table dict created!")
def _GetSectionItems(self, sectionname, parser):
""" Get the list of items in a section.
Parameters: sectionname (string): The name of the section.
sectionitems: The list of section items.
Returns: A dictionary with the parameters for the section,
or None if an error occured.
"""
#Default values:
configuration={'offactivated':'0','startwithoutinternet':'0','offtimeout':'360',
'startwithoutmouse':'0','startwithoutsound':'0','offteacher':'0','offwithoutlogin':'******',
'startwithoutmessages':'0', 'classroomname':MyUtils.classroomName(),'allowburstwol':'1'}
# Now, go through all the remaining items in the section,
# and check for options
try:
itemlist = self._ConfigParser.items(sectionname)
except:
return configuration
for i in itemlist:
# First, check to see if it is a parameter already dealt with.
if i[0] in self._OptionalItems:
optionname = i[0]
if self._translatedict.has_key(optionname):
optionname=self._translatedict[optionname]
configuration[optionname]=str(i[1]).replace('"','')
return configuration
def __init__(self, screen_w, screen_h, screen_x, screen_y, proc_width,
proc_height, mask_img, model, keys, keys_to_onehot,
onehot_names):
self.screen_w = screen_w
self.screen_h = screen_h
self.screen_x = screen_x
self.screen_y = screen_y
self.proc_width = proc_width
self.proc_height = proc_height
self.keys = keys
self.keys_to_onehot = keys_to_onehot
self.onehot_names = onehot_names
self.paused = False
self.mask = cv2.imread(mask_img, cv2.IMREAD_GRAYSCALE)
self.model = model
if self.mask is None:
print "Image mask not found"
exit(1)
elif self.mask.shape[1] != proc_width or self.mask.shape[
0] != proc_height:
print "Image mask has invalid size"
exit(1)
self.scc = MyUtils.ScreenCapture({
'width': screen_w,
'top': screen_y,
'height': screen_h,
'left': screen_x
})
cv2.namedWindow("input", cv2.WINDOW_NORMAL)
def create_db_hiscore():
with MySQL.UseDatabase(config) as cursor:
cursor.execute("""drop table if exists hiscore;""")
cursor.execute(
"""create table hiscore(hiscore_id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, score_amt INT NOT NULL, player_sn varchar(16) NOT NULL);"""
)
print("Table hiscore created!")
def hiscore_to_db():
if check_score_list():
with MySQL.UseDatabase(config) as cursor:
for item in scorelist:
cursor.execute(
"""INSERT INTO hiscore(player_sn, score_amt) VALUES(%s, %s);""",
(item[0], item[1]))
print("Scores inserted into database!")
def spatial_reduction(image_name, neighbourhood):
pix = mu.get_image_as_cv2(image_name)
pix = pix.astype(np.uint16)
row, col = neighbourhood
row_half = int(np.floor(row / 2))
col_half = int(np.floor(col / 2))
h, w, c = pix.shape
print(pix.shape)
new_img = pix.copy()
max0 = lambda t: max(t, 0)
max0func = np.vectorize(max0)
minw = lambda t: min(t, w)
minwfunc = np.vectorize(minw)
minh = lambda t: min(t, h)
minhfunc = np.vectorize(minh)
c1 = np.array(list(range(0, w))) - row_half
c1 = max0func(c1)
c2 = np.array(list(range(0, w))) + row_half
c2 = minwfunc(c2)
c3 = np.array(list(range(0, h))) - col_half
c3 = max0func(c3)
c4 = np.array(list(range(0, h))) + col_half
c4 = minhfunc(c4)
"""sumbox = lambda t: np.sum(pix[t[0]:t[1]],pix[t[2]:t[3]])
sumboxfunc = np.vectorize(sumbox)
sumbox = lambda t: np.sum(pix[t[0]:t[1],t[2]:t[3],t[5]])
sumboxfunc = np.vectorize(sumbox)"""
new_img = pix.copy()
for i in c1:
for j in c3:
for k in range(0, c):
temp = pix[c3[j]:c4[j] + 1, c1[i]:c2[i] + 1, k]
new_img[j, i, k] = np.sum(temp)
divisor = (neighbourhood[0] * neighbourhood[1]) - 1
new_img = (new_img - pix) / divisor
dest_name = mu.save_img(new_img, image_name,
str(neighbourhood).replace(',', '-'))
def train(self, task_id, training_set, testing_set=None, **kwargs):
print 'vm trainer called!'
# connect to database manager
self.db_manager = MongoDBManager(HogSvmTrainer.dbname, ip_addr='128.2.213.107')
self.storage_manager = StorageManager('test1')
local_tmp_dir = os.path.abspath(HogSvmTrainer.local_tmp_dir)
# download images to cache and get all bounding_boxes in memory:
obj_image_bounding_box = \
MyUtils.downloadImgWithBx(local_tmp_dir, self.db_manager, self.storage_manager, training_set)
# create output dir structure
output_dir_prefix = os.path.abspath('./classifier/hog_svm_')
# create an output folder for each task
if 'id' in kwargs:
output_dir = output_dir_prefix + 'id'
else:
timestamp = time.strftime("%m_%d_%y_%H_%M_%S")
output_dir = output_dir_prefix + timestamp
MyUtils.create_dir(output_dir)
# generate vm specific input files
# for each object, create a folder for train.txt, test.txt, and final classifier svm
obj_output_dirs = {}
for each_obj, obj_info in obj_image_bounding_box.iteritems():
train_obj_info, test_obj_info = self.split_train_file(obj_info, 10)
cur_obj_dir = output_dir + '/' + each_obj
obj_output_dirs[each_obj] = cur_obj_dir
MyUtils.create_dir(cur_obj_dir)
self.genInputFileForObj(each_obj, train_obj_info, cur_obj_dir+'/' + 'train_'+each_obj+'.xml')
self.genInputFileForObj(each_obj, test_obj_info, cur_obj_dir+'/' + 'test_'+each_obj+'.xml')
# split the training file randomly for generating testing file
for each_obj, each_obj_dir in obj_output_dirs.iteritems():
print 'calling dlib to train at folder {}'.format(each_obj_dir)
detector_name = each_obj + '.svm'
detector_path = each_obj_dir + '/' + detector_name
accuracy = self.train_with_dlib(each_obj_dir, detector_path, 'train_'+each_obj +'.xml', 'test_'+each_obj+'.xml')
# save the classifier into s3 and database
accuracy = "Testing accuracy: {}".format(accuracy)
MyUtils.zipdir(output_dir)
zip_file = output_dir + '.zip'
print 'uploading file to s3...'
key = self.storage_manager.uploadFile(zip_file)
print 'updating database...'
# TODO: maybe send this information to controller?
# add meta data info to database
classifier_metadata={'name': key,
'training_set':training_set,
'testing_set':testing_set,
'accuracy': 'see log',
'filepath': key}
self.db_manager.addClassifier(classifier_metadata)
return 'succesfully added ' + str(classifier_metadata)
def start_track(self, frame, droi):
self.selection=MyUtils.drectangle_to_tuple(droi)
hsv = cv2.cvtColor(frame, cv2.COLOR_RGB2HSV)
mask = cv2.inRange(hsv, np.array((0., 60., 32.)), np.array((180., 255., 255.)))
x0, y0, x1, y1 = self.selection
self.track_window = (x0, y0, x1-x0, y1-y0)
hsv_roi = hsv[y0:y1, x0:x1]
mask_roi = mask[y0:y1, x0:x1]
hist = cv2.calcHist( [hsv_roi], [0], mask_roi, [16], [0, 180] )
cv2.normalize(hist, hist, 0, 255, cv2.NORM_MINMAX)
self.hist = hist.reshape(-1)
def test_extract_sameids():
MyUtils.init_logging("ExploreQuestions.log")
q_df = pd.read_csv(RQ.QA_TRAINSUBSET_DFPATH, sep="_")
q_df_sorted = q_df.sort_values(by=["asin"],
axis=0,
ascending=True,
inplace=False,
kind='quicksort',
na_position='last')
q_df_asins = q_df_sorted["asin"].copy()
q_df_asins.drop_duplicates(inplace=True)
print(len(q_df_asins))
for asin in q_df_asins:
q_df_subset = q_df_sorted[q_df_sorted["asin"] == asin].copy()
subset_duplicates = q_df_subset[q_df_subset.duplicated(
'unixTime', keep=False) == True]
if len(subset_duplicates) > 0:
subset_duplicates.to_csv("ExploreQuestions.log", mode="a", sep="_")
def print_sell_change(content):
db_data = util.get_sell_history_data()
content += '\n'
for row in db_data:
fund_code = row[2]
# 根据基金码获取基金信息
fund_dict = get_fund_value_json(fund_code)
percentage = (float(fund_dict['dwjz']) - float(row[3])) / float(row[3])
# print(fund_dict['dwjz'])
content += '<' + fund_dict['name'] + '>截止到' + fund_dict[
'jzrq'] + '基金涨跌幅为：' + '%.2f%%' % (percentage * 100)
# print(content)
return content
def notify_status_to_server(self):
if self.identifier == "":
message = MyUtils.MessageBuilder([self.my_port_to_listen],
'new_client').get_message()
else:
message = MyUtils.MessageBuilder(
[self.my_port_to_listen, self.identifier],
'new_client').get_message()
print("Sending state to server: " + message.decode())
self.socket_to_server.send(message)
result = self.socket_to_server.recv(1024)
split_message = MyUtils.MessageHandler(result).message_loads()
print(result.decode())
if split_message[0] != 'error':
self.identifier = split_message[0]
i = 1
while i < len(split_message):
self.add_page_number(split_message[i])
i = i + 1
else:
print("error: " + split_message[1])
exit()
def start(self, interval, normalized_keys):
self.scc = MyUtils.ScreenCapture({
'width': self.screen_w,
'top': self.screen_y,
'height': self.screen_h,
'left': self.screen_x
})
self.interval = interval
self.last_time = time.time()
self.normalized_keys = normalized_keys
self.trainData = []
self.loadRawData()
self.paused = False
def dict_to_db():
if check_dict_lists():
with MySQL.UseDatabase(config) as cursor:
#All short words of guesslist
for guess in guesslist:
if len(guess) < 7:
cursor.execute("""INSERT INTO dict(word_sn) VALUES(%s);""",
(guess, ))
#Any other words in sourcelist (filtered for unwinnable)
for source in sourcelist:
cursor.execute("""INSERT INTO dict(word_sn) VALUES(%s);""",
(source, ))
print("Dictionaries inserted into database!")
def getdevice(device, position=None, brand=None):
url = apiUrl + 'getdevice'
postdata = {
'brand': brand,
'device': device,
'position': position,
'token': apiKey
}
headers = {'content-type': 'application/json'}
result = MyUtils.sendpostdata(url, postdata, headers)
try:
return result.json()
except AttributeError:
return result
def run(self):
with open(self.path2, 'w') as f:
f.write(
"# ################################## Sub2 ############################################### #\n"
)
for k, v in self.json_conf.items():
MyUtils.DebugInfo(GeneratePyFileByJson.getType(v),
"getFileType", 0)
if GeneratePyFileByJson.getType(v) in ("integer", "list"):
f.write(str(k) + ' = ' + str(v) + '\n')
elif GeneratePyFileByJson.getType(v) == "String":
f.write(str(k) + ' = \"' + str(v) + '\"\n')
else:
f.write(str(k) + ' = \"' + str(v) + '\"\n')
f.write("\n")
def __init__(self, dbname, debug=False, ip_addr=None):
# constants
self.IMG_COLL_NAME = 'image'
self.CLASSIFIER_COLL_NAME = 'classifier'
#vars
self.client=None
if ip_addr != None:
self.client = MongoClient(ip_addr)
else:
self.client = MongoClient()
self.db = self.client[dbname]
self.img_coll = self.db[self.IMG_COLL_NAME]
self.createIndexes()
self.logger = MyUtils.getLogger('mongdbmanager')
if debug:
pdb.set_trace()
def readOmdCurpus(fileName):
path = './OMD/' + fileName + '.csv'
inFile = []
with open(path, 'r', encoding='utf-8') as csvfile:
csvReader = csv.reader(csvfile)
for row in csvReader:
inFile.append(row[0:2])
tokenziedInput = [
inp[0:1] + mu.getTokenizedSentenceForDrKeshavarz(inp[1])
for inp in inFile
]
with open(fileName + '_lemmatized_' + '.csv', 'w',
encoding='utf-8') as csvfile:
spamwriter = csv.writer(csvfile, quoting=csv.QUOTE_MINIMAL)
for row in tokenziedInput:
spamwriter.writerow(row[0:1] + [' '.join(row[1:])])
def __init__(self):
# vm registries
self.label_vm = []
self.feature_vm = []
self.classify_vm = []
self.logger = MyUtils.getLogger('controller')
#file manager
self.data_manager = StorageManager('test1')
self.db_manager = DBManager()
#engines
# rpc server for now for handling client request
rpc_paths = ('/RPC2',)
# Create server
self.server = SimpleXMLRPCServer(("0.0.0.0", 8888))
self.server.register_introspection_functions()
self.server.register_function(self.addImage, 'addImage')
self.server.serve_forever()
def check_plan():
sql = 'select name,plan_url,detail_url from fund_plan'
db_data = util.get_all_db_data(sql)
content = ''
count = 0
for row in db_data:
# name = row[0]
plan_url = row[1]
detail_url = row[2]
_json = get_url_json(plan_url)
last_trade_date_fmt = _json.get('data').get('last_trade_date_fmt')
now_date_fmt = time.strftime('%Y-%m-%d', time.localtime(time.time()))
# 发车日期等于今天
if (last_trade_date_fmt == now_date_fmt):
detail_json = get_url_json(detail_url)
content = '<' + _json.get('data').get('plan_name') + '> 已发车，发车时间：' + get_current_time() +\
'.发车方案：'
for item in detail_json.get('data').get('items'):
content += '<' + item.get('fd_name') + '>'
send_dingding(count, content)
count += 1
def testing():
testing = r'FmtMessageBox(_LS(IDS_SDSw_Cannot_Find_Text_Editot_Program), sGENEdit);'
FuncConv = MyUtils.FunctionConvertor()
MyUtils.ConvertLiteralString2TCHARString(testing)
def main():
nSizeArg = len(sys.argv)
if nSizeArg < 2:
print('''잘못된 인수입니다.
"File Path" [extension] [/Temp]
"File Path" : 해당 폴더 포함 하위의 모든 파일이 수정 됩니다.
"extendsion" : 세미콜론으로 구분된 확장자가 들어갑니다. 파일을 필터링 할 때 사용됩니다.
[/Temp] : /Temp 입력 시 파일에 파일에 _Temp 을 붙여 생성합니다. 값 입력이 없을 경우 덮어쓰기 합니다.
[/NoneRecursive] : /NoneRecursive 입력 시 하위 폴더는 제외하고 해당 경로의 파일만 검색합니다.
ex) "C:\\MIDAS\\wbs\\src" ".cpp;.h"''')
src_path = sys.argv[1]
extension = []
bMakeTemp = "/Temp" in sys.argv
bNotFindUnder = "/NoneRecursive" in sys.argv
bFileListTXT = False
if os.path.isfile(src_path):
if os.path.splitext(src_path)[-1] == '.txt':
bFileListTXT = True
if nSizeArg > 2:
extension = sys.argv[2].split(';')
FileList = []
if bFileListTXT == False:
FileList = MyUtils.GetAllFileWithExt(src_path, extension,
bNotFindUnder)
else:
FileList = MyUtils.GetFileListFromTextFile(src_path, extension)
encoding_list = ['cp949', 'gb2312', 'utf-8', 'utf-8-sig', 'utf-16']
ErrorFiles = []
for FilePath in tqdm(FileList):
file_encoding = ''
lines = []
for encoding in encoding_list:
file_encoding = encoding
try:
file = open(FilePath, 'r', encoding=encoding)
lines = file.readlines()
except UnicodeDecodeError:
continue
break
if len(lines) == 0:
ErrorFiles.append(FilePath)
continue
new_lines = MyUtils.ConvertAll(lines)
split_list = os.path.splitext(FilePath)
file_out = file
if bMakeTemp == True:
file_out = open("".join(split_list[0:-1]) + '_Temp' +
split_list[-1],
'w',
encoding=file_encoding)
else:
file_out = open(FilePath, 'w', encoding=file_encoding)
file_out.writelines(new_lines)
for ErrorFile in ErrorFiles:
print("Can't Encoding This File! Path : {}".format(ErrorFile))
return True
ax.format_coord = Formatter(im)
plt.show()
in_size = 32*32
#im_w = 32
#im_h = 32
#in_filter_w = 3
#in_filter_h = 3
#model.add(Convolution2D(32, 3, in_filter_h, in_filter_w, border_mode='full'))
#im_w = im_w + in_filter_w - 1
#im_h = im_h + in_filter_h -1
result_image = MyUtils.ImageToTheanoTensor('3wolfmoon.jpg')
two_samples = np.vstack((result_image,result_image))
nhid=100
model = Sequential()
model.add(Convolution2D(1, 2, 3, 3, border_mode='full'))
#model.add(Convolution2D(1, 1, 3, 3, border_mode='valid'))
#model.add(Convolution2D(3, 1, 3, 3, border_mode='full'))
model.add(Activation('sigmoid', target=0.05))
#model.add(Dense(nhid, 256))
(X_train, y_train), (X_test, y_test) = keras.datasets.cifar10.load_data(test_split=0.1, seed=113)
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
import MyUtils
IDs = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21
]
for pt in IDs:
# subband = 'full'
# print ("pt= " + str(pt+1))
# MyUtils.TrainTest(pt, subband)
subband = 'bandAlpha'
print("pt= " + str(pt + 1))
MyUtils.TrainTest(pt, subband)
FileName_BestModel = 'BestModel_' + subband + '_excl' + str(pt) + '.h5'
MyUtils.NewTest(FileName_BestModel, subband + 'Filtered', pt)
subband = 'bandBeta'
print("pt= " + str(pt + 1))
MyUtils.TrainTest(pt, subband)
FileName_BestModel = 'BestModel_' + subband + '_excl' + str(pt) + '.h5'
MyUtils.NewTest(FileName_BestModel, subband + 'Filtered', pt)
subband = 'bandDelta'
print("pt= " + str(pt + 1))
MyUtils.TrainTest(pt, subband)
FileName_BestModel = 'BestModel_' + subband + '_excl' + str(pt) + '.h5'
MyUtils.NewTest(FileName_BestModel, subband + 'Filtered', pt)
# Removed 8,9 i.e. 7, 8
IDs = [0, 1, 2, 3, 4, 5, 6, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 21]
PATH = "C:/Elham/EEG_PatientIdentification/P1/P1_SHAM_TASK_AK_Saliency/from GPU_OLD/Models/"
subband = 'full'
ef1_S = np.zeros(22)
ef2_S = np.zeros(22)
ef3_S = np.zeros(22)
for pt in IDs:
print("pt= " + str(pt + 1))
FileName_BestModel = PATH + 'BestModel_full_excl' + str(pt) + '.h5'
part = 1
ef1_S[pt] = MyUtils.Test_Temporal3Parts_6bars_ShortLong(
FileName_BestModel, subband, pt, part, 'S')
part = 2
ef2_S[pt] = MyUtils.Test_Temporal3Parts_6bars_ShortLong(
FileName_BestModel, subband, pt, part, 'S')
part = 3
ef3_S[pt] = MyUtils.Test_Temporal3Parts_6bars_ShortLong(
FileName_BestModel, subband, pt, part, 'S')
new_ef1_S = [n for n in ef1_S if n > 0]
new_ef2_S = [n for n in ef2_S if n > 0]
new_ef3_S = [n for n in ef3_S if n > 0]
data = [new_ef1_S, new_ef2_S, new_ef3_S]
fig, ax = plt.subplots()
def train(self, task_id, training_set, testing_set, **kwargs):
print 'caffe train called'
self.output_dir_prefix = os.path.abspath('./classifier/caffenet_')
# create an output folder for each task
if None != task_id:
self.output_dir = self.output_dir_prefix + str(task_id)
else:
timestamp = time.strftime("%m_%d_%y_%H_%M_%S")
self.output_dir = self.output_dir_prefix + timestamp
# TODO: changed to user input parameter?
max_iter = 500
if 'iteration' in kwargs:
max_iter = kwargs['iteration']
MyUtils.create_dir(self.output_dir)
# connect to database manager
self.db_manager = MongoDBManager(CaffeNetTrainer.dbname, ip_addr='128.2.213.107')
self.storage_manager = StorageManager('test1')
# make intermediate folder
self.ori_image_dir = CaffeNetTrainer.local_tmp_dir + '/' +'original'
self.crop_image_dir = CaffeNetTrainer.local_tmp_dir + '/' +'crop'
self.cache_manager = cacheManager(self.ori_image_dir)
MyUtils.create_dir(self.ori_image_dir)
MyUtils.create_dir(self.crop_image_dir)
# get image file list and bounding boxes
# get objects regardless of their videos, since same object in different video belongs to same identitfy
# summarize to get all objects
obj_set = {}
for obj, vid in training_set.iteritems():
if obj not in obj_set:
obj_set[obj] = []
obj_set[obj].extend(vid)
print obj_set
# for each object, get images
training_set_obj_file_path ={}
for obj, vids in obj_set.iteritems():
# 1. download image
# 2. crop image based on bounding boxes
obj_image_path = []
obj_bounding_boxes_with_image = {}
for vid in vids:
obj_bounding_boxes_each_vid = self.db_manager.getBoundingBoxWithImageByVidAndObj(vid, obj)
# TODO: probably can be a bit more efficient. right now is downloading vid repeatedly if it appears
# multiple times in train yaml
self.cache_manager.cacheImageFiles(vid, dir=False)
obj_bounding_boxes_with_image.update(obj_bounding_boxes_each_vid)
for image, bx_list in obj_bounding_boxes_with_image.iteritems():
# self.storage_manager.download(image, self.ori_image_dir)
idx =0
for bx in bx_list:
im = Image.open(self.ori_image_dir+'/'+image)
left_x = bx[0]
left_y = bx[1]
right_x = left_x + bx[2]
right_y = left_y + bx[3]
output_file_path = self.crop_image_dir + '/' +os.path.splitext(image)[0] + '_' + str(obj) + str(idx) \
+ os.path.splitext(image)[1]
im.crop((left_x,left_y, right_x, right_y)).save(output_file_path)
obj_image_path.append(output_file_path)
idx +=1
training_set_obj_file_path[obj] = obj_image_path
# generate training file
training_set_obj_file_path, testing_set_obj_file_path =self.split_train_file(training_set_obj_file_path, 10)
self.generate_caffe_train_file(training_set_obj_file_path, self.output_dir + '/train.txt')
self.generate_caffe_train_file(testing_set_obj_file_path, self.output_dir +'/test.txt')
# generate label file for corresponds label to object names
self.generate_caffe_label_file(training_set_obj_file_path, self.output_dir + '/label.txt')
# modify network prototxt
num_output_category = len(obj_set)
train_file_path = os.path.abspath(self.output_dir + '/train.txt')
test_file_path = os.path.abspath(self.output_dir + '/test.txt')
output_train_val_path = self.output_dir + '/' + 'train_val_custom.prototxt'
output_solver_path = self.output_dir + '/' + 'solver_custom.prototxt'
output_deploy_path = self.output_dir + '/' + 'deploy_custom.prototxt'
# fine-tuning output
self.custom_net_path = self.output_dir + '/train_val_custom.prototxt'
self.custom_snapshot_prefix= self.output_dir + '/caffenet_custom'
self.output_layer_name ="fc8_custom"
self.output_net_name="CaffeNetCustom"
# original model
self.original_output_layer_name ='fc8'
# reference design file locations
input_file_prefix = os.path.abspath('./bvlc_reference_caffenet')
train_file ='train_val.prototxt'
solver_file ='solver.prototxt'
deploy_file ='deploy.prototxt'
self.mod_caffe_net(input_file_prefix+'/' + train_file, num_output_category, train_file_path,
test_file_path, output_train_val_path)
self.mod_caffe_solver(input_file_prefix+ '/' +solver_file,
max_iter, output_solver_path)
self.mod_caffe_deploy(input_file_prefix + '/' + deploy_file,
num_output_category, output_deploy_path)
cmd ="/opt/caffe/build/tools/caffe train"
cmd += " -solver " + output_solver_path
cmd += " -weights ./bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel"
cmd += " | tee " + self.output_dir + '/' + 'log.txt'
print cmd
# issue train
os.system(cmd)
MyUtils.zipdir(self.output_dir)
zip_file = self.output_dir + '.zip'
print 'uploading file to s3...'
key = self.storage_manager.uploadFile(zip_file)
print 'updating database...'
# TODO: maybe send this information to controller?
# add meta data info to database
classifier_metadata={'name': key,
'training_set':training_set,
'testing_set':testing_set,
'accuracy': 'see log',
'filepath': key}
self.db_manager.addClassifier(classifier_metadata)
return
def __init__(self, name, **kwargs):
"""Constructor
Args:
name: Nombre del objeto creado
kwargs: Lista de parámetros aceptados:
loopback_window: Time-steps a pasado para utilizar en la predicción (24)
forward_window: Time-steps a futuro a predecir (4)
num_lstm_layers: Número de capas LSTM (3)
num_cells: Número de neuronas en cada capa (128)
num_dense_layers: Número de capas Dense intermedias (0)
batch_size: Número de muestras del batch (32)
suffle_enable: Flag para hacer 'suffling' en el entrenamiento (True)
tvt_csv_file: Archivo csv con datos históricos train-validate-test
verbose: Flag de depuración
"""
self.name = name
self.lbw = None
self.fww = None
self.nll = None
self.nlc = None
self.ndl = None
self.ndc = None
self.bs = None
self.sf = None
_dbg = False
self.df = None
self.dfapp = None
self.num_outputs = 0
self.num_inputs = 0
self.num_in_steps = self.lbw
self.num_out_steps = self.fww
self.sts_df = None
self.sts_src = None
self.sts_scaled = None
self.scaler = None
self.x_train = None
self.y_train = None
self.x_validation = None
self.y_validation = None
self.x_test = None
self.y_test = None
self.model = None
self.callbacks_list = None
self.columns_to_include = None
self.columns_to_exclude = None
self.indicators_to_generate = None
for key,val in kwargs.items():
if key=='loopback_window':
self.lbw = val
self.num_in_steps = val
if key=='forward_window':
self.fww = val
self.num_out_steps = val
if key=='num_lstm_layers':
self.nll = val
if key=='num_cells':
self.nlc = val
self.ndc = val
if key=='num_dense_layers':
self.ndl = val
if key=='batch_size':
self.bs = val
if key=='suffle_enabled':
self.sf = val
if key=='verbose':
if val=='full':
_dbg = True
if key=='tvt_csv_file':
if _dbg:
print('Cargando histórico...')
self.df = utils.load_hist(val, sep=';', remove_weekends=True)
if _dbg:
print(self.df.head())
print('Incluyendo indicadores...')
self.df = utils.add_indicators(self.df, applied=['HIGH','LOW'], base_cols=['OPEN','HIGH','LOW','CLOSE','OC2','HLC3','OHLC4'])
self.num_outputs = 2
self.df['OUT_HIGH'] = self.df.HIGH.shift(-1)
self.df['OUT_LOW'] = self.df.LOW.shift(-1)
self.num_inputs = self.df.shape[1] - self.num_outputs
if _dbg:
print(self.df.head())
print('Parseando a supervisado con ins={}, outs={}...'.format(self.num_inputs,self.num_outputs))
self.sts_df = self.series_to_supervised(self.df, self.num_inputs, self.num_outputs, self.lbw, self.fww)
if _dbg:
print(self.sts_df.head())
print('Normalizando...')
self.sts_scaled, self.scaler = self.normalize_data(self.sts_df, None, self.name+'_scaler.save')
if _dbg:
print(self.sts_scaled.head())
print('Preparando pares XY...')
self.x_train, self.y_train, self.x_validation, self.y_validation, self.x_test, self.y_test = self.prepare_training_data(self.sts_scaled, self.bs, 4, 1, 0.8, _dbg)
if _dbg:
print('Contruyendo red...')
self.model, self.callbacks_list = self.build_net(_dbg)
if _dbg:
print(self.model.summary())
def train():
hparams = define_flags()
hparams.seq_len = episode_len = hparams.n * hparams.k + 1
if hparams.dataset == "omniglot":
if not os.path.exists(
"/youedata/home/zhangyuhan/Data/OmniglotNPZ/train.npz"
) or not os.path.exists(
"/youedata/home/zhangyuhan/Data/OmniglotNPZ/test.npz"):
MyUtils.read_omniglot()
input_path = "/youedata/home/zhangyuhan/Data/OmniglotNPZ/train.npz"
valid_path = "/youedata/home/zhangyuhan/Data/OmniglotNPZ/test.npz"
else:
raise NotImplementedError
if hparams.dataset == "omniglot":
input_size = (episode_len, 28, 28, 1)
else:
raise NotImplementedError
with open(input_path, "rb") as f:
input_npz = np.load(f)
inputs = {}
for filename in input_npz.files:
inputs[filename] = input_npz[filename]
with open(valid_path, "rb") as f:
valid_npz = np.load(f)
valid_inputs = {}
for filename in valid_npz.files:
valid_inputs[filename] = valid_npz[
filename] #filename is the class label ,each class has 20*4 samples
with tf.Graph().as_default():
q = FewShotInputQueue(inputs.keys(), inputs, hparams.n, hparams.k)
valid_q = FewShotInputQueue(valid_inputs.keys(), valid_inputs,
hparams.n, hparams.k)
generated_input, generated_label = tf.py_func(q.make_one_data, [],
[tf.float32, tf.int32])
batch_tensors = tf.train.batch([generated_input, generated_label],
batch_size=hparams.batch_size,
num_threads=4,
shapes=[input_size, (episode_len, )],
capacity=hparams.batch_size * 5)
valid_input, valid_label = tf.py_func(valid_q.make_one_data, [],
[tf.float32, tf.int32])
valid_batch_tensors = tf.train.batch(
[valid_input, valid_label],
batch_size=hparams.batch_size,
num_threads=4,
shapes=[input_size, (episode_len, )],
capacity=hparams.batch_size * 5)
#each batch with the shape(128,)
with tf.variable_scope("networks"):
embed_network = OmniglotEmbedNetwork(batch_tensors,
hparams.batch_size)
gnn = GNN_FSL(hparams, embed_network.output,
embed_network.label_placeholder, True)
with tf.variable_scope("networks", reuse=True):
valid_embed_network = OmniglotEmbedNetwork(valid_batch_tensors,
hparams.batch_size)
valid_gnn = GNN_FSL(hparams, valid_embed_network.output,
valid_embed_network.label_placeholder, False)
params_to_str = f"M_Nets_{hparams.input_dim}_{hparams.lr}_{hparams.n}_{hparams.k}_{hparams.num_gcn_blocks}_{hparams.label_cut}_{hparams.nn_option}"
log_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", params_to_str))
# Summaries
tf.summary.scalar("train_loss", gnn.loss)
tf.summary.scalar("train_acc", gnn.accuracy)
tf.summary.scalar("valid_loss", valid_gnn.loss)
tf.summary.scalar("valid_acc", valid_gnn.accuracy)
tf.summary.image("inputs",
valid_embed_network.input_placeholder[0],
max_outputs=episode_len)
# Supervisor
supervisor = tf.train.Supervisor(
logdir=log_dir,
save_summaries_secs=240,
save_model_secs=600,
global_step=gnn.global_step,
)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
print("Training start")
#使用supervisor 有checkpoint就从checkpoint中载入，没有就自动初始化哦
with supervisor.managed_session(config=config) as sess:
min_dev_loss = 10000
min_step = -1
STEP_NUM = 10000000
EARLY_STOP = 3000000
print_every = 500
HUGE_VALIDATION_CYCLE = print_every * 20
# writer = tf.summary.FileWriter(logdir=log_dir,graph=sess.graph)
# writer.close()
last_dev = time.time()
for step in range(STEP_NUM):
if supervisor.should_stop():
break
if step - min_step > EARLY_STOP:
print("Early stopping...")
break
if step % print_every != 0:
_, loss, acc, global_step = sess.run([
gnn.train_step, gnn.loss, gnn.accuracy, gnn.global_step
])
# print(gnn.predict_label.shape)
else:
_, loss, acc, global_step = sess.run([
gnn.train_step, gnn.loss, gnn.accuracy, gnn.global_step
])
loss, acc = sess.run([valid_gnn.loss, valid_gnn.accuracy])
current_time = time.time()
print(
f'Evaluate(Step {step}/{global_step} : valid loss({loss}), acc({acc}) in {current_time - last_dev} s'
)
_, train_data, loss, acc, target_label, predictlabel, propagation, diff, Lap, simi, cmpr = sess.run(
[valid_gnn.train_step, valid_gnn.concated_input, valid_gnn.loss, valid_gnn.accuracy, valid_gnn.target_label,
valid_gnn.predict_label, \
valid_gnn.propagation, valid_gnn.diff, valid_gnn.Lap, valid_gnn.simi, valid_gnn.cmpr])
print("predict is \n", predictlabel)
print('target is \n', target_label)
# print('diff is: \n', diff, '\n')
# print('compare is \n', cmpr)
# print('simi is', '\n', simi, '\n')
print('Lap is \n', Lap)
print('propagation is', '\n', propagation, '\n')
print("loss and acc \n", loss, acc, '\n')
# HUGE VALIDATION
if step != 0 and step % HUGE_VALIDATION_CYCLE == 0:
total_loss = total_acc = 0.
BATCH_NUM = 40
for _ in range(BATCH_NUM):
loss, acc = sess.run(
[valid_gnn.loss, valid_gnn.accuracy])
total_loss += loss * hparams.batch_size
total_acc += acc * hparams.batch_size
total_loss /= BATCH_NUM * hparams.batch_size
total_acc /= BATCH_NUM * hparams.batch_size
huge_data_acc_summary = tf.Summary()
huge_data_acc_summary.value.add(
tag="huge_data_accuracy", simple_value=total_acc)
supervisor.summary_computed(sess,
huge_data_acc_summary,
global_step=global_step)
huge_data_loss_summary = tf.Summary()
huge_data_loss_summary.value.add(
tag="huge_data_loss", simple_value=total_loss)
supervisor.summary_computed(sess,
huge_data_loss_summary,
global_step=global_step)
last_dev = current_time
if loss < min_dev_loss:
min_dev_loss = loss
min_step = step
if modified:
configfile=open(newconfig, 'wb')
new_configparser.write(configfile)
configfile.close()
def config_exist(newconfig):
"""Checks if a classroom is already configured
in the current config file"""
new_configparser = ConfigParser.ConfigParser()
new_configparser.read(newconfig)
aula = RootConfigs['classroomname']
return new_configparser.has_section(aula)
########################################################
APP_DIR=os.path.join(MyUtils.getHomeUser(),'.controlaula')
WWWPAGES='/usr/share/controlaula/frontend/www'
LANG='/usr/share/controlaula/lang'
TEACHER_UID=''
if not os.path.isdir(APP_DIR):
os.mkdir(APP_DIR)
if MyUtils. getLoginName()!='root':
IMAGES_DIR=os.path.join(APP_DIR,'loginimages')
if not os.path.isdir(IMAGES_DIR):
os.mkdir(IMAGES_DIR)
if not os.path.isdir(IMAGES_DIR + '/screenshots'):
os.mkdir(IMAGES_DIR + '/screenshots')
FILES_DIR=os.path.join(APP_DIR,'sendfile')
def __init__(self, server_ip=u"ws://localhost", server_port=9000):
self.logger=MyUtils.getLogger(__name__)
self.logger.setLevel(logging.DEBUG)
server_ip_port = server_ip + ':' +str(server_port)
self.ws=create_connection(server_ip_port)
