def on_upload_update_request(request, upload_id): files = request.FILES.getlist('files[]') if not len(files): raise MissingRequiredParameterException( params={'files[]'}, payload=(['hint', 'Please upload some files.'], )) if len(files) > 1: from upload_service.middleware.exception.InvalidRequestException import InvalidRequestException raise InvalidRequestException( params={'files[]'}, payload=(['hint', 'Please upload only one file.'], )) for file in files: if 'image/' not in file.content_type: from upload_service.middleware.exception.InvalidRequestException import InvalidRequestException raise InvalidRequestException( params={'files[]'}, payload=(['hint', 'Please upload only images.'], )) from models import Uploads from utility import auth, delete_file upload = Uploads.manager.filterUpload({ 'id': upload_id, 'user_id': auth(request).id }) if not len(upload): from upload_service.middleware.exception.UnauthorizedException import UnauthorizedException raise UnauthorizedException(payload=( ['hint', "You don't have permission to update this file"], )) else: delete_file(upload[0].path) return True
def validate_windows(input_windows, crop_path, sift_path): # Validate a window given the SIFT result of it # If the result is an empty SIFT file, it is invalid valid_windows = [] for w in input_windows: i = str(w.index) sift_file = sift_path + i + '_sift.txt' if is_non_zero_file(fpath=sift_file): valid_windows.append(w) else: delete_file(sift_file) delete_file(crop_path + i + '.jpg') return valid_windows
def batch_one_image_pre_VLAD(input_image_path, image_name, metadata_path, output_parent_path, unit_ratio_list, overlap_ratio): print "------Begin batch for image", image_name output_path = "%s%s/" % (output_parent_path, image_name) delete_file(file_path=output_path, isDir=True) temp_crop_path = '%stemp_crop/' % output_path image_path = '%s%s.jpg' % (input_image_path, image_name) print "\t======Generating windows" windows = exhaustive_search(image_path, metadata_path, unit_ratio_list, overlap_ratio) crop_window(image_path, temp_crop_path, windows=windows) temp_sift_path = '%stemp_sift/' % output_path print "\t======Sifting windows" sift_image_batch(input_path=temp_crop_path, output_path=temp_sift_path) windows = validate_windows(input_windows=windows, crop_path=temp_crop_path, sift_path=temp_sift_path) save_window_txt(windows, output_path, image_name) return temp_sift_path
def delete(self, request, upload_id): from models import Uploads from utility import delete_file, auth upload = Uploads.manager.filterUpload({ 'id': upload_id, 'user_id': auth(request).id }) if len(upload): delete_file(upload[0].path) return upload[0].delete() else: from upload_service.middleware.exception.NotFoundException import NotFoundException raise NotFoundException( params={'upload-id'}, payload=(['hint', 'Please provide a valid upload id'], ))
def reset_storage_location(self, testcase=None): """ Empty all downloaded clip files from the storage location. INPUT: testcase: a testcase object supplied when executing function as part of a testcase step. OUTPUT: successful: whether the function executed successfully or not. """ self.log.debug("Resetting the storage location at %s ..." % self.storage_loc) result = {'successful': False} try: # make list of all files in storage location files = listdir(self.storage_loc) # remove files in storage location for file in files: file_path = self.storage_loc + file # delete any free-floating files, or clear out directories if not a file deleted = delete_file(self.log, file_path, silent_warnings=True)['verified'] self.log.trace("Storage location reset.") result['successful'] = True except BaseException, e: self.handle_exception(e, operation="reset the storage location at %s" % self.storage_loc)
def del_file(self, orig_file, target_file): ret_o = 0 ret_t = 0 # orig_file, target_file = get_filename( # YM, SAVEDIR, browser, element, vars, data_dic) if (os.access(orig_file, os.R_OK)): ret_o = delete_file(orig_file) if (os.access(target_file, os.R_OK)): ret_t = delete_file(target_file) # retval[__RETCODE__]= ret_o+ret_t # if(retval[__RETCODE__]!=0): # retval[__RETMSG__]="del_file error " + orig_file + " " + target_file # retval[__STEP__]=0 return ret_o + ret_t
def util_funcs_test(): test_folder_path = "test/test_folder" t_name_main_pre = "Utility Functions Test" # Test 'does_file_exist' function name = t_name_main_pre + ": Does path exist? [No]" expected = False actual = util.does_file_exist(test_folder_path) test_util.compare_results(expected, actual, name) name = t_name_main_pre + ": Does path exist? [Yes]" expected = True actual = util.does_file_exist("test") test_util.compare_results(expected, actual, name) # Test 'create_folder_if_dne' function name = t_name_main_pre + ": Create folder that DNE" test_folder_path = util.create_folder_if_dne(test_folder_path) actual = util.does_file_exist(test_folder_path) expected = True test_util.compare_results(expected, actual, name) # Test 'delete_empty_folder' function name = t_name_main_pre + ": Delete empty folder" util.delete_empty_folder(test_folder_path) actual = util.does_file_exist(test_folder_path) expected = False test_util.compare_results(expected, actual, name) # Test 'create_file_if_dne' function name = t_name_main_pre + ": Create file" test_fp = "test/test_file.txt" util.create_file_if_dne( test_fp, ["This is a test \nWow! A new line and no need for writelines()!"]) expected = True actual = util.does_file_exist(test_fp) test_util.compare_results(expected, actual, name) # Test 'delete_file' function name = t_name_main_pre + ": Delete file" util.delete_file(test_fp) expected = False actual = util.does_file_exist(test_fp) test_util.compare_results(expected, actual, name)
def sift_image(input_path, image_name, output_path, params): # Process an image and save the results in a file, # by using the sift exe file with command lines, image_path = input_path + image_name params = "--edge-thresh 10 --peak-thresh 5" if params is None else params file_extension = image_name[image_name.rfind('.') + 1:] image_name = image_name[:image_name.rfind('.')] if file_extension != 'pgm': # create a pgm file im = Image.open(image_path).convert('L') image_path = '%s%s.pgm' % (input_path, image_name) im.save(image_path) result_path = "%s%s_sift.txt" % (output_path, image_name) if not exists(output_path): makedirs(output_path) cmmd = "./sift %s -o %s %s" % (image_path, result_path, params) system(cmmd) delete_file(image_path) return result_path
def batch_all_images(input_image_path, annotation_path, output_parent_path, unit_ratio_list, overlap_ratio, target, target_pos_path, vladVector, target_count=20, pca=True, k=30, max_iter=30, preVLAD=False, voca_path=None, dataset_mode=False, overlap_threshold=0.5): global_sift_path = '%sglobal_sift.txt' % (output_parent_path) image_name_list = get_target_pos_names(input_path=target_pos_path, target=target, target_count=target_count) sift_path_L = [] if preVLAD: delete_file(file_path=global_sift_path) for image_name in image_name_list: metadata_path = '%s%s.xml' % (annotation_path, image_name) win_sift_path = batch_one_image_pre_VLAD(input_image_path, image_name, metadata_path, output_parent_path, unit_ratio_list, overlap_ratio) sift_path_L.append(win_sift_path) append_file(dest_file=global_sift_path, input_path=win_sift_path) print "----------pre-VLAD Done" else: all_dir = list_all_files(input_path=output_parent_path + 'windows/', onlyDir=True) for d in all_dir: sift_path_L.append("%s/windows/%s/temp_sift/" % (output_parent_path, d)) print "----------pre-VLAD is enabled" if voca_path is None or not isfile(voca_path): print "~~~~~~~Learning vocabulary by the sift vectors of all windows of all images" vector_matrix = None if pca: vector_matrix = pca_dataset(input_path=global_sift_path) vocabulary = learn_vocabulary(input_path=global_sift_path, k=k, max_iter=max_iter, single_file=True, vector_matrix=vector_matrix) save_matrix(v=vocabulary, output_path=voca_path) print "~~~~~~~Learning vocabulary done" elif vladVector: print "~~~~~~~Loading existing vocabulary" vocabulary = load_matrix(input_path=voca_path) if vladVector: for i in xrange(len(image_name_list)): image_name = image_name_list[i] output_path = "%swindows/%s/%s" % (output_parent_path, image_name, image_name) print "\t======Creating VLAD vectors" vlad_vector_batch(input_path=sift_path_L[i], output_path=output_path, vocabulary=vocabulary) print "\t======VLAD Done for", image_name else: print "##########No VLAD vector generated...." if dataset_mode: print "^^^^^^^^^^Generate data set for global windows and VLAD" global_X_path = output_parent_path + "global_X.txt" global_Y_path = output_parent_path + "global_Y.txt" delete_file(global_X_path) delete_file(global_Y_path) for img_name in image_name_list: img_window_path = "%s/windows/%s/%s_windows.txt" % (output_parent_path, img_name , img_name) img_vlad_path = "%s/windows/%s/%s_vlad.txt" % (output_parent_path, img_name , img_name) metadata_path = '%s%s.xml' % (annotation_path, img_name) batch_one_image_dataset(global_X_path, global_Y_path, img_window_path, img_vlad_path, metadata_path, target, overlap_threshold=overlap_threshold) print "\tData set done for", img_name print "....................All done"
def config_status_test(): t_name_prefix = "Configuration JSON file Status Check Test" # Enable test mode config.TEST_ACTIVE = True # Test status checker with no file in test config path name = t_name_prefix + ": File not found" actual = config.config_status() expected = "DNE" test_util.compare_results(expected, actual, name) # Test status checker with template file in test config path name = t_name_prefix + ": Create New Template File" # Make the config file config.create_config_file() actual = config.config_status() expected = "template" test_util.compare_results(expected, actual, name) # Make test folders which will be used in setting up our config file test_dl_folder = util.create_folder_if_dne("test/test_downloads") test_edit_folder = util.create_folder_if_dne("test/test_edit") test_final_folder = util.create_folder_if_dne("test/test_final") # Test 'set_downloads_folder' function name = t_name_prefix + ": Setting downloads folder" config.set_downloads_folder(test_dl_folder) expected = test_dl_folder actual = config.get_downloads_folder() test_util.compare_results(expected, actual, name) name = t_name_prefix + ": Setting downloads folder [2]" config.set_downloads_folder("This Should Not Overwrite") expected = test_dl_folder actual = config.get_downloads_folder() test_util.compare_results(expected, actual, name) # Test 'update_downloads_folder' function name = t_name_prefix + ": Update downloads folder" config.update_downloads_folder("This Should Overwrite") expected = "This Should Overwrite" actual = config.get_downloads_folder() test_util.compare_results(expected, actual, name) # Test 'config_status' with one item filled out (and correctly) name = t_name_prefix + ": 1/3 with one bogus entry at pos 1" expected = "path_dne" actual = config.config_status() test_util.compare_results(expected, actual, name) # Test 'update_downloads_folder' function (again) name = t_name_prefix + ": Update downloads folder [2]" config.update_downloads_folder(test_dl_folder) expected = test_dl_folder actual = config.get_downloads_folder() test_util.compare_results(expected, actual, name) # Test 'config_status' with one item filled out (and correctly) name = t_name_prefix + ": 1/3 Complete" expected = "incomplete" actual = config.config_status() test_util.compare_results(expected, actual, name) # Test 'set_editing_folder' function name = t_name_prefix + ": Setting editing folder" config.set_editing_folder(test_edit_folder) expected = test_edit_folder actual = config.get_editing_folder() test_util.compare_results(expected, actual, name) name = t_name_prefix + ": Setting editing folder [2]" config.set_editing_folder("This Should Not Overwrite") expected = test_edit_folder actual = config.get_editing_folder() test_util.compare_results(expected, actual, name) # Test 'update_editing_folder' function name = t_name_prefix + ": Update editing folder" config.update_editing_folder("This Should Overwrite") expected = "This Should Overwrite" actual = config.get_editing_folder() test_util.compare_results(expected, actual, name) # Test 'config_status' with one item filled out (and correctly) name = t_name_prefix + ": 2/3 with one bogus entry at pos 2" expected = "path_dne" actual = config.config_status() test_util.compare_results(expected, actual, name) # Test 'update_editing_folder' function (again) name = t_name_prefix + ": Update editing folder [2]" config.update_editing_folder(test_edit_folder) expected = test_edit_folder actual = config.get_editing_folder() test_util.compare_results(expected, actual, name) # Test 'config_status' with 2 items filled out name = t_name_prefix + ": 2/3 Complete" expected = "incomplete" actual = config.config_status() test_util.compare_results(expected, actual, name) # Test 'set_destination_folder' function name = t_name_prefix + ": Setting destination folder" config.set_destination_folder(test_final_folder) expected = test_final_folder actual = config.get_destination_folder() test_util.compare_results(expected, actual, name) name = t_name_prefix + ": Setting destination folder [2]" config.set_destination_folder("This Should Not Overwrite") expected = test_final_folder actual = config.get_destination_folder() test_util.compare_results(expected, actual, name) # Test 'update_editing_folder' function name = t_name_prefix + ": Update destination folder" config.update_destination_folder("This Should Overwrite") expected = "This Should Overwrite" actual = config.get_destination_folder() test_util.compare_results(expected, actual, name) # Test 'config_status' with one item filled out (and correctly) name = t_name_prefix + ": 3/3 with one bogus entry at pos 3" expected = "path_dne" actual = config.config_status() test_util.compare_results(expected, actual, name) # Test 'update_editing_folder' function (again) name = t_name_prefix + ": Update destination folder [2]" config.update_destination_folder(test_final_folder) expected = test_final_folder actual = config.get_destination_folder() test_util.compare_results(expected, actual, name) # Test 'config_status' with everything filled out and existent name = t_name_prefix + ": Completed Config File" expected = "complete" actual = config.config_status() test_util.compare_results(expected, actual, name) # Return config module to its default state config.TEST_ACTIVE = False config.CONFIG_JSON = {} util.delete_file(config.TEST_CONFIG_FILE_PATH) # Delete temp files util.delete_empty_folder(test_dl_folder) util.delete_empty_folder(test_edit_folder) util.delete_empty_folder(test_final_folder)
def reset_vim_server(self, testcase=None, max_attempts=5, using_custom_database=False): """ Reset the ViM server to freshly installed state. INPUT testcase: a testcase object supplied when executing function as part of a testcase step. max attempts: the maximum number of attempts to perform the operation. using custom database: whether using a custom database or not (should not be deleted). OUPUT successful: whether the function executed successfully or not. """ self.log.debug("Resetting the ViM server to a fresh install state ...") result = {'successful': False} attempt = 1 while not result['successful'] and attempt <= max_attempts: try: # stop the server self.stop_vim_server(testcase=testcase) # disconnect from database self.db.disconnect_from_database() self.db.db_handle = None # define booleans for all operations needed to be done database_deleted = False db_shm_deleted = False db_wal_deleted = False license_deleted = False if not using_custom_database: # delete database files database_deleted = delete_file(self.log, self.database_path)['verified'] db_shm_deleted = delete_file(self.log, self.database_path + "-shm")['verified'] db_wal_deleted = delete_file(self.log, self.database_path + "-wal")['verified'] # delete license license_deleted = delete_file(self.log, self.license_path)['verified'] # reset files and folders storage_reset = self.reset_storage_location()['successful'] properties_deleted = delete_file(self.log, self.properties_path)['verified'] if (not using_custom_database and database_deleted and db_shm_deleted and db_wal_deleted and license_deleted and properties_deleted) or (using_custom_database and storage_reset and properties_deleted): self.log.trace("Reset the ViM server.") result['successful'] = True # restart the server self.start_vim_server(testcase=testcase) self.db.connect_to_database() self.db.db_handle = self.db.establish_handle_to_database()['handle'] # set up initial config sleep(3) self.log_in_to_vim(testcase=testcase, max_attempts=5) self.configure_vim_system_settings(testcase=testcase) elif attempt <= max_attempts: self.log.trace("Failed to reset the ViM server (attempt %s). " "Re-attempting in 5 seconds ..." % attempt) # stop the server self.stop_vim_server(testcase=testcase) elif attempt >= max_attempts: self.log.error("Failed to reset the ViM server.") break except BaseException, e: self.handle_exception(e, operation="reset the ViM server") result['successful'] = False # increment attempt += 1 sleep(5)
for da in output_dell_asset_L: if da is not None and da.svctag != "" and da.is_translation_updated: temp_path = existing_dell_asset_dir + da.svctag + history_DA_file_format save_object_to_path(value=da, output_path=temp_path) if save_dell_asset_excel(output_dell_asset_L, dell_asset_output_path): logger.info("保存结果到Excel文件 %s" % dell_asset_output_path) else: logger.error("保存Excel文件出错") else: logger.warn("-------Output for this job is empty") except Exception, e: logger.error(str(e)) logger.error(traceback.format_exc()) logger.info("\nFINISH\n") additional_text += "\n总用时 %s\n总共 %s个结果" % (diff_two_datetime( start_time, get_current_datetime()), len(output_dell_asset_L)) logger.info(additional_text) subject = subject_temp % ("[查询任务结束] ", get_current_datetime(is_date=True), svctag) if logger.has_error: additional_text += "\n查询程序出现错误,请等待解决。" save_object_to_path(value=logger, output_path=log_output_path) send_email(subject=subject, text=additional_text, config=config, cc_mode=logger.has_error or need_translation, attachment_path_L=[log_output_path, dell_asset_output_path]) delete_file(dell_asset_output_path) delete_file(log_output_path)
model.add(Dense(10)) model.add(Activation(relu)) model.add(Dense(1)) model.summary() model.compile(optimizer=Adam(learning_rate), loss="mse", ) # create two generators for training and validation train_gen = generate_next_batch() validation_gen = generate_next_batch() history = model.fit_generator(train_gen, samples_per_epoch=samples_per_epoch, nb_epoch=epochs, validation_data=validation_gen, nb_val_samples=number_of_validation_samples, verbose=1) # finally save our model and weights delete_file(model_name) delete_file(weights_name) json_string = model.to_json() with open(model_name, 'w') as outfile: json.dump(json_string, outfile) model.save_weights(weights_name)
def batch_all_images(input_image_path, annotation_path, output_parent_path, unit_ratio_list, overlap_ratio, target, target_pos_path, vladVector, target_count=20, pca=True, k=30, max_iter=30, preVLAD=False, voca_path=None, dataset_mode=False, overlap_threshold=0.5): global_sift_path = '%sglobal_sift.txt' % (output_parent_path) image_name_list = get_target_pos_names(input_path=target_pos_path, target=target, target_count=target_count) sift_path_L = [] if preVLAD: delete_file(file_path=global_sift_path) for image_name in image_name_list: metadata_path = '%s%s.xml' % (annotation_path, image_name) win_sift_path = batch_one_image_pre_VLAD(input_image_path, image_name, metadata_path, output_parent_path, unit_ratio_list, overlap_ratio) sift_path_L.append(win_sift_path) append_file(dest_file=global_sift_path, input_path=win_sift_path) print "----------pre-VLAD Done" else: all_dir = list_all_files(input_path=output_parent_path + 'windows/', onlyDir=True) for d in all_dir: sift_path_L.append("%s/windows/%s/temp_sift/" % (output_parent_path, d)) print "----------pre-VLAD is enabled" if voca_path is None or not isfile(voca_path): print "~~~~~~~Learning vocabulary by the sift vectors of all windows of all images" vector_matrix = None if pca: vector_matrix = pca_dataset(input_path=global_sift_path) vocabulary = learn_vocabulary(input_path=global_sift_path, k=k, max_iter=max_iter, single_file=True, vector_matrix=vector_matrix) save_matrix(v=vocabulary, output_path=voca_path) print "~~~~~~~Learning vocabulary done" elif vladVector: print "~~~~~~~Loading existing vocabulary" vocabulary = load_matrix(input_path=voca_path) if vladVector: for i in xrange(len(image_name_list)): image_name = image_name_list[i] output_path = "%swindows/%s/%s" % (output_parent_path, image_name, image_name) print "\t======Creating VLAD vectors" vlad_vector_batch(input_path=sift_path_L[i], output_path=output_path, vocabulary=vocabulary) print "\t======VLAD Done for", image_name else: print "##########No VLAD vector generated...." if dataset_mode: print "^^^^^^^^^^Generate data set for global windows and VLAD" global_X_path = output_parent_path + "global_X.txt" global_Y_path = output_parent_path + "global_Y.txt" delete_file(global_X_path) delete_file(global_Y_path) for img_name in image_name_list: img_window_path = "%s/windows/%s/%s_windows.txt" % ( output_parent_path, img_name, img_name) img_vlad_path = "%s/windows/%s/%s_vlad.txt" % (output_parent_path, img_name, img_name) metadata_path = '%s%s.xml' % (annotation_path, img_name) batch_one_image_dataset(global_X_path, global_Y_path, img_window_path, img_vlad_path, metadata_path, target, overlap_threshold=overlap_threshold) print "\tData set done for", img_name print "....................All done"
additional_text += "查询结果存在保修需要翻译" else: logger.info("No additional translation needed") if len(output_dell_asset_L) > 0: logger.info("~~~~~~~%s output results in total" % len(output_dell_asset_L)) # Save output into the csv_path and also existing dell asset logger.info("~~~~~~~Save output as existing dell assets") for da in output_dell_asset_L: if da is not None and da.svctag != "" and da.is_translation_updated: temp_path = existing_dell_asset_dir + da.svctag + history_DA_file_format save_object_to_path(value=da, output_path=temp_path) if save_dell_asset_excel(output_dell_asset_L, dell_asset_output_path): logger.info("保存结果到Excel文件 %s" % dell_asset_output_path) else: logger.error("保存Excel文件出错") else: logger.warn("-------Output for this job is empty") except Exception, e: logger.error(str(e)) logger.error(traceback.format_exc()) logger.info("\nFINISH\n") additional_text += "\n总用时 %s\n总共 %s个结果" % (diff_two_datetime(start_time, get_current_datetime()), len(output_dell_asset_L)) logger.info(additional_text) subject = subject_temp % ("[查询任务结束] ", get_current_datetime(is_date=True), svctag) if logger.has_error: additional_text += "\n查询程序出现错误,请等待解决。" save_object_to_path(value=logger, output_path=log_output_path) send_email(subject=subject, text=additional_text, config=config, cc_mode=logger.has_error or need_translation, attachment_path_L=[log_output_path, dell_asset_output_path]) delete_file(dell_asset_output_path) delete_file(log_output_path)