def build_walley_data(): # Read data from csv and build data set into list fm = fManager.FileManager() ld_prot = fm.csv2list(settings.walley_prot) ld_gene = fm.csv2list(settings.walley_gene) print (len(ld_prot), len(ld_prot[0])) print (len(ld_gene), len(ld_gene[0])) # Insert into database dbManager = dbm.DBManagerPG(settings.conn_string) dbManager.build_walley_data(ld_prot, 'p') dbManager.build_walley_data(ld_gene, 'g')
line = line.split() word = ' '.join(line[:-300]) embedding = [num for num in line[-300:]] bigEmbedding[word] = embedding print("getting small embedding..") wordbag.append('<PAD>') resultEmb.append(' '.join([str(0.0)] * 300)) for filename in filelist: data = read_tsv(filename) for line in data: text = line[0] text2id = [] sep_text = sepratewords(text).split() for word in sep_text: if word in smallEmbedding: continue elif word not in bigEmbedding: temp_emb = np.random.rand(300) smallEmbedding[word] = ' '.join( [str(num) for num in list(temp_emb)]) wordbag.append(word) resultEmb.append(smallEmbedding[word]) else: smallEmbedding[word] = ' '.join(bigEmbedding[word]) wordbag.append(word) resultEmb.append(smallEmbedding[word]) FileManager("../data/restaurant_wordbag.data", 'w', wordbag) FileManager("../data/restaurant_embedding.data", 'w', resultEmb)
exit(-1) if not os.path.isdir(local_dir): print(f'`local_dir` 的值 "{local_dir}" 所指向的目录不存在') exit(-1) if direction not in ('local-to-remote', 'remote-to-local'): print('`direction` 的值只能是 `local-to-remote` 或者 `remote-to-local`') exit(-1) with open(oss_config_file, 'rt', encoding='utf-8') as fp: oss_config = json.load(fp) if oss_type == 'tencent-cos': bucket = TencentCOSBucket(oss_config) else: bucket = AliyunOSSBucket(oss_config) file_manager = FileManager(local_dir) oss_synchronizer = OSSSynchronizer(file_manager, bucket) if direction == 'local-to-remote': print( f'正在同步 {local_dir}(本地)-> {oss_config.get("bucket", "Unknown Bucket")}(OSS)' ) oss_synchronizer.sync_from_local_to_oss() else: print( f'正在同步 {oss_config.get("bucket", "Unknown Bucket")}(OSS) -> {local_dir}(本地)' ) oss_synchronizer.sync_from_oss_to_local()
from utils import DicomFileManager, FileManager, BinaryClassifierComparator from visualizers import DataVisualizer import plotly if __name__ == "__main__": test_data_visualizer = False test_pre_processor = False test_dicom_file_manager = False test_file_manager = False test_classifier_comparator = True # ----------------------------------- FileManager ----------------------------------- if test_file_manager: fm = FileManager() training_path = 'covid19_repo/data/x_rays/covid' shape = (64, 64) shuffle = True x, y = fm.get_classifier_image_training_data( training_data_path=training_path, shape=shape, shuffle=shuffle) print(x.shape, y.shape) # ----------------------------------- DataVisualizer ----------------------------------- if test_data_visualizer: visualize_pca = False visualize_cnn_activations = True # ---------- Visualizing PCA ---------
import json import os import sys from tencent_cos import TencentCOSBucket from aliyun_oss import AliyunOSSBucket from utils import FileManager, OSSSynchronizer if __name__ == '__main__': with open('config/keybrl-mines.json', 'rt', encoding='utf-8') as fp: cos = TencentCOSBucket(json.load(fp)) local_file = FileManager('../dist') OSSSynchronizer(local_file, cos).sync_from_local_to_oss()
def main() -> None: """主函数 """ # 解析命令行参数 args = parser_args() # 开启调试模式 if args.debug: logger.addHandler(debug_console_handler) logger.setLevel(logging.DEBUG) logger.debug('DEBUG 模式已开启') else: logger.addHandler(normal_console_handler) logger.setLevel(logging.INFO) logger.debug('DEBUG 模式关闭') main_config_path = args.config or default_main_config_path config_encoding = args.config_encoding or default_config_encoding # 加载主配置文件 config = load_configs(config_path=main_config_path, validator=main_config_validator, encoding=config_encoding) if config is None: logger.error(f'加载主配置文件 "{main_config_path}" 失败。') exit(1) for config_item in config: oss_type = config_item['oss_type'] oss_config_path = config_item['oss_config'] local_dir = config_item['local_dir'] direction = config_item['direction'] # 加载 OSS 配置文件 oss_config = load_configs(config_path=oss_config_path, validator=None, encoding=config_encoding) if oss_config is None: logger.error(f'加载 OSS 配置文件 "{oss_config_path}" 失败。') exit(1) if oss_type == 'tencent-cos': bucket = QcloudCosBucket(oss_config) else: bucket = AliyunOssBucket(oss_config) file_manager = FileManager(local_dir) oss_synchronizer = OSSSynchronizer(file_manager, bucket) if direction == 'local-to-remote': logger.info( f'开始同步 {local_dir}(本地)-> {oss_config.get("bucket", "Unknown Bucket")}(OSS)' ) oss_synchronizer.sync_from_local_to_oss() else: logger.info( f'开始同步 {oss_config.get("bucket", "Unknown Bucket")}(OSS) -> {local_dir}(本地)' ) oss_synchronizer.sync_from_oss_to_local()
x = np.random.random((100, 10)) n_pca_components = 10 p_components = pr.pca(x, n_pca_components=n_pca_components) # -------------------------------------- Segment lungs -------------------------------------- if segment_lungs: pr = ChestXRayPreProcessor() training_path = 'covid19_repo/data/x_rays/covid' model_path = 'covid19_repo/u_net_model/Unet_model.json' weights_path = 'covid19_repo/u_net_model/Unet_model.h5' batch_size = 16 threshold = 0.1 verbose = 1 fm = FileManager() x = np.load('data/x_512.npy') segmented_lungs = pr.segment_lungs(x, model_path=model_path, weights_path=weights_path, batch_size=batch_size, threshold=threshold, verbose=verbose) plt.figure() plt.imshow(segmented_lungs[np.random.randint(0, len(segmented_lungs))]) if pre_process_cnn: pr = ImagePreProcessor() x = np.load('data/x_512.npy')[0:10] x = pr.pre_process_built_cnn(x, type_cnn='resnet50') plt.figure()