def build_walley_data():
# Read data from csv and build data set into list
fm = fManager.FileManager()
ld_prot = fm.csv2list(settings.walley_prot)
ld_gene = fm.csv2list(settings.walley_gene)
print (len(ld_prot), len(ld_prot[0]))
print (len(ld_gene), len(ld_gene[0]))
# Insert into database
dbManager = dbm.DBManagerPG(settings.conn_string)
dbManager.build_walley_data(ld_prot, 'p')
dbManager.build_walley_data(ld_gene, 'g')
line = line.split()
word = ' '.join(line[:-300])
embedding = [num for num in line[-300:]]
bigEmbedding[word] = embedding
print("getting small embedding..")
wordbag.append('<PAD>')
resultEmb.append(' '.join([str(0.0)] * 300))
for filename in filelist:
data = read_tsv(filename)
for line in data:
text = line[0]
text2id = []
sep_text = sepratewords(text).split()
for word in sep_text:
if word in smallEmbedding:
continue
elif word not in bigEmbedding:
temp_emb = np.random.rand(300)
smallEmbedding[word] = ' '.join(
[str(num) for num in list(temp_emb)])
wordbag.append(word)
resultEmb.append(smallEmbedding[word])
else:
smallEmbedding[word] = ' '.join(bigEmbedding[word])
wordbag.append(word)
resultEmb.append(smallEmbedding[word])
FileManager("../data/restaurant_wordbag.data", 'w', wordbag)
FileManager("../data/restaurant_embedding.data", 'w', resultEmb)
exit(-1)
if not os.path.isdir(local_dir):
print(f'`local_dir` 的值 "{local_dir}" 所指向的目录不存在')
exit(-1)
if direction not in ('local-to-remote', 'remote-to-local'):
print('`direction` 的值只能是 `local-to-remote` 或者 `remote-to-local`')
exit(-1)
with open(oss_config_file, 'rt', encoding='utf-8') as fp:
oss_config = json.load(fp)
if oss_type == 'tencent-cos':
bucket = TencentCOSBucket(oss_config)
else:
bucket = AliyunOSSBucket(oss_config)
file_manager = FileManager(local_dir)
oss_synchronizer = OSSSynchronizer(file_manager, bucket)
if direction == 'local-to-remote':
print(
f'正在同步 {local_dir}（本地）-> {oss_config.get("bucket", "Unknown Bucket")}（OSS）'
)
oss_synchronizer.sync_from_local_to_oss()
else:
print(
f'正在同步 {oss_config.get("bucket", "Unknown Bucket")}（OSS） -> {local_dir}（本地）'
)
oss_synchronizer.sync_from_oss_to_local()
from utils import DicomFileManager, FileManager, BinaryClassifierComparator
from visualizers import DataVisualizer
import plotly
if __name__ == "__main__":
test_data_visualizer = False
test_pre_processor = False
test_dicom_file_manager = False
test_file_manager = False
test_classifier_comparator = True
# ----------------------------------- FileManager -----------------------------------
if test_file_manager:
fm = FileManager()
training_path = 'covid19_repo/data/x_rays/covid'
shape = (64, 64)
shuffle = True
x, y = fm.get_classifier_image_training_data(
training_data_path=training_path, shape=shape, shuffle=shuffle)
print(x.shape, y.shape)
# ----------------------------------- DataVisualizer -----------------------------------
if test_data_visualizer:
visualize_pca = False
visualize_cnn_activations = True
# ---------- Visualizing PCA ---------
import json
import os
import sys
from tencent_cos import TencentCOSBucket
from aliyun_oss import AliyunOSSBucket
from utils import FileManager, OSSSynchronizer
if __name__ == '__main__':
with open('config/keybrl-mines.json', 'rt', encoding='utf-8') as fp:
cos = TencentCOSBucket(json.load(fp))
local_file = FileManager('../dist')
OSSSynchronizer(local_file, cos).sync_from_local_to_oss()
def main() -> None:
"""主函数
"""
# 解析命令行参数
args = parser_args()
# 开启调试模式
if args.debug:
logger.addHandler(debug_console_handler)
logger.setLevel(logging.DEBUG)
logger.debug('DEBUG 模式已开启')
else:
logger.addHandler(normal_console_handler)
logger.setLevel(logging.INFO)
logger.debug('DEBUG 模式关闭')
main_config_path = args.config or default_main_config_path
config_encoding = args.config_encoding or default_config_encoding
# 加载主配置文件
config = load_configs(config_path=main_config_path,
validator=main_config_validator,
encoding=config_encoding)
if config is None:
logger.error(f'加载主配置文件 "{main_config_path}" 失败。')
exit(1)
for config_item in config:
oss_type = config_item['oss_type']
oss_config_path = config_item['oss_config']
local_dir = config_item['local_dir']
direction = config_item['direction']
# 加载 OSS 配置文件
oss_config = load_configs(config_path=oss_config_path,
validator=None,
encoding=config_encoding)
if oss_config is None:
logger.error(f'加载 OSS 配置文件 "{oss_config_path}" 失败。')
exit(1)
if oss_type == 'tencent-cos':
bucket = QcloudCosBucket(oss_config)
else:
bucket = AliyunOssBucket(oss_config)
file_manager = FileManager(local_dir)
oss_synchronizer = OSSSynchronizer(file_manager, bucket)
if direction == 'local-to-remote':
logger.info(
f'开始同步 {local_dir}（本地）-> {oss_config.get("bucket", "Unknown Bucket")}（OSS）'
)
oss_synchronizer.sync_from_local_to_oss()
else:
logger.info(
f'开始同步 {oss_config.get("bucket", "Unknown Bucket")}（OSS） -> {local_dir}（本地）'
)
oss_synchronizer.sync_from_oss_to_local()
x = np.random.random((100, 10))
n_pca_components = 10
p_components = pr.pca(x, n_pca_components=n_pca_components)
# -------------------------------------- Segment lungs --------------------------------------
if segment_lungs:
pr = ChestXRayPreProcessor()
training_path = 'covid19_repo/data/x_rays/covid'
model_path = 'covid19_repo/u_net_model/Unet_model.json'
weights_path = 'covid19_repo/u_net_model/Unet_model.h5'
batch_size = 16
threshold = 0.1
verbose = 1
fm = FileManager()
x = np.load('data/x_512.npy')
segmented_lungs = pr.segment_lungs(x,
model_path=model_path,
weights_path=weights_path,
batch_size=batch_size,
threshold=threshold,
verbose=verbose)
plt.figure()
plt.imshow(segmented_lungs[np.random.randint(0, len(segmented_lungs))])
if pre_process_cnn:
pr = ImagePreProcessor()
x = np.load('data/x_512.npy')[0:10]
x = pr.pre_process_built_cnn(x, type_cnn='resnet50')
plt.figure()
