def cp_mount(source, destination):
try: # local..../date/CT/serid /media/tx-deepocean/Data/DICOMS/CT/pid/serid
if os.path.exists(destination) and len(os.listdir(destination)) == len(os.listdir(source)):
print '已存在 {}'.format(destination)
elif os.path.exists(destination) and len(os.listdir(destination)) != len(os.listdir(source)):
cmd = 'rm -rf '+destination
os.system(cmd)
if os.path.exists(source) and not os.path.exists(destination):
if len(os.listdir(source)) in range(70,999):
shutil.copytree(source, destination)
if len(os.listdir(destination)) != len(os.listdir(source)):
cmd = 'rm -rf '+destination
os.system(cmd)
else:
gdcmconv(destination)
re_name(destination)
filter_slice_protocol(destination)
desensitize_dir(destination)
else:
print '数量不足 {}'.format(source)
else:
print '目录不存在 {}'.format(source)
except:
out('磁盘挂载下载失败 {}'.format(source))
out(traceback.format_exc())
if os.path.exists(destination):
return len(os.listdir(destination))
else:
return 0
def isnotdicom(s):
try:
ds = dicom.read_file(s)
except:
try:
os.remove(s)
except:
out('没有该路径 {}'.format(s))
warn(traceback.format_exc())
return True
def main(redis_client, channel, TEST):
try:
patient_list = select_id_list_from_study_status(
TEST) # [patientid,body,exam_item,patient_source,modality,status]
patient_list = get_patient_dicom_path(redis_client, patient_list) #
out("current patient number {}".format(len(patient_list)))
if len(patient_list) > 0:
Multi(redis_client, channel, patient_list, TEST) #start
except:
out("main run error")
err(traceback.print_exc())
def unzip(source):
"""
解压提取
:param source: 文件路径
:return:
"""
try:
with zipfile.ZipFile(source, "r") as zf:
zf.extractall()
zf.close()
except:
out("解压失败 {}".format(source))
err(traceback.print_exc())
def unzip(source, destination):
"""
解压提取
:param source: 文件路径
:return:
"""
try:
with zipfile.ZipFile(source, "r") as zf:
zf.extractall(path=destination)
zf.close()
except:
out("解压失败 {}".format(source))
warn(traceback.format_exc())
def receive_analysis_response(redis_client, channel):
"""
接收分析返回
:param redis_client:
:param channel:
:return:
"""
try:
ps = redis_client.pubsub()
ps.subscribe(channel)
for item in ps.listen():
print item
if item['type'] == 'message':
if item['data'].lower() == "kill":
break
else:
hjson = json.loads(item['data'])
patid = hjson['patid']
predict = hjson['predict']
if select_patient_rows(patid) > 0:
update_patient_predict_information(predict) #更新病人预测信息
out("预测结果更新TXDB成功 {}".format(patid))
else:
out("预测结果更新TXDB失败，数据库ID匹配不成功 {}".format(patid))
except:
out("receive predict message err")
err(traceback.print_exc())
def Multi(redis_client, channel, patient_list, TEST):
"""
开始获取符合条件的病人，进行预测
:param redis_client:
:param channel:
:return:
"""
try:
starttime = time.time()
cpuCount = cpu_count() # 计算本机CPU核数
#cpuCount = 1
print 'cpuCount is :', cpuCount
multiprocessing = []
for i in xrange(0, cpuCount): # 创建cpu_count()个进程
p = Process(target=Worker, args=(redis_client, TEST))
p.daemon = True
p.start()
multiprocessing.append(p)
global CNT
for i in range(len(patient_list)):
q.put([channel + str(CNT % channelCount), patient_list[i]])
CNT += 1
q.join()
for i in xrange(0, cpuCount):
q.put(None)
for p in multiprocessing:
p.join()
elapsed = (time.time() - starttime)
out("cpuCount: {} Finished with time:{}".format(cpuCount, elapsed))
except:
out("Multi run error")
err(traceback.print_exc())
def ftp_download(obj, source, destination):
"""
下载文件
:param source: FTP文件路径
:param destination: dicom存储路径
:return: dicom个数
"""
try:
ftpsync = RsyncExtra(user=obj.username.value, pwd=obj.password.value, host=obj.host.value,
port=obj.port.value, timeout= float(obj.timeout.value))
#ftpsync.func = unzip
ftpsync.sync(source, destination,tree= False)
gdcmconv(destination)
re_name(destination)
filter_slice_protocol(destination)
desensitize_dir(destination)
except:
out("ftp下载病人影像失败 {}".format(source))
out(traceback.format_exc())
if os.path.exists(destination):
return len(os.listdir(destination))
else:
return 0
def main(redis_client, channel, TEST):
try:
#patient_list = select_id_list_from_study_status(TEST) # 查询patient （医院RIS）
out('开始数据库查询，请稍后....')
patient_list = select_id_list_from_sqls(TEST)
patient_list = get_patient_dicom_path(
redis_client, patient_list)[:100] # 查询patient dicom path (医院pacs)
out('从数据库查到 {} 位病人'.format(len(patient_list)))
if len(patient_list) > 0:
Multi(redis_client, channel, patient_list, TEST) #start
except:
out("main run error")
err(traceback.print_exc())
def ftp_download(obj, source,
destination): #config_obj.FTP, dicompath, outpath
"""
下载文件
:param source: FTP文件路径
:param destination: dicom存储路径
:return: dicom个数
"""
try:
ftpsync = RsyncExtra(user=obj.username.value,
pwd=obj.password.value,
host=obj.host.value,
port=obj.port.value,
timeout=60)
#ftpsync.func = unzip
out('开始下载 {}'.format(os.path.split(destination)[1]))
ftpsync.sync(source, destination, tree=False)
out('下载解压删除完毕 {}'.format(os.path.split(destination)[1]))
filter_dicom(destination)
out('过滤完毕 {}'.format(os.path.split(destination)[1]))
except:
out("ftp下载病人影像失败 {}".format(source[0]))
err(traceback.format_exc())
return len(os.listdir(destination))
def sync(self,
source,
destination,
tree=False): #source = dicompath, destination = outpath
"""
function:同步ftp目录到本地目录
params：source >> ftp目录
destination >> 本地存放目录
tree >> True 保留原始分支结构
"""
try:
source_from = []
for ftp_path in source:
if 'CT' in ftp_path:
source_from.append(ftp_path)
dicompath = source_from[-1]
dicompath = os.path.split(dicompath)[0]
dicompath = dicompath.replace('ftp://172.19.20.30/', '')
self.ftp.cwd(dicompath) # 设置FTP路径
self.mkdirs(destination) #创建本地目录
os.chdir(destination) #设置本地路径
fileList = self.ftp.nlst()
print fileList
unzip_list = []
for i in fileList:
v = os.path.join(destination, i)
self.downloadFile(
i, v) # downloadFile(self, remotepath, localpath):
unzip_list.append(v)
time.sleep(1)
for unzip_path in unzip_list:
if unzip_path[-2:] == 'PK':
unzip(unzip_path, destination)
os.remove(unzip_path)
except OSError:
out("Error: could not change the local path to " + destination)
except self.error_perm:
out("Error: could not change to " + dicompath)
out(traceback.format_exc())
#sys.exit("Ending Application")
finally:
self.ftp.quit()
def service(redis_client, channel, patient, TEST):
"""
预测当前病人
:param redis_client:
:param channel:
:param patient:
"""
try:
patient_id = patient[0]
#print patient
out("start download image for patid {}".format(patient_id))
if TEST:
time.sleep(10)
redis_client.set(patient_id, "", ex=24 * 60 * 60 * 2)
# 正式使用改成True
elif True:
outpath = os.path.join(config_obj.STORE.dicom_store.value,
patient_id) # 文件存储地址
save_path = os.path.join(config_obj.STORE.image_store.value,
patient_id) # 预测结果保存路径
# 四种取病人dicom方式，四选一
if True:
dicompath = patient[1]
file_number = ftp_download(config_obj.FTP, dicompath,
outpath) # FTP 下载病人dicom
else:
file_number = 0
out("Choose a way that you want to achieve !")
if file_number > 150: # number > X
out("download image count {} {}".format(
file_number, patient_id))
redis_client.set(patient_id, outpath, ex=24 * 60 * 60 * 2)
#Dicomfilter().CT(outpath)# 过滤病人dicom
#gdcmconv(outpath) #解压dicom
desensitize_dir(outpath)
hjson = get_request_json(patient_id, outpath, save_path)
send_analysis_request(redis_client, channel,
hjson) # 发信预测信号到dlserver
out("the sending channl succeed for patid {}".format(
patient_id))
else:
out("the sending channl failed ,download image count {} patid is {}"
.format(file_number, patient_id))
except:
out("service run err {}".format(patient_id))
err(traceback.print_exc())
Multi(redis_client, channel, patient_list, TEST) #start
except:
out("main run error")
err(traceback.print_exc())
if __name__ == '__main__':
mkdir_recursive('./log')
try:
rcon = redis.Redis()
channel = "CT"
channelCount = 1
"""
receive 接收预测服务返回
"""
if False:
res_channel = "prediction"
receive_analysis_response(rcon, res_channel) #更新预测结果到数据库
out("receive predict server run start , channel : {}".format(
res_channel))
while True:
out("server run start !")
q = JoinableQueue()
main(rcon, channel, TEST=False) # 测试两分钟无误后修改为False
time.sleep(float(config_obj.OTHER.server_sleep.value))
except:
out("run error")
out(traceback.print_exc())
def service(redis_client, channel, patient, TEST): #patient : ['',[,,,,]]
"""
预测当前病人
:param redis_client:
:param channel:
:param patient:
"""
try:
patient_id = patient[0]
out("start download image for patid {}".format(patient_id))
# 正式使用改成True
if True:
outpath = os.path.join(config_obj.STORE.dicom_store.value,
patient_id) # 文件存储地址
save_path = os.path.join(config_obj.STORE.image_store.value,
patient_id) # 预测结果保存路径
dicompath_list = patient[1] # pid ---- ['','',''...]
#四种取病人dicom方式，四选一
#---------------------------------------------------------------------------------------------
# if True:
# dicompath_list = patient[1]
# for dicompath in dicompath_list:
# out_path = os.path.join(outpath,dicompath.split('/')[2])
# file_number = ftp_download(config_obj.FTP, dicompath, out_path) # FTP 下载病人dicom
# #out('{}脱敏完成'.format(os.path.join(out_path.split('/')[-2],out_path.split('/')[-1])))
# if os.path.exists(out_path) and file_number > 0:
# out("{} 共下载图像 {} 张".format(patient_id, file_number))
# # gdcmconv(outpath) #解压dicom
# #desensitize_dir(out_path)
# out("脱敏完成 {} ".format(out_path))
# out('是否脱敏:'+str(is_not_desensitized(out_path)))
# if is_not_desensitized(out_path):
# redis_client.set(patient_id, out_path, ex=24*60*60*2)
# hjson = get_request_json(patient_id, out_path, save_path)
# send_analysis_request(redis_client, channel, hjson) # 发信预测信号到dlserver
# else:
# out("脱敏失败 {} ".format(out_path))
# out("发送病人 {} 预测信号到频道 {} 成功".format(patient_id,channel))
# elif os.path.exists(out_path) and file_number == 0:
# pass
# elif not os.path.exists(out_path):
# pass
# else:
# out("the sending channl failed ,download image count {} patid is {}".format(file_number, out_path))
# err(traceback.format_exc())
#----------------------------------------------------------------------------------------------
if True:
for dicompath in dicompath_list: # .....local/date/CT/serid
out_path = os.path.join(
outpath,
dicompath.split(
'/')[7]) #/..../.../Data/DICOMS/CT/pid/serid
file_number = cp_mount(dicompath,
out_path) # copy 挂载PACS存储
#out('{}脱敏完成'.format(os.path.join(out_path.split('/')[-2],out_path.split('/')[-1])))
if os.path.exists(out_path) and file_number > 0:
out("{} 共下载图像 {} 张".format(patient_id, file_number))
# gdcmconv(outpath) #解压dicom
#desensitize_dir(out_path)
out("脱敏完成 {} ".format(out_path))
out('是否脱敏:' + str(is_not_desensitized(out_path)))
if is_not_desensitized(out_path):
redis_client.set(patient_id,
out_path,
ex=24 * 60 * 60 * 2)
hjson = get_request_json(patient_id, out_path,
save_path)
send_analysis_request(redis_client, channel,
hjson) # 发信预测信号到dlserver
else:
out("脱敏失败 {} ".format(out_path))
out("发送病人 {} 预测信号到频道 {} 成功".format(
patient_id, channel))
elif os.path.exists(out_path) and file_number == 0:
pass
elif not os.path.exists(out_path):
pass
else:
out("the sending channl failed ,download image count {} patid is {}"
.format(file_number, out_path))
err(traceback.format_exc())
except:
out("service run err {}".format(patient_id))
err(traceback.format_exc())
patient_list = get_patient_dicom_path(
redis_client, patient_list)[:100] # 查询patient dicom path (医院pacs)
out('从数据库查到 {} 位病人'.format(len(patient_list)))
if len(patient_list) > 0:
Multi(redis_client, channel, patient_list, TEST) #start
except:
out("main run error")
err(traceback.print_exc())
if __name__ == '__main__':
mkdir_recursive('./log')
try:
rcon = redis.Redis()
channel = "CT"
channelCount = 4
"""
receive 接收预测服务返回
"""
while True:
out("server run start !")
q = JoinableQueue()
main(rcon, channel, TEST=False) # 测试两分钟无误后修改为False
time.sleep(float(config_obj.OTHER.server_sleep.value))
except:
out("run error")
out(traceback.print_exc())
