def get_original_data_paths(cls, original_path, label_search_name):
"""
从原始数据中获取所有 inputs 和 labels 的路径，返回为array.
Arguments:
original_path -- 原始数据目录的路径
label_search_name -- 路径下查到 label 的关键字
Returns:
train_data_paths -- 存放所有image 和 label路径的数组,数组的每个元素对应一个字典 {image : ... , label :...}
"""
# 获取 original_path 下的所有文件夹路径
all_paths = glob(os.path.join(original_path, "*"), recursive=False)
train_data_paths = []
# 遍历每个文件夹, 搜索需要的 input 和 label ,并把查到的路径存放到相对应的路径数组中
for sub_path in all_paths:
all_files = glob(os.path.join(sub_path, "*"), recursive=False)
tmp_dict = {}
for file in all_files:
if split_filename(file) == 'mprage_anonymized.nii':
tmp_dict['image'] = file
if re.search(label_search_name, split_filename(file)):
tmp_dict['label'] = file
train_data_paths.append(tmp_dict)
return train_data_paths
def create_train_data(self):
"""
对原始的 inputs，labels 进行 resample, resize 后保存到本地的路径.
"""
for data_paths in self.train_data_paths:
img = data_paths['image']
label = data_paths['label']
dir_name = label.split(os.sep)[-2]
dir_path = os.path.join(self.nii_ouput_path, dir_name)
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
print('create dir:' + dir_path)
img_ouput_path = os.path.join(dir_path, split_filename(img))
label_ouput_path = os.path.join(dir_path, split_filename(label))
# 对 img 和 label 进行训练的预处理
img_data = self.process_image(img, img_ouput_path)
label_data = self.process_image(label,
label_ouput_path,
key='label')
# 对 img 和 label 进行切块并保存为 npy 文件
ouput_path = os.path.join(self.output_path, dir_name)
self.slice_and_save_data(img_data, label_data, ouput_path)
print('-----------> Create train Data done! (SS)')
def create_train_data(cls, train_data_paths, output_path, standard_shape):
"""
对原始的 inputs，labels 进行 resample, resize 后保存到本地的路径.
Arguments:
train_data_paths -- 存放原始 input 路径数组, 可通过 utils.py 中函数 get_original_data_paths() 获取
output_path -- 新的 label 存放的路径
standard_shape -- 标准的 shape(即网络输入的shpae), reslice 后会把结果 padding or clips 到标准的 shape
"""
def pre_data(nii_path, output_path, key='input'):
"""
对单个 nii 数据 preprocess 后并在 save_path 路径下保存为.nii文件
:param nii_path: 需要 reslice 的 nii 文件路径
:param output_path: reslice 后新的 nii 文件输出路径
:param key： 'input' or 'label' 决定了插值的方法
"""
nii = image.load_img(nii_path)
transpese_data, main_axis, padding = cls.preprocess(
nii, standard_shape)
# 一些奇怪的label数据得到的不是整数,如输出值是2,确变成了1.999999,这里需要四舍五入
if key == 'label':
transpese_data = np.rint(transpese_data)
transpese_nii = nib.Nifti1Image(transpese_data,
affine=nii.affine,
header=nii.header)
nib.save(transpese_nii, output_path)
for data_paths in train_data_paths:
img = data_paths['image']
label = data_paths['label']
dir_name = label.split(os.sep)[-2]
dir_path = os.path.join(output_path, dir_name)
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
print('create dir:' + dir_path)
img_ouput_path = os.path.join(dir_path, split_filename(img))
label_ouput_path = os.path.join(dir_path, split_filename(label))
pre_data(img, img_ouput_path)
pre_data(label, label_ouput_path, key='label')
print(img_ouput_path)
print(label_ouput_path)
print(
'-------------------------------------------------------------->'
)
print('----------->reslice done!')
def diversification_spacing(cls, train_data_paths, output_path, ratio=2.0):
def enhance_spacing(nii_path, output_path, key='input'):
nii = image.load_img(nii_path)
target_sapcing = (nii.header['pixdim'][1] / ratio,
nii.header['pixdim'][2] / ratio,
nii.header['pixdim'][3] / ratio)
reslice_data, reslice_affine = reslice_nii(
nii, target_zooms=target_sapcing, key=key)
# 一些奇怪的label数据得到的不是整数,如输出值是2,确变成了1.999999,这里需要四舍五入
if key == 'label':
reslice_data = np.rint(reslice_data)
transpese_nii = nib.Nifti1Image(reslice_data,
affine=reslice_affine,
header=nii.header)
nib.save(transpese_nii, output_path)
for data_paths in train_data_paths:
img = data_paths['image']
label = data_paths['label']
dir_name = label.split(os.sep)[-2]
dir_name = dir_name + '_' + str(ratio)
dir_path = os.path.join(output_path, dir_name)
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
print('create dir:' + dir_path)
img_ouput_path = os.path.join(dir_path, split_filename(img))
label_ouput_path = os.path.join(dir_path, split_filename(label))
enhance_spacing(img, img_ouput_path)
enhance_spacing(label, label_ouput_path, key='label')
print(img_ouput_path)
print(label_ouput_path)
print(
'-------------------------------------------------------------->'
)
print('----------->reslice done!')
def get_labels(data_file, min_extent=100):
from nipype.utils.filemanip import split_filename
from scipy.ndimage import label
import os
import scipy.io as sio
import numpy as np
data = sio.loadmat(data_file)
labels, nlabels = label(data["k"] == 0)
for idx in range(1, nlabels + 1):
if np.sum(labels == idx) < min_extent:
labels[labels == idx] = 0
nlabels = len(np.unique(labels))
_, name, ext = split_filename(data_file)
outfile = os.path.abspath(name + "_label.mat")
sio.savemat(outfile, {"label": labels})
return outfile, nlabels
def split_k(in_file):
import numpy as np
from matplotlib.pyplot import imread, imsave
from nipype.utils.filemanip import split_filename
import os
import scipy.io as sio
data = np.asarray(sio.loadmat(in_file)["kmeans"])
vals = np.unique(data)
_, name, ext = split_filename(in_file)
outfiles = []
for v in vals:
outfile = os.path.abspath(name + "_k%02d" % (v) + ".mat")
print outfile
sio.savemat(outfile, {"k": data == v})
outfiles.append(outfile)
return outfiles
def split_k(in_file):
import numpy as np
from matplotlib.pyplot import imread, imsave
from nipype.utils.filemanip import split_filename
import os
import scipy.io as sio
data = np.asarray(sio.loadmat(in_file)["kmeans"])
vals = np.unique(data)
_, name, ext = split_filename(in_file)
outfiles = []
for v in vals:
outfile = os.path.abspath(name+"_k%02d"%(v)+".mat")
print outfile
sio.savemat(outfile,{"k":data==v})
outfiles.append(outfile)
return outfiles
def get_labels(data_file, min_extent=100):
from nipype.utils.filemanip import split_filename
from scipy.ndimage import label
import os
import scipy.io as sio
import numpy as np
data = sio.loadmat(data_file)
labels, nlabels = label(data["k"] == 0)
for idx in range(1, nlabels+1):
if np.sum(labels==idx)<min_extent:
labels[labels==idx] = 0
nlabels = len(np.unique(labels))
_,name,ext = split_filename(data_file)
outfile = os.path.abspath(name+"_label.mat")
sio.savemat(outfile,{"label":labels})
return outfile, nlabels
def kmeans(img, nlabels=5):
import os
from matplotlib.pyplot import imread, imsave
from sklearn import cluster
from nipype.utils.filemanip import split_filename
import numpy as np
import scipy.io as sio
lena = imread(img)
X = lena.reshape((-1, 1)) # We need an (n_sample, n_feature) array
k_means = cluster.KMeans(n_clusters=nlabels, n_init=1)
k_means.fit(X)
values = k_means.cluster_centers_.squeeze()
labels = k_means.labels_
outkmeans = labels.reshape(lena.shape)
loc, name, ext = split_filename(img)
outfile = os.path.abspath(name + "_kmeans.mat")
sio.savemat(outfile, {"kmeans": outkmeans})
print outfile, "exists", os.path.exists(outfile)
return outfile
def kmeans(img,nlabels=5):
import os
from matplotlib.pyplot import imread, imsave
from sklearn import cluster
from nipype.utils.filemanip import split_filename
import numpy as np
import scipy.io as sio
lena = imread(img)
X = lena.reshape((-1, 1)) # We need an (n_sample, n_feature) array
k_means = cluster.KMeans(n_clusters=nlabels, n_init=1)
k_means.fit(X)
values = k_means.cluster_centers_.squeeze()
labels = k_means.labels_
outkmeans = labels.reshape(lena.shape)
loc, name, ext = split_filename(img)
outfile = os.path.abspath(name+"_kmeans.mat")
sio.savemat(outfile,{"kmeans":outkmeans})
print outfile, "exists", os.path.exists(outfile)
return outfile
def plotallimages(cy3file, cy5file, labelsfile, Clustersfile):
from matplotlib.pyplot import imread, savefig, subplots, cm
from nipype.utils.filemanip import split_filename
import os
import numpy as np
import scipy.io as sio
def get_centroids(labels):
labelnums = np.unique(labels)
centroids = []
for label in labelnums:
centroids.append(
np.mean(np.asarray(np.nonzero(labels == label)), axis=1))
return centroids
cy3 = imread(cy3file)
cy5 = imread(cy5file)
Clusters = np.asarray(sio.loadmat(labelsfile)["label"])
labels = np.asarray(sio.loadmat(Clustersfile)["k"])
_, name, ext = split_filename(Clustersfile)
outfile = os.path.abspath(name + "_img_all.png")
fig, ax = subplots(ncols=4, nrows=1, figsize=(36, 12))
ax[0].imshow(cy3, cmap=cm.Greys)
k = len(np.unique(labels))
cmap = cm.get_cmap('jet', k)
l = ax[1].imshow(labels, cmap=cmap)
ax[1].set_title('K Means Clustering')
nlabels = len(np.unique(Clusters))
cmap = cm.get_cmap('jet', nlabels)
b = ax[2].imshow(Clusters, cmap=cmap, alpha=0.3)
ax[2].set_title("ROI Labelling")
centroids = get_centroids(Clusters)
for i, c in enumerate(centroids):
ax[2].annotate('%d' % i, c[::-1], color="black")
ax[3].imshow(cy5, cmap=cm.Greys)
ax[3].set_title('Cy5 - Defect')
ax[0].set_title('Cy3 - Object')
savefig(outfile, bbox_inches="tight")
return outfile
def plotallimages(cy3file,cy5file,labelsfile,Clustersfile):
from matplotlib.pyplot import imread, savefig,subplots,cm
from nipype.utils.filemanip import split_filename
import os
import numpy as np
import scipy.io as sio
def get_centroids(labels):
labelnums = np.unique(labels)
centroids = []
for label in labelnums:
centroids.append(np.mean(np.asarray(np.nonzero(labels==label)), axis = 1))
return centroids
cy3 = imread(cy3file)
cy5=imread(cy5file)
Clusters = np.asarray(sio.loadmat(labelsfile)["label"])
labels = np.asarray(sio.loadmat(Clustersfile)["k"])
_,name,ext = split_filename(Clustersfile)
outfile = os.path.abspath(name+"_img_all.png")
fig,ax = subplots(ncols=4,nrows=1,figsize=(36,12))
ax[0].imshow(cy3,cmap=cm.Greys)
k = len(np.unique(labels))
cmap = cm.get_cmap('jet', k)
l = ax[1].imshow(labels,cmap=cmap)
ax[1].set_title('K Means Clustering')
nlabels = len(np.unique(Clusters))
cmap = cm.get_cmap('jet', nlabels)
b=ax[2].imshow(Clusters,cmap=cmap,alpha=0.3)
ax[2].set_title("ROI Labelling")
centroids = get_centroids(Clusters)
for i,c in enumerate(centroids):
ax[2].annotate('%d'%i,c[::-1],color="black");
ax[3].imshow(cy5,cmap=cm.Greys)
ax[3].set_title('Cy5 - Defect')
ax[0].set_title('Cy3 - Object')
savefig(outfile,bbox_inches="tight")
return outfile
def download_images(dest_dir,images_df=None,target=None,resample=True):
"""Downloads images dataframe and resamples them to a common space"""
orig_path = os.path.join(dest_dir, "original")
mkdir_p(orig_path)
if resample == True:
if not target:
print "To resample you must specify a target!"
return
resampled_path = os.path.join(dest_dir, "resampled")
mkdir_p(resampled_path)
target_nii = nb.load(target)
if not isinstance(images_df,pandas.DataFrame):
images_df = get_images()
out_df = images_df.copy()
for row in images_df.iterrows():
# Downloading the file to the "original" subfolder
_, _, ext = split_filename(row[1]['file'])
orig_file = os.path.join(orig_path, "%04d%s" % (row[1]['image_id'], ext))
if not os.path.exists(orig_file):
try:
print "Downloading %s" % orig_file
urllib.urlretrieve(row[1]['file'], orig_file)
if resample == True:
# Compute the background and extrapolate outside of the mask
print "Extrapolating %s" % orig_file
niimg = nb.load(orig_file)
affine = niimg.get_affine()
data = niimg.get_data().squeeze()
niimg = nb.Nifti1Image(data, affine,header=niimg.get_header())
bg_mask = compute_background_mask(niimg).get_data()
# Test if the image has been masked:
out_of_mask = data[np.logical_not(bg_mask)]
if np.all(np.isnan(out_of_mask)) or len(np.unique(out_of_mask)) == 1:
# Need to extrapolate
data = _extrapolate_out_mask(data.astype(np.float), bg_mask,iterations=3)[0]
niimg = nb.Nifti1Image(data, affine,header=niimg.get_header())
del out_of_mask, bg_mask
# Resampling the file to target and saving the output in the "resampled" folder
resampled_file = os.path.join(resampled_path,"%06d%s" % (row[1]['image_id'], ext))
print "Resampling %s" % orig_file
resampled_nii = resample_img(niimg, target_nii.get_affine(),target_nii.shape)
resampled_nii = nb.Nifti1Image(resampled_nii.get_data().squeeze(),
resampled_nii.get_affine(),
header=niimg.get_header())
if len(resampled_nii.shape) == 3:
resampled_nii.to_filename(resampled_file)
else:
# We have a 4D file
assert len(resampled_nii.shape) == 4
resampled_data = resampled_nii.get_data()
affine = resampled_nii.get_affine()
for index in range(resampled_nii.shape[-1]):
# First save the files separately
this_nii = nb.Nifti1Image(resampled_data[..., index],affine)
this_id = int("%i%i" % (-row[1]['image_id'], index))
this_file = os.path.join(resampled_path,"%06d%s" % (this_id, ext))
this_nii.to_filename(this_file)
# Second, fix the dataframe
out_df = out_df[out_df.image_id != row[1]['image_id']]
this_row = row[1].copy()
this_row.image_id = this_id
out_df = out_df.append(this_row)
except:
print "Error downloading image id %s, retry this image." %(row[1]["image_id"])
return out_df
if __name__ == '__main__':
# parse command-line arguments
parser = argparse.ArgumentParser()
parser.add_argument('infile', help='input expression matrix (genes x samples)')
parser.add_argument('outfile', help='output label file')
parser.add_argument('--partitions', help='partition label file')
parser.add_argument('--n-partitions', help='number of partitions to generate', type=int, default=0)
parser.add_argument('--method', help='partitioning method to use when generating partitions', default='uniform', choices=['random', 'uniform'])
args = parser.parse_args()
# determine basename for output files
basename, _ = utils.split_filename(args.infile)
# load input expression matrix
print('Loading expression matrix...')
emx = utils.load_dataframe(args.infile)
# load partition labels from file
if args.partitions != None:
print('Loading partition file...')
pairs = pd.read_csv(args.partitions, header=None, names=['sample', 'label'], sep='\t')
# or generate partitions
elif args.n_partitions != 0:
print('Generating partitions using the %s method...' % (args.method))
def query_file_list(self, parent_item_id):
item_list = DataDao.query_data_item_by_parent(parent_item_id,
True,
limit=1000)
params = []
for item in item_list:
_item_path = item.path
txt = item.filename
if item.aliasname:
txt = item.aliasname
item_fuzzy_id = obfuscate_id(item.id)
format_size = scale_size(item.size)
# print("id:", item.id, ",item_fuzzy_id:", item_fuzzy_id)
params.append({
"id": item_fuzzy_id,
"text": txt,
"data": {
"path": _item_path,
"_id": item_fuzzy_id,
"format_size": format_size,
"fs_id": item.fs_id,
"category": item.category,
"source": "local",
"isdir": item.isdir
},
"children": True,
"icon": "folder"
})
# print("dirs total:", len(params))
sp: SearchParams = SearchParams.build_params(0, 1000)
# sp.add_must(is_match=False, field="path", value=parent_path)
sp.add_must(is_match=False, field="parent", value=parent_item_id)
sp.add_must(is_match=False, field="isdir", value=0)
es_body = build_query_item_es_body(sp)
# logger.info("es_body:{}".format(es_body))
es_result = es_dao_local().es_search_exec(es_body)
hits_rs = es_result["hits"]
total = hits_rs["total"]
logger.info("files es total:{}".format(total))
for _s in hits_rs["hits"]:
icon_val = "file"
fn_name = _s["_source"]["filename"]
category = _s["_source"]["category"]
format_size = scale_size(_s["_source"]["size"])
media_type = self.check_data_item_media_type(category, fn_name)
txt = fn_name
aliasname = None
if "aliasname" in _s["_source"] and _s["_source"]["aliasname"]:
aliasname = _s["_source"]["aliasname"]
if aliasname:
fn_name, extname = split_filename(fn_name)
alias_fn, alias_extname = split_filename(aliasname)
if not alias_extname:
alias_extname = extname
aliasname = "{}{}".format(
alias_fn,
"." + alias_extname if alias_extname.strip() else "")
# __idx = fn_name.rfind(".")
# if __idx > 0:
# fn_name = fn_name[0:__idx]
txt = "[{}]{}".format(fn_name, aliasname)
f_type = guess_file_type(txt)
if f_type:
icon_val = "file file-%s" % f_type
item_fuzzy_id = obfuscate_id(_s["_source"]["id"])
params.append({
"id": obfuscate_id(_s["_source"]["id"]),
"text": txt,
"data": {
"path": _s["_source"]["path"],
"isdir": _s["_source"]["isdir"],
"source": "local",
"media_type": media_type,
"format_size": format_size,
"category": category,
"fs_id": _s["_source"]["fs_id"],
"_id": item_fuzzy_id
},
"children": False,
"icon": icon_val
})
return params
def download_images(dest_dir, images_df=None, target=None, resample=True):
"""Downloads images dataframe and resamples them to a common space"""
orig_path = os.path.join(dest_dir, "original")
mkdir_p(orig_path)
if resample == True:
if not target:
print "To resample you must specify a target!"
return
resampled_path = os.path.join(dest_dir, "resampled")
mkdir_p(resampled_path)
target_nii = nb.load(target)
if not isinstance(images_df, pandas.DataFrame):
images_df = get_images()
out_df = images_df.copy()
for row in images_df.iterrows():
# Downloading the file to the "original" subfolder
_, _, ext = split_filename(row[1]['file'])
orig_file = os.path.join(orig_path,
"%04d%s" % (row[1]['image_id'], ext))
if not os.path.exists(orig_file):
try:
print "Downloading %s" % orig_file
urllib.urlretrieve(row[1]['file'], orig_file)
if resample == True:
# Compute the background and extrapolate outside of the mask
print "Extrapolating %s" % orig_file
niimg = nb.load(orig_file)
affine = niimg.get_affine()
data = niimg.get_data().squeeze()
niimg = nb.Nifti1Image(data,
affine,
header=niimg.get_header())
bg_mask = compute_background_mask(niimg).get_data()
# Test if the image has been masked:
out_of_mask = data[np.logical_not(bg_mask)]
if np.all(np.isnan(out_of_mask)) or len(
np.unique(out_of_mask)) == 1:
# Need to extrapolate
data = _extrapolate_out_mask(data.astype(np.float),
bg_mask,
iterations=3)[0]
niimg = nb.Nifti1Image(data,
affine,
header=niimg.get_header())
del out_of_mask, bg_mask
# Resampling the file to target and saving the output in the "resampled" folder
resampled_file = os.path.join(
resampled_path, "%06d%s" % (row[1]['image_id'], ext))
print "Resampling %s" % orig_file
resampled_nii = resample_img(niimg,
target_nii.get_affine(),
target_nii.shape)
resampled_nii = nb.Nifti1Image(
resampled_nii.get_data().squeeze(),
resampled_nii.get_affine(),
header=niimg.get_header())
if len(resampled_nii.shape) == 3:
resampled_nii.to_filename(resampled_file)
else:
# We have a 4D file
assert len(resampled_nii.shape) == 4
resampled_data = resampled_nii.get_data()
affine = resampled_nii.get_affine()
for index in range(resampled_nii.shape[-1]):
# First save the files separately
this_nii = nb.Nifti1Image(
resampled_data[..., index], affine)
this_id = int("%i%i" %
(-row[1]['image_id'], index))
this_file = os.path.join(resampled_path,
"%06d%s" % (this_id, ext))
this_nii.to_filename(this_file)
# Second, fix the dataframe
out_df = out_df[
out_df.image_id != row[1]['image_id']]
this_row = row[1].copy()
this_row.image_id = this_id
out_df = out_df.append(this_row)
except:
print "Error downloading image id %s, retry this image." % (
row[1]["image_id"])
return out_df
def query_share_list(self, parent_item_id):
params = []
sp: SearchParams = SearchParams.build_params(0, 1000)
# sp.add_must(is_match=False, field="path", value=parent_path)
if parent_item_id:
sp.add_must(is_match=False, field="parent", value=parent_item_id)
else:
sp.add_must(is_match=False, field="pos", value=SHARE_ES_TOP_POS)
sp.add_must(is_match=False, field="parent", value=0)
# sp.add_must(is_match=False, field="isdir", value=0)
es_body = build_query_item_es_body(sp)
print("query_share_list es_body:", es_body)
es_result = es_dao_share().es_search_exec(es_body)
hits_rs = es_result["hits"]
total = hits_rs["total"]
print("query_share_list files es total:", total)
for _s in hits_rs["hits"]:
icon_val = "jstree-file"
fn_name = _s["_source"]["filename"]
txt = fn_name
aliasname = ''
if "aliasname" in _s["_source"] and _s["_source"]["aliasname"]:
aliasname = _s["_source"]["aliasname"]
if aliasname:
fn_name, extname = split_filename(fn_name)
alias_fn, alias_extname = split_filename(aliasname)
if not alias_extname:
alias_extname = extname
aliasname = "{}{}".format(
alias_fn,
"." + alias_extname if alias_extname.strip() else "")
txt = "[{}]{}".format(fn_name, aliasname)
has_children = False
a_attr = {}
if _s["_source"]["isdir"] == 1:
# icon_val = "jstree-folder"
icon_val = "folder"
has_children = True
if _s["_source"]["pin"] == 1:
a_attr = {'style': 'color:red'}
else:
f_type = guess_file_type(txt)
if f_type:
icon_val = "jstree-file file-%s" % f_type
node_text = txt
format_size = scale_size(_s["_source"]["size"])
if format_size:
if _s["_source"]["isdir"] == 1:
node_text = "{}(shared)({})".format(node_text, format_size)
else:
node_text = "{}({})".format(node_text, format_size)
node_param = {
"id": "s_%s" % _s["_source"]["id"],
"text": node_text,
"data": {
"path": _s["_source"]["path"],
"fs_id": _s["_source"]["fs_id"],
"server_ctime": _s["_source"].get("server_ctime", 0),
"isdir": _s["_source"]["isdir"],
"source": _s["_source"]["source"],
"pin": _s["_source"]["pin"],
"_id": _s["_source"]["id"],
"sourceid": _s["_source"]["sourceid"],
"p_id": _s["_source"]["fs_id"]
},
"children": has_children,
"icon": icon_val
}
if a_attr:
node_param['a_attr'] = a_attr
params.append(node_param)
return params
def getname(filename):
_, name, ext = split_filename(filename)
oldname = os.path.join('uploads', name + ext)
newname = os.path.join('downloads', name + '.jpg')
return oldname, newname
def getname(filename):
_,name,ext = split_filename(filename)
oldname = os.path.join('uploads',name+ext)
newname = os.path.join('downloads',name+'.jpg')
return oldname, newname
def query_file_list(self, parent_item_id):
# item_list = CommunityDao.query_data_item_by_parent(parent_item_id, True, pan_id, limit=1000)
params = []
sp: SearchParams = SearchParams.build_params(0, 1000)
# sp.add_must(is_match=False, field="path", value=parent_path)
sp.add_must(is_match=False, field="parent", value=parent_item_id)
# sp.add_must(is_match=False, field="isdir", value=0)
# if pan_id and pan_id > 0:
# sp.add_must(is_match=False, field="sourceid", value=pan_id)
es_body = build_query_item_es_body(sp)
print("local es_body:", es_body)
es_result = es_dao_local().es_search_exec(es_body)
hits_rs = es_result["hits"]
total = hits_rs["total"]
print("local files es total:", total)
for _s in hits_rs["hits"]:
icon_val = "jstree-file"
ori_fn_name = _s["_source"]["filename"]
ori_aliasname = ''
if "aliasname" in _s["_source"] and _s["_source"]["aliasname"]:
ori_aliasname = _s["_source"]["aliasname"]
aliasname = ori_aliasname
fn_name = ori_fn_name
txt = fn_name
if aliasname:
fn_name, extname = split_filename(fn_name)
alias_fn, alias_extname = split_filename(aliasname)
if not alias_extname:
alias_extname = extname
aliasname = "{}{}".format(
alias_fn,
"." + alias_extname if alias_extname.strip() else "")
txt = "[{}]{}".format(fn_name, aliasname)
is_dir = _s["_source"]["isdir"] == 1
t_tag = ES_TAG_MAP['FREE']
is_free = False
tags = _s["_source"]["tags"]
if not tags:
tags = []
isp = False
has_children = False
a_attr = {}
if not is_dir and _s["_source"]["pin"] == 1:
a_attr = {'style': 'color:red'}
if PRODUCT_TAG in tags:
if not a_attr:
a_attr = {'style': 'color:green'}
isp = True
if t_tag in tags:
is_free = True
if is_dir:
# icon_val = "jstree-folder"
icon_val = "folder"
has_children = True
else:
f_type = guess_file_type(txt)
if f_type:
icon_val = "jstree-file file-%s" % f_type
node_text = txt
format_size = scale_size(_s["_source"]["size"])
price = self.parse_price(format_size, 2)
if format_size:
node_text = "{}({})".format(node_text, format_size)
if is_free:
node_text = "[{}]{}".format(t_tag, node_text)
if not a_attr:
a_attr = {'style': 'color:green'}
if isp:
node_text = "[{}]{}".format(PRODUCT_TAG, node_text)
fs_id = _s["_source"]["fs_id"]
item_id = _s["_source"]["id"]
item_fuzzy_id = obfuscate_id(item_id)
node_param = {
"id": item_fuzzy_id,
"text": node_text,
"data": {
"path": _s["_source"]["path"],
"server_ctime": _s["_source"].get("server_ctime", 0),
"isdir": _s["_source"]["isdir"],
"source": _s["_source"]["source"],
"fs_id": fs_id,
"pin": _s["_source"]["pin"],
"_id": item_fuzzy_id,
"isp": isp,
"tags": tags,
"sourceid": _s["_source"]["sourceid"],
"p_id": _s["_source"]["id"],
"price": price,
"fn": ori_fn_name,
"alias": ori_aliasname
},
"children": has_children,
"icon": icon_val
}
if a_attr:
node_param['a_attr'] = a_attr
params.append(node_param)
return params
