def load_model():
model = [{
'filters': 16,
'kernel_size_x': 4,
'kernel_size_y': 5,
'strides_x': 1,
'strides_y': 2
}, {
'filters': 15,
'kernel_size_x': 1,
'kernel_size_y': 2,
'strides_x': 1,
'strides_y': 1
}, {
'filters': 3,
'kernel_size_x': 3,
'kernel_size_y': 3,
'strides_x': 2,
'strides_y': 1
}]
encoder = Transformer.chromosome_to_encoder(model)
decoder = Transformer.chromosome_to_decoder(model)
loaded_autoencoder = Autoencoder.Autoencoder(encoder, decoder)
loaded_autoencoder.compile(optimizer='adam',
loss=tf.keras.losses.MeanSquaredError())
return loaded_autoencoder
def Test29():
Trans, BatchDatas, SrcDict, TgtDict, MaxLength = T.TestBuildTransformer()
for BatchInd, Batch in enumerate(BatchDatas):
SrcSent = Batch["SrcSent"]
SrcLength = Batch["SrcLength"]
SrcMask = T.BatchLengthToBoolTensorMask(SrcLength, MaxLength)
#Output=Trans(SrcSent,TgtSent,SrcMask,TgtMask)
print("Step")
print(BatchInd + 1)
def get_marker_array(data_input, marker_array=MarkerArray(), type="anchor"):
obj_pos_in_vel = None
offset = None
marker_size = None
color = None
if type == "anchor":
obj_pos_in_anchor = np.argwhere(
data_input[:, :, :, :, 0] > data_input[:, :, :, :, 1])[:, 1:4]
obj_pos_in_vel = Transformer.anchor_to_vel_coordinate(
obj_pos_in_anchor)
offset = param.STRIDE_STEP / 2.0
marker_size = param.RESOLUTION * param.SCALE
color = (1, 0, 0, 0.8)
elif type == "voxel":
obj_pos_in_voxel = np.argwhere(data_input[:, :, :, :, 0] > 0)[:, 1:4]
obj_pos_in_vel = Transformer.voxel_to_vel_coordinate(obj_pos_in_voxel)
offset = param.RESOLUTION / 2.0
marker_size = param.RESOLUTION
color = (0, 1, 0, 0.2)
elif type == "ground_truth":
obj_pos_in_vel = data_input
offset = param.STRIDE_STEP / 2.0
marker_size = param.RESOLUTION * param.SCALE
color = (1, 0, 0, 0.8)
marker_array_size = obj_pos_in_vel.shape[0]
print("Total %d objects detected" % marker_array_size)
id = 0
for i in range(marker_array_size):
marker = Marker()
marker.header.frame_id = "/cnn_3d"
marker.header.stamp = rospy.Time.now()
marker.type = marker.CUBE
marker.action = marker.ADD
marker.id = id
id += 1
marker.color.r = color[0]
marker.color.g = color[1]
marker.color.b = color[2]
marker.color.a = color[3]
marker.scale.x = marker_size
marker.scale.y = marker_size
marker.scale.z = marker_size
marker.pose.position.x = obj_pos_in_vel[i, 0] + offset
marker.pose.position.y = obj_pos_in_vel[i, 1] + offset
marker.pose.position.z = obj_pos_in_vel[i, 2] + offset
marker_array.markers.append(marker)
return marker_array
def preprocess(tweet):
abbv_dict = json.load(open("../other/abbreviations.json"))
emo_lexica_dict = json.load(open("../other/emoticons.json"))
for emoticon in emo_lexica_dict[u'emoticons']:
abbv_dict[emoticon] = ' '
for word in emo_lexica_dict[u'words']:
abbv_dict[word] = ' '
hash_transformer = Transformer.HashtagTransformer()
sub_transformer = Transformer.SubstitutionTransformer(abbv_dict)
preprocessor = Preprocessor([hash_transformer, sub_transformer])
tweet = ' '.join(tokenize(tweet))
tweet = preprocessor.transform(tweet)
return tweet
def Test28():
Trans, BatchDatas, SrcDict, TgtDict, MaxLength = T.TrainBuildTransformer()
for BatchInd, Batch in enumerate(BatchDatas):
SrcSent = Batch["SrcSent"]
SrcLength = Batch["SrcLength"]
TgtSent = Batch["TgtSent"]
TgtLength = Batch["TgtLength"]
SrcMask = T.BatchLengthToBoolTensorMask(SrcLength, MaxLength)
TgtMask = T.BatchLengthToBoolTensorMask(TgtLength, MaxLength)
Output = Trans(SrcSent, TgtSent, SrcMask, TgtMask)
print("Step")
print(BatchInd + 1)
print(Output.size())
print(Output[0][2])
def label_parser(label_file, rt_cam_to_vel):
location = []
shape = [] # height, width, length
rotation = []
if param.LABEL_FORMAT == 'txt':
location, shape, rotation = read_label_from_txt(label_file)
if len(location) == 0 or len(shape) == 0 or len(rotation) == 0:
return None, None, None
rotation = np.pi / 2 - rotation
location = np.c_[location, np.ones(location.shape[0])]
location = np.dot(rt_cam_to_vel, location.transpose())[:3, :]
location = location.transpose()
location_idx = Transformer.clipper(location, rotation, shape)
if len(location_idx) == 0:
return None, None, None
location = location[location_idx]
else:
print "Invalid label format!!!"
return location, rotation, shape
def Test7():
B, H, L, E = 2, 2, 6, 50
PE = T.PositionalEmbedding(E, L)
SrcMask = t.BoolTensor(np.array([[1, 1, 1, 1, 0, 0], [1, 1, 1, 0, 0, 0]]))
x = PE(SrcMask)
print(x.size())
print(x)
def Do(self):
for BatchInd, Batch in enumerate(self.BatchDatas):
SrcSent = Batch["SrcSent"]
SrcLength = Batch["SrcLength"]
BatchSampleNum = SrcSent.size()[0]
#print(type(BatchSampleNum))
CurrentBatchOuput = TranslateOutput(
self.TgtDict, self.MaxLength).Init(BatchSampleNum)
TgtIndexSent = CurrentBatchOuput.GetCurrentIndexTensor()
SrcMask = Tr.BatchLengthToBoolTensorMask(SrcLength, self.MaxLength)
for Step in range(self.MaxLength):
TgtMask = self.GetTgtMask(Step, BatchSampleNum)
if self.UseGPU:
SrcSent=SrcSent.cuda()
TgtMask=TgtMask.cuda()
SrcMask=SrcMask.cuda()
TgtMask=TgtMask.cuda()
ProOutput = self.Model(SrcSent, TgtIndexSent, SrcMask, TgtMask)
ProOutput=ProOutput.cpu()
LocalMaxPro, Idx = self.PickWord(ProOutput, Step)
CurrentBatchOuput.Add(Idx)
if CurrentBatchOuput.AllFinish():
print("Appending Ouput of Batch "+str(BatchInd))
CurrentBatchOuput.ToFile(self.ResultPath)
break
TgtIndexSent = CurrentBatchOuput.GetCurrentIndexTensor()
def __init__(self, sLength, numberOfLayers, selfAttentionSize,
interMediumDim, wordEmbeddingSize):
super(TransformerXLEncoder, self).__init__()
self.L = numberOfLayers
self.tsfEncoderList = [
tsf.TransformerEncoder(h=selfAttentionSize,
interMediumDim=interMediumDim,
dk=wordEmbeddingSize)
for _ in range(numberOfLayers)
]
self.cacheMatrix = [
tf.zeros(shape=[sLength, wordEmbeddingSize], dtype=tf.float32)
for _ in range(numberOfLayers)
]
self.WQ = [
keras.layers.Dense(wordEmbeddingSize)
for _ in range(numberOfLayers)
]
self.WK = [
keras.layers.Dense(wordEmbeddingSize)
for _ in range(numberOfLayers)
]
self.WV = [
keras.layers.Dense(wordEmbeddingSize)
for _ in range(numberOfLayers)
]
def Parser(config):
#Define the eval environment
env = {
'tr': Transformer.Transformer(),
}
config = eval(config, env)
#Load input
df = getattr(InputAdapter.InputAdapter,
config['input'][0])(config['input'][1])
# Version checking
if not 'config' in config:
raise SyntaxError("This doesn't look like config format")
if config['config'] > 1:
raise ValueError("This is for a newer config")
# Run all preprocessing functions
for function in config['preprocess']:
#The preprocessing functions change the dataframe, however they don't
#require assignment of new columns.
function(df)
#Run all transformation functions
for row in config['transformations']:
#In the new column, run the function and assign the return value.
df[row[0]] = row[1](df)
#Output the file
df = getattr(OutputAdapter.OutputAdapter,
config['output'][0])(df, config['output'][1])
def Test6():
B, H, L, E = 2, 2, 6, 4
En = T.Encoder(H, E, L, 3)
SrcMask = t.BoolTensor(np.array([[1, 1, 1, 1, 0, 0], [1, 1, 1, 0, 0, 0]]))
Src = t.Tensor(np.random.randn(B, L, E))
x = En(Src, SrcMask)
print(x.size())
def Test10():
VacabularySize = 4
B, L, E = 1, 2, 4
Gen = T.Generator(VacabularySize, E)
Tgt = t.Tensor(np.random.randn(B, L, E))
x = Gen(Tgt)
print(x.size())
print(x)
def Test8():
B, H, L, E = 2, 2, 6, 4
De = T.DecoderLayer(H, E, L)
SrcMask = t.BoolTensor(np.array([[1, 1, 1, 1, 0, 0], [1, 1, 1, 0, 0, 0]]))
TgtMask = t.BoolTensor(np.array([[1, 1, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0]]))
Memory = t.Tensor(np.random.randn(B, L, E))
Tgt = t.Tensor(np.random.randn(B, L, E))
De(Tgt, Memory, SrcMask, TgtMask)
def _calc_diff_ft(self, img_no, axis, normalize=False):
img0 = self.mri0.get_image(img_no, axis, rotate=0)
img1 = self.mri1.get_image(img_no, axis, rotate=0)
if (normalize):
M0 = np.amax(img0)
img0 = img0 / M0
M1 = np.amax(img1)
img1 = img1 / M1
T0 = tf.Transformer(img0)
T1 = tf.Transformer(img1)
f0 = T0.get_ft_raw()
f1 = T1.get_ft_raw()
diff = np.sum(np.abs(f0 - f1))
return diff
def run(self):
"""
Start processing.
"""
# parse the command line arguments and set logging options
try:
self.args = self.parser.parse_args()
self.configureLogging()
self.logger.info("Started with {0}".format(' '.join(sys.argv[1:])))
except Exception as e:
self.parser.print_help()
sys.exit(e)
# load the configuration file
try:
with open(self.args.config) as f:
self.config.readfp(f)
except Exception as e:
self.logger.critical("Could not load the specified configuration file")
sys.exit(e)
# set options
Cfg.LOG_EXC_INFO = self.args.trace
# execute commands
with Timer.Timer() as t:
if self.args.crawl:
import Crawler
Crawler.crawl(self.config, self.args.update)
if self.args.clean:
import Cleaner
Cleaner.clean(self.config, self.args.update)
if self.args.infer:
import Facter
Facter.infer(self.config, self.args.update)
if self.args.graph:
import Grapher
Grapher.graph(self.config, self.args.update)
if self.args.transform:
import Transformer
Transformer.transform(self.config)
if self.args.post:
import Poster
Poster.post(self.config)
if self.args.analyze:
import Analyzer
Analyzer.analyze(self.config, self.args.update)
self.logger.info("Indexer finished in {0}:{1}:{2}".format(t.hours, t.minutes, t.seconds))
def step(hparams, tokens, past=None):
lm_output = Transformer.model(hparams=hparams,
X=tokens,
past=past,
reuse=tf.AUTO_REUSE)
logits = lm_output['logits'][:, :, :hparams.
n_vocab] # keep output dimension the same
# tf.gather collect the corresponding logits
logits = tf.gather(logits, reduced_dict_index)
# logits = logits * dict_mask
presents = lm_output['present']
presents.set_shape(
Transformer.past_shape(hparams=hparams, batch_size=batch_size))
return {
'logits': logits,
'presents': presents,
}
def Test9():
B, H, L, E = 2, 2, 6, 4
De = T.Decoder(H, E, L, 2)
SrcMask = t.BoolTensor(np.array([[1, 1, 1, 1, 0, 0], [1, 1, 1, 0, 0, 0]]))
TgtMask = t.BoolTensor(np.array([[1, 1, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0]]))
Memory = t.Tensor(np.random.randn(B, L, E))
Tgt = t.Tensor(np.random.randn(B, L, E))
x = De(Tgt, Memory, SrcMask, TgtMask)
print(x.size())
def Test19():
B, H, L, E = 2, 4, 6, 8
SrcVocab = 10
TgtVocab = 12
EnLayer = 2
Delayer = 3
Trans = T.TransformerNMTModel(H, E, SrcVocab, TgtVocab, L, EnLayer,
Delayer)
Trans.Save("Model/Trans.0")
def __init__(self,batchSize,sLength,wordEmbeddingSize,
numberOfTransformerLayers,selfAttentionSize,interMediumDim,
outDim,numberOfMaskTwoStreamLayer,numberOfDecoder):
super(XLNet,self).__init__()
self.dn = numberOfDecoder
self.encoder = XLNetEncoder(batchSize,sLength,wordEmbeddingSize,
numberOfTransformerLayers,selfAttentionSize,interMediumDim,
numberOfMaskTwoStreamLayer)
self.decoders = [tsf.TransformerDecoder(selfAttentionSize,interMediumDim,wordEmbeddingSize) for _ in range(numberOfDecoder)]
self.dense = keras.layers.Dense(outDim)
def predict(sentence):
max_len = 20
tokenizer = spacy.load('en_core_web_sm')
device = torch.device('cpu')
word_to_idx = pickle.load(
open(CONFIG.INPUT_PATH + 'word_to_idx.pickle', 'rb'))
idx_to_word = pickle.load(
open(CONFIG.INPUT_PATH + 'idx_to_word.pickle', 'rb'))
vocab_size = len(word_to_idx)
pad_idx = word_to_idx["<PAD>"]
model = Transformer.Transformer(input_vocab_size=vocab_size,
out_vocab_size=vocab_size,
max_len=CONFIG.max_len,
embed_dims=CONFIG.embed_dims,
pad_idx=pad_idx,
heads=CONFIG.heads,
forward_expansion=CONFIG.forward_expansion,
num_layers=CONFIG.num_layers,
dropout=CONFIG.dropout,
device=device)
model.load_state_dict(torch.load(CONFIG.MODEL_PATH))
model.eval()
sentence_idx = []
for word in tokenizer(sentence):
word = str(word.text.lower())
if word in word_to_idx:
sentence_idx.append(word_to_idx[word])
else:
sentence_idx.append(word_to_idx["<UNK>"])
summary_idx_ = [word_to_idx["<SOS>"]]
sentence_idx = torch.tensor(sentence_idx).unsqueeze(0)
summary = []
with torch.no_grad():
for _ in range(max_len):
summary_idx = torch.tensor(summary_idx_).unsqueeze(0)
output = model(sentence_idx, summary_idx)
output = torch.softmax(output, dim=-1)
output = torch.argmax(output, dim=-1)[:, -1].item()
summary_idx_.append(output)
if output == word_to_idx["<EOS>"]:
break
summary.append(idx_to_word[output])
summary = ' '.join(summary)
print(f'[ARTICLE] -> {sentence}\n')
print(f'[SUMMARY] -> {summary}')
def Test18():
B, H, L, E = 2, 4, 6, 8
SrcVocab = 10
TgtVocab = 12
EnLayer = 2
Delayer = 3
Trans = T.TransformerNMTModel(H, E, SrcVocab, TgtVocab, L, EnLayer,
Delayer)
print(Trans)
for key in Trans.state_dict().keys():
print(key)
def compare(self, img, transList):
score = 0
for image in transList:
score = Transformer.getDif(img, image)
# print("score is: ", score)
if score > ci:
# print("match found")
# image.save("_ANSWERIMAGE.png")
return transList.index(image)
# print("Nothing matched")
return -1
def display_single(self, kernel):
img_conv = signal.convolve2d(self.img_, kernel, boundary = 'symm', mode='same');
tF = td.Transformer(self.img_);
img_frr = tF.get_ft_spectrum();
fig, axarr = plt.subplots(1, 4, figsize = (15, 5));
axarr[0].imshow(kernel);
axarr[1].imshow(img_conv);
axarr[2].imshow(img_frr);
axarr[3].imshow(img);
plt.tight_layout();
plt.show();
def __init__(self, ModelPath, ResultPath):
self.UseGPU=torch.cuda.is_available()
Trans, BatchDatas, SrcDict, TgtDict, MaxLength = Tr.TestBuildTransformer()
self.Model = Trans
self.BatchDatas = BatchDatas
self.SrcDict = SrcDict
self.TgtDict = TgtDict
self.MaxLength = MaxLength
self.ResultPath = ResultPath
self.Model.Load(ModelPath)
self.Model.Eval()
if self.UseGPU:
self.Model=self.Model.cuda()
def Test1():
B, H, L, E = 1, 1, 5, 4
#SrcMask=t.BoolTensor(np.random.binomial(1,0.5,[1,5]))
#TgtMask=t.BoolTensor(np.random.binomial(1,0.5,[1,5]))\
SrcMask = t.BoolTensor(np.array([[1, 1, 1, 1, 0]]))
TgtMask = t.BoolTensor(np.array([[1, 1, 0, 0, 0]]))
Src = t.Tensor(np.random.randn(B, H, L, E))
Tgt = t.Tensor(np.random.randn(B, H, L, E))
x, a = T.Attention(Tgt, Src, Src, TgtMask, SrcMask)
print(x.size())
print(SrcMask)
print(TgtMask)
print(x)
print(a)
def Train(x,
y,
model=None,
epochs=100,
batch_size=128,
LR=0.001,
n_layers=3,
layer_size=128,
filters=32,
dropout=False,
MaxPooling=False,
Embedding=False,
vocab_size=None,
embedding_dim=64,
loss="binary_crossentropy",
train=False):
if model == None:
print("Please define the model: [CNN,DNN,RNN,GRU,LSTM,Transformer]")
elif model == "CNN":
Trained_Model = CNN.Train(x, y, epochs, batch_size, LR, n_layers,
filters, dropout, MaxPooling, Embedding,
vocab_size, embedding_dim, loss, train)
elif model == "DNN":
Trained_Model = DNN.Train(x, y, epochs, batch_size, LR, n_layers,
layer_size, dropout, MaxPooling, Embedding,
vocab_size, embedding_dim, loss, train)
elif model == "RNN":
Trained_Model = RNN.Train(x, y, epochs, batch_size, LR, n_layers,
layer_size, dropout, MaxPooling, Embedding,
vocab_size, embedding_dim, loss, train)
elif model == "GRU":
Trained_Model = GRU.Train(x, y, epochs, batch_size, LR, n_layers,
layer_size, dropout, MaxPooling, Embedding,
vocab_size, embedding_dim, loss, train)
elif model == "LSTM":
Trained_Model = LSTM.Train(x, y, epochs, batch_size, LR, n_layers,
layer_size, dropout, MaxPooling, Embedding,
vocab_size, embedding_dim, loss, train)
elif model == "Transformer":
Trained_Model = Transformer.Train(x,
y,
epochs,
batch_size,
LR,
vocab_size,
loss,
embedding_dim,
train=True)
return Trained_Model
def get_test_voxel(data_path):
point_cloud = None
if param.DATA_FORMAT == 'pcd':
point_cloud = load_from_pcd(data_path)
elif param.DATA_FORMAT == 'bin':
point_cloud = load_from_bin(data_path)
point_cloud = angle_filter(point_cloud)
voxel = Transformer.raw_to_voxel(point_cloud)
return point_cloud, voxel[np.newaxis, :]
def Test11():
B, H, L, E = 2, 4, 6, 8
SrcVocab = 10
TgtVocab = 12
EnLayer = 2
Delayer = 3
Trans = T.TransformerNMTModel(H, E, SrcVocab, TgtVocab, L, EnLayer,
Delayer)
SrcMask = t.BoolTensor(np.array([[1, 1, 1, 1, 0, 0], [1, 1, 1, 0, 0, 0]]))
TgtMask = t.BoolTensor(np.array([[1, 1, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0]]))
Src = t.LongTensor(np.random.binomial(SrcVocab, 0.5, [B, L]))
Tgt = t.LongTensor(np.random.binomial(SrcVocab, 0.5, [B, L]))
x = Trans(Src, Tgt, SrcMask, TgtMask)
print(x.size())
print(x[0][0])
def Test26():
MaxLength = 30
BatchSize = 2
EmbeddingSize = 4
HeadNum = 2
EnLayer = 2
DeLayer = 2
SrcIndSentences, SrcLength, SrcDict = DL.LoadData("src.sents", "src.vocab",
MaxLength)
TgtIndSentences, TgtLength, TgtDict = DL.LoadData("tgt.sents", "tgt.vocab",
MaxLength)
TrainDataset = DL.TrainCorpusDataset(SrcIndSentences, SrcLength,
TgtIndSentences, TgtLength)
BatchDatas = DL.TrainDataLoaderCreator(TrainDataset, BatchSize)
SrcVocabularySize = SrcDict.VocabularySize()
TgtVocabularySize = TgtDict.VocabularySize()
Trans = T.TransformerNMTModel(HeadNum, EmbeddingSize, SrcVocabularySize,
TgtVocabularySize, MaxLength, EnLayer,
DeLayer)
for BatchInd, Batch in enumerate(BatchDatas):
print("BegingBatch")
SrcSent = Batch["SrcSent"]
print(SrcSent.size())
SrcLength = Batch["SrcLength"]
#print(SrcLength.size())
TgtSent = Batch["TgtSent"]
print(TgtSent.size())
TgtLength = Batch["TgtLength"]
#print(TgtLength.size())
SrcMask = T.BatchLengthToBoolTensorMask(SrcLength, MaxLength)
TgtMask = T.BatchLengthToBoolTensorMask(TgtLength, MaxLength)
Output = Trans(SrcSent, TgtSent, SrcMask, TgtMask)
print("Step")
print(BatchInd + 1)
print(Output.size())
print(Output[0][2])
def Test3():
B, H, L = 2, 4, 6
E = H * 2
MH = T.MultiHeadAttention(HeadNum=H, EmbeddingSize=E, MaxLength=L)
#SrcMask=t.BoolTensor(np.random.binomial(1,0.5,[1,5]))
#TgtMask=t.BoolTensor(np.random.binomial(1,0.5,[1,5]))\
SrcMask = t.BoolTensor(np.array([[1, 1, 1, 1, 0, 0], [1, 1, 1, 0, 0, 0]]))
TgtMask = t.BoolTensor(np.array([[1, 1, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0]]))
Src = t.Tensor(np.random.randn(B, L, E))
Tgt = t.Tensor(np.random.randn(B, L, E))
#x,a=T.Attention(Tgt,Src,Src,TgtMask,SrcMask)
x = MH(Tgt, Src, Src, TgtMask, SrcMask)
print(x.size())
print(SrcMask)
print(TgtMask)
print(x)
def _update_common(self, slider_no, val):
# Remember the slider's value is already set; you need only to update
# the kernel matrix and the display
print ('The slider no is: ' + str(slider_no));
[u, v] = self.non_zeros[slider_no];
self.my_kernel[u, v] = val;
self.img_conv = signal.convolve2d(self.img_, self.my_kernel, boundary = 'symm', mode='same');
tF = td.Transformer(self.img_);
self.img_frr = tF.get_ft_spectrum();
self.ax[0].imshow(self.my_kernel);
self.ax[1].imshow(self.img_conv);
self.ax[2].imshow(self.img_frr);
self.ax[3].imshow(self.img_);
print(repr(self.my_kernel))
self.fig.canvas.draw_idle();
# transform a binary file into workable file
tf = tf.Transformer(fname,ftype,fwgeodir,fwsrcdir,nxyz=nxyz_ct,dxyz=dxyz_ct)
print tf.nx,tf.ny,tf.nz,tf.dx,tf.dy,tf.dz
# convert a binary file to a volume
tf.Bin2Vol()
# get the appropriate info about the volume for G4file writing
tf.GetPixelDim()
organinfofile = '{}/OrgantagvsName.txt'.format(frdir)
tf.GetOrganInfo(organinfofile)
# copy the necessary files to G4 folders
ecompfile = '{}/ECompDensity.txt'.format(frdir)
shutil.copy2(organinfofile,fwgeodir)
shutil.copy2(ecompfile,fwgeodir)
# write G4files of the transformed volume
tf.Vol2G4file()
# write sourcemap based on the geometry volume
srclist = range(4,24)
tumorname = ['Tumor%d' % n for n in range(1,13)]
srcname = organname + tumorname
#print srclist
#print srcname
for i in range(len(srcname)):
tf.makeSourceMapFile(srcname[i],srclist[i])
( options, args ) = parser.parse_args()
if ( options.index and options.transfer ) or ( options.index and options.speller ) or ( options.speller and options.transfer ):
parser.error( "-x , -t and -p are exclusives!" )
if options.index:
if len( args ) == 2:
SOURCE = args[0]
DESTINATION = args[1]
else:
parser.error( "Choose SOURCE_PATH and DISTINATION_PATH" )
if options.index == "main":
T = Transformer( ixpath = DESTINATION , dypypath = None, dbpath = SOURCE )
ayaSchema = T.build_schema( tablename = 'aya' )
T.build_docindex( ayaSchema )
elif options.index == "extend":
E = ZekrModelsImporter( pathindex = DESTINATION, pathstore = SOURCE )
E.load_translationModels()
elif options.index == "word" :
T = Transformer( ixpath = DESTINATION , dypypath = None, dbpath = SOURCE )
wordqcSchema = T.build_schema( tablename = 'wordqc' )
T.build_docindex( wordqcSchema, tablename = 'wordqc' )
if options.transfer:
# check to see if xxxx/binIM/xxxx/GeoVol.bin exists
geobinfname = '{}/GeometryIM/binIM/{}/GeoVol.bin'.format(param_dict['fwdir'],param_dict['geotag'])
if os.path.isfile(geobinfname):
print 'WooHoo! GeoVol.bin exist: load the volume and write the source maps!'
# convert the segmented volume to G4-friendly files
tf = tf.Transformer(param_dict['binfwtype'],param_dict['VHDMSDdir'],ptdir,param_dict['fwdir'],param_dict['geotag'],param_dict['ecomptag'],nxyz=param_dict['nxyz'],dxyz=param_dict['dxyz'])
# convert a binary file to a volume
tf.Bin2Vol()
# get the appropriate info about the volume for G4file writing
tf.GetPixelDim()
tf.GetOrganInfo()
# write sourcemap based on the geometry volume
tf.makeSourceMapFile(param_dict['srcname'])
else: # build the volume from the raw CT images to GeoVol.bin
print 'wait up! GeoVol.bin is not found: let\'s start from the groundup to get the geometry volume ready and write the source maps!'
# basic CT segmentation based on amide's VOIs
# 'voifname' include all the names of all amide VOIs (include organs and tumors)
thedir = '{}/GeoIMdata/{}'.format(ptdir,param_dict['geotag'])
for nn in param_dict['organvoiname']:
fname = '{0}/roi_raw_data_{{{1}}}.tsv'.format(thedir,nn)
vol = xv.Xyz2Vol(fname,param_dict['frtype'],param_dict['dxyz'],param_dict['xyz0'],param_dict['nxyz'],0,param_dict['frindx'])
binfname = '{}/{}.bin'.format(thedir,nn)
xv.SaveFlattenVol(vol,binfname,param_dict['binfwtype'])
# get the CT dicom volume
ctdxyz,ctnxyz,ctvol = xv.Dicom2Vol(param_dict['ctdir'])
# segment via threshold
