def create_stack_from_DataFrame(df,
render,
name=None,
stackResolutionX=None,
stackResolutionY=None,
stackResolutionZ=None):
"""Creates a `render-ws` stack from given DataFrame
Parameters
----------
df : `pd.DataFrame`
DataFrame of tile data
render : `renderapi.render.RenderClient`
`render-ws` instance
name : str
Name of stack
Looks for 'stack' column in `df` if not provided
"""
# Set stack name
if name is None:
stack = df.iloc[0]['stack']
else:
stack = name
# Loop through tiles
out = f"Creating tile specifications for \033[1m{stack}\033[0m..."
print(out)
tile_specs = []
for i, tile in df.iterrows():
# Create `TileSpec`s
ts = TileSpec(**tile.to_dict())
# Ensure integer min, max intensity
ts.minint = int(tile['minint'])
ts.maxint = int(tile['maxint'])
# Collect `TileSpec`s
tile_specs.append(ts)
# Create stack
create_stack(stack=stack,
stackResolutionX=stackResolutionX,
stackResolutionY=stackResolutionY,
stackResolutionZ=stackResolutionZ,
render=render)
# Import TileSpecs to render
out = f"Importing tile specifications to \033[1m{stack}\033[0m..."
print(out)
import_tilespecs(stack=stack, tilespecs=tile_specs, render=render)
# Close stack
set_stack_state(stack=stack, state='COMPLETE', render=render)
out = f"Stack \033[1m{stack}\033[0m created successfully."
print(out)
def import_trakem2_project(stack, xml_filepath, render):
"""Import render stack from TrakEM2 xml file"""
# Soupify TrakEM2 xml file
soup = Soup(xml_filepath.read_bytes(), 'lxml')
# Iterate through layers to collect tile specifications
tile_specs = []
out = f"Creating tile specifications for \033[1m{stack}\033[0m..."
print(out)
for layer in tqdm(soup.find_all('t2_layer')):
# Iterate through patches
for patch in layer.find_all('t2_patch'):
# Get patch data as dict
d = patch.attrs
# Parse transform data
M00, M10, M01, M11, B0, B1 = [
float(i)
for i in re.findall(r'-?[\d.]+(?:[Ee]-?\d+)?', d['transform'])
]
A = AffineModel(M00, M01, M10, M11, B0, B1)
# Define layout
z = float(layer.attrs['z'])
col, row = [int(i) for i in re.findall(r'\d+', d['title'])][-2:]
layout = Layout(sectionId=f'S{int(z):03d}',
imageRow=row,
imageCol=col)
# Create tile specification
ts = TileSpec(tileId=d['title'],
z=z,
width=d['width'],
height=d['height'],
imageUrl=d['file_path'],
minint=d['min'],
maxint=d['max'],
layout=layout,
tforms=[A])
# Collect tile specification
tile_specs.append(ts)
# Create stack
create_stack(stack=stack, render=render)
# Import TileSpecs to render
out = f"Importing tile specifications to \033[1m{stack}\033[0m..."
print(out)
import_tilespecs(stack=stack, tilespecs=tile_specs, render=render)
# Close stack
set_stack_state(stack=stack, state='COMPLETE', render=render)
out = f"Stack \033[1m{stack}\033[0m created successfully."
print(out)
def run(self):
print mod.args
self.logger.error(
'WARNING NEEDS TO BE TESTED, TALK TO FORREST IF BROKEN')
if not os.path.isdir(self.args['outputXMLdir']):
os.makedirs(self.args['outputXMLdir'])
xmlDir = self.args['outputXMLdir']
#fill in missing bounds with the input stack bounds
bounds = self.render.run(renderapi.stack.get_stack_bounds,
self.args['inputStack'])
for key in bounds.keys():
self.args[key] = self.args.get(key, bounds[key])
EMz = renderapi.stack.get_z_values_for_stack(self.args['inputStack'],
render=self.render)
tilespecsfiles = []
shiftTransform = AffineModel(B0=self.args['minX'],
B1=self.args['minY'])
for z in EMz:
infile = os.path.join(xmlDir, '%05d.xml' % z)
outfile = os.path.join(xmlDir, '%05d.json' % z)
newoutfile = os.path.join(xmlDir, '%05d-new.json' % z)
self.convert_trakem2_project(infile, xmlDir, outfile)
newtilejson = json.load(open(outfile, 'r'))
newEMtilespecs = [TileSpec(json=tsj) for tsj in newtilejson]
EMtilespecs = renderapi.tilespec.get_tile_specs_from_minmax_box(
self.args['inputStack'],
z,
self.args['minX'],
self.args['maxX'],
self.args['minY'],
self.args['maxY'],
render=self.render)
for ts in EMtilespecs:
nts = next(t for t in newEMtilespecs if t.tileId == ts.tileId)
ts.tforms = nts.tforms
ts.tforms.append(shiftTransform)
tilespecsfiles.append(newoutfile)
renderapi.utils.renderdump(EMtilespecs, open(newoutfile, 'w'))
sv = renderapi.stack.get_stack_metadata(self.args['inputStack'],
render=self.render)
renderapi.stack.create_stack(self.args['outputStack'],
render=self.render)
renderapi.stack.set_stack_metadata(self.args['outputStack'],
sv,
render=self.render)
renderapi.client.import_jsonfiles_parallel(self.args['outputStack'],
tilespecsfiles,
render=self.render)
def run(self):
print mod.args
if not os.path.isdir(self.args['outputXMLdir']):
os.makedirs(self.args['outputXMLdir'])
xmlDir = self.args['outputXMLdir']
#fill in missing bounds with the input stack bounds
bounds = self.render.run(renderapi.stack.get_stack_bounds,
self.args['inputStack'])
for key in bounds.keys():
self.args[key] = self.args.get(key, bounds[key])
EMz = renderapi.stack.get_z_values_for_stack(self.args['inputStack'],
render=self.render)
tilespecsfiles = []
lmstack = self.args['LMstacks'][self.args['LMstack_index']]
for z in EMz:
infile = os.path.join(xmlDir, '%05d.xml' % z)
outfile = os.path.join(xmlDir, '%05d.json' % z)
newoutfile = os.path.join(xmlDir, '%05d-newLM.json' % z)
self.convert_trakem2_project(infile, xmlDir, outfile)
newtilejson = json.load(open(outfile, 'r'))
newtilespecs = [TileSpec(json=tsj) for tsj in newtilejson]
LMtilespecs = renderapi.tilespec.get_tile_specs_from_z(
lmstack, z, render=self.render)
LMtilespecssubset = []
for ts in newtilespecs:
try:
nts = next(t for t in LMtilespecs if t.tileId == ts.tileId)
print nts.tileId
LMtilespecssubset.append(nts)
except:
pass
tilespecsfiles.append(newoutfile)
with open(newoutfile, 'w') as fp:
renderapi.utils.renderdump(LMtilespecssubset, fp)
#renderapi.stack.delete_stack(self.args['outputStack'],render=self.render)
sv = renderapi.stack.get_stack_metadata(lmstack, render=self.render)
renderapi.stack.create_stack(self.args['outputStack'],
render=self.render)
renderapi.stack.set_stack_metadata(self.args['outputStack'],
sv,
render=self.render)
renderapi.client.import_jsonfiles_parallel(self.args['outputStack'],
tilespecsfiles,
render=self.render)
def make_tilespec(tileDirectory,tilespecdir,outputProject,outputOwner,outputStack,minval=0,maxval=50000):
imagefiles = glob.glob(tileDirectory + "/*.tif")
imagefiles.sort()
z = 0
tilespecpaths = []
tilespeclist = []
for f in imagefiles:
#print f
filepath=f
sectionId="SEC%04d"%z
tileId = "%04d"%z
frameId = "FRAME%04d"%z
width = 1388
height = 1040
fileparts=filepath.split(os.path.sep)[1:]
if not os.path.isdir(tilespecdir):
os.makedirs(tilespecdir)
layout = Layout(sectionId=sectionId,
scopeId='Leica',
cameraId='zyla',
imageRow=0,
imageCol=0,
stageX = 0.0,
stageY = 0.0,
rotation = 0.0,
pixelsize = 0.103)
tform = AffineModel(M00=1.0,M01 = 0.0, M10 = 0.0, M11 = 1.0, B0 = 0.0, B1 = 0.0)
mipmap0 = MipMapLevel(level=0,imageUrl=filepath)
mipmaplevels=[mipmap0]
#tilespeclist.append(TileSpec(tileId=tileId,frameId = frameId, z=z, width=width, height=height, tforms=[tform],minint=minval,maxint=maxval,layout= layout))
t = TileSpec(tileId=tileId,frameId = frameId, z=z, width=width, height=height, mipMapLevels=mipmaplevels, tforms=[tform],minint=minval,maxint=maxval,layout= layout)
tilespeclist.append(t)
json_file = os.path.join(tilespecdir,outputProject+'_'+outputOwner+'_'+outputStack+'_%04d.json'%z)
fd=open(json_file, "w")
renderapi.utils.renderdump([t],fd)
fd.close()
tilespecpaths.append(json_file)
z = z+1
return tilespecpaths,tilespeclist
def run(self):
print mod.args
self.logger.error('WARNING NEEDS TO BE TESTED, TALK TO FORREST IF BROKEN')
if not os.path.isdir(self.args['outputXMLdir']):
os.makedirs(self.args['outputXMLdir'])
xmlDir = self.args['outputXMLdir']
EMz = renderapi.stack.get_z_values_for_stack(self.args['EMstack'],render=self.render)
tilespecsfiles = []
shiftTransform = AffineModel(B0=args['minX'],B1=args['minY'])
for z in EMz:
infile = os.path.join(xmlDir,'%05d.xml'%z)
outfile = os.path.join(xmlDir,'%05d.json'%z)
newoutfile = os.path.join(xmlDir,'%05d-new.json'%z)
self.convert_trakem2_project(self,infile,xmlDir,outfile)
newtilejson = json.load(open(outfile,'r'))
newEMtilespecs = [TileSpec(json=tsj) for tsj in newtilejson]
EMtilespecs = renderapi.tilespec.get_tile_specs_from_minmax_box(
EMstack,
z,
self.args['minX'],
self.args['maxX'],
self.args['minY'],
self.args['maxY'],
render=self.render)
for ts in EMtilespecs:
nts = next(t for t in newEMtilespecs if t.tileId == ts.tileId )
ts.tforms=nts.tforms
ts.tforms.append(shiftTransform)
tilespecsfiles.append(newoutfile)
renderapi.utils.renderdump(EMtilespecs,open(newoutfile,'w'))
renderapi.stack.delete_stack(self.args['outputEMStack'],render=self.render)
renderapi.stack.create_stack(self.args['outputEMStack'],render=self.render)
renderapi.client.import_jsonfiles_parallel(self.args['outputEMStack'],tilespecsfiles,render=self.render)
def run(self):
zvalues = self.render.run(renderapi.stack.get_z_values_for_stack,
self.args['inputStack'])
minZ = self.args.get('minZ', int(np.min(zvalues)))
maxZ = self.args.get('maxZ', int(np.max(zvalues)))
if self.args['doChunk']:
allchunks = createchunks(minZ, maxZ, self.args['chunkSize'])
raise Exception('do chunk output not yet implemented')
else:
allchunks = []
ck = Chunk()
ck.first = minZ
ck.last = maxZ
ck.dir = str(ck.first) + "-" + str(ck.last)
allchunks.append(ck)
for x in allchunks:
indir = os.path.join(self.args['outputXMLdir'], x.dir)
infile = os.path.join(indir, 'project.xml')
outfile = os.path.join(indir, 'tilespec.json')
self.convert_trakem2_project(infile, indir, outfile)
with open(outfile, 'r') as fp:
tsjson = json.load(fp)
output_tilespecs = [TileSpec(json=tsj) for tsj in tsjson]
shiftTransform = AffineModel(B0=self.args['minX'],
B1=self.args['minY'])
jsonfiles = []
for layerid in range(x.first, x.last + 1):
self.logger.debug('layerid {}'.format(x.first))
jsonfilename = os.path.join(self.args['outputXMLdir'],
'%05d.json' % layerid)
output_tilespec_list = []
tilespecs_original = renderapi.tilespec.get_tile_specs_from_minmax_box(
self.args['inputStack'],
layerid,
self.args['minX'],
self.args['maxX'],
self.args['minY'],
self.args['maxY'],
render=self.render)
for tso in tilespecs_original:
#tileid4 = "%04d"%(int(output_tilespecs[0].tileId))
#matches = [ts for ts in output_tilespecs if ts.tileId==tso.tileId]
#print "This is tileid : ", tileid4
#print tso.tileId
matches = [
ts for ts in output_tilespecs
if int(ts.tileId) == int(tso.tileId)
]
if len(matches) > 0:
tsm = matches[0]
tso.tforms = tsm.tforms
tso.tforms.append(shiftTransform)
output_tilespec_list.append(tso)
with open(jsonfilename, 'w') as fp:
renderapi.utils.renderdump(output_tilespec_list,
fp,
indent=4)
jsonfiles.append(jsonfilename)
if not self.args['doChunk']:
#outfile = os.path.join(indir,'tilespec_old.json')
#renderapi.stack.delete_stack(self.args['outputStack'],render=r)
sv = renderapi.stack.get_stack_metadata(
self.args['inputStack'], render=self.render)
renderapi.stack.create_stack(self.args['outputStack'],
render=self.render)
renderapi.stack.set_stack_metadata(self.args['outputStack'],
sv,
render=self.render)
print outfile
renderapi.client.import_jsonfiles_parallel(
self.args['outputStack'], jsonfiles, render=self.render)
def process_siteset(render,siteset, sectionset, doc, project_path,lm_dataset='test',lm_stack='ACQDAPI_1'):
project = doc['F-BioSEM-Project']['BioSemProject']
# find the light dataset name
dataset = find_node_by_field(doc, 'Name', lm_dataset)
imported_data = find_node_by_field(doc, 'Name', 'Imported Data')
# get the important transforms from atlas file
# transform from EM data>root
at = AtlasTransform(dataset['ParentTransform'])
# transform from LMdata > root
id_at = AtlasTransform(imported_data['ParentTransform'])
project_dir, project_file = os.path.split(project_path)
project_base = os.path.splitext(project_file)[0]
ribnum = project_path.split(os.path.sep)[-2]
ribnum = int(ribnum.lower().replace('ribbon', ''))
nodes, paths=find_nodes_by_field(project, 'Name', siteset['Name'])
ods=[ods for ods, p in zip(nodes, paths) if 'OrderedDataSet' in p]
sitename=siteset['Name'].replace(' ', '')
json_files = []
if len(ods) > 0:
# print "found it"
ods=ods[0]
outdir=os.path.join(os.path.join(project_dir, 'TEM2_import_files'))
if not os.path.isdir(outdir):
os.makedirs(outdir)
for site in siteset['Site']:
sitefile=os.path.join(
outdir, site['Name'].replace(' ', '') + '.csv')
# print sitefile
df=pd.DataFrame(columns=('Path', 'M00', 'M01',
'M10', 'M11', 'dx', 'dy'))
section=[section for section in sectionset['Section']
if section['UID'] == site['LinkedToUID']][0]
if type(ods['PlaceableMosaic']) is not type([]):
ods['PlaceableMosaic']=[ods['PlaceableMosaic']]
mosaic=[pm for pm in ods['PlaceableMosaic'] if pm['UID']
== site['AcquisitionSpec']['AcquiredDataUID']]
if len(mosaic) == 0:
continue
else:
mosaic=mosaic[0]
# print mosaic['Name']
# this is the transform that describes how to transform from the coodinate system of the EM stage
# to the new coordiante transform of this mosaic site
mt=AtlasTransform(mosaic['ParentTransform'])
center=np.array([[0.5, 0.5]])
center_stage=mt.tform(center)
# these are now the light microscopy stage coordinates that correspond with this site
lm_coords=at.inverse_tform(center_stage)
# print mt
# print 'center_stage',center_stage
# print 'lm coords',lm_coords
# print 'at',at
# print site['AcquisitionSpec'].keys()
pid=site['AcquisitionSpec']['WorkingProtocolUID']
pixsize, width, height=get_protocol_metadata(pid, project)
# print site['Name'],',',section['Name'],',',siteset['Name'],',',mosaic['Name']
# print site['AcquisitionSpec']['AcquiredDataUID']
relpath=mosaic['FileName'][mosaic['FileName'].find(
project_base):]
# print relpath
unixpath=relpath.replace('\\', '/')
unixpath=os.path.join(project_dir, unixpath)
basedir=os.path.split(unixpath)[0]
base=os.path.splitext(unixpath)[0]
updatespath=base + '.ve-updates'
with open(updatespath) as fd:
mosaicdoc=xmltodict.parse(fd.read())['ATLAS-Stitch-Info']
rootdir=mosaicdoc['OriginalRoot']
if type(mosaicdoc['Tile']) is not type([]):
mosaicdoc['Tile']=[mosaicdoc['Tile']]
sectnum=int(section['SectionIndex'])
#print 'section number',sectnum
sectionId='%d' % (1000 * ribnum + sectnum)
sectionZ=renderapi.stack.get_section_z_value(
lm_stack, sectionId, render=render)
tilespecs=renderapi.tilespec.get_tile_specs_from_z(
lm_stack, sectionZ, render=render)
LMtile_xy=np.array(
[[ts.layout.stageX, ts.layout.stageY] for ts in tilespecs])
#print 'sectionZ',sectionZ
image_corners=np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
# print "at"
# print at
# print "mt"
# print mt
tilespeclist=[]
for i, tile in enumerate(mosaicdoc['Tile']):
distances=np.zeros(len(tilespecs))
imagepath=os.path.join(basedir, tile['Name'])
maskpath=os.path.join(
basedir, tile['Name'][0:-4] + '_mask.tif')
flippath=os.path.join(
basedir, tile['Name'][0:-4] + '_flip.jpg')
tform=AtlasTransform(tile['ParentTransform'])
# shift = np.array([[150,-80]])
shift=np.array([[0, 0]])
# tranform the EM tile through the transforms to get LM stage coordinates of corners
EMtile_corners_lm_stage=at.inverse_tform(
id_at.inverse_tform(mt.tform(tform.tform(image_corners) + shift)))
# EMtile_corners_lm_stage[:,1]+=(2048*.107)
# EMtile_corners_lm_stage[:,1]*=-1
#print "EMtile_corners_lm_stage",EMtile_corners_lm_stage
# tranform the EM tile through the transforms to get LM stage coordinates of center
# print tform
EMtile_center_lm_stage=at.inverse_tform(
id_at.inverse_tform(mt.tform(tform.tform(center) + shift)))
# EMtile_center_lm_stage[:,1]+=(2048*.107)
#print 'EMtile_center_lm_stage',EMtile_center_lm_stage
# EMtile_center_lm_stage[:,1]*=-1
# print [(ts['layout']['stageX']-tile_lm_stage[0,0])**2+(ts['layout']['stageY']+tile_lm_stage[0,1])**2 for ts in tilespecs]
# EMtile_xy = np.array([tile_lm_stage[0,0],-tile_lm_stage[0,1]])
# figure out which of the LM tiles is closest to this EM tile in terms of stage coordinates
# the distance vector
dist_xy=LMtile_xy - \
np.tile(EMtile_center_lm_stage, (len(tilespecs), 1))
# the euclidean distance in microns
d=np.sqrt(np.sum(dist_xy**2, axis=1))
LMtile_i=np.argmin(d) # pick out the index of the smallest
# this is the tilespec of the closest tile
close_spec=tilespecs[LMtile_i]
# print 'tile ',LMtile_i,'id',close_spec['tileId'], 'closest at ','%4.2f'%d[LMtile_i],' um'
#print close_spec.tileId
# this calculates the delta from the EM stage coordinates to the LM stage coordinates
# and divides by the size of LM pixels, to get delta in pixels from the center of the LM tile
# to each of the corners of the EM tile
# note hardcoded LM pixel size
delt=(EMtile_corners_lm_stage - \
LMtile_xy[LMtile_i, :]) / 0.107
delt_center=(EMtile_center_lm_stage - \
LMtile_xy[LMtile_i, :]) / 0.107
# this is the location in terms of pixels within the LM tile of the EM corners
# this is with pixel 0,0 in the upper left, and assuming that the stage coordinates
# are in such a manner that positive y is down, and positive x is to the right
EMtile_corners_local_pixels=delt + \
np.array(
[[close_spec.width / 2.0, close_spec.height / 2.0]])
EMtile_center_local_pixels=delt_center + \
np.array(
[[close_spec.width / 2.0, close_spec.height / 2.0]])
# use renderapi to map these local pixel coordinates to the global space
EMtile_corners_world_coords=renderapi.transform.estimate_dstpts(close_spec.tforms,EMtile_corners_local_pixels)
EMtile_center_world_coords=renderapi.transform.estimate_dstpts(close_spec.tforms,EMtile_center_local_pixels)
#print lm_stack
#print "localmap",renderapi.transform.estimate_dstpts(close_spec.tforms,EMtile_center_local_pixels)
#print "EMtile_center_local_pixels",EMtile_center_local_pixels
#print "EMtile_center_world_coords",EMtile_center_world_coords
# these are the local coordinates of the corners of the EM tile
# listed in the same order as the "corners" variable, but noting
# that the definition in what is 0,0 is different from ATLAS definition
# this transformation of the tile needs to account for this
EMpixel_corners=np.array([
[0.0, float(tile['Height'])],
[0.0, 0.0],
[float(tile['Width']),
float(tile['Height'])],
[float(tile['Width']), 0]
])
# print "from\n",EMpixel_corners
# print "to\n",EMtile_corners_world_coords
emtform=cv2.getAffineTransform(np.float32(
EMpixel_corners[0:3, :]), np.float32(EMtile_corners_world_coords[0:3, :]))
atlas_emt=AtlasTransform()
atlas_emt.load_from_openCV(emtform)
# print atlas_emt.tform(np.array([[0,0],[5000,5000],[0,5000],[5000,0]]))
s=np.sqrt(-np.linalg.det(atlas_emt.M[0:2, 0:2]))
# print atlas_emt.M[0:2,0:2]/s
row, col=tile['Name'].split('_')[1].split('-')
row=int(row[1:])
col=int(col[1:])
layout=Layout(sectionId=sectionId,
scopeId="Gemini",
cameraId='SEM3nm5K',
imageRow=row,
imageCol=col,
stageX=tform.tform(center)[0, 0],
stageY=tform.tform(center)[0, 1],
rotation=0.0,
pixelsize=0.03)
# flip=AffineModel(M00=1,
# M01=0,
# M10=0,
# M11=1,
# B0=5000 * (col - 1),
# B1=-5000 * (row - 1))
# am=AffineModel(M00=emtform[0, 0],
# M01=emtform[0, 1],
# M10=emtform[1, 0],
# M11=emtform[1, 1],
# B0=emtform[0, 2],
# B1=emtform[1, 2])
amf=AffineModel(M00=emtform[0, 0],
M01=emtform[0, 1],
M10=-emtform[1, 0],
M11=-emtform[1, 1],
B0=emtform[0, 2],
B1=emtform[1, 2])
tilespec=TileSpec(tileId=tile['UID'],
z=sectionZ,
width=int(tile['Width']),
height=int(tile['Height']),
imageUrl='file:' + \
flippath.replace(" ", "%20"),
frameId=tile['UID'],
maskUrl='file:' + \
maskpath.replace(" ", "%20"),
tforms=[amf],
minint=0,
maxint=255,
layout=layout)
tilespeclist.append(tilespec)
# row= (str(path),) + tform.to_tuple()
# df.loc[i]=row
# print tilespec.to_dict()
# for tile in mosaicdoc['Tiles']['Tile']:
# print tile['UID'],tile['@row'],tile['@col'],tile['StageX'],tile['StageY']
# df.to_csv(sitefile,index=False,header=False)
tilespec_path = os.path.join(project_dir,'tilespecs')
if not os.path.isdir(tilespec_path):
os.makedirs(tilespec_path)
json_file=os.path.join(
tilespec_path, 'EM_rib%04dsect%04d_%s.json' % (ribnum, sectnum, sitename))
with open(json_file,'w') as fp:
renderapi.utils.renderdump(tilespeclist,fp)
json_files.append(json_file)
return json_files
def make_tilespec_from_statetable (df,rootdir,outputProject,outputOwner,outputStack,minval=0,maxval=50000):
df = df[df['zstack']==0]
#ribbons = df.groupby('ribbon')
#zoffset=0
#for ribbnum,ribbon in ribbons:
# ribbon.loc[ribbon.index,'z']=ribbon['section']+zoffset
# zoffset += ribbon['section'].max()+1
# df.loc[ribbon.index,'z']=ribbon['z'].values
mipmap_args = []
tilespecpaths = []
for ((ch,sess),chgroup) in df.groupby(['ch_name','session']):
print ch,sess
for ((rib,sect),group) in chgroup.groupby(['ribbon','section']):
tilespeclist=[]
z=0
for ind,row in group.iterrows():
filepath=row.full_path
fileparts=filepath.split(os.path.sep)[1:]
tilespecdir = rootdir + "/processed/downsamp_tilespec/"+fileparts[5]+"/"+fileparts[6]+"/"+fileparts[7]
#print tilespecdir
if not os.path.isdir(tilespecdir):
os.makedirs(tilespecdir)
downdir = rootdir+"/processed/downsamp_images/"+fileparts[5]+"/"+fileparts[6]+"/"+fileparts[7]
#print "This is the Down Sampled Directory: %s"%downdir
if not os.path.exists(downdir):
os.makedirs(downdir)
#construct command for creating mipmaps for this tilespec
#downcmd = ['python','create_mipmaps.py','--inputImage',filepath,'--outputDirectory',downdir,'--mipmaplevels','1','2','3']
#cmds.append(downcmd)
mipmap_args.append((filepath,downdir))
layout = Layout(sectionId=row.ribbon*1000+row.section,
scopeId='Leica',
cameraId='zyla',
imageRow=0,
imageCol=0,
stageX = row.xstage,
stageY = row.ystage,
rotation = 0.0,
pixelsize = row.scale_x)
mipmap0 = MipMapLevel(level=0,imageUrl=row.full_path)
mipmaplevels=[mipmap0]
filename = "%s_S%04d_F%04d_Z%02d.tif"%(row.ch_name,row.section,row.frame,0)
for i in range(1,4):
scUrl = 'file:' + os.path.join(downdir,filename[0:-4]+'_mip0%d.jpg'%i)
mml = MipMapLevel(level=i,imageUrl=scUrl)
mipmaplevels.append(mml)
tform = AffineModel(M00=row.a00,
M01=row.a01,
M10=row.a10,
M11=row.a11,
B0=row.a02,
B1=row.a12)
tilespeclist.append(TileSpec(tileId=row.tileID,
frameId = row.frame,
z=row.z,
width=row.width,
height=row.height,
mipMapLevels=mipmaplevels,
tforms=[tform],
minint=minval,
maxint=maxval,
layout= layout))
z = row.z
json_text=json.dumps([t.to_dict() for t in tilespeclist],indent=4)
json_file = os.path.join(tilespecdir,outputProject+'_'+outputOwner+'_'+outputStack+'_%04d.json'%z)
fd=open(json_file, "w")
fd.write(json_text)
fd.close()
tilespecpaths.append(json_file)
return tilespecpaths,mipmap_args
def make_tilespec_from_llp (rootdir,outputProject,outputOwner,outputStack,minval=0,maxval=50000):
mipmap_args = []
tilespecpaths = []
basename ='LLP'
#cwd = os.getcwd()
#load database
db = sqlite3.connect('llp.sqlite')
c=db.cursor()
c.execute('SELECT * FROM image_item')
keys = [description[0] for description in c.description]
imdb = list()
mags = list()
# import data
for row in c.execute('SELECT * FROM image_item ORDER BY image_id'):
im_item = dict()
for idx,key in enumerate(keys):
im_item[key] = row[idx]
mags.append(row[3])
imdb.append(im_item)
mags=np.array(mags)
# histogram information for intensity scaling
c.execute('SELECT histgramRangeMax FROM histgram')
h_max = c.fetchall()[1][0]
c.execute('SELECT histgramRangeMin FROM histgram')
h_min = c.fetchall()[1][0]
c.execute('SELECT histgramAverage FROM histgram')
h_av = c.fetchall()[1][0]
c.execute('SELECT histgramStdev FROM histgram')
h_sd = c.fetchall()[1][0]
db.close()
#minmax iwill be based on average value, otherwise use given values
# if int_av:
# h_max = int(min(65535,h_av + int_width * h_sd))
# h_min = int(max(0,h_av - int_width * h_sd))
# process each mag separately
for thismag in np.unique(mags):
# thismag=1000
# if thismag==1000:
itemname = basename + '_'+str(thismag)+'x'
# outfile = os.path.join(dirname,itemname)
# outfile = outfile + outformat
im_idx = np.where(mags==thismag)
setup_id=0
tile_id=0
numslices = np.shape(im_idx)[1]
digits = len(str(numslices))
tilespeclist=[]
z=0
for tile_id,imx in enumerate(im_idx[0]):
# thisview=dict()
tile = imdb[imx]
# im = io.imread(thisim['filename'])
f1 = os.path.realpath(tile['filename'])
fbase = os.path.splitext(os.path.basename(f1))[0]
tilespecdir = os.path.join('processed','tilespec')
filepath= groupsharepath(f1)
#print tilespecdir
if not os.path.isdir(tilespecdir):
os.makedirs(tilespecdir)
downdir = os.path.join("processed","downsamp_images")
#print "This is the Down Sampled Directory: %s"%downdir
if not os.path.exists(downdir):
os.makedirs(downdir)
downdir1 = groupsharepath(os.path.realpath(downdir))
pxs = 1/tile['pixel_per_nm']
#construct command for creating mipmaps for this tilespec
#downcmd = ['python','create_mipmaps.py','--inputImage',filepath,'--outputDirectory',downdir,'--mipmaplevels','1','2','3']
#cmds.append(downcmd)
mipmap_args.append((f1,os.path.realpath(downdir)))
layout = Layout(sectionId=z,
scopeId='JEOL',
cameraId='Matataki',
imageRow=0,
imageCol=0,
stageX = tile['location_x_nm'],
stageY = tile['location_y_nm'],
rotation = tile['stage_x_axis_rotation_degree_'],
pixelsize = pxs)
mipmap0 = MipMapLevel(level=0,imageUrl='file://' + filepath)
mipmaplevels=[mipmap0]
for i in range(1,4):
scUrl = 'file://' + os.path.join(downdir1,fbase) + '_mip0%d.jpg'%i
mml = MipMapLevel(level=i,imageUrl=scUrl)
mipmaplevels.append(mml)
# transformation
# 1) The scale and rotation information
th = np.radians(thisim['image_degree'])
ct = np.cos(th)
st = np.sin(th)
rotmat = pxs * np.array([[ct,-st,0],[st,ct,0],[0,0,1]])
# 2) The translation matrix to position the object in space (lower left corner)
# mat_t = np.concatenate((np.eye(2),[[0,thisim['location_x_nm']/1000],[0,thisim['location_y_nm']/1000]]),axis=1)
# mat_t = np.concatenate((mat_t,[[0,0,1,0],[0,0,0,1]]))
# tf_tr = tf.matrix_to_transformation(mat_t).tolist()
tform = AffineModel(M00=rotmat[0,0],
M01=rotmat[0,0],
M10=rotmat[0,0],
M11=rotmat[0,0],
B0=thisim['location_x_nm'],
B1=thisim['location_y_nm'])
tilespeclist.append(TileSpec(tileId=itemname+'_t'+('{:0'+str(digits)+'}').format(tile_id),
frameId = itemname,
z=z,
width=tile['image_width_px'],
height=tile['image_height_px'],
mipMapLevels=mipmaplevels,
tforms=[tform],
minint=minval,
maxint=maxval,
layout= layout))
json_file = os.path.realpath(os.path.join(tilespecdir,outputProject+'_'+outputOwner+'_'+outputStack+'_%04d.json'%z))
fd=open(json_file, "w")
renderapi.utils.renderdump(tilespeclist,fd,sort_keys=True, indent=4, separators=(',', ': '))
fd.close()
tilespecpaths.append(json_file)
return tilespecpaths,mipmap_args
# ----------------------
if not os.path.exists('meta'): print('Change to proper directory!');exit()
mfile0 = os.path.join('meta','logs','imagelist_')
mfiles = glob.glob(mfile0+'*')
tiles = list()
views = list()
idx = 0
for mfile in mfiles:
with open(mfile) as mf: ml = mf.read().splitlines()
mdfile = os.path.join('meta','logs','metadata'+mfile[mfile.rfind('_'):])
with open(mdfile) as mdf: mdl = mdf.read().splitlines()
conffile = os.path.join('meta','logs','config'+mfile[mfile.rfind('_'):])
with open(conffile) as cf: cl = cf.read().splitlines()
config = parse_adoc(cl)
pxs = float(config['grab_frame_pixel_size'][0])#/1000 # in um
z_thick = float(config['slice_thickness'][0])#/1000 # in um
# generate the individual transformation matrices
# 1) The scale and rotation information form the map item
mat = np.diag((pxs,pxs,z_thick))
mat_s = np.concatenate((mat,[[0],[0],[0]]),axis=1)
mat_s = np.concatenate((mat_s,[[0,0,0,1]]))
for line in mdl:
if line.startswith('TILE: '):
tile = bdv.str2dict(line[line.find('{'):])
tiles.append(tile)
# 2) The translation matrix to position the object in space (lower left corner)
mat_t = np.concatenate((np.eye(3),[[tile['glob_x']],[tile['glob_y']],[tile['glob_z']]]),axis=1)
mat_t = np.concatenate((mat_t,[[0,0,0,1]]))
def make_tilespec_from_sbemimage(rootdir,
outputProject,
outputOwner,
outputStack,
minval=0,
maxval=50000):
mipmap_args = []
tilespecpaths = []
if not os.path.exists('meta'):
print('Change to proper directory!')
exit()
mfile0 = os.path.join('meta', 'logs', 'imagelist_')
mfiles = glob.glob(mfile0 + '*')
tiles = list()
views = list()
idx = 0
for mfile in mfiles:
with open(mfile) as mf:
ml = mf.read().splitlines()
mdfile = os.path.join('meta', 'logs',
'metadata' + mfile[mfile.rfind('_'):])
with open(mdfile) as mdf:
mdl = mdf.read().splitlines()
conffile = os.path.join('meta', 'logs',
'config' + mfile[mfile.rfind('_'):])
with open(conffile) as cf:
cl = cf.read().splitlines()
config = parse_adoc(cl)
pxs = float(config['grab_frame_pixel_size'][0]) #/1000 # in um
z_thick = float(config['slice_thickness'][0]) #/1000 # in um
# generate the individual transformation matrices
# 1) The scale and rotation information form the map item
mat = np.diag((pxs, pxs, z_thick))
mat_s = np.concatenate((mat, [[0], [0], [0]]), axis=1)
mat_s = np.concatenate((mat_s, [[0, 0, 0, 1]]))
tilespeclist = []
z = 0
for line in mdl:
if line.startswith('TILE: '):
tile = bdv.str2dict(line[line.find('{'):])
tiles.append(tile)
# 2) The translation matrix to position the object in space (lower left corner)
mat_t = np.concatenate(
(np.eye(3), [[tile['glob_x']], [tile['glob_y']],
[tile['glob_z']]]),
axis=1)
mat_t = np.concatenate((mat_t, [[0, 0, 0, 1]]))
f1 = os.path.realpath(tile['filename'])
fbase = os.path.splitext(os.path.basename(f1))[0]
tilespecdir = os.path.join('processed', 'tilespec')
filepath = groupsharepath(f1)
#print tilespecdir
if not os.path.isdir(tilespecdir):
os.makedirs(tilespecdir)
downdir = os.path.join("processed", "downsamp_images")
#print "This is the Down Sampled Directory: %s"%downdir
if not os.path.exists(downdir):
os.makedirs(downdir)
downdir1 = groupsharepath(os.path.realpath(downdir))
#construct command for creating mipmaps for this tilespec
#downcmd = ['python','create_mipmaps.py','--inputImage',filepath,'--outputDirectory',downdir,'--mipmaplevels','1','2','3']
#cmds.append(downcmd)
mipmap_args.append((f1, os.path.realpath(downdir)))
layout = Layout(sectionId=tile['slice_counter'],
scopeId='3View',
cameraId='3View',
imageRow=0,
imageCol=0,
stageX=tile['glob_x'] / 10,
stageY=tile['glob_y'] / 10,
rotation=0.0,
pixelsize=pxs)
mipmap0 = MipMapLevel(level=0, imageUrl='file://' + filepath)
mipmaplevels = [mipmap0]
filename = tile['tileid']
for i in range(1, 4):
scUrl = 'file://' + os.path.join(downdir1,
fbase) + '_mip0%d.jpg' % i
mml = MipMapLevel(level=i, imageUrl=scUrl)
mipmaplevels.append(mml)
tform = AffineModel(M00=1,
M01=0,
M10=0,
M11=1,
B0=tile['glob_x'] / 10,
B1=tile['glob_y'] / 10)
tilespeclist.append(
TileSpec(tileId=tile['tileid'],
frameId=tile['tileid'][:tile['tileid'].find('.')],
z=tile['glob_z'],
width=tile['tile_width'],
height=tile['tile_height'],
mipMapLevels=mipmaplevels,
tforms=[tform],
minint=minval,
maxint=maxval,
layout=layout))
z = tile['glob_z']
json_file = os.path.realpath(
os.path.join(
tilespecdir, outputProject + '_' + outputOwner + '_' +
outputStack + '_%04d.json' % z))
fd = open(json_file, "w")
renderapi.utils.renderdump(tilespeclist,
fd,
sort_keys=True,
indent=4,
separators=(',', ': '))
fd.close()
tilespecpaths.append(json_file)
return tilespecpaths, mipmap_args