def import_jsonfiles(stack, jsonfiles, transformFile=None,
subprocess_mode=None, client_script=None, memGB=None,
host=None, port=None, owner=None, project=None,
close_stack=True, render=None, **kwargs):
"""import jsons using client script serially
Parameters
----------
jsonfiles : :obj:`list` of :obj:`str`
iterator of filenames to be uploaded
transformFile : str
path to a jsonfile that contains shared
transform references (if necessary)
close_stack : bool
mark render stack as COMPLETE after successful import
render : renderapi.render.Render
render connect object
"""
set_stack_state(stack, 'LOADING', host, port, owner, project)
importJsonClient(stack, jsonfiles, transformFile,
subprocess_mode=subprocess_mode, host=host, port=port,
owner=owner, project=project, client_script=client_script,
memGB=memGB, **kwargs)
if close_stack:
set_stack_state(stack, 'COMPLETE', host, port, owner, project)
def importTransformChangesClient(stack, targetStack, transformFile,
targetOwner=None, targetProject=None,
changeMode=None, close_stack=True,
subprocess_mode=None,
host=None, port=None, owner=None,
project=None, client_script=None, memGB=None,
render=None, **kwargs):
"""run ImportTransformChangesClient.java
Parameters
----------
stack : str
stack from which tiles will be transformed
targetStack : str
stack that will hold results of transforms
transformFile : str
locaiton of json file in format defined below
::
[{{"tileId": <tileId>,
"transform": <transformDict>}},
{{"tileId": ...}},
...
]
targetOwner : str
owner of target stack
targetProject : str
project of target stack
changeMode : str
method to apply transform to tiles. Options are:
'APPEND' -- add transform to tilespec's list of transforms
'REPLACE_LAST' -- change last transform in tilespec's
list of transforms to match transform
'REPLACE_ALL' -- overwrite tilespec's transforms field to match
transform
Raises
------
ClientScriptError
if changeMode is not valid
"""
if changeMode not in ['APPEND', 'REPLACE_LAST', 'REPLACE_ALL']:
raise ClientScriptError(
'changeMode {} is not valid!'.format(changeMode))
argvs = (make_stack_params(host, port, owner, project, stack) +
['--targetStack', targetStack] +
['--transformFile', transformFile] +
get_param(targetOwner, '--targetOwner') +
get_param(targetProject, '--targetProject') +
get_param(changeMode, '--changeMode'))
call_run_ws_client(
'org.janelia.render.client.ImportTransformChangesClient', memGB=memGB,
client_script=client_script, subprocess_mode=subprocess_mode,
add_args=argvs, **kwargs)
if close_stack:
set_stack_state(stack, 'COMPLETE', host, port, owner, project)
def import_trakem2_project(stack, xml_filepath, render):
"""Import render stack from TrakEM2 xml file"""
# Soupify TrakEM2 xml file
soup = Soup(xml_filepath.read_bytes(), 'lxml')
# Iterate through layers to collect tile specifications
tile_specs = []
out = f"Creating tile specifications for \033[1m{stack}\033[0m..."
print(out)
for layer in tqdm(soup.find_all('t2_layer')):
# Iterate through patches
for patch in layer.find_all('t2_patch'):
# Get patch data as dict
d = patch.attrs
# Parse transform data
M00, M10, M01, M11, B0, B1 = [
float(i)
for i in re.findall(r'-?[\d.]+(?:[Ee]-?\d+)?', d['transform'])
]
A = AffineModel(M00, M01, M10, M11, B0, B1)
# Define layout
z = float(layer.attrs['z'])
col, row = [int(i) for i in re.findall(r'\d+', d['title'])][-2:]
layout = Layout(sectionId=f'S{int(z):03d}',
imageRow=row,
imageCol=col)
# Create tile specification
ts = TileSpec(tileId=d['title'],
z=z,
width=d['width'],
height=d['height'],
imageUrl=d['file_path'],
minint=d['min'],
maxint=d['max'],
layout=layout,
tforms=[A])
# Collect tile specification
tile_specs.append(ts)
# Create stack
create_stack(stack=stack, render=render)
# Import TileSpecs to render
out = f"Importing tile specifications to \033[1m{stack}\033[0m..."
print(out)
import_tilespecs(stack=stack, tilespecs=tile_specs, render=render)
# Close stack
set_stack_state(stack=stack, state='COMPLETE', render=render)
out = f"Stack \033[1m{stack}\033[0m created successfully."
print(out)
def import_jsonfiles_validate_client(stack,
jsonfiles,
transformFile=None,
client_script=None,
host=None,
port=None,
owner=None,
project=None,
close_stack=True,
mem=6,
validator=None,
subprocess_mode=None,
memGB=None,
render=None,
**kwargs):
"""Uses java client for parallelization and validation
Parameters
----------
stack: str
stack to which jsonfiles should be uploaded
jsonfiles: :obj:`list` of :obj:`str`
tilespecs in json files
transformFile: str, optional
json file listing transformspecs with ids which are referenced
in tilespecs contained in jsonfiles
"""
transform_params = (['--transformFile', transformFile]
if transformFile is not None else [])
if validator is None:
validator_params = [
'--validatorClass',
'org.janelia.alignment.spec.validator.TemTileSpecValidator',
'--validatorData', 'minCoordinate:-500,maxCoordinate:100000,'
'minSize:500,maxSize:10000'
]
else:
raise NotImplementedError('No custom validation handling!')
stack_params = make_stack_params(host, port, owner, project, stack)
set_stack_state(stack, 'LOADING', host, port, owner, project)
call_run_ws_client('org.janelia.render.client.ImportJsonClient',
stack_params + validator_params + transform_params +
jsonfiles,
client_script=client_script,
memGB=memGB,
subprocess_mode=subprocess_mode,
**kwargs)
if close_stack:
set_stack_state(stack, 'COMPLETE', host, port, owner, project)
def create_stack_from_DataFrame(df,
render,
name=None,
stackResolutionX=None,
stackResolutionY=None,
stackResolutionZ=None):
"""Creates a `render-ws` stack from given DataFrame
Parameters
----------
df : `pd.DataFrame`
DataFrame of tile data
render : `renderapi.render.RenderClient`
`render-ws` instance
name : str
Name of stack
Looks for 'stack' column in `df` if not provided
"""
# Set stack name
if name is None:
stack = df.iloc[0]['stack']
else:
stack = name
# Loop through tiles
out = f"Creating tile specifications for \033[1m{stack}\033[0m..."
print(out)
tile_specs = []
for i, tile in df.iterrows():
# Create `TileSpec`s
ts = TileSpec(**tile.to_dict())
# Ensure integer min, max intensity
ts.minint = int(tile['minint'])
ts.maxint = int(tile['maxint'])
# Collect `TileSpec`s
tile_specs.append(ts)
# Create stack
create_stack(stack=stack,
stackResolutionX=stackResolutionX,
stackResolutionY=stackResolutionY,
stackResolutionZ=stackResolutionZ,
render=render)
# Import TileSpecs to render
out = f"Importing tile specifications to \033[1m{stack}\033[0m..."
print(out)
import_tilespecs(stack=stack, tilespecs=tile_specs, render=render)
# Close stack
set_stack_state(stack=stack, state='COMPLETE', render=render)
out = f"Stack \033[1m{stack}\033[0m created successfully."
print(out)
def import_jsonfiles_and_transforms_parallel_by_z(stack,
jsonfiles,
transformfiles,
poolsize=20,
mpPool=WithPool,
client_scripts=None,
host=None,
port=None,
owner=None,
project=None,
close_stack=True,
render=None,
**kwargs):
"""imports json files and transform files in parallel
Parameters
----------
stack : str
the stack to import within
jsonfiles : :obj:`list` of :obj:`str`
"list of tilespec" json paths to import
transformfiles : :obj:`list` of :obj:`str`
"list of transform files" paths which matches
in a 1-1 way with jsonfiles, so referenced transforms
are shared only within a single element of these matched lists.
Useful cases where there is as single z transforms shared
by all tiles within a single z, but not across z's
poolsize : int, optional
number of processes for multiprocessing pool
close_stack : bool, optional
whether to mark render stack as COMPLETE after successful import
render : renderapi.render.Render
render connect object
**kwargs
arbitrary keyword arguments
"""
set_stack_state(stack, 'LOADING', host, port, owner, project)
partial_import = partial(import_single_json_file,
stack,
render=render,
client_scripts=client_scripts,
host=host,
port=port,
owner=owner,
project=project)
with mpPool(poolsize) as pool:
pool.map(partial_import, jsonfiles, transformfiles)
if close_stack:
set_stack_state(stack, 'COMPLETE', host, port, owner, project)
def import_jsonfiles_parallel(stack,
jsonfiles,
poolsize=20,
transformFile=None,
mpPool=WithPool,
client_scripts=None,
host=None,
port=None,
owner=None,
project=None,
close_stack=True,
render=None,
**kwargs):
"""import jsons using client script in parallel
Parameters
----------
stack : str
the stack to upload into
jsonfiles : :obj:`list` of :obj:`str`
list of jsonfile paths to upload
poolsize : int
number of upload processes spawned by multiprocessing pool
transformFile : str
a single json file path containing transforms referenced
in the jsonfiles
close_stack : bool
whether to mark render stack as COMPLETE after successful import
render : renderapi.render.Render
render connect object
**kwargs
arbitrary keyword arguments
"""
set_stack_state(stack, 'LOADING', host, port, owner, project)
partial_import = partial(import_single_json_file,
stack,
render=render,
transformFile=transformFile,
client_scripts=client_scripts,
host=host,
port=port,
owner=owner,
project=project,
**kwargs)
with mpPool(poolsize) as pool:
pool.map(partial_import, jsonfiles)
if close_stack:
set_stack_state(stack, 'COMPLETE', host, port, owner, project)
def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None,
subprocess_mode=None, poolsize=20,
mpPool=WithPool,
close_stack=True, max_tilespecs_per_group=None,
host=None, port=None,
owner=None, project=None,
client_script=None, memGB=None, render=None,
**kwargs):
"""method to import tilesepcs directly from
:class:`renderapi.tilespec.TileSpec` objects using
pathos.multiprocessing to parallelize
Parameters
----------
stack : str
stack to which tilespecs will be added
tilespecs : :obj:`list` of :class:`renderapi.tilespec.TileSpec`
list of tilespecs to import
sharedTransforms : obj:`list` of :obj:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional
list of shared referenced transforms to be ingested
poolsize : int
degree of parallelism to use
subprocess_mode : str
subprocess mode used when calling client side java
close_stack : bool
mark render stack as COMPLETE after successful import
max_tilespecs_per_group: int
maximum tilespecs per import process, default to len(tilespecs)/poolsize
render : :class:renderapi.render.Render
render connect object
kwargs: dict .. all other kwargs to pass on to renderapi.client.import_tilespecs
""" # noqa: E501
tslists = (
max((len(tilespecs) // max_tilespecs_per_group) + 1, poolsize) if
max_tilespecs_per_group is not None else poolsize)
set_stack_state(stack, 'LOADING', host, port, owner, project)
partial_import = partial(
import_tilespecs, stack, sharedTransforms=sharedTransforms,
subprocess_mode=subprocess_mode, host=host, port=port,
owner=owner, project=project, client_script=client_script,
render=render, memGB=memGB, **kwargs)
# TODO this is a weird way to do splits.... is that okay?
tilespec_groups = [g for g in
(tilespecs[i::tslists] for i in range(tslists)) if g]
with mpPool(poolsize) as pool:
pool.map(partial_import, tilespec_groups)
if close_stack:
set_stack_state(stack, 'COMPLETE', host, port, owner, project)