def runInpainting(self, image, mask, folder): basename = os.path.splitext(image['name'])[0] outPath = VolumePath(basename + '_result.jpg') artifactPath = VolumePath('job_artifacts') job = docker_run.delay( 'zachmullen/inpainting:latest', container_args=[ GirderFileIdToVolume(image['_id']), GirderFileIdToVolume(mask['_id']), outPath, '--artifacts-dir', artifactPath, '--progress-pipe', ProgressPipe() ], girder_job_title='Inpainting: %s' % image['name'], girder_result_hooks=[ GirderUploadVolumePathToFolder(outPath, folder['_id'], upload_kwargs={ 'reference': json.dumps({ 'inpaintedImage': True, 'folderId': str(folder['_id']), }) }), # GirderUploadVolumePathJobArtifact(artifactPath) ]).job folder['inpaintingJobId'] = job['_id'] Folder().save(folder) job['inpaintingImageId'] = image['_id'] job['inpaintingMaskId'] = mask['_id'] job['inpaintingFolderId'] = folder['_id'] return Job().save(job)
def run_albany(self, params): """Run albany on a folder that is on girder. Will store the output in the specified output folder. """ inputFolderId = params.get('inputFolderId') outputFolderId = params.get('outputFolderId') filename = 'input.yaml' folder_name = 'workingDir' volume = GirderFolderIdToVolume(inputFolderId, volume=TemporaryVolume.default, folder_name=folder_name) outputDir = inputFolderId + '/' + folder_name + '/output.exo' volumepath = VolumePath(outputDir, volume=TemporaryVolume.default) result = docker_run.delay(ALBANY_IMAGE, pull_image=False, container_args=[filename], entrypoint='/usr/local/albany/bin/AlbanyT', remove_container=True, working_dir=volume, girder_result_hooks=[ GirderUploadVolumePathToFolder( volumepath, outputFolderId) ]) # Set the multiscale meta data and return the job jobId = result.job['_id'] return utils.setMultiscaleMetaData(jobId, inputFolderId, outputFolderId)
def run_dream3d(self, params): """Run Dream3D on a folder that is on girder. Will store the output in the specified output folder. """ inputFolderId = params.get('inputFolderId') outputFolderId = params.get('outputFolderId') folder_name = 'workingDir' volume = GirderFolderIdToVolume(inputFolderId, volume=TemporaryVolume.default, folder_name=folder_name) outputDir = inputFolderId + '/' + folder_name + '/output' volumepath = VolumePath(outputDir, volume=TemporaryVolume.default) result = docker_run.delay( DREAM3D_IMAGE, pull_image=False, container_args=[ '-c', 'bash /root/runPipelineRunner $(ls *.json | head -1)' ], remove_container=True, working_dir=volume, entrypoint='bash', girder_result_hooks=[ GirderUploadVolumePathToFolder(volumepath, outputFolderId) ]) # Set the multiscale meta data and return the job jobId = result.job['_id'] return utils.setMultiscaleMetaData(jobId, inputFolderId, outputFolderId)
def test_docker_run(self, params): result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=['stdio', '-m', 'hello docker!'], remove_container=True) return result.job
def selectBest(initWorkingSetName, stepName, requestInfo, jobId, outputFolder, imageFiles, dsmFile): """ Run a Girder Worker job to select the best image pair. Requirements: - Danesfield Docker image is available on host :param initWorkingSetName: The name of the top-level working set. :type initWorkingSetName: str :param stepName: The name of the step. :type stepName: str (DanesfieldStep) :param requestInfo: HTTP request and authorization info. :type requestInfo: RequestInfo :param jobId: Job ID. :type jobId: str :param outputFolder: Output folder document. :type outputFolder: dict :param imageFiles: List of image files. :type imageFiles: list[dict] :param file: DSM image file document. :type file: dict :returns: Job document. """ gc = createGirderClient(requestInfo) # Docker container arguments containerArgs = list(itertools.chain( [ 'danesfield/tools/select_best.py', '--dsm', GirderFileIdToVolume(dsmFile['_id'], gc=gc) ], [ GirderFileIdToVolume(imageFile['_id'], gc=gc) for imageFile in imageFiles ] )) asyncResult = docker_run.delay( **createDockerRunArguments( image=DockerImage.DANESFIELD, containerArgs=containerArgs, jobTitle='[%s] Select best' % initWorkingSetName, jobType=stepName, user=requestInfo.user ) ) # Add info for job event listeners job = asyncResult.job job = addJobInfo(job, jobId=jobId, stepName=stepName) return job
def _runUtm(folder, paramsFile, outputFolder): outpath = VolumePath('__results__') return docker_run.delay('samuelgerber/utm', container_args=[ GirderFolderIdToVolume(folder['_id']), GirderFileIdToVolume(paramsFile['_id']), '--workingfolder', outpath ], girder_job_title='UTM: ' + folder['name'], girder_result_hooks=[ GirderUploadVolumePathToFolder(outpath, outputFolder['_id']) ]).job
def test_docker_run_progress_pipe(self, params): progressions = params.get('progressions') progress_pipe = ProgressPipe() result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=['progress', '-p', progress_pipe, '--progressions', progressions], remove_container=True) return result.job
def test_docker_run_girder_file_to_volume(self, params): file_id = params.get('fileId') result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=['read_write', '-i', GirderFileIdToVolume(file_id), '-o', Connect(NamedOutputPipe('out'), HostStdOut())], remove_container=True) return result.job
def test_docker_run_cancel(self, params): mode = params.get('mode') result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=[mode], remove_container=True) assert wait_for_status(self.getCurrentUser(), result.job, JobStatus.RUNNING) result.revoke() return result.job
def test_docker_run_file_upload_to_item(self, params): item_id = params.get('itemId') contents = params.get('contents') volumepath = VolumePath('test_file') result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=['write', '-p', volumepath, '-m', contents], remove_container=True, girder_result_hooks=[GirderUploadVolumePathToItem(volumepath, item_id)]) return result.job
def test_docker_run_mount_volume(self, params): fixture_dir = params.get('fixtureDir') filename = 'read.txt' mount_dir = '/mnt/test' mount_path = os.path.join(mount_dir, filename) volumes = {fixture_dir: {'bind': mount_dir, 'mode': 'ro'}} result = docker_run.delay(TEST_IMAGE, pull_image=True, container_args=['read', '-p', mount_path], remove_container=True, volumes=volumes) return result.job
def test_docker_run_mount_idiomatic_volume(self, params): fixture_dir = params.get('fixtureDir') filename = 'read.txt' mount_dir = '/mnt/test' mount_path = os.path.join(mount_dir, filename) volume = BindMountVolume(fixture_dir, mount_path, 'ro') volumepath = VolumePath(filename, volume) result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=['read', '-p', volumepath], remove_container=True, volumes=[volume]) return result.job
def test_docker_run_temporary_volume_root(self, params): prefix = params.get('prefix') root = os.path.join(tempfile.gettempdir(), prefix) # We set the mode to 0o777 because the worker container is # running as the 'worker' user and needs to be able to have # read/write access to the TemporaryVolume volume = TemporaryVolume(host_dir=root, mode=0o777) result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=['print_path', '-p', volume], remove_container=True, volumes=[volume]) return result.job
def test_docker_run_girder_file_to_named_pipe_on_temp_vol(self, params): """ This is a simplified version of test_docker_run_girder_file_to_named_pipe it uses the TemporaryVolume, rather than having to setup the volumes 'manually', this is the approach we should encourage. """ file_id = params.get('fileId') pipe_name = 'input_pipe' connect = Connect(GirderFileIdToStream(file_id), NamedInputPipe(pipe_name)) result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=['read', '-p', connect], remove_container=True) return result.job
def test_docker_run_girder_file_to_named_pipe(self, params): tmp_dir = params.get('tmpDir') file_id = params.get('fileId') mount_dir = '/mnt/girder_worker/data' pipe_name = 'input_pipe' volumes = {tmp_dir: {'bind': mount_dir, 'mode': 'rw'}} connect = Connect(GirderFileIdToStream(file_id), NamedInputPipe(pipe_name, mount_dir, tmp_dir)) result = docker_run.delay(TEST_IMAGE, pull_image=True, container_args=['read', '-p', connect], remove_container=True, volumes=volumes) return result.job
def test_docker_run_named_pipe_output(self, params): tmp_dir = params.get('tmpDir') message = params.get('message') mount_dir = '/mnt/girder_worker/data' pipe_name = 'output_pipe' volumes = {tmp_dir: {'bind': mount_dir, 'mode': 'rw'}} connect = Connect(NamedOutputPipe(pipe_name, mount_dir, tmp_dir), HostStdOut()) result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=['write', '-p', connect, '-m', message], remove_container=True, volumes=volumes) return result.job
def test_docker_run_transfer_encoding_stream(self, params): item_id = params.get('itemId') file_id = params.get('fileId') delimiter = params.get('delimiter') headers = { 'Girder-Token': str(Token().createToken(getCurrentUser())['_id']) } url = '%s/%s?itemId=%s&delimiter=%s' % ( getApiUrl(), 'integration_tests/docker/input_stream', item_id, delimiter) container_args = [ 'read_write', '-i', GirderFileIdToVolume(file_id), '-o', Connect(NamedOutputPipe('out'), ChunkedTransferEncodingStream(url, headers)) ] result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=container_args, remove_container=True) return result.job
def _createThumbnail(item): # Remove previously attached thumbnails _removeThumbnails(item, saveItem=True) outdir = VolumePath('__thumbnails_output__') return docker_run.delay( 'girder/dicom_thumbnailer:latest', container_args=[ '--slices', str(THUMB_SLICES), '--width', str(THUMB_WIDTH), '--height', str(THUMB_HEIGHT), GirderItemIdToVolume(item['_id'], item_name=item['name']), outdir ], girder_job_title='DICOM thumbnail generation: %s' % item['name'], girder_result_hooks=[ GirderUploadVolumePathToItem( outdir, item['_id'], upload_kwargs={ 'reference': json.dumps({'interactive_thumbnail': True}) }) ]).job
def _createThumbnail(item, preset): # Remove previously attached thumbnails _removeThumbnails(item, saveItem=True) outdir = VolumePath('__thumbnails_output__') return docker_run.delay( 'zachmullen/3d_thumbnails:latest', container_args=[ '--angle-step', str(_ANGLE_STEP), '--width', str(_SIZE), '--height', str(_SIZE), '--preset', preset, GirderItemIdToVolume(item['_id'], item_name=item['name']), outdir ], girder_job_title='Interactive thumbnail generation: %s' % item['name'], girder_result_hooks=[ GirderUploadVolumePathToItem( outdir, item['_id'], upload_kwargs={ 'reference': json.dumps({'interactive_thumbnail': True}) }) ]).job
def run_smtk_mesh_placement(self, params): """Run an smtk mesh placement on a folder that is on girder. Will store the output in the specified output folder. """ inputFolderId = params.get('inputFolderId') outputFolderId = params.get('outputFolderId') folder_name = 'workingDir' volume = GirderFolderIdToVolume(inputFolderId, volume=TemporaryVolume.default, folder_name=folder_name) outputDir = inputFolderId + '/' + folder_name + '/output/' volumepath = VolumePath(outputDir, volume=TemporaryVolume.default) result = docker_run.delay( SMTK_IMAGE, pull_image=False, container_args=[ '-c', ('. ~/setupEnvironment; ' 'python /usr/local/afrl-automation/runner.py input.json; ' 'mkdir output; ' 'mv input.yaml output/; ' 'mv elastic.yaml output/;' 'mv *BC.exo output/') ], entrypoint='bash', remove_container=True, working_dir=volume, girder_result_hooks=[ GirderUploadVolumePathToFolder(volumepath, outputFolderId) ]) # Set the multiscale meta data and return the job jobId = result.job['_id'] return utils.setMultiscaleMetaData(jobId, inputFolderId, outputFolderId)
def runMetrics(initWorkingSetName, stepName, requestInfo, jobId, outputFolder, referenceFolder, referencePrefix, dtmFile, dsmFile, clsFile, mtlFile): """ Run a Girder Worker job to compute metrics on output files. Requirements: - Danesfield Docker image is available on host :param initWorkingSetName: The name of the top-level working set. :type initWorkingSetName: str :param stepName: The name of the step. :type stepName: str (DanesfieldStep) :param requestInfo: HTTP request and authorization info. :type requestInfo: RequestInfo :param jobId: Job ID. :type jobId: str :param outputFolder: Output folder document. :type outputFolder: dict :param referenceFolder: Reference directory. :type referenceFolder: dict :param referencePrefix: Reference file prefix. :type referencePrefix: str :param dtmFile: DTM file document. :type dtmFile: dict :param dsmFile: DSM file document. :type dsmFile: dict :param clsFile: CLS file document. :type clsFile: dict :param mtlFile: MTL file document. :type mtlFile: dict :returns: Job document. """ gc = createGirderClient(requestInfo) if referencePrefix == "STANDARD": # We know that there's no reference data with this selection containerArgs = ['echo', 'No ground truth selected for scoring'] asyncResult = docker_run.delay( **createDockerRunArguments(image=DockerImage.DANESFIELD, containerArgs=containerArgs, jobTitle='[%s] Run metrics' % initWorkingSetName, jobType=stepName, user=requestInfo.user)) else: # Otherwise we assume the reference data exists, and try to # run the metrics outputVolumePath = VolumePath('__output__') # Docker container arguments containerArgs = [ 'danesfield/tools/run_metrics.py', '--output-dir', outputVolumePath, '--ref-dir', GirderFolderIdToVolume(referenceFolder['_id'], gc=gc), '--ref-prefix', referencePrefix, '--dsm', GirderFileIdToVolume(dsmFile['_id'], gc=gc), '--cls', GirderFileIdToVolume(clsFile['_id'], gc=gc), '--mtl', GirderFileIdToVolume(mtlFile['_id'], gc=gc), '--dtm', GirderFileIdToVolume(dtmFile['_id'], gc=gc) ] # Result hooks # - Upload output files to output folder # - Provide upload metadata upload_kwargs = createUploadMetadata(jobId, stepName) resultHooks = [ GirderUploadVolumePathToFolder(outputVolumePath, outputFolder['_id'], upload_kwargs=upload_kwargs, gc=gc) ] asyncResult = docker_run.delay( **createDockerRunArguments(image=DockerImage.DANESFIELD, containerArgs=containerArgs, jobTitle='[%s] Run metrics' % initWorkingSetName, jobType=stepName, user=requestInfo.user, resultHooks=resultHooks)) # Add info for job event listeners job = asyncResult.job job = addJobInfo(job, jobId=jobId, stepName=stepName) return job
def unetSemanticSegmentation(initWorkingSetName, stepName, requestInfo, jobId, outputFolder, dsmFile, dtmFile, msiImageFile, rgbImageFile, configFile, modelFile): """ Run a Girder Worker job to segment buildings using UNet semantic segmentation. Requirements: - Danesfield Docker image is available on host :param initWorkingSetName: The name of the top-level working set. :type initWorkingSetName: strps :param stepName: The name of the step. :type stepName: str (DanesfieldStep) :param requestInfo: HTTP request and authorization info. :type requestInfo: RequestInfo :param jobId: Job ID. :type jobId: str :param outputFolder: Output folder document. :type outputFolder: dict :param dsmFile: DSM file document. :type dsmFile: dict :param dtmFile: DTM file document. :type dtmFile: dict :param msiImageFile: Pansharpened MSI image file document. :type msiImageFile: dict :param rgbImageFile: RGB image file document. :type rgbImageFile: dict :param configFile: Configuration file document. :type configFile: dict :param modelFile: Model file document. :type modelFile: dict :returns: Job document. """ gc = createGirderClient(requestInfo) # Set output directory outputVolumePath = VolumePath('.') # Docker container arguments containerArgs = [ 'danesfield/tools/kwsemantic_segment.py', # Configuration file GirderFileIdToVolume(configFile['_id'], gc=gc), # Model file GirderFileIdToVolume(modelFile['_id'], gc=gc), # RGB image GirderFileIdToVolume(rgbImageFile['_id'], gc=gc), # DSM GirderFileIdToVolume(dsmFile['_id'], gc=gc), # DTM GirderFileIdToVolume(dtmFile['_id'], gc=gc), # MSI image GirderFileIdToVolume(msiImageFile['_id'], gc=gc), # Output directory outputVolumePath, # Output file prefix 'semantic' ] # Result hooks # - Upload output files to output folder # - Provide upload metadata upload_kwargs = createUploadMetadata(jobId, stepName) resultHooks = [ GirderUploadVolumePathToFolder(outputVolumePath, outputFolder['_id'], upload_kwargs=upload_kwargs, gc=gc) ] asyncResult = docker_run.delay( **createDockerRunArguments(image=DockerImage.DANESFIELD, containerArgs=containerArgs, jobTitle=( '[%s] UNet semantic segmentation: %s' % (initWorkingSetName, dsmFile['name'])), jobType=stepName, user=requestInfo.user, resultHooks=resultHooks)) # Add info for job event listeners job = asyncResult.job job = addJobInfo(job, jobId=jobId, stepName=stepName) return job
def computeNdvi(initWorkingSetName, stepName, requestInfo, jobId, outputFolder, imageFiles, outputNdviFilename): """ Run a Girder Worker job to compute the NDVI from a set of images Requirements: - Danesfield Docker image is available on host :param initWorkingSetName: The name of the top-level working set. :type initWorkingSetName: str :param stepName: The name of the step. :type stepName: str (DanesfieldStep) :param requestInfo: HTTP request and authorization info. :type requestInfo: RequestInfo :param jobId: Job ID. :type jobId: str :param outputFolder: Output folder document. :type outputFolder: dict :param imageFiles: List of pansharpened image files. :type imageFiles: list[dict] :param outputNdviFilename: Filename for output NDVI :type outputNdviFilename: str :returns: Job document. """ gc = createGirderClient(requestInfo) # Set output file names ndviOutputVolumePath = VolumePath(outputNdviFilename) # Docker container arguments containerArgs = list( itertools.chain([ 'danesfield/tools/compute_ndvi.py', ], [ GirderFileIdToVolume(imageFile['_id'], gc=gc) for imageFile in imageFiles ], [ndviOutputVolumePath])) # Result hooks # - Upload output files to output folder # - Provide upload metadata upload_kwargs = createUploadMetadata(jobId, stepName) resultHooks = [ GirderUploadVolumePathToFolder(ndviOutputVolumePath, outputFolder['_id'], upload_kwargs=upload_kwargs, gc=gc) ] asyncResult = docker_run.delay( **createDockerRunArguments(image=DockerImage.DANESFIELD, containerArgs=containerArgs, jobTitle='[%s] Compute NDVI' % initWorkingSetName, jobType=stepName, user=requestInfo.user, resultHooks=resultHooks)) # Add info for job event listeners job = asyncResult.job job = addJobInfo(job, jobId=jobId, stepName=stepName) return job
def runPhotomorph(self, folder, maskRect): user = self.getCurrentUser() mp4Out = VolumePath('__output_mp4s__/') gifOut = VolumePath('__output_gifs__/') parent = Folder().load(folder['parentId'], level=AccessType.WRITE, exc=True, user=user) outputFolder = Folder().createFolder(parent, '_output', public=False, creator=user, reuseExisting=True) outputMp4 = Folder().createFolder(outputFolder, 'mp4s', public=False, creator=user, reuseExisting=True) outputGif = Folder().createFolder(outputFolder, 'gifs', public=False, creator=user, reuseExisting=True) parent['photomorphOutputFolderId'] = outputFolder['_id'] parent['photomorphOutputItems'] = {} parent['photomorphMaskRect'] = maskRect parent['photomorphJobStatus'] = JobStatus.QUEUED parent['photomorphOutputItems'] = {'gif': [], 'mp4': []} job = docker_run.delay( 'zachmullen/photomorph:latest', container_args=[ '--mp4-out', mp4Out, '--gif-out', gifOut, '--mask-rect', ','.join(str(i) for i in itertools.chain(*maskRect)), GirderFolderIdToVolume(folder['_id'], folder_name='_input') ], girder_job_title='Timelapse creation: %s' % parent['name'], girder_result_hooks=[ GirderUploadVolumePathToFolder(mp4Out, outputMp4['_id'], upload_kwargs={ 'reference': json.dumps({ 'photomorph': True, 'folderId': str(parent['_id']), 'resultType': 'mp4' }) }), GirderUploadVolumePathToFolder(gifOut, outputGif['_id'], upload_kwargs={ 'reference': json.dumps({ 'photomorph': True, 'folderId': str(parent['_id']), 'resultType': 'gif' }) }) ]).job parent['photomorphJobId'] = job['_id'] Folder().save(parent) job['photomorphId'] = parent['_id'] return Job().save(job)
def getRoadVector(initWorkingSetName, stepName, requestInfo, jobId, outputFolder, left, bottom, right, top): """ Run a Girder Worker job to segment buildings by comparing a DSM to a DTM. Requirements: - Danesfield Docker image is available on host :param initWorkingSetName: The name of the top-level working set. :type initWorkingSetName: str :param stepName: The name of the step. :type stepName: str (DanesfieldStep) :param requestInfo: HTTP request and authorization info. :type requestInfo: RequestInfo :param jobId: Job ID. :type jobId: str :param outputFolder: Output folder document. :type outputFolder: dict :param left: Longitude of left / westernmost side of bounding box :type left: float :param bottom: Latitude of bottom / southernmost side of bounding box :type bottom: float :param right: Longitude of right / easternmost side of bounding box :type right: float :param top: Latitude of top / northernmost side of bounding box :type top: float :returns: Job document. """ gc = createGirderClient(requestInfo) # Set output file names outputVolumePath = VolumePath('__output__') # Docker container arguments containerArgs = [ 'danesfield/tools/get_road_vector.py', '--left', str(left), '--bottom', str(bottom), '--right', str(right), '--top', str(top), '--output-dir', outputVolumePath, ] # Result hooks # - Upload output files to output folder # - Provide upload metadata upload_kwargs = createUploadMetadata(jobId, stepName) resultHooks = [ GirderUploadVolumePathToFolder( outputVolumePath, outputFolder['_id'], upload_kwargs=upload_kwargs, gc=gc) ] asyncResult = docker_run.delay( **createDockerRunArguments( image=DockerImage.DANESFIELD, containerArgs=containerArgs, jobTitle='[%s] Get OSM road vector data' % initWorkingSetName, jobType=stepName, user=requestInfo.user, resultHooks=resultHooks ) ) # Add info for job event listeners job = asyncResult.job job = addJobInfo(job, jobId=jobId, stepName=stepName) return job
def generateDsm(initWorkingSetName, stepName, requestInfo, jobId, outputFolder, pointCloudFile, outputPrefix): """ Run a Girder Worker job to generate a Digital Surface Model (DSM) from a point cloud. Requirements: - Danesfield Docker image is available on host :param initWorkingSetName: The name of the top-level working set. :type initWorkingSetName: str :param stepName: The name of the step. :type stepName: str (DanesfieldStep) :param requestInfo: HTTP request and authorization info. :type requestInfo: RequestInfo :param jobId: Job ID. :type jobId: str :param outputFolder: Output folder document. :type outputFolder: dict :param pointCloudFile: Point cloud file document. :type pointCloudFile: dict :param outputPrefix: The prefix of the output file name. :type outputPrefix: str :returns: Job document. """ gc = createGirderClient(requestInfo) # Set output file name based on point cloud file dsmName = outputPrefix + '_P3D_DSM.tif' outputVolumePath = VolumePath(dsmName) # Docker container arguments containerArgs = [ 'danesfield/tools/generate_dsm.py', outputVolumePath, '--source_points', GirderFileIdToVolume(pointCloudFile['_id'], gc=gc) ] # Result hooks # - Upload output files to output folder # - Provide upload metadata upload_kwargs = createUploadMetadata(jobId, stepName) resultHooks = [ GirderUploadVolumePathToFolder(outputVolumePath, outputFolder['_id'], upload_kwargs=upload_kwargs, gc=gc) ] asyncResult = docker_run.delay( **createDockerRunArguments(image=DockerImage.DANESFIELD, containerArgs=containerArgs, jobTitle=('[%s] Generate DSM: %s' % (initWorkingSetName, pointCloudFile['name'])), jobType=stepName, user=requestInfo.user, resultHooks=resultHooks)) # Add info for job event listeners job = asyncResult.job job = addJobInfo(job, jobId=jobId, stepName=stepName) return job
def test_docker_run_raises_exception(self, params): result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=['raise_exception'], remove_container=True) return result.job
def textureMapping(initWorkingSetName, stepName, requestInfo, jobId, outputFolder, objFiles, imageFiles, dsmFile, dtmFile): """ Run a Girder Worker job to run texture mapping. Requirements: - Danesfield Docker image is available on host :param initWorkingSetName: The name of the top-level working set. :type initWorkingSetName: str :param stepName: The name of the step. :type stepName: str (DanesfieldStep) :param requestInfo: HTTP request and authorization info. :type requestInfo: RequestInfo :param jobId: Job ID. :type jobId: str :param outputFolder: Output folder document. :type outputFolder: dict :param objFiles: List of OBJ files. :type objFiles: list[dict] :param imageFiles: List of cropped and pansharpened image files. :type imageFiles: list[dict] :param dsmFile: DSM file document. :type dsmFile: dict :param dtmFile: DTM file document. :type dtmFile: dict :returns: Job document. """ gc = createGirderClient(requestInfo) # Set output directory outputVolumePath = VolumePath('__output__') # Set output path for occlusion mesh occlusionMeshName = 'xxxx.obj' occlusionMeshVolumePath = VolumePath(occlusionMeshName) containerArgs = [ 'danesfield/tools/texture_mapping.py', GirderFileIdToVolume(dsmFile['_id'], gc=gc), GirderFileIdToVolume(dtmFile['_id'], gc=gc), outputVolumePath, occlusionMeshVolumePath, '--crops' ] containerArgs.extend( [GirderFileIdToVolume(f['_id'], gc=gc) for f in imageFiles]) containerArgs.append('--buildings') containerArgs.extend( [GirderFileIdToVolume(f['_id'], gc=gc) for f in objFiles]) # Result hooks # - Upload output files to output folder # - Provide upload metadata upload_kwargs = createUploadMetadata(jobId, stepName) resultHooks = [ GirderUploadVolumePathToFolder(outputVolumePath, outputFolder['_id'], upload_kwargs=upload_kwargs, gc=gc), GirderUploadVolumePathToFolder(occlusionMeshVolumePath, outputFolder['_id'], upload_kwargs=upload_kwargs, gc=gc) ] asyncResult = docker_run.delay( **createDockerRunArguments(image=DockerImage.DANESFIELD, containerArgs=containerArgs, jobTitle='[%s] Texture mapping' % initWorkingSetName, jobType=stepName, user=requestInfo.user, resultHooks=resultHooks)) # Add info for job event listeners job = asyncResult.job job = addJobInfo(job, jobId=jobId, stepName=stepName) return job
def generatePointCloud(initWorkingSetName, stepName, requestInfo, jobId, outputFolder, imageFiles, longitude, latitude, longitudeWidth, latitudeWidth): """ Run a Girder Worker job to generate a 3D point cloud from 2D images. Requirements: - P3D Girder Worker Docker image is available on host - Host folder /mnt/GTOPO30 contains GTOPO 30 data :param initWorkingSetName: The name of the top-level working set. :type initWorkingSetName: str :param stepName: The name of the step. :type stepName: str (DanesfieldStep) :param requestInfo: HTTP request and authorization info. :type requestInfo: RequestInfo :param jobId: Job ID. :type jobId: str :param outputFolder: Output folder document. :type outputFolder: dict :param imageFiles: List of input image files. :type imageFiles: list[dict] :param longitude: :type longitude: :param latitude: :type latitude: :param longitudeWidth: :type longitudeWidth: :param latitudeWidth: :type latitudeWidth: :returns: Job document. """ gc = createGirderClient(requestInfo) # Docker volumes volumes = [ BindMountVolume(host_path='/mnt/GTOPO30', container_path='/P3D/GTOPO30', mode='ro') ] outputVolumePath = VolumePath('__output__') # Docker container arguments # TODO: Consider a solution where args are written to a file, in # case of very long command lines containerArgs = list(itertools.chain( [ 'python', '/P3D/RTN_distro/scripts/generate_point_cloud.pyc', '--out', outputVolumePath, '--longitude', str(longitude), '--latitude', str(latitude), '--longitudeWidth', str(longitudeWidth), '--latitudeWidth', str(latitudeWidth), '--firstProc', '0', '--threads', '8', '--images' ], [GirderFileIdToVolume(imageFile['_id'], gc=gc) for imageFile in imageFiles], )) # Result hooks # - Upload output files to output folder # - Provide upload metadata upload_kwargs = createUploadMetadata(jobId, stepName) resultHooks = [ GirderUploadVolumePathToFolder( outputVolumePath, outputFolder['_id'], upload_kwargs=upload_kwargs, gc=gc) ] asyncResult = docker_run.delay( volumes=volumes, **createDockerRunArguments( image=DockerImage.P3D, containerArgs=containerArgs, jobTitle='[%s] Generate point cloud' % initWorkingSetName, jobType=stepName, user=requestInfo.user, resultHooks=resultHooks ) ) # Add info for job event listeners job = asyncResult.job job = addJobInfo(job, jobId=jobId, stepName=stepName) return job
def segmentByHeight(initWorkingSetName, stepName, requestInfo, jobId, outputFolder, dsmFile, dtmFile, ndviFile, roadVectorFile): """ Run a Girder Worker job to segment buildings by comparing a DSM to a DTM. Requirements: - Danesfield Docker image is available on host :param initWorkingSetName: The name of the top-level working set. :type initWorkingSetName: str :param stepName: The name of the step. :type stepName: str (DanesfieldStep) :param requestInfo: HTTP request and authorization info. :type requestInfo: RequestInfo :param jobId: Job ID. :type jobId: str :param outputFolder: Output folder document. :type outputFolder: dict :param dsmFile: DSM file document. :type dsmFile: dict :param dtmFile: DTM file document. :type dtmFile: dict :param ndviFile: NDVI file document. :type ndviFile: dict :param roadVectorFile: Road vector file. :type roadVectorFile: dict :returns: Job document. """ gc = createGirderClient(requestInfo) # Set output file names # TODO: Danesfield master script hardcodes these without any # prefix; do the same here thresholdOutputVolumePath = VolumePath('threshold_CLS.tif') roadRasterOutputVolumePath = VolumePath('road_rasterized.tif') roadBridgeRasterOutputVolumePath = VolumePath('road_rasterized_bridge.tif') # Docker container arguments containerArgs = [ 'danesfield/tools/segment_by_height.py', # DSM GirderFileIdToVolume(dsmFile['_id'], gc=gc), # DTM GirderFileIdToVolume(dtmFile['_id'], gc=gc), # Threshold output image thresholdOutputVolumePath, # Normalized Difference Vegetation Index image '--input-ndvi', GirderFileIdToVolume(ndviFile['_id'], gc=gc), '--road-vector', GirderFileIdToVolume(roadVectorFile['_id'], gc=gc), '--road-rasterized', roadRasterOutputVolumePath, '--road-rasterized-bridge', roadBridgeRasterOutputVolumePath ] # Result hooks # - Upload output files to output folder # - Provide upload metadata upload_kwargs = createUploadMetadata(jobId, stepName) resultHooks = [ GirderUploadVolumePathToFolder(thresholdOutputVolumePath, outputFolder['_id'], upload_kwargs=upload_kwargs, gc=gc) ] asyncResult = docker_run.delay( **createDockerRunArguments(image=DockerImage.DANESFIELD, containerArgs=containerArgs, jobTitle='[%s] Segment by height: %s' % (initWorkingSetName, dsmFile['name']), jobType=stepName, user=requestInfo.user, resultHooks=resultHooks)) # Add info for job event listeners job = asyncResult.job job = addJobInfo(job, jobId=jobId, stepName=stepName) return job