def run(cls, ifnm, ofnm, args, **kw ):
'''converts input filename into output using exact arguments as provided in args'''
if not cls.installed:
return None
failonread = kw.get('failonread') or (not block_reads)
tmp = None
with Locks(ifnm, ofnm, failonexist=True) as l:
if l.locked: # the file is not being currently written by another process
command = [cls.CONVERTERCOMMAND]
command.extend(args)
log.debug('Run command: [%s]', misc.toascii(command))
proceed = True
if ofnm is not None and os.path.exists(ofnm) and os.path.getsize(ofnm)>16:
if kw.get('nooverwrite', False) is True:
proceed = False
log.warning ('Run: output exists before command [%s], skipping', misc.toascii(ofnm))
else:
log.warning ('Run: output exists before command [%s], overwriting', misc.toascii(ofnm))
if proceed is True:
command, tmp = misc.start_nounicode_win(ifnm, command)
try:
retcode = call (command)
except Exception:
retcode = 100
log.exception('Error running command: %s', command)
misc.end_nounicode_win(tmp)
if retcode == 99:
# in case of a timeout
log.info ('Run: timed-out for [%s]', misc.toascii(command))
if ofnm is not None and os.path.exists(ofnm):
os.remove(ofnm)
raise ImageServiceException(412, 'Requested timeout reached')
if retcode!=0:
log.info ('Run: returned [%s] for [%s]', retcode, misc.toascii(command))
return None
if ofnm is None:
return str(retcode)
# output file does not exist for some operations, like tiles
# tile command does not produce a file with this filename
# if not os.path.exists(ofnm):
# log.error ('Run: output does not exist after command [%s]', ofnm)
# return None
elif l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1,15))
# make sure the write of the output file have finished
if ofnm is not None and os.path.exists(ofnm):
with Locks(ofnm, failonread=failonread) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1,15))
# safeguard for incorrectly converted files, sometimes only the tiff header can be written
# empty lock files are automatically removed before by lock code
if os.path.exists(ofnm) and os.path.getsize(ofnm) < cls.MINIMUM_FILE_SIZE:
log.error ('Run: output file is smaller than %s bytes, probably an error, removing [%s]', cls.MINIMUM_FILE_SIZE, ofnm)
os.remove(ofnm)
return None
return ofnm
def run_read(cls, ifnm, command ):
with Locks(ifnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1,10))
command, tmp = misc.start_nounicode_win(ifnm, command)
log.debug('run_read command: [%s]', misc.toascii(command))
out = cls.run_command( command )
misc.end_nounicode_win(tmp)
return out
def meta_dicom(cls, ifnm, series=0, xml=None, **kw):
'''appends nodes to XML'''
if os.path.basename(
ifnm
) == 'DICOMDIR': # skip reading metadata for teh index file
return
try:
import dicom
except (ImportError, OSError):
log.warn(
'pydicom is needed for DICOM support, skipping DICOM metadata...'
)
return
def recurse_tree(dataset, parent, encoding='latin-1'):
for de in dataset:
if de.tag == ('7fe0', '0010'):
continue
node = etree.SubElement(parent,
'tag',
name=de.name,
type=':///DICOM#%04.x,%04.x' %
(de.tag.group, de.tag.element))
if de.VR == "SQ": # a sequence
for i, dataset in enumerate(de.value):
recurse_tree(dataset, node, encoding)
else:
if isinstance(de.value, dicom.multival.MultiValue):
value = ','.join(
safedecode(i, encoding) for i in de.value)
else:
value = safedecode(de.value, encoding)
try:
node.set('value', value.strip())
except (ValueError, Exception):
pass
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
ds = dicom.read_file(tmp or ifnm)
except (Exception):
misc.end_nounicode_win(tmp)
return
encoding = dicom_init_encoding(ds)
recurse_tree(ds, xml, encoding=encoding)
misc.end_nounicode_win(tmp)
return
def tile(cls, token, ofnm, level=None, x=None, y=None, sz=None, **kw):
'''extract tile from image
default interface:
Level,X,Y tile from input filename into output in TIFF format
alternative interface, not required to support and may return None in this case
scale=scale, x1=x1, y1=y1, x2=x2, y2=y2, arbitrary_size=False '''
# open slide driver does not support arbitrary size interface
if kw.get('arbitrary_size',
False) == True or level is None or sz is None:
return None
ifnm = token.first_input_file()
series = token.series
if not cls.supported(token):
return None
log.debug('Tile: %s %s %s %s %s for [%s]', level, x, y, sz, series,
ifnm)
level = misc.safeint(level, 0)
x = misc.safeint(x, 0)
y = misc.safeint(y, 0)
sz = misc.safeint(sz, 0)
with Locks(ifnm, ofnm, failonexist=True) as l:
if l.locked: # the file is not being currently written by another process
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
slide = openslide.OpenSlide(tmp or ifnm)
dz = deepzoom.DeepZoomGenerator(slide,
tile_size=sz,
overlap=0)
img = dz.get_tile(dz.level_count - level - 1, (x, y))
img.save(ofnm, 'TIFF', compression='LZW')
slide.close()
misc.end_nounicode_win(tmp)
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
misc.end_nounicode_win(tmp)
return None
# make sure the file was written
with Locks(ofnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
return ofnm
def supported(cls, token, **kw):
'''return True if the input file format is supported'''
if not cls.installed:
return False
return False #!!! TODO: re-enable
ifnm = token.first_input_file()
log.debug('Supported for: %s', ifnm)
#if token.is_multifile_series() is True:
# return False
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
s = openslide.OpenSlide.detect_format(tmp or ifnm)
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
s = None
misc.end_nounicode_win(tmp)
return (s is not None and s != 'generic-tiff')
def thumbnail(cls, token, ofnm, width, height, **kw):
'''converts input filename into output thumbnail'''
ifnm = token.first_input_file()
series = token.series
if not cls.supported(token):
return None
log.debug('Thumbnail: %s %s %s for [%s]', width, height, series, ifnm)
fmt = kw.get('fmt', 'jpeg').upper()
with Locks(ifnm, ofnm, failonexist=True) as l:
if l.locked: # the file is not being currently written by another process
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
slide = openslide.OpenSlide(tmp or ifnm)
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
misc.end_nounicode_win(tmp)
return None
img = slide.get_thumbnail((width, height))
try:
img.save(ofnm, fmt)
except IOError:
tmp = '%s.tif' % ofnm
img.save(tmp, 'TIFF')
ConverterImgcnv.thumbnail(ProcessToken(ifnm=tmp),
ofnm=ofnm,
width=width,
height=height,
**kw)
slide.close()
misc.end_nounicode_win(tmp)
elif l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
# make sure the file was written
with Locks(ofnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
return ofnm
def writeHistogram(cls, token, ofnm, **kw):
'''writes Histogram in libbioimage format'''
if not cls.supported(token):
return None
ifnm = token.first_input_file()
log.debug('Writing histogram for %s into: %s', ifnm, ofnm)
# estimate best level for histogram approximation
preferred_side = 1024
dims = token.dims or {}
num_x = int(dims.get('image_num_x', 0))
num_y = int(dims.get('image_num_y', 0))
width = preferred_side
height = preferred_side
#scales = [float(i) for i in dims.get('image_resolution_level_scales', '').split(',')]
# scales metadata is provided according to original data storage, not deepzoom virtual view
sz = max(num_x, num_y)
num_levels = int(
round(
math.ceil(math.log(sz, 2)) -
math.ceil(math.log(min_level_size, 2)) + 1))
scales = [1 / float(pow(2, i)) for i in range(0, num_levels)]
log.debug('scales: %s', scales)
sizes = [(round(num_x * i), round(num_y * i)) for i in scales]
log.debug('scales: %s', sizes)
relatives = [
max(width / float(sz[0]), height / float(sz[1])) for sz in sizes
]
log.debug('relatives: %s', relatives)
relatives = [i if i >= 1 else 1000000 for i in relatives
] # only allow levels containing entire blocks
log.debug('relatives: %s', relatives)
level = relatives.index(min(relatives))
log.debug('Chosen level: %s', level)
# extract desired level and compute histogram
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
slide = openslide.OpenSlide(tmp or ifnm)
dz = deepzoom.DeepZoomGenerator(slide,
tile_size=preferred_side,
overlap=0)
img = dz.get_tile(dz.level_count - level - 1, (0, 0))
slide.close()
misc.end_nounicode_win(tmp)
hist = img.histogram()
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
misc.end_nounicode_win(tmp)
log.debug('histogram: %s', hist)
# currently openslide only supports 8 bit 3 channel images
# need to generate a histogram file uniformly distributed from 0..255
channels = 3
i = 0
import struct
with open(ofnm, 'wb') as f:
f.write(struct.pack('<cccc', 'B', 'I', 'M', '1')) # header
f.write(struct.pack('<cccc', 'I', 'H', 'S', '1')) # spec
f.write(struct.pack('<L', channels)) # number of histograms
# write histograms
for c in range(channels):
f.write(struct.pack('<cccc', 'B', 'I', 'M', '1')) # header
f.write(struct.pack('<cccc', 'H', 'S', 'T', '1')) # spec
# write bim::HistogramInternal
f.write(struct.pack('<H',
8)) # uint16 data_bpp; // bits per pixel
f.write(struct.pack(
'<H', 1)) # uint16 data_fmt; // signed, unsigned, float
f.write(struct.pack('<d', 0.0)) # double shift;
f.write(struct.pack('<d', 1.0)) # double scale;
f.write(struct.pack('<d', 0.0)) # double value_min;
f.write(struct.pack('<d', 255.0)) # double value_max;
# write data
f.write(struct.pack('<L', 256)) # histogram size, here 256
for i in range(256):
f.write(
struct.pack('<Q', hist[i])
) # histogram data, here each color has freq of 100
return ofnm
def meta(cls, token, **kw):
ifnm = token.first_input_file()
if not cls.supported(token):
return {}
log.debug('Meta for: %s', ifnm)
with Locks(ifnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 10))
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
slide = openslide.OpenSlide(tmp or ifnm)
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
misc.end_nounicode_win(tmp)
return {}
rd = {
'format':
slide.properties.get(openslide.PROPERTY_NAME_VENDOR),
'image_num_series':
0,
'image_num_x':
slide.dimensions[0],
'image_num_y':
slide.dimensions[1],
'image_num_z':
1,
'image_num_t':
1,
'image_num_c':
3,
'image_num_resolution_levels':
slide.level_count,
'image_resolution_level_scales':
','.join([str(1.0 / i) for i in slide.level_downsamples]),
'image_pixel_format':
'unsigned integer',
'image_pixel_depth':
8,
'magnification':
slide.properties.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER),
'channel_0_name':
'red',
'channel_1_name':
'green',
'channel_2_name':
'blue',
'channel_color_0':
'255,0,0',
'channel_color_1':
'0,255,0',
'channel_color_2':
'0,0,255',
# new format
'channels/channel_00000/name':
'red',
'channels/channel_00000/color':
'255,0,0',
'channels/channel_00001/name':
'green',
'channels/channel_00001/color':
'0,255,0',
'channels/channel_00002/name':
'blue',
'channels/channel_00002/color':
'0,0,255',
}
if slide.properties.get(openslide.PROPERTY_NAME_MPP_X,
None) is not None:
rd.update({
'pixel_resolution_x':
slide.properties.get(openslide.PROPERTY_NAME_MPP_X, 0),
'pixel_resolution_y':
slide.properties.get(openslide.PROPERTY_NAME_MPP_Y, 0),
'pixel_resolution_unit_x':
'microns',
'pixel_resolution_unit_y':
'microns'
})
# custom - any other tags in proprietary files should go further prefixed by the custom parent
for k, v in slide.properties.iteritems():
rd['custom/%s' % k.replace('.', '/')] = v
slide.close()
# read metadata using imgcnv since openslide does not decode all of the info
meta = ConverterImgcnv.meta(
ProcessToken(ifnm=tmp or ifnm, series=token.series), **kw)
meta.update(rd)
rd = meta
misc.end_nounicode_win(tmp)
return rd
def info(cls, token, **kw):
'''returns a dict with file info'''
ifnm = token.first_input_file()
series = token.series
if not cls.supported(token):
return {}
log.debug('Info for: %s', ifnm)
with Locks(ifnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 10))
if not os.path.exists(ifnm):
return {}
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
slide = openslide.OpenSlide(tmp or ifnm)
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
misc.end_nounicode_win(tmp)
return {}
info2 = {
'format':
slide.properties[openslide.PROPERTY_NAME_VENDOR],
'image_num_series':
0,
'image_series_index':
0,
'image_num_x':
slide.dimensions[0],
'image_num_y':
slide.dimensions[1],
'image_num_z':
1,
'image_num_t':
1,
'image_num_c':
3,
'image_num_resolution_levels':
slide.level_count,
'image_resolution_level_scales':
','.join([str(1.0 / i) for i in slide.level_downsamples]),
'image_pixel_format':
'unsigned integer',
'image_pixel_depth':
8
}
if slide.properties.get(openslide.PROPERTY_NAME_MPP_X,
None) is not None:
info2.update({
'pixel_resolution_x':
slide.properties.get(openslide.PROPERTY_NAME_MPP_X, 0),
'pixel_resolution_y':
slide.properties.get(openslide.PROPERTY_NAME_MPP_Y, 0),
'pixel_resolution_unit_x':
'microns',
'pixel_resolution_unit_y':
'microns'
})
slide.close()
# read metadata using imgcnv since openslide does not decode all of the info
info = ConverterImgcnv.info(
ProcessToken(ifnm=tmp or ifnm, series=series), **kw)
misc.end_nounicode_win(tmp)
info.update(info2)
return info
return {}
def meta_dicom_parsed(cls, ifnm, xml=None, **kw):
'''appends nodes to XML'''
try:
import dicom
except (ImportError, OSError):
log.warn(
'pydicom is needed for DICOM support, skipping DICOM metadata...'
)
return
def append_tag(dataset,
tag,
parent,
name=None,
fmt=None,
safe=True,
encoding='latin-1'):
de = dataset.get(tag, None)
if de is None:
return
name = name or de.name
typ = ':///DICOM#%04.x,%04.x' % (de.tag.group, de.tag.element)
if fmt is None:
if isinstance(de.value, dicom.multival.MultiValue):
value = ','.join(safedecode(i, encoding) for i in de.value)
else:
value = safedecode(de.value, encoding)
else:
if safe is True:
try:
value = fmt(safedecode(de.value, encoding))
except (Exception):
value = safedecode(de.value, encoding)
else:
value = fmt(safedecode(de.value, encoding))
value = value.strip()
if len(value) > 0:
node = etree.SubElement(parent,
'tag',
name=name,
value=value,
type=typ)
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
ds = dicom.read_file(tmp or ifnm)
except (Exception):
misc.end_nounicode_win(tmp)
return
encoding = dicom_init_encoding(ds)
append_tag(ds, ('0010', '0020'), xml, encoding=encoding) # Patient ID
try:
append_tag(ds, ('0010', '0010'),
xml,
name='Patient\'s First Name',
safe=False,
fmt=lambda x: x.split('^', 1)[1],
encoding=encoding) # Patient's Name
append_tag(ds, ('0010', '0010'),
xml,
name='Patient\'s Last Name',
safe=False,
fmt=lambda x: x.split('^', 1)[0],
encoding=encoding) # Patient's Name
except (Exception):
append_tag(ds, ('0010', '0010'), xml,
encoding=encoding) # Patient's Name
append_tag(ds, ('0010', '0040'), xml,
encoding=encoding) # Patient's Sex 'M'
append_tag(ds, ('0010', '1010'), xml,
encoding=encoding) # Patient's Age '019Y'
append_tag(ds, ('0010', '0030'), xml,
fmt=dicom_parse_date) # Patient's Birth Date
append_tag(ds, ('0012', '0062'), xml,
encoding=encoding) # Patient Identity Removed
append_tag(ds, ('0008', '0020'), xml,
fmt=dicom_parse_date) # Study Date
append_tag(ds, ('0008', '0030'), xml,
fmt=dicom_parse_time) # Study Time
append_tag(ds, ('0008', '0060'), xml, encoding=encoding) # Modality
append_tag(ds, ('0008', '1030'), xml,
encoding=encoding) # Study Description
append_tag(ds, ('0008', '103e'), xml,
encoding=encoding) # Series Description
append_tag(ds, ('0008', '0080'), xml,
encoding=encoding) # Institution Name
append_tag(ds, ('0008', '0090'), xml,
encoding=encoding) # Referring Physician's Name
append_tag(ds, ('0008', '0008'), xml) # Image Type
append_tag(ds, ('0008', '0012'), xml,
fmt=dicom_parse_date) # Instance Creation Date
append_tag(ds, ('0008', '0013'), xml,
fmt=dicom_parse_time) # Instance Creation Time
append_tag(ds, ('0008', '1060'), xml,
encoding=encoding) # Name of Physician(s) Reading Study
append_tag(ds, ('0008', '2111'), xml,
encoding=encoding) # Derivation Description
misc.end_nounicode_win(tmp)