def _convert_list(self, value):
"""Convert a string into a typed numpy array.
If it is not possible it returns a numpy string.
"""
try:
numpy_values = []
values = value.split(" ")
types = set([])
for string_value in values:
v = self._convert_scalar_value(string_value)
numpy_values.append(v)
types.add(v.dtype.type)
result_type = numpy.result_type(*types)
if issubclass(result_type.type, (numpy.string_, six.binary_type)):
# use the raw data to create the result
return numpy.string_(value)
elif issubclass(result_type.type, (numpy.unicode_, six.text_type)):
# use the raw data to create the result
return numpy.unicode_(value)
else:
return numpy.array(numpy_values, dtype=result_type)
except ValueError:
return numpy.string_(value)
def write(self, struct_name, data_dict):
"""write data_dict under the group struct_name in the open hdf5 file
:param struct_name: the identificatioon of the structure to write in the hdf5
:param data_dict: The python dictionnary containing the informations to write
"""
if self.file is None:
info = "No file currently open"
logger.info(info)
return
group_l1 = self.file.create_group(struct_name)
group_l1.attrs['OCTAVE_GLOBAL'] = np.uint8(1)
group_l1.attrs['OCTAVE_NEW_FORMAT'] = np.uint8(1)
group_l1.create_dataset("type", data=np.string_('scalar struct'), dtype="|S14")
group_l2 = group_l1.create_group('value')
for ftparams in data_dict:
group_l3 = group_l2.create_group(ftparams)
group_l3.attrs['OCTAVE_NEW_FORMAT'] = np.uint8(1)
if type(data_dict[ftparams]) == str:
group_l3.create_dataset("type", (), data=np.string_('sq_string'), dtype="|S10")
if self.octave_targetted_version < 3.8:
group_l3.create_dataset("value", data=np.string_(data_dict[ftparams] + '0'))
else:
group_l3.create_dataset("value", data=np.string_(data_dict[ftparams]))
else:
group_l3.create_dataset("type", (), data=np.string_('scalar'), dtype="|S7")
group_l3.create_dataset("value", data=data_dict[ftparams])
def to_hdf5(self, group):
"""Write reaction to an HDF5 group
Parameters
----------
group : h5py.Group
HDF5 group to write to
"""
group.attrs['mt'] = self.mt
if self.mt in REACTION_NAME:
group.attrs['label'] = np.string_(REACTION_NAME[self.mt])
else:
group.attrs['label'] = np.string_(self.mt)
group.attrs['Q_value'] = self.q_value
group.attrs['center_of_mass'] = 1 if self.center_of_mass else 0
for T in self.xs:
Tgroup = group.create_group(T)
if self.xs[T] is not None:
dset = Tgroup.create_dataset('xs', data=self.xs[T].y)
if hasattr(self.xs[T], '_threshold_idx'):
threshold_idx = self.xs[T]._threshold_idx + 1
else:
threshold_idx = 1
dset.attrs['threshold_idx'] = threshold_idx
for i, p in enumerate(self.products):
pgroup = group.create_group('product_{}'.format(i))
p.to_hdf5(pgroup)
def setup_root_attr(f, extension=None):
"""
Write the root metadata for this file
Parameter
---------
f : an h5py.File object
The file in which to write the data
"""
if extension is None:
extension = "ED-PIC"
# Required attributes
f.attrs["openPMD"] = numpy.string_("1.0.0")
f.attrs["openPMDextension"] = ext_list[extension]
f.attrs["basePath"] = numpy.string_("/data/%T/")
f.attrs["meshesPath"] = numpy.string_("meshes/")
f.attrs["particlesPath"] = numpy.string_("particles/")
f.attrs["iterationEncoding"] = numpy.string_("groupBased")
f.attrs["iterationFormat"] = numpy.string_("/data/%T/")
# Recommended attributes
f.attrs["author"] = numpy.string_("NN")
f.attrs["software"] = numpy.string_("simex_platform")
f.attrs["softwareVersion"] = numpy.string_("0.2")
f.attrs["date"] = numpy.string_( datetime.datetime.now(tzlocal()).strftime('%Y-%m-%d %H:%M:%S %z'))
def _write_buffer(self):
self.attrs['version'] = np.string_(__version__)
self.attrs['type'] = np.string_(type(self).__name__)
self['charge'] = self.buffer.charge
self['charge'].attrs['unit'] = self.buffer.charge_unit
self['numMol'] = self.buffer.numMol
self['volume'] = self.buffer.volume
self['volume'].attrs['unit'] = self.buffer.volume_unit
self['temperature'] = self.buffer.temperature
self['temperature'].attrs['unit'] = self.buffer.temperature_unit
self['timeLags'] = self.buffer.timeLags
self['timeLags'].attrs['unit'] = self.buffer.timeLags_unit
self['nCorr'] = self.buffer.nCorr
self['nCorr'].attrs['unit'] = self.buffer.nCorr_unit
def do_dec(dectype):
dec_group = self.require_group(dectype.value)
buf = getattr(self.buffer, dectype.value)
dec_group['decBins'] = buf.decBins
dec_group['decCorr'] = buf.decCorr
dec_group['decPairCount'] = buf.decPairCount
dec_group['decBins'].attrs['unit'] = buf.decBins_unit
dec_group['decCorr'].attrs['unit'] = buf.decCorr_unit
for type_ in DecType:
if getattr(self.buffer, type_.value) is not None:
do_dec(type_)
def test_roundtrip_strings_with_fill_value(self):
values = np.array(['ab', 'cdef', np.nan], dtype=object)
encoding = {'_FillValue': np.string_('X'), 'dtype': np.dtype('S1')}
original = Dataset({'x': ('t', values, {}, encoding)})
expected = original.copy(deep=True)
expected['x'][:2] = values[:2].astype('S')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
original = Dataset({'x': ('t', values, {}, {'_FillValue': '\x00'})})
if not isinstance(self, Only32BitTypes):
# these stores can save unicode strings
expected = original.copy(deep=True)
if isinstance(self, BaseNetCDF4Test):
# netCDF4 can't keep track of an empty _FillValue for VLEN
# variables
expected['x'][-1] = ''
elif (isinstance(self, (NetCDF3ViaNetCDF4DataTest,
NetCDF4ClassicViaNetCDF4DataTest)) or
(has_netCDF4 and type(self) is GenericNetCDFDataTest)):
# netCDF4 can't keep track of an empty _FillValue for nc3, either:
# https://github.com/Unidata/netcdf4-python/issues/273
expected['x'][-1] = np.string_('')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
def _cal_cesaro(self):
"""
Calculate Cesaro data
"""
def cesaro_integrate(y, x):
cesaro = integrate.cumtrapz(y, x, initial=0)
cesaro = integrate.cumtrapz(cesaro, x, initial=0)
return cesaro
self.buffer.nDCesaro = cesaro_integrate(self.buffer.nCorr,
self.buffer.timeLags)
# Unit: nCorr (L^2 T^-2), nDCesaro (L^2)
self.buffer.nDCesaro_unit = np.string_(
self.buffer.nCorr_unit.decode().split()[0])
self.buffer.nDTotalCesaro = np.sum(
self.buffer.nDCesaro *
(self.zz * self.ww)[:, np.newaxis], axis=0)
self.buffer.nDTotalCesaro_unit = np.string_(
self.buffer.nCorr_unit.decode().split()[0])
def do_dec(buf):
buf.decDCesaro = cesaro_integrate(
buf.decCorr, self.buffer.timeLags)
buf.decDCesaro_unit = np.string_(
buf.decCorr_unit.decode().split()[0])
for type_ in DecType:
buf = getattr(self.buffer, type_.value)
if buf is not None:
do_dec(buf)
def setupimgh5(f, Nframetotal:int, Nrow:int, Ncol:int, dtype=np.uint16,
writemode='r+', key='/rawimg',cmdlog:str=''):
"""
f: HDF5 handle (or filename)
h: HDF5 dataset handle
"""
if isinstance(f, (str,Path)): # assume new HDF5 file wanted
print('creating', f)
with h5py.File(f, writemode, libver='latest') as F:
return setupimgh5(F,Nframetotal,Nrow,Ncol,dtype,writemode,key)
elif isinstance(f, h5py.File):
h = f.create_dataset(key,
shape = (Nframetotal,Nrow,Ncol),
dtype=dtype,
chunks= (1,Nrow,Ncol), # each image is a chunk
compression='gzip',
compression_opts=1, #no difference in filesize from 1 to 5, except much faster to use lower numbers!
shuffle= True,
fletcher32= True,
track_times= True)
h.attrs["CLASS"] = np.string_("IMAGE")
h.attrs["IMAGE_VERSION"] = np.string_("1.2")
h.attrs["IMAGE_SUBCLASS"] = np.string_("IMAGE_GRAYSCALE")
h.attrs["DISPLAY_ORIGIN"] = np.string_("LL")
h.attrs['IMAGE_WHITE_IS_ZERO'] = np.uint8(0)
if cmdlog and isinstance(cmdlog,str):
f['/cmdlog'] = cmdlog
else:
raise TypeError(f'{type(f)} is not correct, must be filename or h5py.File HDF5 file handle')
return h
def _readCIF(cls, instream):
"""
- Check if the filename containing the CIF data exists
- read the cif file
- removes the comments
@param instream: opened file object containing the CIF data
@return: a set of bytes (8-bit string) containing the raw data
@rtype: string
"""
if "read" not in dir(instream):
raise RuntimeError("CIF._readCIF(instream): I expected instream to be an opened file,\
here I got %s type %s" % (instream, type(instream)))
out_bytes = numpy.string_("")
for sLine in instream:
nline = numpy.string_(sLine)
pos = nline.find(cls.HASH)
if pos >= 0:
if cls.isAscii(nline):
out_bytes += nline[:pos] + numpy.string_(os.linesep)
if pos > 80:
logger.warning("This line is too long and could cause problems in PreQuest: %s", sLine)
else:
out_bytes += nline
if len(sLine.strip()) > 80:
logger.warning("This line is too long and could cause problems in PreQuest: %s", sLine)
return out_bytes
def store_image(h5group, data, compression):
if len(data.shape) == 2:
# single event
data = data.reshape(1, data.shape[0], data.shape[1])
if "image" not in h5group:
maxshape = (None, data.shape[1], data.shape[2])
dset = h5group.create_dataset("image",
data=data,
dtype=np.uint8,
maxshape=maxshape,
chunks=True,
fletcher32=True,
compression=compression)
# Create and Set image attributes:
# HDFView recognizes this as a series of images.
# Use np.string_ as per
# http://docs.h5py.org/en/stable/strings.html#compatibility
dset.attrs.create('CLASS', np.string_('IMAGE'))
dset.attrs.create('IMAGE_VERSION', np.string_('1.2'))
dset.attrs.create('IMAGE_SUBCLASS', np.string_('IMAGE_GRAYSCALE'))
else:
dset = h5group["image"]
oldsize = dset.shape[0]
dset.resize(oldsize + data.shape[0], axis=0)
dset[oldsize:] = data
def new_entry(self, entry="entry", program_name="pyFAI",
title="description of experiment",
force_time=None, force_name=False):
"""
Create a new entry
:param entry: name of the entry
:param program_name: value of the field as string
:param title: value of the field as string
:param force_time: enforce the start_time (as string!)
:param force_name: force the entry name as such, without numerical suffix.
:return: the corresponding HDF5 group
"""
if not force_name:
nb_entries = len(self.get_entries())
entry = "%s_%04i" % (entry, nb_entries)
entry_grp = self.h5.require_group(entry)
entry_grp.attrs["NX_class"] = numpy.string_("NXentry")
entry_grp["title"] = numpy.string_(title)
entry_grp["program_name"] = numpy.string_(program_name)
if force_time:
entry_grp["start_time"] = numpy.string_(force_time)
else:
entry_grp["start_time"] = numpy.string_(get_isotime())
self.to_close.append(entry_grp)
return entry_grp
def _write_dataset(f, ds):
f[ds.name] = ds.data
dsobj = f[ds.name]
if ds.comment:
dsobj.attrs['COMMENT'] = np.string_(ds.comment)
if ds._display_name:
dsobj.attrs['DISPLAY_NAME'] = np.string_(ds.display_name)
if ds.quantity:
dsobj.attrs['QUANTITY'] = np.string_(ds.quantity)
if ds.unit:
dsobj.attrs['UNIT'] = np.string_(ds.unit)
if ds.display_unit != ds.unit:
dsobj.attrs['DISPLAY_UNIT'] = np.string_(ds.display_unit)
for ri in range(len(ds.scales)):
s = ds.scales[ri]
sobj = f[s.name]
dsobj.dims.create_scale(sobj, s.scale_name)
dsobj.dims[ri].attach_scale(sobj)
return dsobj
def get_roi_hdf5(hdf5FileName, hdf5FileName_ROI, run, rois, detector_names, pede_thr=-1, dark_file=""):
if rois == []:
for d in detector_names:
rois.append([])
if len(rois) != len(detector_names):
print "ERROR: please put one ROI per detector!"
sys.exit(-1)
f = h5py.File(hdf5FileName, 'r')
f_out = h5py.File(hdf5FileName_ROI, 'a', )
if dark_file != "":
f_dark = h5py.File(dark_file, "r")
run_dst = f["/run_%06d" % run]
detectors_list = []
detector_dstnames = [i for i in run_dst.keys() if i.find("detector_2d") != -1]
for d in detector_dstnames:
if run_dst[d + "/detector_info/detector_name"].value in detector_names:
detectors_list.append(d)
tag_list = f["/run_%06d/event_info/tag_number_list" % run][:]
DET_INFO_DSET = "/detector_info/detector_name"
RUN_INFO_DST = ["event_info", "exp_info", "run_info"]
file_info = f["file_info"]
f.copy(file_info, f_out)
try:
f_out.create_group("/run_%06d" % run)
except:
print sys.exc_info()[1]
for info_dst in RUN_INFO_DST:
info = run_dst[info_dst]
f.copy(info, f_out["/run_%06d" % run])
for i, dreal in enumerate(detectors_list):
detector_dsetname = "/run_%06d/%s" % (run, dreal)
print detector_dsetname
try:
fout_grp = f_out.create_group(detector_dsetname)
except:
print sys.exc_info()[1]
info = f[detector_dsetname]["detector_info"]
f.copy(info, f_out[detector_dsetname])
if dark_file != "":
print "With dark correction"
sacla_hdf5.get_roi_data(f[detector_dsetname], f_out[detector_dsetname], tag_list, rois[i], pede_thr=pede_thr, dark_matrix=f_dark[dark_dset_names[i]][:])
f_out[detector_dsetname].attrs['dark_filename'] = np.string_(dark_file.split("/")[-1])
print np.string_(dark_file.split("/")[-1])
else:
sacla_hdf5.get_roi_data(f[detector_dsetname], f_out[detector_dsetname], tag_list, rois[i], pede_thr=pede_thr)
#asciiList = [n.encode("ascii", "ignore") for n in strList]
#f_out[detector_dsetname + "/dark_fname"] = np.string_("aaaaaaaa")
f_out.close()
print "Run %s done!" % str(run)
def GetCheckPoint (self) :
"""
pickle the current iteration of the optimization
"""
return {"population" : np.string_(pickle.dumps(self.optimization_pop)),
"halloffame" : np.string_(pickle.dumps(self.optimization_hof)) }
##########################################################################
def init(self, lima_cfg):
"""
Initializes the HDF5 file for writing. Part of prepareAcq.
@param lima_cfg: dictionnary with parameters coming from Lima at the "prepareAcq"
"""
with self._sem:
self.filename = lima_cfg.get("directory")
if not self.filename.endswith('/'): self.filename += '/'
self.filename += lima_cfg.get("prefix")+'.h5'
# silence non serious error messages, which are printed
# because we use h5py in a new thread (not in the main one)
# this is a bug seems to be fixed on newer version !!
# see this h5py issue 206: https://code.google.com/p/h5py/issues/detail?id=206
h5py._errors.silence_errors()
try:
self.hdf5 = h5py.File(self.filename, 'a')
except IOError:
os.unlink(self.filename)
print ("here e e ")
self.hdf5 = h5py.File(self.filename)
prefix = lima_cfg.get("prefix") or self.CONFIG_ITEMS["hpath"]
if not prefix.endswith("_"):
prefix+="_"
entries = len([i.startswith(prefix) for i in self.hdf5])
self.hpath = posixpath.join("%s%04d"%(prefix,entries),self.lima_grp)
self.group = self.hdf5.require_group(self.hpath)
self.group.parent.attrs["NX_class"] = "NXentry"
self.group.attrs["NX_class"] = "NXdata"
cfg_grp = self.hdf5.require_group(posixpath.join(self.hpath, self.metadata_grp))
cfg_grp["detector_name"] = numpy.string_(self.detector_name)
for k, v in lima_cfg.items():
if type(v) in types.StringTypes:
cfg_grp[k] = numpy.string_(v)
else:
cfg_grp[k] = v
self.number_of_frames = (max(1, lima_cfg["number_of_frames"]) + self.min_size - 1) // self.min_size * self.min_size
self.min_size = max(1, self.min_size)
self.shape = (self.number_of_frames , lima_cfg.get("dimY", 1), lima_cfg.get("dimX", 1))
self.chunk = (self.min_size, lima_cfg.get("dimY", 1), lima_cfg.get("dimX", 1))
if "dtype" in lima_cfg:
self.dtype = numpy.dtype(lima_cfg["dtype"])
else:
self.dtype = numpy.int32
self.dataset = self.group.require_dataset(self.dataset_name, self.shape, dtype=self.dtype, chunks=self.chunk,
maxshape=(None,) + self.chunk[1:])
self.dataset.attrs["interpretation"] = "image"
self.dataset.attrs["metadata"] = self.metadata_grp
self.dataset.attrs["signal"] = "1"
self.group.parent["title"] = numpy.string_("Raw frames")
self.group.parent["program"] = numpy.string_("LImA HDF5 plugin")
self.group.parent["start_time"] = numpy.string_(self.getIsoTime())
def writeh5img(h,dat,path):
fimg = h.create_dataset(path,data=dat,
compression='gzip', compression_opts=5,
fletcher32=True,shuffle=True,track_times=True)
fimg.attrs["CLASS"] = string_("IMAGE")
fimg.attrs["IMAGE_VERSION"] = string_("1.2")
fimg.attrs["IMAGE_SUBCLASS"] = string_("IMAGE_GRAYSCALE")
fimg.attrs["DISPLAY_ORIGIN"] = string_("LL")
fimg.attrs['IMAGE_WHITE_IS_ZERO'] = uint8(0)
def test_export_averages_hdf5(self):
time_avg_path = os.path.join(self.filepath, 'time_avg.h5')
self.ds.export_averages(time_avg_path, fmt='HDF5', scale_values=False)
h5_time_avg = h5py.File(time_avg_path, 'r')['time_average']
assert_equal(self.ds.time_averages.astype('uint16'), h5_time_avg)
assert_equal(np.string_(self.ds.channel_names),
np.string_(h5_time_avg.attrs['channel_names']))
dim_labels = [dim.label for dim in h5_time_avg.dims]
assert_equal(['z', 'y', 'x', 'c'], dim_labels)
def finish(self):
self.flush()
self.h5file.root._v_attrs.km3pipe = np.string_(kp.__version__)
self.h5file.root._v_attrs.pytables = np.string_(tb.__version__)
self.h5file.root._v_attrs.jpp = np.string_(get_jpp_revision())
self.h5file.root._v_attrs.format_version = np.string_(FORMAT_VERSION)
self.log.info("Adding index tables.")
for where, idx_tab in self.indices.items():
self.log.debug("Creating index table for '%s'" % where)
h5loc = idx_tab.h5loc
self.log.info(" -> {0}".format(h5loc))
indices = Table({
"index": idx_tab.data["indices"],
"n_items": idx_tab.data["n_items"]
},
h5loc=h5loc)
self._write_table(h5loc, indices, title='Indices')
self.log.info(
"Creating pytables index tables. "
"This may take a few minutes..."
)
for tab in self._tables.values():
if 'frame_id' in tab.colnames:
tab.cols.frame_id.create_index()
if 'slice_id' in tab.colnames:
tab.cols.slice_id.create_index()
if 'dom_id' in tab.colnames:
tab.cols.dom_id.create_index()
if 'event_id' in tab.colnames:
try:
tab.cols.event_id.create_index()
except NotImplementedError:
log.warning(
"Table '{}' has an uint64 column, "
"not indexing...".format(tab._v_name)
)
if 'group_id' in tab.colnames:
try:
tab.cols.group_id.create_index()
except NotImplementedError:
log.warning(
"Table '{}' has an uint64 column, "
"not indexing...".format(tab._v_name)
)
tab.flush()
if "HDF5MetaData" in self.services:
self.log.info("Writing HDF5 meta data.")
metadata = self.services["HDF5MetaData"]
for name, value in metadata.items():
self.h5file.set_node_attr("/", name, value)
if not self.keep_open:
self.h5file.close()
self.print("HDF5 file written to: {}".format(self.filename))
def _write_data(lock, im, index, offset, abs_offset, imfilename, timestamp, projorder, tot_files,
provenance_dt, outfile, dsetname, outshape, outtype, logfilename, itime, n_images):
"""To do...
"""
lock.acquire()
try:
# Open the HDF5 file to be populated with projections (or sinograms):
t0 = time.time()
f_out = getHDF5(outfile, 'a')
f_out_dset = f_out.require_dataset(dsetname, outshape, outtype, chunks=tdf.get_dset_chunks(outshape[0]))
# Write the projection file or sinogram file:
if projorder:
tdf.write_tomo(f_out_dset, index - abs_offset, im)
else:
tdf.write_sino(f_out_dset, index - abs_offset, im)
# Set minimum and maximum (without Infs and NaNs):
tmp = im[:].astype(numpy.float32)
tmp = tmp[numpy.nonzero(numpy.isfinite(tmp))]
if (numpy.amin(tmp[:]) < float(f_out_dset.attrs['min'])):
f_out_dset.attrs['min'] = str(numpy.amin(tmp[:]))
if (numpy.amax(tmp[:]) > float(f_out_dset.attrs['max'])):
f_out_dset.attrs['max'] = str(numpy.amax(tmp[:]))
f_out_dset.attrs['avg'] = str(float(f_out_dset.attrs['avg']) + numpy.mean(tmp[:])/(1.0*n_images) )
# Save provenance metadata:
provenance_dset = f_out.require_dataset('provenance/detector_output', (tot_files,), dtype=provenance_dt)
provenance_dset["filename", offset - abs_offset + index] = numpy.string_(os.path.basename(imfilename))
provenance_dset["timestamp", offset - abs_offset + index] = numpy.string_(datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3])
# Close the HDF5:
f_out.close()
t1 = time.time()
# Print out execution time:
log = open(logfilename,"a")
log.write(os.linesep + "\t%s processed (I: %0.3f sec - O: %0.3f sec)." % (os.path.basename(imfilename), itime, t1 - t0))
log.close()
except:
io_warning = True
# Print out execution time:
log = open(logfilename,"a")
log.write(os.linesep + "\tError when writing to TDF %s. File skipped." % (os.path.basename(imfilename)))
log.close()
pass
finally:
lock.release()
def processFile(file):
global BINARY_IMAGE_SIZE, photoCopies, emissionsSamples, vin
BINARY_IMAGE_SIZE = 114173
photoCopies = 0
emissionsSamples = 0
vin = ""
linesIn = 0
for line in open(file): # read the CSV file
linesIn += 1
fields = line.split(",") # tokenize the input
if(fields[0] != '""' and linesIn != 1):
if(opaqueImage == "0"):
f = h5py.File(targetDir + fields[0] + ".hdf5a", "w") # open the output HDF5 File
else:
f = h5py.File(targetDir + fields[0] + ".hdf5b", "w") # open the output HDF5 File
grp = f.create_group(fields[0]) # create vehicle group (VIN id)
grp['manufacturer'] = str.strip(fields[1], '"') # populate the vehicle group ...
grp['modelYear'] = int(fields[2])
grp['vehicleType'] = fields[3]
grp['oilChangeDistance'] = float(fields[4])
grp['odometer'] = float(fields[5])
grp['comments'] = str.strip(fields[6], '"')
photoCopies = int(fields[13])
if(opaqueImage == "0"):
# write as image
for i in range(photoCopies):
image = Image.open(imageFile) # read the image file
grp['photo' + str(i)] = image # add the image member to the group
grp['photo' + str(i)].attrs['CLASS'] = np.string_("IMAGE")
grp['photo' + str(i)].attrs['IMAGE_VERSION'] = np.string_("1.2")
grp['photo' + str(i)].attrs['IMAGE_SUBCLASS'] = np.string_("IMAGE_TRUECOLOR")
grp['photo' + str(i)].attrs["IMAGE_COLORMODEL"] = np.string_("RGB")
grp['photo' + str(i)].attrs['INTERLACE_MODE'] = np.string_("INTERLACE_PIXEL")
else:
# write as opaque binary data
dtOpaque = np.dtype('V' + str(BINARY_IMAGE_SIZE))
for i in range(photoCopies):
with open(imageFile, "rb") as f2:
imageBytes = f2.read(BINARY_IMAGE_SIZE)
opd = f.create_dataset((fields[0] + '/photo' + str(i)), (BINARY_IMAGE_SIZE,), dtype=dtOpaque)
opd = imageBytes
vin = fields[0] # preallocate compound array
emissionsSamples = fields[12]
dt = ([("Date", np.dtype('a10')), ("Ex HC", np.dtype('f4')), ("Non-Ex HC", np.dtype('f4')), ("Ex CO", np.dtype('f4')), ("Ex NO2", np.dtype('f4'))])
ds = f.create_dataset(fields[0] + "/emissions", (int(emissionsSamples),), dtype=dt)
emissionPtr = 0
if(linesIn != 1): # skip the header, write the emissions record ...
ds[emissionPtr:emissionPtr+1] = [(fields[7], float(fields[8]), float(fields[9]), float(fields[10]), float(fields[11]))]
emissionPtr += 1
def makeSkeleton(name):
entry = f.create_group(name)
entry.attrs['NX_class'] = numpy.string_('NXentry')
dset = entry.create_dataset('definition',(1,),dtype="S70")
dset[0] = "NXdepends"
sample = entry.create_group("sample")
sample.attrs['NX_class'] = numpy.string_('NXsample')
rot = sample.create_dataset('rotation_angle',(1,),'f32')
rot[0] = 22.8
trans = sample.create_group('transform')
trans.attrs['NX_class'] = numpy.string_('NXtransformations')
dset = trans.create_dataset('depends_on',(1,),dtype="S70")
def save(filename, group):
with h5py.File(filename, 'w') as f:
if(group.comment != None):
f.attrs['COMMENT'] = np.string_(group.comment.encode('utf8'))
for key, value in group.attributes.items():
if type(value) is str:
value = np.string_(value.encode('utf8'))
f.attrs[key] = value
for ds in group.datasets:
_write_dataset(f, ds)
def main(argv):
# parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('-e', '--ens')
parser.add_argument('-d', '--dir')
parser.add_argument('-x', '--exclude', type=float)
args = parser.parse_args()
# switch directory if needed
if args.dir:
os.chdir(args.dir)
# load target ensemble
ens = wtc.loadEnsemble(args.ens)
results = Parallel(n_jobs=-1,
verbose=50)(delayed(calcCorrelationFunctionsFromFile)(traj.h5obj.filename,
traj.strain,
traj.wormID,
excludeEdge=args.exclude)
for traj in ens)
# save results
ensName = os.path.splitext(args.ens)[0]
f = h5py.File('{0}_corrfun.h5'.format(ensName), 'w')
f.create_dataset('taus', (len(lags),), dtype=float)
f['taus'][...] = lags/11.5
f.create_dataset('x_speed', (len(x),), dtype=float)
f['x_speed'][...] = x
f.create_dataset('wormID', (len(ens),), dtype='S10')
f.create_dataset('strain', (len(ens),), dtype='S10')
f.create_dataset('MSD', (len(ens), len(lags)), dtype=float)
f.create_dataset('VACF', (len(ens), len(lags)), dtype=float)
f.create_dataset('s_ACF', (len(ens), len(lags)), dtype=float)
f.create_dataset('psi_ACF', (len(ens), len(lags)), dtype=float)
f.create_dataset('dpsi_ACF', (len(ens), len(lags)), dtype=float)
f.create_dataset('psi_trend', (len(ens), len(lags)), dtype=float)
for i in xrange(len(ens)):
result = results[i]
traj = ens[i]
f['strain'][i] = np.string_(traj.strain)
f['wormID'][i] = np.string_(traj.wormID)
f['MSD'][i] = result[0].filled(np.nan)
f['VACF'][i] = result[1].filled(np.nan)
f['s_ACF'][i]= result[2].filled(np.nan)
f['psi_ACF'][i] = result[3].filled(np.nan)
f['dpsi_ACF'][i] = result[4].filled(np.nan)
f['psi_trend'][i] = result[5].filled(np.nan)
f['s_dist'][i]= result[6].filled(np.nan)
f.close()
def createHdf5v2(matFileDir,hdf5File):
hdf5F = h5py.File(hdf5File, 'w')
lstMatFiles = os.listdir(matFileDir)
for i in range(len(lstMatFiles)):
_f = h5py.File(matFileDir+lstMatFiles[i],'r')
arrAttrs = lstMatFiles[i].split('_')
_eeg = _f['eEEG']
dset = hdf5F.create_dataset('d'+str(100+i), data=_eeg, dtype=np.float16)
dset.attrs['studentno'] = np.string_(arrAttrs[3])
dset.attrs['convdiv'] = np.string_(arrAttrs[5])
dset.attrs['expunexp'] = np.string_(arrAttrs[6])
dset.attrs['isi'] = np.string_(arrAttrs[7].split('.')[0].split('-')[1])
return hdf5F
def testEndTimeInArray(self):
model = hdf5.Hdf5TreeModel()
h5 = _mock.File("/foo/bar/1.mock")
h5.create_group("a").create_dataset("end_time", numpy.array([numpy.string_("2015")]))
h5.create_group("b").create_dataset("end_time", numpy.array([numpy.string_("2013")]))
h5.create_group("c").create_dataset("end_time", numpy.array([numpy.string_("2014")]))
model.insertH5pyObject(h5)
proxy = hdf5.NexusSortFilterProxyModel()
proxy.setSourceModel(model)
proxy.sort(0, qt.Qt.DescendingOrder)
names = self.getChildNames(proxy, proxy.index(0, 0, qt.QModelIndex()))
self.assertListEqual(names, ["a", "c", "b"])
def write(self, filename, compression=True):
'''
Write out to a standard dust file, including calculations of the mean
opacities and optionally thermal emissivities.
'''
# Check that the optical properties have been set
self.optical_properties.ensure_all_set()
# Compute mean opacities if not already existent
self._compute_mean_opacities()
# Check that emissivities are set (before computing mean opacities)
if not self.emissivities.all_set():
logger.info("Computing emissivities assuming LTE")
self.emissivities.set_lte(self.optical_properties, self.mean_opacities)
# Create dust table set
if isinstance(filename, six.string_types):
dt = h5py.File(filename, 'w')
else:
dt = filename
# Add standard keywords to header
dt.attrs['version'] = 2
dt.attrs['type'] = 1
dt.attrs['python_version'] = np.string_(__version__)
if self.md5:
dt.attrs['asciimd5'] = np.string_(self.md5)
# Add optical properties and scattering angle tables
self.optical_properties.to_hdf5_group(dt)
# Add mean opacities table
self.mean_opacities.to_hdf5_group(dt)
# Add emissivities and emissivity variable tables
self.emissivities.to_hdf5_group(dt)
# Dust sublimation parameters
self._write_dust_sublimation(dt)
# Check that there are no NaN values in the file - if there are, a
# warning is emitted.
check_for_nans(dt)
# Close dust file
if isinstance(dt, h5py.File):
dt.close()
self._file = (filename, self.hash())
def testStartTime(self):
"""If it is not NXentry, start_time is not used"""
model = hdf5.Hdf5TreeModel()
h5 = commonh5.File("/foo/bar/1.mock", "w")
h5.create_group("a").create_dataset("start_time", data=numpy.string_("2015"))
h5.create_group("b").create_dataset("start_time", data=numpy.string_("2013"))
h5.create_group("c").create_dataset("start_time", data=numpy.string_("2014"))
model.insertH5pyObject(h5)
proxy = hdf5.NexusSortFilterProxyModel()
proxy.setSourceModel(model)
proxy.sort(0, qt.Qt.AscendingOrder)
names = self.getChildNames(proxy, proxy.index(0, 0, qt.QModelIndex()))
self.assertListEqual(names, ["a", "b", "c"])
def testNXentryEndTimeInArray(self):
"""Test NXentry with end_time"""
model = hdf5.Hdf5TreeModel()
h5 = commonh5.File("/foo/bar/1.mock", "w")
create_NXentry(h5, "a").create_dataset("end_time", data=numpy.array([numpy.string_("2015")]))
create_NXentry(h5, "b").create_dataset("end_time", data=numpy.array([numpy.string_("2013")]))
create_NXentry(h5, "c").create_dataset("end_time", data=numpy.array([numpy.string_("2014")]))
model.insertH5pyObject(h5)
proxy = hdf5.NexusSortFilterProxyModel()
proxy.setSourceModel(model)
proxy.sort(0, qt.Qt.DescendingOrder)
names = self.getChildNames(proxy, proxy.index(0, 0, qt.QModelIndex()))
self.assertListEqual(names, ["a", "c", "b"])
def init(self, lima_cfg=None):
"""
Initializes the HDF5 file for writing. Part of prepareAcq.
@param lima_cfg: dictionnary with parameters coming from Lima at the "prepareAcq"
"""
logger.debug("HDF5 writer init with %s" % lima_cfg)
with self._sem:
if h5py:
try:
self.hdf5 = h5py.File(self.filename)
except IOError:
logger.error("typically a corrupted HDF5 file ! : %s" % self.filename)
os.unlink(self.filename)
self.hdf5 = h5py.File(self.filename)
else:
err = "No h5py library, no chance"
logger.error(err)
raise RuntimeError(err)
prefix = lima_cfg.get("prefix") or self.CONFIG_ITEMS["hpath"]
if not prefix.endswith("_"):
prefix+="_"
entries = len([i.startswith(prefix) for i in self.hdf5])
self.hpath = posixpath.join("%s%04d"%(prefix,entries),self.lima_grp)
self.group = self.hdf5.require_group(self.hpath)
self.group.parent.attrs["NX_class"] = "NXentry"
self.group.attrs["NX_class"] = "NXdata"
cfg_grp = self.hdf5.require_group(posixpath.join(self.hpath, self.metadata_grp))
cfg_grp["detector_name"] = numpy.string_(self.detector_name)
for k, v in lima_cfg.items():
if type(v) in types.StringTypes:
cfg_grp[k] = numpy.string_(v)
else:
cfg_grp[k] = v
number_of_frames = (max(1, lima_cfg["number_of_frames"]) + self.min_size - 1) // self.min_size * self.min_size
self.min_size = max(1, self.min_size)
self.shape = (number_of_frames , lima_cfg.get("dimY", 1), lima_cfg.get("dimX", 1))
self.chunk = (self.min_size, lima_cfg.get("dimY", 1), lima_cfg.get("dimX", 1))
if "dtype" in lima_cfg:
self.dtype = numpy.dtype(lima_cfg["dtype"])
else:
self.dtype = numpy.int32
self.dataset = self.group.require_dataset(self.dataset_name, self.shape, dtype=self.dtype, chunks=self.chunk,
maxshape=(None,) + self.chunk[1:])
self.dataset.attrs["interpretation"] = "image"
self.dataset.attrs["metadata"] = self.metadata_grp
self.dataset.attrs["signal"] = "1"
self.group.parent["title"] = numpy.string_("Raw frames")
self.group.parent["program"] = numpy.string_("LImA HDF5 plugin")
self.group.parent["start_time"] = numpy.string_(getIsoTime())
def new_detector(self, name="detector", entry="entry", subentry="pyFAI"):
"""
Create a new entry/pyFAI/Detector
@param detector: name of the detector
@param entry: name of the entry
@param subentry: all pyFAI description of detectors should be in a pyFAI sub-entry
"""
entry_grp = self.new_entry(entry)
pyFAI_grp = self.new_class(entry_grp, subentry, "NXsubentry")
pyFAI_grp["definition_local"] = numpy.string_("pyFAI")
pyFAI_grp["definition_local"].attrs["version"] = numpy.string_(version)
det_grp = self.new_class(pyFAI_grp, name, "NXdetector")
return det_grp
def append_h5(file_h5, df, group_new, OUT_DIR, tag=None, overwrite=False, datasets='all'):
if type(tag) != type(None):
if tag[0] != '_':
tag = '_%s' % tag
else:
tag = ''
file_h5_base = os.path.basename(file_h5)
i = file_h5_base.index('ATL')
file_h5_base = file_h5_base[i+6:]
file_h5_new = os.path.join(OUT_DIR, file_h5_base)
file_h5_new = '.'.join(file_h5_new.split('.')[:-1]) + tag + '.h5'
if os.path.exists(file_h5) and not os.path.exists(file_h5_new):
# first time making the file
# make new file regardless of overwrite, since
# there's nothing to overwrite
# initialize
os.system('cp {} {}'.format(file_h5, file_h5_new))
elif not os.path.exists(file_h5) and not os.path.exists(file_h5_new):
# first time making new file, and original file doesn't exist
print('warning: cannot make h5 file %s as the original (%s) does not exist' % (file_h5_new, file_h5))
return 0
# can't do anything
elif os.path.exists(file_h5) and os.path.exists(file_h5_new):
# new file already made, and original file exists
if overwrite:
# overwrite new file
os.system('cp {} {}'.format(file_h5, file_h5_new))
elif not os.path.exists(file_h5) and os.path.exists(file_h5_new):
# new file already made, and original file is gone
if overwrite:
# cannot overwrite since original file is gone,
# can only remove _%dm groups, if anything
print('warning: cannot overwrite %s as original file (%s) does not exist' % (file_h5_new, file_h5))
filt_datasets = False
if datasets != 'all':
filt_datasets = True
datasets = list(datasets)
if not os.path.exists(file_h5_new):
print('error: %s not found' % file_h5_new)
return 0
repack = False
with h5.File(file_h5_new, 'a') as fp:
if group_new in fp:
del fp[group_new]
repack = True
fp_df = fp.create_group(group_new)
dset = {}
for key in df:
if filt_datasets:
if not (key in datasets):
continue
data = np.array(df[key])
dtype = data.dtype
if dtype == 'O':
data = np.array([np.string_(val) for val in data])
dtype = data.dtype
n = len(data)
dset[key] = fp_df.create_dataset(key, (n,), dtype=dtype, compression=1) # [1,9]
dset[key][:] = data
if repack:
file_h5_new_copy = ''.join(os.path.basename(file_h5_new).split('.')[:-1]) + '_copy.h5'
file_h5_new_copy = os.path.join(os.path.dirname(file_h5_new), file_h5_new_copy)
os.system('cp {} {}'.format(file_h5_new, file_h5_new_copy))
os.system('h5repack {} {}'.format(file_h5_new_copy, file_h5_new))
os.system('rm {}'.format(file_h5_new_copy))
def da2cf(dataarray,
epoch=EPOCH,
flatten_attrs=False,
exclude_attrs=None,
compression=None):
"""Convert the dataarray to something cf-compatible.
Args:
dataarray (xr.DataArray):
The data array to be converted
epoch (str):
Reference time for encoding of time coordinates
flatten_attrs (bool):
If True, flatten dict-type attributes
exclude_attrs (list):
List of dataset attributes to be excluded
"""
if exclude_attrs is None:
exclude_attrs = []
new_data = dataarray.copy()
# Remove area as well as user-defined attributes
for key in ['area'] + exclude_attrs:
new_data.attrs.pop(key, None)
anc = [
ds.attrs['name']
for ds in new_data.attrs.get('ancillary_variables', [])
]
if anc:
new_data.attrs['ancillary_variables'] = ' '.join(anc)
# TODO: make this a grid mapping or lon/lats
# new_data.attrs['area'] = str(new_data.attrs.get('area'))
for key, val in new_data.attrs.copy().items():
if val is None:
new_data.attrs.pop(key)
if key == 'ancillary_variables' and val == []:
new_data.attrs.pop(key)
new_data.attrs.pop('_last_resampler', None)
if compression is not None:
new_data.encoding.update(compression)
if 'time' in new_data.coords:
new_data['time'].encoding['units'] = epoch
new_data['time'].attrs['standard_name'] = 'time'
new_data['time'].attrs.pop('bounds', None)
if 'time' not in new_data.dims:
new_data = new_data.expand_dims('time')
if 'x' in new_data.coords:
new_data['x'].attrs['standard_name'] = 'projection_x_coordinate'
new_data['x'].attrs['units'] = 'm'
if 'y' in new_data.coords:
new_data['y'].attrs['standard_name'] = 'projection_y_coordinate'
new_data['y'].attrs['units'] = 'm'
if 'crs' in new_data.coords:
new_data = new_data.drop('crs')
new_data.attrs.setdefault('long_name', new_data.attrs.pop('name'))
if 'prerequisites' in new_data.attrs:
new_data.attrs['prerequisites'] = [
np.string_(str(prereq))
for prereq in new_data.attrs['prerequisites']
]
# Flatten dict-type attributes, if desired
if flatten_attrs:
new_data.attrs = flatten_dict(new_data.attrs)
# Encode attributes to netcdf-compatible datatype
new_data.attrs = encode_attrs_nc(new_data.attrs)
return new_data
def main():
parser = make_parser()
args = parser.parse_args()
if args.tf_seed != -1:
tf.random.set_random_seed(args.tf_seed)
if not args.no_shuffle and args.shuffle_seed != -1:
np.random.seed(args.shuffle_seed)
#Define params for model:
SEED = args.tf_seed
BATCH_SIZE = args.train_batch_size
CHANNEL_SIZE = args.input_dim[2]
NUM_EPOCHS = args.num_epochs
IMG_DIM = args.input_dim[0]
TRAIN_DIR = 'train/'
TEST_DIR = 'test/'
CLASSES = {
0: "No DR",
1: "Mild",
2: "Moderate",
3: "Severe",
4: "Proliferative DR"
}
df_train = pd.read_csv(os.path.join(args.data_dir, "train.csv"))
df_test = pd.read_csv(os.path.join(args.data_dir, "test.csv"))
print("Training set has {} samples".format(df_train.shape[0]))
print("Testing set has {} samples".format(df_test.shape[0]))
#Process image directories into exact file name (include .png):
def append_ext(fn):
return fn + ".png"
df_train["id_code"] = df_train["id_code"].apply(append_ext)
# load data into generator:
# For some reason the generator wants diagnostic labels in string form:
df_train['diagnosis'] = df_train['diagnosis'].astype(str)
_validation_split = 0.20
#x_train_shape = (int(np.round(df_train.shape[0] * (1 - _validation_split))), IMG_DIM, IMG_DIM, CHANNEL_SIZE)
#x_test_shape = (int(np.round(df_train.shape[0] * _validation_split)), IMG_DIM, IMG_DIM, CHANNEL_SIZE)
y_train_shape = (int(np.round(df_train.shape[0] *
(1 - _validation_split))), None)
y_test_shape = (int(np.round(df_train.shape[0] * _validation_split)), None)
train_datagen = ImageDataGenerator(rescale=1. / 255,
validation_split=_validation_split)
train_generator = train_datagen.flow_from_dataframe(
dataframe=df_train,
directory=args.data_dir + TRAIN_DIR,
x_col="id_code",
y_col="diagnosis",
batch_size=BATCH_SIZE,
class_mode="categorical",
target_size=(IMG_DIM, IMG_DIM),
subset='training',
seed=SEED)
val_generator = train_datagen.flow_from_dataframe(
dataframe=df_train,
directory=args.data_dir + TRAIN_DIR,
x_col="id_code",
y_col="diagnosis",
batch_size=BATCH_SIZE,
class_mode="categorical",
target_size=(IMG_DIM, IMG_DIM),
subset='validation',
seed=SEED)
# build model
if args.arch == 'basic':
model = network_builders.build_basic_model(args)
elif args.arch == 'fc':
model = network_builders.build_network_fc(args)
elif args.arch == 'fc_cust':
model = network_builders.build_fc_adjustable(args)
elif args.arch == 'lenet':
model = network_builders.build_lenet_conv(args)
elif args.arch == 'allcnn':
model = network_builders.build_all_cnn(args)
elif args.arch == 'resnet':
model = network_builders.build_resnet(args)
elif args.arch == 'vgg':
model = network_builders.build_vgg_half(args)
else:
raise Error("Unknown architeciture {}".format(args.arch))
init_model(model, args)
define_training(model, args)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
for collection in ['tb_train_step'
]: # 'eval_train' and 'eval_test' added manually later
tf.summary.scalar(collection + '_acc',
model.accuracy,
collections=[collection])
tf.summary.scalar(collection + '_loss',
model.loss,
collections=[collection])
tb_writer, hf = None, None
dsets = {}
if args.output_dir:
tb_writer = tf.summary.FileWriter(args.output_dir, sess.graph)
# set up output for gradients/weights
if args.save_weights:
dim_sum = sum([
tf.size(var).eval()
for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
])
total_iters = args.num_epochs * int(
y_train_shape[0] / args.train_batch_size)
total_chunks = int(total_iters / args.save_every)
hf = h5py.File(args.output_dir + '/weights', 'w-')
# write metadata
var_shapes = np.string_(';'.join([
str(var.get_shape())
for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
]))
hf.attrs['var_shapes'] = var_shapes
var_names = np.string_(';'.join([
str(var.name)
for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
]))
hf.attrs['var_names'] = var_names
# all individual weights at every iteration, where all_weights[i] = weights before iteration i:
dsets['all_weights'] = hf.create_dataset(
'all_weights', (total_chunks + 1, dim_sum),
dtype='f8',
compression='gzip')
if args.save_training_grads:
dsets['training_grads'] = hf.create_dataset(
'training_grads', (total_chunks, dim_sum),
dtype='f8',
compression='gzip')
train_and_eval(sess, model, y_train_shape, train_generator, y_test_shape,
val_generator, tb_writer, dsets, args)
if tb_writer:
tb_writer.close()
if hf:
hf.close()
def write_h5(self, hs):
hs.attrs['segfile'] = np.string_(self.segfile)
hs.attrs['imgfile'] = np.string_(self.obscat.detfile)
for source in self:
source.write_h5(hs)
def sgfs_to_hdf5(self,
sgf_files,
hdf5_file,
bd_size=19,
ignore_errors=True,
verbose=False):
"""Convert all files in the iterable sgf_files into an hdf5 group to be stored in hdf5_file
Arguments:
- sgf_files : an iterable of relative or absolute paths to SGF files
- hdf5_file : the name of the HDF5 where features will be saved
- bd_size : side length of board of games that are loaded
- ignore_errors : if True, issues a Warning when there is an unknown
exception rather than halting. Note that sgf.ParseException and
go.IllegalMove exceptions are always skipped
The resulting file has the following properties:
states : dataset with shape (n_data, n_features, board width, board height)
actions : dataset with shape (n_data, 2) (actions are stored as x,y tuples of
where the move was played)
file_offsets : group mapping from filenames to tuples of (index, length)
For example, to find what positions in the dataset come from 'test.sgf':
index, length = file_offsets['test.sgf']
test_states = states[index:index+length]
test_actions = actions[index:index+length]
"""
# make a hidden temporary file in case of a crash.
# on success, this is renamed to hdf5_file
tmp_file = os.path.join(os.path.dirname(hdf5_file),
".tmp." + os.path.basename(hdf5_file))
h5f = h5.File(tmp_file, 'w')
try:
# see http://docs.h5py.org/en/latest/high/group.html#Group.create_dataset
states = h5f.require_dataset(
'states',
dtype=np.uint8,
shape=(1, self.n_features, bd_size, bd_size),
maxshape=(None, self.n_features, bd_size,
bd_size), # 'None' == arbitrary size
exact=
False, # allow non-uint8 datasets to be loaded, coerced to uint8
chunks=(64, self.n_features, bd_size,
bd_size), # approximately 1MB chunks
compression="lzf")
actions = h5f.require_dataset('actions',
dtype=np.uint8,
shape=(1, 2),
maxshape=(None, 2),
exact=False,
chunks=(1024, 2),
compression="lzf")
# 'file_offsets' is an HDF5 group so that 'file_name in file_offsets' is fast
file_offsets = h5f.require_group('file_offsets')
# Store comma-separated list of feature planes in the scalar field 'features'. The
# string can be retrieved using h5py's scalar indexing: h5f['features'][()]
h5f['features'] = np.string_(','.join(
self.feature_processor.feature_list))
if verbose:
print("created HDF5 dataset in {}".format(tmp_file))
next_idx = 0
for file_name in sgf_files:
if verbose:
print(file_name)
# count number of state/action pairs yielded by this game
n_pairs = 0
file_start_idx = next_idx
try:
for state, move in self.convert_game(file_name, bd_size):
if next_idx >= len(states):
states.resize((next_idx + 1, self.n_features,
bd_size, bd_size))
actions.resize((next_idx + 1, 2))
states[next_idx] = state
actions[next_idx] = move
n_pairs += 1
next_idx += 1
except go.IllegalMove:
warnings.warn("Illegal Move encountered in %s\n"
"\tdropping the remainder of the game" %
file_name)
except sgf.ParseException:
warnings.warn("Could not parse %s\n\tdropping game" %
file_name)
except SizeMismatchError:
warnings.warn("Skipping %s; wrong board size" % file_name)
except Exception as e:
# catch everything else
if ignore_errors:
warnings.warn("Unkown exception with file %s\n\t%s" %
(file_name, e),
stacklevel=2)
else:
raise e
finally:
if n_pairs > 0:
# '/' has special meaning in HDF5 key names, so they
# are replaced with ':' here
file_name_key = file_name.replace('/', ':')
file_offsets[file_name_key] = [file_start_idx, n_pairs]
if verbose:
print("\t%d state/action pairs extracted" %
n_pairs)
elif verbose:
print("\t-no usable data-")
except Exception as e:
print("sgfs_to_hdf5 failed")
os.remove(tmp_file)
raise e
if verbose:
print("finished. renaming %s to %s" % (tmp_file, hdf5_file))
# processing complete; rename tmp_file to hdf5_file
h5f.close()
os.rename(tmp_file, hdf5_file)
names = []
model = VGGNet()
for i, img_path in enumerate(img_list):
norm_feat = model.extract_feat(img_path)
img_name = os.path.split(img_path)[1]
feats.append(norm_feat)
names.append(img_name)
print("extracting feature from image No. %d , %d images in total" %
((i + 1), len(img_list)))
feats = np.array(feats)
# print(feats)
# directory for storing extracted features
# output = args["index"]
output = "featureCNN.h5"
# print(feats)
# print(feats.shape)
print("--------------------------------------------------")
# print(names)
print("--------------------------------------------------")
print(" writing feature extraction results ... ")
print("--------------------------------------------------")
h5f = h5py.File(output, 'w')
h5f.create_dataset('dataset_1', data=feats)
# h5f.create_dataset('dataset_2', data = names)
h5f.create_dataset('dataset_2', data=np.string_(names))
h5f.close()
def index_to_h5(index, key, grp):
if index.dtype == 'object':
grp.create_dataset(key,
data=np.array([np.string_(str(x)) for x in index]))
else:
grp.create_dataset(key, data=np.array(index))
def generate(self, regen=False):
"""Calculate the total Fisher matrix and bias and save to a file.
Parameters
----------
regen : boolean, optional
Force regeneration if products already exist (default `False`).
"""
if mpiutil.rank0:
st = time.time()
print("======== Starting PS calculation ========")
ffile = self.psdir + "/fisher.hdf5"
if os.path.exists(ffile) and not regen:
print("Fisher matrix file: %s exists. Skipping..." % ffile)
return
mpiutil.barrier()
# Pre-compute all the angular power spectra for the bands
self.genbands()
# Calculate Fisher and bias for each m
# Pair up each list item with its position.
zlist = list(enumerate(range(self.telescope.mmax + 1)))
# Partition list based on MPI rank
llist = mpiutil.partition_list_mpi(zlist)
# Operate on sublist
fisher_bias_list = [self.fisher_bias_m(item) for ind, item in llist]
# Unpack into separate lists of the Fisher matrix and bias
fisher_loc, bias_loc = zip(*fisher_bias_list)
# Sum over all local m-modes to get the over all Fisher and bias pe process
fisher_loc = np.sum(np.array(fisher_loc),
axis=0).real # Be careful of the .real here
bias_loc = np.sum(np.array(bias_loc),
axis=0).real # Be careful of the .real here
self.fisher = mpiutil.allreduce(fisher_loc, op=MPI.SUM)
self.bias = mpiutil.allreduce(bias_loc, op=MPI.SUM)
# Write out all the PS estimation products
if mpiutil.rank0:
et = time.time()
print("======== Ending PS calculation (time=%f) ========" %
(et - st))
# Check to see ensure that Fisher matrix isn't all zeros.
if not (self.fisher == 0).all():
# Generate derived quantities (covariance, errors..)
cv = la.pinv(self.fisher, rcond=1e-8)
err = cv.diagonal()**0.5
cr = cv / np.outer(err, err)
else:
cv = np.zeros_like(self.fisher)
err = cv.diagonal()
cr = np.zeros_like(self.fisher)
f = h5py.File(self.psdir + "/fisher.hdf5", "w")
f.attrs["bandtype"] = np.string_(
self.bandtype) # HDF5 string issues
f.create_dataset("fisher/", data=self.fisher)
f.create_dataset("bias/", data=self.bias)
f.create_dataset("covariance/", data=cv)
f.create_dataset("errors/", data=err)
f.create_dataset("correlation/", data=cr)
f.create_dataset("band_power/", data=self.band_power)
if self.bandtype == "polar":
f.create_dataset("k_start/", data=self.k_start)
f.create_dataset("k_end/", data=self.k_end)
f.create_dataset("k_center/", data=self.k_center)
f.create_dataset("theta_start/", data=self.theta_start)
f.create_dataset("theta_end/", data=self.theta_end)
f.create_dataset("theta_center/", data=self.theta_center)
f.create_dataset("k_bands", data=self.k_bands)
f.create_dataset("theta_bands", data=self.theta_bands)
elif self.bandtype == "cartesian":
f.create_dataset("kpar_start/", data=self.kpar_start)
f.create_dataset("kpar_end/", data=self.kpar_end)
f.create_dataset("kpar_center/", data=self.kpar_center)
f.create_dataset("kperp_start/", data=self.kperp_start)
f.create_dataset("kperp_end/", data=self.kperp_end)
f.create_dataset("kperp_center/", data=self.kperp_center)
f.create_dataset("kpar_bands", data=self.kpar_bands)
f.create_dataset("kperp_bands", data=self.kperp_bands)
f.close()
def write(self, filename=None, compression=True, copy=True,
absolute_paths=False, wall_dtype=float,
physics_dtype=float, overwrite=True):
'''
Write the model input parameters to an HDF5 file
Parameters
----------
filename : str
The name of the input file to write. If no name is specified, the
filename is constructed from the model name.
compression : bool
Whether to compress the datasets inside the HDF5 file.
copy : bool
Whether to copy all external content into the input file, or
whether to just link to external content.
absolute_paths : bool
If copy=False, then if absolute_paths is True, absolute filenames
are used in the link, otherwise the path relative to the input
file is used.
wall_dtype : type
Numerical type to use for wall positions.
physics_dtype : type
Numerical type to use for physical grids.
overwrite : bool
Whether to overwrite any pre-existing file
'''
# If no filename has been specified, use the model name to construct
# one. If neither have been specified, raise an exception.
if filename is None:
if self.name is not None:
filename = self.name + '.rtin'
else:
raise ValueError("filename= has not been specified and model "
"has no name")
# Remove previous file if it exists
if overwrite and os.path.exists(filename):
os.remove(filename)
# Check that grid has been set up
if self.grid is None:
raise Exception("No coordinate grid has been set up")
# Check that containing directory exists to give a more understandable
# message than 'File does not exist', since this might confuse users
# (it is the output directory that does not exist)
if not os.path.dirname(filename) == "":
if not os.path.exists(os.path.dirname(filename)):
raise IOError("Directory %s does not exist" %
os.path.dirname(filename))
# Create output file
delete_file(filename)
root = h5py.File(filename, 'w')
# Add Python version
root.attrs['python_version'] = np.string_(__version__.encode('utf-8'))
# Create all the necessary groups and sub-groups
g_sources = root.create_group('Sources')
g_output = root.create_group('Output')
g_peeled = g_output.create_group('Peeled')
g_binned = g_output.create_group('Binned')
# Output sources
for i, source in enumerate(self.sources):
if isinstance(source, MapSource):
source.write(g_sources, 'source_%05i' % (i + 1), self.grid,
compression=compression, map_dtype=physics_dtype)
else:
source.write(g_sources, 'source_%05i' % (i + 1))
# Output configuration for peeled images/SEDs
for i, peel in enumerate(self.peeled_output):
if not self._frequencies is None:
if not peel._monochromatic:
raise Exception("Peeled images need to be set to monochromatic mode")
peel.write(g_peeled.create_group('group_%05i' % (i + 1)))
# Output configuration for binned images/SEDs
if self.binned_output is not None:
if self.forced_first_interaction:
raise Exception("can't use binned images with forced first interaction - use set_forced_first_interaction(False) to disable")
self.binned_output.write(g_binned.create_group('group_00001'))
# Write monochromatic configuration
self._write_monochromatic(root, compression=compression)
# Write run-time and output configuration
self.write_run_conf(root)
self.conf.output.write(g_output)
if isinstance(self.grid, GridOnDisk):
g_grid = link_or_copy(root, 'Grid', self.grid.link, copy=copy, absolute_paths=absolute_paths)
else:
# Create group
g_grid = root.create_group('Grid')
# Check self-consistency of grid
self.grid._check_array_dimensions()
# Write the geometry and physical quantity arrays to the input file
self.grid.write(g_grid, copy=copy, absolute_paths=absolute_paths, compression=compression, physics_dtype=physics_dtype)
if 'density' in self.grid:
# Check if dust types are specified for each
if self.dust is None:
raise Exception("No dust properties specified")
if isinstance(self.dust, h5py.ExternalLink):
link_or_copy(root, 'Dust', self.dust, copy, absolute_paths=absolute_paths)
elif isinstance(self.dust, h5py.Group):
root.copy(self.dust, 'Dust')
elif type(self.dust) == list:
g_dust = root.create_group('Dust')
if self.grid['density'].n_dust != len(self.dust):
raise Exception("Number of density grids should match number of dust types")
# Output dust file, avoiding writing the same dust file multiple times
present = {}
for i, dust in enumerate(self.dust):
short_name = 'dust_%03i' % (i + 1)
if copy:
if isinstance(dust, six.string_types):
dust = SphericalDust(dust)
if dust.hash() in present:
g_dust[short_name] = h5py.SoftLink(present[dust.hash()])
else:
dust.write(g_dust.create_group(short_name))
present[dust.hash()] = short_name
else:
if type(dust) != str:
if dust._file is None:
raise ValueError("Dust properties are not located in a file, so cannot link. Use copy=True or write the dust properties to a file first")
else:
# Check that has still matches file
if dust.hash() != dust._file[1]:
raise ValueError("Dust properties have been modified since "
"being read in, so cannot link to dust file "
"on disk. You can solve this by writing out "
"the dust properties to a new file, or by "
"using copy=True.")
dust = dust._file[0]
if absolute_paths:
path = os.path.abspath(dust)
else:
# Relative path should be relative to input file, not current directory.
path = os.path.relpath(dust, os.path.dirname(filename))
g_dust[short_name] = h5py.ExternalLink(path, '/')
else:
raise ValueError("Unknown type for dust attribute: %s" % type(self.dust))
_n_dust = len(root['Dust'])
# Write minimum specific energy
if self._minimum_temperature is not None:
if np.isscalar(self._minimum_temperature):
_minimum_temperature = [self._minimum_temperature for i in range(_n_dust)]
elif len(self._minimum_temperature) != _n_dust:
raise Exception("Number of minimum_temperature values should match number of dust types")
else:
_minimum_temperature = self._minimum_temperature
_minimum_specific_energy = []
for i, dust in enumerate(root['Dust']):
d = SphericalDust(root['Dust'][dust])
_minimum_specific_energy.append(d.temperature2specific_energy(_minimum_temperature[i]))
elif self._minimum_specific_energy is not None:
if np.isscalar(self._minimum_specific_energy):
_minimum_specific_energy = [self._minimum_specific_energy for i in range(_n_dust)]
elif len(self._minimum_specific_energy) != _n_dust:
raise Exception("Number of minimum_specific_energy values should match number of dust types")
else:
_minimum_specific_energy = self._minimum_specific_energy
else:
_minimum_specific_energy = None
if isinstance(self.grid, GridOnDisk):
if _minimum_specific_energy is not None:
raise ValueError("Cannot set minimum specific energy or temperature when using grid from disk")
elif _minimum_specific_energy is not None:
g_grid['Quantities'].attrs["minimum_specific_energy"] = [float(x) for x in _minimum_specific_energy]
else:
root.create_group('Dust')
# Check that there are no NaN values in the file - if there are, a
# warning is emitted.
check_for_nans(root)
root.close()
self.filename = filename
def _parse_csv_line(headers, values, req_site_params):
"""
Parse a single line from data file.
:param headers:
A list of header names, the strings from the first line of csv file.
:param values:
A list of values of a single row to parse.
:returns:
A tuple of the following values (in specified order):
sctx
An instance of :class:`openquake.hazardlib.gsim.base.SitesContext`
with attributes populated by the information from in row in a form
of single-element numpy arrays.
rctx
An instance of
:class:`openquake.hazardlib.gsim.base.RuptureContext`.
dctx
An instance of
:class:`openquake.hazardlib.gsim.base.DistancesContext`.
stddev_types
An empty list, if the ``result_type`` column says "MEAN"
for that row, otherwise it is a list with one item --
a requested standard deviation type.
expected_results
A dictionary mapping IMT-objects to one-element arrays of expected
result values. Those results represent either standard deviation
or mean value of corresponding IMT depending on ``result_type``.
result_type
A string literal, one of ``'STDDEV'`` or ``'MEAN'``. Value
is taken from column ``result_type``.
"""
rctx = RuptureContext()
sctx = SitesContext(slots=req_site_params)
dctx = DistancesContext()
expected_results = {}
stddev_types = result_type = damping = None
for param, value in zip(headers, values):
if param == 'result_type':
value = value.upper()
if value.endswith('_STDDEV'):
# the row defines expected stddev results
result_type = 'STDDEV'
stddev_types = [getattr(const.StdDev, value[:-len('_STDDEV')])]
else:
# the row defines expected exponents of mean values
assert value == 'MEAN'
stddev_types = []
result_type = 'MEAN'
elif param == 'damping':
damping = float(value)
elif param.startswith('site_'):
# value is sites context object attribute
if param == 'site_vs30measured' or param == 'site_backarc':
value = float(value) != 0
elif param in ('site_siteclass', 'site_ec8', 'site_ec8_p18',
'site_geology'):
value = numpy.string_(value)
else:
value = float(value)
# site_lons, site_lats, site_depths -> lon, lat, depth
if param.endswith(('lons', 'lats', 'depths')):
attr = param[len('site_'):-1]
else: # vs30s etc
attr = param[len('site_'):]
setattr(sctx, attr, numpy.array([value]))
elif param.startswith('dist_'):
# value is a distance measure
value = float(value)
setattr(dctx, param[len('dist_'):], numpy.array([value]))
elif param.startswith('rup_'):
# value is a rupture context attribute
try:
value = float(value)
except ValueError:
if value != 'undefined':
raise
setattr(rctx, param[len('rup_'):], value)
elif param == 'component_type':
pass
else:
# value is the expected result (of result_type type)
value = float(value)
if param == 'arias': # ugly legacy corner case
param = 'ia'
try: # The title of the column should be IMT(args)
imt = from_string(param.upper())
except KeyError: # Then it is just a period for SA
imt = registry['SA'](float(param), damping)
expected_results[imt] = numpy.array([value])
assert result_type is not None
return sctx, rctx, dctx, stddev_types, expected_results, result_type
def export_to_hdf5(self, path, mode='a', libver='earliest'):
"""Export incident photon data to an HDF5 file.
Parameters
----------
path : str
Path to write HDF5 file to
mode : {'r', r+', 'w', 'x', 'a'}
Mode that is used to open the HDF5 file. This is the second argument
to the :class:`h5py.File` constructor.
libver : {'earliest', 'latest'}
Compatibility mode for the HDF5 file. 'latest' will produce files
that are less backwards compatible but have performance benefits.
"""
# Open file and write version
f = h5py.File(str(path), mode, libver=libver)
f.attrs['filetype'] = np.string_('data_photon')
if 'version' not in f.attrs:
f.attrs['version'] = np.array(HDF5_VERSION)
group = f.create_group(self.name)
group.attrs['Z'] = Z = self.atomic_number
# Determine union energy grid
union_grid = np.array([])
for rx in self:
union_grid = np.union1d(union_grid, rx.xs.x)
group.create_dataset('energy', data=union_grid)
# Write cross sections
shell_group = group.create_group('subshells')
designators = []
for mt, rx in self.reactions.items():
name, key = _REACTION_NAME[mt]
if mt in (502, 504, 515, 517, 522, 525):
sub_group = group.create_group(key)
elif mt >= 534 and mt <= 572:
# Subshell
designators.append(key)
sub_group = shell_group.create_group(key)
# Write atomic relaxation
if key in self.atomic_relaxation.subshells:
self.atomic_relaxation.to_hdf5(sub_group, key)
else:
continue
rx.to_hdf5(sub_group, union_grid, Z)
shell_group.attrs['designators'] = np.array(designators, dtype='S')
# Write Compton profiles
if self.compton_profiles:
compton_group = group.create_group('compton_profiles')
profile = self.compton_profiles
compton_group.create_dataset('num_electrons',
data=profile['num_electrons'])
compton_group.create_dataset('binding_energy',
data=profile['binding_energy'])
# Get electron momentum values
compton_group.create_dataset('pz', data=profile['J'][0].x)
# Create/write 2D array of profiles
J = np.array([Jk.y for Jk in profile['J']])
compton_group.create_dataset('J', data=J)
# Write bremsstrahlung
if self.bremsstrahlung:
brem_group = group.create_group('bremsstrahlung')
for key, value in self.bremsstrahlung.items():
if key == 'I':
brem_group.attrs[key] = value
else:
brem_group.create_dataset(key, data=value)
def log_and_save_initial(self):
"""
Perform the initial log and save.
"""
if self.should_save:
# Notify the user where the file is being saved.
print("QOC is saving this optimization run to {}."
"".format(self.save_file_path))
save_count, save_count_remainder = np.divmod(
self.iteration_count, self.save_iteration_step)
density_count = len(self.initial_densities)
# If the final iteration doesn't fall on a save step, add a save step.
if save_count_remainder != 0:
save_count += 1
try:
with self.save_file_lock:
with h5py.File(self.save_file_path, "w") as save_file:
save_file["complex_controls"] = self.complex_controls
save_file["control_count"] = self.control_count
save_file[
"control_eval_count"] = self.control_eval_count
save_file["controls"] = np.zeros(
(
save_count,
self.control_eval_count,
self.control_count,
),
dtype=self.initial_controls.dtype)
save_file["cost_eval_step"] = self.cost_eval_step
save_file["cost_names"] = np.array([
np.string_("{}".format(cost))
for cost in self.costs
])
save_file["error"] = np.repeat(
np.finfo(np.float64).max, save_count)
save_file["evolution_time"] = self.evolution_time
save_file["final_densities"] = np.zeros(
(save_count, density_count, self.hilbert_size,
self.hilbert_size),
dtype=np.complex128)
save_file["grads"] = np.zeros(
(save_count, self.control_eval_count,
self.control_count),
dtype=self.initial_controls.dtype)
save_file["initial_controls"] = self.initial_controls
save_file["initial_densities"] = self.initial_densities
if self.save_intermediate_densities_:
save_file["intermediate_densities"] = np.zeros(
(save_count, self.system_eval_count,
*self.initial_densities.shape),
dtype=np.complex128)
save_file["interpolation_policy"] = "{}".format(
self.interpolation_policy)
save_file["iteration_count"] = self.iteration_count
save_file["max_control_norms"] = self.max_control_norms
save_file["method"] = self.method
save_file["optimizer"] = "{}".format(self.optimizer)
save_file["program_type"] = self.program_type.value
save_file["system_eval_count"] = self.system_eval_count
#ENDWITH
#ENDWITH
except Timeout:
print("Could not perform initial save.")
#ENDIF
if self.should_log:
print("iter | total error | grads_l2 \n"
"=========================================")
def init(self, fai_cfg=None, lima_cfg=None):
"""
Initializes the HDF5 file for writing
:param fai_cfg: the configuration of the worker as a dictionary
"""
logger.debug("in init")
Writer.init(self, fai_cfg, lima_cfg)
with self._sem:
if logger.isEnabledFor(logging.DEBUG):
# TODO: this is Debug statement
open("fai_cfg.debug.json",
"w").write(json.dumps(self.fai_cfg, indent=4))
open("lima_cfg.debug.json",
"w").write(json.dumps(self.lima_cfg, indent=4))
self.fai_cfg["nbpt_rad"] = self.fai_cfg.get("nbpt_rad", 1000)
if h5py:
try:
self.hdf5 = h5py.File(self.filename)
except IOError: # typically a corrupted HDF5 file !
os.unlink(self.filename)
self.hdf5 = h5py.File(self.filename)
else:
logger.error("No h5py library, no chance")
raise RuntimeError("No h5py library, no chance")
self.group = self.hdf5.require_group(self.hpath)
self.group.attrs["NX_class"] = numpy.string_("NXentry")
self.pyFAI_grp = self.hdf5.require_group(
posixpath.join(self.hpath, self.CONFIG))
self.pyFAI_grp.attrs["desc"] = numpy.string_(
"PyFAI worker configuration")
for key, value in self.fai_cfg.items():
if value is None:
continue
try:
self.pyFAI_grp[key] = value
except Exception as e:
logger.error("Unable to set %s: %s", key, value)
logger.debug("Backtrace", exc_info=True)
raise RuntimeError(e.args[0])
rad_name, rad_unit = str(self.fai_cfg.get("unit",
"2th_deg")).split(
"_", 1)
self.radial_values = self.group.require_dataset(
rad_name, (self.fai_cfg["nbpt_rad"], ), numpy.float32)
if self.fai_cfg.get("nbpt_azim", 0) > 1:
self.azimuthal_values = self.group.require_dataset(
"chi", (self.fai_cfg["nbpt_azim"], ), numpy.float32)
self.azimuthal_values.attrs["unit"] = numpy.string_("deg")
self.azimuthal_values.attrs["interpretation"] = numpy.string_(
"scalar")
self.azimuthal_values.attrs["long name"] = numpy.string_(
"Azimuthal angle")
self.radial_values.attrs["unit"] = numpy.string_(rad_unit)
self.radial_values.attrs["interpretation"] = numpy.string_(
"scalar")
self.radial_values.attrs["long name"] = numpy.string_(
"diffraction radial direction")
if self.fast_scan_width:
self.fast_motor = self.group.require_dataset(
"fast", (self.fast_scan_width, ), numpy.float32)
self.fast_motor.attrs["long name"] = numpy.string_(
"Fast motor position")
self.fast_motor.attrs["interpretation"] = numpy.string_(
"scalar")
self.fast_motor.attrs["axis"] = numpy.string_("1")
self.radial_values.attrs["axis"] = numpy.string_("2")
if self.azimuthal_values is not None:
chunk = 1, self.fast_scan_width, self.fai_cfg[
"nbpt_azim"], self.fai_cfg["nbpt_rad"]
self.ndim = 4
self.azimuthal_values.attrs["axis"] = numpy.string_("3")
else:
chunk = 1, self.fast_scan_width, self.fai_cfg["nbpt_rad"]
self.ndim = 3
else:
self.radial_values.attrs["axis"] = numpy.string_("1")
if self.azimuthal_values is not None:
chunk = 1, self.fai_cfg["nbpt_azim"], self.fai_cfg[
"nbpt_rad"]
self.ndim = 3
self.azimuthal_values.attrs["axis"] = numpy.string_("2")
else:
chunk = 1, self.fai_cfg["nbpt_rad"]
self.ndim = 2
if self.DATASET_NAME in self.group:
del self.group[self.DATASET_NAME]
shape = list(chunk)
if self.lima_cfg.get("number_of_frames", 0) > 0:
if self.fast_scan_width is not None:
shape[0] = 1 + self.lima_cfg[
"number_of_frames"] // self.fast_scan_width
else:
shape[0] = self.lima_cfg["number_of_frames"]
dtype = self.lima_cfg.get("dtype") or self.fai_cfg.get("dtype")
if dtype is None:
dtype = numpy.float32
else:
dtype = numpy.dtype(dtype)
self.dataset = self.group.require_dataset(self.DATASET_NAME,
shape,
dtype=dtype,
chunks=chunk,
maxshape=(None, ) +
chunk[1:])
if self.fai_cfg.get("nbpt_azim", 0) > 1:
self.dataset.attrs["interpretation"] = numpy.string_("image")
else:
self.dataset.attrs["interpretation"] = numpy.string_(
"spectrum")
self.dataset.attrs["signal"] = numpy.string_("1")
self.chunk = chunk
self.shape = chunk
name = "Mapping " if self.fast_scan_width else "Scanning "
name += "2D" if self.fai_cfg.get("nbpt_azim", 0) > 1 else "1D"
name += " experiment"
self.group["title"] = numpy.string_(name)
self.group["program"] = numpy.string_("PyFAI")
self.group["start_time"] = numpy.string_(get_isotime())
def do_write(self, sim: 'Simulation', h5file: File) -> None:
gg = h5file.create_group('SpatialMesh')
sim.mesh.export_h5(gg)
for i, c in enumerate('xyz'):
gg[f'electric_field_{c}'] = sim.electric_field.array.data[
..., i].flatten()
gg['charge_density'] = sim.charge_density.data.flatten()
gg['potential'] = sim.potential.data.flatten()
sim.time_grid.export_h5(h5file.create_group('TimeGrid'))
g = h5file.create_group('ParticleSources')
g.attrs['number_of_sources'] = [len(sim.particle_sources)]
for s in sim.particle_sources:
s.export_h5(g.create_group(s.name))
for p in sim.particle_arrays:
s = next(s for s in sim.particle_sources
if s.charge == p.charge and s.mass == p.mass)
p.export_h5(g[s.name])
for s in sim.particle_sources:
if 'particle_id' not in g[s.name]:
ParticleArray([], s.charge, s.mass, np.empty((0, 3)),
np.empty((0, 3)), True).export_h5(g[s.name])
g = h5file.create_group('InnerRegions')
g.attrs['number_of_regions'] = [len(sim.inner_regions)]
for s in sim.inner_regions:
s.export_h5(g.create_group(s.name))
g = h5file.create_group('ExternalFields')
if sim.electric_fields.__class__.__name__ == "FieldZero":
ff = []
elif sim.electric_fields.__class__.__name__ == "FieldSum":
ff = sim.electric_fields.fields
else:
ff = [sim.electric_fields]
g.attrs['number_of_electric_fields'] = len(ff)
for s in ff:
sg = g.create_group(s.name)
if s.__class__ is FieldUniform:
ft = 'electric_uniform'
for i, c in enumerate('xyz'):
sg.attrs[
f'electric_uniform_field_{c}'] = s.uniform_field_vector[
i]
elif s.__class__ is FieldExpression:
ft = 'electric_tinyexpr'
for i, c in enumerate('xyz'):
expr = getattr(s, f'expression_{c}')
expr = np.string_(expr.encode('utf8')) + b'\x00'
sg.attrs[f'electric_tinyexpr_field_{c}'] = expr
elif s.__class__ is FieldFromCSVFile:
ft = 'electric_on_regular_grid'
sg.attrs['electric_h5filename'] = np.string_(
s.field_filename.encode('utf8') + b'\x00')
sg.attrs['field_type'] = np.string_(ft.encode('utf8') + b'\x00')
if sim.magnetic_fields.__class__.__name__ == "FieldZero":
ff = []
elif sim.magnetic_fields.__class__.__name__ == "FieldSum":
ff = sim.magnetic_fields.fields
else:
ff = [sim.magnetic_fields]
g.attrs['number_of_magnetic_fields'] = len(ff)
for s in ff:
sg = g.create_group(s.name)
if s.__class__ is FieldUniform:
ft = 'magnetic_uniform'
sg.attrs['speed_of_light'] = speed_of_light
for i, c in enumerate('xyz'):
sg.attrs[
f'magnetic_uniform_field_{c}'] = s.uniform_field_vector[
i]
elif s.__class__ is FieldExpression:
ft = 'magnetic_tinyexpr'
sg.attrs['speed_of_light'] = speed_of_light
for i, c in enumerate('xyz'):
expr = getattr(s, f'expression_{c}')
expr = np.string_(expr.encode('utf8')) + b'\x00'
sg.attrs[f'magnetic_tinyexpr_field_{c}'] = expr
elif s.__class__ is FieldFromCSVFile:
ft = 'magnetic_on_regular_grid'
sg.attrs['magnetic_h5filename'] = np.string_(
s.field_filename.encode('utf8') + b'\x00')
sg.attrs['field_type'] = np.string_(ft.encode('utf8') + b'\x00')
g = h5file.create_group('ParticleInteractionModel')
g.attrs['particle_interaction_model'] = \
np.string_(sim.particle_interaction_model.name.encode('utf8') + b'\x00')
def _write_pickled(self, obj, path):
"""Write a pickled object to file."""
if path in self.fh:
del self.fh[path]
pickled_obj = np.string_(pickle.dumps(obj))
self.fh[path] = pickled_obj
if n_candidates_remaining != 10000:
for i in range(n_candidates_remaining - 1):
print("receiving")
dat = queue.receive()
print("received")
candidates.append(unpack_message(dat))
print(unpack_message(dat))
print(1)
dy_norm = read_buffer(reader)
for i in range(
len(candidates)
): # for each candidate, calculate what part of data to save and put it
# in an h5 file
s1 = int(candidates[i][2]) - int(candidates[i][0]) - 1000
s1 = s1 * (s1 > 0)
s2 = min(
s1 + int(disp_delay(float(candidates[i][6]), dt, 1.53, 1.28) + 1) +
2**int(candidates[i][4]) + 1000, 200000)
save_cand = np.array([dy_norm[:, s1:s2], candidates[i]])
save_cand[1][-1] = np.string_(save_cand[1][-1])
save_cand[1][-5] = np.string_(save_cand[1][-5])
adict = dict(data=save_cand[0], metadata=save_cand[1])
with h5py.File('/home/user/candidates_train/candidates.h5', 'a') as hf:
grp = hf.create_group('candidate_%s_%s' %
(int(candidates[i][2]), i))
for k, v in adict.items():
grp.create_dataset(k, data=v)
print("Saving candidate %s" % candidates[i][2])
count += 1
reader.disconnect() # disconnect from the buffer
def save(self):
"""Method to save the class under a systematic name."""
# Create a temporary file and swap it to the original name in case
# data needs to be loaded while saving
tmp_name = uuid.uuid4()
with h5py.File("{}.hdf5".format(tmp_name), "w") as f:
# Save geometry (atoms and positions need to be separate):
d_geom = f.create_group("geometry")
if not isinstance(self.geometry, basestring):
atoms = [numpy.string_(item[0]) for item in self.geometry]
positions = numpy.array(
[list(item[1]) for item in self.geometry])
else:
atoms = numpy.string_(self.geometry)
positions = None
d_geom.create_dataset("atoms",
data=(atoms if atoms is not None else False))
d_geom.create_dataset(
"positions",
data=(positions if positions is not None else False))
# Save basis:
f.create_dataset("basis", data=numpy.string_(self.basis))
# Save multiplicity:
f.create_dataset("multiplicity", data=self.multiplicity)
# Save charge:
f.create_dataset("charge", data=self.charge)
# Save description:
f.create_dataset("description",
data=numpy.string_(self.description))
# Save name:
f.create_dataset("name", data=numpy.string_(self.name))
# Save n_atoms:
f.create_dataset("n_atoms", data=self.n_atoms)
# Save atoms:
f.create_dataset("atoms", data=numpy.string_(self.atoms))
# Save protons:
f.create_dataset("protons", data=self.protons)
# Save n_electrons:
f.create_dataset("n_electrons", data=self.n_electrons)
# Save generic attributes from calculations:
f.create_dataset("n_orbitals",
data=(self.n_orbitals
if self.n_orbitals is not None else False))
f.create_dataset(
"n_qubits",
data=(self.n_qubits if self.n_qubits is not None else False))
f.create_dataset(
"nuclear_repulsion",
data=(self.nuclear_repulsion
if self.nuclear_repulsion is not None else False))
# Save attributes generated from SCF calculation.
f.create_dataset(
"hf_energy",
data=(self.hf_energy if self.hf_energy is not None else False))
f.create_dataset(
"canonical_orbitals",
data=(self.canonical_orbitals
if self.canonical_orbitals is not None else False),
compression=("gzip"
if self.canonical_orbitals is not None else None))
f.create_dataset(
"overlap_integrals",
data=(self.overlap_integrals
if self.overlap_integrals is not None else False),
compression=("gzip"
if self.overlap_integrals is not None else None))
f.create_dataset(
"orbital_energies",
data=(self.orbital_energies
if self.orbital_energies is not None else False))
# Save attributes generated from integrals.
f.create_dataset(
"one_body_integrals",
data=(self.one_body_integrals
if self.one_body_integrals is not None else False),
compression=("gzip"
if self.one_body_integrals is not None else None))
f.create_dataset(
"two_body_integrals",
data=(self.two_body_integrals
if self.two_body_integrals is not None else False),
compression=("gzip"
if self.two_body_integrals is not None else None))
# Save attributes generated from MP2 calculation.
f.create_dataset("mp2_energy",
data=(self.mp2_energy
if self.mp2_energy is not None else False))
# Save attributes generated from CISD calculation.
f.create_dataset("cisd_energy",
data=(self.cisd_energy
if self.cisd_energy is not None else False))
f.create_dataset("cisd_one_rdm",
data=(self.cisd_one_rdm if self.cisd_one_rdm
is not None else False),
compression=("gzip" if self.cisd_one_rdm
is not None else None))
f.create_dataset("cisd_two_rdm",
data=(self.cisd_two_rdm if self.cisd_two_rdm
is not None else False),
compression=("gzip" if self.cisd_two_rdm
is not None else None))
# Save attributes generated from exact diagonalization.
f.create_dataset("fci_energy",
data=(self.fci_energy
if self.fci_energy is not None else False))
f.create_dataset(
"fci_one_rdm",
data=(self.fci_one_rdm
if self.fci_one_rdm is not None else False),
compression=("gzip" if self.fci_one_rdm is not None else None))
f.create_dataset(
"fci_two_rdm",
data=(self.fci_two_rdm
if self.fci_two_rdm is not None else False),
compression=("gzip" if self.fci_two_rdm is not None else None))
# Save attributes generated from CCSD calculation.
f.create_dataset("ccsd_energy",
data=(self.ccsd_energy
if self.ccsd_energy is not None else False))
f.create_dataset(
"ccsd_single_amps",
data=(self.ccsd_single_amps
if self.ccsd_single_amps is not None else False),
compression=("gzip"
if self.ccsd_single_amps is not None else None))
f.create_dataset(
"ccsd_double_amps",
data=(self.ccsd_double_amps
if self.ccsd_double_amps is not None else False),
compression=("gzip"
if self.ccsd_double_amps is not None else None))
# Save general calculation data
key_list = list(self.general_calculations.keys())
f.create_dataset(
"general_calculations_keys",
data=([numpy.string_(key)
for key in key_list] if len(key_list) > 0 else False))
f.create_dataset(
"general_calculations_values",
data=([self.general_calculations[key]
for key in key_list] if len(key_list) > 0 else False))
# Remove old file first for compatibility with systems that don't allow
# rename replacement. Catching OSError for when file does not exist
# yet
try:
os.remove("{}.hdf5".format(self.filename))
except OSError:
pass
shutil.move("{}.hdf5".format(tmp_name),
"{}.hdf5".format(self.filename))
print(dset[0])
# 超出长度部分截断
dset[0] = 'aksdjfhaksdfjhasdfjahsdkfjhasdk'.encode()
print(dset[0])
dt = np.dtype('S3')
a = np.array(['a', 'ab', 'abc', 'abcd'], dtype=dt)
print(a)
dt = h5py.special_dtype(vlen=str)
print(repr(dt))
print(dt.kind)
dset = f.create_dataset('vlen_dataset', (100, ), dtype=dt)
dset[0] = 'Hello'
dset[1] = np.string_('Hello2')
dset[3] = 'X' * 10000
# 读取一个元素时 返回str
out = dset[0]
print(type(out))
# 读取多个元素时 返回object
print(dset[0:2])
out = dset[0:1]
print(out.dtype)
args = parser.parse_args()
N = args.n_ref + 1
h5f = h5py.File('periodic_ref_output.hdf5',
'w',
driver='mpio',
comm=MPI.COMM_WORLD)
h5f.create_dataset("its", (N, ), dtype=np.int32)
h5f.create_dataset("num_dofs", (N, ), dtype=np.int32)
sd = h5f.create_dataset("solve_time", (N, MPI.COMM_WORLD.size),
dtype=np.float64)
solver = "BoomerAMG" if args.boomeramg else "GAMG"
ct = "Tet" if args.tetra else "Hex"
sd.attrs["solver"] = np.string_(solver)
sd.attrs["degree"] = np.string_(str(int(args.degree)))
sd.attrs["ct"] = np.string_(ct)
for i in range(N):
if MPI.COMM_WORLD.rank == 0:
set_log_level(LogLevel.INFO)
log(LogLevel.INFO, "Run {0:1d} in progress".format(i))
set_log_level(LogLevel.ERROR)
reference_periodic(args.tetra,
r_lvl=i,
out_hdf5=h5f,
xdmf=args.xdmf,
boomeramg=args.boomeramg,
kspview=args.kspview,
degree=args.degree)
def remote_ff_single_node(**data): #paramFileName startLayerNr endLayerNr timePath StartFileNrFirstLayer NrFilesPerSweep FileStem SeedFolder StartNr EndNr
import numpy as np
import os, subprocess
from pathlib import Path
import h5py
import warnings
import matplotlib.pyplot as plt
warnings.filterwarnings('ignore')
def getValueFromParamFile(paramfn,searchStr,nLines=1,wordNr=1,nWords=1):
ret_list = []
nrLines = 0
f = open(paramfn,'r')
PSContents = f.readlines()
for line in PSContents:
if line.startswith(searchStr+' '):
words = line.replace('\t',' ').replace('\n',' ').split(' ')
words = [_f for _f in words if _f]
ret_list.append(words[wordNr:wordNr+nWords])
nrLines += 1
if (nrLines == nLines):
return ret_list
return ret_list
paramFN = data.get('paramFileName')
startLayerNr = int(data.get('startLayerNr'))
endLayerNr = int(data.get('endLayerNr'))
time_path = data.get('timePath')
startNrFirstLayer = int(data.get('StartFileNrFirstLayer'))
nrFilesPerSweep = int(data.get('NrFilesPerSweep'))
fStem = data.get('FileStem')
topdir = data.get('SeedFolder')
startNr = int(data.get('StartNr'))
endNr = int(data.get('EndNr'))
darkFN = data.get('darkFN')
nFrames = data.get('nFrames')
numProcs = data.get('numProcs')
numBlocks = 1
blockNr = 0
os.chdir(topdir)
paramContents = open(paramFN).readlines()
baseNameParamFN = paramFN.split('/')[-1]
homedir = os.path.expanduser('~')
nFrames = endNr - startNr + 1
resArr = []
headSpots = 'GrainID SpotID Omega DetectorHor DetectorVert OmeRaw Eta RingNr YLab ZLab Theta StrainError OriginalRadiusFileSpotID IntegratedIntensity Omega(degrees) YCen(px) ZCen(px) IMax MinOme(degrees) MaxOme(degress) Radius(px) Theta(degrees) Eta(degrees) DeltaOmega NImgs RingNr GrainVolume GrainRadius PowderIntensity SigmaR SigmaEta NrPx'
for layerNr in range(startLayerNr,endLayerNr+1):
thisStartNr = startNrFirstLayer + (layerNr-1)*nrFilesPerSweep
folderName = fStem + '_Layer_' + str(layerNr).zfill(4) + '_Analysis_Time_' + time_path
thisDir = topdir + '/' + folderName + '/'
Path(thisDir).mkdir(parents=True,exist_ok=True)
outdir = f'{thisDir}/{fStem}_Layer_{str(layerNr).zfill(4)}_Analysis_Time_{time_path}/'
Path(outdir).mkdir(parents=True,exist_ok=True)
os.chdir(thisDir)
thisParamFN = thisDir + baseNameParamFN
thisPF = open(thisParamFN,'w')
for line in paramContents:
thisPF.write(line)
thisPF.write('RawFolder '+topdir+'\n')
thisPF.write('SeedFolder '+topdir+'\n')
thisPF.write('Dark '+topdir+'/'+darkFN+'\n')
thisPF.write('Folder '+thisDir+'\n')
thisPF.write('LayerNr '+str(layerNr)+'\n')
thisPF.write('StartFileNr '+str(thisStartNr)+'\n')
thisPF.close()
Path(thisDir+'/Temp').mkdir(parents=True,exist_ok=True)
Path(thisDir+'Output').mkdir(parents=True,exist_ok=True)
Path(thisDir+'Results').mkdir(parents=True,exist_ok=True)
stdoutf = open(thisDir+'output.txt','w')
stdoutf2 = open(thisDir+'output2.txt','w')
subprocess.call(os.path.expanduser("~/opt/MIDAS/FF_HEDM/bin/GetHKLList")+" "+baseNameParamFN,shell=True,stdout=stdoutf)
subprocess.call(os.path.expanduser("~/opt/MIDAS/FF_HEDM/bin/PeaksFittingOMP")+' '+baseNameParamFN+' '+ str(blockNr) + ' ' + str(numBlocks) + ' '+str(nFrames)+' '+str(numProcs),shell=True,stdout=stdoutf,stderr=stdoutf2)
subprocess.call(os.path.expanduser("~/opt/MIDAS/FF_HEDM/bin/MergeOverlappingPeaksAll")+' '+baseNameParamFN,shell=True,stdout=stdoutf)
subprocess.call(os.path.expanduser("~/opt/MIDAS/FF_HEDM/bin/CalcRadiusAll")+' '+baseNameParamFN,shell=True,stdout=stdoutf)
subprocess.call(os.path.expanduser("~/opt/MIDAS/FF_HEDM/bin/FitSetup")+' '+baseNameParamFN,shell=True,stdout=stdoutf)
subprocess.call(os.path.expanduser("~/opt/MIDAS/FF_HEDM/bin/SaveBinData"),shell=True,stdout=stdoutf)
nSpotsToIndex = len(open('SpotsToIndex.csv').readlines())
rc = subprocess.call(os.path.expanduser("~/opt/MIDAS/FF_HEDM/bin/IndexerOMP")+' paramstest.txt '+str(blockNr)+' '+str(numBlocks)+' '+str(nSpotsToIndex)+' 32',shell=True,stdout=stdoutf,stderr=stdoutf2)#+str(numProcs),shell=True,stdout=stdoutf)
if (rc < = 0):
return('Something went wrong')
subprocess.call(os.path.expanduser("~/opt/MIDAS/FF_HEDM/bin/FitPosOrStrainsOMP")+' paramstest.txt '+str(blockNr)+' '+str(numBlocks)+' '+str(nSpotsToIndex)+' '+str(numProcs),shell=True,stdout=stdoutf)
subprocess.call(os.path.expanduser('~/opt/MIDAS/FF_HEDM/bin/ProcessGrains') + ' ' + baseNameParamFN,shell=True,stdout=stdoutf)
paramFile = baseNameParamFN
outFN = f'{outdir}/result_{fStem}_LayerNr_{layerNr}_Analysis_time_{time_path}.hdf'
pad = int(getValueFromParamFile(paramFile,'Padding')[0][0])
Grains = np.genfromtxt('Grains.csv',skip_header=9)
SpotMatrix = np.genfromtxt('SpotMatrix.csv',skip_header=1)
IDRings = np.genfromtxt('IDRings.csv',skip_header=1)
IDsHash = np.genfromtxt('IDsHash.csv',skip_header=1)
InputAll = np.genfromtxt('InputAll.csv',skip_header=1)
InputAllExtra = np.genfromtxt('InputAllExtraInfoFittingAll.csv',skip_header=1)
SpotsToIndex = np.genfromtxt('SpotsToIndex.csv',skip_header=1)
HKLs = np.genfromtxt('hkls.csv',skip_header=1)
outFile = h5py.File(outFN,'w')
f = open('Grains.csv','r')
nGrains = int(f.readline().split()[1])
beamCenter = float(f.readline().split()[1])
beamThickness = float(f.readline().split()[1])
globalPosition = float(f.readline().split()[1])
f.readline()
f.readline()
f.readline()
f.readline()
hGr = f.readline()
f.close()
outFile.attrs['Software'] = np.string_("MIDAS")
outFile.attrs['Version'] = np.string_("6.0")
outFile.attrs['Contact'] = np.string_("*****@*****.**")
outFile.create_dataset('ParametersFile',data=np.string_(open(paramFile).read()))
group1 = outFile.create_group('RawFiles')
group1.create_dataset('paramstest',data=np.string_(open('paramstest.txt').read()))
sm = group1.create_dataset('SpotMatrix',data=SpotMatrix)
sm.attrs['head'] = np.string_(open('SpotMatrix.csv').readline())
gr = group1.create_dataset('AllGrains',data=Grains)
gr.attrs['head'] = np.string_(hGr)
idr = group1.create_dataset('IDRings',data=IDRings)
idr.attrs['head'] = np.string_(open('IDRings.csv').readline())
idh = group1.create_dataset('IDsHash',data=IDsHash)
idh.attrs['head'] = np.string_(open('IDsHash.csv').readline())
ipa = group1.create_dataset('InputAll',data=InputAll)
ipa.attrs['head'] = np.string_(open('InputAll.csv').readline())
ipe = group1.create_dataset('InputAllExtraInfo',data=InputAllExtra)
ipe.attrs['head'] = np.string_(open('InputAllExtraInfoFittingAll.csv').readline())
group1.create_dataset('SpotsToIndex',data=SpotsToIndex)
hk = group1.create_dataset('HKLs',data=HKLs)
hk.attrs['head'] = np.string_(open('hkls.csv').readline())
# We have a merged filesystem now, not according to rings
# Put Temp data
group2 = group1.create_group('Temp')
for fNr in range(startNr,endNr+1):
fileName = f'{os.getcwd()}/Temp/{fStem}_{layerNr}_{str(fNr).zfill(pad)}_PS.csv'
if os.path.exists(fileName):
arr = np.genfromtxt(fileName,skip_header=1)
if arr.shape[0] > 0:
tmpd = group2.create_dataset(os.path.basename(fileName),data=arr)
tmpd.attrs['head'] = np.string_(open(fileName).readline())
# Put Radii
fileName = f'{os.getcwd()}/Radius_StartNr_{startNr}_EndNr_{endNr}.csv'
arr = np.genfromtxt(fileName,skip_header=1)
nSps,nColsRad = arr.shape
radd = group1.create_dataset(os.path.basename(fileName),data=arr)
radd.attrs['head'] = np.string_(open(fileName).readline())
radii = arr
# Put Merge Result
fileName = f'{os.getcwd()}/Result_StartNr_{startNr}_EndNr_{endNr}.csv'
arr = np.genfromtxt(fileName,skip_header=1)
nSps,nTrs = arr.shape
resd = group1.create_dataset(os.path.basename(fileName),data=arr)
resd.attrs['head'] = np.string_(open(fileName).readline())
resarr = arr
gg = outFile.create_group('Grains')
for counter,grain in enumerate(Grains):
thisID = int(grain[0])
print(f'Processing grain {counter+1} out of {Grains.shape[0]} grains.')
grg = gg.create_group('GrainID_'+str(thisID))
grd = grg.create_dataset('GrainInfo',data=grain)
grd.attrs['header'] = hGr
spotsThisGrain = SpotMatrix[SpotMatrix[:,0] == thisID]
RadiusInfo = np.empty((spotsThisGrain.shape[0],nColsRad))
for ctr,spot in enumerate(spotsThisGrain):
spotID = int(spot[1])
orig_ID = int(IDRings[IDRings[:,2]==spotID,1])
ringNr = int(IDRings[IDRings[:,2]==spotID,0])
subInfo = radii[orig_ID-1]
RadiusInfo[ctr,:] = subInfo
RadiusInfo = np.hstack((spotsThisGrain,RadiusInfo))
spd = grg.create_dataset('SpotMatrix_Radius',data=RadiusInfo)
spd.attrs['header'] = headSpots
outFile.close()
# Make and save plots
plt.scatter(Grains[:,10],Grains[:,11]); plt.xlabel('X [\mu m]'); plt.ylabel('Y [\mu m]'); plt.savefig(outdir+'/XY.png'); plt.clf()
plt.scatter(Grains[:,11],Grains[:,12]); plt.xlabel('Y [\mu m]'); plt.ylabel('Z [\mu m]'); plt.savefig(outdir+'/YZ.png'); plt.clf()
plt.scatter(Grains[:,10],Grains[:,12]); plt.xlabel('X [\mu m]'); plt.ylabel('Z [\mu m]'); plt.savefig(outdir+'/XZ.png'); plt.clf()
plt.scatter(Grains[:,22],Grains[:,19]); plt.xlabel('Grain Radius [\mu m]'); plt.ylabel('PosErr [\mu m]'); plt.savefig(outdir+'/PosvsRad.png'); plt.clf()
plt.scatter(Grains[:,22],Grains[:,21]); plt.xlabel('Grain Radius [\mu m]'); plt.ylabel('InternalAngle [Degrees]'); plt.savefig(outdir+'/IAvsRad.png'); plt.clf()
plt.scatter(Grains[:,22],Grains[:,33]); plt.xlabel('Grain Radius [\mu m]'); plt.ylabel('E_XX'); plt.savefig(outdir+'/eXXvsRad.png'); plt.clf()
plt.scatter(Grains[:,22],Grains[:,37]); plt.xlabel('Grain Radius [\mu m]'); plt.ylabel('E_YY'); plt.savefig(outdir+'/eYYvsRad.png'); plt.clf()
plt.scatter(Grains[:,22],Grains[:,41]); plt.xlabel('Grain Radius [\mu m]'); plt.ylabel('E_ZZ'); plt.savefig(outdir+'/eZZvsRad.png'); plt.clf()
resArr.append([outFN,open('Grains.csv','r').readline()])
os.chdir(topdir)
subprocess.call('tar -czf recon_'+time_path+'.tar.gz *_Analysis_Time_'+time_path+'*',shell=True)
return resArr
def write(self, fname):
"""
write the file in CBF format
:param str fname: name of the file
"""
if self.data is None:
raise RuntimeError("CBF image contains no data")
# The shape is provided by self.data
self._shape = None
dim2, dim1 = self.shape
binary_blob = compByteOffset(self.data)
dtype = "Unknown"
for key, value in DATA_TYPES.items():
if value == self.data.dtype:
dtype = key
binary_block = [
b"--CIF-BINARY-FORMAT-SECTION--",
b"Content-Type: application/octet-stream;",
b' conversions="x-CBF_BYTE_OFFSET"',
b'Content-Transfer-Encoding: BINARY',
numpy.string_("X-Binary-Size: %d" % (len(binary_blob))),
b"X-Binary-ID: 1",
numpy.string_('X-Binary-Element-Type: "%s"' % (dtype)),
b"X-Binary-Element-Byte-Order: LITTLE_ENDIAN",
b"Content-MD5: " + md5sum(binary_blob),
numpy.string_("X-Binary-Number-of-Elements: %d" % (dim1 * dim2)),
numpy.string_("X-Binary-Size-Fastest-Dimension: %d" % dim1),
numpy.string_("X-Binary-Size-Second-Dimension: %d" % dim2),
b"X-Binary-Size-Padding: 1", b"", self.STARTER + binary_blob, b"",
b"--CIF-BINARY-FORMAT-SECTION----"
]
if "_array_data.header_contents" not in self.header:
nonCifHeaders = []
else:
nonCifHeaders = [
i.strip()[2:]
for i in self.header["_array_data.header_contents"].split("\n")
if i.find("# ") >= 0
]
for key in self.header:
if key.startswith("_"):
if key not in self.cif or self.cif[key] != self.header[key]:
self.cif[key] = self.header[key]
elif key.startswith("X-Binary-"):
pass
elif key.startswith("Content-"):
pass
elif key.startswith("conversions"):
pass
elif key.startswith("filename"):
pass
elif key in self.header:
nonCifHeaders.append("%s %s" % (key, self.header[key]))
if len(nonCifHeaders) > 0:
self.cif["_array_data.header_contents"] = "\r\n".join(
["# %s" % i for i in nonCifHeaders])
self.cbf = b"\r\n".join(binary_block)
block = b"\r\n".join([
b"",
self.CIF_BINARY_BLOCK_KEY.encode("ASCII"), b";", self.cbf, b";"
])
self.cif.pop(self.CIF_BINARY_BLOCK_KEY, None)
with open(fname, "wb") as out_file:
out_file.write(self.cif.tostring(fname, "\r\n").encode("ASCII"))
out_file.write(block)
def main(_):
with open(FLAGS.filelist, 'r') as fin:
filelist = json.load(fin)
filelist = filelist['path']
filelist = [
np.string_(os.path.basename(filename)) for filename in filelist
]
if FLAGS.prefix == 'facescrub':
filelist = [filename.replace(b" ", b"_") for filename in filelist]
filelist = [replace_extension(filename) for filename in filelist]
file_dict = {filename: i for i, filename in enumerate(filelist)}
num_samples = len(filelist)
config = tf.estimator.RunConfig(
model_dir=FLAGS.model_dir,
keep_checkpoint_every_n_hours=1,
save_summary_steps=50,
save_checkpoints_secs=60 * 20,
)
estimator = tf.estimator.Estimator(model_fn=model.model_fn, config=config)
predictions = estimator.predict(input_fn=lambda: model.imagenet_iterator(
is_training=FLAGS.is_training),
yield_single_examples=True,
predict_keys=[
"true_labels", "true_label_texts",
"small_embeddings", "filename"
])
if not os.path.exists('embeddings_reranking'):
os.mkdir('embeddings_reranking')
if FLAGS.output is None:
output_file = FLAGS.prefix + "_" + os.path.basename(
FLAGS.model_dir.rstrip('/')) + '.hdf5'
else:
output_file = FLAGS.output
output_file = os.path.join('embeddings_reranking', output_file)
if os.path.exists(output_file):
print('%s exists. Exiting' % output_file)
sys.exit()
print("*" * 10, "Writing to", output_file)
sparsity = []
embeddings = []
labels = []
idx = -np.ones(num_samples, dtype=np.int32)
for step, prediction in enumerate(predictions):
embedding = prediction["small_embeddings"]
true_label = prediction["true_labels"][0]
true_label_text = np.string_(prediction["true_label_texts"])
filename = np.string_(prediction["filename"])
idxs = np.where(np.abs(embedding) >= FLAGS.zero_threshold)[0]
sparsity.append(len(idxs))
embeddings.append(embedding)
labels.append(true_label)
id = file_dict[filename]
assert (idx[id] == -1)
idx[id] = step
if step % 10000 == 0:
print('Step %d Av. Sparsity %f' % (step, np.mean(sparsity)))
sys.stdout.flush()
if FLAGS.debug and step >= 10000:
break
embeddings = np.asarray(embeddings)
embeddings = embeddings[idx]
print("embeddings shape", embeddings.shape)
labels = np.asarray(labels)
labels = labels[idx]
print("labels shape", labels.shape)
h5file = h5py.File(output_file, mode='w')
h5file.create_dataset(name='embedding', data=embeddings)
h5file.create_dataset(name='label', data=labels)
h5file.close()
print("Done writing to", output_file)
def Perform_Test():
'''
Main function that performs entire spectrometer test.
Finishes setup phase, then waits for button input to continue.
Setup phase takes a while, mainly due to DAQ_Speed_Test() and DAQ_Speed_Test().
'''
global Spec_Is_Read, Spec_Init_Done, DAQ_Is_Read, Power_Is_Read, Timer_Is_Over
Spec_Is_Read = Value('i', 0)
Spec_Is_Read.value = 0
Spec_Init_Done = Value('i', 0)
Spec_Init_Done.value = 0
DAQ_Is_Read = Value('i', 0)
DAQ_Is_Read.value = 0
Power_Is_Read = Value('i', 0)
Power_Is_Read.value = 0
Timer_Is_Over = Value('i', 0)
Timer_Is_Over.value = 0
Integration_Time = 100 # Integration time in ms
Spec1.setTriggerMode(3) # It is set for free running mode
#Spec1.setIntegrationTime(Integration_Time*1000) # Integration time is in microseconds when using the library
# Check to see if the folder called Records exist
Path_to_Records = os.path.abspath(os.path.join(os.getcwd())) + "/Records"
if not os.path.exists(Path_to_Records):
os.makedirs(Path_to_Records)
DAQ1.writePort(Green_Shutter, 0)
DAQ1.writePort(Blue_Shutter, 0)
# Initializing the variables
Integration_list_MilSec = [8, 16, 32, 64, 128, 256, 512, 1024
] #Integration time for the spectrometer in ms
Shutter_Delay = 4 #ms
No_DAQ_Tests = 20000
DAQ_SamplingRate = DAQ_Speed_Test(
No_DAQ_Tests) * 1000 #Shows the sampling speed in ms
global Wavelengths, Min_Wave_Index, Max_Wave_Index, Full_Spec_Records2, Spec_Time, Spec_Index
Wavelengths = Spec1.readWavelength()
Min_Wave_Index = max(bisect.bisect(Wavelengths,
float(min_len.get()) - 1), 0)
Max_Wave_Index = bisect.bisect(Wavelengths, float(max_len.get()))
Wavelengths = Wavelengths[Min_Wave_Index:Max_Wave_Index]
Current_Spec_Record = Array(
'd', np.zeros(shape=(len(Wavelengths), 1), dtype=float))
No_Spec_Tests = 500
Full_Spec_Records2 = Array(
'd', np.zeros(shape=(len(Wavelengths) * No_Spec_Tests, 1),
dtype=float))
Spec_Time = Array('d', np.zeros(shape=(No_Spec_Tests, 1), dtype=float))
Spec_Index = Array('i', np.zeros(shape=(1, 1), dtype=int))
Spec_SamplingRate = Spec_Speed_Test(No_Spec_Tests)
Integration_Buffer_Time = 100 #ms # This is for the spectrometer. This is the time from the integration started till shutter opens
#DurationOfReading = np.sum(Integration_list_MilSec) + len(Integration_list_MilSec)*Delay_Between_Integrations # Duration of reading in seconds.
DurationOfReading = (
Integration_list_MilSec[-1] + Integration_Buffer_Time + Shutter_Delay *
3) * len(Integration_list_MilSec) # Duration of reading in seconds.
No_BakGro_Spec = 10 # This is for continious reading and refers to the last few spectrom reading wich are background and the laser is off
# Finding parameter values from GUI
# - Port to use, either Green_Shutter or Blue_Shutter
# - Integration time in ms
# - Total recording duration in s
# - Prefix of filename
global Shutter_Port
Shutter_Port = shut_mode.get()
Integration_Continious = float(int_time.get())
DurationOfReading = float(
rec_time.get()) * 1000.0 + No_BakGro_Spec * float(int_time.get())
File_name_PreFix = filename.get()
if File_name_PreFix == "":
File_name_PreFix = "OptrodeData"
if (Power_meter.Error == 0):
#Powermeter_SamplingRate = 5.1 #ms
No_Power_Tests = 200
Power_SamplingRate = Power_Speed_Test(
No_Power_Tests) * 1000 #Shows the sampling speed in ms
#%% ############## Defining the size of the arrays and matrices for recording the signals beased on the duration of the recording #######
#No_DAC_Sample = int(round((DurationOfReading + DurationOfReading/4) /DAQ_SamplingRate)) # Number of samples for DAQ analogue to digital converter (AINx).
No_DAC_Sample = int((2. * DurationOfReading) / DAQ_SamplingRate)
if (Power_meter.Error == 0):
No_Power_Sample = int((1.5 * DurationOfReading) / Power_SamplingRate)
else:
No_Power_Sample = 0
#No_Power_Sample = int(round(DurationOfReading/Powermeter_SamplingRate))
# Number of samples for P100D Power meter to read.
# Roughly P100 can read the power every 2.7 ms.
if (par_mode.get() == 'c'): # Continious paradigm
No_Spec_Sample = int(
round(
float(DurationOfReading) / float(float(Integration_Continious))
)) # Number of samples for spectrometer to read.
else:
No_Spec_Sample = len(Integration_list_MilSec
) # Number of samples for spectrometer to read.
Rerun = 'First'
while True:
################################ Variables initializations #############################################
Current_Spec_Record = Array(
'd', np.zeros(shape=(len(Wavelengths), 1), dtype=float))
#Last_Spec_Record = Array('d', np.zeros(shape=( len(Wavelengths) ,1), dtype = float ))
Spec_Index = Array('i', np.zeros(shape=(1, 1), dtype=int))
Full_Spec_Records = np.zeros(shape=(len(Wavelengths), No_Spec_Sample),
dtype=float)
Full_Spec_Records2 = Array(
'd',
np.zeros(shape=(len(Wavelengths) * No_Spec_Sample, 1),
dtype=float))
Spec_Time = Array('d', np.zeros(shape=(No_Spec_Sample, 1),
dtype=float))
global DAQ_Signal, DAQ_Time, DAQ_Index
DAQ_Signal = Array('d', np.zeros(shape=(No_DAC_Sample, 1),
dtype=float))
DAQ_Time = Array('d', np.zeros(shape=(No_DAC_Sample, 1), dtype=float))
DAQ_Index = Array('i', np.zeros(shape=(1, 1), dtype=int))
#DAQ_Index_Total = Array('i', np.zeros(shape=( 1 ,1), dtype = int ))
#Ref_Signal = Array('d', np.zeros(shape=( No_DAC_Sample ,1), dtype = float ))
#Ref_Time = Array('d', np.zeros(shape=( No_DAC_Sample ,1), dtype = float ))
if (Power_meter.Error == 0):
Power_Signal = Array(
'd', np.zeros(shape=(No_Power_Sample, 1), dtype=float))
Power_Time = Array(
'd', np.zeros(shape=(No_Power_Sample, 1), dtype=float))
Power_Index = Array('i', np.zeros(shape=(1, 1), dtype=int))
##### Finished Setup
##### Waiting for button press
but_start.config(state=NORMAL)
debug("Ready to start paradigm. Press start to begin.")
but_setup.wait_variable(wait_var)
but_start.config(state=DISABLED)
# Starting the chosen paradigm
if (par_mode.get() == 'm'):
Multi_Integration_Paradigm(Integration_list_MilSec,
Integration_Buffer_Time, Shutter_Delay,
No_Power_Sample)
else:
Continious_Paradigm(float(Integration_Continious), No_Spec_Sample,
No_DAC_Sample, No_Power_Sample, No_BakGro_Spec)
# Loading the Spectrometer Array to a matrix before saving and plotting ###############
Wave_len = len(Wavelengths)
for I in range(Spec_Index[0]):
Full_Spec_Records[:, I] = Full_Spec_Records2[I * Wave_len:(I + 1) *
Wave_len]
# Closing the devices
Spec_Details = Spec1.readDetails()
DAQ_Details = DAQ1.getDetails()
DAQ1.writePort(Shutter_Port, 0)
# ########### The file containing the records (HDF5 format)###########
#Path_to_Records = os.path.abspath(os.path.join( os.getcwd(), os.pardir)) + "/Records"
#Path_to_Records = os.path.abspath(os.path.join( os.getcwd())) + "/Records"
os.chdir(Path_to_Records)
if is_suff.get() == 1:
File_name_Suffix = str('%s' % datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d-%H-%M-%S'))
File_name = File_name_PreFix + '-' + File_name_Suffix + ".hdf5"
else:
File_name = File_name_PreFix + ".hdf5"
f = h5py.File(File_name, "w")
# Saving the recorded signals in HDF5 format
Optrode_DAQ = f.create_group('DAQT7')
f.create_dataset('DAQT7/PhotoDiode', data=np.asanyarray(DAQ_Signal[:]))
f.create_dataset('DAQT7/TimeIndex', data=np.asanyarray(DAQ_Time[:]))
Optrode_DAQ.attrs['DAQT7 Details'] = np.string_(DAQ_Details)
Optrode_Spectrometer = f.create_group('Spectrometer')
f.create_dataset('Spectrometer/Intensities',
data=np.asanyarray(Full_Spec_Records))
f.create_dataset('Spectrometer/Time_Index',
data=np.asanyarray(Spec_Time))
f.create_dataset('Spectrometer/WaveLength',
data=np.asanyarray(Wavelengths))
Optrode_Spectrometer.attrs['Spectrometer Details'] = np.string_(
Spec_Details)
if (Power_meter.Error == 0):
Optrode_Power = f.create_group('PM100_PowerMeter')
f.create_dataset('PM100_PowerMeter/Power',
data=np.asanyarray(Power_Signal[:]))
f.create_dataset('PM100_PowerMeter/TimeIndex',
data=np.asanyarray(Power_Time[:]))
#Optrode_DAQ.attrs['PowerMeter Details'] = np.string_(DAQ_Details)
f.close()
Path_to_Fred_Codes = os.path.abspath(
os.path.join(os.getcwd(), os.pardir))
os.chdir(Path_to_Fred_Codes)
# Plotting the spectrumeter and the photodiod recordings ########
if plots[0].get() == 1:
plt.figure()
plt.plot(
np.asarray(DAQ_Time[0:DAQ_Index[0]]) - DAQ_Time[0],
np.asanyarray(DAQ_Signal[0:DAQ_Index[0]]))
plt.title('Photo diode')
plt.xlabel('Ellapsed time (s)')
plt.ylabel('Voltage (v)')
'''
plt.figure()
plt.plot(Spec1.readWavelength()[2:],Full_Spec_Records[2:])
plt.title('Specrometer recordings')
plt.xlabel('Wavelength (nano meter)')
plt.ylabel('Intensity')
#plt.plot(np.asarray(Ref_Time[0:DAQ_Index[0]]) - DAQ_Time[0],Ref_Signal[0:DAQ_Index[0]])
'''
# Estimate the latencies of the devices ###################################
if plots[1].get() == 1:
plt.figure()
plt.subplot(1, 2, 1)
DAQ_Latency = np.asanyarray(DAQ_Time[0:DAQ_Index[0]])
DAQ_Latency[0] = 0
for I in range(1, DAQ_Index[0]):
DAQ_Latency[I] = DAQ_Time[I] - DAQ_Time[I - 1]
#plt.subplot(1,3,1)
plt.plot(DAQ_Latency)
plt.ylabel("Time (s)")
plt.title("DAQ latencies")
plt.show()
plt.subplot(1, 2, 2)
Spec_Latency = np.asarray(Spec_Time[0:np.int(Spec_Index[0])])
Spec_Latency[0] = 0
for I in range(1, Spec_Index[0]):
Spec_Latency[I] = np.float(Spec_Time[I] - Spec_Time[I - 1])
plt.plot(Spec_Latency[1:])
plt.ylabel("Time (s)")
plt.title("Spectrometer integration durations")
plt.show()
if plots[2].get() == 1 and Power_meter.Error == 0:
plt.figure()
plt.subplot(1, 2, 1)
Power_Latency = np.asanyarray(Power_Time[0:Power_Index[0]])
Power_Latency[0] = 0
for I in range(1, int(Power_Index[0])):
Power_Latency[I] = Power_Time[I] - Power_Time[I - 1]
#plt.subplot(1,3,1)
plt.plot(Power_Latency)
plt.ylabel("Time (s)")
plt.title("Power latencies")
plt.show()
plt.subplot(1, 2, 2)
plt.plot(
np.asarray(Power_Time[0:Power_Index[0]]) - Power_Time[0],
np.asanyarray(Power_Signal[0:Power_Index[0]]))
plt.title('Power Meter')
plt.xlabel('Ellapsed time (s)')
plt.ylabel('Power (w)')
plt.show()
print('\n')
print('Data is saved. \n')
# Wait for button press
but_rerun.config(state=NORMAL)
but_change.config(state=NORMAL)
debug("Re-run test with same parameters, change parameters, or quit.")
but_rerun.wait_variable(wait_var)
but_rerun.config(state=DISABLED)
but_change.config(state=DISABLED)
# If we clicked the 'change' button, quit loop, otherwise keep going.
if wait_var.get() == 2:
but_setup.config(state=NORMAL)
Disable_UI(root, False)
debug(" ")
break
class CIF(dict):
"""
This is the CIF class, it represents the CIF dictionary;
and as a a python dictionary thus inherits from the dict built in class.
keys are always unicode (str in python3)
values are bytes
"""
EOL = [numpy.string_(i) for i in ("\r", "\n", "\r\n", "\n\r")]
BLANK = [numpy.string_(i) for i in (" ", "\t")] + EOL
SINGLE_QUOTE = numpy.string_("'")
DOUBLE_QUOTE = numpy.string_('"')
SEMICOLUMN = numpy.string_(';')
START_COMMENT = (SINGLE_QUOTE, DOUBLE_QUOTE)
BINARY_MARKER = numpy.string_("--CIF-BINARY-FORMAT-SECTION--")
HASH = numpy.string_("#")
LOOP = numpy.string_("loop_")
UNDERSCORE = ord("_") if six.PY3 else b"_"
QUESTIONMARK = ord("?") if six.PY3 else b"?"
STOP = numpy.string_("stop_")
GLOBAL = numpy.string_("global_")
DATA = numpy.string_("data_")
SAVE = numpy.string_("save_")
def __init__(self, _strFilename=None):
"""
Constructor of the class.
:param _strFilename: the name of the file to open
:type _strFilename: filename (str) or file object
"""
dict.__init__(self)
self._ordered = []
if _strFilename is not None: # load the file)
self.loadCIF(_strFilename)
def __setitem__(self, key, value):
if key not in self._ordered:
self._ordered.append(key)
return dict.__setitem__(self, key, value)
def pop(self, key, default=None):
if key in self._ordered:
self._ordered.remove(key)
return dict.pop(self, key, default)
def popitem(self, key, default=None):
if key in self._ordered:
self._ordered.remove(key)
return dict.popitem(self, key, None)
def loadCIF(self, _strFilename, _bKeepComment=False):
"""Load the CIF file and populates the CIF dictionary into the object
:param str _strFilename: the name of the file to open
:return: None
"""
own_fd = False
if isinstance(_strFilename, (six.binary_type, six.text_type)):
if os.path.isfile(_strFilename):
infile = open(_strFilename, "rb")
own_fd = True
else:
raise RuntimeError("CIF.loadCIF: No such file to open: %s" %
_strFilename)
elif "read" in dir(_strFilename):
infile = _strFilename
else:
raise RuntimeError("CIF.loadCIF: what is %s type %s" %
(_strFilename, type(_strFilename)))
if _bKeepComment:
self._parseCIF(numpy.string_(infile.read()))
else:
self._parseCIF(CIF._readCIF(infile))
if own_fd:
infile.close()
readCIF = loadCIF
@staticmethod
def isAscii(text):
"""
Check if all characters in a string are ascii,
:param str text: input string
:return: boolean
:rtype: boolean
"""
try:
text.decode("ascii")
except UnicodeDecodeError:
return False
else:
return True
@classmethod
def _readCIF(cls, instream):
"""
- Check if the filename containing the CIF data exists
- read the cif file
- removes the comments
:param file instream: opened file object containing the CIF data
:return: a set of bytes (8-bit string) containing the raw data
:rtype: string
"""
if "read" not in dir(instream):
raise RuntimeError(
"CIF._readCIF(instream): I expected instream to be an opened file,\
here I got %s type %s" % (instream, type(instream)))
out_bytes = numpy.string_("")
for sLine in instream:
nline = numpy.string_(sLine)
pos = nline.find(cls.HASH)
if pos >= 0:
if cls.isAscii(nline):
out_bytes += nline[:pos] + numpy.string_(os.linesep)
if pos > 80:
logger.warning(
"This line is too long and could cause problems in PreQuest: %s",
sLine)
else:
out_bytes += nline
if len(sLine.strip()) > 80:
logger.warning(
"This line is too long and could cause problems in PreQuest: %s",
sLine)
return out_bytes
def _parseCIF(self, bytes_text):
"""
- Parses the text of a CIF file
- Cut it in fields
- Find all the loops and process
- Find all the keys and values
:param bytes_text: the content of the CIF - file
:type bytes_text: 8-bit string (str in python2 or bytes in python3)
:return: Nothing, the data are incorporated at the CIF object dictionary
:rtype: None
"""
loopidx = []
looplen = []
loop = []
fields = split_tokens(bytes_text)
logger.debug("After split got %s fields of len: %s", len(fields),
[len(i) for i in fields])
for idx, field in enumerate(fields):
if field.lower() == self.LOOP:
loopidx.append(idx)
if loopidx:
for i in loopidx:
loopone, length, keys = CIF._analyseOneLoop(fields, i)
loop.append([keys, loopone])
looplen.append(length)
for i in range(len(loopidx) - 1, -1, -1):
f1 = fields[:loopidx[i]] + fields[loopidx[i] + looplen[i]:]
fields = f1
self[self.LOOP.decode("ASCII")] = loop
for i in range(len(fields) - 1):
if len(fields[i + 1]) == 0:
fields[i + 1] = self.QUESTIONMARK
if fields[i][0] == self.UNDERSCORE and fields[
i + 1][0] != self.UNDERSCORE:
try:
data = fields[i + 1].decode("ASCII")
except UnicodeError:
logger.warning("Unable to decode in ascii: %s" %
fields[i + 1])
data = fields[i + 1]
self[(fields[i]).decode("ASCII")] = data
@classmethod
def _splitCIF(cls, bytes_text):
"""
Separate the text in fields as defined in the CIF
:param bytes_text: the content of the CIF - file
:type bytes_text: 8-bit string (str in python2 or bytes in python3)
:return: list of all the fields of the CIF
:rtype: list
"""
fields = []
while True:
if len(bytes_text) == 0:
break
elif bytes_text[0] == cls.SINGLE_QUOTE:
idx = 0
finished = False
while not finished:
idx += 1 + bytes_text[idx + 1:].find(cls.SINGLE_QUOTE)
if idx >= len(bytes_text) - 1:
fields.append(bytes_text[1:-1].strip())
bytes_text = numpy.string_("")
finished = True
break
if bytes_text[idx + 1] in cls.BLANK:
fields.append(bytes_text[1:idx].strip())
tmp_text = bytes_text[idx + 1:]
bytes_text = tmp_text.strip()
finished = True
elif bytes_text[0] == cls.DOUBLE_QUOTE:
idx = 0
finished = False
while not finished:
idx += 1 + bytes_text[idx + 1:].find(cls.DOUBLE_QUOTE)
if idx >= len(bytes_text) - 1:
fields.append(bytes_text[1:-1].strip())
bytes_text = numpy.string_("")
finished = True
break
if bytes_text[idx + 1] in cls.BLANK:
fields.append(bytes_text[1:idx].strip())
tmp_text = bytes_text[idx + 1:]
bytes_text = tmp_text.strip()
finished = True
elif bytes_text[0] == cls.SEMICOLUMN:
if bytes_text[1:].strip().find(cls.BINARY_MARKER) == 0:
idx = bytes_text[32:].find(cls.BINARY_MARKER)
if idx == -1:
idx = 0
else:
idx += 32 + len(cls.BINARY_MARKER)
else:
idx = 0
finished = False
while not finished:
idx += 1 + bytes_text[idx + 1:].find(cls.SEMICOLUMN)
if bytes_text[idx - 1] in cls.EOL:
fields.append(bytes_text[1:idx - 1].strip())
tmp_text = bytes_text[idx + 1:]
bytes_text = tmp_text.strip()
finished = True
else:
res = bytes_text.split(None, 1)
if len(res) == 2:
first, second = bytes_text.split(None, 1)
if cls.isAscii(first):
fields.append(first)
bytes_text = second.strip()
continue
start_binary = bytes_text.find(cls.BINARY_MARKER)
if start_binary > 0:
end_binary = bytes_text[start_binary + 1:].find(
cls.BINARY_MARKER) + start_binary + 1 + len(
cls.BINARY_MARKER)
fields.append(bytes_text[:end_binary])
bytes_text = bytes_text[end_binary:].strip()
else:
fields.append(bytes_text)
bytes_text = numpy.string_("")
break
return fields
@classmethod
def _analyseOneLoop(cls, fields, start_idx):
"""Processes one loop in the data extraction of the CIF file
:param list fields: list of all the words contained in the cif file
:param int start_idx: the starting index corresponding to the "loop_" key
:return: the list of loop dictionaries, the length of the data
extracted from the fields and the list of all the keys of the loop.
:rtype: tuple
"""
loop = []
keys = []
i = start_idx + 1
finished = False
while not finished:
if fields[i][0] == cls.UNDERSCORE:
keys.append(fields[i])
i += 1
else:
finished = True
data = []
while True:
if i >= len(fields):
break
elif len(fields[i]) == 0:
break
elif fields[i][0] == cls.UNDERSCORE:
break
elif fields[i] in (cls.LOOP, cls.STOP, cls.GLOBAL, cls.DATA,
cls.SAVE):
break
else:
data.append(fields[i])
i += 1
k = 0
if len(data) < len(keys):
element = {}
for j in keys:
if k < len(data):
element[j] = data[k]
else:
element[j] = cls.QUESTIONMARK
k += 1
loop.append(element)
else:
for i in range(len(data) // len(keys)):
element = {}
for j in keys:
element[j] = data[k]
k += 1
loop.append(element)
return loop, 1 + len(keys) + len(data), keys
##########################################
# everything needed to write a CIF file #
##########################################
def saveCIF(self,
_strFilename="test.cif",
linesep=os.linesep,
binary=False):
"""Transforms the CIF object in string then write it into the given file
:param _strFilename: the of the file to be written
:param linesep: line separation used (to force compatibility with windows/unix)
:param binary: Shall we write the data as binary (True only for imageCIF/CBF)
:return: None
"""
if binary:
mode = "wb"
else:
mode = "w"
try:
fFile = open(_strFilename, mode)
except IOError:
logger.error("Error during the opening of file for write: %s",
_strFilename)
return
fFile.write(self.tostring(_strFilename, linesep))
try:
fFile.close()
except IOError:
logger.error("Error during the closing of file for write: %s",
_strFilename)
def tostring(self, _strFilename=None, linesep=os.linesep):
"""
Converts a cif dictionnary to a string according to the CIF syntax.
:param str _strFilename: the name of the filename to be appended in the
header of the CIF file.
:param linesep: default line separation (can be '\\n' or '\\r\\n').
:return: a string that corresponds to the content of the CIF-file.
"""
lstStrCif = ["# " + i for i in __version__]
if "_chemical_name_common" in self:
t = self["_chemical_name_common"].split()[0]
elif _strFilename is not None:
t = os.path.splitext(os.path.split(
str(_strFilename).strip())[1])[0]
else:
t = ""
lstStrCif.append("data_%s" % (t))
# first of all get all the keys:
lKeys = list(self.keys())
lKeys.sort()
for key in lKeys[:]:
if key in self._ordered:
lKeys.remove(key)
self._ordered += lKeys
for sKey in self._ordered:
if sKey == "loop_":
continue
if sKey not in self:
self._ordered.remove(sKey)
logger.debug(
"Skipping key %s from ordered list as no more present in dict"
)
continue
sValue = str(self[sKey])
if sValue.find("\n") > -1: # should add value between ;;
lLine = [sKey, ";", sValue, ";", ""]
elif len(sValue.split()) > 1: # should add value between ''
sLine = "%s '%s'" % (sKey, sValue)
if len(sLine) > 80:
lLine = [str(sKey), sValue]
else:
lLine = [sLine]
else:
sLine = "%s %s" % (sKey, sValue)
if len(sLine) > 80:
lLine = [str(sKey), sValue]
else:
lLine = [sLine]
lstStrCif += lLine
if "loop_" in self:
for loop in self["loop_"]:
lstStrCif.append("loop_ ")
lKeys = loop[0]
llData = loop[1]
lstStrCif += [" %s" % (sKey) for sKey in lKeys]
for lData in llData:
sLine = " "
for key in lKeys:
sRawValue = lData[key]
if sRawValue.find(
"\n") > -1: # should add value between ;;
lstStrCif += [sLine, ";", str(sRawValue), ";"]
sLine = " "
else:
if len(sRawValue.split()
) > 1: # should add value between ''
value = "'%s'" % (sRawValue)
else:
value = str(sRawValue)
if len(sLine) + len(value) > 78:
lstStrCif += [sLine]
sLine = " " + value
else:
sLine += " " + value
lstStrCif.append(sLine)
lstStrCif.append("")
return linesep.join(lstStrCif)
def exists(self, sKey):
"""
Check if the key exists in the CIF and is non empty.
:param str sKey: CIF key
:param cif: CIF dictionary
:return: True if the key exists in the CIF dictionary and is non empty
:rtype: boolean
"""
bExists = False
if sKey in self:
if len(self[sKey]) >= 1:
if self[sKey][0] not in (self.QUESTIONMARK,
numpy.string_(".")):
bExists = True
return bExists
def existsInLoop(self, sKey):
"""
Check if the key exists in the CIF dictionary.
:param str sKey: CIF key
:param cif: CIF dictionary
:return: True if the key exists in the CIF dictionary and is non empty
:rtype: boolean
"""
if not self.exists(self.LOOP):
return False
bExists = False
if not bExists:
for i in self[self.LOOP]:
for j in i[0]:
if j == sKey:
bExists = True
return bExists
def loadCHIPLOT(self, _strFilename):
"""
Load the powder diffraction CHIPLOT file and returns the
pd_CIF dictionary in the object
:param str _strFilename: the name of the file to open
:return: the CIF object corresponding to the powder diffraction
:rtype: dictionary
"""
if not os.path.isfile(_strFilename):
errStr = "I cannot find the file %s" % _strFilename
logger.error(errStr)
raise IOError(errStr)
lInFile = open(_strFilename, "r").readlines()
self[
"_audit_creation_method"] = 'From 2-D detector using FIT2D and CIFfile'
self["_pd_meas_scan_method"] = "fixed"
self["_pd_spec_description"] = lInFile[0].strip()
try:
iLenData = int(lInFile[3])
except ValueError:
iLenData = None
lOneLoop = []
try:
f2ThetaMin = float(lInFile[4].split()[0])
last = ""
for sLine in lInFile[-20:]:
if sLine.strip() != "":
last = sLine.strip()
f2ThetaMax = float(last.split()[0])
limitsOK = True
except (ValueError, IndexError):
limitsOK = False
f2ThetaMin = 180.0
f2ThetaMax = 0
# print "limitsOK:", limitsOK
for sLine in lInFile[4:]:
sCleaned = sLine.split("#")[0].strip()
data = sCleaned.split()
if len(data) == 2:
if not limitsOK:
f2Theta = float(data[0])
if f2Theta < f2ThetaMin:
f2ThetaMin = f2Theta
if f2Theta > f2ThetaMax:
f2ThetaMax = f2Theta
lOneLoop.append({"_pd_meas_intensity_total": data[1]})
if not iLenData:
iLenData = len(lOneLoop)
assert (iLenData == len(lOneLoop))
self["_pd_meas_2theta_range_inc"] = "%.4f" % (
(f2ThetaMax - f2ThetaMin) / (iLenData - 1))
if self["_pd_meas_2theta_range_inc"] < 0:
self["_pd_meas_2theta_range_inc"] = abs(
self["_pd_meas_2theta_range_inc"])
tmp = f2ThetaMax
f2ThetaMax = f2ThetaMin
f2ThetaMin = tmp
self["_pd_meas_2theta_range_max"] = "%.4f" % f2ThetaMax
self["_pd_meas_2theta_range_min"] = "%.4f" % f2ThetaMin
self["_pd_meas_number_of_points"] = str(iLenData)
self[self.LOOP] = [[["_pd_meas_intensity_total"], lOneLoop]]
@staticmethod
def LoopHasKey(loop, key):
"Returns True if the key (string) exist in the array called loop" ""
try:
loop.index(key)
return True
except ValueError:
return False
def _splitCIF(cls, bytes_text):
"""
Separate the text in fields as defined in the CIF
:param bytes_text: the content of the CIF - file
:type bytes_text: 8-bit string (str in python2 or bytes in python3)
:return: list of all the fields of the CIF
:rtype: list
"""
fields = []
while True:
if len(bytes_text) == 0:
break
elif bytes_text[0] == cls.SINGLE_QUOTE:
idx = 0
finished = False
while not finished:
idx += 1 + bytes_text[idx + 1:].find(cls.SINGLE_QUOTE)
if idx >= len(bytes_text) - 1:
fields.append(bytes_text[1:-1].strip())
bytes_text = numpy.string_("")
finished = True
break
if bytes_text[idx + 1] in cls.BLANK:
fields.append(bytes_text[1:idx].strip())
tmp_text = bytes_text[idx + 1:]
bytes_text = tmp_text.strip()
finished = True
elif bytes_text[0] == cls.DOUBLE_QUOTE:
idx = 0
finished = False
while not finished:
idx += 1 + bytes_text[idx + 1:].find(cls.DOUBLE_QUOTE)
if idx >= len(bytes_text) - 1:
fields.append(bytes_text[1:-1].strip())
bytes_text = numpy.string_("")
finished = True
break
if bytes_text[idx + 1] in cls.BLANK:
fields.append(bytes_text[1:idx].strip())
tmp_text = bytes_text[idx + 1:]
bytes_text = tmp_text.strip()
finished = True
elif bytes_text[0] == cls.SEMICOLUMN:
if bytes_text[1:].strip().find(cls.BINARY_MARKER) == 0:
idx = bytes_text[32:].find(cls.BINARY_MARKER)
if idx == -1:
idx = 0
else:
idx += 32 + len(cls.BINARY_MARKER)
else:
idx = 0
finished = False
while not finished:
idx += 1 + bytes_text[idx + 1:].find(cls.SEMICOLUMN)
if bytes_text[idx - 1] in cls.EOL:
fields.append(bytes_text[1:idx - 1].strip())
tmp_text = bytes_text[idx + 1:]
bytes_text = tmp_text.strip()
finished = True
else:
res = bytes_text.split(None, 1)
if len(res) == 2:
first, second = bytes_text.split(None, 1)
if cls.isAscii(first):
fields.append(first)
bytes_text = second.strip()
continue
start_binary = bytes_text.find(cls.BINARY_MARKER)
if start_binary > 0:
end_binary = bytes_text[start_binary + 1:].find(
cls.BINARY_MARKER) + start_binary + 1 + len(
cls.BINARY_MARKER)
fields.append(bytes_text[:end_binary])
bytes_text = bytes_text[end_binary:].strip()
else:
fields.append(bytes_text)
bytes_text = numpy.string_("")
break
return fields
def _ensure_fill_value_valid(data, attributes):
# work around for netCDF4/scipy issue where _FillValue has the wrong type:
# https://github.com/Unidata/netcdf4-python/issues/271
if data.dtype.kind == "S" and "_FillValue" in attributes:
attributes["_FillValue"] = np.string_(attributes["_FillValue"])
def save(self, filename):
"""
Saves the detector description into a NeXus file, adapted from:
http://download.nexusformat.org/sphinx/classes/base_classes/NXdetector.html
Main differences:
* differentiate pixel center from pixel corner offsets
* store all offsets are ndarray according to slow/fast dimension (not x, y)
:param filename: name of the file on the disc
"""
if not io.h5py:
logger.error("h5py module missing: NeXus detectors not supported")
raise RuntimeError("H5py module is missing")
with io.Nexus(filename, "a") as nxs:
det_grp = nxs.new_detector(name=self.name.replace(" ", "_"))
det_grp["API_VERSION"] = numpy.string_(self.API_VERSION)
det_grp["IS_FLAT"] = self.IS_FLAT
det_grp["IS_CONTIGUOUS"] = self.IS_CONTIGUOUS
det_grp["pixel_size"] = numpy.array([self.pixel1, self.pixel2],
dtype=numpy.float32)
det_grp["force_pixel"] = self.force_pixel
det_grp["force_pixel"].attrs[
"info"] = "The detector class specifies the pixel size"
if self.max_shape is not None:
det_grp["max_shape"] = numpy.array(self.max_shape,
dtype=numpy.int32)
if self.shape is not None:
det_grp["shape"] = numpy.array(self.shape, dtype=numpy.int32)
if self.binning is not None:
det_grp["binning"] = numpy.array(self._binning,
dtype=numpy.int32)
if self.flatfield is not None:
dset = det_grp.create_dataset("flatfield",
data=self.flatfield,
compression="gzip",
compression_opts=9,
shuffle=True)
dset.attrs["interpretation"] = "image"
if self.darkcurrent is not None:
dset = det_grp.create_dataset("darkcurrent",
data=self.darkcurrent,
compression="gzip",
compression_opts=9,
shuffle=True)
dset.attrs["interpretation"] = "image"
if self.mask is not None:
dset = det_grp.create_dataset("mask",
data=self.mask,
compression="gzip",
compression_opts=9,
shuffle=True)
dset.attrs["interpretation"] = "image"
if not (self.uniform_pixel and self.IS_FLAT):
# Get ready for the worse case: 4 corner per pixel, position 3D: z,y,x
dset = det_grp.create_dataset("pixel_corners",
data=self.get_pixel_corners(),
compression="gzip",
compression_opts=9,
shuffle=True)
dset.attrs["interpretation"] = "vertex"
"line3": "SE3",
"triangle": "TR3",
"triangle6": "TR6",
"quad": "QU4",
"quad8": "QU8",
"tetra": "TE4",
"tetra10": "T10",
"hexahedron": "HE8",
"hexahedron20": "H20",
"pyramid": "PY5",
"pyramid13": "P13",
"wedge": "PE6",
"wedge15": "P15",
}
med_to_meshio_type = {v: k for k, v in meshio_to_med_type.items()}
numpy_void_str = np.string_("")
def read(filename):
import h5py
f = h5py.File(filename, "r")
# Mesh ensemble
mesh_ensemble = f["ENS_MAA"]
meshes = mesh_ensemble.keys()
if len(meshes) != 1:
raise ReadError("Must only contain exactly 1 mesh, found {}.".format(
len(meshes)))
mesh_name = list(meshes)[0]
mesh = mesh_ensemble[mesh_name]
def do_califa(outname='NGC4047',
gallist=['NGC4047'],
linelbl='co',
seq='smo7',
hexgrid=False,
allpix=False,
debug=False,
califa_natv='fitsdata',
califa_smo='fitsdata',
comom='../img_comom/fitsdata',
nfiles=5,
astrom='x',
ortpar='edge_leda.csv',
distpar='edge_califa.csv',
distcol='caDistP3d',
discard_cdmatrix=False):
"""
Extract Pipe3D products into an HDF5 database. This script assumes
there are 5 native resolution and 5 smoothed resolution files per galaxy.
Parameters
----------
outname : str
Prefix of the output filename
gallist : list of str
List of galaxy names
linelbl : str
Identifier for reference line in the CO FITS filenames
seq : str
Identifier, generally to indicate smoothing resolution for CO
hexgrid : boolean
True to sample on a hexagonal grid (experimental)
allpix : boolean
True to dump every pixel, otherwise every 3rd pixel in x and y is used.
debug : boolean
True to generate some additional output
califa_natv : str
Path to the directory where native res CALIFA FITS files reside
califa_smo : str
Path to the directory where smoothed res CALIFA FITS files reside
comom : str
Path to the directory where CO moments FITS files reside
nfiles : int
Number of Pipe3D files per galaxy. Should be 5 (old) or 1 (packed).
astrom : str
String at start of filename for native resolution images with astrometry.
This is ignored in nfiles=1.
ortpar : filename
Name of the EdgeTable which has LEDA orientation parameters for the sample
distpar : filename
Name of the EdgeTable which has distances for converting \Sigma_*.
distcol : str
Name of the distance column in 'distpar' to use. Default is 'caDistP3d'
taken from 'DL' column in get_proc_elines_CALIFA.csv.
discard_cdmatrix : boolean
True to discard CD matrix in CALIFA files. Use with care since this
relies on the CDELT1 and CDELT2 being correct.
"""
if allpix:
stride = [1, 1, 1]
else:
stride = [3, 3, 1]
# cuts for when to apply BD correction
hacut = 0.06 # 1e-16 erg / (cm2 s)
hbcut = 0.04 # 1e-16 erg / (cm2 s)
ahalo = 0 # mag
ahahi = 6 # mag
# FITS keywords important for astrometry
wcskeys = [
'CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CRPIX1', 'CRPIX2', 'CDELT1',
'CDELT2'
]
cdkeys = [
'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2', 'CD1_3', 'CD2_3', 'CD3_1', 'CD3_2',
'CD3_3'
]
dimkeys = ['NAXIS1', 'NAXIS2']
# Get the orientation parameters from LEDA
orttbl = EdgeTable(ortpar)
orttbl.add_index('Name')
# Get the distance from the CALIFA table
disttbl = EdgeTable(distpar)
disttbl.add_index('Name')
# Read the FITS data
# The columns to save are defined in fitsextract.py
prodtype = ['ELINES', 'SFH', 'SSP', 'indices', 'flux_elines']
leadstr = ['', '', '', 'indices.CS.', 'flux_elines.']
tailstr = ['.ELINES', '.SFH', '.SSP', '', '']
tailstr = [s + '.cube.fits.gz' for s in tailstr]
for i_prod, prod in enumerate(prodtype):
zsel, labels, units, nsel = getlabels(prod)
default_len = len(zsel)
rglist = []
smlist = []
if len(gallist) == 0:
raise RuntimeError('Error: gallist is empty!')
for gal in gallist:
print('\nWorking on galaxy {} product {} nsel={}'.format(
gal, prod, nsel))
# Generate output header using CO astrometry
cofile = os.path.join(
comom, gal + '.' + linelbl + '.' + seq + '_dil.snrpk.fits.gz')
if not os.path.exists(cofile):
print('####### Cannot find', cofile)
continue
cohd = fits.getheader(cofile)
# CALIFA files with x in name have optical astrometry
if nfiles == 5:
cafile = os.path.join(
califa_natv,
astrom + leadstr[i_prod] + gal + tailstr[i_prod])
else:
cafile = os.path.join(califa_natv,
gal + '.Pipe3D.cube.fits.gz')
if not os.path.exists(cafile):
print('####### Cannot find', cafile)
continue
if nfiles == 5:
hdus = fits.open(cafile, ignore_missing_end=True)
cahd = hdus[0].header
#cahd = fits.getheader(cafile, ignore_missing_end=True)
else:
hdus = fits.open(cafile)
# The header for the selected extension
cahd = hdus[hdus.index_of(prod)].header
# Blanking of CTYPE3 so that fitsextract treats as pseudocube
cahd['CTYPE3'] = ''
# Use HDU 0 'ORG_HDR' when possible
cahd0 = hdus[0].header
for key in cdkeys + wcskeys:
if key in cahd0.keys():
cahd[key] = cahd0[key]
# Set CDELT3 to 1 since this will be its value in template
for key in ['CDELT3', 'CD3_3']:
if key in cahd.keys():
cahd[key] = 1.
# Copy the CALIFA header and replace wcskeys with CO values
outhd = cahd.copy()
for key in dimkeys + wcskeys:
if key in cohd.keys():
outhd[key] = cohd[key]
# Need to discard CD matrix which would override the new wcskeys
if 'CDELT1' in cohd.keys() and 'CDELT2' in cohd.keys():
for key in cdkeys:
if key in outhd.keys():
del outhd[key]
# Optionally discard CD matrix in CALIFA files and fall back on CDELTs
if discard_cdmatrix:
for key in cdkeys:
if key in cahd.keys():
del cahd[key]
# First process the native resolution file (tab0) with astrometry
if nfiles == 5:
#hdu = fits.open(cafile, ignore_missing_end=True)[0]
hdu = hdus[0]
else:
hdu = hdus[hdus.index_of(prod)]
if debug:
print('\nINPUT', WCS(hdu.header))
print('\nOUTPUT', WCS(outhd))
newim = reproject_interp(hdu,
outhd,
order=0,
return_footprint=False)
nz = newim.shape[0]
if debug:
print('nz=', nz)
#fits.writeto(base.replace('fits','rg.fits'), newim, outhd, overwrite=True)
rglabels = [s + '_rg' for s in labels]
# Add smoothed Ha and Hb columns for extinction estimates
if prod == 'ELINES' or prod == 'flux_elines':
kernel = Gaussian2DKernel(3)
if prod == 'ELINES':
hb_idx = 5
ha_idx = 6
rglabels += ['Hbeta_sm3_rg', 'Halpha_sm3_rg']
outhd['DESC_20'] = ' Hbeta after 3as smooth'
outhd['DESC_21'] = ' Halpha after 3as smooth'
else:
hb_idx = 28
ha_idx = 45
rglabels += ['flux_Hbeta_sm3_rg', 'flux_Halpha_sm3_rg']
outhd['NAME408'] = ' Hbeta after 3as smooth'
outhd['NAME409'] = ' Halpha after 3as smooth'
hb_conv = convolve(newim[hb_idx, :, :],
kernel,
preserve_nan=True)
ha_conv = convolve(newim[ha_idx, :, :],
kernel,
preserve_nan=True)
newim = np.concatenate(
(newim, hb_conv[np.newaxis], ha_conv[np.newaxis]))
if len(zsel) == default_len:
zsel = list(zsel) + [nz, nz + 1]
if len(units) == default_len:
units += ['10^-16 erg cm^-2 s^-1', '10^-16 erg cm^-2 s^-1']
tab0 = fitsextract(newim,
header=outhd,
keepnan=True,
stride=stride,
bunit=units,
col_lbl=rglabels,
zselect=zsel,
ra_gc=15 * orttbl.loc[gal]['ledaRA'],
dec_gc=orttbl.loc[gal]['ledaDE'],
pa=orttbl.loc[gal]['ledaPA'],
inc=orttbl.loc[gal]['ledaAxIncl'],
ortlabel='LEDA',
first=True,
use_hexgrid=hexgrid)
gname = Column([np.string_(gal)] * len(tab0),
name='Name',
description='Galaxy Name')
tab0.add_column(gname, index=0)
rglist.append(tab0)
# Then process the smoothed file (tab1)
smofile = os.path.join(califa_smo,
leadstr[i_prod] + gal + tailstr[i_prod])
hdu = fits.open(smofile, ignore_missing_end=True)[0]
hdu.header = cahd
newim = reproject_interp(hdu,
outhd,
order=0,
return_footprint=False)
# if debug:
# fits.writeto(base.replace('fits','sm.fits'), newim, outhd, overwrite=True)
smlabels = [s + '_sm' for s in labels]
# Add smoothed Ha and Hb for extinction estimates
if prod == 'ELINES' or prod == 'flux_elines':
kernel = Gaussian2DKernel(5)
if prod == 'ELINES':
hb_idx = 5
ha_idx = 6
smlabels += ['Hbeta_sm5_sm', 'Halpha_sm5_sm']
outhd['DESC_20'] = ' Hbeta after 5as smooth'
outhd['DESC_21'] = ' Halpha after 5as smooth'
else:
hb_idx = 28
ha_idx = 45
smlabels += ['flux_Hbeta_sm5_sm', 'flux_Halpha_sm5_sm']
outhd['NAME408'] = ' Hbeta after 5as smooth'
outhd['NAME409'] = ' Halpha after 5as smooth'
hb_conv = convolve(newim[hb_idx, :, :],
kernel,
preserve_nan=True)
ha_conv = convolve(newim[ha_idx, :, :],
kernel,
preserve_nan=True)
newim = np.concatenate(
(newim, hb_conv[np.newaxis], ha_conv[np.newaxis]))
tab1 = fitsextract(newim,
header=outhd,
keepnan=True,
stride=stride,
bunit=units,
col_lbl=smlabels,
zselect=zsel,
ra_gc=15 * orttbl.loc[gal]['ledaRA'],
dec_gc=orttbl.loc[gal]['ledaDE'],
pa=orttbl.loc[gal]['ledaPA'],
inc=orttbl.loc[gal]['ledaAxIncl'],
ortlabel='LEDA',
first=True,
use_hexgrid=hexgrid)
gname = Column([np.string_(gal)] * len(tab1),
name='Name',
description='Galaxy Name')
tab1.add_column(gname, index=0)
smlist.append(tab1)
# Add additional columns
if prod == 'ELINES' or prod == 'flux_elines':
if prod == 'ELINES':
prfx = ''
else:
prfx = 'flux_'
#
# Native resolution
# sfr0 is SFR from Halpha without extinction correction
sfr0_rg = sfr_ha(tab0[prfx + 'Halpha_rg'],
imf='salpeter',
name=prfx + 'sigsfr0_rg')
e_sfr0_rg = Column(
sfr0_rg * abs(tab0['e_' + prfx + 'Halpha_rg'] /
tab0[prfx + 'Halpha_rg']),
name='e_' + prfx + 'sigsfr0_rg',
dtype='f4',
unit=sfr0_rg.unit,
description='error of uncorrected SFR surface density')
tab0.add_columns([sfr0_rg, e_sfr0_rg])
# Balmer decrement corrected SFR
sfr_rg, sfrext_rg, e_sfr_rg, e_sfrext_rg = sfr_ha(
tab0[prfx + 'Halpha_rg'],
flux_hb=tab0[prfx + 'Hbeta_rg'],
e_flux_ha=tab0['e_' + prfx + 'Halpha_rg'],
e_flux_hb=tab0['e_' + prfx + 'Hbeta_rg'],
imf='salpeter',
name=prfx + 'sigsfr_corr_rg')
tab0.add_columns([sfr_rg, e_sfr_rg, sfrext_rg, e_sfrext_rg])
# Halpha extinction and SFR after 3" smoothing and clipping
A_Ha3_rg = Column(get_AHa(tab0[prfx + 'Halpha_sm3_rg'],
tab0[prfx + 'Hbeta_sm3_rg'],
np.log10),
name=prfx + 'AHa_smooth3_rg',
dtype='f4',
unit='mag',
description='Ha extinction after 3as smooth')
clip = ((tab0[prfx + 'Halpha_sm3_rg'] < hacut) |
(tab0[prfx + 'Hbeta_sm3_rg'] < hbcut) |
(A_Ha3_rg > ahahi) | (A_Ha3_rg < ahalo))
sfr3_rg = Column(
sfr0_rg * 10**(0.4 * A_Ha3_rg),
name=prfx + 'sigsfr_adopt_rg',
dtype='f4',
unit=sfr0_rg.unit,
description='smooth+clip BD corrected SFR surface density')
sfr3_rg[clip] = sfr0_rg[clip]
# A_Ha3_rg[clip] = np.nan
tab0.add_columns([A_Ha3_rg, sfr3_rg])
#
# Smoothed resolution
# sfr0 is SFR from Halpha without extinction correction
sfr0_sm = sfr_ha(tab1[prfx + 'Halpha_sm'],
imf='salpeter',
name=prfx + 'sigsfr0_sm')
e_sfr0_sm = Column(
sfr0_sm * abs(tab1['e_' + prfx + 'Halpha_sm'] /
tab1[prfx + 'Halpha_sm']),
name='e_' + prfx + 'sigsfr0_sm',
dtype='f4',
unit=sfr0_sm.unit,
description='error of uncorrected SFR surface density')
tab1.add_columns([sfr0_sm, e_sfr0_sm])
# Balmer decrement corrected SFR
sfr_sm, sfrext_sm, e_sfr_sm, e_sfrext_sm = sfr_ha(
tab1[prfx + 'Halpha_sm'],
flux_hb=tab1[prfx + 'Hbeta_sm'],
e_flux_ha=tab1['e_' + prfx + 'Halpha_sm'],
e_flux_hb=tab1['e_' + prfx + 'Hbeta_sm'],
imf='salpeter',
name=prfx + 'sigsfr_corr_sm')
tab1.add_columns([sfr_sm, e_sfr_sm, sfrext_sm, e_sfrext_sm])
# Halpha extinction and SFR after 5" smoothing and clipping
A_Ha5_sm = Column(get_AHa(tab1[prfx + 'Halpha_sm5_sm'],
tab1[prfx + 'Hbeta_sm5_sm'],
np.log10),
name=prfx + 'AHa_smooth5_sm',
dtype='f4',
unit='mag',
description='Ha extinction after 5as smooth')
clip = ((tab1[prfx + 'Halpha_sm5_sm'] < hacut) |
(tab1[prfx + 'Hbeta_sm5_sm'] < hbcut) |
(A_Ha5_sm > ahahi) | (A_Ha5_sm < ahalo))
sfr5_sm = Column(
sfr0_sm * 10**(0.4 * A_Ha5_sm),
name=prfx + 'sigsfr_adopt_sm',
dtype='f4',
unit=sfr0_rg.unit,
description='smooth+clip BD corrected SFR surface density')
sfr5_sm[clip] = sfr0_sm[clip]
# A_Ha5_sm[clip] = np.nan
tab1.add_columns([A_Ha5_sm, sfr5_sm])
#
# BPT requires flux_elines since EW(Ha) is part of classification
if prod == 'flux_elines':
BPT0, BPT0sf, p_BPT0 = bpt_type(tab0,
ext='_rg',
name='BPT_rg',
prob=True)
tab0.add_columns([BPT0, p_BPT0, BPT0sf])
BPT1, BPT1sf, p_BPT1 = bpt_type(tab1,
ext='_sm',
name='BPT_sm',
prob=True)
tab1.add_columns([BPT1, p_BPT1, BPT1sf])
#
zoh0, zoherr0 = ZOH_M13(tab0,
ext='_rg',
name='ZOH_rg',
err=True)
tab0.add_columns([zoh0, zoherr0])
zoh1, zoherr1 = ZOH_M13(tab1,
ext='_sm',
name='ZOH_sm',
err=True)
tab1.add_columns([zoh1, zoherr1])
elif prod == 'SSP':
# For stellar surface density we need distance
star0 = stmass_pc2(tab0['mass_ssp_rg'],
dz=tab0['cont_dezon_rg'],
dist=disttbl.loc[gal][distcol],
name='sigstar_rg')
star1 = stmass_pc2(tab1['mass_ssp_sm'],
dz=tab1['cont_dezon_sm'],
dist=disttbl.loc[gal][distcol],
name='sigstar_sm')
avstar0 = stmass_pc2(tab0['mass_Avcor_ssp_rg'],
dz=tab0['cont_dezon_rg'],
dist=disttbl.loc[gal][distcol],
name='sigstar_Avcor_rg')
avstar0.description += ' dust corrected'
avstar1 = stmass_pc2(tab1['mass_Avcor_ssp_sm'],
dz=tab1['cont_dezon_sm'],
dist=disttbl.loc[gal][distcol],
name='sigstar_Avcor_sm')
avstar1.description += ' dust corrected'
ferr0 = Column(
abs(tab0['e_medflx_ssp_rg'] / tab0['medflx_ssp_rg']),
name='fe_medflx_rg',
dtype='f4',
unit='fraction',
description='fractional error in continuum flux')
ferr1 = Column(
abs(tab1['e_medflx_ssp_sm'] / tab1['medflx_ssp_sm']),
name='fe_medflx_sm',
dtype='f4',
unit='fraction',
description='fractional error in continuum flux')
tab0.add_columns([star0, avstar0, ferr0])
tab1.add_columns([star1, avstar1, ferr1])
if len(rglist) > 0:
rg_merge = vstack(rglist)
rg_merge.meta['date'] = datetime.today().strftime('%Y-%m-%d')
if debug:
print(rg_merge.colnames)
print('There are', len(rg_merge), 'rows in native table')
if len(smlist) > 0:
sm_merge = vstack(smlist)
sm_merge.meta['date'] = datetime.today().strftime('%Y-%m-%d')
if debug:
print(sm_merge.colnames)
print('There are', len(sm_merge), 'rows in smoothed table')
if prod == prodtype[0]:
rg_merge.write(outname + '.pipe3d.hdf5',
path=prod + '_rg',
overwrite=True,
serialize_meta=True,
compression=True)
else:
rg_merge.write(outname + '.pipe3d.hdf5',
path=prod + '_rg',
append=True,
serialize_meta=True,
compression=True)
sm_merge.write(outname + '.pipe3d.hdf5',
path=prod + '_sm',
append=True,
serialize_meta=True,
compression=True)
return
img_names = []
for i, img_path in tqdm(enumerate(gallery)):
feature = extractFeatures_fc(img_path,model)
img_name = os.path.split(img_path)[1]
feature /= np.linalg.norm(feature)
features.append(feature)
img_names.append(img_name)
features = np.array(features)
features = np.squeeze(features)
return features, img_names
rmac_features, rmac_img_names = get_feature_map_rmac()
h5f = h5py.File('/content/drive/Shared drives/my_new_pan/4096-bn-rmac', 'w')
h5f.create_dataset('dataset_1', data=rmac_features)
h5f.create_dataset('dataset_2', data=np.string_(rmac_img_names))
h5f.close()
fc_features, fc_img_names = get_feature_map_fc()
h5f = h5py.File('/content/drive/Shared drives/my_new_pan/4096-bn-rmac', 'w')
h5f.create_dataset('dataset_1', data=fc_features)
h5f.create_dataset('dataset_2', data=np.string_(fc_img_names))
h5f.close()
query = get_imlist('test_data_A/query')
len(query)
def get_query_feature_map_rmac():
#first step inception
