def convertImageFormat(image: bytes, quality: Optional[int] = 80) -> bytes:
"""Convert picture format to solve the problem of unrecognizable pictures
Parameters
----------
image : bytes
Read out the bytes of the image
quality : int, optional
Image compression quality, by default 80
Returns
-------
bytes
Returns the converted picture bytes
"""
from .tmpFile import tmpFile
quality = quality or 80
with tmpFile() as file1, tmpFile() as file2:
with open(file1, "wb") as f:
f.write(image)
with Image.open(file1) as f:
f.save(file2, "BMP")
for i in reversed(range(quality, 100, 5)):
with Image.open(file2) as f:
f.save(file1, "PNG", optimize=True, quality=i)
if getFileSize(file1) <= MAX_IMAGE_SIZE:
break
with open(file1, "rb") as f:
readData = f.read()
return readData + b"\x00" * 16 + token_bytes(16)
def read_gals(self, model_name, first_file, last_file, thissnap):
# The input galaxy structure:
Galdesc_full = [('SnapNum', np.int32), ('Type', np.int16),
('isFlyby', np.int16), ('GalaxyIndex', np.int64),
('CentralGalaxyIndex', np.int64),
('SAGEHaloIndex', np.int32),
('SAGETreeIndex', np.int32),
('SimulationHaloIndex', np.int64),
('mergeType', np.int32), ('mergeIntoID', np.int32),
('mergeIntoSnapNum', np.int32), ('dT', np.float32),
('Pos', (np.float32, 3)), ('Vel', (np.float32, 3)),
('Spin', (np.float32, 3)), ('Len', np.int32),
('Mvir', np.float32), ('CentralMvir', np.float32),
('Rvir', np.float32), ('Vvir', np.float32),
('Vmax', np.float32), ('VelDisp', np.float32),
('ColdGas', np.float32), ('StellarMass', np.float32),
('BulgeMass', np.float32), ('HotGas', np.float32),
('EjectedMass', np.float32),
('BlackHoleMass', np.float32),
('IntraClusterStars', np.float32),
('MetalsColdGas', np.float32),
('MetalsStellarMass', np.float32),
('MetalsBulgeMass', np.float32),
('MetalsHotGas', np.float32),
('MetalsEjectedMass', np.float32),
('MetalsIntraClusterStars', np.float32),
('SfrDisk', np.float32), ('SfrBulge', np.float32),
('SfrDiskZ', np.float32), ('SfrBulgeZ', np.float32),
('DiskRadius', np.float32), ('Cooling', np.float32),
('Heating', np.float32),
('QuasarModeBHaccretionMass', np.float32),
('TimeOfLastMajorMerger', np.float32),
('TimeOfLastMinorMerger', np.float32),
('OutflowRate', np.float32),
('infallMvir', np.float32), ('infallVvir', np.float32),
('infallVmax', np.float32)]
names = [Galdesc_full[i][0] for i in xrange(len(Galdesc_full))]
formats = [Galdesc_full[i][1] for i in xrange(len(Galdesc_full))]
Galdesc = np.dtype({'names': names, 'formats': formats}, align=True)
# Initialize variables.
TotNTrees = 0
TotNGals = 0
FileIndexRanges = []
goodfiles = 0
if thissnap in self.SMFsnaps:
print
print "Determining array storage requirements."
# Read each file and determine the total number of galaxies to be read in
for fnr in xrange(first_file, last_file + 1):
fname = model_name + '_' + str(fnr) # Complete filename
if not os.path.isfile(fname):
# print "File\t%s \tdoes not exist! Skipping..." % (fname)
continue
if getFileSize(fname) == 0:
print "File\t%s \tis empty! Skipping..." % (fname)
continue
fin = open(fname, 'rb') # Open the file
Ntrees = np.fromfile(fin, np.dtype(np.int32),
1) # Read number of trees in file
NtotGals = np.fromfile(fin, np.dtype(np.int32),
1)[0] # Read number of gals in file.
TotNTrees = TotNTrees + Ntrees # Update total sim trees number
TotNGals = TotNGals + NtotGals # Update total sim gals number
goodfiles = goodfiles + 1 # Update number of files read for volume calculation
fin.close()
print "Input files contain:\t%d trees ;\t%d galaxies ." % (
TotNTrees, TotNGals)
# Initialize the storage array
G = np.empty(TotNGals, dtype=Galdesc)
if thissnap in self.SMFsnaps:
offset = 0 # Offset index for storage array
# Open each file in turn and read in the preamble variables and structure.
print "Reading in files."
for fnr in xrange(first_file, last_file + 1):
fname = model_name + '_' + str(fnr) # Complete filename
if not os.path.isfile(fname):
continue
if getFileSize(fname) == 0:
continue
fin = open(fname, 'rb') # Open the file
Ntrees = np.fromfile(fin, np.dtype(np.int32),
1) # Read number of trees in file
NtotGals = np.fromfile(fin, np.dtype(np.int32),
1)[0] # Read number of gals in file.
GalsPerTree = np.fromfile(fin, np.dtype(
(np.int32,
Ntrees)), 1) # Read the number of gals in each tree
print ": Reading N=", NtotGals, " \tgalaxies from file: ", fname
GG = np.fromfile(fin, Galdesc,
NtotGals) # Read in the galaxy structures
FileIndexRanges.append((offset, offset + NtotGals))
# Slice the file array into the global array
# N.B. the copy() part is required otherwise we simply point to
# the GG data which changes from file to file
# NOTE THE WAY PYTHON WORKS WITH THESE INDICES!
G[offset:offset + NtotGals] = GG[0:NtotGals].copy()
del (GG)
offset = offset + NtotGals # Update the offset position for the global array
fin.close() # Close the file
print "Total galaxies considered:", TotNGals
print
# Convert the Galaxy array into a recarray
G = G.view(np.recarray)
# Calculate the volume given the first_file and last_file
self.volume = self.BoxSize**3.0 * goodfiles / self.MaxTreeFiles
return G
def read_gals(self, model_name, first_file, last_file, thissnap):
# The input galaxy structure:
Galdesc_full = [
('SnapNum' , np.int32),
('Type' , np.int32),
('GalaxyIndex' , np.int64),
('CentralGalaxyIndex' , np.int64),
('SAGEHaloIndex' , np.int32),
('SAGETreeIndex' , np.int32),
('SimulationFOFHaloIndex' , np.int32),
('mergeType' , np.int32),
('mergeIntoID' , np.int32),
('mergeIntoSnapNum' , np.int32),
('dT' , np.float32),
('Pos' , (np.float32, 3)),
('Vel' , (np.float32, 3)),
('Spin' , (np.float32, 3)),
('Len' , np.int32),
('Mvir' , np.float32),
('CentralMvir' , np.float32),
('Rvir' , np.float32),
('Vvir' , np.float32),
('Vmax' , np.float32),
('VelDisp' , np.float32),
('ColdGas' , np.float32),
('StellarMass' , np.float32),
('BulgeMass' , np.float32),
('HotGas' , np.float32),
('EjectedMass' , np.float32),
('BlackHoleMass' , np.float32),
('IntraClusterStars' , np.float32),
('MetalsColdGas' , np.float32),
('MetalsStellarMass' , np.float32),
('MetalsBulgeMass' , np.float32),
('MetalsHotGas' , np.float32),
('MetalsEjectedMass' , np.float32),
('MetalsIntraClusterStars' , np.float32),
('SfrDisk' , np.float32),
('SfrBulge' , np.float32),
('SfrDiskZ' , np.float32),
('SfrBulgeZ' , np.float32),
('DiskRadius' , np.float32),
('Cooling' , np.float32),
('Heating' , np.float32),
('r_heat' , np.float32),
('QuasarModeBHaccretionMass' , np.float32),
('TimeSinceMajorMerger' , np.float32),
('TimeSinceMinorMerger' , np.float32),
('OutflowRate' , np.float32),
('infallMvir' , np.float32),
('infallVvir' , np.float32),
('infallVmax' , np.float32),
('Qjet' , np.float32),
('Rcocoon' , np.float32),
('Rshocked' , np.float32),
('t_AGN_returne' , np.float32),
('t_AGN_on' , np.float32),
('Tshocked' , np.float32),
('Mshocked' , np.float32),
('RadioLuminosity' , (np.float32,7)),
('RadioAGNaccretionRate' , np.float32),
('rho_zero_Makino' , np.float32),
('rho_zero_Capelo' , np.float32),
('rho_zero_iso' , np.float32),
('b_gas' , np.float32),
('Rs' , np.float32),
('concentration' , np.float32),
('Temp_Gas' , np.float32),
('Lx_bol' , np.float32),
('R_index' , np.float32),
('Q_index' , np.float32),
('R_cool' , np.float32),
('fcool' , np.float32),
('t_static' , np.float32),
('t_AGN_off' , np.float32),
('time_to_next_on' , np.float32),
('delta' , np.float32)
]
names = [Galdesc_full[i][0] for i in xrange(len(Galdesc_full))]
formats = [Galdesc_full[i][1] for i in xrange(len(Galdesc_full))]
Galdesc = np.dtype({'names':names, 'formats':formats}, align=True)
# Initialize variables.
TotNTrees = 0
TotNGals = 0
FileIndexRanges = []
goodfiles = 0
if thissnap in self.SMFsnaps:
print
print "Determining array storage requirements."
# Read each file and determine the total number of galaxies to be read in
for fnr in xrange(first_file,last_file+1):
fname = model_name+'_'+str(fnr) # Complete filename
if not os.path.isfile(fname):
# print "File\t%s \tdoes not exist! Skipping..." % (fname)
continue
if getFileSize(fname) == 0:
print "File\t%s \tis empty! Skipping..." % (fname)
continue
fin = open(fname, 'rb') # Open the file
Ntrees = np.fromfile(fin,np.dtype(np.int32),1) # Read number of trees in file
NtotGals = np.fromfile(fin,np.dtype(np.int32),1)[0] # Read number of gals in file.
TotNTrees = TotNTrees + Ntrees # Update total sim trees number
TotNGals = TotNGals + NtotGals # Update total sim gals number
goodfiles = goodfiles + 1 # Update number of files read for volume calculation
fin.close()
print "Input files contain:\t%d trees ;\t%d galaxies ." % (TotNTrees, TotNGals)
# Initialize the storage array
G = np.empty(TotNGals, dtype=Galdesc)
if thissnap in self.SMFsnaps:
offset = 0 # Offset index for storage array
# Open each file in turn and read in the preamble variables and structure.
print "Reading in files."
for fnr in xrange(first_file,last_file+1):
fname = model_name+'_'+str(fnr) # Complete filename
if not os.path.isfile(fname):
continue
if getFileSize(fname) == 0:
continue
fin = open(fname, 'rb') # Open the file
Ntrees = np.fromfile(fin, np.dtype(np.int32), 1) # Read number of trees in file
NtotGals = np.fromfile(fin, np.dtype(np.int32), 1)[0] # Read number of gals in file.
GalsPerTree = np.fromfile(fin, np.dtype((np.int32, Ntrees)),1) # Read the number of gals in each tree
print ": Reading N=", NtotGals, " \tgalaxies from file: ", fname
GG = np.fromfile(fin, Galdesc, NtotGals) # Read in the galaxy structures
FileIndexRanges.append((offset,offset+NtotGals))
# Slice the file array into the global array
# N.B. the copy() part is required otherwise we simply point to
# the GG data which changes from file to file
# NOTE THE WAY PYTHON WORKS WITH THESE INDICES!
G[offset:offset+NtotGals]=GG[0:NtotGals].copy()
del(GG)
offset = offset + NtotGals # Update the offset position for the global array
fin.close() # Close the file
print "Total galaxies considered:", TotNGals
print
# Convert the Galaxy array into a recarray
G = G.view(np.recarray)
# Calculate the volume given the first_file and last_file
self.volume = self.BoxSize**3.0 * goodfiles / self.MaxTreeFiles
return G
def read_gals(self, model_name, first_file, last_file):
# The input galaxy structure:
floattypef = np.float32
Galdesc_full = [
('Type', np.int32), ('GalaxyIndex', np.int64),
('HaloIndex', np.int32), ('SimulationHaloIndex', np.int32),
('TreeIndex', np.int32), ('SnapNum', np.int32),
('CentralGalaxyIndex', np.int64), ('CentralMvir', floattypef),
('mergeType', np.int32), ('mergeIntoID', np.int32),
('mergeIntoSnapNum', np.int32), ('dT', floattypef),
('Pos', (floattypef, 3)), ('Vel', (floattypef, 3)),
('Spin', (floattypef, 3)), ('Len', np.int32), ('LenMax', np.int32),
('Mvir', floattypef), ('Rvir', floattypef), ('Vvir', floattypef),
('Vmax', floattypef), ('VelDisp', floattypef),
('DiscRadii', (floattypef, 31)), ('ColdGas', floattypef),
('StellarMass', floattypef), ('ClassicalBulgeMass', floattypef),
('SecularBulgeMass', floattypef), ('HotGas', floattypef),
('EjectedMass', floattypef), ('BlackHoleMass', floattypef),
('IntraClusterStars', floattypef), ('DiscGas', (floattypef, 30)),
('DiscStars', (floattypef, 30)), ('SpinStars', (floattypef, 3)),
('SpinGas', (floattypef, 3)),
('SpinClassicalBulge', (floattypef, 3)),
('StarsInSitu', floattypef), ('StarsInstability', floattypef),
('StarsMergeBurst', floattypef), ('DiscHI', (floattypef, 30)),
('DiscH2', (floattypef, 30)), ('DiscSFR', (floattypef, 30)),
('MetalsColdGas', floattypef), ('MetalsStellarMass', floattypef),
('ClassicalMetalsBulgeMass', floattypef),
('SecularMetalsBulgeMass', floattypef),
('MetalsHotGas', floattypef), ('MetalsEjectedMass', floattypef),
('MetalsIntraClusterStars', floattypef),
('DiscGasMetals', (floattypef, 30)),
('DiscStarsMetals', (floattypef, 30)), ('SfrDisk', floattypef),
('SfrBulge', floattypef), ('SfrDiskZ', floattypef),
('SfrBulgeZ', floattypef), ('DiskScaleRadius', floattypef),
('Cooling', floattypef), ('Heating', floattypef),
('LastMajorMerger', floattypef), ('LastMinorMerger', floattypef),
('OutflowRate', floattypef), ('infallMvir', floattypef),
('infallVvir', floattypef), ('infallVmax', floattypef)
]
names = [Galdesc_full[i][0] for i in xrange(len(Galdesc_full))]
formats = [Galdesc_full[i][1] for i in xrange(len(Galdesc_full))]
Galdesc = np.dtype({'names': names, 'formats': formats}, align=True)
# Initialize variables.
TotNTrees = 0
TotNGals = 0
FileIndexRanges = []
print("Determining array storage requirements.")
# Read each file and determine the total number of galaxies to be read in
goodfiles = 0
for fnr in xrange(first_file, last_file + 1):
fname = model_name + '_' + str(fnr) # Complete filename
if not os.path.isfile(fname):
# print "File\t%s \tdoes not exist! Skipping..." % (fname)
continue
if getFileSize(fname) == 0:
print("File\t%s \tis empty! Skipping..." % (fname))
continue
fin = open(fname, 'rb') # Open the file
Ntrees = np.fromfile(fin, np.dtype(np.int32),
1) # Read number of trees in file
NtotGals = np.fromfile(fin, np.dtype(np.int32),
1)[0] # Read number of gals in file.
TotNTrees = TotNTrees + Ntrees # Update total sim trees number
TotNGals = TotNGals + NtotGals # Update total sim gals number
goodfiles = goodfiles + 1 # Update number of files read for volume calculation
fin.close()
print()
print("Input files contain:\t%d trees ;\t%d galaxies ." %
(TotNTrees, TotNGals))
print()
# Initialize the storage array
G = np.empty(TotNGals, dtype=Galdesc)
offset = 0 # Offset index for storage array
# Open each file in turn and read in the preamble variables and structure.
print("Reading in files.")
for fnr in xrange(first_file, last_file + 1):
fname = model_name + '_' + str(fnr) # Complete filename
if not os.path.isfile(fname):
continue
if getFileSize(fname) == 0:
continue
fin = open(fname, 'rb') # Open the file
Ntrees = np.fromfile(fin, np.dtype(np.int32),
1) # Read number of trees in file
NtotGals = np.fromfile(fin, np.dtype(np.int32),
1)[0] # Read number of gals in file.
GalsPerTree = np.fromfile(fin, np.dtype(
(np.int32, Ntrees)), 1) # Read the number of gals in each tree
print(": Reading N=", NtotGals, " \tgalaxies from file: ",
fname)
GG = np.fromfile(fin, Galdesc,
NtotGals) # Read in the galaxy structures
FileIndexRanges.append((offset, offset + NtotGals))
# Slice the file array into the global array
# N.B. the copy() part is required otherwise we simply point to
# the GG data which changes from file to file
# NOTE THE WAY PYTHON WORKS WITH THESE INDICES!
G[offset:offset + NtotGals] = GG[0:NtotGals].copy()
del (GG)
offset = offset + NtotGals # Update the offset position for the global array
fin.close() # Close the file
print()
print("Total galaxies considered:", TotNGals)
# Convert the Galaxy array into a recarray
G = G.view(np.recarray)
w = np.where(G.StellarMass > 1.0)[0]
print("Galaxies more massive than 10^10Msun/h:", len(w))
print()
# Calculate the volume given the first_file and last_file
self.volume = self.BoxSize**3.0 * goodfiles / self.MaxTreeFiles
# w = np.where(G.TreeIdx == 8)[0]
# for i in xrange(len(w)):
# print i, G.TreeIdx[w[i]], G.Type[w[i]], G.GalaxyIndex[w[i]], G.mergeType[w[i]], G.mergeIntoID[w[i]], G.mergeIntoSnapNum[w[i]]
return G