def getSubMatricesFromFits(ccdData,
expID,
ccd,
n=6,
perc=100,
ccdOption='name'):
subMatrices = []
# try to split in n submatrices
## best behaviour with n=6
### if not feasible, raise warning and skip this ccd from this expID
try:
left, right = np.hsplit(ccdData, 2)
if getConfig("Image",
"ccdByNumOrName") == 'name' and ccd.startswith("F"):
heightDiv = n / 3 # heightDiv = 2
else:
heightDiv = n / 2 # heightDiv = 3
subMatrices = np.append(np.asarray(np.vsplit(left, heightDiv)),
np.asarray(np.vsplit(right, heightDiv)),
axis=0)
except:
failSplitMsg = "Failed to split CCD {} from expID {} into submatrices. Skipping this CCD.".format(
ccd, expID)
logging.warning(failSplitMsg)
for subMatIndex, subMatrix in enumerate(subMatrices):
# fits_io.maskFits performs the image clip (border drop)
## if masking (keeping the same size), replace "--" with median value
if not getConfig("Image", "maskBorder"):
subMatrix[subMatrix == "--"] = np.median(subMatrix)
saveSubMatrix(subMatrix, expID, ccd, subMatIndex, perc)
return subMatrices
def maskFits(ccdData, ccd):
maskBorder = getConfig("Image", "maskBorder")
if maskBorder:
maskBorder = int(maskBorder)
return ccdData[maskBorder:-maskBorder, maskBorder:-maskBorder]
else:
mask = getMask(ccdData, ccd, getConfig("Paths", "maskPath"),
getConfig("Image", "maskExtension"))
return ma.masked_array(ccdData, mask=mask, copy=False)
def getListOfCompressedFiles():
inputPath = getConfig("Paths", "inputPath")
if (not inputPath):
logging.critical("inputPath NOT defined! Exiting.")
exit()
singleFile = getConfig("InputFiles", "singleFZFile")
if (singleFile):
return [inputPath + singleFile]
return glob(inputPath + "*.fits.fz")
def getTrainSize(groupedList):
trainFrac = getConfig("Training","trainSizeFraction")
trainFrac = float(trainFrac) if trainFrac else 0.9 # default = 0.9
trainFracMsg = "trainFrac: {}".format(trainFrac)
logging.debug(trainFracMsg)
return int(trainFrac * len(groupedList))
def createLabelMap(outDataDir):
labelMapFileName = getConfig("Training","labelMapFile")
if labelMapFileName:
labelMapFileName = outDataDir + labelMapFileName
else:
labelMapFileName = outDataDir + 'labelmap.pbtxt'
content = "item {{\n id: 1\n name: '{0}'\n}}".format(LABEL)
with open(labelMapFileName, "w") as labelMapFile:
labelMapFile.write(content)
def checkOutputPath():
outputPath = getConfig("Paths", "outputPath")
if not path.isdir(outputPath):
try:
makedirs(outputPath)
except OSError as exc: # Python >2.5 # Python >3 has exist_ok flag
if exc.errno == errno.EEXIST and path.isdir(outputPath):
pass
else:
raise
def getDfFileNames(outDataDir):
totalFile = getConfig("Training", "totalFile")
totalFile = totalFile if totalFile else "totalDF.csv"
trainFile = getConfig("Training", "trainFile")
trainFile = trainFile if trainFile else "train.csv"
testFile = getConfig("Training", "testFile")
testFile = testFile if testFile else "test.csv"
totalMsg = "totalFile: {}".format(totalFile)
trainMsg = "trainFile: {}".format(trainFile)
testMsg = "testFile: {}".format(testFile)
logging.debug(totalMsg)
logging.debug(trainMsg)
logging.debug(testMsg)
files = [totalFile, trainFile, testFile]
return [outDataDir+file for file in files]
def getCCDNames(fitsFileName):
ccdOption = getConfig("Image", "ccdByNumOrName")
ccdInfo = 'CCDNUM' if ccdOption == 'num' else 'EXTNAME'
try:
with fits.open(fitsFileName) as fitsFile:
return [
fitsFile[i].header[ccdInfo] for i in range(1, len(fitsFile))
]
except:
failMsg = "Unexpected error: {}".format(exc_info()[0])
logging.critical(failMsg)
exit()
def getMask(ccdData, ccd, maskPath, maskExtension):
fullPathMaskFile = maskPath + ccd + "." + maskExtension
if not path.isfile(fullPathMaskFile):
maskMsg = "maskPath or CCD {} or extension {} NOT found. Creating mask.".format(
ccd, maskExtension)
logging.info(maskMsg)
maskBorder = getConfig("Image", "maskBorder")
try:
maskBorder = int(maskBorder) if maskBorder else maskBorder
return createMask(ccdData, maskBorder)
except:
return createMask(ccdData)
return loadMaskForCCD(fullPathMaskFile)
def getTotalDF():
parentDir = getConfig("Training","parentDirWithCSVs")
filesInParentDir = glob(parentDir+"*.csv")
if filesInParentDir:
logging.info('parentDirWithCSVs has at least one csv file. Return dataframe with csv(s) in this level.')
return pd.concat([pd.read_csv(dfFile) for dfFile in filesInParentDir])
logging.info('parentDirWithCSVs does not have csv files in this level. Return dataframe with csv in children\'s level.')
childrenDFs = []
for childDir in glob(parentDir+"*/"):
childCSVs = glob(childDir+"*.csv")
if childCSVs:
childrenDFs.append(pd.concat([pd.read_csv(childFile) for childFile in childCSVs]))
return pd.concat(childrenDFs, ignore_index=True)
def createDataDir():
outputPath = getConfig("Training","parentDirWithCSVs")
outDataDir = outputPath+"data/"
if not path.isdir(outDataDir):
makedirs(outDataDir)
return outDataDir
def getSubMatrixFileName(expID, ccd, subMatIndex):
separator = "_"
return getConfig("Paths", "outputPath") + separator.join(
[expID, ccd, str(subMatIndex)])