def writeMetricsToXML(dMetricsArg, visitID, sourceDir, xmlFilePath,
modelEngineRootNode, modelVersion):
log = Logger(__name__)
tree = ET.ElementTree(ET.Element(modelEngineRootNode))
nodMeta = ET.SubElement(tree.getroot(), 'Meta')
datetag = ET.SubElement(nodMeta, DATECREATEDFIELD)
datetag.text = datetime.datetime.now().isoformat()
version = ET.SubElement(nodMeta, VERSIONFIELD)
version.text = modelVersion
nodVisitID = ET.SubElement(nodMeta, 'VisitID')
nodVisitID.text = str(visitID)
nodMetrics = ET.SubElement(tree.getroot(), 'Metrics')
_writeDictionaryToXML(nodMetrics, dMetricsArg)
rough_string = ET.tostring(tree.getroot(), 'utf-8')
reparsed = xml.dom.minidom.parseString(rough_string)
pretty = reparsed.toprettyxml(indent="\t")
with open(xmlFilePath, "wb") as f:
f.write(pretty)
log.info("Wrote metrics to file: {}".format(xmlFilePath))
def __init__(self, message):
# Call the base class constructor with the parameters it needs
super(MissingException, self).__init__(message)
self.returncode = 4
logg = Logger('MissingException')
logg.error(message)
class Raster:
def __init__(self, sfilename):
self.filename = sfilename
self.log = Logger("Raster")
self.errs = ""
try:
if (path.isfile(self.filename)):
src_ds = gdal.Open( self.filename )
else:
self.log.error("Missing file: {}".format(self.filename))
raise MissingException("Could not find raster file: {}".format(path.basename(self.filename)))
except RuntimeError, e:
raise DataException("Raster file exists but has problems: {}".format(path.basename(self.filename)))
try:
# Read Raster Properties
srcband = src_ds.GetRasterBand(1)
self.bands = src_ds.RasterCount
self.driver = src_ds.GetDriver().LongName
self.gt = src_ds.GetGeoTransform()
self.nodata = srcband.GetNoDataValue()
""" Turn a Raster with a single band into a 2D [x,y] = v array """
self.array = srcband.ReadAsArray()
# Now mask out any NAN or nodata values (we do both for consistency)
if self.nodata is not None:
# To get over the issue where self.nodata may be imprecisely set we may need to use the array's
# true nodata, taken directly from the array
workingNodata = self.nodata
self.min = np.nanmin(self.array)
if isclose(self.min, self.nodata, rel_tol=1e-03):
workingNodata = self.min
self.array = np.ma.array(self.array, mask=(np.isnan(self.array) | (self.array == workingNodata)))
self.dataType = srcband.DataType
self.min = np.nanmin(self.array)
self.max = np.nanmax(self.array)
self.proj = src_ds.GetProjection()
# Remember:
# [0]/* top left x */
# [1]/* w-e pixel resolution */
# [2]/* rotation, 0 if image is "north up" */
# [3]/* top left y */
# [4]/* rotation, 0 if image is "north up" */
# [5]/* n-s pixel resolution */
self.left = self.gt[0]
self.cellWidth = self.gt[1]
self.top = self.gt[3]
self.cellHeight = self.gt[5]
self.cols = src_ds.RasterXSize
self.rows = src_ds.RasterYSize
# Important to throw away the srcband
srcband.FlushCache()
srcband = None
except RuntimeError as e:
print('Could not retrieve meta Data for %s' % self.filepath, e)
raise e
def __init__(self, *args, **kwargs):
# First initialize our logger and our empty metrics
self.log = Logger(self.__module__)
self.metrics = deepcopy(self.TEMPLATE)
# Now try a calculation
try:
self.calc(*args, **kwargs)
except (DataException, MissingException), e:
self.log.warning("Could not complete metric calculation")
self.metrics = deepcopy(self.TEMPLATE)
def __init__(self, sfilename):
self.filename = sfilename
self.log = Logger("Raster")
self.errs = ""
try:
if (path.isfile(self.filename)):
src_ds = gdal.Open( self.filename )
else:
self.log.error("Missing file: {}".format(self.filename))
raise MissingException("Could not find raster file: {}".format(path.basename(self.filename)))
except RuntimeError, e:
raise DataException("Raster file exists but has problems: {}".format(path.basename(self.filename)))
def __init__(self):
super(Tokenator, self).__init__()
if not self._initdone:
self.log = Logger("API:Tokenator")
self.log.info("Init Token Settings")
self.TOKEN = None
self.tokenfile = os.environ.get('KEYSTONE_TOKENFILE')
self.load()
if self.TOKEN is None:
self.getToken()
else:
self.log.debug("re-using security token")
self._initdone = True
else:
self.log.debug("re-using security token")
class CHaMPMetric(object):
# This is the dictionary representation of the object
# in its null state.
TEMPLATE = {}
def __init__(self, *args, **kwargs):
# First initialize our logger and our empty metrics
self.log = Logger(self.__module__)
self.metrics = deepcopy(self.TEMPLATE)
# Now try a calculation
try:
self.calc(*args, **kwargs)
except (DataException, MissingException), e:
self.log.warning("Could not complete metric calculation")
self.metrics = deepcopy(self.TEMPLATE)
def loadChannelUnitsFromAPI(vid):
apiUnits = APIGet('visits/{}/measurements/Channel Unit'.format(vid))
dUnits = {}
for nodUnit in apiUnits['values']:
value = nodUnit['value']
nCUNumber = int(value['ChannelUnitNumber'])
tier1 = value['Tier1']
tier2 = value['Tier2']
segment = value['ChannelSegmentID']
dUnits[nCUNumber] = (tier1, tier2, segment)
log = Logger("Channel Units")
log.info("{0} channel units loaded from XML file".format(len(dUnits)))
return dUnits
def __init__(self, sFilename=None):
self.filename = sFilename
self.driver = ogr.GetDriverByName("ESRI Shapefile")
self.datasource = None
self.fields = {}
self.features = {}
self.log = Logger("Shapefile")
self.loaded = False
if sFilename and os.path.isfile(sFilename):
self.load(sFilename)
self.loaded = True
def querychoices(title, options, question):
# cls()
if len(options) == 1:
return options[0]
log = Logger("querychoices")
log.title(title, "=")
for idx, cstr in enumerate(options):
log.info("({0}) {1}".format(idx + 1, cstr))
while True:
log.info(question + " [Choose One] ")
choice = raw_input()
try:
nchoice = int(choice.strip())
if nchoice < 1 or nchoice > len(options):
raise ValueError("Choice is out of range")
return options[nchoice - 1]
except ValueError as e:
log.warning("Please respond with a single integer value. \n")
def loadChannelUnitsFromXML(xmlFilePath):
if not path.isfile(xmlFilePath):
raise "Missing channel unit file at {0}".format(xmlFilePath)
tree = ET.parse(xmlFilePath)
nodRoot = tree.getroot()
dUnits = {}
for nodUnit in nodRoot.findall('Units/Unit'):
nCUNumber = int(nodUnit.find('ChannelUnitNumber').text)
tier1 = nodUnit.find('Tier1').text
tier2 = nodUnit.find('Tier2').text
segment = nodUnit.find('Segment').text
dUnits[nCUNumber] = (tier1, tier2, segment)
log = Logger("Channel Units")
log.info("{0} channel units loaded from XML file".format(len(dUnits)))
return dUnits
def query_yes_no(question, default="yes"):
"""Ask a yes/no question via raw_input() and return their answer.
"question" is a string that is presented to the user.
"default" is the presumed answer if the user just hits <Enter>.
It must be "yes" (the default), "no" or None (meaning
an answer is required of the user).
The "answer" return value is True for "yes" or False for "no".
"""
log = Logger("query_yes_no")
valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
if default is None:
prompt = " [y/n] "
elif default == "yes":
prompt = " [Y/n] "
elif default == "no":
prompt = " [y/N] "
else:
raise ValueError("invalid default answer: '%s'" % default)
while True:
log.info(question + prompt)
choice = raw_input().lower()
if default is not None and choice == '':
return valid[default]
elif choice in valid:
return valid[choice]
else:
log.warning("Please respond with 'yes' or 'no' (or 'y' or 'n').\n")
def loadChannelUnitsFromJSON(jsonFilePath):
if jsonFilePath is not None and not path.isfile(jsonFilePath):
raise "Missing channel unit file at {0}".format(jsonFilePath)
dUnits = {}
with open(jsonFilePath) as data_file:
data = json.load(data_file)
for nodUnit in data['values']:
value = nodUnit['value']
nCUNumber = int(value['ChannelUnitNumber'])
tier1 = value['Tier1']
tier2 = value['Tier2']
segment = value['ChannelSegmentID']
dUnits[nCUNumber] = (tier1, tier2, segment)
log = Logger("Channel Units")
log.info("{0} channel units loaded from XML file".format(len(dUnits)))
return dUnits
def deleteRaster(sFullPath):
"""
:param path:
:return:
"""
log = Logger("Delete Raster")
if path.isfile(sFullPath):
try:
# Delete the raster properly
driver = gdal.GetDriverByName('GTiff')
gdal.Driver.Delete(driver, sFullPath)
log.debug("Raster Successfully Deleted: {0}".format(sFullPath))
except Exception as e:
log.error("Failed to remove existing clipped raster at {0}".format(sFullPath))
raise e
else:
log.debug("No raster file to delete at {0}".format(sFullPath))
def getAbsInsensitivePath(abs_insensitive_path, ignoreAbsent=False):
"""
Will sanitize cases and return the correct case.
:param abs_insensitive_path:
:param ignoreAbsent: if true this will not throw an exception and just return the path
:return:
"""
log = Logger("getAbsInsensitivePath")
if len(abs_insensitive_path) == 0:
raise IOError("Zero-length path used: getAbsInsensitivePath()")
if os.path.sep == "/":
pathreplaced = abs_insensitive_path.replace("\\", os.path.sep)
else:
pathreplaced = abs_insensitive_path.replace("/", os.path.sep)
parts = pathreplaced.split(os.path.sep)
improved_parts = []
for part in parts:
if part == ".." or part == "." or part == "":
improved_parts.append(part)
else:
improved_path = os.path.sep.join(improved_parts)
if len(improved_path) == 0:
improved_path = os.path.sep
try:
found = False
for name in os.listdir(improved_path):
if part.lower() == name.lower():
improved_parts.append(name)
found = True
if not found:
raise OSError("Not found")
except OSError, e:
if not ignoreAbsent:
raise MissingException(
"Could not find case-insensitive path: {}".format(
abs_insensitive_path))
else:
return abs_insensitive_path
def __init__(self, sProjPath):
"""
:param sProjPath: Either the folder containing the project.rs.xml or the filepath of the actual project.rs.xml
"""
log = Logger('TopoProject')
try:
if path.isfile(sProjPath):
self.projpath = path.dirname(sProjPath)
self.projpathxml = sProjPath
elif path.isdir(sProjPath):
self.projpath = sProjPath
self.projpathxml = path.join(sProjPath, "project.rs.xml")
else:
raise MissingException(
"No project file or directory with the name could be found: {}"
.format(sProjPath))
except Exception, e:
raise MissingException(
"No project file or directory with the name could be found: {}"
.format(sProjPath))
class Tokenator(TokenatorBorg):
"""
Read up on the Borg pattern if you don't already know it. Super useful
"""
RETRIES_ALLOWED = 5
RETRY_DELAY = 4
def __init__(self):
super(Tokenator, self).__init__()
if not self._initdone:
self.log = Logger("API:Tokenator")
self.log.info("Init Token Settings")
self.TOKEN = None
self.tokenfile = os.environ.get('KEYSTONE_TOKENFILE')
self.load()
if self.TOKEN is None:
self.getToken()
else:
self.log.debug("re-using security token")
self._initdone = True
else:
self.log.debug("re-using security token")
def getToken(self):
self.log.info("Getting security token")
if os.environ.get('KEYSTONE_URL') is None:
raise Exception("Missing KEYSTONE_URL")
if os.environ.get('KEYSTONE_USER') is None:
raise Exception("Missing KEYSTONE_USER")
if os.environ.get('KEYSTONE_PASS') is None:
raise Exception("Missing KEYSTONE_PASS")
if os.environ.get('KEYSTONE_CLIENT_ID') is None:
raise Exception("Missing KEYSTONE_CLIENT_ID")
if os.environ.get('KEYSTONE_CLIENT_SECRET') is None:
raise Exception("Missing KEYSTONE_CLIENT_SECRET")
retry = True
retries = 0
response = None
while retry:
retries += 1
try:
response = requests.post(
os.environ.get('KEYSTONE_URL'),
data={
"username": os.environ.get('KEYSTONE_USER'),
"password": os.environ.get('KEYSTONE_PASS'),
"grant_type": "password",
"client_id": os.environ.get('KEYSTONE_CLIENT_ID'),
"client_secret":
os.environ.get('KEYSTONE_CLIENT_SECRET'),
"scope": 'keystone openid profile'
},
verify=verification())
except Exception, e:
errmsg = "Connection Exception: Request exception caught and will be retried: `{}` Retry: {}/{}".format(
e.message, retries, Tokenator.RETRIES_ALLOWED)
if retry and retries >= Tokenator.RETRIES_ALLOWED:
raise NetworkException(errmsg)
else:
self.log.error(errmsg)
continue
if response.status_code != 200:
errMsg = "Could not retrieve SitkaAPI Access Token with error: {0}. Retry#: {1}/{2}".format(
response.status_code, retries, Tokenator.RETRIES_ALLOWED)
if retries >= Tokenator.RETRIES_ALLOWED:
raise NetworkException(errMsg)
else:
self.log.error(
"{0} Retrying after {1} seconds. Retry#: {2}/{3}".
format(errMsg, Tokenator.RETRY_DELAY, retries,
Tokenator.RETRIES_ALLOWED))
time.sleep(Tokenator.RETRY_DELAY)
else:
retry = False
respObj = json.loads(response.content)
self.TOKEN = "bearer " + respObj['access_token']
self.store()