def test_cmf_path_validation(self):
cmf_partial_fail = os.path.join(
self.parent_dir, 'tests', 'testfiles',
'fixture_cmf_description_one_file_and_one_dir_not_valid.json')
# test validation on creation (for failing case) using default parameters
self.assertRaises(ValueError, CrashMoveFolder, cmf_partial_fail)
# force validation on creation, for a json file we know would otherwise fail:
self.assertRaises(ValueError,
CrashMoveFolder,
cmf_partial_fail,
verify_on_creation=True)
# test validation is correctly disabled on creation, for a json file we know would otherwise fail:
test_cmf = CrashMoveFolder(cmf_partial_fail, verify_on_creation=False)
self.assertIsInstance(test_cmf, CrashMoveFolder)
# check message included in the ValueError:
with self.assertRaises(ValueError) as cm:
test_cmf = CrashMoveFolder(cmf_partial_fail,
verify_on_creation=True)
if six.PY2:
self.assertRegexpMatches(str(cm.exception), "map_templates")
self.assertNotRegexpMatches(str(cm.exception), "original_data")
else:
self.assertRegex(str(cm.exception), "map_templates")
self.assertNotRegex(str(cm.exception), "original_data")
# create a valid CMF object and then test paths, after creation
test_cmf_path = os.path.join(self.parent_dir, 'example',
'cmf_description_flat_test.json')
test_cmf = CrashMoveFolder(test_cmf_path)
self.assertTrue(test_cmf.verify_paths())
test_cmf_path = os.path.join(
self.parent_dir, 'example',
'cmf_description_relative_paths_test.json')
test_cmf = CrashMoveFolder(test_cmf_path)
self.assertTrue(test_cmf.verify_paths())
test_cmf.active_data = os.path.join(self.parent_dir, 'DOES-NOT-EXIST')
self.assertFalse(test_cmf.verify_paths())
class BaseRunnerPlugin(object):
def __init__(self, hum_event, **kwargs):
self.hum_event = hum_event
self.cmf = CrashMoveFolder(self.hum_event.cmf_descriptor_path)
if not self.cmf.verify_paths():
raise ValueError(
"Cannot find paths and directories referenced by cmf {}".
format(self.cmf.path))
if self.__class__ is BaseRunnerPlugin:
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and cannot be instantiated directly'
)
def get_projectfile_extension(self, **kwargs):
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and the `get_projectfile_extension`'
' method cannot be called directly')
def get_lyr_render_extension(self, **kwargs):
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and the `get_lyr_render_extension`'
' method cannot be called directly')
def _get_all_templates_by_regex(self, recipe):
"""
Gets the fully qualified filenames of map templates, which exist in `self.cmf.map_templates` whose
filenames match the regex `recipe.template`.
@param recipe: A MapRecipe object.
@returns: A list of all of the templates, stored in `cmf.map_templates` whose
filename matches the regex `recipe.template` and that have the extention
`self.get_projectfile_extension()`
"""
def _is_relevant_file(f):
extension = os.path.splitext(f)[1]
logger.debug(
'checking file "{}", with extension "{}", against pattern "{}" and "{}"'
.format(f, extension, recipe.template,
self.get_projectfile_extension()))
if re.search(recipe.template, f):
logger.debug('file {} matched regex'.format(f))
f_path = os.path.join(self.cmf.map_templates, f)
logger.debug(
'file {} joined with self.cmf.map_templates "{}"'.format(
f, f_path))
return (os.path.isfile(f_path)) and (
extension == self.get_projectfile_extension())
else:
return False
# TODO: This results in calling `os.path.join` twice for certain files
logger.debug('searching for map templates in; {}'.format(
self.cmf.map_templates))
all_filenames = os.listdir(self.cmf.map_templates)
logger.debug('all available template files:\n\t{}'.format(
'\n\t'.join(all_filenames)))
relevant_filenames = [
os.path.realpath(os.path.join(self.cmf.map_templates, fi))
for fi in all_filenames if _is_relevant_file(fi)
]
logger.debug('possible template files:\n\t{}'.format(
'\n\t'.join(relevant_filenames)))
return relevant_filenames
def _get_template_by_aspect_ratio(self, template_aspect_ratios, target_ar):
"""
Selects the template which best matches the required aspect ratio.
@param possible_aspect_ratios: A list of tuples. For each tuple the first element is the path to the
template. The second element is the relevant aspect ratio for that
template. Typically this would be generated by
`get_aspect_ratios_of_templates()`.
@param target_ar: The taret aspect ratio - typically the aspect ratio of the bounding box for the country
being mapped.
@returns: The path of the template with the best matching aspect ratio.
"""
logger.info(
'Selecting from available templates based on the most best matching aspect ratio'
)
# Target is more landscape than the most landscape template
most_landscape = max(template_aspect_ratios, key=itemgetter(1))
if most_landscape[1] < target_ar:
logger.info(
'Target area of interest is more landscape than the most landscape template'
)
return most_landscape[0]
# Target is more portrait than the most portrait template
most_portrait = min(template_aspect_ratios, key=itemgetter(1))
if most_portrait[1] > target_ar:
logger.info(
'Target area of interest is more portrait than the most portrait template'
)
return most_portrait[0]
# The option with the smallest aspect ratio that is larger than target_ar
larger_ar = min([(templ_path, templ_ar)
for templ_path, templ_ar in template_aspect_ratios
if templ_ar >= target_ar],
key=itemgetter(1))
# The option with the largest aspect ratio that is smaller than target_ar
smaller_ar = max([(templ_path, templ_ar)
for templ_path, templ_ar in template_aspect_ratios
if templ_ar <= target_ar],
key=itemgetter(1))
# Linear combination:
# if (2*target_ar) > (larger_ar[1] + smaller_ar[1]):
# return larger_ar[0]
# asmith: personally I think that this is the better option, but will go with the linear combination for now
# logarithmic combination
if (2 * math.log(target_ar)) > (math.log(larger_ar[1]) +
math.log(smaller_ar[1])):
logger.info(
'Aspect ratio of the target area of interest lies between the aspect ratios of the'
' available templates')
return larger_ar[0]
return smaller_ar[0]
def get_aspect_ratios_of_templates(self, possible_templates, recipe):
"""
Plugins are required to implement this method.
The implementation should calculate the aspect ratio of the principal map frame within the list of
templates. The definition of "principal" is left to the plugin, though is typically the largest map
frame.
@param possible_templates: A list of paths to possible templates
@returns: A list of tuples. For each tuple the first element is the path to the template. The second
element is the aspect ratio of the largest* map frame within that template.
See `_get_largest_map_frame` for the description of how largest is determined.
@raises NotImplementedError: In the base class.
"""
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and the `get_aspect_ratios_of_templates`'
' method cannot be called directly')
def _get_aspect_ratio_of_bounds(self, bounds):
minx, miny, maxx, maxy = bounds
dx = (
maxx - minx
) % 360 # Accounts for the case where the bounds stradles the 180 meridian
dy = maxy - miny
return float(dx) / dy
def get_templates(self, **kwargs):
"""
Updates the recipe's `template_path` value. The result is the absolute path to the template.
To select the appropriate template it uses two inputs.
* The `recipe.template` value, which is a regex for the filename of the possible templates
* The target asspect ratio. If the aspect ratio of the target data can be determined then this is
also used to select the best matching template, amogst those which match the regex. If the
target ratio cannot be determined fromsource gis data, then the target ratio of 1.0 will be
used.
"""
recipe = kwargs['state']
# If there already is a valid `recipe.map_project_path` just skip with method
if recipe.map_project_path:
if os.path.exists(recipe.map_project_path):
return recipe
else:
raise ValueError(
'Unable to locate map project file: {}'.format(
recipe.map_project_path))
# use `recipe.template` as regex to locate one or more templates
possible_templates = self._get_all_templates_by_regex(recipe)
# Select the template with the most appropriate aspect ratio
possible_aspect_ratios = self.get_aspect_ratios_of_templates(
possible_templates, recipe)
mf = recipe.get_frame(recipe.principal_map_frame)
# Default value
target_aspect_ratio = 1.0
# If the MapFrame's target extent is not None, then use that:
if mf.extent:
target_aspect_ratio = self._get_aspect_ratio_of_bounds(mf.extent)
# use logic to workout which template has best aspect ratio
# obviously not this logic though:
recipe.template_path = self._get_template_by_aspect_ratio(
possible_aspect_ratios, target_aspect_ratio)
# TODO re-enable "Have the input files changed?"
# Have the input shapefiles changed?
return recipe
# TODO: asmith 2020/03/03
# 1) Please avoid hardcoding the naming convention for the mxds wherever possible. The Naming Convention
# classes can avoid the need to hardcode the naming convention for the input mxd templates. It might be
# possible to avoid the need to hardcode the naming convention for the output mxds using a
# String.Template be specified within the Cookbook?
# https://docs.python.org/2/library/string.html#formatspec
# https://www.python.org/dev/peps/pep-3101/
#
# 2) This only checks the filename for the mxd - it doesn't check the values within the text element of
# the map layout view (and hence the output metadata).
def get_next_map_version_number(self, mapNumberDirectory, mapNumber,
mapFileName):
versionNumber = 0
files = glob.glob(mapNumberDirectory + os.sep + mapNumber +
'-v[0-9][0-9]-' + mapFileName + '.mxd')
for file in files:
versionNumber = int(
os.path.basename(file).replace(mapNumber + '-v', '').replace(
('-' + mapFileName + '.mxd'), '')) # noqa
versionNumber = versionNumber + 1
return versionNumber
# TODO Is it possible to aviod the need to hardcode the naming convention for the output mxds? Eg could a
# String.Template be specified within the Cookbook?
# https://docs.python.org/2/library/string.html#formatspec
# https://www.python.org/dev/peps/pep-3101/
def create_ouput_map_project(self, **kwargs):
recipe = kwargs['state']
# Create `mapNumberDirectory` for output
output_dir = os.path.join(self.cmf.map_projects, recipe.mapnumber)
if not (os.path.isdir(output_dir)):
os.mkdir(output_dir)
# Construct output MXD/QPRJ name
logger.debug(
'About to create new map project file for product "{}"'.format(
recipe.product))
output_map_base = slugify(recipe.product)
logger.debug('Set output name for new map project file to "{}"'.format(
output_map_base))
recipe.version_num = self.get_next_map_version_number(
output_dir, recipe.mapnumber, output_map_base)
output_map_name = '{}-v{}-{}{}'.format(
recipe.mapnumber,
str(recipe.version_num).zfill(2), output_map_base,
self.get_projectfile_extension())
recipe.map_project_path = os.path.abspath(
os.path.join(output_dir, output_map_name))
logger.debug('Path for new map project file; {}'.format(
recipe.map_project_path))
logger.debug('Map Version number; {}'.format(recipe.version_num))
# Copy `src_template` to `recipe.map_project_path`
copyfile(recipe.template_path, recipe.map_project_path)
return recipe
def export_maps(self, **kwargs):
"""
Generates all file for export.
Accumulate some of the parameters for export XML, then calls
_do_export(....) to do that actual work
"""
recipe = kwargs['state']
export_params = {}
properties = {} # For properties from MapAction Toolbox
try:
export_params = self._create_export_dir(export_params, recipe)
if 'properties' in kwargs:
properties = kwargs['properties']
for key in list(properties.keys()):
export_params[str(key)] = properties[str(key)]
export_params = self._do_export(export_params, recipe)
except Exception as exp:
logger.error(
'Failed to export the map. export_params = "{}"'.format(
export_params))
logger.error(exp)
self.zip_exported_files(export_params)
def _create_export_dir(self, export_params, recipe):
# Accumulate parameters for export XML
version_str = "v" + str(recipe.version_num).zfill(2)
export_directory = os.path.abspath(
os.path.join(self.cmf.export_dir, recipe.mapnumber, version_str))
export_params["exportDirectory"] = export_directory
try:
os.makedirs(export_directory)
except OSError as exc: # Python >2.5
# Note 'errno.EEXIST' is not a typo. There should be two 'E's.
# https://docs.python.org/2/library/errno.html#errno.EEXIST
if exc.errno == errno.EEXIST and os.path.isdir(export_directory):
pass
else:
raise
return export_params
def _do_export(self, export_params, recipe):
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and the `export_maps`'
' method cannot be called directly')
def zip_exported_files(self, export_params):
# Get key params as local variables
core_file_name = export_params['coreFileName']
export_dir = export_params['exportDirectory']
mdr_xml_file_path = export_params['exportXmlFileLocation']
jpg_path = export_params['jpgFileLocation']
png_thumbnail_path = export_params['pngThumbNailFileLocation']
# And now Zip
zipFileName = core_file_name + ".zip"
zipFileLocation = os.path.join(export_dir, zipFileName)
with ZipFile(zipFileLocation, 'w') as zipObj:
zipObj.write(mdr_xml_file_path,
os.path.basename(mdr_xml_file_path))
zipObj.write(jpg_path, os.path.basename(jpg_path))
zipObj.write(png_thumbnail_path,
os.path.basename(png_thumbnail_path))
if (len(export_params.get("emfFileLocation", "")) > 0):
zipObj.write(
export_params['emfFileLocation'],
os.path.basename(export_params['emfFileLocation']))
# TODO: asmith 2020/03/03
# Given we are explictly setting the pdfFileName for each page within the DDPs
# it is possible return a list of all of the filenames for all of the PDFs. Please
# can we use this list to include in the zip file. There are edge cases where just
# adding all of the pdfs in a particular directory might not behave correctly (eg if
# the previous run had crashed midway for some reason)
for pdf in os.listdir(export_dir):
if pdf.endswith(".pdf"):
zipObj.write(
os.path.join(export_dir, pdf),
os.path.basename(os.path.join(export_dir, pdf)))
print("Export complete to " + export_dir)
def build_project_files(self, **kwargs):
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and the `build_project_files`'
' method cannot be called directly')