def xmlExport(fc):
"""
Export metadata from the item and convert to a stand-alone xml file
Parameters
----------
fc: Data Element;Layer
The item whose metadata will be converted or a stand-alone XML file that will be converted.
Returns
-------
output_xml: File
XML file
"""
# arcgis install directory
dir = arcpy.GetInstallInfo("desktop")["InstallDir"]
# exact copy xslt transformation
xslt = dir + "Metadata/Stylesheets/gpTools/exact copy of.xslt"
# set output directory
out_dir = arcpy.env.scratchFolder
# set arcpy workspace
arcpy.env.workspace = out_dir
# set workspace overwrite true
arcpy.env.overwriteOutput = True
#assemble output xml file name
output_xml = path.join(out_dir, path.basename(fc) + ".xml")
#XSLT transforamtion
arcpy.XSLTransform_conversion(fc, xslt, output_xml)
return output_xml
def export_featureclass(input_path, output_name):
print("Upgrading downloaded metadata to ArcGIS standard")
arcpy.UpgradeMetadata_conversion(input_path, 'FGDC_TO_ARCGIS')
print("Downloaded metadata upgraded to ArcGIS standard")
print("Overwriting original metadata with DCP standard")
arcpy.MetadataImporter_conversion(
os.path.join(template_path, '{}.xml'.format(output_name)),
input_path)
print("Original metadata overwritten")
tree = ET.parse('{}.xml'.format(input_path))
root = tree.getroot()
for title in root.iter('title'):
title.text = output_name.replace('_', ' ')
for pubdate_fgdc in root.iter('pubdate'):
pubdate_fgdc.text = last_update_date_str
for descrip_fgdc in root.iter('abstract'):
descrip_fgdc.text += ' Dataset last updated: {}. Dataset last downloaded: {}'.format(
last_update_date_meta, today)
print("Writing updated metadata to {}".format(input_path))
tree.write('{}.xml'.format(input_path))
print("Metadata update complete for {}".format(input_path))
print("Upgrading metadata format for {}".format(input_path))
arcpy.UpgradeMetadata_conversion(input_path, 'FGDC_TO_ARCGIS')
print("Metadata format upgraded for {}".format(input_path))
print("Exporting shapefile to SDE PROD as {}".format(output_name))
arcpy.FeatureClassToFeatureClass_conversion(input_path, sde_path,
output_name)
print("Removing local storage info")
arcpy.XSLTransform_conversion(
os.path.join(sde_path, output_name), xslt_storage,
os.path.join(zip_dir_path, '{}_storage.xml'.format(input_path)))
arcpy.XSLTransform_conversion('{}_storage.xml'.format(input_path),
xslt_geoprocess,
'{}_geoprocess.xml'.format(input_path))
print("Importing final metadata to {}".format(output_name))
arcpy.MetadataImporter_conversion(
os.path.join(zip_dir_path, "{}_geoprocess.xml".format(input_path)),
os.path.join(sde_path, output_name))
def RemoveUnwantedMetadata(fgdb, fc):
# set environments & variables
arcpy.env.workspace = fgdb
# the full path of feature class is passed, and we need the base name
featureClass = os.path.basename(fc)
# the below construction makes it installation independent
installDir = arcpy.GetInstallInfo()['InstallDir']
ssPath1 = 'Metadata\\Stylesheets\\gpTools\\remove geoprocessing history.xslt'
ssPath2 = 'Metadata\\Stylesheets\\gpTools\\remove local storage info.xslt'
xsltPath1 = installDir + ssPath1
xsltPath2 = installDir + ssPath2
# delete the output directory if it already exists so you don't have to overwrite
outXml = 'c:\\XML_out'
if os.path.exists(outXml):
shutil.rmtree(outXml)
os.mkdir(outXml)
# Remove geoprocessing history and local machine name/paths
# output xml for removing gp history
nameXml1 = outXml + os.sep + str(featureClass) + ".xml"
# output xml for removing local storage info
nameXml2 = outXml + os.sep + str(featureClass) + "2.xml"
try:
arcpy.XSLTransform_conversion(featureClass, xsltPath1, nameXml1, "")
arcpy.MetadataImporter_conversion(nameXml1, featureClass)
arcpy.XSLTransform_conversion(featureClass, xsltPath2, nameXml2, "")
arcpy.MetadataImporter_conversion(nameXml2, featureClass)
return (True, "Succeeded")
except Exception as e:
error = "Failed: metadata removal for " + str(fc) + " " + str(e)
return (False, error)
finally:
# clean up
shutil.rmtree(outXml)
def updateFeatureClassAbstract(featureClassLocation, featureClassName,
abstractText):
# set up scratch file location for XML
metadata_file = metadataScratchFilePath(featureClassName)
if os.path.exists(metadata_file):
os.remove(metadata_file)
# create XML scratch file
arcpy.env.workspace = featureClassLocation
arcpy.XSLTransform_conversion(featureClassName, XSLTFilePath(),
metadata_file)
# update XML file with new abstract
updateMetaDataFileAbstract(metadata_file, abstractText)
# write updates to the feature class
arcpy.ImportMetadata_conversion(metadata_file,
"FROM_ARCGIS",
featureClassName,
Enable_automatic_updates=False)
def xmlExport(fc):
"""
export metadata from feature class and save as xml file
"""
# arcgis install directory
dir = arcpy.GetInstallInfo("desktop")["InstallDir"]
# exact copy xslt transformation
xslt = dir + "Metadata/Stylesheets/gpTools/exact copy of.xslt"
# set output directory
out_dir = outputFolder
# set arcpy workspace
arcpy.env.workspace = out_dir
# set workspace overwrite true
arcpy.env.overwriteOutput = True
#assemble output xml file name
output_xml = path.join(out_dir, path.basename(fc) + ".xml")
#XSLT transforamtion
arcpy.XSLTransform_conversion(fc, xslt, output_xml)
def export_metadata():
"""Exports the feature class metadata to an xml file
Returns:
None
"""
folder = 'metadata'
name = get_dataset_filename()
# Create a metadata folder in the temp directory if it does not exist
temp_working_folder = os.path.join(temp_workspace, folder)
create_folder(temp_working_folder, True)
# Set the destinion of the metadata export
source = staging_feature_class
raw_metadata_export = os.path.join(temp_working_folder, name + "_raw.xml")
# Export the metadata
arcpy.env.workspace = temp_working_folder
installDir = arcpy.GetInstallInfo("desktop")["InstallDir"]
translator = installDir + "Metadata/Translator/ARCGIS2FGDC.xml"
arcpy.ExportMetadata_conversion(source, translator, raw_metadata_export)
# Process: XSLT Transformation to remove any sensitive info or format
destination = os.path.join(temp_working_folder, name + ".xml")
if os.path.exists(args.metadata_xslt):
logger.info("Applying metadata XSLT: " + args.metadata_xslt)
arcpy.XSLTransform_conversion(raw_metadata_export, args.metadata_xslt,
destination, "")
# Reimport the clean metadata into the FGDB
logger.debug("Reimporting metadata to file geodatabase " + destination)
arcpy.MetadataImporter_conversion(destination, staging_feature_class)
else:
# If no transformation exists, just rename and publish the raw metadata
logger.warn("Metadata XSLT not found")
os.rename(raw_metadata_export, destination)
# Publish the metadata to the download folder
publish_file(temp_working_folder, name + ".xml", "metadata")
def GetRawXML(InputXML_or_DataLayer, OutputRawXML):
'''
Obtains the raw metadata from a dataset.
'''
#Finds the install directory and obtains the xslt used to grab all of the xml contents, unedited
installDir = arcpy.GetInstallInfo("desktop")["InstallDir"]
xsltPath = "Metadata/Stylesheets/gpTools/exact copy of.xslt"
Exact_Copy_XSLT = os.path.join(installDir,xsltPath)
Exact_Copy_XSLT = os.path.realpath(Exact_Copy_XSLT)
arcpy.AddMessage("The raw xml is now being extracted into a temporary file for further processing... \n")
#Process: Get Metadata
try:
if os.path.exists(OutputRawXML):
os.remove(OutputRawXML)
except IOError:
raise Exception, "Error. This tool requires read/write access to the directories where the temporary and final outputs are saved. Please choose another directory for the tool outputs."
try:
arcpy.XSLTransform_conversion(InputXML_or_DataLayer, Exact_Copy_XSLT, OutputRawXML)
#arcpy.AddMessage(arcpy.GetMessages(0))
except:
raise Exception, str(arcpy.GetMessages(2))
def CreateCopyMDRecord(InputXML_or_DataLayer,
MDRecordCopy): #Creates unmodified backup of input.
'''
Obtains the raw metadata from a dataset. This makes an exact copy.
'''
#Finds the install directory and obtains the xslt used to grab all of the xml contents, unedited
installDir = arcpy.GetInstallInfo("desktop")["InstallDir"]
xsltPath = "Metadata/Stylesheets/gpTools/exact copy of.xslt"
Exact_Copy_XSLT = os.path.join(installDir, xsltPath)
Exact_Copy_XSLT = os.path.realpath(Exact_Copy_XSLT)
#Process: Get Metadata
try:
if os.path.exists(MDRecordCopy):
os.remove(MDRecordCopy)
except IOError:
raise Exception, "This tool requires read/write access to the directories where the temporary and final outputs are saved. Please choose another directory for the tool outputs."
try:
arcpy.XSLTransform_conversion(InputXML_or_DataLayer, Exact_Copy_XSLT,
MDRecordCopy)
except:
raise Exception, str(arcpy.GetMessages(2))
def __init__(self, dataset=None, metadata_file=None, items=None,
temp_folder=metadata_temp_folder, loglevel='INFO'):
screen_handler = None
self.logger = logging.getLogger("__name__")
if not len(self.logger.handlers):
screen_handler = logging.StreamHandler() # set up the logging level at debug, but only write INFO or higher to the screen
self.logger.setLevel(logging.DEBUG)
self.logger.addHandler(screen_handler)
else:
for handler in self.logger.handlers:
# take the first one available
if isinstance(handler, logging.StreamHandler):
screen_handler = handler
break
# just making sure that there is a screenhandler
if screen_handler is None:
screen_handler = logging.StreamHandler()
self.logger.setLevel(logging.DEBUG)
self.logger.addHandler(screen_handler)
if loglevel.upper() == "CRITICAL":
screen_handler.setLevel(logging.CRITICAL)
elif loglevel.upper() == "ERROR":
screen_handler.setLevel(logging.ERROR)
elif loglevel.upper() == "WARNING":
screen_handler.setLevel(logging.WARNING)
elif loglevel.upper() == "INFO":
screen_handler.setLevel(logging.INFO)
elif loglevel.upper() == "DEBUG":
screen_handler.setLevel(logging.DEBUG)
else:
screen_handler.setLevel(logging.NOTSET)
self.logger.debug("Set logging mode to {0}".format(loglevel))
if items is None:
items = list()
self.items = items
self.metadata_file = metadata_file
self.elements = xml.etree.ElementTree.ElementTree()
self.temp_folder = temp_folder
self.dataset = dataset
self._gdb_datasets = ["FeatureClass", "Table", "RasterDataset", "RasterCatalog", "MosaicDataset"]
self._simple_datasets = ["ShapeFile", "RasterDataset", "Layer"]
self._layers = ["FeatureLayer"]
if self.dataset: # Check if dataset is set
# export the metadata to the temporary location
self.data_type = self.get_datatype()
# for layers get the underlying dataset and start over
if self.data_type in self._layers:
desc = arcpy.Describe(self.dataset)
self.data_type = desc.dataElement.dataType
self.dataset = desc.dataElement.catalogPath # overwrite path to dataset with layer's data source
self._workspace = self.get_workspace()
self._workspace_type = self.get_workspace_type()
# Datasets in Filesystem have metadata attached as XML file
# we can directly write to it
if self._workspace_type == 'FileSystem':
if self.data_type in self._simple_datasets:
xml_file = self.dataset + ".xml"
#if no XML file exists create one and add most basic metadata item to it
if not os.path.exists(xml_file):
self._create_xml_file(xml_file)
self.metadata_file = xml_file
else:
raise TypeError("Cannot read {0}. Data type is not supported".format(self.dataset))
# Metadata for GDB datasets are stored inside the GDB itself.
# We need to first export them to a temporary file, modify them and then import them back
else:
if self.data_type in self._gdb_datasets:
metadata_filename = os.path.basename(self.dataset) + ".xml"
self.metadata_file = os.path.join(self.temp_folder, metadata_filename)
if os.path.exists(self.metadata_file):
os.remove(self.metadata_file)
self.logger.debug("Exporting metadata to temporary file {0!s}".format(self.metadata_file))
arcpy.XSLTransform_conversion(self.dataset, xslt, self.metadata_file)
else:
raise TypeError("Cannot read {0}. Data type is not supported".format(self.dataset))
elif self.metadata_file: # Check if metadata file is set instead
self.data_type = 'MetadataFile'
if self.metadata_file.endswith('.xml'):
if not os.path.exists(self.metadata_file):
self._create_xml_file(self.metadata_file)
self._workspace_type = 'FileSystem'
else:
raise TypeError("Metadata file is not an XML file. Check file extension")
self.elements.parse(self.metadata_file)
# create these all after the parsing happens so that if they have any self initialization, they can correctly perform it
for name in elements.keys():
if "sync" in elements[name].keys():
sync = elements[name]["sync"]
else:
sync = None
if "unsupported" in elements[name].keys():
if self.data_type in elements[name]["unsupported"]:
self.logger.debug("{0} not supported for {1}. SKIP".format(name, self.data_type))
continue
setattr(self, "_{0!s}".format(name), None)
if elements[name]['type'] in ["string", "datetime", "date", "time", "integer", "float"]:
setattr(self, "_{0}".format(name), MetadataItem(elements[name]['path'], name, self, sync))
if self.__dict__["_{0}".format(name)].value is not None:
setattr(self, name, self.__dict__["_{0}".format(name)].value.strip())
else:
setattr(self, name, self.__dict__["_{0}".format(name)].value)
elif elements[name]['type'] == "attribute":
setattr(self, "_{0}".format(name), MetadataItem(elements[name]['path'], name, self, sync))
if isinstance(self.__dict__["_{0}".format(name)].attributes, dict):
key = elements[name]['key']
values = elements[name]['values']
if key in self.__dict__["_{0}".format(name)].attributes.keys():
v = self.__dict__["_{0}".format(name)].attributes[elements[name]['key']]
for value in values:
if v in value:
setattr(self, name, value[0])
break
else:
setattr(self, name, None)
elif elements[name]['type'] == "list":
setattr(self, "_{0}".format(name), MetadataValueList(elements[name]["tagname"], elements[name]['path'], name, self, sync))
#setattr(self, name, self.__dict__["_{}".format(name)].value)
#setattr(self, name, ListValues(self.__dict__["_{}".format(name)], name))
elif elements[name]['type'] == "language":
setattr(self, "_{0}".format(name), MetadataLanguage(elements[name]['path'], name, self, sync))
if self.__dict__["_{0}".format(name)].value is not None:
setattr(self, name, self.__dict__["_{0}".format(name)].value.strip())
else:
setattr(self, name, self.__dict__["_{0}".format(name)].value)
# TODO: turn on sync
elif elements[name]['type'] == "parent_item":
setattr(self, "_{0}".format(name), MetadataParentItem(elements[name]['path'], self, elements[name]['elements']))
setattr(self, name, self.__dict__["_{0}".format(name)])
elif elements[name]['type'] == "object_list":
setattr(self, "_{0}".format(name), MetadataObjectList(elements[name]["tagname"], elements[name]['path'], self, elements[name]['elements'], sync))
#setattr(self, name, self.__dict__["_{}".format(name)])
if elements[name] in self.__dict__.keys():
self.items.append(getattr(self, "_{0}".format(elements[name])))
if items:
self.initialize_items()
def write_metadata(input_items, template_xml, xslt_file, summary, description,
tags, data_credits, use_constraints, overwrite,
token_header):
"""Writes metadata."""
updated = 0
errors = 0
skipped = 0
global processed_count
for item in input_items:
try:
id = item[1]
path = item[0]
# Temporary XML file
temp_xml = tempfile.NamedTemporaryFile(suffix='.xml',
delete=True).name
# Export xml
try:
arcpy.XSLTransform_conversion(path, xslt_file, temp_xml)
except arcpy.ExecuteError:
src_xml = os.path.join(
arcpy.Describe(path).path,
'{0}.xml'.format(os.path.basename(path)))
shutil.copyfile(template_xml, src_xml)
arcpy.XSLTransform_conversion(src_xml, xslt_file, temp_xml)
# Read in XML
tree = eTree.parse(temp_xml)
root = tree.getroot()
changes = 0
# ISO allows many dataIdInfo groups; ArcGIS generally supports only one.
data_id_elements = root.findall(".//dataIdInfo")
if not data_id_elements:
data_id_elements = [eTree.SubElement(root, 'dataIdInfo')]
for data_id_element in data_id_elements:
# Write summary.
summary_element = root.findall(".//idPurp")
if not summary_element:
summary_element = eTree.SubElement(data_id_element,
'idPurp')
summary_element.text = summary
changes += 1
else:
for element in summary_element:
if summary and (overwrite or element.text is None):
element.text = summary
changes += 1
# Write description.
description_element = root.findall(".//idAbs")
if not description_element:
description_element = eTree.SubElement(
data_id_element, 'idAbs')
description_element.text = description
changes += 1
else:
for element in description_element:
if description and (overwrite or element.text is None):
element.text = description
changes += 1
# Write tags.
tags = task_utils.get_unique_strings(tags)
search_keys = root.findall(".//searchKeys")
if not search_keys:
search_element = eTree.SubElement(data_id_element,
'searchKeys')
for tag in tags:
new_tag = eTree.SubElement(search_element, "keyword")
new_tag.text = tag
changes += 1
elif not overwrite:
# Still add any new tags.
for search_element in search_keys:
if tags:
for tag in tags:
if tag.lower() not in [
se.text.lower() for se in
search_element.findall('.//keyword')
]:
new_tag = eTree.SubElement(
search_element, "keyword")
new_tag.text = tag
changes += 1
else:
if tags:
for search_element in search_keys:
[
search_element.remove(e)
for e in search_element.findall('.//keyword')
]
for tag in tags:
new_tag = eTree.SubElement(
search_element, "keyword")
new_tag.text = tag
changes += 1
# Write credits.
credits_element = root.findall(".//idCredit")
if not credits_element:
credits_element = eTree.SubElement(data_id_element,
'idCredit')
credits_element.text = data_credits
changes += 1
else:
for element in credits_element:
if data_credits and (overwrite
or element.text is None):
element.text = data_credits
changes += 1
# Write use constraints.
res_constraints = root.findall(".//resConst")
if not res_constraints:
res_constraint_element = eTree.SubElement(
data_id_element, 'resConst')
const_element = eTree.SubElement(res_constraint_element,
'Consts')
new_constraint = eTree.SubElement(const_element,
'useLimit')
new_constraint.text = use_constraints
changes += 1
elif not overwrite:
constraint_elements = root.findall('.//Consts')
for element in constraint_elements:
if use_constraints:
new_constraint = eTree.SubElement(
element, 'useLimit')
new_constraint.text = use_constraints
changes += 1
else:
if use_constraints:
constraint_elements = root.findall('.//Consts')
if constraint_elements:
[
constraint_elements[0].remove(e) for e in
constraint_elements[0].findall('.//useLimit')
]
new_constraint = eTree.SubElement(
constraint_elements[0], 'useLimit')
new_constraint.text = use_constraints
changes += 1
if changes > 0:
# Save modifications to the temporary XML file.
tree.write(temp_xml)
# Import the XML file to the item; existing metadata is replaced.
arcpy.MetadataImporter_conversion(temp_xml, path)
status_writer.send_percent(
processed_count / result_count,
_('Metadata updated for: {0}').format(path),
'write_metadata')
processed_count += 1
try:
index_item(id, token_header)
except (IndexError, urllib2.HTTPError, urllib2.URLError) as e:
status_writer.send_status(e.message)
pass
updated += 1
else:
processed_count += 1
status_writer.send_percent(
processed_count / result_count,
_('No metadata changes for: {0}').format(path),
'write_metadata')
skipped_reasons[path] = _(
'No metadata changes for: {0}').format(path)
skipped += 1
except Exception as ex:
processed_count += 1
status_writer.send_percent(processed_count / result_count,
_('Skipped: {0}').format(path),
'write_metadata')
status_writer.send_status(_('FAIL: {0}').format(repr(ex)))
errors_reasons[path] = repr(ex)
errors += 1
pass
return updated, errors, skipped
def __init__(self, dataset=None, metadata_file=None, items=None,
temp_folder=metadata_temp_folder):
if items is None:
items = list()
self.items = items
self.dataset = dataset
self.metadata_file = metadata_file
self.elements = xml.etree.ElementTree.ElementTree()
self.temp_folder = temp_folder
self._gdb_datasets = ["FeatureClass", "Table", "RasterDataset", "RasterCatalog", "MosaicDataset"]
self._simple_datasets = ["ShapeFile", "RasterDataset"]
self._layers = ["FeatureLayer", "Layer"]
if self.dataset: # for both, we want to export the metadata out
# export the metadata to the temporary location
self.data_type = self.get_datatype()
# for layers get the underlying dataset and start over
if self.data_type in self._layers:
desc = arcpy.Describe(self.dataset)
self.data_type = desc.dataElement.dataType
self.dataset = desc.dataElement.catalogPath
self._workspace = self.get_workspace()
self._workspace_type = self.get_workspace_type()
# Datasets in Filesystem have metadata attached as XML file
# we can directly write to it
if self._workspace_type == 'FileSystem':
if self.data_type in self._simple_datasets:
xml_file = self.dataset + ".xml"
# if no XML file exists create one and add most basic metadata item to it
if not os.path.exists(xml_file):
with open(xml_file, "w") as f:
f.write('<metadata xml:lang="en"></metadata>')
self.metadata_file = xml_file
else:
raise TypeError("Datatype is not supported")
# Metadata for GDB datasets are stored inside the GDB itself.
# We need to first export them to a temporary file, modify them and then import them back
else:
if self.data_type in self._gdb_datasets:
metadata_filename = os.path.basename(self.dataset) + ".xml"
self.metadata_file = os.path.join(self.temp_folder, metadata_filename)
if os.path.exists(self.metadata_file):
os.remove(self.metadata_file)
logwrite("Exporting metadata to temporary file %s" % self.metadata_file)
arcpy.XSLTransform_conversion(self.dataset, xslt, self.metadata_file)
else:
raise TypeError("Datatype is not supported")
self.elements.parse(self.metadata_file)
# create these all after the parsing happens so that if they have any self initialization,
# they can correctly perform it
for name in elements.keys():
setattr(self, "_%s" % name, None)
if elements[name]['type'] in ["string", "date", "integer", "float"]:
setattr(self, "_{}".format(name), MetadataItem(elements[name]['path'], name, self))
if self.__dict__["_{}".format(name)].value is not None:
setattr(self, name, self.__dict__["_{}".format(name)].value.strip())
else:
setattr(self, name, self.__dict__["_{}".format(name)].value)
elif elements[name]['type'] == "list":
setattr(self, "_{}".format(name), MetadataList(elements[name]["tagname"], elements[name]['path'], name, self))
setattr(self, name, self.__dict__["_{}".format(name)].value)
elif elements[name]['type'] == "language":
setattr(self, "_{}".format(name), MetadataLanguage(elements[name]['path'], name, self))
if self.__dict__["_{}".format(name)].value is not None:
setattr(self, name, self.__dict__["_{}".format(name)].value.strip())
else:
setattr(self, name, self.__dict__["_{}".format(name)].value)
elif elements[name]['type'] == "local":
setattr(self, name, MetadataLocals(elements[name]['path'], name, self))
elif elements[name]['type'] == "contact":
setattr(self, "_{}".format(name), MetadataContact(elements[name]['path'], name, self))
setattr(self, name, self.__dict__["_{}".format(name)])
if elements[name] in self.__dict__.keys():
self.items.append(getattr(self, "_{}".format(elements[name])))
if items:
self.initialize_items()
def export_featureclass(input_path, output_name, modified_path):
'''
Index BIN field, create new double field called PLUTO_BBL populate new field with BBL values with conditionals
accounting for incorrect BBL values (either short or missing) and reorder output tables to include new field
within the previous standard
'''
print("Creating PLUTO BBL field")
arcpy.AddField_management(input_path, 'PLUTO_BBL', 'DOUBLE')
print("PLUTO BBL field created")
cursor = arcpy.da.UpdateCursor(input_path, ['BASE_BBL', 'MPLUTO_BBL', 'PLUTO_BBL', 'BIN'])
for row in cursor:
print("Parsing BASE_BBLS: {} and MPLUTO_BBLS: {}".format(row[0], row[1]))
if len(row[0]) != 10:
error_list.add("Short BBL. BASE_BBL = {} ; MPLUTO_BBL = {} ; BIN = {}".format(row[0], row[1], row[3]))
if row[1].isspace() == True:
if row[0].isspace() == True:
error_list.add("Missing BBL. BIN = {}".format(row[3]))
continue
if row[0].isspace() == False and r"`" not in row[0]:
row[2] = float(row[0])
cursor.updateRow(row)
continue
'''
Case where ` character is included in BBL value. Interim value of 1 to replace this character until I can confirm with Matt.
This would create the correct BBL based on DOB Property Profile Overview lookup information
'''
if r"`" in row[0]:
new_bbl = row[0].replace("`", "1")
row[2] = float(new_bbl)
cursor.updateRow(row)
continue
else:
row[2] = float(row[0])
cursor.updateRow(row)
continue
if row[1] == r'':
if row[0] == r'':
error_list.add("Missing BBL. BIN = {}".format(row[3]))
if r"`" in row[0]:
new_bbl = row[0].replace("`", "1")
row[2] = float(new_bbl)
cursor.updateRow(row)
continue
else:
row[2] = float(row[0])
cursor.updateRow(row)
continue
else:
row[2] = float(row[1])
cursor.updateRow(row)
continue
print("Creating field map in order to re-order the export tables")
fieldMap = arcpy.FieldMappings()
fieldMap.addTable(input_path)
newFieldMap = arcpy.FieldMappings()
print("Field mapping created")
print("Adding fields to new field map")
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('NAME')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('BIN')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('CNSTRCT_YR')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('LSTMODDATE')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('LSTSTATYPE')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('DOITT_ID')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('HEIGHTROOF')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('FEAT_CODE')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('GROUNDELEV')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('BASE_BBL')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('MPLUTO_BBL')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('PLUTO_BBL')))
newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('GEOMSOURCE')))
print("All fields added to field map")
print("Exporting as modified shapefile in temporary directory")
arcpy.FeatureClassToFeatureClass_conversion(in_features=input_path,
out_path=zip_dir_path,
out_name=modified_path.split('.')[0],
field_mapping=newFieldMap)
print("Modified shapefile exported")
print("Upgrading downloaded metadata to ArcGIS standard")
arcpy.env.workspace = zip_dir_path
arcpy.env.overwriteOutput = True
metadata_path = os.path.join(zip_dir_path, modified_path)
arcpy.UpgradeMetadata_conversion(metadata_path, 'FGDC_TO_ARCGIS')
print("Downloaded metadata upgraded to ArcGIS standard")
print("Overwriting original metadata with DCP standard")
arcpy.MetadataImporter_conversion(os.path.join(template_path, '{}.xml'.format(output_name)),
metadata_path)
print("Original metadata overwritten")
tree = ET.parse('{}.xml'.format(metadata_path))
root = tree.getroot()
for title in root.iter('title'):
title.text = output_name.replace('_', ' ')
for pubdate_fgdc in root.iter('pubdate'):
pubdate_fgdc.text = last_update_date_str
for descrip_fgdc in root.iter('abstract'):
descrip_fgdc.text += ' Dataset last updated: {}. Dataset last downloaded: {}'.format(
last_update_date_meta, today)
print("Writing updated metadata to {}".format(metadata_path))
tree.write('{}.xml'.format(metadata_path))
print("Metadata update complete for {}".format(metadata_path))
print("Upgrading metadata format for {}".format(metadata_path))
arcpy.UpgradeMetadata_conversion(metadata_path, 'FGDC_TO_ARCGIS')
print("Metadata format upgraded for {}".format(metadata_path))
arcpy.env.workspace = sde_path
arcpy.env.overwriteOutput = True
print("Exporting shapefile to SDE PROD as {}".format(output_name))
arcpy.FeatureClassToFeatureClass_conversion(metadata_path, sde_path, output_name)
print("Removing local storage info")
print("Adding index to BIN field")
arcpy.AddIndex_management(os.path.join(sde_path, output_name), ['BIN'], 'BIN_Index')
print("Index added to BIN field")
print("Adding index to PLUTO_BBL field")
arcpy.AddIndex_management(os.path.join(sde_path, output_name), ['PLUTO_BBL'], 'PLUTO_BBL_Index')
print("Index added to PLUTO_BBL field")
arcpy.XSLTransform_conversion(os.path.join(sde_path, output_name),
xslt_storage,
os.path.join(zip_dir_path, '{}_storage.xml'.format(modified_path.split('.')[0])))
arcpy.XSLTransform_conversion(os.path.join(zip_dir_path, '{}_storage.xml'.format(modified_path.split('.')[0])),
xslt_geoprocess,
os.path.join(zip_dir_path, '{}_geoprocess.xml'.format(modified_path.split('.')[0])))
print("Importing final metadata to {}".format(output_name))
arcpy.MetadataImporter_conversion(os.path.join(zip_dir_path, "{}_geoprocess.xml".format(modified_path.split('.')[0])),
os.path.join(sde_path, output_name))
remove_lcl_storage_xslt = Arcdir + "Metadata/Stylesheets/gpTools/remove local storage info.xslt"
print("Arcdir set")
print("Exporting previous cycle's POPS metadata")
arcpy.env.workspace = current_meta_dir
arcpy.env.overwriteOutput = True
if arcpy.Exists(sde_pops_path):
arcpy.ExportMetadata_conversion(
sde_pops_path, translator,
os.path.join(current_meta_dir, "nyc_pops_meta.xml"))
print("Previous cycle's POPS metadata exported")
arcpy.UpgradeMetadata_conversion(
os.path.join(current_meta_dir, 'nyc_pops_meta.xml'),
'FGDC_TO_ARCGIS')
arcpy.XSLTransform_conversion(
os.path.join(current_meta_dir, 'nyc_pops_meta.xml'),
remove_geoprocess_xslt,
os.path.join(current_meta_dir, 'nyc_pops_meta.xml'))
# Convert csv to shapefile
spatial_ref = config.get('PATHS', 'spatial_ref')
for csv in os.listdir(current_csv_dir):
if csv.endswith('.csv') and 'nycpops' in csv:
print("Converting csv to in-memory DBase table")
arcpy.TableToTable_conversion(os.path.join(current_csv_dir,
csv), 'in_memory',
'{}'.format(csv.split('.')[0]))
arcpy.env.workspace = 'in_memory'
print("Creating XY Event Layer from DBase table")
arcpy.MakeXYEventLayer_management(
def copy_modify_fc(fc, gdb_path):
arcpy.env.workspace = gdb_path
arcpy.env.overwriteOutput = True
desc = arcpy.Describe(fc)
if hasattr(desc, "dataType"):
print("Data set Data Type - {}".format(desc.dataType))
if desc.dataType == "FeatureClass":
print("Copying {} to SDE".format(fc))
arcpy.env.workspace = sde_path
arcpy.env.overwriteOutput = True
arcpy.FeatureClassToFeatureClass_conversion(os.path.join(gdb_path, fc), sde_path, "CSCL_{}".format(fc))
print("{} copy complete".format(fc))
arcpy.ExportMetadata_conversion(os.path.join(sde_path, "CSCL_{}".format(fc)),
translator,
os.path.join(metadata_path, "{}.xml".format(fc)))
print("Exporting metadata with geoprocessing history removed")
arcpy.XSLTransform_conversion(os.path.join(metadata_path, "{}.xml".format(fc)),
stylesheet,
os.path.join(metadata_path, "{}_xslt.xml".format(fc)))
print("Metadata exported")
tree = ET.parse(os.path.join(metadata_path, "{}_xslt.xml".format(fc)))
root = tree.getroot()
print("Removing Publication Date since it is not currently maintained")
for citeinfo in root.iter("citeinfo"):
for pubdate in citeinfo.findall("pubdate"):
citeinfo.remove(pubdate)
print("Publication Date removed")
print("Appending download date to metadata description")
for descrip in root.iter("purpose"):
descrip.text = descrip.text + " Dataset Last Downloaded: {}".format(today_longform)
tree.write(os.path.join(metadata_path, "{}_xslt_moded.xml".format(fc)))
print("Download date appended to metadata description")
print("Importing altered metadata to SDE")
arcpy.MetadataImporter_conversion(os.path.join(metadata_path, "{}_xslt_moded.xml".format(fc)),
os.path.join(sde_path, "CSCL_{}".format(fc)))
print("Metadata imported")
arcpy.UpgradeMetadata_conversion(os.path.join(sde_path, "CSCL_{}".format(fc)), "FGDC_TO_ARCGIS")
print("Metadata upgraded")
if desc.dataType == "Table":
print("Copying {} to SDE".format(fc))
arcpy.env.workspace = sde_path
arcpy.env.overwriteOutput = True
arcpy.TableToTable_conversion(os.path.join(gdb_path, fc), sde_path, "CSCL_{}".format(fc))
print("{} copy complete".format(fc))
arcpy.ExportMetadata_conversion(os.path.join(sde_path, "CSCL_{}".format(fc)),
translator,
os.path.join(metadata_path, "{}.xml".format(fc)))
print("Exporting metadata with geoprocessing history removed")
arcpy.XSLTransform_conversion(os.path.join(metadata_path, "{}.xml".format(fc)),
stylesheet,
os.path.join(metadata_path, "{}_xslt.xml".format(fc)))
print("Metadata exported")
tree = ET.parse(os.path.join(metadata_path, "{}_xslt.xml".format(fc)))
root = tree.getroot()
print("Removing Publication Date since it is not currently maintained")
for citeinfo in root.iter("citeinfo"):
for pubdate in citeinfo.findall("pubdate"):
citeinfo.remove(pubdate)
print("Publication Date removed")
print("Appending download date to metadata description")
for descrip in root.iter("purpose"):
descrip.text = descrip.text + " Dataset Last Downloaded: {}".format(today_longform)
tree.write(os.path.join(metadata_path, "{}_xslt_moded.xml".format(fc)))
print("Download date appended to metadata description")
print("Importing altered metadata to SDE")
arcpy.MetadataImporter_conversion(os.path.join(metadata_path, "{}_xslt_moded.xml".format(fc)),
os.path.join(sde_path, "CSCL_{}".format(fc)))
print("Metadata imported")
arcpy.UpgradeMetadata_conversion(os.path.join(sde_path, "CSCL_{}".format(fc)), "FGDC_TO_ARCGIS")
print("Metadata upgraded")
sde = 'Database Connections\GIS_gis_sde.sde'
conn = arcpy.ArcSDESQLExecute(sde)
sql = '''select owner, table_name from layers order by owner, table_name'''
query = conn.execute(sql)
for x in query:
fc = str(x[0]) + '.' + str(x[1])
print(fc)
acct = x[0]
conn_file = 'Database Connections' + os.sep + 'GIS_' + str(acct) + '.sde'
print(conn_file)
try:
nameXml1 = outXML + os.sep + x[1] + '1.xml'
nameXml2 = outXML + os.sep + x[1] + '2.xml'
nameXml3 = outXML + os.sep + x[1] + '3.xml'
arcpy.XSLTransform_conversion(conn_file + os.sep + x[1], sspath1,
nameXml1, '')
arcpy.MetadataImporter_conversion(nameXml1, conn_file + os.sep + x[1])
arcpy.XSLTransform_conversion(conn_file + os.sep + x[1], sspath2,
nameXml2, '')
arcpy.MetadataImporter_conversion(nameXml2, conn_file + os.sep + x[1])
arcpy.XSLTransform_conversion(conn_file + os.sep + x[1], sspath3,
nameXml3, '')
arcpy.MetadataImporter_conversion(nameXml3, conn_file + os.sep + x[1])
print('complete')
except:
print('something broke...')
continue
shutil.rmtree(outXML)
total = datetime.now() - start
## Needs to be run in the 32-bit python environment installed with ArcGIS C:\Python27\ArcGIS10.3
##Make sure that the workspace has all of the xmls you want to transform as well as the FGDC Plus.xsl file in that location
import os, sys, arcpy
from arcpy import env
work_dir = "L:/Priv/CORFiles/Geospatial_Library/Data/Project/StreamCat/FTP_Staging/StreamCat/Documentation/Metadata/XMLs"
env.workspace = work_dir
html_dir = "L:/Priv/CORFiles/Geospatial_Library/Data/Project/StreamCat/FTP_Staging/StreamCat/Documentation/Metadata"
xslt = "L:/Priv/CORFiles/Geospatial_Library/Data/Project/StreamCat/Metadata/FGDC Plus.xsl"
for xml in arcpy.ListFiles("*.xml"):
html = html_dir + '/' + xml.split(".")[0] + '.html'
if not os.path.exists(html):
# print html
arcpy.XSLTransform_conversion(xml, xslt, html)
def UpdateFCMetadata(fcPathName, metaConnStr):
# Set connection strings and get other settings from configuration file
parser = ConfigParser.SafeConfigParser()
parser.read('LibMgr.ini')
disclaimerFile = parser.get('Metadata', 'disclaimerFile')
idCredit = parser.get('Metadata', 'idCredit')
constraint_useLimit = parser.get('Metadata', 'constraint_useLimit')
organization = parser.get('Metadata', 'organization')
timeperd_current = parser.get('Metadata', 'timeperd_current')
addrtype = parser.get('Metadata', 'addrtype')
address = parser.get('Metadata', 'address')
city = parser.get('Metadata', 'city')
state = parser.get('Metadata', 'state')
zip = parser.get('Metadata', 'zip')
country = parser.get('Metadata', 'country')
phone = parser.get('Metadata', 'phone')
librarian = parser.get('Metadata', 'librarian')
thumbnailsPath = parser.get('Metadata', 'thumbnailsPath')
num_elements = 0
featName = fcPathName.split('.')[-1]
conn = odbcconn(metaConnStr)
metaqry = 'SELECT [FULL_NAME],[COVER_NAME],[ABSTRACT],[UPDATEDATE],[OWNERNAME]' +\
',[PATH],[METAACCESS],[ONMAINT],[MAINTFREQ],[KNOWNERROR],[LINEAGE]' +\
',[DOMAIN],[RECTIFIED],[MAINTORG],[MAINTDESC],[LIBINPUT],[SOURCNAME]' +\
',[SOURCCONTACT],[SOURCDOCNAME],[SOURCDATE],[SOURCSCALE],[SOURCFORMAT]' +\
',[SOUR2NAME],[SOUR2CONTACT],[SOUR2DOCNAME],[SOUR2DATE],[SOUR2SCALE]' +\
',[SOUR2FORMAT],[ONMG],[MGLAYERNAME],[MGSCALELOW],[MGSCALEHIGH] ' +\
'FROM [dbo].[metadata] WHERE [COVER_NAME] = \'' + featName + '\''
df_FCMeta = readsqlqry(
metaqry, conn) # pandas 0.19.1 load query result to pandas dataframe
df_row = df_FCMeta.iloc[0]
qry = 'SELECT [FieldName] AS \'ndx\',[FieldName],[Description] FROM [dbo].[master_metafield] WHERE [CoverName] = \'' + featName + '\''
df_fieldMeta = readsqlqry(
qry, conn,
index_col='ndx') # pandas 0.19.1 load query result to pandas dataframe
arcpy.env.overwriteOutput = True
# install location
dir = arcpy.GetInstallInfo('desktop')['InstallDir']
# stylesheet to use
copy_xslt = r'{0}'.format(
os.path.join(dir, 'Metadata\Stylesheets\gpTools\exact copy of.xslt'))
# temporary XML file
xmlfile = arcpy.CreateScratchName('.xml',
workspace=arcpy.env.scratchFolder)
# export xml
arcpy.XSLTransform_conversion(fcPathName, copy_xslt, xmlfile, '')
# read in XML
tree = ET.parse(xmlfile)
root = tree.getroot()
# build the supplemental info string
sSuppInfo = BuildMetadataSupString(df_row)
# get the dataIdInfo element
dataIdInfoEl = root.find('dataIdInfo')
# dataIdInfo purpose element
subEl = ET.SubElement(dataIdInfoEl, 'idPurp')
subEl.text = df_row['FULL_NAME']
num_elements += 1
# dataIdInfo abstract element
subEl = ET.SubElement(dataIdInfoEl, 'idAbs')
subEl.text = df_row['ABSTRACT'] + sSuppInfo
num_elements += 1
# dataIdInfo access constraint element
subEl = ET.SubElement(dataIdInfoEl, 'accconst')
subEl.text = df_row['METAACCESS']
num_elements += 1
# dataIdInfo credit element
subEl = ET.SubElement(dataIdInfoEl, 'idCredit')
subEl.text = idCredit
num_elements += 1
# dataIdInfo maintenance frequency element
subEl = ET.SubElement(dataIdInfoEl, 'resMaint')
subEl = ET.SubElement(subEl, 'usrDefFreq')
subEl = ET.SubElement(subEl, 'duration')
subEl.text = df_row['MAINTFREQ']
num_elements += 1
# dataIdInfo use limit element
subEl = ET.SubElement(dataIdInfoEl, 'resConst')
subEl = ET.SubElement(subEl, 'Consts')
subEl = ET.SubElement(subEl, 'useLimit')
subEl.text = constraint_useLimit
num_elements += 1
# dataIdInfo keyword elements obtained from FULL_NAME
searchKeysEl = ET.SubElement(dataIdInfoEl, 'searchKeys')
keywords = df_row['FULL_NAME'].split(' ')
for keyword in keywords:
newKeyEl = ET.SubElement(searchKeysEl, 'keyword')
newKeyEl.text = keyword
num_elements += 1
# create the idInfo element
idInfoEl = ET.SubElement(root, 'idInfo')
# idinfo use constraint element
with open(disclaimerFile,
'r') as file: # read the disclaimer text file to a string
disclaimer = file.read()
subEl = ET.SubElement(idInfoEl, 'useconst')
subEl.text = disclaimer
num_elements += 1
# idinfo citation onlink element
# remove the server name portion of the path
path = df_row['PATH'].split('\\')[2]
pathRoot = '\\\\' + path + '\\'
onlink = df_row['PATH'].replace(pathRoot, '')
subEl = ET.SubElement(idInfoEl, 'citation')
citeinfoEl = ET.SubElement(subEl, 'citeinfo')
subEl = ET.SubElement(subEl, 'onlink')
subEl.text = onlink
num_elements += 1
# idinfo citation origin element
subEl = ET.SubElement(citeinfoEl, 'origin')
subEl.text = organization
num_elements += 1
# idinfo citation pubdate element
subEl = ET.SubElement(citeinfoEl, 'pubdate')
subEl.text = datetime.datetime.now().strftime("%B %d, %Y")
num_elements += 1
# create the idInfo timeperd element
timeperdEl = ET.SubElement(idInfoEl, 'timeperd')
# idinfo timeperd update date comment element
subEl = ET.SubElement(timeperdEl, 'current')
subEl.text = timeperd_current
num_elements += 1
# idinfo timeperd update date element
subEl = ET.SubElement(timeperdEl, 'timeinfo')
subEl = ET.SubElement(subEl, 'sngdate')
subEl = ET.SubElement(subEl, 'caldate')
subEl.text = df_row['UPDATEDATE']
num_elements += 1
# create the idInfo descript element
descriptEl = ET.SubElement(idInfoEl, 'descript')
# idinfo descript abstract element
subEl = ET.SubElement(descriptEl, 'abstract')
subEl.text = df_row['ABSTRACT']
num_elements += 1
# idinfo descript purpose element
subEl = ET.SubElement(descriptEl, 'purpose')
subEl.text = df_row['FULL_NAME']
num_elements += 1
# idinfo descript supplinf element
subEl = ET.SubElement(descriptEl, 'supplinf')
subEl.text = sSuppInfo
num_elements += 1
# idinfo keywords themekey element
subEl = ET.SubElement(idInfoEl, 'keywords')
subEl = ET.SubElement(subEl, 'theme')
subEl = ET.SubElement(subEl, 'themekey')
subEl.text = df_row['FULL_NAME']
num_elements += 1
# create the idInfo point of contact elements
subEl = ET.SubElement(idInfoEl, 'ptcontac')
cntinfoEl = ET.SubElement(subEl, 'cntinfo')
cntperpEl = ET.SubElement(cntinfoEl, 'cntperp')
cntaddrEl = ET.SubElement(cntinfoEl, 'cntaddr')
cntvoiceEl = ET.SubElement(cntinfoEl, 'cntvoice')
# idinfo point of contact person element
subEl = ET.SubElement(cntperpEl, 'cntper')
subEl.text = df_row['OWNERNAME']
num_elements += 1
# idinfo point of contact organization element
subEl = ET.SubElement(cntperpEl, 'cntorg')
subEl.text = organization
num_elements += 1
# idinfo point of contact address type element
subEl = ET.SubElement(cntaddrEl, 'addrtype')
subEl.text = addrtype
num_elements += 1
# idinfo point of contact address element
subEl = ET.SubElement(cntaddrEl, 'address')
subEl.text = address
num_elements += 1
# idinfo point of contact city element
subEl = ET.SubElement(cntaddrEl, 'city')
subEl.text = city
num_elements += 1
# idinfo point of contact state element
subEl = ET.SubElement(cntaddrEl, 'state')
subEl.text = state
num_elements += 1
# idinfo point of contact zip element
subEl = ET.SubElement(cntaddrEl, 'postal')
subEl.text = zip
num_elements += 1
# idinfo point of contact country element
subEl = ET.SubElement(cntaddrEl, 'country')
subEl.text = country
num_elements += 1
# idinfo point of contact phone element
subEl = ET.SubElement(cntinfoEl, 'cntvoice')
subEl.text = phone
num_elements += 1
# create the metainfo point of contact elements
metainfoEl = ET.SubElement(root, 'metainfo')
subEl = ET.SubElement(metainfoEl, 'metc')
cntinfoEl = ET.SubElement(subEl, 'cntinfo')
cntorgpEl = ET.SubElement(cntinfoEl, 'cntorgp')
cntaddrEl = ET.SubElement(subEl, 'cntaddr')
# metainfo point of contact person element
subEl = ET.SubElement(cntorgpEl, 'cntper')
subEl.text = librarian
num_elements += 1
# metainfo point of contact organization element
subEl = ET.SubElement(cntorgpEl, 'cntorg')
subEl.text = df_row['OWNERNAME'] + '\n' + organization
num_elements += 1
# metainfo point of contact address type element
subEl = ET.SubElement(cntaddrEl, 'addrtype')
subEl.text = addrtype
num_elements += 1
# metainfo point of contact address element
subEl = ET.SubElement(cntaddrEl, 'address')
subEl.text = address
num_elements += 1
# metainfo point of contact city element
subEl = ET.SubElement(cntaddrEl, 'city')
subEl.text = city
num_elements += 1
# metainfo point of contact state element
subEl = ET.SubElement(cntaddrEl, 'state')
subEl.text = state
num_elements += 1
# metainfo point of contact zip element
subEl = ET.SubElement(cntaddrEl, 'postal')
subEl.text = zip
num_elements += 1
# metainfo point of contact country element
subEl = ET.SubElement(cntaddrEl, 'country')
subEl.text = country
num_elements += 1
# metainfo point of contact phone element
subEl = ET.SubElement(cntinfoEl, 'cntvoice')
subEl.text = phone
num_elements += 1
# idinfo maintenance status element
statusEl = ET.SubElement(idInfoEl, 'status')
subEl = ET.SubElement(statusEl, 'progress')
if df_row['ONMAINT'] and df_row['ONMAINT'].upper() == 'Y':
subEl.text = 'Maintained'
num_elements += 1
else:
subEl.text = 'Not Maintained'
num_elements += 1
# idinfo maintenance frequency element
subEl = ET.SubElement(statusEl, 'update')
subEl.text = df_row['MAINTFREQ']
num_elements += 1
# add descriptions from library metadata table master_metafields to the feature class fields
attrEls = root.findall('eainfo/detailed/attr')
for attrEl in attrEls: # iterate feature class fields
lablEl = attrEl.find('attrlabl') # find the attribute name element
if lablEl is not None: # for unknown reason, the root.findall sometimes gets attributes that are empty
fldname = lablEl.text
try:
descrip = df_fieldMeta.loc[fldname][
'Description'] # get the field description from the dataframe
except Exception as e:
#print('\tNo description for field ' + fldname)
pass # ignore error returned by dataframe loc. Field not in field metadata table.
else:
subEl = ET.SubElement(attrEl,
'attrdef') #field description element
subEl.text = descrip
num_elements += 1
subEl = ET.SubElement(
attrEl, 'attrdefs') #field description source element
subEl.text = 'Pima County'
num_elements += 1
# set metadata thumbnail
jpgFile = thumbnailsPath + '/' + featName + '.jpg'
if os.path.exists(jpgFile):
with open(jpgFile, "rb") as img_file:
strEncoded = base64.b64encode(img_file.read())
attrib = {'EsriPropertyType': 'PictureX'}
subEl = ET.SubElement(root, 'Binary')
subEl = ET.SubElement(subEl, 'Thumbnail')
subEl = ET.SubElement(subEl, 'Data', attrib)
subEl.text = strEncoded
num_elements += 1
if num_elements > 0:
# save modifications to XML
try:
tree.write(xmlfile)
arcpy.MetadataImporter_conversion(xmlfile, fcPathName)
except Exception as e:
print(e.message)
else:
print('No changes to save')
# --------------------------------------------------------------------------- # RunXlst.py # Created on: 2013-04-25 SG # Description: Run an xslt function (to upgrade a Gizinta.xml file) # --------------------------------------------------------------------------- # Copyright 2012-2013 Vertex3 Inc # This work is licensed under the Creative Commons Attribution-ShareAlike 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by-sa/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. # Import arcpy module import arcpy # Local variables: Gizinta_xml = arcpy.GetParameterAsText(0) xsl = "UpgradeGizintaXml.xsl" Gizinta2_xml = arcpy.GetParameterAsText(1) # Process: XSLT Transformation arcpy.XSLTransform_conversion(Gizinta_xml, xsl, Gizinta2_xml, "")
## Needs to be run in the 32-bit python environment installed with ArcGIS C:\Python27\ArcGIS10.3
##Make sure that the workspace has all of the xmls you want to transform as well as the FGDC Plus.xsl file in that location
import os, sys, arcpy
from arcpy import env
work_dir = "L:/Priv/CORFiles/Geospatial_Library_Projects/StreamCat/FTP_Staging/StreamCat/Documentation/Metadata/XMLs"
env.workspace = work_dir
html_dir = "L:/Priv/CORFiles/Geospatial_Library_Projects/StreamCat/FTP_Staging/StreamCat/Documentation/Metadata"
xslt = "L:/Priv/CORFiles/Geospatial_Library_Projects/SSWR1.1B/Metadata/FGDC Plus.xsl"
for xml in arcpy.ListFiles("*.xml"):
print(xml)
html = html_dir + '/' + xml.split(".")[0] + '.html'
if not os.path.exists(html):
# print html
arcpy.XSLTransform_conversion(source=xml,
xslt=xslt,
output=html,
xsltparam="")
waterfront_fc_name.split('\\')[-1]))
desired_metadata_xml_name = r'{}.xml'.format(
os.path.join(desired_meta_path,
waterfront_fc_name.split('\\')[-1]))
print(temp_metadata_xml_name)
metadata_xml_nostorage_name = temp_metadata_xml_name.replace(
'.xml', '_nostorage.xml')
print(metadata_xml_nostorage_name)
metadata_xml_final_name = temp_metadata_xml_name.replace(
'.xml', '_final.xml')
print(metadata_xml_final_name)
print("Exporting xml metadata to temporary location for cleaning")
arcpy.ExportMetadata_conversion(waterfront_fc_name, translator,
temp_metadata_xml_name)
arcpy.XSLTransform_conversion(temp_metadata_xml_name,
xslt_remove_storage,
metadata_xml_nostorage_name)
arcpy.XSLTransform_conversion(metadata_xml_nostorage_name,
xslt_remove_geoproc_hist,
metadata_xml_final_name)
print("Metadata xml final name - {}".format(metadata_xml_final_name))
print('Waterfront FC name - {}'.format(waterfront_fc_name))
arcpy.MetadataImporter_conversion(metadata_xml_final_name,
waterfront_fc_name)
print("Exporting xml metadata to desired location on BytesProd")
arcpy.ExportMetadata_conversion(metadata_xml_final_name, translator,
desired_metadata_xml_name)
print(desired_metadata_xml_name.replace('.xml', '.html'))
arcpy.XSLTransform_conversion(
desired_metadata_xml_name, xslt_html,
desired_metadata_xml_name.replace('.xml', '.html'))
def ARCGIS(workspace):
"""
Set workspace (where datasets to be processed are located) below
"""
arcpy.env.workspace = workspace
ws = arcpy.env.workspace
print 'ws: ', ws
"""
Get ArcGIS Install Location
"""
ARCGIS_INSTALL_LOCATION = arcpy.GetInstallInfo()['InstallDir']
"""
Get location of XSL Translator to go from ArcGIS to FGDC
"""
FGDC_XSL_FILE = ARCGIS_INSTALL_LOCATION + "Metadata\\Translator\\Transforms\\ArcGIS2FGDC.xsl"
"""
Set output location for OGP metadata and log file, then instantiate Logger
"""
if os.path.exists(ws + "\\output\\"):
OUTPUT_LOCATION = ws + "\\output\\"
else:
os.mkdir(ws + "\\output\\")
OUTPUT_LOCATION = ws + "\\output\\"
print 'output location: ', OUTPUT_LOCATION
d = datetime.today()
LOG_NAME = "OGP_MD_LOG_" + d.strftime("%y%m%d%M%S") + ".txt"
sys.stdout = Logger(OUTPUT_LOCATION, LOG_NAME)
files = arcpy.ListFiles()
datasets = arcpy.ListDatasets()
fields = [
'LayerId', 'Name', 'CollectionId', 'Institution', 'InstitutionSort',
'Access', 'DataType', 'DataTypeSort', 'Availability',
'LayerDisplayName', 'LayerDisplayNameSort', 'Publisher',
'PublisherSort', 'Originator', 'OriginatorSort', 'ThemeKeywords',
'PlaceKeywords', 'Abstract', 'MaxY', 'MinY', 'MinX', 'MaxX', 'CenterX',
'CenterY', 'HalfWidth', 'HalfHeight', 'Area', 'ContentDate', 'Location'
]
if os.path.exists(ws + "\\temp.xml"):
print "removed temp xml file"
os.remove(ws + "\\temp.xml")
"""
Set UTC timezone for ContentDate field
"""
utc = pytz.utc
for i in datasets[:5]:
"""
start the clock! obviously not at all necessary
"""
start = clock()
"""
converting metadata from ArcGIS to FGDC using XSL transformation (super fast) and
writing it to a temporary XML file that is then parsed and translated into
final OGP format
"""
print "======================================"
print "Current item is " + i
print "======================================\n"
print "Converting ArcGIS metadata into temporary FGDC XML file"
XSLtrans = arcpy.XSLTransform_conversion(i, FGDC_XSL_FILE, "temp.xml")
print XSLtrans.getMessages(), '\n'
FGDCtree = ElementTree()
root = FGDCtree.parse(ws + "\\temp.xml")
DATATYPE = dataTypeParseFGDC(root)
DATATYPESORT = DATATYPE
#create string representation of FGDC md to be appended at end of OGP md
FGDC_TEXT = ET.tostring(root)
#layerID equals filename sans extension
LAYERID = i[:i.find(".")]
#name equals FGDC title field
#NAME = root.find("idinfo/citation/citeinfo/title").text
try:
NAME = root.findtext("idinfo/citation/citeinfo/title")
except AttributeError as e:
print "Name field doesn't exist! Setting to UNKNOWN for now"
NAME = "UNKNOWN"
COLLECTIONID = "initial collection"
INSTITUTION = "University of Minnesota"
INSTITUTIONSORT = "University of Minnesota"
try:
ACCESS = root.find("idinfo/accconst").text
except AttributeError as e:
print "Access field doesn't exist! Setting to UNKNOWN for now"
ACCESS = "UNKNOWN"
AVAILABILITY = "Online"
LAYERDISPLAYNAME = NAME
LAYERDISPLAYNAMESORT = NAME
try:
#PUBLISHER = root.find("idinfo/citation/citeinfo/pubinfo/publish").text
PUBLISHER = root.findtext(
"idinfo/citation/citeinfo/pubinfo/publish")
except AttributeError as e:
print "Publisher field doesn't exist! Setting to UNKNOWN for now"
PUBLISHER = "UNKNOWN"
finally:
PUBLISHERSORT = PUBLISHER
try:
ORIGINATOR = root.find("idinfo/citation/citeinfo/origin").text
except AttributeError as e:
print "Originator field doesn't exist! Setting to UNKNOWN for now"
ORIGINATOR = "UNKNOWN"
finally:
ORIGINATORSORT = ORIGINATOR
#Combine multiple Keyword elements into a string rep
THEMEKEYWORDS = keywordParse(FGDCtree, "themekey")
PLACEKEYWORDS = keywordParse(FGDCtree, "placekey")
try:
ABSTRACT = root.find("idinfo/descript/abstract").text
except AttributeError as e:
print "No abstract found. Setting to UNKNOWN for now"
ABSTRACT = "UNKNOWN"
try:
MINX = root.find("idinfo/spdom/bounding/westbc").text
MINY = root.find("idinfo/spdom/bounding/southbc").text
MAXX = root.find("idinfo/spdom/bounding/eastbc").text
MAXY = root.find("idinfo/spdom/bounding/northbc").text
except AttributeError as e:
print "extent information not found!"
#SRSPROJECTIONCODE=
#try:
if root.find("idinfo/timeperd/timeinfo/sngdate/caldate") is not None:
dateText = root.find(
"idinfo/timeperd/timeinfo/sngdate/caldate").text
year = int(dateText[0:4])
month = int(dateText[4:6])
day = int(dateText[6:8])
date = datetime(year, month, day)
CONTENTDATE = date.isoformat() + "Z"
elif root.find(
"idinfo/timeperd/timeinfo/rngdates/begdate") is not None:
dateText = root.find(
"idinfo/timeperd/timeinfo/rngdates/begdate").text
year = int(dateText[0:4])
month = int(dateText[4:6])
day = int(dateText[6:8])
date = datetime(year, month, day)
CONTENTDATE = date.isoformat() + "Z"
#except AttributeError as e:
# print "No content date found! setting to UNKNOWN for now"
# CONTENTDATE = "UNKNOWN"
"""
LOCATION
<field name="Location">
{
"ArcGIS93Rest": ["(the url for the REST service)"],
"tilecache": ["(the url for the tilecache, which I believe ArcGIS server does not have)"],
"download": "(downloading URL for the layer)","wfs": "(WFS URL, if has one)"
}
</field>
"""
## #base URL for all images
## REST_URL_BASE = "http://lib-geo1.lib.umn.edu:6080/arcgis/rest/services/OGP/mosaic_ogp_4/ImageServer/exportImage?"
##
## #parameters to pass to URL, in the form of a dictionary
## DOWNLOAD_URL_PARAMS = {}
## DOWNLOAD_URL_PARAMS['bbox'] = MINX+','+MINY+','+MAXX+','+MAXY
## DOWNLOAD_URL_PARAMS['bboxSR'] = '4269'
## DOWNLOAD_URL_PARAMS['imageSR'] = '26915'
## DOWNLOAD_URL_PARAMS['format'] = 'tiff'
## DOWNLOAD_URL_PARAMS['pixelType'] = 'U8'
## DOWNLOAD_URL_PARAMS['size'] = '3000,3000'
## DOWNLOAD_URL_PARAMS['restrictUTurns'] = 'esriNoDataMatchAny'
## DOWNLOAD_URL_PARAMS['interpolation'] = 'RSP_BilinearInterpolation'
## cursor = arcpy.SearchCursor(u"\\mosaic245.gdb\\mosaic_ogp_1","name = '"+ LAYERID+"'")
## raster_ID = cursor.next().OBJECTID
## DOWNLOAD_URL_PARAMS['mosaicRule'] ={}
## DOWNLOAD_URL_PARAMS['mosaicRule']['mosaicMethod'] = 'esriMosaicLockRaster'
## DOWNLOAD_URL_PARAMS['mosaicRule']['lockRasterIds'] = [raster_ID]
## DOWNLOAD_URL_PARAMS['f']='image'
##
## DOWNLOAD_URL = REST_URL_BASE + urllib.urlencode(DOWNLOAD_URL_PARAMS)
##
## REST_URL= REST_URL_BASE + urllib.urlencode({'mosaicRule': DOWNLOAD_URL_PARAMS['mosaicRule']})
##
## LOCATION = '{"ArcGIS93Rest" : "' + REST_URL + '", "tilecache" : "None", "download" : "' + DOWNLOAD_URL + '", "wfs" : "None"}'
"""
Put it all together to make the OGP metadata xml file
"""
OGPtree = ElementTree()
OGProot = Element("add", allowDups="false")
doc = SubElement(OGProot, "doc")
OGPtree._setroot(OGProot)
try:
for field in fields:
fieldEle = SubElement(doc, "field", name=field)
fieldEle.text = locals()[field.upper()]
except KeyError as e:
print "Nonexistant key: ", field
fgdc_text = SubElement(doc, "field", name="FgdcText")
fgdc_text.text = '<?xml version="1.0"?>' + FGDC_TEXT
if os.path.exists(OUTPUT_LOCATION + LAYERID + "_OGP.xml"):
existsPrompt = raw_input(
'OGP metadata already exists! Overwrite? (Y/N): ')
if existsPrompt == 'Y':
OGPtree.write(OUTPUT_LOCATION + LAYERID + "_OGP.xml")
print "Output file: " + OUTPUT_LOCATION + LAYERID + "_OGP.xml"
else:
pass
else:
OGPtree.write(OUTPUT_LOCATION + LAYERID + "_OGP.xml")
print "Output file: " + OUTPUT_LOCATION + LAYERID + "_OGP.xml"
os.remove(ws + "\\temp.xml")
print "Next!\n"
end = clock()
if (end - start) > 60:
mins = str(math.ceil((end - start) / 60))
secs = str(math.ceil((end - start) % 60))
print str(
len(datasets)
) + " datasets processed in " + mins + " minutes and " + secs + " seconds"
else:
secs = str(math.ceil(end - start))
print str(
len(datasets)) + " datasets processed in " + secs + " seconds"
sys.stdout = sys.__stdout__
print 'Done. See log file for information'
