def xmlExport(fc): """ Export metadata from the item and convert to a stand-alone xml file Parameters ---------- fc: Data Element;Layer The item whose metadata will be converted or a stand-alone XML file that will be converted. Returns ------- output_xml: File XML file """ # arcgis install directory dir = arcpy.GetInstallInfo("desktop")["InstallDir"] # exact copy xslt transformation xslt = dir + "Metadata/Stylesheets/gpTools/exact copy of.xslt" # set output directory out_dir = arcpy.env.scratchFolder # set arcpy workspace arcpy.env.workspace = out_dir # set workspace overwrite true arcpy.env.overwriteOutput = True #assemble output xml file name output_xml = path.join(out_dir, path.basename(fc) + ".xml") #XSLT transforamtion arcpy.XSLTransform_conversion(fc, xslt, output_xml) return output_xml
def export_featureclass(input_path, output_name): print("Upgrading downloaded metadata to ArcGIS standard") arcpy.UpgradeMetadata_conversion(input_path, 'FGDC_TO_ARCGIS') print("Downloaded metadata upgraded to ArcGIS standard") print("Overwriting original metadata with DCP standard") arcpy.MetadataImporter_conversion( os.path.join(template_path, '{}.xml'.format(output_name)), input_path) print("Original metadata overwritten") tree = ET.parse('{}.xml'.format(input_path)) root = tree.getroot() for title in root.iter('title'): title.text = output_name.replace('_', ' ') for pubdate_fgdc in root.iter('pubdate'): pubdate_fgdc.text = last_update_date_str for descrip_fgdc in root.iter('abstract'): descrip_fgdc.text += ' Dataset last updated: {}. Dataset last downloaded: {}'.format( last_update_date_meta, today) print("Writing updated metadata to {}".format(input_path)) tree.write('{}.xml'.format(input_path)) print("Metadata update complete for {}".format(input_path)) print("Upgrading metadata format for {}".format(input_path)) arcpy.UpgradeMetadata_conversion(input_path, 'FGDC_TO_ARCGIS') print("Metadata format upgraded for {}".format(input_path)) print("Exporting shapefile to SDE PROD as {}".format(output_name)) arcpy.FeatureClassToFeatureClass_conversion(input_path, sde_path, output_name) print("Removing local storage info") arcpy.XSLTransform_conversion( os.path.join(sde_path, output_name), xslt_storage, os.path.join(zip_dir_path, '{}_storage.xml'.format(input_path))) arcpy.XSLTransform_conversion('{}_storage.xml'.format(input_path), xslt_geoprocess, '{}_geoprocess.xml'.format(input_path)) print("Importing final metadata to {}".format(output_name)) arcpy.MetadataImporter_conversion( os.path.join(zip_dir_path, "{}_geoprocess.xml".format(input_path)), os.path.join(sde_path, output_name))
def RemoveUnwantedMetadata(fgdb, fc): # set environments & variables arcpy.env.workspace = fgdb # the full path of feature class is passed, and we need the base name featureClass = os.path.basename(fc) # the below construction makes it installation independent installDir = arcpy.GetInstallInfo()['InstallDir'] ssPath1 = 'Metadata\\Stylesheets\\gpTools\\remove geoprocessing history.xslt' ssPath2 = 'Metadata\\Stylesheets\\gpTools\\remove local storage info.xslt' xsltPath1 = installDir + ssPath1 xsltPath2 = installDir + ssPath2 # delete the output directory if it already exists so you don't have to overwrite outXml = 'c:\\XML_out' if os.path.exists(outXml): shutil.rmtree(outXml) os.mkdir(outXml) # Remove geoprocessing history and local machine name/paths # output xml for removing gp history nameXml1 = outXml + os.sep + str(featureClass) + ".xml" # output xml for removing local storage info nameXml2 = outXml + os.sep + str(featureClass) + "2.xml" try: arcpy.XSLTransform_conversion(featureClass, xsltPath1, nameXml1, "") arcpy.MetadataImporter_conversion(nameXml1, featureClass) arcpy.XSLTransform_conversion(featureClass, xsltPath2, nameXml2, "") arcpy.MetadataImporter_conversion(nameXml2, featureClass) return (True, "Succeeded") except Exception as e: error = "Failed: metadata removal for " + str(fc) + " " + str(e) return (False, error) finally: # clean up shutil.rmtree(outXml)
def updateFeatureClassAbstract(featureClassLocation, featureClassName, abstractText): # set up scratch file location for XML metadata_file = metadataScratchFilePath(featureClassName) if os.path.exists(metadata_file): os.remove(metadata_file) # create XML scratch file arcpy.env.workspace = featureClassLocation arcpy.XSLTransform_conversion(featureClassName, XSLTFilePath(), metadata_file) # update XML file with new abstract updateMetaDataFileAbstract(metadata_file, abstractText) # write updates to the feature class arcpy.ImportMetadata_conversion(metadata_file, "FROM_ARCGIS", featureClassName, Enable_automatic_updates=False)
def xmlExport(fc): """ export metadata from feature class and save as xml file """ # arcgis install directory dir = arcpy.GetInstallInfo("desktop")["InstallDir"] # exact copy xslt transformation xslt = dir + "Metadata/Stylesheets/gpTools/exact copy of.xslt" # set output directory out_dir = outputFolder # set arcpy workspace arcpy.env.workspace = out_dir # set workspace overwrite true arcpy.env.overwriteOutput = True #assemble output xml file name output_xml = path.join(out_dir, path.basename(fc) + ".xml") #XSLT transforamtion arcpy.XSLTransform_conversion(fc, xslt, output_xml)
def export_metadata(): """Exports the feature class metadata to an xml file Returns: None """ folder = 'metadata' name = get_dataset_filename() # Create a metadata folder in the temp directory if it does not exist temp_working_folder = os.path.join(temp_workspace, folder) create_folder(temp_working_folder, True) # Set the destinion of the metadata export source = staging_feature_class raw_metadata_export = os.path.join(temp_working_folder, name + "_raw.xml") # Export the metadata arcpy.env.workspace = temp_working_folder installDir = arcpy.GetInstallInfo("desktop")["InstallDir"] translator = installDir + "Metadata/Translator/ARCGIS2FGDC.xml" arcpy.ExportMetadata_conversion(source, translator, raw_metadata_export) # Process: XSLT Transformation to remove any sensitive info or format destination = os.path.join(temp_working_folder, name + ".xml") if os.path.exists(args.metadata_xslt): logger.info("Applying metadata XSLT: " + args.metadata_xslt) arcpy.XSLTransform_conversion(raw_metadata_export, args.metadata_xslt, destination, "") # Reimport the clean metadata into the FGDB logger.debug("Reimporting metadata to file geodatabase " + destination) arcpy.MetadataImporter_conversion(destination, staging_feature_class) else: # If no transformation exists, just rename and publish the raw metadata logger.warn("Metadata XSLT not found") os.rename(raw_metadata_export, destination) # Publish the metadata to the download folder publish_file(temp_working_folder, name + ".xml", "metadata")
def GetRawXML(InputXML_or_DataLayer, OutputRawXML): ''' Obtains the raw metadata from a dataset. ''' #Finds the install directory and obtains the xslt used to grab all of the xml contents, unedited installDir = arcpy.GetInstallInfo("desktop")["InstallDir"] xsltPath = "Metadata/Stylesheets/gpTools/exact copy of.xslt" Exact_Copy_XSLT = os.path.join(installDir,xsltPath) Exact_Copy_XSLT = os.path.realpath(Exact_Copy_XSLT) arcpy.AddMessage("The raw xml is now being extracted into a temporary file for further processing... \n") #Process: Get Metadata try: if os.path.exists(OutputRawXML): os.remove(OutputRawXML) except IOError: raise Exception, "Error. This tool requires read/write access to the directories where the temporary and final outputs are saved. Please choose another directory for the tool outputs." try: arcpy.XSLTransform_conversion(InputXML_or_DataLayer, Exact_Copy_XSLT, OutputRawXML) #arcpy.AddMessage(arcpy.GetMessages(0)) except: raise Exception, str(arcpy.GetMessages(2))
def CreateCopyMDRecord(InputXML_or_DataLayer, MDRecordCopy): #Creates unmodified backup of input. ''' Obtains the raw metadata from a dataset. This makes an exact copy. ''' #Finds the install directory and obtains the xslt used to grab all of the xml contents, unedited installDir = arcpy.GetInstallInfo("desktop")["InstallDir"] xsltPath = "Metadata/Stylesheets/gpTools/exact copy of.xslt" Exact_Copy_XSLT = os.path.join(installDir, xsltPath) Exact_Copy_XSLT = os.path.realpath(Exact_Copy_XSLT) #Process: Get Metadata try: if os.path.exists(MDRecordCopy): os.remove(MDRecordCopy) except IOError: raise Exception, "This tool requires read/write access to the directories where the temporary and final outputs are saved. Please choose another directory for the tool outputs." try: arcpy.XSLTransform_conversion(InputXML_or_DataLayer, Exact_Copy_XSLT, MDRecordCopy) except: raise Exception, str(arcpy.GetMessages(2))
def __init__(self, dataset=None, metadata_file=None, items=None, temp_folder=metadata_temp_folder, loglevel='INFO'): screen_handler = None self.logger = logging.getLogger("__name__") if not len(self.logger.handlers): screen_handler = logging.StreamHandler() # set up the logging level at debug, but only write INFO or higher to the screen self.logger.setLevel(logging.DEBUG) self.logger.addHandler(screen_handler) else: for handler in self.logger.handlers: # take the first one available if isinstance(handler, logging.StreamHandler): screen_handler = handler break # just making sure that there is a screenhandler if screen_handler is None: screen_handler = logging.StreamHandler() self.logger.setLevel(logging.DEBUG) self.logger.addHandler(screen_handler) if loglevel.upper() == "CRITICAL": screen_handler.setLevel(logging.CRITICAL) elif loglevel.upper() == "ERROR": screen_handler.setLevel(logging.ERROR) elif loglevel.upper() == "WARNING": screen_handler.setLevel(logging.WARNING) elif loglevel.upper() == "INFO": screen_handler.setLevel(logging.INFO) elif loglevel.upper() == "DEBUG": screen_handler.setLevel(logging.DEBUG) else: screen_handler.setLevel(logging.NOTSET) self.logger.debug("Set logging mode to {0}".format(loglevel)) if items is None: items = list() self.items = items self.metadata_file = metadata_file self.elements = xml.etree.ElementTree.ElementTree() self.temp_folder = temp_folder self.dataset = dataset self._gdb_datasets = ["FeatureClass", "Table", "RasterDataset", "RasterCatalog", "MosaicDataset"] self._simple_datasets = ["ShapeFile", "RasterDataset", "Layer"] self._layers = ["FeatureLayer"] if self.dataset: # Check if dataset is set # export the metadata to the temporary location self.data_type = self.get_datatype() # for layers get the underlying dataset and start over if self.data_type in self._layers: desc = arcpy.Describe(self.dataset) self.data_type = desc.dataElement.dataType self.dataset = desc.dataElement.catalogPath # overwrite path to dataset with layer's data source self._workspace = self.get_workspace() self._workspace_type = self.get_workspace_type() # Datasets in Filesystem have metadata attached as XML file # we can directly write to it if self._workspace_type == 'FileSystem': if self.data_type in self._simple_datasets: xml_file = self.dataset + ".xml" #if no XML file exists create one and add most basic metadata item to it if not os.path.exists(xml_file): self._create_xml_file(xml_file) self.metadata_file = xml_file else: raise TypeError("Cannot read {0}. Data type is not supported".format(self.dataset)) # Metadata for GDB datasets are stored inside the GDB itself. # We need to first export them to a temporary file, modify them and then import them back else: if self.data_type in self._gdb_datasets: metadata_filename = os.path.basename(self.dataset) + ".xml" self.metadata_file = os.path.join(self.temp_folder, metadata_filename) if os.path.exists(self.metadata_file): os.remove(self.metadata_file) self.logger.debug("Exporting metadata to temporary file {0!s}".format(self.metadata_file)) arcpy.XSLTransform_conversion(self.dataset, xslt, self.metadata_file) else: raise TypeError("Cannot read {0}. Data type is not supported".format(self.dataset)) elif self.metadata_file: # Check if metadata file is set instead self.data_type = 'MetadataFile' if self.metadata_file.endswith('.xml'): if not os.path.exists(self.metadata_file): self._create_xml_file(self.metadata_file) self._workspace_type = 'FileSystem' else: raise TypeError("Metadata file is not an XML file. Check file extension") self.elements.parse(self.metadata_file) # create these all after the parsing happens so that if they have any self initialization, they can correctly perform it for name in elements.keys(): if "sync" in elements[name].keys(): sync = elements[name]["sync"] else: sync = None if "unsupported" in elements[name].keys(): if self.data_type in elements[name]["unsupported"]: self.logger.debug("{0} not supported for {1}. SKIP".format(name, self.data_type)) continue setattr(self, "_{0!s}".format(name), None) if elements[name]['type'] in ["string", "datetime", "date", "time", "integer", "float"]: setattr(self, "_{0}".format(name), MetadataItem(elements[name]['path'], name, self, sync)) if self.__dict__["_{0}".format(name)].value is not None: setattr(self, name, self.__dict__["_{0}".format(name)].value.strip()) else: setattr(self, name, self.__dict__["_{0}".format(name)].value) elif elements[name]['type'] == "attribute": setattr(self, "_{0}".format(name), MetadataItem(elements[name]['path'], name, self, sync)) if isinstance(self.__dict__["_{0}".format(name)].attributes, dict): key = elements[name]['key'] values = elements[name]['values'] if key in self.__dict__["_{0}".format(name)].attributes.keys(): v = self.__dict__["_{0}".format(name)].attributes[elements[name]['key']] for value in values: if v in value: setattr(self, name, value[0]) break else: setattr(self, name, None) elif elements[name]['type'] == "list": setattr(self, "_{0}".format(name), MetadataValueList(elements[name]["tagname"], elements[name]['path'], name, self, sync)) #setattr(self, name, self.__dict__["_{}".format(name)].value) #setattr(self, name, ListValues(self.__dict__["_{}".format(name)], name)) elif elements[name]['type'] == "language": setattr(self, "_{0}".format(name), MetadataLanguage(elements[name]['path'], name, self, sync)) if self.__dict__["_{0}".format(name)].value is not None: setattr(self, name, self.__dict__["_{0}".format(name)].value.strip()) else: setattr(self, name, self.__dict__["_{0}".format(name)].value) # TODO: turn on sync elif elements[name]['type'] == "parent_item": setattr(self, "_{0}".format(name), MetadataParentItem(elements[name]['path'], self, elements[name]['elements'])) setattr(self, name, self.__dict__["_{0}".format(name)]) elif elements[name]['type'] == "object_list": setattr(self, "_{0}".format(name), MetadataObjectList(elements[name]["tagname"], elements[name]['path'], self, elements[name]['elements'], sync)) #setattr(self, name, self.__dict__["_{}".format(name)]) if elements[name] in self.__dict__.keys(): self.items.append(getattr(self, "_{0}".format(elements[name]))) if items: self.initialize_items()
def write_metadata(input_items, template_xml, xslt_file, summary, description, tags, data_credits, use_constraints, overwrite, token_header): """Writes metadata.""" updated = 0 errors = 0 skipped = 0 global processed_count for item in input_items: try: id = item[1] path = item[0] # Temporary XML file temp_xml = tempfile.NamedTemporaryFile(suffix='.xml', delete=True).name # Export xml try: arcpy.XSLTransform_conversion(path, xslt_file, temp_xml) except arcpy.ExecuteError: src_xml = os.path.join( arcpy.Describe(path).path, '{0}.xml'.format(os.path.basename(path))) shutil.copyfile(template_xml, src_xml) arcpy.XSLTransform_conversion(src_xml, xslt_file, temp_xml) # Read in XML tree = eTree.parse(temp_xml) root = tree.getroot() changes = 0 # ISO allows many dataIdInfo groups; ArcGIS generally supports only one. data_id_elements = root.findall(".//dataIdInfo") if not data_id_elements: data_id_elements = [eTree.SubElement(root, 'dataIdInfo')] for data_id_element in data_id_elements: # Write summary. summary_element = root.findall(".//idPurp") if not summary_element: summary_element = eTree.SubElement(data_id_element, 'idPurp') summary_element.text = summary changes += 1 else: for element in summary_element: if summary and (overwrite or element.text is None): element.text = summary changes += 1 # Write description. description_element = root.findall(".//idAbs") if not description_element: description_element = eTree.SubElement( data_id_element, 'idAbs') description_element.text = description changes += 1 else: for element in description_element: if description and (overwrite or element.text is None): element.text = description changes += 1 # Write tags. tags = task_utils.get_unique_strings(tags) search_keys = root.findall(".//searchKeys") if not search_keys: search_element = eTree.SubElement(data_id_element, 'searchKeys') for tag in tags: new_tag = eTree.SubElement(search_element, "keyword") new_tag.text = tag changes += 1 elif not overwrite: # Still add any new tags. for search_element in search_keys: if tags: for tag in tags: if tag.lower() not in [ se.text.lower() for se in search_element.findall('.//keyword') ]: new_tag = eTree.SubElement( search_element, "keyword") new_tag.text = tag changes += 1 else: if tags: for search_element in search_keys: [ search_element.remove(e) for e in search_element.findall('.//keyword') ] for tag in tags: new_tag = eTree.SubElement( search_element, "keyword") new_tag.text = tag changes += 1 # Write credits. credits_element = root.findall(".//idCredit") if not credits_element: credits_element = eTree.SubElement(data_id_element, 'idCredit') credits_element.text = data_credits changes += 1 else: for element in credits_element: if data_credits and (overwrite or element.text is None): element.text = data_credits changes += 1 # Write use constraints. res_constraints = root.findall(".//resConst") if not res_constraints: res_constraint_element = eTree.SubElement( data_id_element, 'resConst') const_element = eTree.SubElement(res_constraint_element, 'Consts') new_constraint = eTree.SubElement(const_element, 'useLimit') new_constraint.text = use_constraints changes += 1 elif not overwrite: constraint_elements = root.findall('.//Consts') for element in constraint_elements: if use_constraints: new_constraint = eTree.SubElement( element, 'useLimit') new_constraint.text = use_constraints changes += 1 else: if use_constraints: constraint_elements = root.findall('.//Consts') if constraint_elements: [ constraint_elements[0].remove(e) for e in constraint_elements[0].findall('.//useLimit') ] new_constraint = eTree.SubElement( constraint_elements[0], 'useLimit') new_constraint.text = use_constraints changes += 1 if changes > 0: # Save modifications to the temporary XML file. tree.write(temp_xml) # Import the XML file to the item; existing metadata is replaced. arcpy.MetadataImporter_conversion(temp_xml, path) status_writer.send_percent( processed_count / result_count, _('Metadata updated for: {0}').format(path), 'write_metadata') processed_count += 1 try: index_item(id, token_header) except (IndexError, urllib2.HTTPError, urllib2.URLError) as e: status_writer.send_status(e.message) pass updated += 1 else: processed_count += 1 status_writer.send_percent( processed_count / result_count, _('No metadata changes for: {0}').format(path), 'write_metadata') skipped_reasons[path] = _( 'No metadata changes for: {0}').format(path) skipped += 1 except Exception as ex: processed_count += 1 status_writer.send_percent(processed_count / result_count, _('Skipped: {0}').format(path), 'write_metadata') status_writer.send_status(_('FAIL: {0}').format(repr(ex))) errors_reasons[path] = repr(ex) errors += 1 pass return updated, errors, skipped
def __init__(self, dataset=None, metadata_file=None, items=None, temp_folder=metadata_temp_folder): if items is None: items = list() self.items = items self.dataset = dataset self.metadata_file = metadata_file self.elements = xml.etree.ElementTree.ElementTree() self.temp_folder = temp_folder self._gdb_datasets = ["FeatureClass", "Table", "RasterDataset", "RasterCatalog", "MosaicDataset"] self._simple_datasets = ["ShapeFile", "RasterDataset"] self._layers = ["FeatureLayer", "Layer"] if self.dataset: # for both, we want to export the metadata out # export the metadata to the temporary location self.data_type = self.get_datatype() # for layers get the underlying dataset and start over if self.data_type in self._layers: desc = arcpy.Describe(self.dataset) self.data_type = desc.dataElement.dataType self.dataset = desc.dataElement.catalogPath self._workspace = self.get_workspace() self._workspace_type = self.get_workspace_type() # Datasets in Filesystem have metadata attached as XML file # we can directly write to it if self._workspace_type == 'FileSystem': if self.data_type in self._simple_datasets: xml_file = self.dataset + ".xml" # if no XML file exists create one and add most basic metadata item to it if not os.path.exists(xml_file): with open(xml_file, "w") as f: f.write('<metadata xml:lang="en"></metadata>') self.metadata_file = xml_file else: raise TypeError("Datatype is not supported") # Metadata for GDB datasets are stored inside the GDB itself. # We need to first export them to a temporary file, modify them and then import them back else: if self.data_type in self._gdb_datasets: metadata_filename = os.path.basename(self.dataset) + ".xml" self.metadata_file = os.path.join(self.temp_folder, metadata_filename) if os.path.exists(self.metadata_file): os.remove(self.metadata_file) logwrite("Exporting metadata to temporary file %s" % self.metadata_file) arcpy.XSLTransform_conversion(self.dataset, xslt, self.metadata_file) else: raise TypeError("Datatype is not supported") self.elements.parse(self.metadata_file) # create these all after the parsing happens so that if they have any self initialization, # they can correctly perform it for name in elements.keys(): setattr(self, "_%s" % name, None) if elements[name]['type'] in ["string", "date", "integer", "float"]: setattr(self, "_{}".format(name), MetadataItem(elements[name]['path'], name, self)) if self.__dict__["_{}".format(name)].value is not None: setattr(self, name, self.__dict__["_{}".format(name)].value.strip()) else: setattr(self, name, self.__dict__["_{}".format(name)].value) elif elements[name]['type'] == "list": setattr(self, "_{}".format(name), MetadataList(elements[name]["tagname"], elements[name]['path'], name, self)) setattr(self, name, self.__dict__["_{}".format(name)].value) elif elements[name]['type'] == "language": setattr(self, "_{}".format(name), MetadataLanguage(elements[name]['path'], name, self)) if self.__dict__["_{}".format(name)].value is not None: setattr(self, name, self.__dict__["_{}".format(name)].value.strip()) else: setattr(self, name, self.__dict__["_{}".format(name)].value) elif elements[name]['type'] == "local": setattr(self, name, MetadataLocals(elements[name]['path'], name, self)) elif elements[name]['type'] == "contact": setattr(self, "_{}".format(name), MetadataContact(elements[name]['path'], name, self)) setattr(self, name, self.__dict__["_{}".format(name)]) if elements[name] in self.__dict__.keys(): self.items.append(getattr(self, "_{}".format(elements[name]))) if items: self.initialize_items()
def export_featureclass(input_path, output_name, modified_path): ''' Index BIN field, create new double field called PLUTO_BBL populate new field with BBL values with conditionals accounting for incorrect BBL values (either short or missing) and reorder output tables to include new field within the previous standard ''' print("Creating PLUTO BBL field") arcpy.AddField_management(input_path, 'PLUTO_BBL', 'DOUBLE') print("PLUTO BBL field created") cursor = arcpy.da.UpdateCursor(input_path, ['BASE_BBL', 'MPLUTO_BBL', 'PLUTO_BBL', 'BIN']) for row in cursor: print("Parsing BASE_BBLS: {} and MPLUTO_BBLS: {}".format(row[0], row[1])) if len(row[0]) != 10: error_list.add("Short BBL. BASE_BBL = {} ; MPLUTO_BBL = {} ; BIN = {}".format(row[0], row[1], row[3])) if row[1].isspace() == True: if row[0].isspace() == True: error_list.add("Missing BBL. BIN = {}".format(row[3])) continue if row[0].isspace() == False and r"`" not in row[0]: row[2] = float(row[0]) cursor.updateRow(row) continue ''' Case where ` character is included in BBL value. Interim value of 1 to replace this character until I can confirm with Matt. This would create the correct BBL based on DOB Property Profile Overview lookup information ''' if r"`" in row[0]: new_bbl = row[0].replace("`", "1") row[2] = float(new_bbl) cursor.updateRow(row) continue else: row[2] = float(row[0]) cursor.updateRow(row) continue if row[1] == r'': if row[0] == r'': error_list.add("Missing BBL. BIN = {}".format(row[3])) if r"`" in row[0]: new_bbl = row[0].replace("`", "1") row[2] = float(new_bbl) cursor.updateRow(row) continue else: row[2] = float(row[0]) cursor.updateRow(row) continue else: row[2] = float(row[1]) cursor.updateRow(row) continue print("Creating field map in order to re-order the export tables") fieldMap = arcpy.FieldMappings() fieldMap.addTable(input_path) newFieldMap = arcpy.FieldMappings() print("Field mapping created") print("Adding fields to new field map") newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('NAME'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('BIN'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('CNSTRCT_YR'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('LSTMODDATE'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('LSTSTATYPE'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('DOITT_ID'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('HEIGHTROOF'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('FEAT_CODE'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('GROUNDELEV'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('BASE_BBL'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('MPLUTO_BBL'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('PLUTO_BBL'))) newFieldMap.addFieldMap(fieldMap.getFieldMap(fieldMap.findFieldMapIndex('GEOMSOURCE'))) print("All fields added to field map") print("Exporting as modified shapefile in temporary directory") arcpy.FeatureClassToFeatureClass_conversion(in_features=input_path, out_path=zip_dir_path, out_name=modified_path.split('.')[0], field_mapping=newFieldMap) print("Modified shapefile exported") print("Upgrading downloaded metadata to ArcGIS standard") arcpy.env.workspace = zip_dir_path arcpy.env.overwriteOutput = True metadata_path = os.path.join(zip_dir_path, modified_path) arcpy.UpgradeMetadata_conversion(metadata_path, 'FGDC_TO_ARCGIS') print("Downloaded metadata upgraded to ArcGIS standard") print("Overwriting original metadata with DCP standard") arcpy.MetadataImporter_conversion(os.path.join(template_path, '{}.xml'.format(output_name)), metadata_path) print("Original metadata overwritten") tree = ET.parse('{}.xml'.format(metadata_path)) root = tree.getroot() for title in root.iter('title'): title.text = output_name.replace('_', ' ') for pubdate_fgdc in root.iter('pubdate'): pubdate_fgdc.text = last_update_date_str for descrip_fgdc in root.iter('abstract'): descrip_fgdc.text += ' Dataset last updated: {}. Dataset last downloaded: {}'.format( last_update_date_meta, today) print("Writing updated metadata to {}".format(metadata_path)) tree.write('{}.xml'.format(metadata_path)) print("Metadata update complete for {}".format(metadata_path)) print("Upgrading metadata format for {}".format(metadata_path)) arcpy.UpgradeMetadata_conversion(metadata_path, 'FGDC_TO_ARCGIS') print("Metadata format upgraded for {}".format(metadata_path)) arcpy.env.workspace = sde_path arcpy.env.overwriteOutput = True print("Exporting shapefile to SDE PROD as {}".format(output_name)) arcpy.FeatureClassToFeatureClass_conversion(metadata_path, sde_path, output_name) print("Removing local storage info") print("Adding index to BIN field") arcpy.AddIndex_management(os.path.join(sde_path, output_name), ['BIN'], 'BIN_Index') print("Index added to BIN field") print("Adding index to PLUTO_BBL field") arcpy.AddIndex_management(os.path.join(sde_path, output_name), ['PLUTO_BBL'], 'PLUTO_BBL_Index') print("Index added to PLUTO_BBL field") arcpy.XSLTransform_conversion(os.path.join(sde_path, output_name), xslt_storage, os.path.join(zip_dir_path, '{}_storage.xml'.format(modified_path.split('.')[0]))) arcpy.XSLTransform_conversion(os.path.join(zip_dir_path, '{}_storage.xml'.format(modified_path.split('.')[0])), xslt_geoprocess, os.path.join(zip_dir_path, '{}_geoprocess.xml'.format(modified_path.split('.')[0]))) print("Importing final metadata to {}".format(output_name)) arcpy.MetadataImporter_conversion(os.path.join(zip_dir_path, "{}_geoprocess.xml".format(modified_path.split('.')[0])), os.path.join(sde_path, output_name))
remove_lcl_storage_xslt = Arcdir + "Metadata/Stylesheets/gpTools/remove local storage info.xslt" print("Arcdir set") print("Exporting previous cycle's POPS metadata") arcpy.env.workspace = current_meta_dir arcpy.env.overwriteOutput = True if arcpy.Exists(sde_pops_path): arcpy.ExportMetadata_conversion( sde_pops_path, translator, os.path.join(current_meta_dir, "nyc_pops_meta.xml")) print("Previous cycle's POPS metadata exported") arcpy.UpgradeMetadata_conversion( os.path.join(current_meta_dir, 'nyc_pops_meta.xml'), 'FGDC_TO_ARCGIS') arcpy.XSLTransform_conversion( os.path.join(current_meta_dir, 'nyc_pops_meta.xml'), remove_geoprocess_xslt, os.path.join(current_meta_dir, 'nyc_pops_meta.xml')) # Convert csv to shapefile spatial_ref = config.get('PATHS', 'spatial_ref') for csv in os.listdir(current_csv_dir): if csv.endswith('.csv') and 'nycpops' in csv: print("Converting csv to in-memory DBase table") arcpy.TableToTable_conversion(os.path.join(current_csv_dir, csv), 'in_memory', '{}'.format(csv.split('.')[0])) arcpy.env.workspace = 'in_memory' print("Creating XY Event Layer from DBase table") arcpy.MakeXYEventLayer_management(
def copy_modify_fc(fc, gdb_path): arcpy.env.workspace = gdb_path arcpy.env.overwriteOutput = True desc = arcpy.Describe(fc) if hasattr(desc, "dataType"): print("Data set Data Type - {}".format(desc.dataType)) if desc.dataType == "FeatureClass": print("Copying {} to SDE".format(fc)) arcpy.env.workspace = sde_path arcpy.env.overwriteOutput = True arcpy.FeatureClassToFeatureClass_conversion(os.path.join(gdb_path, fc), sde_path, "CSCL_{}".format(fc)) print("{} copy complete".format(fc)) arcpy.ExportMetadata_conversion(os.path.join(sde_path, "CSCL_{}".format(fc)), translator, os.path.join(metadata_path, "{}.xml".format(fc))) print("Exporting metadata with geoprocessing history removed") arcpy.XSLTransform_conversion(os.path.join(metadata_path, "{}.xml".format(fc)), stylesheet, os.path.join(metadata_path, "{}_xslt.xml".format(fc))) print("Metadata exported") tree = ET.parse(os.path.join(metadata_path, "{}_xslt.xml".format(fc))) root = tree.getroot() print("Removing Publication Date since it is not currently maintained") for citeinfo in root.iter("citeinfo"): for pubdate in citeinfo.findall("pubdate"): citeinfo.remove(pubdate) print("Publication Date removed") print("Appending download date to metadata description") for descrip in root.iter("purpose"): descrip.text = descrip.text + " Dataset Last Downloaded: {}".format(today_longform) tree.write(os.path.join(metadata_path, "{}_xslt_moded.xml".format(fc))) print("Download date appended to metadata description") print("Importing altered metadata to SDE") arcpy.MetadataImporter_conversion(os.path.join(metadata_path, "{}_xslt_moded.xml".format(fc)), os.path.join(sde_path, "CSCL_{}".format(fc))) print("Metadata imported") arcpy.UpgradeMetadata_conversion(os.path.join(sde_path, "CSCL_{}".format(fc)), "FGDC_TO_ARCGIS") print("Metadata upgraded") if desc.dataType == "Table": print("Copying {} to SDE".format(fc)) arcpy.env.workspace = sde_path arcpy.env.overwriteOutput = True arcpy.TableToTable_conversion(os.path.join(gdb_path, fc), sde_path, "CSCL_{}".format(fc)) print("{} copy complete".format(fc)) arcpy.ExportMetadata_conversion(os.path.join(sde_path, "CSCL_{}".format(fc)), translator, os.path.join(metadata_path, "{}.xml".format(fc))) print("Exporting metadata with geoprocessing history removed") arcpy.XSLTransform_conversion(os.path.join(metadata_path, "{}.xml".format(fc)), stylesheet, os.path.join(metadata_path, "{}_xslt.xml".format(fc))) print("Metadata exported") tree = ET.parse(os.path.join(metadata_path, "{}_xslt.xml".format(fc))) root = tree.getroot() print("Removing Publication Date since it is not currently maintained") for citeinfo in root.iter("citeinfo"): for pubdate in citeinfo.findall("pubdate"): citeinfo.remove(pubdate) print("Publication Date removed") print("Appending download date to metadata description") for descrip in root.iter("purpose"): descrip.text = descrip.text + " Dataset Last Downloaded: {}".format(today_longform) tree.write(os.path.join(metadata_path, "{}_xslt_moded.xml".format(fc))) print("Download date appended to metadata description") print("Importing altered metadata to SDE") arcpy.MetadataImporter_conversion(os.path.join(metadata_path, "{}_xslt_moded.xml".format(fc)), os.path.join(sde_path, "CSCL_{}".format(fc))) print("Metadata imported") arcpy.UpgradeMetadata_conversion(os.path.join(sde_path, "CSCL_{}".format(fc)), "FGDC_TO_ARCGIS") print("Metadata upgraded")
sde = 'Database Connections\GIS_gis_sde.sde' conn = arcpy.ArcSDESQLExecute(sde) sql = '''select owner, table_name from layers order by owner, table_name''' query = conn.execute(sql) for x in query: fc = str(x[0]) + '.' + str(x[1]) print(fc) acct = x[0] conn_file = 'Database Connections' + os.sep + 'GIS_' + str(acct) + '.sde' print(conn_file) try: nameXml1 = outXML + os.sep + x[1] + '1.xml' nameXml2 = outXML + os.sep + x[1] + '2.xml' nameXml3 = outXML + os.sep + x[1] + '3.xml' arcpy.XSLTransform_conversion(conn_file + os.sep + x[1], sspath1, nameXml1, '') arcpy.MetadataImporter_conversion(nameXml1, conn_file + os.sep + x[1]) arcpy.XSLTransform_conversion(conn_file + os.sep + x[1], sspath2, nameXml2, '') arcpy.MetadataImporter_conversion(nameXml2, conn_file + os.sep + x[1]) arcpy.XSLTransform_conversion(conn_file + os.sep + x[1], sspath3, nameXml3, '') arcpy.MetadataImporter_conversion(nameXml3, conn_file + os.sep + x[1]) print('complete') except: print('something broke...') continue shutil.rmtree(outXML) total = datetime.now() - start
## Needs to be run in the 32-bit python environment installed with ArcGIS C:\Python27\ArcGIS10.3 ##Make sure that the workspace has all of the xmls you want to transform as well as the FGDC Plus.xsl file in that location import os, sys, arcpy from arcpy import env work_dir = "L:/Priv/CORFiles/Geospatial_Library/Data/Project/StreamCat/FTP_Staging/StreamCat/Documentation/Metadata/XMLs" env.workspace = work_dir html_dir = "L:/Priv/CORFiles/Geospatial_Library/Data/Project/StreamCat/FTP_Staging/StreamCat/Documentation/Metadata" xslt = "L:/Priv/CORFiles/Geospatial_Library/Data/Project/StreamCat/Metadata/FGDC Plus.xsl" for xml in arcpy.ListFiles("*.xml"): html = html_dir + '/' + xml.split(".")[0] + '.html' if not os.path.exists(html): # print html arcpy.XSLTransform_conversion(xml, xslt, html)
def UpdateFCMetadata(fcPathName, metaConnStr): # Set connection strings and get other settings from configuration file parser = ConfigParser.SafeConfigParser() parser.read('LibMgr.ini') disclaimerFile = parser.get('Metadata', 'disclaimerFile') idCredit = parser.get('Metadata', 'idCredit') constraint_useLimit = parser.get('Metadata', 'constraint_useLimit') organization = parser.get('Metadata', 'organization') timeperd_current = parser.get('Metadata', 'timeperd_current') addrtype = parser.get('Metadata', 'addrtype') address = parser.get('Metadata', 'address') city = parser.get('Metadata', 'city') state = parser.get('Metadata', 'state') zip = parser.get('Metadata', 'zip') country = parser.get('Metadata', 'country') phone = parser.get('Metadata', 'phone') librarian = parser.get('Metadata', 'librarian') thumbnailsPath = parser.get('Metadata', 'thumbnailsPath') num_elements = 0 featName = fcPathName.split('.')[-1] conn = odbcconn(metaConnStr) metaqry = 'SELECT [FULL_NAME],[COVER_NAME],[ABSTRACT],[UPDATEDATE],[OWNERNAME]' +\ ',[PATH],[METAACCESS],[ONMAINT],[MAINTFREQ],[KNOWNERROR],[LINEAGE]' +\ ',[DOMAIN],[RECTIFIED],[MAINTORG],[MAINTDESC],[LIBINPUT],[SOURCNAME]' +\ ',[SOURCCONTACT],[SOURCDOCNAME],[SOURCDATE],[SOURCSCALE],[SOURCFORMAT]' +\ ',[SOUR2NAME],[SOUR2CONTACT],[SOUR2DOCNAME],[SOUR2DATE],[SOUR2SCALE]' +\ ',[SOUR2FORMAT],[ONMG],[MGLAYERNAME],[MGSCALELOW],[MGSCALEHIGH] ' +\ 'FROM [dbo].[metadata] WHERE [COVER_NAME] = \'' + featName + '\'' df_FCMeta = readsqlqry( metaqry, conn) # pandas 0.19.1 load query result to pandas dataframe df_row = df_FCMeta.iloc[0] qry = 'SELECT [FieldName] AS \'ndx\',[FieldName],[Description] FROM [dbo].[master_metafield] WHERE [CoverName] = \'' + featName + '\'' df_fieldMeta = readsqlqry( qry, conn, index_col='ndx') # pandas 0.19.1 load query result to pandas dataframe arcpy.env.overwriteOutput = True # install location dir = arcpy.GetInstallInfo('desktop')['InstallDir'] # stylesheet to use copy_xslt = r'{0}'.format( os.path.join(dir, 'Metadata\Stylesheets\gpTools\exact copy of.xslt')) # temporary XML file xmlfile = arcpy.CreateScratchName('.xml', workspace=arcpy.env.scratchFolder) # export xml arcpy.XSLTransform_conversion(fcPathName, copy_xslt, xmlfile, '') # read in XML tree = ET.parse(xmlfile) root = tree.getroot() # build the supplemental info string sSuppInfo = BuildMetadataSupString(df_row) # get the dataIdInfo element dataIdInfoEl = root.find('dataIdInfo') # dataIdInfo purpose element subEl = ET.SubElement(dataIdInfoEl, 'idPurp') subEl.text = df_row['FULL_NAME'] num_elements += 1 # dataIdInfo abstract element subEl = ET.SubElement(dataIdInfoEl, 'idAbs') subEl.text = df_row['ABSTRACT'] + sSuppInfo num_elements += 1 # dataIdInfo access constraint element subEl = ET.SubElement(dataIdInfoEl, 'accconst') subEl.text = df_row['METAACCESS'] num_elements += 1 # dataIdInfo credit element subEl = ET.SubElement(dataIdInfoEl, 'idCredit') subEl.text = idCredit num_elements += 1 # dataIdInfo maintenance frequency element subEl = ET.SubElement(dataIdInfoEl, 'resMaint') subEl = ET.SubElement(subEl, 'usrDefFreq') subEl = ET.SubElement(subEl, 'duration') subEl.text = df_row['MAINTFREQ'] num_elements += 1 # dataIdInfo use limit element subEl = ET.SubElement(dataIdInfoEl, 'resConst') subEl = ET.SubElement(subEl, 'Consts') subEl = ET.SubElement(subEl, 'useLimit') subEl.text = constraint_useLimit num_elements += 1 # dataIdInfo keyword elements obtained from FULL_NAME searchKeysEl = ET.SubElement(dataIdInfoEl, 'searchKeys') keywords = df_row['FULL_NAME'].split(' ') for keyword in keywords: newKeyEl = ET.SubElement(searchKeysEl, 'keyword') newKeyEl.text = keyword num_elements += 1 # create the idInfo element idInfoEl = ET.SubElement(root, 'idInfo') # idinfo use constraint element with open(disclaimerFile, 'r') as file: # read the disclaimer text file to a string disclaimer = file.read() subEl = ET.SubElement(idInfoEl, 'useconst') subEl.text = disclaimer num_elements += 1 # idinfo citation onlink element # remove the server name portion of the path path = df_row['PATH'].split('\\')[2] pathRoot = '\\\\' + path + '\\' onlink = df_row['PATH'].replace(pathRoot, '') subEl = ET.SubElement(idInfoEl, 'citation') citeinfoEl = ET.SubElement(subEl, 'citeinfo') subEl = ET.SubElement(subEl, 'onlink') subEl.text = onlink num_elements += 1 # idinfo citation origin element subEl = ET.SubElement(citeinfoEl, 'origin') subEl.text = organization num_elements += 1 # idinfo citation pubdate element subEl = ET.SubElement(citeinfoEl, 'pubdate') subEl.text = datetime.datetime.now().strftime("%B %d, %Y") num_elements += 1 # create the idInfo timeperd element timeperdEl = ET.SubElement(idInfoEl, 'timeperd') # idinfo timeperd update date comment element subEl = ET.SubElement(timeperdEl, 'current') subEl.text = timeperd_current num_elements += 1 # idinfo timeperd update date element subEl = ET.SubElement(timeperdEl, 'timeinfo') subEl = ET.SubElement(subEl, 'sngdate') subEl = ET.SubElement(subEl, 'caldate') subEl.text = df_row['UPDATEDATE'] num_elements += 1 # create the idInfo descript element descriptEl = ET.SubElement(idInfoEl, 'descript') # idinfo descript abstract element subEl = ET.SubElement(descriptEl, 'abstract') subEl.text = df_row['ABSTRACT'] num_elements += 1 # idinfo descript purpose element subEl = ET.SubElement(descriptEl, 'purpose') subEl.text = df_row['FULL_NAME'] num_elements += 1 # idinfo descript supplinf element subEl = ET.SubElement(descriptEl, 'supplinf') subEl.text = sSuppInfo num_elements += 1 # idinfo keywords themekey element subEl = ET.SubElement(idInfoEl, 'keywords') subEl = ET.SubElement(subEl, 'theme') subEl = ET.SubElement(subEl, 'themekey') subEl.text = df_row['FULL_NAME'] num_elements += 1 # create the idInfo point of contact elements subEl = ET.SubElement(idInfoEl, 'ptcontac') cntinfoEl = ET.SubElement(subEl, 'cntinfo') cntperpEl = ET.SubElement(cntinfoEl, 'cntperp') cntaddrEl = ET.SubElement(cntinfoEl, 'cntaddr') cntvoiceEl = ET.SubElement(cntinfoEl, 'cntvoice') # idinfo point of contact person element subEl = ET.SubElement(cntperpEl, 'cntper') subEl.text = df_row['OWNERNAME'] num_elements += 1 # idinfo point of contact organization element subEl = ET.SubElement(cntperpEl, 'cntorg') subEl.text = organization num_elements += 1 # idinfo point of contact address type element subEl = ET.SubElement(cntaddrEl, 'addrtype') subEl.text = addrtype num_elements += 1 # idinfo point of contact address element subEl = ET.SubElement(cntaddrEl, 'address') subEl.text = address num_elements += 1 # idinfo point of contact city element subEl = ET.SubElement(cntaddrEl, 'city') subEl.text = city num_elements += 1 # idinfo point of contact state element subEl = ET.SubElement(cntaddrEl, 'state') subEl.text = state num_elements += 1 # idinfo point of contact zip element subEl = ET.SubElement(cntaddrEl, 'postal') subEl.text = zip num_elements += 1 # idinfo point of contact country element subEl = ET.SubElement(cntaddrEl, 'country') subEl.text = country num_elements += 1 # idinfo point of contact phone element subEl = ET.SubElement(cntinfoEl, 'cntvoice') subEl.text = phone num_elements += 1 # create the metainfo point of contact elements metainfoEl = ET.SubElement(root, 'metainfo') subEl = ET.SubElement(metainfoEl, 'metc') cntinfoEl = ET.SubElement(subEl, 'cntinfo') cntorgpEl = ET.SubElement(cntinfoEl, 'cntorgp') cntaddrEl = ET.SubElement(subEl, 'cntaddr') # metainfo point of contact person element subEl = ET.SubElement(cntorgpEl, 'cntper') subEl.text = librarian num_elements += 1 # metainfo point of contact organization element subEl = ET.SubElement(cntorgpEl, 'cntorg') subEl.text = df_row['OWNERNAME'] + '\n' + organization num_elements += 1 # metainfo point of contact address type element subEl = ET.SubElement(cntaddrEl, 'addrtype') subEl.text = addrtype num_elements += 1 # metainfo point of contact address element subEl = ET.SubElement(cntaddrEl, 'address') subEl.text = address num_elements += 1 # metainfo point of contact city element subEl = ET.SubElement(cntaddrEl, 'city') subEl.text = city num_elements += 1 # metainfo point of contact state element subEl = ET.SubElement(cntaddrEl, 'state') subEl.text = state num_elements += 1 # metainfo point of contact zip element subEl = ET.SubElement(cntaddrEl, 'postal') subEl.text = zip num_elements += 1 # metainfo point of contact country element subEl = ET.SubElement(cntaddrEl, 'country') subEl.text = country num_elements += 1 # metainfo point of contact phone element subEl = ET.SubElement(cntinfoEl, 'cntvoice') subEl.text = phone num_elements += 1 # idinfo maintenance status element statusEl = ET.SubElement(idInfoEl, 'status') subEl = ET.SubElement(statusEl, 'progress') if df_row['ONMAINT'] and df_row['ONMAINT'].upper() == 'Y': subEl.text = 'Maintained' num_elements += 1 else: subEl.text = 'Not Maintained' num_elements += 1 # idinfo maintenance frequency element subEl = ET.SubElement(statusEl, 'update') subEl.text = df_row['MAINTFREQ'] num_elements += 1 # add descriptions from library metadata table master_metafields to the feature class fields attrEls = root.findall('eainfo/detailed/attr') for attrEl in attrEls: # iterate feature class fields lablEl = attrEl.find('attrlabl') # find the attribute name element if lablEl is not None: # for unknown reason, the root.findall sometimes gets attributes that are empty fldname = lablEl.text try: descrip = df_fieldMeta.loc[fldname][ 'Description'] # get the field description from the dataframe except Exception as e: #print('\tNo description for field ' + fldname) pass # ignore error returned by dataframe loc. Field not in field metadata table. else: subEl = ET.SubElement(attrEl, 'attrdef') #field description element subEl.text = descrip num_elements += 1 subEl = ET.SubElement( attrEl, 'attrdefs') #field description source element subEl.text = 'Pima County' num_elements += 1 # set metadata thumbnail jpgFile = thumbnailsPath + '/' + featName + '.jpg' if os.path.exists(jpgFile): with open(jpgFile, "rb") as img_file: strEncoded = base64.b64encode(img_file.read()) attrib = {'EsriPropertyType': 'PictureX'} subEl = ET.SubElement(root, 'Binary') subEl = ET.SubElement(subEl, 'Thumbnail') subEl = ET.SubElement(subEl, 'Data', attrib) subEl.text = strEncoded num_elements += 1 if num_elements > 0: # save modifications to XML try: tree.write(xmlfile) arcpy.MetadataImporter_conversion(xmlfile, fcPathName) except Exception as e: print(e.message) else: print('No changes to save')
# --------------------------------------------------------------------------- # RunXlst.py # Created on: 2013-04-25 SG # Description: Run an xslt function (to upgrade a Gizinta.xml file) # --------------------------------------------------------------------------- # Copyright 2012-2013 Vertex3 Inc # This work is licensed under the Creative Commons Attribution-ShareAlike 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by-sa/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. # Import arcpy module import arcpy # Local variables: Gizinta_xml = arcpy.GetParameterAsText(0) xsl = "UpgradeGizintaXml.xsl" Gizinta2_xml = arcpy.GetParameterAsText(1) # Process: XSLT Transformation arcpy.XSLTransform_conversion(Gizinta_xml, xsl, Gizinta2_xml, "")
## Needs to be run in the 32-bit python environment installed with ArcGIS C:\Python27\ArcGIS10.3 ##Make sure that the workspace has all of the xmls you want to transform as well as the FGDC Plus.xsl file in that location import os, sys, arcpy from arcpy import env work_dir = "L:/Priv/CORFiles/Geospatial_Library_Projects/StreamCat/FTP_Staging/StreamCat/Documentation/Metadata/XMLs" env.workspace = work_dir html_dir = "L:/Priv/CORFiles/Geospatial_Library_Projects/StreamCat/FTP_Staging/StreamCat/Documentation/Metadata" xslt = "L:/Priv/CORFiles/Geospatial_Library_Projects/SSWR1.1B/Metadata/FGDC Plus.xsl" for xml in arcpy.ListFiles("*.xml"): print(xml) html = html_dir + '/' + xml.split(".")[0] + '.html' if not os.path.exists(html): # print html arcpy.XSLTransform_conversion(source=xml, xslt=xslt, output=html, xsltparam="")
waterfront_fc_name.split('\\')[-1])) desired_metadata_xml_name = r'{}.xml'.format( os.path.join(desired_meta_path, waterfront_fc_name.split('\\')[-1])) print(temp_metadata_xml_name) metadata_xml_nostorage_name = temp_metadata_xml_name.replace( '.xml', '_nostorage.xml') print(metadata_xml_nostorage_name) metadata_xml_final_name = temp_metadata_xml_name.replace( '.xml', '_final.xml') print(metadata_xml_final_name) print("Exporting xml metadata to temporary location for cleaning") arcpy.ExportMetadata_conversion(waterfront_fc_name, translator, temp_metadata_xml_name) arcpy.XSLTransform_conversion(temp_metadata_xml_name, xslt_remove_storage, metadata_xml_nostorage_name) arcpy.XSLTransform_conversion(metadata_xml_nostorage_name, xslt_remove_geoproc_hist, metadata_xml_final_name) print("Metadata xml final name - {}".format(metadata_xml_final_name)) print('Waterfront FC name - {}'.format(waterfront_fc_name)) arcpy.MetadataImporter_conversion(metadata_xml_final_name, waterfront_fc_name) print("Exporting xml metadata to desired location on BytesProd") arcpy.ExportMetadata_conversion(metadata_xml_final_name, translator, desired_metadata_xml_name) print(desired_metadata_xml_name.replace('.xml', '.html')) arcpy.XSLTransform_conversion( desired_metadata_xml_name, xslt_html, desired_metadata_xml_name.replace('.xml', '.html'))
def ARCGIS(workspace): """ Set workspace (where datasets to be processed are located) below """ arcpy.env.workspace = workspace ws = arcpy.env.workspace print 'ws: ', ws """ Get ArcGIS Install Location """ ARCGIS_INSTALL_LOCATION = arcpy.GetInstallInfo()['InstallDir'] """ Get location of XSL Translator to go from ArcGIS to FGDC """ FGDC_XSL_FILE = ARCGIS_INSTALL_LOCATION + "Metadata\\Translator\\Transforms\\ArcGIS2FGDC.xsl" """ Set output location for OGP metadata and log file, then instantiate Logger """ if os.path.exists(ws + "\\output\\"): OUTPUT_LOCATION = ws + "\\output\\" else: os.mkdir(ws + "\\output\\") OUTPUT_LOCATION = ws + "\\output\\" print 'output location: ', OUTPUT_LOCATION d = datetime.today() LOG_NAME = "OGP_MD_LOG_" + d.strftime("%y%m%d%M%S") + ".txt" sys.stdout = Logger(OUTPUT_LOCATION, LOG_NAME) files = arcpy.ListFiles() datasets = arcpy.ListDatasets() fields = [ 'LayerId', 'Name', 'CollectionId', 'Institution', 'InstitutionSort', 'Access', 'DataType', 'DataTypeSort', 'Availability', 'LayerDisplayName', 'LayerDisplayNameSort', 'Publisher', 'PublisherSort', 'Originator', 'OriginatorSort', 'ThemeKeywords', 'PlaceKeywords', 'Abstract', 'MaxY', 'MinY', 'MinX', 'MaxX', 'CenterX', 'CenterY', 'HalfWidth', 'HalfHeight', 'Area', 'ContentDate', 'Location' ] if os.path.exists(ws + "\\temp.xml"): print "removed temp xml file" os.remove(ws + "\\temp.xml") """ Set UTC timezone for ContentDate field """ utc = pytz.utc for i in datasets[:5]: """ start the clock! obviously not at all necessary """ start = clock() """ converting metadata from ArcGIS to FGDC using XSL transformation (super fast) and writing it to a temporary XML file that is then parsed and translated into final OGP format """ print "======================================" print "Current item is " + i print "======================================\n" print "Converting ArcGIS metadata into temporary FGDC XML file" XSLtrans = arcpy.XSLTransform_conversion(i, FGDC_XSL_FILE, "temp.xml") print XSLtrans.getMessages(), '\n' FGDCtree = ElementTree() root = FGDCtree.parse(ws + "\\temp.xml") DATATYPE = dataTypeParseFGDC(root) DATATYPESORT = DATATYPE #create string representation of FGDC md to be appended at end of OGP md FGDC_TEXT = ET.tostring(root) #layerID equals filename sans extension LAYERID = i[:i.find(".")] #name equals FGDC title field #NAME = root.find("idinfo/citation/citeinfo/title").text try: NAME = root.findtext("idinfo/citation/citeinfo/title") except AttributeError as e: print "Name field doesn't exist! Setting to UNKNOWN for now" NAME = "UNKNOWN" COLLECTIONID = "initial collection" INSTITUTION = "University of Minnesota" INSTITUTIONSORT = "University of Minnesota" try: ACCESS = root.find("idinfo/accconst").text except AttributeError as e: print "Access field doesn't exist! Setting to UNKNOWN for now" ACCESS = "UNKNOWN" AVAILABILITY = "Online" LAYERDISPLAYNAME = NAME LAYERDISPLAYNAMESORT = NAME try: #PUBLISHER = root.find("idinfo/citation/citeinfo/pubinfo/publish").text PUBLISHER = root.findtext( "idinfo/citation/citeinfo/pubinfo/publish") except AttributeError as e: print "Publisher field doesn't exist! Setting to UNKNOWN for now" PUBLISHER = "UNKNOWN" finally: PUBLISHERSORT = PUBLISHER try: ORIGINATOR = root.find("idinfo/citation/citeinfo/origin").text except AttributeError as e: print "Originator field doesn't exist! Setting to UNKNOWN for now" ORIGINATOR = "UNKNOWN" finally: ORIGINATORSORT = ORIGINATOR #Combine multiple Keyword elements into a string rep THEMEKEYWORDS = keywordParse(FGDCtree, "themekey") PLACEKEYWORDS = keywordParse(FGDCtree, "placekey") try: ABSTRACT = root.find("idinfo/descript/abstract").text except AttributeError as e: print "No abstract found. Setting to UNKNOWN for now" ABSTRACT = "UNKNOWN" try: MINX = root.find("idinfo/spdom/bounding/westbc").text MINY = root.find("idinfo/spdom/bounding/southbc").text MAXX = root.find("idinfo/spdom/bounding/eastbc").text MAXY = root.find("idinfo/spdom/bounding/northbc").text except AttributeError as e: print "extent information not found!" #SRSPROJECTIONCODE= #try: if root.find("idinfo/timeperd/timeinfo/sngdate/caldate") is not None: dateText = root.find( "idinfo/timeperd/timeinfo/sngdate/caldate").text year = int(dateText[0:4]) month = int(dateText[4:6]) day = int(dateText[6:8]) date = datetime(year, month, day) CONTENTDATE = date.isoformat() + "Z" elif root.find( "idinfo/timeperd/timeinfo/rngdates/begdate") is not None: dateText = root.find( "idinfo/timeperd/timeinfo/rngdates/begdate").text year = int(dateText[0:4]) month = int(dateText[4:6]) day = int(dateText[6:8]) date = datetime(year, month, day) CONTENTDATE = date.isoformat() + "Z" #except AttributeError as e: # print "No content date found! setting to UNKNOWN for now" # CONTENTDATE = "UNKNOWN" """ LOCATION <field name="Location"> { "ArcGIS93Rest": ["(the url for the REST service)"], "tilecache": ["(the url for the tilecache, which I believe ArcGIS server does not have)"], "download": "(downloading URL for the layer)","wfs": "(WFS URL, if has one)" } </field> """ ## #base URL for all images ## REST_URL_BASE = "http://lib-geo1.lib.umn.edu:6080/arcgis/rest/services/OGP/mosaic_ogp_4/ImageServer/exportImage?" ## ## #parameters to pass to URL, in the form of a dictionary ## DOWNLOAD_URL_PARAMS = {} ## DOWNLOAD_URL_PARAMS['bbox'] = MINX+','+MINY+','+MAXX+','+MAXY ## DOWNLOAD_URL_PARAMS['bboxSR'] = '4269' ## DOWNLOAD_URL_PARAMS['imageSR'] = '26915' ## DOWNLOAD_URL_PARAMS['format'] = 'tiff' ## DOWNLOAD_URL_PARAMS['pixelType'] = 'U8' ## DOWNLOAD_URL_PARAMS['size'] = '3000,3000' ## DOWNLOAD_URL_PARAMS['restrictUTurns'] = 'esriNoDataMatchAny' ## DOWNLOAD_URL_PARAMS['interpolation'] = 'RSP_BilinearInterpolation' ## cursor = arcpy.SearchCursor(u"\\mosaic245.gdb\\mosaic_ogp_1","name = '"+ LAYERID+"'") ## raster_ID = cursor.next().OBJECTID ## DOWNLOAD_URL_PARAMS['mosaicRule'] ={} ## DOWNLOAD_URL_PARAMS['mosaicRule']['mosaicMethod'] = 'esriMosaicLockRaster' ## DOWNLOAD_URL_PARAMS['mosaicRule']['lockRasterIds'] = [raster_ID] ## DOWNLOAD_URL_PARAMS['f']='image' ## ## DOWNLOAD_URL = REST_URL_BASE + urllib.urlencode(DOWNLOAD_URL_PARAMS) ## ## REST_URL= REST_URL_BASE + urllib.urlencode({'mosaicRule': DOWNLOAD_URL_PARAMS['mosaicRule']}) ## ## LOCATION = '{"ArcGIS93Rest" : "' + REST_URL + '", "tilecache" : "None", "download" : "' + DOWNLOAD_URL + '", "wfs" : "None"}' """ Put it all together to make the OGP metadata xml file """ OGPtree = ElementTree() OGProot = Element("add", allowDups="false") doc = SubElement(OGProot, "doc") OGPtree._setroot(OGProot) try: for field in fields: fieldEle = SubElement(doc, "field", name=field) fieldEle.text = locals()[field.upper()] except KeyError as e: print "Nonexistant key: ", field fgdc_text = SubElement(doc, "field", name="FgdcText") fgdc_text.text = '<?xml version="1.0"?>' + FGDC_TEXT if os.path.exists(OUTPUT_LOCATION + LAYERID + "_OGP.xml"): existsPrompt = raw_input( 'OGP metadata already exists! Overwrite? (Y/N): ') if existsPrompt == 'Y': OGPtree.write(OUTPUT_LOCATION + LAYERID + "_OGP.xml") print "Output file: " + OUTPUT_LOCATION + LAYERID + "_OGP.xml" else: pass else: OGPtree.write(OUTPUT_LOCATION + LAYERID + "_OGP.xml") print "Output file: " + OUTPUT_LOCATION + LAYERID + "_OGP.xml" os.remove(ws + "\\temp.xml") print "Next!\n" end = clock() if (end - start) > 60: mins = str(math.ceil((end - start) / 60)) secs = str(math.ceil((end - start) % 60)) print str( len(datasets) ) + " datasets processed in " + mins + " minutes and " + secs + " seconds" else: secs = str(math.ceil(end - start)) print str( len(datasets)) + " datasets processed in " + secs + " seconds" sys.stdout = sys.__stdout__ print 'Done. See log file for information'