Esempio n. 1
0
    def vector_to_raster(self, input_fc):

        if self.vector_to_raster_output:

            temp_dir = util.create_temp_dir(self.scratch_workspace)
            arcpy.CreateFileGDB_management(temp_dir, 'temp.gdb')

            # Get spatial reference of output
            sr = arcpy.Describe(self.vector_to_raster_output).spatialReference

            logging.debug('Starting to project input vector FC to the spatial reference of the output raster')
            out_projected_fc = os.path.join(temp_dir, 'temp.gdb', 'src_prj_to_ras_sr')
            arcpy.Project_management(input_fc, out_projected_fc, sr)

            util.add_field_and_calculate(out_projected_fc, 'ras_val', 'SHORT', '', 1, self.gfw_env)

            # Get cell size of output
            cell_size = int(arcpy.GetRasterProperties_management(self.vector_to_raster_output,
                                                                 'CELLSIZEX').getOutput(0))

            arcpy.env.pyramid = "NONE"
            arcpy.env.snapRaster = self.vector_to_raster_output

            logging.debug('Rasterizing and outputting as tif')
            out_raster = os.path.join(temp_dir, 'out.tif')

            arcpy.PolygonToRaster_conversion(out_projected_fc, 'ras_val', out_raster, "CELL_CENTER", "", cell_size)

            # Stop service that has a lock on the raster
            service = r'image_services/analysis'
            # arcgis_server.set_service_status(service, 'stop')

            logging.debug('Sleeping for 10 seconds to let the lock files resolve themselves')
            time.sleep(10)

            logging.debug('Copying raster {0} to output {1}'.format(out_raster, self.vector_to_raster_output))
            arcpy.Delete_management(self.vector_to_raster_output)

            # Move all related tif files to final destination
            # Much faster than using CopyRaster_management-- just need to physically move the files
            src_dir = os.path.dirname(out_raster)
            src_file_name = os.path.splitext(os.path.basename(out_raster))[0]

            out_dir = os.path.dirname(self.vector_to_raster_output)
            out_file_name = os.path.splitext(os.path.basename(self.vector_to_raster_output))[0]

            for extension in ['.tif', '.tfw', '.tif.aux.xml', '.tif.vat.cpg', '.tif.vat.dbf', '.tif.xml']:
                src_file = os.path.join(src_dir, src_file_name + extension)
                out_file = os.path.join(out_dir, out_file_name + extension)

                try:
                    shutil.move(src_file, out_file)
                except IOError:
                    print 'No such file {0}'.format(src_file)

            # Restart the service after we're finished
            # arcgis_server.set_service_status(service, 'start')

        else:
            pass
Esempio n. 2
0
 def add_country_code(self):
     """
     If there's a value for sell.add_country_value, add a country field and populate it
     :return:
     """
     if self.add_country_value:
         logging.info('Starting vector_layer.add_country_value for {0}, '
                      'country val {1}'.format(self.name, self.add_country_value))
         util.add_field_and_calculate(self.source, "country", 'TEXT', 3, self.add_country_value, self.gfw_env)
Esempio n. 3
0
 def add_country_code(self):
     """
     If there's a value for sell.add_country_value, add a country field and populate it
     :return:
     """
     if self.add_country_value:
         logging.info('Starting vector_layer.add_country_value for {0}, '
                      'country val {1}'.format(self.name,
                                               self.add_country_value))
         util.add_field_and_calculate(self.source, "country", 'TEXT', 3,
                                      self.add_country_value, self.gfw_env)
    def add_and_populate_country_field(self, in_fc):
        """
        Add a field for country if it doesn't exist and populate it with country value
        :param in_fc: fc to add 'country' to
        :return: True/False based on whether country added; will be used later to delete the field if added
        """
        country_added = False

        if 'country' not in util.list_fields(in_fc, self.gfw_env):
            util.add_field_and_calculate(in_fc, 'country', 'TEXT', 3, self.add_country_value, self.gfw_env)
            country_added = True

        return country_added
Esempio n. 5
0
    def add_and_populate_country_field(self, in_fc):
        """
        Add a field for country if it doesn't exist and populate it with country value
        :param in_fc: fc to add 'country' to
        :return: True/False based on whether country added; will be used later to delete the field if added
        """
        country_added = False

        if 'country' not in util.list_fields(in_fc, self.gfw_env):
            util.add_field_and_calculate(in_fc, 'country', 'TEXT', 3,
                                         self.add_country_value, self.gfw_env)
            country_added = True

        return country_added
Esempio n. 6
0
    def clean_source_shps(self, shp_list):
        """
        After the data has been unzipped, repair geometry, remove fields, and add date and orig_fname
        :param shp_list: list of cleaned shapefiles ready to be appended to final output
        :return:
        """
        cleaned_shp_list = []

        for shp in shp_list:
            
            shp_name = os.path.basename(shp).replace('-', '_')
            
            single_part_path = os.path.join(os.path.dirname(shp), shp_name.replace('.shp', '') + '_singlepart.shp')

            logging.info('Starting multipart to singlepart for ' + shp_name)
            arcpy.MultipartToSinglepart_management(shp, single_part_path)

            arcpy.RepairGeometry_management(single_part_path, "DELETE_NULL")

            # Must have one field before we delete all the other ones. So says arcgis anyway
            orig_oid_field = 'orig_oid'
            util.add_field_and_calculate(single_part_path, orig_oid_field, 'Text', '255', '!FID!', self.gfw_env)
            self.remove_all_fields_except(single_part_path, keep_field_list=[orig_oid_field])

            imazon_date_str = self.get_date_from_filename(os.path.basename(shp))

            util.add_field_and_calculate(single_part_path, 'Date', 'DATE', "", imazon_date_str, self.gfw_env)
            util.add_field_and_calculate(single_part_path, 'data_type', 'TEXT', "255", self.data_type(shp),
                                         self.gfw_env)
            util.add_field_and_calculate(single_part_path, 'orig_fname', 'TEXT', "255", shp_name, self.gfw_env)

            cleaned_shp_list.append(single_part_path)

        return cleaned_shp_list
Esempio n. 7
0
    def clean_source_shps(self, shp_list):
        """
        After the data has been unzipped, repair geometry, remove fields, and add date and orig_fname
        :param shp_list: list of cleaned shapefiles ready to be appended to final output
        :return:
        """
        cleaned_shp_list = []

        for shp in shp_list:
            
            shp_name = os.path.basename(shp).replace('-', '_')
            
            single_part_path = os.path.join(os.path.dirname(shp), shp_name.replace('.shp', '') + '_singlepart.shp')

            # unclear why this extra garbage is added to the filename, but it is
            if 'desmatamento' in shp and int(shp[shp.index('2018')+5:shp.index('2018')+7]) <= 7:
                shp = os.path.splitext(shp)[0] + '_01102018.shp'

            # sometimes the zip files have dashes after the month, sometimes underscores
            # who can ever know why
            if not os.path.exists(shp):
                idx_2018 = shp.index('2018') + 4
                shp = shp[:idx_2018] + '-' + shp[idx_2018 + 1:]

            logging.info('Starting multipart to singlepart for ' + shp_name)
            arcpy.MultipartToSinglepart_management(shp, single_part_path)

            arcpy.RepairGeometry_management(single_part_path, "DELETE_NULL")

            # Must have one field before we delete all the other ones. So says arcgis anyway
            orig_oid_field = 'orig_oid'
            util.add_field_and_calculate(single_part_path, orig_oid_field, 'Text', '255', '!FID!', self.gfw_env)
            self.remove_all_fields_except(single_part_path, keep_field_list=[orig_oid_field])

            imazon_date_str = self.get_date_from_filename(os.path.basename(shp))

            util.add_field_and_calculate(single_part_path, 'Date', 'DATE', "", imazon_date_str, self.gfw_env)
            util.add_field_and_calculate(single_part_path, 'date_alias', 'DATE', "", imazon_date_str,
                                         self.gfw_env)
            util.add_field_and_calculate(single_part_path, 'data_type', 'TEXT', "255", self.data_type(shp),
                                         self.gfw_env)
            util.add_field_and_calculate(single_part_path, 'orig_fname', 'TEXT', "255", shp_name, self.gfw_env)

            self.calculate_area_ha_eckert_iv(single_part_path)

            cleaned_shp_list.append(single_part_path)

        return cleaned_shp_list
Esempio n. 8
0
    def vector_to_raster(self, input_fc):

        if self.vector_to_raster_output:

            temp_dir = util.create_temp_dir(self.scratch_workspace)
            arcpy.CreateFileGDB_management(temp_dir, 'temp.gdb')

            # Get spatial reference of output
            sr = arcpy.Describe(self.vector_to_raster_output).spatialReference

            logging.debug('Starting to project input vector FC to the spatial reference of the output raster')
            out_projected_fc = os.path.join(temp_dir, 'temp.gdb', 'src_prj_to_ras_sr')
            arcpy.Project_management(input_fc, out_projected_fc, sr)

            util.add_field_and_calculate(out_projected_fc, 'ras_val', 'SHORT', '', 1, self.gfw_env)

            # Get cell size of output
            cell_size = int(arcpy.GetRasterProperties_management(self.vector_to_raster_output,
                                                                 'CELLSIZEX').getOutput(0))

            arcpy.env.pyramid = "NONE"
            arcpy.env.snapRaster = self.vector_to_raster_output

            logging.debug('Rasterizing and outputting as tif')
            out_raster = os.path.join(temp_dir, 'out.tif')

            arcpy.PolygonToRaster_conversion(out_projected_fc, 'ras_val', out_raster, "CELL_CENTER", "", cell_size)

            logging.debug('Copying raster {0} to output {1}'.format(out_raster, self.vector_to_raster_output))
            arcpy.Delete_management(self.vector_to_raster_output)

            # Move all related tif files to final destination
            # Much faster than using CopyRaster_management-- just need to physically move the files
            src_dir = os.path.dirname(out_raster)
            src_file_name = os.path.splitext(os.path.basename(out_raster))[0]

            out_dir = os.path.dirname(self.vector_to_raster_output)
            out_file_name = os.path.splitext(os.path.basename(self.vector_to_raster_output))[0]

            for extension in ['.tif', '.tfw', '.tif.aux.xml', '.tif.vat.cpg', '.tif.vat.dbf', '.tif.xml']:
                src_file = os.path.join(src_dir, src_file_name + extension)
                out_file = os.path.join(out_dir, out_file_name + extension)

                shutil.move(src_file, out_file)

        else:
            pass
Esempio n. 9
0
    def clean_source_shps(self, shp_list):
        """
        After the data has been unzipped, repair geometry, remove fields, and add date and orig_fname
        :param shp_list: list of cleaned shapefiles ready to be appended to final output
        :return:
        """
        cleaned_shp_list = []

        for shp in shp_list:

            shp_name = os.path.basename(shp).replace('-', '_')

            single_part_path = os.path.join(
                os.path.dirname(shp),
                shp_name.replace('.shp', '') + '_singlepart.shp')

            logging.info('Starting multipart to singlepart for ' + shp_name)
            arcpy.MultipartToSinglepart_management(shp, single_part_path)

            arcpy.RepairGeometry_management(single_part_path, "DELETE_NULL")

            # Must have one field before we delete all the other ones. So says arcgis anyway
            orig_oid_field = 'orig_oid'
            util.add_field_and_calculate(single_part_path, orig_oid_field,
                                         'Text', '255', '!FID!', self.gfw_env)
            self.remove_all_fields_except(single_part_path,
                                          keep_field_list=[orig_oid_field])

            imazon_date_str = self.get_date_from_filename(
                os.path.basename(shp))

            util.add_field_and_calculate(single_part_path, 'Date', 'DATE', "",
                                         imazon_date_str, self.gfw_env)
            util.add_field_and_calculate(single_part_path, 'date_alias',
                                         'DATE', "", imazon_date_str,
                                         self.gfw_env)
            util.add_field_and_calculate(single_part_path,
                                         'data_type', 'TEXT', "255",
                                         self.data_type(shp), self.gfw_env)
            util.add_field_and_calculate(single_part_path, 'orig_fname',
                                         'TEXT', "255", shp_name, self.gfw_env)

            self.calculate_area_ha_eckert_iv(single_part_path)

            cleaned_shp_list.append(single_part_path)

        return cleaned_shp_list
Esempio n. 10
0
    def vector_to_raster(self, input_fc):

        if self.vector_to_raster_output:

            temp_dir = util.create_temp_dir(self.scratch_workspace)
            arcpy.CreateFileGDB_management(temp_dir, 'temp.gdb')

            # Get spatial reference of output
            sr = arcpy.Describe(self.vector_to_raster_output).spatialReference

            logging.debug(
                'Starting to project input vector FC to the spatial reference of the output raster'
            )
            out_projected_fc = os.path.join(temp_dir, 'temp.gdb',
                                            'src_prj_to_ras_sr')
            arcpy.Project_management(input_fc, out_projected_fc, sr)

            util.add_field_and_calculate(out_projected_fc, 'ras_val', 'SHORT',
                                         '', 1, self.gfw_env)

            # Get cell size of output
            cell_size = int(
                arcpy.GetRasterProperties_management(
                    self.vector_to_raster_output, 'CELLSIZEX').getOutput(0))

            arcpy.env.pyramid = "NONE"
            arcpy.env.snapRaster = self.vector_to_raster_output

            logging.debug('Rasterizing and outputting as tif')
            out_raster = os.path.join(temp_dir, 'out.tif')

            arcpy.PolygonToRaster_conversion(out_projected_fc, 'ras_val',
                                             out_raster, "CELL_CENTER", "",
                                             cell_size)

            # Stop service that has a lock on the raster
            service = r'image_services/analysis'
            # arcgis_server.set_service_status(service, 'stop')

            logging.debug(
                'Sleeping for 10 seconds to let the lock files resolve themselves'
            )
            time.sleep(10)

            logging.debug('Copying raster {0} to output {1}'.format(
                out_raster, self.vector_to_raster_output))
            arcpy.Delete_management(self.vector_to_raster_output)

            # Move all related tif files to final destination
            # Much faster than using CopyRaster_management-- just need to physically move the files
            src_dir = os.path.dirname(out_raster)
            src_file_name = os.path.splitext(os.path.basename(out_raster))[0]

            out_dir = os.path.dirname(self.vector_to_raster_output)
            out_file_name = os.path.splitext(
                os.path.basename(self.vector_to_raster_output))[0]

            for extension in [
                    '.tif', '.tfw', '.tif.aux.xml', '.tif.vat.cpg',
                    '.tif.vat.dbf', '.tif.xml'
            ]:
                src_file = os.path.join(src_dir, src_file_name + extension)
                out_file = os.path.join(out_dir, out_file_name + extension)

                try:
                    shutil.move(src_file, out_file)
                except IOError:
                    print 'No such file {0}'.format(src_file)

            # Restart the service after we're finished
            # arcgis_server.set_service_status(service, 'start')

        else:
            pass