Пример #1
0
    def __init__(self, source_datacube=None, tile_type_id=1):
        """Constructor
        Arguments:
            source_datacube: Optional DataCube object whose connection and data will be shared
            tile_type_id: Optional tile_type_id value (defaults to 1)
        """

        if source_datacube:
            # Copy values from source_datacube and then override command line args
            self.__dict__ = copy(source_datacube.__dict__)

            args = self.parse_args()
            # Set instance attributes for every value in command line arguments file
            for attribute_name in args.__dict__.keys():
                attribute_value = args.__dict__[attribute_name]
                self.__setattr__(attribute_name, attribute_value)

        else:
            DataCube.__init__(self)
            # Call inherited constructor
        self.temp_dir = os.path.join(
            self.temp_dir, re.sub('^/', '', os.path.abspath(self.source_dir)))
        self.create_directory(self.temp_dir)
        logger.debug('self.temp_dir = %s', self.temp_dir)

        if self.debug:
            console_handler.setLevel(logging.DEBUG)
Пример #2
0
    def __init__(self, source_datacube=None, tile_type_id=1):
        """Constructor
        Arguments:
            source_datacube: Optional DataCube object whose connection and data will be shared
            tile_type_id: Optional tile_type_id value (defaults to 1)
        """
        
        if source_datacube:
            # Copy values from source_datacube and then override command line args
            self.__dict__ = copy(source_datacube.__dict__)
            
            args = self.parse_args()
            # Set instance attributes for every value in command line arguments file
            for attribute_name in args.__dict__.keys():
                attribute_value = args.__dict__[attribute_name]
                self.__setattr__(attribute_name, attribute_value)

        else:
            DataCube.__init__(self); # Call inherited constructor        
        self.temp_dir = os.path.join(self.temp_dir, re.sub('^/', '', os.path.abspath(self.source_dir)))
        self.create_directory(self.temp_dir)
        logger.debug('self.temp_dir = %s', self.temp_dir)
            
        if self.debug:
            console_handler.setLevel(logging.DEBUG)
Пример #3
0
    def __init__(self, source_datacube=None, default_tile_type_id=1):
        '''
        Constructor for TileRemover class
        '''
        self.dataset_records = {}
        self.acquisition_records = {}
        self.all_tile_records = {}
        self.tile_records_to_delete = {}
        self.tile_records_to_update = {}

        if source_datacube:
            # Copy values from source_datacube and then override command line args
            self.__dict__ = copy(source_datacube.__dict__)
            
            args = self.parse_args()
            # Set instance attributes for every value in command line arguments file
            for attribute_name in args.__dict__.keys():
                attribute_value = args.__dict__[attribute_name]
                self.__setattr__(attribute_name, attribute_value)

        else:
            DataCube.__init__(self) # Call inherited constructor
            
        if self.debug:
            logger.setLevel(logging.DEBUG)
            
        if self.action and type(self.action) == str:
            self.action = TileRemover.action_dict.get(self.action[0].lower()) or 'report'
        else:
            self.action = 'report'
            
        if self.target and type(self.target) == str:
            self.target = TileRemover.target_dict.get(self.target[0].lower()) or 'acquisition'
        else:
            self.target = 'acquisition'
            
        if self.dataset_name: # Dataset list specified at command line
            self.dataset_name_list = self.dataset_name.split(',')
        elif self.dataset_list: # Dataset list file specified
            dataset_list_file = open(self.dataset_list, 'r')
            self.dataset_name_list = [dataset_name.replace('\n', '') for dataset_name in dataset_list_file.readlines()]            
            dataset_list_file.close()
        else:
            raise Exception('No dataset IDs or dataset name list file specified')
        
        assert self.dataset_name_list, 'No dataset names specified'
        self.dataset_name_list = sorted(self.dataset_name_list)
        
        # Only need one cursor - create it here
        self.db_cursor = self.db_connection.cursor()
        
        # Populate field name lists for later use
        self.dataset_field_list = self.get_field_names('dataset', ['xml_text'])
        self.acquisition_field_list = self.get_field_names('acquisition', ['mtl_text'])
        self.tile_field_list = self.get_field_names('tile')
        
        self.satellite_dict = self.get_satellite_dict()
        
        log_multiline(logger.debug, self.__dict__, 'self.__dict__', '\t')
Пример #4
0
    def __init__(self, source_datacube=None, default_tile_type_id=1):
        '''
        Constructor for TileRemover class
        '''
        self.dataset_records = {}
        self.acquisition_records = {}
        self.all_tile_records = {}
        self.tile_records_to_delete = {}
        self.tile_records_to_update = {}

        if source_datacube:
            # Copy values from source_datacube and then override command line args
            self.__dict__ = copy(source_datacube.__dict__)
            
            args = self.parse_args()
            # Set instance attributes for every value in command line arguments file
            for attribute_name in args.__dict__.keys():
                attribute_value = args.__dict__[attribute_name]
                self.__setattr__(attribute_name, attribute_value)

        else:
            DataCube.__init__(self) # Call inherited constructor
            
        if self.debug:
            console_handler.setLevel(logging.DEBUG)
            
        if self.action and type(self.action) == str:
            self.action = TileRemover.action_dict.get(self.action[0].lower()) or 'report'
        else:
            self.action = 'report'
            
        if self.target and type(self.target) == str:
            self.target = TileRemover.target_dict.get(self.target[0].lower()) or 'acquisition'
        else:
            self.target = 'acquisition'
            
        if self.dataset_name: # Dataset list specified at command line
            self.dataset_name_list = self.dataset_name.split(',')
        elif self.dataset_list: # Dataset list file specified
            dataset_list_file = open(self.dataset_list, 'r')
            self.dataset_name_list = [dataset_name.replace('\n', '') for dataset_name in dataset_list_file.readlines()]            
            dataset_list_file.close()
        else:
            raise Exception('No dataset IDs or dataset name list file specified')
        
        assert self.dataset_name_list, 'No dataset names specified'
        self.dataset_name_list = sorted(self.dataset_name_list)
        
        # Only need one cursor - create it here
        self.db_cursor = self.db_connection.cursor()
        
        # Populate field name lists for later use
        self.dataset_field_list = self.get_field_names('dataset', ['xml_text'])
        self.acquisition_field_list = self.get_field_names('acquisition', ['mtl_text'])
        self.tile_field_list = self.get_field_names('tile')
        
        self.satellite_dict = self.get_satellite_dict()
        
        log_multiline(logger.debug, self.__dict__, 'self.__dict__', '\t')
Пример #5
0
    def __init__(self, source_datacube=None, default_tile_type_id=1):
        """Constructor
        Arguments:
            source_datacube: Optional DataCube object whose connection and data will be shared
            tile_type_id: Optional tile_type_id value (defaults to config file value = 1)
        """

        if source_datacube:
            # Copy values from source_datacube and then override command line args
            self.__dict__ = copy(source_datacube.__dict__)

            args = self.parse_args()
            # Set instance attributes for every value in command line arguments file
            for attribute_name in args.__dict__.keys():
                attribute_value = args.__dict__[attribute_name]
                self.__setattr__(attribute_name, attribute_value)

        else:
            DataCube.__init__(self)
            # Call inherited constructor

        if self.debug:
            console_handler.setLevel(logging.DEBUG)

        # Turn autocommit OFF so that transaction can cover all queries for each dataset
        self.db_connection.autocommit = False
        self.db_connection.set_isolation_level(
            psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED)

        # Attempt to parse dates from command line arguments or config file
        try:
            self.default_tile_type_id = int(self.default_tile_type_id)
        except:
            self.default_tile_type_id = default_tile_type_id
        try:
            self.start_date = datetime.strptime(self.start_date,
                                                '%d/%m/%Y').date()
        except:
            self.start_date = None
        try:
            self.end_date = datetime.strptime(self.end_date, '%d/%m/%Y').date()
        except:
            self.end_date = None
        try:
            self.min_path = int(self.min_path)
        except:
            self.min_path = None
        try:
            self.max_path = int(self.max_path)
        except:
            self.max_path = None
        try:
            self.min_row = int(self.min_row)
        except:
            self.min_row = None
        try:
            self.max_row = int(self.max_row)
        except:
            self.max_row = None
Пример #6
0
    def __init__(self, source_datacube=None, default_tile_type_id=1):
        """Constructor
        Arguments:
            source_datacube: Optional DataCube object whose connection and data will be shared
            tile_type_id: Optional tile_type_id value (defaults to config file value = 1)
        """

        if source_datacube:
            # Copy values from source_datacube and then override command line args
            self.__dict__ = copy(source_datacube.__dict__)

            args = self.parse_args()
            # Set instance attributes for every value in command line arguments file
            for attribute_name in args.__dict__.keys():
                attribute_value = args.__dict__[attribute_name]
                self.__setattr__(attribute_name, attribute_value)

        else:
            DataCube.__init__(self)
            # Call inherited constructor

        if self.debug:
            console_handler.setLevel(logging.DEBUG)

        # Turn autocommit OFF so that transaction can cover all queries for each dataset
        self.db_connection.autocommit = False
        self.db_connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED)

        # Attempt to parse dates from command line arguments or config file
        try:
            self.default_tile_type_id = int(self.default_tile_type_id)
        except:
            self.default_tile_type_id = default_tile_type_id
        try:
            self.start_date = datetime.strptime(self.start_date, "%d/%m/%Y").date()
        except:
            self.start_date = None
        try:
            self.end_date = datetime.strptime(self.end_date, "%d/%m/%Y").date()
        except:
            self.end_date = None
        try:
            self.min_path = int(self.min_path)
        except:
            self.min_path = None
        try:
            self.max_path = int(self.max_path)
        except:
            self.max_path = None
        try:
            self.min_row = int(self.min_row)
        except:
            self.min_row = None
        try:
            self.max_row = int(self.max_row)
        except:
            self.max_row = None
Пример #7
0
 def __init__(self, args):
     self.my_args = args
     DataCube.__init__(self)
Пример #8
0
 def __init__(self, args):
     self.my_args = args
     DataCube.__init__(self)
Пример #9
0
    def __init__(self, source_datacube=None, default_tile_type_id=1):
        """Constructor
        Arguments:
            source_datacube: Optional DataCube object whose connection and data will be shared
            tile_type_id: Optional tile_type_id value (defaults to 1)
        """
        
        if source_datacube:
            # Copy values from source_datacube and then override command line args
            self.__dict__ = copy(source_datacube.__dict__)
            
            args = self.parse_args()
            # Set instance attributes for every value in command line arguments file
            for attribute_name in args.__dict__.keys():
                attribute_value = args.__dict__[attribute_name]
                self.__setattr__(attribute_name, attribute_value)

        else:
            DataCube.__init__(self) # Call inherited constructor
            
        # Attempt to parse dates from command line arguments or config file
        try:
            self.start_date = datetime.strptime(self.start_date, '%Y%m%d').date()
        except:
            try:
                self.start_date = datetime.strptime(self.start_date, '%d/%m/%Y').date()
            except:
                try:
                    self.start_date = datetime.strptime(self.start_date, '%Y-%m-%d').date()
                except:
                    self.start_date= None      
                          
        try:
            self.end_date = datetime.strptime(self.end_date, '%Y%m%d').date()
        except:
            try:
                self.end_date = datetime.strptime(self.end_date, '%d/%m/%Y').date()
            except:
                try:
                    self.end_date = datetime.strptime(self.end_date, '%Y-%m-%d').date()
                except:
                    self.end_date= None            
          

        try:
            self.thumbnail_size = int(self.thumbnail_size)
        except:
            self.thumbnail_size = 512
            
        # Other variables set from config file only - not used
        try:
            self.min_path = int(self.min_path) 
        except:
            self.min_path = None
        try:
            self.max_path = int(self.max_path) 
        except:
            self.max_path = None
        try:
            self.min_row = int(self.min_row) 
        except:
            self.min_row = None
        try:
            self.max_row = int(self.max_row) 
        except:
            self.max_row = None
            
        self.style_dict = {
#                           'IconStyle': {'scale': 0.4, 'Icon': {'href': 'http://maps.google.com/mapfiles/kml/shapes/star.png'}},
                           'LabelStyle': {'color': '9900ffff', 'scale': 1},
                           'LineStyle': {'color': '990000ff', 'width': 2},
                           'PolyStyle': {'color': '997f7fff', 'fill': 1, 'outline': 1}
                          }
Пример #10
0
    def __init__(self, source_datacube=None, default_tile_type_id=1):
        """Constructor
        Arguments:
            source_datacube: Optional DataCube object whose connection and data will be shared
            tile_type_id: Optional tile_type_id value (defaults to 1)
        """
        
        if source_datacube:
            # Copy values from source_datacube and then override command line args
            self.__dict__ = copy(source_datacube.__dict__)
            
            args = self.parse_args()
            # Set instance attributes for every value in command line arguments file
            for attribute_name in args.__dict__.keys():
                attribute_value = args.__dict__[attribute_name]
                self.__setattr__(attribute_name, attribute_value)

        else:
            DataCube.__init__(self) # Call inherited constructor
            
        # Attempt to parse dates from command line arguments or config file
        try:
            self.start_date = datetime.strptime(self.start_date, '%Y%m%d').date()
        except:
            try:
                self.start_date = datetime.strptime(self.start_date, '%d/%m/%Y').date()
            except:
                try:
                    self.start_date = datetime.strptime(self.start_date, '%Y-%m-%d').date()
                except:
                    self.start_date= None      
                          
        try:
            self.end_date = datetime.strptime(self.end_date, '%Y%m%d').date()
        except:
            try:
                self.end_date = datetime.strptime(self.end_date, '%d/%m/%Y').date()
            except:
                try:
                    self.end_date = datetime.strptime(self.end_date, '%Y-%m-%d').date()
                except:
                    self.end_date= None            
          

        # Other variables set from config file only - not used
        try:
            self.min_path = int(self.min_path) 
        except:
            self.min_path = None
        try:
            self.max_path = int(self.max_path) 
        except:
            self.max_path = None
        try:
            self.min_row = int(self.min_row) 
        except:
            self.min_row = None
        try:
            self.max_row = int(self.max_row) 
        except:
            self.max_row = None
            
        self.thredds_root = '/g/data1/v27/projects/EOS_delivery/LANDSAT/'
Пример #11
0
    def __init__(self, source_datacube=None, default_tile_type_id=1):
        """Constructor
        Arguments:
            source_datacube: Optional DataCube object whose connection and data will be shared
            tile_type_id: Optional tile_type_id value (defaults to 1)
        """

        if source_datacube:
            # Copy values from source_datacube and then override command line args
            self.__dict__ = copy(source_datacube.__dict__)

            args = self.parse_args()
            # Set instance attributes for every value in command line arguments file
            for attribute_name in args.__dict__.keys():
                attribute_value = args.__dict__[attribute_name]
                self.__setattr__(attribute_name, attribute_value)

        else:
            DataCube.__init__(self)  # Call inherited constructor

        # Attempt to parse dates from command line arguments or config file
        try:
            self.start_date = datetime.strptime(self.start_date,
                                                '%Y%m%d').date()
        except:
            try:
                self.start_date = datetime.strptime(self.start_date,
                                                    '%d/%m/%Y').date()
            except:
                try:
                    self.start_date = datetime.strptime(
                        self.start_date, '%Y-%m-%d').date()
                except:
                    self.start_date = None

        try:
            self.end_date = datetime.strptime(self.end_date, '%Y%m%d').date()
        except:
            try:
                self.end_date = datetime.strptime(self.end_date,
                                                  '%d/%m/%Y').date()
            except:
                try:
                    self.end_date = datetime.strptime(self.end_date,
                                                      '%Y-%m-%d').date()
                except:
                    self.end_date = None

        # Other variables set from config file only - not used
        try:
            self.min_path = int(self.min_path)
        except:
            self.min_path = None
        try:
            self.max_path = int(self.max_path)
        except:
            self.max_path = None
        try:
            self.min_row = int(self.min_row)
        except:
            self.min_row = None
        try:
            self.max_row = int(self.max_row)
        except:
            self.max_row = None

        self.thredds_root = '/g/data1/v27/projects/EOS_delivery/LANDSAT/'
Пример #12
0
    def __init__(self, source_datacube=None, default_tile_type_id=1):
        """Constructor
        Arguments:
            source_datacube: Optional DataCube object whose connection and data will be shared
            tile_type_id: Optional tile_type_id value (defaults to 1)
        """

        if source_datacube:
            # Copy values from source_datacube and then override command line args
            self.__dict__ = copy(source_datacube.__dict__)

            args = self.parse_args()
            # Set instance attributes for every value in command line arguments file
            for attribute_name in args.__dict__.keys():
                attribute_value = args.__dict__[attribute_name]
                self.__setattr__(attribute_name, attribute_value)

        else:
            DataCube.__init__(self)  # Call inherited constructor

        # Attempt to parse dates from command line arguments or config file
        try:
            self.start_date = datetime.strptime(self.start_date,
                                                '%Y%m%d').date()
        except:
            try:
                self.start_date = datetime.strptime(self.start_date,
                                                    '%d/%m/%Y').date()
            except:
                try:
                    self.start_date = datetime.strptime(
                        self.start_date, '%Y-%m-%d').date()
                except:
                    self.start_date = None

        try:
            self.end_date = datetime.strptime(self.end_date, '%Y%m%d').date()
        except:
            try:
                self.end_date = datetime.strptime(self.end_date,
                                                  '%d/%m/%Y').date()
            except:
                try:
                    self.end_date = datetime.strptime(self.end_date,
                                                      '%Y-%m-%d').date()
                except:
                    self.end_date = None

        try:
            self.thumbnail_size = int(self.thumbnail_size)
        except:
            self.thumbnail_size = 512

        # Other variables set from config file only - not used
        try:
            self.min_path = int(self.min_path)
        except:
            self.min_path = None
        try:
            self.max_path = int(self.max_path)
        except:
            self.max_path = None
        try:
            self.min_row = int(self.min_row)
        except:
            self.min_row = None
        try:
            self.max_row = int(self.max_row)
        except:
            self.max_row = None

        self.style_dict = {
            #                           'IconStyle': {'scale': 0.4, 'Icon': {'href': 'http://maps.google.com/mapfiles/kml/shapes/star.png'}},
            'LabelStyle': {
                'color': '9900ffff',
                'scale': 1
            },
            'LineStyle': {
                'color': '990000ff',
                'width': 2
            },
            'PolyStyle': {
                'color': '997f7fff',
                'fill': 1,
                'outline': 1
            }
        }