Exemplo n.º 1
0
    def newshp(self, task):
        """Tiles a shapefile specified in the task and registers metadata with
        GAE.

        Arguments:
            task - an item from the queue expected to have shapfile path
        """
        # Validates task:
        if task is None:
            logging.warn('newshp task was None')
            del self.g.QUEUED_LAYERS[task[self.g.Q_ITEM_FULL_PATH]]
            return
        if not task.has_key(self.g.Q_ITEM_FULL_PATH):
            logging.warn('newshp task does not have %s' % self.g.Q_ITEM_FULL_PATH)
            del self.g.QUEUED_LAYERS[task[self.g.Q_ITEM_FULL_PATH]]
            return
        fullpath = task[self.g.Q_ITEM_FULL_PATH]
        if fullpath is None or len(fullpath.strip()) == 0:
            logging.warn('newshp task has invalid path ' % fullpath)
            del self.g.QUEUED_LAYERS[task[self.g.Q_ITEM_FULL_PATH]]
            return

        logging.info('Starting new task')
        # Executes the job      
        layer = None
        try:
            layer = Layer(fullpath, self.g.TILE_DIR, self.g.ERR_DIR,
                          self.g.SRC_DIR, self.g.DST_DIR, self.g.MAP_XML,
                          self.g.TILE_URL,
                          self.g.LAYER_URL,
                          self.g.VALID_ID_SERVICE_URL,
                          zoom=self.g.TILE_MAX_ZOOM)                          
            logging.info('Layer created: ' + fullpath)
            layer.totiles(self.g)
            logging.info('Layers tiled in ' + self.g.TILE_DIR)
            logging.info('Layer metadata getting registered...')
            layer.register()
            logging.info('Layer getting cleaned up...')
            layer.cleanup()

        
        except (SpeciesIdError), e:
            logging.info('SpeciesIdError ' + fullpath)
            id = Layer.idfrompath(fullpath)[0]
            logging.warn('id=' + id)
            err_dir = self.g.ERR_DIR
            src_dir = self.g.SRC_DIR
            err_dir = os.path.join(err_dir, 'animalia/species')
            # Copies to errors directory:
            for file in os.listdir(src_dir):
                if file.startswith(id):
                    shutil.copy2(os.path.join(src_dir, file), err_dir)
            # Removes from source directory:
            for file in os.listdir(src_dir):
                if file.startswith(id):
                    os.remove(os.path.join(src_dir, file))
Exemplo n.º 2
0
    def test_cleanup(self):
        if True:
            return
        # ----------------------------------------------------------------------
        # Cleanup without errors:

        # Sets up directories for testing:
        tiledir = tempfile.mkdtemp()
        errdir = tempfile.mkdtemp()
        newdir = tempfile.mkdtemp()
        archivedir = tempfile.mkdtemp()
        mapxml = '/ftp/tile/mapfile.xml'
        shutil.copytree(EXAMPLE_SHAPEFILE_PATH, os.path.join(newdir, EXAMPLE_SHAPEFILE_ID))
        path = os.path.join(newdir, '%s/%s.shp' % (EXAMPLE_SHAPEFILE_ID, EXAMPLE_SHAPEFILE_ID))

        # Creates the layer
        layer = Layer(path, tiledir, errdir, newdir, archivedir, mapxml)
        self.assertNotEqual(layer, None)

        # Tiles and cleans up without errors:
        layer.totiles()
        layer.cleanup()
        self.assertFalse(os.path.exists(os.path.join(newdir, EXAMPLE_SHAPEFILE_ID)))
        self.assertFalse(os.path.exists(os.path.join(errdir, EXAMPLE_SHAPEFILE_ID)))
        self.assertTrue(os.path.exists(os.path.join(archivedir, EXAMPLE_SHAPEFILE_ID)))
        self.assertTrue(os.path.exists(os.path.join(tiledir, EXAMPLE_SHAPEFILE_ID)))

        # Removes test directories:
        shutil.rmtree(tiledir)
        shutil.rmtree(errdir)
        shutil.rmtree(newdir)
        shutil.rmtree(archivedir)

        # ----------------------------------------------------------------------
        # Cleanup with errors:

        tiledir = tempfile.mkdtemp()
        errdir = tempfile.mkdtemp()
        newdir = tempfile.mkdtemp()
        archivedir = tempfile.mkdtemp()
        mapxml = '/ftp/tile/mapfile.xml'
        shutil.copytree(EXAMPLE_SHAPEFILE_PATH, os.path.join(newdir, EXAMPLE_SHAPEFILE_ID))
        path = os.path.join(newdir, '%s/%s.shp' % (EXAMPLE_SHAPEFILE_ID, EXAMPLE_SHAPEFILE_ID))

        # Creates the layer
        layer = Layer(path, tiledir, errdir, newdir, archivedir, mapxml)
        self.assertNotEqual(layer, None)

        # Tiles and cleans with errors:
        layer.totiles()
        layer.cleanup(error=True)
        self.assertTrue(os.path.exists(os.path.join(newdir, EXAMPLE_SHAPEFILE_ID)))
        self.assertTrue(os.path.exists(os.path.join(errdir, EXAMPLE_SHAPEFILE_ID)))
        self.assertFalse(os.path.exists(os.path.join(archivedir, EXAMPLE_SHAPEFILE_ID)))
        self.assertFalse(os.path.exists(os.path.join(tiledir, EXAMPLE_SHAPEFILE_ID)))

        # Removes test directories:
        shutil.rmtree(tiledir)
        shutil.rmtree(errdir)
        shutil.rmtree(newdir)
        shutil.rmtree(archivedir)