def newshp(self, task):
"""Tiles a shapefile specified in the task and registers metadata with
GAE.
Arguments:
task - an item from the queue expected to have shapfile path
"""
# Validates task:
if task is None:
logging.warn('newshp task was None')
del self.g.QUEUED_LAYERS[task[self.g.Q_ITEM_FULL_PATH]]
return
if not task.has_key(self.g.Q_ITEM_FULL_PATH):
logging.warn('newshp task does not have %s' % self.g.Q_ITEM_FULL_PATH)
del self.g.QUEUED_LAYERS[task[self.g.Q_ITEM_FULL_PATH]]
return
fullpath = task[self.g.Q_ITEM_FULL_PATH]
if fullpath is None or len(fullpath.strip()) == 0:
logging.warn('newshp task has invalid path ' % fullpath)
del self.g.QUEUED_LAYERS[task[self.g.Q_ITEM_FULL_PATH]]
return
logging.info('Starting new task')
# Executes the job
layer = None
try:
layer = Layer(fullpath, self.g.TILE_DIR, self.g.ERR_DIR,
self.g.SRC_DIR, self.g.DST_DIR, self.g.MAP_XML,
self.g.TILE_URL,
self.g.LAYER_URL,
self.g.VALID_ID_SERVICE_URL,
zoom=self.g.TILE_MAX_ZOOM)
logging.info('Layer created: ' + fullpath)
layer.totiles(self.g)
logging.info('Layers tiled in ' + self.g.TILE_DIR)
logging.info('Layer metadata getting registered...')
layer.register()
logging.info('Layer getting cleaned up...')
layer.cleanup()
except (SpeciesIdError), e:
logging.info('SpeciesIdError ' + fullpath)
id = Layer.idfrompath(fullpath)[0]
logging.warn('id=' + id)
err_dir = self.g.ERR_DIR
src_dir = self.g.SRC_DIR
err_dir = os.path.join(err_dir, 'animalia/species')
# Copies to errors directory:
for file in os.listdir(src_dir):
if file.startswith(id):
shutil.copy2(os.path.join(src_dir, file), err_dir)
# Removes from source directory:
for file in os.listdir(src_dir):
if file.startswith(id):
os.remove(os.path.join(src_dir, file))
def test_cleanup(self):
if True:
return
# ----------------------------------------------------------------------
# Cleanup without errors:
# Sets up directories for testing:
tiledir = tempfile.mkdtemp()
errdir = tempfile.mkdtemp()
newdir = tempfile.mkdtemp()
archivedir = tempfile.mkdtemp()
mapxml = '/ftp/tile/mapfile.xml'
shutil.copytree(EXAMPLE_SHAPEFILE_PATH, os.path.join(newdir, EXAMPLE_SHAPEFILE_ID))
path = os.path.join(newdir, '%s/%s.shp' % (EXAMPLE_SHAPEFILE_ID, EXAMPLE_SHAPEFILE_ID))
# Creates the layer
layer = Layer(path, tiledir, errdir, newdir, archivedir, mapxml)
self.assertNotEqual(layer, None)
# Tiles and cleans up without errors:
layer.totiles()
layer.cleanup()
self.assertFalse(os.path.exists(os.path.join(newdir, EXAMPLE_SHAPEFILE_ID)))
self.assertFalse(os.path.exists(os.path.join(errdir, EXAMPLE_SHAPEFILE_ID)))
self.assertTrue(os.path.exists(os.path.join(archivedir, EXAMPLE_SHAPEFILE_ID)))
self.assertTrue(os.path.exists(os.path.join(tiledir, EXAMPLE_SHAPEFILE_ID)))
# Removes test directories:
shutil.rmtree(tiledir)
shutil.rmtree(errdir)
shutil.rmtree(newdir)
shutil.rmtree(archivedir)
# ----------------------------------------------------------------------
# Cleanup with errors:
tiledir = tempfile.mkdtemp()
errdir = tempfile.mkdtemp()
newdir = tempfile.mkdtemp()
archivedir = tempfile.mkdtemp()
mapxml = '/ftp/tile/mapfile.xml'
shutil.copytree(EXAMPLE_SHAPEFILE_PATH, os.path.join(newdir, EXAMPLE_SHAPEFILE_ID))
path = os.path.join(newdir, '%s/%s.shp' % (EXAMPLE_SHAPEFILE_ID, EXAMPLE_SHAPEFILE_ID))
# Creates the layer
layer = Layer(path, tiledir, errdir, newdir, archivedir, mapxml)
self.assertNotEqual(layer, None)
# Tiles and cleans with errors:
layer.totiles()
layer.cleanup(error=True)
self.assertTrue(os.path.exists(os.path.join(newdir, EXAMPLE_SHAPEFILE_ID)))
self.assertTrue(os.path.exists(os.path.join(errdir, EXAMPLE_SHAPEFILE_ID)))
self.assertFalse(os.path.exists(os.path.join(archivedir, EXAMPLE_SHAPEFILE_ID)))
self.assertFalse(os.path.exists(os.path.join(tiledir, EXAMPLE_SHAPEFILE_ID)))
# Removes test directories:
shutil.rmtree(tiledir)
shutil.rmtree(errdir)
shutil.rmtree(newdir)
shutil.rmtree(archivedir)
def test_register_error(self):
Layer.register_error(EXAMPLE_SHAPEFILE_ID, 'IOError', 'File not found')