def scene_dir_setup(): asset_bn = 'global-daily-chirps-v2.0.1997.07.01.tif.gz' product_bn = 'global_1997182_chirps_precip.tif' tile_fp = os.path.join(chirps.chirpsRepository.path('tiles'), 'global') scene_fp = os.path.join(tile_fp, '1997182') target_asset_fp = os.path.join(scene_fp, asset_bn) target_product_fp = os.path.join(scene_fp, product_bn) link_target = '/vsigzip/' + target_asset_fp # race condition here, but for integration tests, probably close enough if os.path.lexists(target_product_fp): raise IOError('file obstructing test, aborting: ' + target_product_fp) orm.setup() # setup by moving the asset into the archive, but only if needed use_fake_asset = not os.path.exists(target_asset_fp) if use_fake_asset: source_asset_fp = os.path.join(os.path.dirname(__file__), 'data', asset_bn) made_dirs = [d for d in tile_fp, scene_fp if not os.path.exists(d)] [os.mkdir(d) for d in made_dirs] shutil.copy(source_asset_fp, target_asset_fp) dbinv.rectify_assets(chirps.chirpsAsset) yield target_product_fp, link_target # cleanup (using a fake DB means don't have to rectify post-delete) if os.path.lexists(target_product_fp): os.remove(target_product_fp) if use_fake_asset: os.remove(target_asset_fp) [os.rmdir(d) for d in reversed(made_dirs)] # <- remove in correct order
def migrate_database(): """Migrate the database if the ORM is turned on.""" if not orm.use_orm(): return print 'Migrating database' orm.setup() call_command('migrate', interactive=False)
def t_setup(mocker, setup_complete, use_orm, expected): """Test orm.setup() to confirm it only runs setup at the right times.""" mocker.patch('gips.inventory.orm.setup_complete', setup_complete) m_use_orm = mocker.patch('gips.inventory.orm.use_orm') m_use_orm.return_value = use_orm m_django_setup = mocker.patch('gips.inventory.orm.django.setup') orm.setup() # confirm it ran/didn't run django.setup as expected, and confirm # that the global var got set in any case assert expected == m_django_setup.called and orm.setup_complete
def gips_script_setup(driver_string=None, stop_on_error=False, setup_orm=True): """Run this at the beginning of a GIPS CLI program to do setup.""" global _stop_on_error _stop_on_error = stop_on_error set_error_handler(cli_error_handler) from gips.inventory import orm # avoids a circular import with error_handler(): # must run before orm.setup data_class = None if driver_string is None else import_data_class( driver_string) if setup_orm: orm.setup() return data_class
def t_inventory_setup(override_settings, mocker): override_settings().GIPS_ORM = True setup_mock = mocker.patch.object(gips.inventory.orm.django, 'setup') gips.inventory.orm.setup_complete = False # in case its been called already setup() # orm.setup() assert gips.inventory.orm.setup_complete setup_mock.assert_called_once_with() setup() # call again and confirm that situation still looks right assert gips.inventory.orm.setup_complete setup_mock.assert_called_once_with()
def main(): title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF # argument parsing parser = GIPSParser(description=title) group = parser.add_argument_group('archive options') group.add_argument('--keep', help='Keep files after adding to archive', default=False, action='store_true') group.add_argument('--recursive', help='Iterate through subdirectories', default=False, action='store_true') group.add_argument( '--update', help= 'Update asset if newer version available, (must call gips_process to regenerate products', default=False, action='store_true') group.add_argument( '--path', default='.', help='Path to search for files to archive, defaults to `.`') args = parser.parse_args() utils.gips_script_setup(None, args.stop_on_error) with utils.error_handler('Data archive error'): print title cls = import_data_class(args.command) orm.setup() # set up DB orm in case it's needed for Asset.archive() archived_assets = cls.archive_assets(args.path, args.recursive, args.keep, args.update) # if DB inventory is enabled, update it to contain the newly archived assets if orm.use_orm(): for a in archived_assets: dbinv.update_or_add_asset(asset=a.asset, sensor=a.sensor, tile=a.tile, date=a.date, name=a.archived_filename, driver=cls.name.lower()) utils.gips_exit()
def t_api_inventory(driver, spatial, product, datespec, expected): """ Test gips.data.core.Data.inventory for different API parameter combinations. """ driver_setup.setup_repo_data(driver) from gips.data.modis import modisData, modisAsset from gips.inventory import orm import gips.inventory.dbinv.api as api orm.setup() api.rectify_assets(modisAsset) r = spatial if spatial.endswith('.tif') else None s = spatial if r is None else None inv = modisData.inventory( site=s, rastermask=r, dates=datespec, products=[product], ) assert len(inv.dates) == expected