def _deconstruct_resource_key(key): d, dataset_code, area_type, area_code, filename = key.key.split('/') fsplit = filename.split('_') if len(fsplit) < 2: raise ValueError filename_dataset_code = fsplit[0] filename_area_code = fsplit[1].split('.')[0] if dataset_code != filename_dataset_code or area_code != filename_area_code: print(dataset_code, filename_dataset_code) raise ValueError base = { 'dataset_code': dataset_code, 'filename': filename, 'url': key.generate_url(expires_in=0, query_auth=False), } if area_type in ['half-degree', 'one-degree', 'two-degree']: codes = dsn.code_translate(area_code, area_type, 'tnris-quad') return [ dict(area_code=code, area_type='quad', **base) for code in codes ] else: return [dict(area_code=area_code, area_type=area_type, **base)]
def _migrate_quad_data(old_prefix, new_abbreviation, name_scheme): old_keys = old_bucket.list(prefix=old_prefix) ext = '.zip' for old_key in old_keys: s = old_key.key split = s.split(old_prefix)[1].split(ext)[0].split('_') if name_scheme.startswith('drg_'): area_type = name_scheme[4:] full_code = split[-1].lower() code = full_code[:2] + full_code[3:] if area_type in ['quad', 'half-degree']: code = dsn.code_translate(code, 'usgs-quad', 'tnris-quad') elif name_scheme == 'nwi': area_type = 'quad' code = ''.join(split) # strip out first number of longitude value to match the quad numbering # convention used virtually everywhere else in TNRIS data new_url = _new_url(new_abbreviation, area_type, code, ext) _s3_copy(old_key, new_url)