def cartodb_sync(shp, production_table, where_clause, gfw_env, scratch_workspace):
"""
Function called by VectorLayer and other Layer objects as part of layer.update()
Will carry out the sync process from start to finish-- pushing the shp to a staging table on cartodb, then
to production on cartodb, using a where_clause if included
:param shp: input feature class (can be GDB FC or SDE too)
:param production_table: final output table in cartoDB
:param where_clause: where_clause to use when adding/deleting from final prod table
:param gfw_env: gfw env
:param scratch_workspace: scratch workspace
:return:
"""
# ogr2ogr can't use an SDE fc as an input; must export to GDB
if where_clause or '@localhost).sde' in shp:
shp = util.fc_to_temp_gdb(shp, scratch_workspace, where_clause)
basename = os.path.basename(shp)
staging_table = os.path.splitext(basename)[0] + '_staging'
delete_staging_table_if_exists(staging_table, gfw_env)
# Create a temp ID field (set equal to OBJECTID) that we'll use to manage pushing to cartodb incrementally
temp_id_field = util.create_temp_id_field(shp, gfw_env)
validated_fc_in_sqlite = cartodb_make_valid_geom_local(shp)
cartodb_create(validated_fc_in_sqlite, production_table, staging_table, temp_id_field, gfw_env)
cartodb_delete_where_clause_or_truncate_prod_table(production_table, where_clause, gfw_env)
cartodb_push_to_production(staging_table, production_table, gfw_env)
delete_staging_table_if_exists(staging_table, gfw_env)
def cartodb_sync(shp, production_table, where_clause, gfw_env,
scratch_workspace):
"""
Function called by VectorLayer and other Layer objects as part of layer.update()
Will carry out the sync process from start to finish-- pushing the shp to a staging table on cartodb, then
to production on cartodb, using a where_clause if included
:param shp: input feature class (can be GDB FC or SDE too)
:param production_table: final output table in cartoDB
:param where_clause: where_clause to use when adding/deleting from final prod table
:param gfw_env: gfw env
:param scratch_workspace: scratch workspace
:return:
"""
# ogr2ogr can't use an SDE fc as an input; must export to GDB
if where_clause or 'localhost' in shp:
shp = util.fc_to_temp_gdb(shp, scratch_workspace, where_clause)
basename = os.path.basename(shp)
staging_table = os.path.splitext(basename)[0] + '_staging'
delete_staging_table_if_exists(staging_table, gfw_env)
# Create a temp ID field (set equal to OBJECTID) that we'll use to manage pushing to cartodb incrementally
temp_id_field = util.create_temp_id_field(shp, gfw_env)
validated_fc_in_sqlite = cartodb_make_valid_geom_local(shp)
cartodb_create(validated_fc_in_sqlite, production_table, staging_table,
temp_id_field, gfw_env)
cartodb_delete_where_clause_or_truncate_prod_table(production_table,
where_clause, gfw_env)
cartodb_push_to_production(staging_table, production_table, gfw_env)
delete_staging_table_if_exists(staging_table, gfw_env)
def zip_file(input_fc, temp_zip_dir, download_output=None, archive_output=None, sr_is_local=False):
"""
:param input_fc: feature class/raster to zip
:param temp_zip_dir: output zip dir
:param download_output: path to the download output, if required
:param archive_output: path to the archive output, if requried
:param sr_is_local: if the spatial reference is local, will create a _local.zip in download_output
:return: None
"""
logging.debug('Starting archive.zip_file')
basepath, fname, base_fname = util.gen_paths_shp(input_fc)
temp_dir = util.create_temp_dir(temp_zip_dir)
data_type = arcpy.Describe(input_fc).dataType
if data_type in ['FeatureClass', 'ShapeFile']:
# Try to create a shapefile first, knowing that the data may be too large and may have to use an FGDB instead
logging.debug('trying to zip SHP-----------------')
arcpy.FeatureClassToShapefile_conversion(input_fc, temp_dir)
out_shp = os.path.join(temp_dir, fname)
# If the dir with the shapefile is < 2GB, zip the shapefile
if all_files_less_than_2gb(temp_dir):
temp_zip = zip_shp(out_shp)
else:
logging.debug('Some components of SHP > 2 GB; now exporting to GDB instead')
# Delete shapefile conversion dir and start fresh
temp_dir = util.create_temp_dir(temp_zip_dir)
gdb_fc = util.fc_to_temp_gdb(input_fc, temp_dir)
gdb_dir = os.path.dirname(os.path.dirname(gdb_fc))
temp_zip = zip_dir(gdb_dir)
elif data_type == 'RasterDataset':
temp_zip = zip_tif(input_fc)
else:
logging.error('Unknown data_type: {0}. Exiting the program'.format(data_type))
sys.exit(1)
# Define output path for archive zip file and copy temp zip there
if archive_output:
logging.debug('Archiving {0} in {1}'.format(base_fname, archive_output))
ts = time.time()
timestamp = datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d%H%M%S')
dst = os.path.splitext(archive_output)[0] + '_{0}.zip'.format(timestamp)
shutil.copy(temp_zip, dst)
# Define output path for download zip file and copy temp zip there
if download_output:
logging.debug("Copying {0} to download folder {1}".format(base_fname, download_output))
if sr_is_local:
dst = os.path.splitext(download_output)[0] + "_local.zip"
else:
dst = download_output
shutil.copy(temp_zip, dst)
def zip_file(input_fc, temp_zip_dir, download_output=None, archive_output=None, sr_is_local=False):
"""
:param input_fc: feature class/raster to zip
:param temp_zip_dir: output zip dir
:param download_output: path to the download output, if required
:param archive_output: path to the archive output, if requried
:param sr_is_local: if the spatial reference is local, will create a _local.zip in download_output
:return: None
"""
logging.debug('Starting archive.zip_file')
basepath, fname, base_fname = util.gen_paths_shp(input_fc)
temp_dir = util.create_temp_dir(temp_zip_dir)
data_type = arcpy.Describe(input_fc).dataType
if data_type in ['FeatureClass', 'ShapeFile']:
# Try to create a shapefile first, knowing that the data may be too large and may have to use an FGDB instead
logging.debug('trying to zip SHP-----------------')
arcpy.FeatureClassToShapefile_conversion(input_fc, temp_dir)
out_shp = os.path.join(temp_dir, fname)
# If the dir with the shapefile is < 2GB, zip the shapefile
if all_files_less_than_2gb(temp_dir):
temp_zip = zip_shp(out_shp)
else:
logging.debug('Some components of SHP > 2 GB; now exporting to GDB instead')
# Delete shapefile conversion dir and start fresh
temp_dir = util.create_temp_dir(temp_zip_dir)
gdb_fc = util.fc_to_temp_gdb(input_fc, temp_dir)
gdb_dir = os.path.dirname(os.path.dirname(gdb_fc))
temp_zip = zip_dir(gdb_dir)
elif data_type == 'RasterDataset':
temp_zip = zip_tif(input_fc)
else:
logging.error('Unknown data_type: {0}. Exiting the program'.format(data_type))
sys.exit(1)
# Define output path for archive zip file and copy temp zip there
if archive_output:
logging.debug('Archiving {0} in {1}'.format(base_fname, archive_output))
ts = time.time()
timestamp = datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d%H%M%S')
dst = os.path.splitext(archive_output)[0] + '_{0}.zip'.format(timestamp)
if r's3://' in dst:
subprocess.check_call(['aws', 's3', 'cp', temp_zip, dst])
else:
shutil.copy(temp_zip, dst)
# Define output path for download zip file and copy temp zip there
if download_output:
logging.debug("Copying {0} to download folder {1}".format(base_fname, download_output))
if sr_is_local:
dst = os.path.splitext(download_output)[0] + "_local.zip"
else:
dst = download_output
if r's3://' in dst:
subprocess.check_call(['aws', 's3', 'cp', temp_zip, dst])
else:
shutil.copy(temp_zip, dst)
