def execute(request):
"""Move files to a target folder.
:param request: json as a dict.
"""
moved = 0
skipped = 0
errors = 0
new_folder = False
parameters = request['params']
target_folder = task_utils.get_parameter_value(parameters, 'target_folder', 'value')
flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value')
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
if target_folder:
if not os.path.exists(target_folder):
os.makedirs(target_folder)
new_folder = True
num_results, response_index = task_utils.get_result_count(parameters)
task_utils.CHUNK_SIZE = num_results
# Query the index for results in groups of 25.
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'), 'move_files')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),headers=headers)
input_items = task_utils.get_input_items(results.json()['response']['docs'], True, True)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
result = move_files(input_items, target_folder, flatten_results)
moved += result[0]
errors += result[1]
skipped += result[2]
status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'move_files')
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(request['folder'], '__report.json'), moved, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Copies files to a target folder.
:param request: json as a dict.
"""
copied = 0
skipped = 0
errors = 0
global result_count
parameters = request['params']
target_dirs = ''
target_folder = task_utils.get_parameter_value(parameters, 'target_folder', 'value')
flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value')
if not flatten_results:
target_dirs = os.path.splitdrive(target_folder)[1]
flatten_results = 'false'
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
# Query the index for results in groups of 25.
result_count, response_index = task_utils.get_result_count(parameters)
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'), 'copy_files')
i = 0.
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
input_items = task_utils.get_input_items(results.json()['response']['docs'], list_components=True)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
result = copy_files(input_items, target_folder, flatten_results, target_dirs)
copied += result[0]
errors += result[1]
skipped += result[2]
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(request['folder'], '__report.json'), copied, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Copies files to a target folder.
:param request: json as a dict.
"""
copied = 0
skipped = 0
errors = 0
global result_count
parameters = request['params']
target_dirs = ''
target_folder = task_utils.get_parameter_value(parameters, 'target_folder', 'value')
flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value')
if not flatten_results:
target_dirs = os.path.splitdrive(target_folder)[1]
flatten_results = 'false'
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
# Query the index for results in groups of 25.
result_count, response_index = task_utils.get_result_count(parameters)
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'), 'copy_files')
i = 0.
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)
input_items = task_utils.get_input_items(results.json()['response']['docs'], list_components=True)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
result = copy_files(input_items, target_folder, flatten_results, target_dirs)
copied += result[0]
errors += result[1]
skipped += result[2]
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(request['folder'], '__report.json'), copied, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Remove tags.
:param request: json as a dict.
"""
query = ''
errors = 0
parameters = request['params']
archive_location = request['folder']
if not os.path.exists(archive_location):
os.makedirs(archive_location)
# Parameter values
snapshot_name = task_utils.get_parameter_value(parameters, 'snapshot_name', 'value')
data_folder = task_utils.get_parameter_value(parameters, 'data_folder', 'value')
request_owner = request['owner']
result_count, response_index = task_utils.get_result_count(parameters)
# Get the query index and query (if any).
# query_index = task_utils.QueryIndex(parameters[response_index])
fq = ''
if 'fq' in parameters[response_index]['query']:
if isinstance(parameters[response_index]['query']['fq'], list):
for q in parameters[response_index]['query']['fq']:
if '{!tag=' in q:
q = q.split('}')[1]
fq += q + ' AND '
fq = fq.strip(' AND ')
else:
# Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
fq += '&fq={0}'.format(parameters[response_index]['query']['fq'].replace("\\", ""))
fq = fq.replace(' ', '%20')
# fq = query_index.get_fq()
if fq:
query = fq.replace('&fq=', '')
if query:
result = create_snapshot(snapshot_name, request_owner, query)
else:
result = create_snapshot(snapshot_name, request_owner)
if not result[0]:
errors += 1
errors_reasons[snapshot_name] = result[1]
# Update state if necessary.
if errors > 0:
status_writer.send_state(status.STAT_FAILED)
else:
status_writer.send_status(result[1])
result_name = ''
while not result_name:
file_path = os.path.join(data_folder, 'backup', '*{0}.zip'.format(snapshot_name))
try:
result_name = os.path.basename(glob.glob(file_path)[0])
except IndexError:
continue
shutil.copyfile(os.path.join(data_folder, 'backup', result_name), os.path.join(archive_location, result_name))
task_utils.report(os.path.join(request['folder'], '__report.json'), 1, 0, errors, errors_details=errors_reasons)
def execute(request):
"""Delete items.
:param request: json as a dict.
"""
query = ''
errors = 0
parameters = request['params']
archive_location = request['folder']
if not os.path.exists(archive_location):
os.makedirs(archive_location)
# Parameter values
delete_thumbs = task_utils.get_parameter_value(
parameters, 'delete_thumbnails', 'value') or False
delete_metadata = task_utils.get_parameter_value(
parameters, 'delete_metadata', 'value') or False
delete_layers = task_utils.get_parameter_value(parameters, 'delete_layers',
'value') or False
request_owner = request['owner']
result_count, response_index = task_utils.get_result_count(parameters)
fq = ''
if 'fq' in parameters[response_index]['query']:
if isinstance(parameters[response_index]['query']['fq'], list):
for q in parameters[response_index]['query']['fq']:
if '{!tag=' in q:
q = q.split('}')[1]
fq += q + ' AND '
fq = fq.strip(' AND ')
else:
# Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
fq = parameters[response_index]['query']['fq'].replace("\\", "")
if 'q' in parameters[response_index]['query']:
query = parameters[response_index]['query']['q']
result = delete_items(fq, query, delete_thumbs, delete_metadata,
delete_layers, request_owner)
if not result[0]:
errors += 1
errors_reasons['delete_items'] = result[1]
# Update state if necessary.
if errors > 0:
status_writer.send_state(status.STAT_FAILED)
else:
status_writer.send_status(result[1])
task_utils.report(os.path.join(request['folder'], '__report.json'),
1,
0,
errors,
errors_details=errors_reasons)
def execute(request):
"""Delete items.
:param request: json as a dict.
"""
query = ''
errors = 0
parameters = request['params']
archive_location = request['folder']
if not os.path.exists(archive_location):
os.makedirs(archive_location)
# Parameter values
delete_thumbs = task_utils.get_parameter_value(parameters, 'delete_thumbnails', 'value') or False
delete_metadata = task_utils.get_parameter_value(parameters, 'delete_metadata', 'value') or False
delete_layers = task_utils.get_parameter_value(parameters, 'delete_layers', 'value') or False
request_owner = request['owner']
result_count, response_index = task_utils.get_result_count(parameters)
fq = ''
if 'fq' in parameters[response_index]['query']:
if isinstance(parameters[response_index]['query']['fq'], list):
for q in parameters[response_index]['query']['fq']:
if '{!tag=' in q:
q = q.split('}')[1]
fq += q + ' AND '
fq = fq.strip(' AND ')
else:
# Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
fq = parameters[response_index]['query']['fq'].replace("\\", "")
if 'q' in parameters[response_index]['query']:
query = parameters[response_index]['query']['q']
result = delete_items(fq, query, delete_thumbs, delete_metadata, delete_layers, request_owner)
if not result[0]:
errors += 1
errors_reasons['delete_items'] = result[1]
# Update state if necessary.
if errors > 0:
status_writer.send_state(status.STAT_FAILED)
else:
status_writer.send_status(result[1])
task_utils.report(os.path.join(request['folder'], '__report.json'), 1, 0, errors, errors_details=errors_reasons)
def execute(request):
"""Converts each input dataset to kml (.kmz).
:param request: json as a dict.
"""
converted = 0
skipped = 0
errors = 0
global result_count
parameters = request['params']
out_workspace = os.path.join(request['folder'], 'temp')
if not os.path.exists(out_workspace):
os.makedirs(out_workspace)
# Get the boundary box extent for input to KML tools.
extent = ''
try:
try:
ext = task_utils.get_parameter_value(parameters,
'processing_extent', 'wkt')
if ext:
sr = task_utils.get_spatial_reference("4326")
extent = task_utils.from_wkt(ext, sr)
except KeyError:
ext = task_utils.get_parameter_value(parameters,
'processing_extent',
'feature')
if ext:
extent = arcpy.Describe(ext).extent
except KeyError:
pass
# Get the output file name.
output_file_name = task_utils.get_parameter_value(parameters,
'output_file_name',
'value')
if not output_file_name:
output_file_name = 'kml_results'
result_count, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']),
task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'),
'convert_to_kml')
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
for group in groups:
if fq:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query +
'{0}&ids={1}'.format(fl, ','.join(group)),
headers=headers)
else:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
headers=headers)
docs = results.json()['response']['docs']
input_items = task_utils.get_input_items(docs)
if not input_items:
input_items = task_utils.get_input_items(
parameters[response_index]['response']['docs'])
input_rows = collections.defaultdict(list)
for doc in docs:
if 'path' not in doc:
input_rows[doc['name']].append(doc)
if input_rows:
result = convert_to_kml(input_rows, out_workspace, extent)
converted += result[0]
errors += result[1]
skipped += result[2]
if input_items:
result = convert_to_kml(input_items, out_workspace, extent)
converted += result[0]
errors += result[1]
skipped += result[2]
if not input_items and not input_rows:
status_writer.send_state(
status.STAT_FAILED,
_('No items to process. Check if items exist.'))
return
# Zip up kmz files if more than one.
if converted > 1:
status_writer.send_status("Converted: {}".format(converted))
zip_file = task_utils.zip_data(out_workspace,
'{0}.zip'.format(output_file_name))
shutil.move(
zip_file,
os.path.join(os.path.dirname(out_workspace),
os.path.basename(zip_file)))
shutil.copy2(
os.path.join(os.path.dirname(os.path.dirname(__file__)),
'supportfiles', '_thumb.png'), request['folder'])
elif converted == 1:
try:
kml_file = glob.glob(os.path.join(out_workspace, '*.kmz'))[0]
tmp_lyr = arcpy.KMLToLayer_conversion(kml_file, out_workspace,
'kml_layer')
task_utils.make_thumbnail(
tmp_lyr.getOutput(0),
os.path.join(request['folder'], '_thumb.png'))
except arcpy.ExecuteError:
pass
shutil.move(
kml_file,
os.path.join(request['folder'], os.path.basename(kml_file)))
# Update state if necessary.
if skipped > 0 or errors > 0:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(errors + skipped))
task_utils.report(os.path.join(request['folder'],
'__report.json'), converted, skipped,
errors, errors_reasons, skipped_reasons)
def execute(request):
"""Mosaics input raster datasets into a new raster dataset.
:param request: json as a dict.
"""
parameters = request['params']
out_coordinate_system = task_utils.get_parameter_value(parameters, 'output_projection', 'code')
# Advanced options
output_raster_format = task_utils.get_parameter_value(parameters, 'raster_format', 'value')
compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value')
compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value')
output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
if not output_file_name:
output_file_name = 'output'
arcpy.env.compression = '{0} {1}'.format(compression_method, compression_quality)
clip_area = None
if not output_raster_format == 'MosaicDataset':
# Get the clip region as an extent object.
try:
clip_area_wkt = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt')
if not clip_area_wkt:
clip_area_wkt = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'
if not out_coordinate_system == '0':
clip_area = task_utils.get_clip_region(clip_area_wkt, out_coordinate_system)
else:
clip_area = task_utils.get_clip_region(clip_area_wkt)
except KeyError:
pass
status_writer.send_status(_('Setting the output workspace...'))
out_workspace = os.path.join(request['folder'], 'temp')
if not os.path.exists(out_workspace):
os.makedirs(out_workspace)
if output_raster_format == 'FileGDB' or output_raster_format == 'MosaicDataset':
out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0)
arcpy.env.workspace = out_workspace
status_writer.send_status(_('Starting to process...'))
num_results, response_index = task_utils.get_result_count(parameters)
raster_items = None
if num_results > task_utils.CHUNK_SIZE:
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
for group in groups:
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
input_items = task_utils.get_input_items(results.json()['response']['docs'])
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
raster_items, pixels, bands, skipped = get_items(input_items)
else:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
raster_items, pixels, bands, skipped = get_items(input_items)
if not raster_items:
if skipped == 0:
status_writer.send_state(status.STAT_FAILED, _('Invalid input types'))
skipped_reasons['All Items'] = _('Invalid input types')
task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), num_results, skipped_details=skipped_reasons)
return
else:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons)
return
# Get most common pixel type.
pixel_type = pixel_types[max(set(pixels), key=pixels.count)]
if output_raster_format in ('FileGDB', 'GRID', 'MosaicDataset'):
output_name = arcpy.ValidateTableName('mosaic', out_workspace)
else:
output_name = '{0}.{1}'.format(arcpy.ValidateTableName('mosaic', out_workspace)[:9], output_raster_format.lower())
status_writer.send_status(output_name)
if output_raster_format == 'MosaicDataset':
try:
status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters')))
if out_coordinate_system == '0':
out_coordinate_system = raster_items[0]
else:
out_coordinate_system = None
mosaic_ds = arcpy.CreateMosaicDataset_management(out_workspace,
output_name,
out_coordinate_system,
max(bands),
pixel_type)
arcpy.AddRastersToMosaicDataset_management(mosaic_ds, 'Raster Dataset', raster_items)
arcpy.MakeMosaicLayer_management(mosaic_ds, 'mosaic_layer')
layer_object = arcpy.mapping.Layer('mosaic_layer')
task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png'))
except arcpy.ExecuteError:
status_writer.send_state(status.STAT_FAILED, arcpy.GetMessages(2))
return
else:
try:
if len(bands) > 1:
status_writer.send_state(status.STAT_FAILED, _('Input rasters must have the same number of bands'))
return
status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters')))
if out_coordinate_system == '0':
out_coordinate_system = None
if clip_area:
ext = '{0} {1} {2} {3}'.format(clip_area.XMin, clip_area.YMin, clip_area.XMax, clip_area.YMax)
tmp_mosaic = arcpy.MosaicToNewRaster_management(
raster_items,
out_workspace,
'tm',
out_coordinate_system,
pixel_type,
number_of_bands=bands.keys()[0]
)
status_writer.send_status(_('Clipping...'))
out_mosaic = arcpy.Clip_management(tmp_mosaic, ext, output_name)
arcpy.Delete_management(tmp_mosaic)
else:
out_mosaic = arcpy.MosaicToNewRaster_management(raster_items,
out_workspace,
output_name,
out_coordinate_system,
pixel_type, number_of_bands=bands.keys()[0])
arcpy.MakeRasterLayer_management(out_mosaic, 'mosaic_layer')
layer_object = arcpy.mapping.Layer('mosaic_layer')
task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png'))
except arcpy.ExecuteError:
status_writer.send_state(status.STAT_FAILED, arcpy.GetMessages(2))
return
if arcpy.env.workspace.endswith('.gdb'):
out_workspace = os.path.dirname(arcpy.env.workspace)
zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
# Update state if necessary.
if skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons)
def execute(request):
"""Writes existing metadata for summary, description and tags.
If overwrite is false, existing metadata is untouched unless any
field is empty or does not exist, then it is created.
:param request: json as a dict.
"""
updated = 0
errors = 0
skipped = 0
global result_count
parameters = request['params']
summary = task_utils.get_parameter_value(parameters, 'summary', 'value')
description = task_utils.get_parameter_value(parameters, 'description',
'value')
tags = task_utils.get_parameter_value(parameters, 'tags', 'value')
data_credits = task_utils.get_parameter_value(parameters, 'credits',
'value')
constraints = task_utils.get_parameter_value(parameters, 'constraints',
'value')
# Handle commas, spaces, and/or new line separators.
tags = [tag for tag in re.split(' |,|\n', tags) if not tag == '']
overwrite = task_utils.get_parameter_value(parameters, 'overwrite',
'value')
if not overwrite:
overwrite = False
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
# Stylesheet
xslt_file = os.path.join(
arcpy.GetInstallInfo()['InstallDir'],
'Metadata/Stylesheets/gpTools/exact copy of.xslt')
# Template metadata file.
template_xml = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'supportfiles', 'metadata_template.xml')
result_count, response_index = task_utils.get_result_count(parameters)
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl + ',links'
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']),
task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'),
'write_metadata')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query +
'{0}&ids={1}'.format(fl, ','.join(group)),
headers=headers)
else:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
headers=headers)
docs = results.json()['response']['docs']
if not docs:
docs = parameters[response_index]['response']['docs']
input_items = []
for doc in docs:
if 'path' in doc:
if 'links' in doc:
links = eval(doc['links'])
input_items.append(
(doc['path'], links['links'][0]['link'][0]['id']))
else:
input_items.append((doc['path'], doc['id']))
result = write_metadata(input_items, template_xml, xslt_file, summary,
description, tags, data_credits, constraints,
overwrite, headers)
updated += result[0]
errors += result[1]
skipped += result[2]
status_writer.send_percent(
i / result_count, '{0}: {1:.0f}%'.format("Processed",
i / result_count * 100),
'write_metadata')
# Report state.
if skipped > 0 or errors > 0:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(skipped + errors))
else:
status_writer.send_state(status.STAT_SUCCESS)
task_utils.report(os.path.join(request['folder'],
'__report.json'), updated, skipped, errors,
errors_reasons, skipped_reasons)
def execute(request):
"""Converts each input dataset to kml (.kmz).
:param request: json as a dict.
"""
converted = 0
skipped = 0
errors = 0
global result_count
parameters = request['params']
out_workspace = os.path.join(request['folder'], 'temp')
if not os.path.exists(out_workspace):
os.makedirs(out_workspace)
# Get the boundary box extent for input to KML tools.
extent = ''
try:
try:
ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt')
if ext:
sr = task_utils.get_spatial_reference("4326")
extent = task_utils.from_wkt(ext, sr)
except KeyError:
ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'feature')
if ext:
extent = arcpy.Describe(ext).extent
except KeyError:
pass
# Get the output file name.
output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
if not output_file_name:
output_file_name = 'kml_results'
result_count, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'), 'convert_to_kml')
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
for group in groups:
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
docs = results.json()['response']['docs']
input_items = task_utils.get_input_items(docs)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
input_rows = collections.defaultdict(list)
for doc in docs:
if 'path' not in doc:
input_rows[doc['name']].append(doc)
if input_rows:
result = convert_to_kml(input_rows, out_workspace, extent)
converted += result[0]
errors += result[1]
skipped += result[2]
if input_items:
result = convert_to_kml(input_items, out_workspace, extent)
converted += result[0]
errors += result[1]
skipped += result[2]
if not input_items and not input_rows:
status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.'))
return
# Zip up kmz files if more than one.
if converted > 1:
status_writer.send_status("Converted: {}".format(converted))
zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
shutil.copy2(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', '_thumb.png'), request['folder'])
elif converted == 1:
try:
kml_file = glob.glob(os.path.join(out_workspace, '*.kmz'))[0]
tmp_lyr = arcpy.KMLToLayer_conversion(kml_file, out_workspace, 'kml_layer')
task_utils.make_thumbnail(tmp_lyr.getOutput(0), os.path.join(request['folder'], '_thumb.png'))
except arcpy.ExecuteError:
pass
shutil.move(kml_file, os.path.join(request['folder'], os.path.basename(kml_file)))
# Update state if necessary.
if skipped > 0 or errors > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), converted, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Zips all input files to output.zip.
:param request: json as a dict.
"""
zipped = 0
skipped = 0
parameters = request['params']
flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value')
if not flatten_results:
flatten_results = False
zip_file_location = request['folder']
if not os.path.exists(zip_file_location):
os.makedirs(request['folder'])
output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
if not output_file_name:
output_file_name = 'output'
zip_file = os.path.join(zip_file_location, '{0}.zip'.format(output_file_name))
zipper = task_utils.ZipFileManager(zip_file, 'w', zipfile.ZIP_DEFLATED)
num_results, response_index = task_utils.get_result_count(parameters)
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
#query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'), 'zip_files')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
input_items = task_utils.get_input_items(results.json()['response']['docs'], list_components=True)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'], list_components=True)
result = zip_files(zipper, input_items, zip_file_location, flatten_results)
zipped += result[0]
skipped += result[1]
status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'zip_files')
zipper.close()
if zipped == 0:
status_writer.send_state(status.STAT_FAILED, _('No results were zipped.'))
return
# Update state if necessary.
if skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), zipped, skipped, skipped_details=skipped_reasons)
def execute(request):
"""Exports search results a CSV, shapefile or XML document.
:param request: json as a dict.
"""
chunk_size = task_utils.CHUNK_SIZE
file_name = task_utils.get_parameter_value(request['params'], 'file_name',
'value')
fields = task_utils.get_parameter_value(request['params'], 'fields',
'value')
out_format = task_utils.get_parameter_value(request['params'],
'output_format', 'value')
if not 'path' in fields and 'path:[absolute]' in fields:
fields.append('path')
if 'geo' in fields:
i_geo = fields.index('geo')
fields.remove('geo')
fields.insert(i_geo, '[geo]')
# Create the temporary workspace.
task_folder = os.path.join(request['folder'], 'temp')
if not os.path.exists(task_folder):
os.makedirs(task_folder)
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
num_results, response_index = task_utils.get_result_count(
request['params'])
query = '{0}/select?&wt=json&fl={1}'.format(sys.argv[2].split('=')[1],
','.join(fields))
if 'query' in request['params'][response_index]:
# Voyager Search Traditional UI
for p in request['params']:
if 'query' in p:
request_qry = p['query']
break
if 'voyager.list' in request_qry:
query += '&voyager.list={0}'.format(request_qry['voyager.list'])
# Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
if 'fq' in request_qry:
try:
query += '&fq={0}'.format(request_qry['fq'].replace("\\", ""))
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['fq']:
query += '&fq={0}'.format(qry).replace("\\", "").replace(
' ', '%20')
if 'q' in request_qry:
try:
query += '&q={0}'.format(request_qry['q'].replace("\\", ""))
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['q']:
query += '&q={0}'.format(qry).replace("\\", "").replace(
' ', '%20')
if 'place' in request_qry:
try:
query += '&place={0}'.format(request_qry['place'].replace(
"\\", ""))
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['place']:
query += '&place={0}'.format(qry).replace("\\",
"").replace(
' ', '%20')
if 'place.op' in request_qry:
query += '&place.op={0}'.format(request_qry['place.op'])
query += '&rows={0}&start={1}'
exported_cnt = 0.
for i in xrange(0, num_results, chunk_size):
req = urllib2.Request(query.replace('{0}',
str(chunk_size)).replace(
'{1}', str(i)),
headers=headers)
for n in urllib2.urlopen(req):
jobs = eval(n.replace('null', '"null"'))['response']['docs']
if out_format == 'CSV':
export_to_csv(jobs, file_name, task_folder, fields)
elif out_format == 'XML':
export_to_xml(jobs, file_name, task_folder)
elif out_format == 'SHP':
export_to_shp(jobs, file_name, task_folder)
exported_cnt += chunk_size
if exported_cnt > num_results:
status_writer.send_percent(100, 'exported: 100%',
'export_results')
else:
percent_done = exported_cnt / num_results
status_writer.send_percent(
percent_done,
'{0}: {1:.0f}%'.format("exported", percent_done * 100),
'export_results')
else:
# Voyager Search Portal/Cart UI
ids = []
for p in request['params']:
if 'ids' in p:
ids = p['ids']
break
groups = task_utils.grouper(list(ids), chunk_size, '')
i = 0
for group in groups:
i += len([v for v in group if not v == ''])
req = urllib2.Request(query + '&ids={0}'.format(','.join(group)),
headers=headers)
results = urllib2.urlopen(req)
jobs = eval(results.read())['response']['docs']
if out_format == 'CSV':
export_to_csv(jobs, file_name, task_folder, fields)
elif out_format == 'XML':
export_to_xml(jobs, file_name, task_folder)
elif out_format == 'SHP':
export_to_shp(jobs, file_name, task_folder)
percent_done = float(i) / num_results
status_writer.send_percent(
percent_done, '{0}: {1:.0f}%'.format("exported",
percent_done * 100),
'export_results')
# Zip up outputs.
if exported_count == 0:
task_utils.report(os.path.join(request['folder'], '__report.json'),
exported_count, 0, errors_count, errors_reasons)
else:
task_utils.report(os.path.join(request['folder'], '__report.json'),
exported_count, 0, errors_count, errors_reasons)
zip_file = task_utils.zip_data(task_folder, 'output.zip')
shutil.move(
zip_file,
os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))
def execute(request):
"""Replace the workspace path for layer files and map document layers.
:param request: json as a dict.
"""
updated = 0
skipped = 0
parameters = request['params']
backup = task_utils.get_parameter_value(parameters, 'create_backup', 'value')
old_data_source = task_utils.get_parameter_value(parameters, 'old_data_source', 'value').lower()
new_data_source = task_utils.get_parameter_value(parameters, 'new_data_source', 'value')
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
if not arcpy.Exists(new_data_source):
status_writer.send_state(status.STAT_FAILED, _('{0} does not exist').format(new_data_source))
return
if os.path.splitext(new_data_source)[1] not in ('.gdb', '.mdb', '.sde'):
new_dataset = os.path.basename(new_data_source)
dsc = arcpy.Describe(os.path.dirname(new_data_source))
else:
dsc = arcpy.Describe(new_data_source)
new_dataset = ''
wks_type = 'NONE'
if dsc.dataType == 'FeatureDataset':
new_workspace = dsc.path
wks_type = get_workspace_type(new_workspace)
elif dsc.dataType == 'Workspace':
new_workspace = dsc.catalogPath
wks_type = get_workspace_type(new_workspace)
elif dsc.dataType == 'Folder':
dsc = arcpy.Describe(new_data_source)
new_workspace = dsc.catalogPath
if new_dataset.endswith('.shp'):
wks_type = 'SHAPEFILE_WORKSPACE'
new_dataset = new_dataset.rsplit('.shp')[0]
else:
if arcpy.Describe(new_data_source).dataType == 'RasterDataset':
wks_type = 'RASTER_WORKSPACE'
elif dsc.dataType == 'CadDrawingDataset':
new_workspace = dsc.path
wks_type = 'CAD_WORKSPACE'
else:
new_workspace = os.path.dirname(new_data_source)
num_results, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}'.format(sys.argv[2].split('=')[1], '/select?&wt=json')
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'), 'replace_data_source')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "{0}&rows={1}&start={2}".format(fl, task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers= headers)
else:
results = requests.get(query + "{0}&rows={1}&start={2}".format(fl, task_utils.CHUNK_SIZE, group[0]), headers=headers)
input_items = task_utils.get_input_items(results.json()['response']['docs'], True)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
result = replace_data_source(input_items, old_data_source, new_workspace, new_dataset, wks_type, backup, headers)
updated += result[0]
skipped += result[1]
status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'replace_data_source')
# Update state if necessary.
if skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), updated, skipped, skipped_details=skipped_reasons)
def execute(request):
"""Creates a GeoPDF.
:param request: json as a dict.
"""
added_to_map = 0
errors = 0
skipped = 0
parameters = request["params"]
num_results, response_index = task_utils.get_result_count(parameters)
docs = parameters[response_index]["response"]["docs"]
input_items = task_utils.get_input_items(docs)
input_rows = collections.defaultdict(list)
for doc in docs:
if "path" not in doc:
input_rows[doc["name"]].append(doc)
if num_results > task_utils.CHUNK_SIZE:
status_writer.send_state(status.STAT_FAILED, "Reduce results to 25 or less.")
return
map_template = task_utils.get_parameter_value(parameters, "map_template", "value")
base_map = task_utils.get_parameter_value(parameters, "base_map", "value")
map_title = task_utils.get_parameter_value(parameters, "map_title", "value")
attribute_setting = task_utils.get_parameter_value(parameters, "attribute_settings", "value")
author = task_utils.get_parameter_value(parameters, "map_author", "value")
output_file_name = task_utils.get_parameter_value(parameters, "output_file_name", "value")
if not output_file_name:
output_file_name = "output_pdf"
try:
map_view = task_utils.get_parameter_value(parameters, "map_view", "extent")
except KeyError:
map_view = None
pass
temp_folder = os.path.join(request["folder"], "temp")
if not os.path.exists(temp_folder):
os.makedirs(temp_folder)
if base_map == "NONE":
base_layer = None
else:
base_layer = arcpy.mapping.Layer(
os.path.join(
os.path.dirname(os.path.dirname(__file__)), "supportfiles", "basemaps", "{0}.lyr".format(base_map)
)
)
mxd_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "supportfiles", "frame", map_template)
mxd = arcpy.mapping.MapDocument(mxd_path)
data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
layers = []
all_layers = []
if input_rows:
for name, rows in input_rows.iteritems():
for row in rows:
try:
name = arcpy.CreateUniqueName(name, "in_memory")
# Create the geometry.
geo_json = row["[geo]"]
geom = arcpy.AsShape(geo_json)
arcpy.CopyFeatures_management(geom, name)
feature_layer = arcpy.MakeFeatureLayer_management(name, os.path.basename(name))
layer_file = arcpy.SaveToLayerFile_management(
feature_layer, os.path.join(temp_folder, "{0}.lyr".format(os.path.basename(name)))
)
layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
added_to_map += 1
except KeyError:
skipped += 1
skipped_reasons[name] = "No geographic information"
continue
for i, item in enumerate(input_items, 1):
try:
# Is the item a mxd data frame.
map_frame_name = task_utils.get_data_frame_name(item)
if map_frame_name:
item = item.split("|")[0].strip()
dsc = arcpy.Describe(item)
if dsc.dataType == "Layer":
layers.append(arcpy.mapping.Layer(dsc.catalogPath))
elif dsc.dataType == "FeatureClass" or dsc.dataType == "ShapeFile":
if os.path.basename(item) in [l.name for l in all_layers]:
layer_name = "{0}_{1}".format(os.path.basename(item), i)
else:
layer_name = os.path.basename(item)
feature_layer = arcpy.MakeFeatureLayer_management(item, layer_name)
layer_file = arcpy.SaveToLayerFile_management(
feature_layer, os.path.join(temp_folder, "{0}.lyr".format(layer_name))
)
layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
elif dsc.dataType == "FeatureDataset":
arcpy.env.workspace = item
for fc in arcpy.ListFeatureClasses():
layer_file = arcpy.SaveToLayerFile_management(
arcpy.MakeFeatureLayer_management(fc, "{0}_{1}".format(fc, i)),
os.path.join(temp_folder, "{0}_{1}.lyr".format(fc, i)),
)
layer = arcpy.mapping.Layer(layer_file.getOutput(0))
layer.name = fc
layers.append(layer)
all_layers.append(layer)
elif dsc.dataType == "RasterDataset":
if os.path.basename(item) in [l.name for l in all_layers]:
layer_name = "{0}_{1}".format(os.path.basename(item), i)
else:
layer_name = os.path.basename(item)
raster_layer = arcpy.MakeRasterLayer_management(item, layer_name)
layer_file = arcpy.SaveToLayerFile_management(
raster_layer, os.path.join(temp_folder, "{0}.lyr".format(layer_name))
)
layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
elif dsc.catalogPath.endswith(".kml") or dsc.catalogPath.endswith(".kmz"):
if not os.path.splitext(dsc.name)[0] in layers:
name = os.path.splitext(dsc.name)[0]
else:
name = "{0}_{1}".format(os.path.splitext(dsc.name)[0], i)
arcpy.KMLToLayer_conversion(dsc.catalogPath, temp_folder, name)
layers.append(arcpy.mapping.Layer(os.path.join(temp_folder, "{0}.lyr".format(name))))
all_layers.append(arcpy.mapping.Layer(os.path.join(temp_folder, "{0}.lyr".format(name))))
elif dsc.dataType == "MapDocument":
input_mxd = arcpy.mapping.MapDocument(item)
if map_frame_name:
df = arcpy.mapping.ListDataFrames(input_mxd, map_frame_name)[0]
layers = arcpy.mapping.ListLayers(input_mxd, data_frame=df)
else:
layers = arcpy.mapping.ListLayers(input_mxd)
if layers:
for layer in layers:
status_writer.send_status(_("Adding layer {0}...").format(layer.name))
arcpy.mapping.AddLayer(data_frame, layer)
added_to_map += 1
layers = []
else:
status_writer.send_status(_("Invalid input type: {0}").format(item))
skipped_reasons[item] = "Invalid input type"
skipped += 1
except Exception as ex:
status_writer.send_status(_("FAIL: {0}").format(repr(ex)))
errors += 1
errors_reasons[item] = repr(ex)
pass
if map_view:
extent = map_view.split(" ")
new_extent = data_frame.extent
new_extent.XMin, new_extent.YMin = float(extent[0]), float(extent[1])
new_extent.XMax, new_extent.YMax = float(extent[2]), float(extent[3])
data_frame.extent = new_extent
else:
data_frame.zoomToSelectedFeatures()
# Update text elements in map template.
date_element = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "date")
if date_element:
date_element[0].text = "Date: {0}".format(task_utils.get_local_date())
title_element = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "title")
if title_element:
title_element[0].text = map_title
author_element = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "author")
if author_element:
author_element[0].text = "{0} {1}".format(author_element[0].text, author)
if map_template in ("ANSI_D_LND.mxd", "ANSI_E_LND.mxd"):
coord_elements = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "x*")
coord_elements += arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "y*")
if coord_elements:
for e in coord_elements:
new_text = e.text
if e.name == "xmin":
dms = task_utils.dd_to_dms(data_frame.extent.XMin)
if data_frame.extent.XMin > 0:
new_text = new_text.replace("W", "E")
elif e.name == "xmax":
dms = task_utils.dd_to_dms(data_frame.extent.XMax)
if data_frame.extent.XMax > 0:
new_text = new_text.replace("W", "E")
elif e.name == "ymin":
dms = task_utils.dd_to_dms(data_frame.extent.YMin)
if data_frame.extent.YMin < 0:
new_text = new_text.replace("N", "S")
elif e.name == "ymax":
if data_frame.extent.YMax < 0:
new_text = new_text.replace("N", "S")
dms = task_utils.dd_to_dms(data_frame.extent.YMax)
new_text = new_text.replace("d", str(dms[0]))
new_text = new_text.replace("m", str(dms[1]))
new_text = new_text.replace("s", str(dms[2]))
e.text = new_text
# Do this now so it does not affect zoom level or extent.
if base_layer:
status_writer.send_status(_("Adding basemap {0}...").format(base_map))
arcpy.mapping.AddLayer(data_frame, base_layer, "BOTTOM")
if added_to_map > 0:
status_writer.send_status(_("Exporting to PDF..."))
arcpy.mapping.ExportToPDF(
mxd,
os.path.join(request["folder"], "{0}.pdf".format(output_file_name)),
layers_attributes=attribute_setting,
)
# Create a thumbnail size PNG of the mxd.
task_utils.make_thumbnail(mxd, os.path.join(request["folder"], "_thumb.png"), False)
else:
status_writer.send_state(status.STAT_FAILED, _("No results can be exported to PDF"))
task_utils.report(
os.path.join(request["folder"], "__report.json"), added_to_map, skipped, skipped_details=skipped_reasons
)
return
# Update state if necessary.
if skipped > 0 or errors > 0:
status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(skipped + errors))
task_utils.report(
os.path.join(request["folder"], "__report.json"), added_to_map, skipped, errors, errors_reasons, skipped_reasons
)
def execute(request):
"""Deletes files.
:param request: json as a dict
"""
errors_reasons = {}
errors = 0
published = 0
app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
parameters = request['params']
num_results, response_index = task_utils.get_result_count(parameters)
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
if num_results > task_utils.CHUNK_SIZE:
status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.')
return
server_conn = task_utils.get_parameter_value(parameters, 'server_connection_path', 'value')
service_name = task_utils.get_parameter_value(parameters, 'service_name', 'value')
folder_name = task_utils.get_parameter_value(parameters, 'folder_name', 'value')
if not server_conn:
status_writer.send_state(status.STAT_FAILED, _('A server path is required'))
return
request_folder = os.path.join(request['folder'], 'temp')
if not os.path.exists(request_folder):
os.makedirs(request_folder)
map_template = os.path.join(request_folder, 'output.mxd')
shutil.copyfile(os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'), map_template)
for item in input_items:
try:
# Code required because of an Esri bug - cannot describe a map package (raises IOError).
if item.endswith('.mpk'):
status_writer.send_status(_('Extracting: {0}').format(item))
arcpy.ExtractPackage_management(item, request_folder)
pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
mxd_file = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.mxd')[0])
mxd = arcpy.mapping.MapDocument(mxd_file)
create_service(request_folder, mxd, server_conn, service_name, folder_name)
else:
data_type = arcpy.Describe(item).dataType
if data_type == 'MapDocument':
mxd = arcpy.mapping.MapDocument(item)
create_service(request_folder, mxd, server_conn, service_name, folder_name)
elif data_type == 'Layer':
if item.endswith('.lpk'):
status_writer.send_status(_('Extracting: {0}').format(item))
arcpy.ExtractPackage_management(item, request_folder)
pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
item = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.lyr')[0])
layer = arcpy.mapping.Layer(item)
mxd = arcpy.mapping.MapDocument(map_template)
mxd.description = layer.name
mxd.tags = layer.name
mxd.save()
data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
arcpy.mapping.AddLayer(data_frame, layer)
mxd.save()
create_service(request_folder, mxd, server_conn, service_name, folder_name)
elif data_type in ('FeatureClass', 'ShapeFile', 'RasterDataset'):
if data_type == 'RasterDataset':
arcpy.MakeRasterLayer_management(item, os.path.basename(item))
else:
arcpy.MakeFeatureLayer_management(item, os.path.basename(item))
layer = arcpy.mapping.Layer(os.path.basename(item))
mxd = arcpy.mapping.MapDocument(map_template)
mxd.title = layer.name
data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
arcpy.mapping.AddLayer(data_frame, layer)
mxd.save()
create_service(request_folder, mxd, server_conn, service_name, folder_name)
elif data_type in ('CadDrawingDataset', 'FeatureDataset'):
arcpy.env.workspace = item
mxd = arcpy.mapping.MapDocument(map_template)
data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
for fc in arcpy.ListFeatureClasses():
dataset_name = os.path.splitext(os.path.basename(item))[0]
l = arcpy.MakeFeatureLayer_management(fc, '{0}_{1}'.format(dataset_name, os.path.basename(fc)))
arcpy.mapping.AddLayer(data_frame, l.getOutput(0))
mxd.save()
arcpy.ResetEnvironments()
create_service(request_folder, mxd, server_conn, service_name, folder_name)
published += 1
except task_utils.AnalyzeServiceException as ex:
errors += 1
errors_reasons[item] = repr(ex)
status_writer.send_state(status.STAT_FAILED)
except arcpy.ExecuteError as ex:
errors += 1
errors_reasons[item] = repr(ex)
status_writer.send_state(status.STAT_FAILED)
except Exception as ex:
errors += 1
errors_reasons[item] = repr(ex)
status_writer.send_state(status.STAT_FAILED)
finally:
if errors:
errors_reasons[item] = repr(ex)
status_writer.send_status(_('FAIL: {0}').format(errors_reasons[item]))
task_utils.report(os.path.join(request['folder'], '__report.json'), published, 0, errors, errors_reasons)
def execute(request):
"""Clips selected search results using the clip geometry.
:param request: json as a dict.
"""
clipped = 0
errors = 0
skipped = 0
global result_count
parameters = request['params']
# Retrieve the clip features.
clip_features = task_utils.get_parameter_value(parameters, 'clip_features', 'value')
# Retrieve the coordinate system code.
out_coordinate_system = int(task_utils.get_parameter_value(parameters, 'output_projection', 'code'))
# Retrieve the output format, create mxd and output file name parameter values.
out_format = task_utils.get_parameter_value(parameters, 'output_format', 'value')
create_mxd = task_utils.get_parameter_value(parameters, 'create_mxd', 'value')
output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
if not output_file_name:
output_file_name = 'clip_results'
# Create the temporary workspace if clip_feature_class:
out_workspace = os.path.join(request['folder'], 'temp')
if not os.path.exists(out_workspace):
os.makedirs(out_workspace)
# Set the output coordinate system.
if not out_coordinate_system == 0: # Same as Input
out_sr = task_utils.get_spatial_reference(out_coordinate_system)
arcpy.env.outputCoordinateSystem = out_sr
# Set the output workspace.
status_writer.send_status(_('Setting the output workspace...'))
if not out_format == 'SHP':
out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0)
arcpy.env.workspace = out_workspace
# Query the index for results in groups of 25.
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
result_count, response_index = task_utils.get_result_count(parameters)
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
# Get the Clip features by id.
id = clip_features['id']
clip_query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', "&fl=id,path,fullpath:[absolute],absolute_path:[absolute],[lyrFile],[geo]&q=id:{0}".format(id))
clip_result = requests.get(clip_query, headers=headers)
clipper = clip_result.json()['response']['docs'][0]
if 'absolute_path' in clipper and not clipper['absolute_path'].startswith('s3'):
clip_features = clipper['absolute_path']
elif '[lyrFile]' in clipper:
clip_features = clipper['[lyrFile]']
elif '[geo]' in clipper:
clip_features = arcpy.AsShape(clipper['[geo]']).projectAs(arcpy.SpatialReference(4326))
elif 'absolute_path' in clipper and clipper['absolute_path'].startswith('s3'):
base_name = os.path.basename(clipper['path'])
temp_folder = tempfile.mkdtemp()
if '[downloadURL]' in clipper:
download = os.path.join(temp_folder, os.path.basename(clipper['[downloadURL]']))
response = requests.get(clipper['[downloadURL]'])
with open(download, 'wb') as fp:
fp.write(response.content)
if download.endswith('.zip'):
zip = zipfile.ZipFile(download)
zip.extractall(temp_folder)
clip_features = os.path.join(temp_folder, base_name)
else:
clip_features = download
else:
bbox = clipper['bbox'].split()
extent = arcpy.Extent(*bbox)
pt_array = arcpy.Array([extent.lowerLeft, extent.upperLeft, extent.upperRight, extent.lowerRight])
clip_features = arcpy.Polygon(pt_array, 4326)
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'), 'clip_data')
for group in groups:
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
docs = results.json()['response']['docs']
input_items = task_utils.get_input_items(docs)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
input_rows = collections.defaultdict(list)
for doc in docs:
if 'path' not in doc:
input_rows[doc['name']].append(doc)
if input_rows:
result = clip_data(input_rows, out_workspace, clip_features, out_format)
clipped += result[0]
errors += result[1]
skipped += result[2]
if input_items:
result = clip_data(input_items, out_workspace, clip_features, out_format)
clipped += result[0]
errors += result[1]
skipped += result[2]
if not input_items and not input_rows:
status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.'))
return
if arcpy.env.workspace.endswith('.gdb'):
out_workspace = os.path.dirname(arcpy.env.workspace)
if clipped > 0:
try:
if out_format == 'MPK':
mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
mxd = task_utils.create_mxd(out_workspace, mxd_template, 'output')
status_writer.send_status(_("Packaging results..."))
task_utils.create_mpk(out_workspace, mxd, files_to_package)
shutil.move(os.path.join(out_workspace, 'output.mpk'),
os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name)))
elif out_format == 'LPK':
status_writer.send_status(_("Packaging results..."))
task_utils.create_lpk(out_workspace, output_file_name, files_to_package)
elif out_format == 'KML':
task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb"))
arcpy.env.workspace = ''
arcpy.RefreshCatalog(os.path.join(out_workspace, "output.gdb"))
try:
arcpy.Delete_management(os.path.join(out_workspace, "output.gdb"))
except arcpy.ExecuteError:
pass
zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
else:
if create_mxd:
mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
task_utils.create_mxd(out_workspace, mxd_template, 'output')
zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
except arcpy.ExecuteError as ee:
status_writer.send_state(status.STAT_FAILED, _(ee))
sys.exit(1)
else:
status_writer.send_state(status.STAT_FAILED, _('No output created. Zero inputs were clipped.'))
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), clipped, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Remove tags.
:param request: json as a dict.
"""
query = ''
errors = 0
parameters = request['params']
archive_location = request['folder']
if not os.path.exists(archive_location):
os.makedirs(archive_location)
# Parameter values
search_action = task_utils.get_parameter_value(parameters, 'search_action',
'value')
search_name = task_utils.get_parameter_value(parameters, 'saved_searches',
'value')
search_name = eval(search_name[0])['text']
groups = task_utils.get_parameter_value(parameters, 'groups', 'value')
request_owner = request['owner']
result_count, response_index = task_utils.get_result_count(parameters)
fq = '/'
if 'fq' in parameters[response_index]['query']:
if isinstance(parameters[response_index]['query']['fq'], list):
for q in parameters[response_index]['query']['fq']:
if '{!tag=' in q:
q = q.split('}')[1]
if ':' in q:
facet = q.split(':')[0]
value = q.split(':')[1]
if '(' in value:
value = value.replace('(', '').replace(')', '')
value = urllib.urlencode({'val': value.replace('"', '')})
value = value.split('val=')[1]
facet2 = 'f.{0}='.format(facet)
q = '{0}{1}'.format(facet2,
value) #q.replace(facet + ':', facet2)
fq += '{0}/'.format(q).replace('"', '')
else:
# Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
fq += '&fq={0}'.format(
parameters[response_index]['query']['fq'].replace("\\", ""))
if '{!tag=' in fq:
fq = fq.split('}')[1]
if ':' in fq:
if fq.startswith('/&fq='):
fq = fq.replace('/&fq=', '')
facet = fq.split(':')[0]
value = fq.split(':')[1].replace('(', '').replace(')',
'').replace(
'"', '')
if 'place' not in facet:
value = urllib.urlencode({'val': value}).split('val=')[1]
facet2 = 'f.{0}='.format(facet)
if '(' in value:
fq = ''
if value.split(' '):
for v in value.split(' '):
fq += (facet2 +
v.replace('(', '').replace(')', '') +
'/').replace(':', '')
else:
value = urllib.urlencode({'val': value}).split('val=')[1]
fq = '{0}{1}'.format(facet2, value)
if '{! place.op=' in fq:
relop = find_between(fq, 'place.op=', '}')
fq = fq.replace('}', '').replace('{', '')
fq = fq.replace('! place.op={0}'.format(relop),
'/place.op={0}/'.format(relop))
fq = fq.replace('place:', 'place=')
fq = fq.replace('&fq=', '')
hasQ = False
if 'q' in parameters[response_index]['query']:
query = parameters[response_index]['query']['q']
hasQ = True
if fq:
query += '/'
if fq:
if fq.startswith('/place'):
query += fq.replace('"', '')
elif '!tag' in query and 'OR' in query:
# e.g. "path": "/q=id:(92cdd06e01761c4c d9841b2f59b8a326) OR format:(application%2Fvnd.esri.shapefile)"
q = query.split('}')[1].replace('))/',
'').replace('(',
'').replace('(', '')
q = urllib.urlencode({'val': q.split(':')[1]}).split('val=')[1]
query = query.split(' OR ')[0] + ' OR ' + q
else:
if fq.startswith('f.//'):
fq = fq.replace('f.//', '/').replace('"', '')
if ' place.id' in fq:
fq = fq.replace(' place.id', '/place.id').replace('"', '')
if '{! place.op=' in fq:
relop = find_between(fq, 'place.op=', '}')
fq = fq.replace('}', '').replace('{', '')
fq = fq.replace('! place.op={0}'.format(relop),
'/place.op={0}/'.format(relop)).replace(
'"', '')
query += fq.rstrip('/')
query = query.replace('f./', '')
query = query.replace('&fq=', '')
if search_action == 'Overwrite an existing saved search':
delete_result = delete_saved_search(search_name, request_owner)
if not delete_result[0]:
status_writer.send_state(status.STAT_FAILED, delete_result[1])
return
if query:
result = create_saved_search(search_name, groups, request_owner, query,
hasQ)
else:
result = create_saved_search(search_name, groups, request_owner, "",
hasQ)
if not result[0]:
errors += 1
errors_reasons[search_name] = result[1]
# Update state if necessary.
if errors > 0:
status_writer.send_state(status.STAT_FAILED, result[1])
else:
status_writer.send_status(result[1])
task_utils.report(os.path.join(request['folder'], '__report.json'),
1,
0,
errors,
errors_details=errors_reasons)
def execute(request):
"""Copies data to an existing geodatabase or feature dataset.
:param request: json as a dict.
"""
added = 0
errors = 0
skipped = 0
global result_count
parameters = request["params"]
# Get the target workspace location.
out_gdb = task_utils.get_parameter_value(parameters, "target_workspace", "value")
# Retrieve the coordinate system code.
out_coordinate_system = task_utils.get_parameter_value(parameters, "output_projection", "code")
if not out_coordinate_system == "0": # Same as Input
arcpy.env.outputCoordinateSystem = task_utils.get_spatial_reference(out_coordinate_system)
task_folder = request["folder"]
if not os.path.exists(task_folder):
os.makedirs(task_folder)
# Check if the geodatabase exists or if it is a feature dataset.
is_fds = False
if not os.path.exists(out_gdb):
if out_gdb.endswith(".gdb"):
arcpy.CreateFileGDB_management(os.path.dirname(out_gdb), os.path.basename(out_gdb))
status_writer.send_status(_("Created output workspace: {0}").format(out_gdb))
elif out_gdb.endswith(".mdb"):
arcpy.CreatePersonalGDB_management(os.path.dirname(out_gdb), os.path.basename(out_gdb))
status_writer.send_status(_("Created output workspace: {0}").format(out_gdb))
elif out_gdb.endswith(".sde"):
status_writer.send_state(status.STAT_FAILED, _("{0} does not exist").format(out_gdb))
return
else:
# Possible feature dataset.
is_fds = is_feature_dataset(out_gdb)
if not is_fds:
if os.path.dirname(out_gdb).endswith(".gdb"):
if not os.path.exists(os.path.dirname(out_gdb)):
arcpy.CreateFileGDB_management(
os.path.dirname(os.path.dirname(out_gdb)), os.path.basename(os.path.dirname(out_gdb))
)
arcpy.CreateFeatureDataset_management(os.path.dirname(out_gdb), os.path.basename(out_gdb))
elif os.path.dirname(out_gdb).endswith(".mdb"):
if not os.path.exists(os.path.dirname(out_gdb)):
arcpy.CreatePersonalGDB_management(
os.path.dirname(os.path.dirname(out_gdb)), os.path.basename(os.path.dirname(out_gdb))
)
arcpy.CreateFeatureDataset_management(os.path.dirname(out_gdb), os.path.basename(out_gdb))
status_writer.send_status(_("Setting the output workspace..."))
arcpy.env.workspace = out_gdb
headers = {"x-access-token": task_utils.get_security_token(request["owner"])}
result_count, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = "{0}{1}{2}".format(sys.argv[2].split("=")[1], "/select?&wt=json", fl)
# query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")
query += fq
elif "ids" in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]["ids"]), task_utils.CHUNK_SIZE, "")
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")
status_writer.send_percent(0.0, _("Starting to process..."), "add_to_geodatabase")
for group in groups:
if fq:
results = requests.get(
query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
)
elif "ids" in parameters[response_index]:
results = requests.get(query + "{0}&ids={1}".format(fl, ",".join(group)), headers=headers)
else:
results = requests.get(
query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
)
docs = results.json()["response"]["docs"]
input_items = task_utils.get_input_items(docs)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]["response"]["docs"])
input_rows = collections.defaultdict(list)
for doc in docs:
if "path" not in doc:
input_rows[doc["title"]].append(doc)
if input_rows:
result = add_to_geodatabase(input_rows, out_gdb, is_fds)
added += result[0]
errors += result[1]
skipped += result[2]
if input_items:
result = add_to_geodatabase(input_items, out_gdb, is_fds)
added += result[0]
errors += result[1]
skipped += result[2]
if not input_items and not input_rows:
status_writer.send_state(status.STAT_FAILED, _("No items to process. Check if items exist."))
return
# Update state if necessary.
if skipped > 0 or errors > 0:
status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(skipped + errors))
task_utils.report(
os.path.join(task_folder, "__report.json"), added, skipped, errors, errors_reasons, skipped_reasons
)
def execute(request):
"""Clips selected search results using the clip geometry.
:param request: json as a dict.
"""
clipped = 0
errors = 0
skipped = 0
global result_count
parameters = request['params']
# Retrieve clip geometry.
try:
clip_area = task_utils.get_parameter_value(parameters, 'clip_geometry', 'wkt')
if not clip_area:
clip_area = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'
except KeyError:
clip_area = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'
# Retrieve the coordinate system code.
out_coordinate_system = int(task_utils.get_parameter_value(parameters, 'output_projection', 'code'))
# Retrieve the output format, create mxd parameter and output file name values.
out_format = task_utils.get_parameter_value(parameters, 'output_format', 'value')
create_mxd = task_utils.get_parameter_value(parameters, 'create_mxd', 'value')
output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
if not output_file_name:
output_file_name = 'clip_results'
# Create the temporary workspace if clip_feature_class:
out_workspace = os.path.join(request['folder'], 'temp')
if not os.path.exists(out_workspace):
os.makedirs(out_workspace)
# Set the output coordinate system.
if not out_coordinate_system == 0: # Same as Input
out_sr = task_utils.get_spatial_reference(out_coordinate_system)
arcpy.env.outputCoordinateSystem = out_sr
# Create the clip polygon geometry object in WGS84 projection.
gcs_sr = task_utils.get_spatial_reference(4326)
gcs_clip_poly = task_utils.from_wkt(clip_area, gcs_sr)
if not gcs_clip_poly.area > 0:
gcs_clip_poly = task_utils.from_wkt('POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))', gcs_sr)
# Set the output workspace.
status_writer.send_status(_('Setting the output workspace...'))
if not out_format == 'SHP':
out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0)
arcpy.env.workspace = out_workspace
# Query the index for results in groups of 25.
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
result_count, response_index = task_utils.get_result_count(parameters)
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'), 'clip_data')
for group in groups:
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
# docs = eval(results.read().replace('false', 'False').replace('true', 'True').replace('null', 'None'))['response']['docs']
docs = results.json()['response']['docs']
input_items = task_utils.get_input_items(docs)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
input_rows = collections.defaultdict(list)
for doc in docs:
if 'path' not in doc:
input_rows[doc['name']].append(doc)
if input_rows:
result = clip_data(input_rows, out_workspace, out_coordinate_system, gcs_sr, gcs_clip_poly, out_format)
clipped += result[0]
errors += result[1]
skipped += result[2]
if input_items:
result = clip_data(input_items, out_workspace, out_coordinate_system, gcs_sr, gcs_clip_poly, out_format)
clipped += result[0]
errors += result[1]
skipped += result[2]
if not input_items and not input_rows:
status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.'))
return
if arcpy.env.workspace.endswith('.gdb'):
out_workspace = os.path.dirname(arcpy.env.workspace)
if clipped > 0:
try:
if out_format == 'MPK':
mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
mxd = task_utils.create_mxd(out_workspace, mxd_template, 'output')
status_writer.send_status(_("Packaging results..."))
task_utils.create_mpk(out_workspace, mxd, files_to_package)
shutil.move(os.path.join(out_workspace, 'output.mpk'),
os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name)))
elif out_format == 'LPK':
status_writer.send_status(_("Packaging results..."))
task_utils.create_lpk(out_workspace,output_file_name, files_to_package)
elif out_format == 'KML':
task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb"))
arcpy.env.workspace = ''
try:
arcpy.Delete_management(os.path.join(out_workspace, "output.gdb"))
except arcpy.ExecuteError:
pass
zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
else:
if create_mxd:
mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
task_utils.create_mxd(out_workspace, mxd_template, 'output')
zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
except arcpy.ExecuteError as ee:
status_writer.send_state(status.STAT_FAILED, _(ee))
sys.exit(1)
else:
status_writer.send_state(status.STAT_FAILED, _('No output created. Zero inputs were clipped.'))
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), clipped, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Creates a GeoPDF.
:param request: json as a dict.
"""
added_to_map = 0
errors = 0
skipped = 0
parameters = request['params']
num_results, response_index = task_utils.get_result_count(parameters)
docs = parameters[response_index]['response']['docs']
input_items = task_utils.get_input_items(docs)
input_rows = collections.defaultdict(list)
for doc in docs:
if 'path' not in doc:
input_rows[doc['name']].append(doc)
if num_results > task_utils.CHUNK_SIZE:
status_writer.send_state(status.STAT_FAILED,
'Reduce results to 25 or less.')
return
map_template = task_utils.get_parameter_value(parameters, 'map_template',
'value')
base_map = task_utils.get_parameter_value(parameters, 'base_map', 'value')
map_title = task_utils.get_parameter_value(parameters, 'map_title',
'value')
attribute_setting = task_utils.get_parameter_value(parameters,
'attribute_settings',
'value')
author = task_utils.get_parameter_value(parameters, 'map_author', 'value')
output_file_name = task_utils.get_parameter_value(parameters,
'output_file_name',
'value')
if not output_file_name:
output_file_name = 'output_pdf'
try:
map_view = task_utils.get_parameter_value(parameters, 'map_view',
'extent')
except KeyError:
map_view = None
pass
temp_folder = os.path.join(request['folder'], 'temp')
if not os.path.exists(temp_folder):
os.makedirs(temp_folder)
if base_map == 'NONE':
base_layer = None
else:
base_layer = arcpy.mapping.Layer(
os.path.join(os.path.dirname(os.path.dirname(__file__)),
'supportfiles', 'basemaps',
'{0}.lyr'.format(base_map)))
mxd_path = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'supportfiles', 'frame', map_template)
mxd = arcpy.mapping.MapDocument(mxd_path)
data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
layers = []
all_layers = []
if input_rows:
for name, rows in input_rows.iteritems():
for row in rows:
try:
name = arcpy.CreateUniqueName(name, 'in_memory')
# Create the geometry.
geo_json = row['[geo]']
geom = arcpy.AsShape(geo_json)
arcpy.CopyFeatures_management(geom, name)
feature_layer = arcpy.MakeFeatureLayer_management(
name, os.path.basename(name))
layer_file = arcpy.SaveToLayerFile_management(
feature_layer,
os.path.join(temp_folder,
'{0}.lyr'.format(os.path.basename(name))))
layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
all_layers.append(
arcpy.mapping.Layer(layer_file.getOutput(0)))
added_to_map += 1
except KeyError:
skipped += 1
skipped_reasons[name] = 'No geographic information'
continue
for i, item in enumerate(input_items, 1):
try:
# Is the item a mxd data frame.
map_frame_name = task_utils.get_data_frame_name(item)
if map_frame_name:
item = item.split('|')[0].strip()
dsc = arcpy.Describe(item)
if dsc.dataType == 'Layer':
layers.append(arcpy.mapping.Layer(dsc.catalogPath))
elif dsc.dataType == 'FeatureClass' or dsc.dataType == 'ShapeFile':
if os.path.basename(item) in [l.name for l in all_layers]:
layer_name = '{0}_{1}'.format(os.path.basename(item), i)
else:
layer_name = os.path.basename(item)
feature_layer = arcpy.MakeFeatureLayer_management(
item, layer_name)
layer_file = arcpy.SaveToLayerFile_management(
feature_layer,
os.path.join(temp_folder, '{0}.lyr'.format(layer_name)))
layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
elif dsc.dataType == 'FeatureDataset':
arcpy.env.workspace = item
for fc in arcpy.ListFeatureClasses():
layer_file = arcpy.SaveToLayerFile_management(
arcpy.MakeFeatureLayer_management(
fc, '{0}_{1}'.format(fc, i)),
os.path.join(temp_folder, '{0}_{1}.lyr'.format(fc, i)))
layer = arcpy.mapping.Layer(layer_file.getOutput(0))
layer.name = fc
layers.append(layer)
all_layers.append(layer)
elif dsc.dataType == 'RasterDataset':
if os.path.basename(item) in [l.name for l in all_layers]:
layer_name = '{0}_{1}'.format(os.path.basename(item), i)
else:
layer_name = os.path.basename(item)
raster_layer = arcpy.MakeRasterLayer_management(
item, layer_name)
layer_file = arcpy.SaveToLayerFile_management(
raster_layer,
os.path.join(temp_folder, '{0}.lyr'.format(layer_name)))
layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
elif dsc.catalogPath.endswith('.kml') or dsc.catalogPath.endswith(
'.kmz'):
if not os.path.splitext(dsc.name)[0] in layers:
name = os.path.splitext(dsc.name)[0]
else:
name = '{0}_{1}'.format(os.path.splitext(dsc.name)[0], i)
arcpy.KMLToLayer_conversion(dsc.catalogPath, temp_folder, name)
layers.append(
arcpy.mapping.Layer(
os.path.join(temp_folder, '{0}.lyr'.format(name))))
all_layers.append(
arcpy.mapping.Layer(
os.path.join(temp_folder, '{0}.lyr'.format(name))))
elif dsc.dataType == 'MapDocument':
input_mxd = arcpy.mapping.MapDocument(item)
if map_frame_name:
df = arcpy.mapping.ListDataFrames(input_mxd,
map_frame_name)[0]
layers = arcpy.mapping.ListLayers(input_mxd, data_frame=df)
else:
layers = arcpy.mapping.ListLayers(input_mxd)
if layers:
for layer in layers:
status_writer.send_status(
_('Adding layer {0}...').format(layer.name))
arcpy.mapping.AddLayer(data_frame, layer)
added_to_map += 1
layers = []
else:
status_writer.send_status(
_('Invalid input type: {0}').format(item))
skipped_reasons[item] = 'Invalid input type'
skipped += 1
except Exception as ex:
status_writer.send_status(_('FAIL: {0}').format(repr(ex)))
errors += 1
errors_reasons[item] = repr(ex)
pass
if map_view:
extent = map_view.split(' ')
new_extent = data_frame.extent
new_extent.XMin, new_extent.YMin = float(extent[0]), float(extent[1])
new_extent.XMax, new_extent.YMax = float(extent[2]), float(extent[3])
data_frame.extent = new_extent
else:
data_frame.zoomToSelectedFeatures()
# Update text elements in map template.
date_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT',
'date')
if date_element:
date_element[0].text = 'Date: {0}'.format(task_utils.get_local_date())
title_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT',
'title')
if title_element:
title_element[0].text = map_title
author_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT',
'author')
if author_element:
author_element[0].text = '{0} {1}'.format(author_element[0].text,
author)
if map_template in ('ANSI_D_LND.mxd', 'ANSI_E_LND.mxd'):
coord_elements = arcpy.mapping.ListLayoutElements(
mxd, 'TEXT_ELEMENT', 'x*')
coord_elements += arcpy.mapping.ListLayoutElements(
mxd, 'TEXT_ELEMENT', 'y*')
if coord_elements:
for e in coord_elements:
new_text = e.text
if e.name == 'xmin':
dms = task_utils.dd_to_dms(data_frame.extent.XMin)
if data_frame.extent.XMin > 0:
new_text = new_text.replace('W', 'E')
elif e.name == 'xmax':
dms = task_utils.dd_to_dms(data_frame.extent.XMax)
if data_frame.extent.XMax > 0:
new_text = new_text.replace('W', 'E')
elif e.name == 'ymin':
dms = task_utils.dd_to_dms(data_frame.extent.YMin)
if data_frame.extent.YMin < 0:
new_text = new_text.replace('N', 'S')
elif e.name == 'ymax':
if data_frame.extent.YMax < 0:
new_text = new_text.replace('N', 'S')
dms = task_utils.dd_to_dms(data_frame.extent.YMax)
new_text = new_text.replace('d', str(dms[0]))
new_text = new_text.replace('m', str(dms[1]))
new_text = new_text.replace('s', str(dms[2]))
e.text = new_text
# Do this now so it does not affect zoom level or extent.
if base_layer:
status_writer.send_status(_('Adding basemap {0}...').format(base_map))
arcpy.mapping.AddLayer(data_frame, base_layer, 'BOTTOM')
if added_to_map > 0:
status_writer.send_status(_('Exporting to PDF...'))
arcpy.mapping.ExportToPDF(mxd,
os.path.join(
request['folder'],
'{0}.pdf'.format(output_file_name)),
layers_attributes=attribute_setting)
# Create a thumbnail size PNG of the mxd.
task_utils.make_thumbnail(
mxd, os.path.join(request['folder'], '_thumb.png'), False)
else:
status_writer.send_state(status.STAT_FAILED,
_('No results can be exported to PDF'))
task_utils.report(os.path.join(request['folder'], '__report.json'),
added_to_map,
skipped,
skipped_details=skipped_reasons)
return
# Update state if necessary.
if skipped > 0 or errors > 0:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(request['folder'],
'__report.json'), added_to_map, skipped,
errors, errors_reasons, skipped_reasons)
def execute(request):
"""Builds raster pyramids for input raster datasets.
:param request: json as a dict.
"""
processed = 0
skipped = 0
parameters = request['params']
resampling_method = task_utils.get_parameter_value(parameters, 'resampling_method', 'value')
# Advanced options
compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value')
compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value')
# Create the task folder to hold report files.
task_folder = request['folder']
if not os.path.exists(task_folder):
os.makedirs(task_folder)
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
num_results, response_index = task_utils.get_result_count(parameters)
if num_results > task_utils.CHUNK_SIZE:
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'), 'build_raster_pyramids')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)
input_items = task_utils.get_input_items(results.json()['response']['docs'])
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
result = build_pyramids(input_items, compression_method, compression_quality, resampling_method)
processed += result[0]
skipped += result[1]
status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'build_raster_pyramids')
else:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
processed, skipped = build_pyramids(input_items, compression_method, compression_quality, resampling_method, True)
# Update state if necessary.
if skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), processed, skipped, skipped_details=skipped_reasons)
def execute(request):
"""Builds raster pyramids for input raster datasets.
:param request: json as a dict.
"""
processed = 0
skipped = 0
parameters = request['params']
# Get the extent for for which to use to calculate statistics.
extent = ''
try:
try:
ext = task_utils.get_parameter_value(parameters,
'processing_extent', 'wkt')
if ext:
sr = task_utils.get_spatial_reference("4326")
extent = task_utils.from_wkt(ext, sr)
except KeyError:
ext = task_utils.get_parameter_value(parameters,
'processing_extent',
'feature')
if ext:
extent = arcpy.Describe(ext).extent
except KeyError:
pass
horizontal_skip_factor = task_utils.get_parameter_value(
parameters, 'horizontal_skip_factor', 'value')
vertical_skip_factor = task_utils.get_parameter_value(
parameters, 'vertical_skip_factor', 'value')
ignore_pixel_values = task_utils.get_parameter_value(
parameters, 'ignore_pixel_values', 'value')
# Create the task folder to hold report files.
task_folder = request['folder']
if not os.path.exists(task_folder):
os.makedirs(task_folder)
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
num_results, response_index = task_utils.get_result_count(parameters)
if num_results > task_utils.CHUNK_SIZE:
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1],
'/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results),
task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(
list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE,
'')
else:
groups = task_utils.grouper(range(0, num_results),
task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'),
'calculate_raster_statistics')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(
task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(
query + '{0}&ids={1}'.format(fl, ','.join(group)),
verify=verify_ssl,
headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(
task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
input_items = task_utils.get_input_items(
results.json()['response']['docs'])
if not input_items:
input_items = task_utils.get_input_items(
parameters[response_index]['response']['docs'])
result = calculate_raster_statistics(input_items, extent,
horizontal_skip_factor,
vertical_skip_factor,
ignore_pixel_values)
processed += result[0]
skipped += result[1]
status_writer.send_percent(
i / num_results,
'{0}: {1:.0f}%'.format("Processed", i / num_results * 100),
'calculate_raster_statistics')
else:
input_items = task_utils.get_input_items(
parameters[response_index]['response']['docs'])
processed, skipped = calculate_raster_statistics(
input_items, extent, horizontal_skip_factor, vertical_skip_factor,
ignore_pixel_values, True)
# Update state if necessary.
if skipped > 0:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'),
processed,
skipped,
skipped_details=skipped_reasons)
def execute(request):
"""Deletes files.
:param request: json as a dict
"""
errors_reasons = {}
errors = 0
published = 0
app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
parameters = request['params']
num_results, response_index = task_utils.get_result_count(parameters)
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
if num_results > task_utils.CHUNK_SIZE:
status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.')
return
url = task_utils.get_parameter_value(parameters, 'url', 'value')
username = task_utils.get_parameter_value(parameters, 'username', 'value')
password = task_utils.get_parameter_value(parameters, 'password', 'value')
service_name = task_utils.get_parameter_value(parameters, 'service_name', 'value')
folder_name = task_utils.get_parameter_value(parameters, 'folder_name', 'value')
request_folder = os.path.join(request['folder'], 'temp')
if not os.path.exists(request_folder):
os.makedirs(request_folder)
map_template = os.path.join(request_folder, 'output.mxd')
shutil.copyfile(os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'), map_template)
for item in input_items:
try:
# Code required because of an Esri bug - cannot describe a map package (raises IOError).
if item.endswith('.mpk'):
status_writer.send_status(_('Extracting: {0}').format(item))
arcpy.ExtractPackage_management(item, request_folder)
pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
mxd_file = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.mxd')[0])
mxd = arcpy.mapping.MapDocument(mxd_file)
create_service(request_folder, mxd, url, username, password, service_name, folder_name)
else:
data_type = arcpy.Describe(item).dataType
if data_type == 'MapDocument':
mxd = arcpy.mapping.MapDocument(item)
create_service(request_folder, mxd, url, username, password, service_name, folder_name)
elif data_type == 'Layer':
if item.endswith('.lpk'):
status_writer.send_status(_('Extracting: {0}').format(item))
arcpy.ExtractPackage_management(item, request_folder)
pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
item = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.lyr')[0])
layer = arcpy.mapping.Layer(item)
mxd = arcpy.mapping.MapDocument(map_template)
mxd.description = layer.name
mxd.tags = layer.name
mxd.save()
data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
arcpy.mapping.AddLayer(data_frame, layer)
mxd.save()
create_service(request_folder, mxd, url, username, password, service_name, folder_name)
elif data_type in ('FeatureClass', 'ShapeFile', 'RasterDataset'):
if data_type == 'RasterDataset':
arcpy.MakeRasterLayer_management(item, os.path.basename(item))
else:
arcpy.MakeFeatureLayer_management(item, os.path.basename(item))
layer = arcpy.mapping.Layer(os.path.basename(item))
mxd = arcpy.mapping.MapDocument(map_template)
mxd.description = layer.name
mxd.tags = layer.name
mxd.save()
data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
arcpy.mapping.AddLayer(data_frame, layer)
mxd.save()
create_service(request_folder, mxd, url, username, password, service_name, folder_name)
published += 1
except task_utils.AnalyzeServiceException as ase:
status_writer.send_state(status.STAT_FAILED, _(ase))
errors_reasons[item] = repr(ase)
errors += 1
except requests.RequestException as re:
status_writer.send_state(status.STAT_FAILED, _(re))
errors_reasons[item] = repr(re)
errors += 1
except task_utils.PublishException as pe:
status_writer.send_state(status.STAT_FAILED, _(pe))
errors_reasons[item] = repr(pe)
errors += 1
except arcpy.ExecuteError as ee:
status_writer.send_state(status.STAT_FAILED, _(ee))
errors_reasons[item] = repr(ee)
errors += 1
except Exception as ex:
status_writer.send_state(status.STAT_FAILED, _(ex))
errors_reasons[item] = repr(ex)
errors += 1
finally:
task_utils.report(os.path.join(request['folder'], '__report.json'), published, 0, errors, errors_reasons)
def execute(request):
"""Deletes files.
:param request: json as a dict
"""
errors_reasons = {}
errors = 0
published = 0
app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
parameters = request['params']
num_results, response_index = task_utils.get_result_count(parameters)
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
if num_results > task_utils.CHUNK_SIZE:
status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.')
return
url = task_utils.get_parameter_value(parameters, 'url', 'value')
username = task_utils.get_parameter_value(parameters, 'username', 'value')
password = task_utils.get_parameter_value(parameters, 'password', 'value')
service_name = task_utils.get_parameter_value(parameters, 'service_name', 'value')
folder_name = task_utils.get_parameter_value(parameters, 'folder_name', 'value')
request_folder = os.path.join(request['folder'], 'temp')
if not os.path.exists(request_folder):
os.makedirs(request_folder)
map_template = os.path.join(request_folder, 'output.mxd')
shutil.copyfile(os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'), map_template)
for item in input_items:
try:
# Code required because of an Esri bug - cannot describe a map package (raises IOError).
if item.endswith('.mpk'):
status_writer.send_status(_('Extracting: {0}').format(item))
arcpy.ExtractPackage_management(item, request_folder)
pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
mxd_file = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.mxd')[0])
mxd = arcpy.mapping.MapDocument(mxd_file)
create_service(request_folder, mxd, url, username, password, service_name, folder_name)
else:
data_type = arcpy.Describe(item).dataType
if data_type == 'MapDocument':
mxd = arcpy.mapping.MapDocument(item)
create_service(request_folder, mxd, url, username, password, service_name, folder_name)
elif data_type == 'Layer':
if item.endswith('.lpk'):
status_writer.send_status(_('Extracting: {0}').format(item))
arcpy.ExtractPackage_management(item, request_folder)
pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
item = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.lyr')[0])
layer = arcpy.mapping.Layer(item)
mxd = arcpy.mapping.MapDocument(map_template)
mxd.description = layer.name
mxd.tags = layer.name
mxd.save()
data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
arcpy.mapping.AddLayer(data_frame, layer)
mxd.save()
create_service(request_folder, mxd, url, username, password, service_name, folder_name)
elif data_type in ('FeatureClass', 'ShapeFile', 'RasterDataset'):
if data_type == 'RasterDataset':
arcpy.MakeRasterLayer_management(item, os.path.basename(item))
else:
arcpy.MakeFeatureLayer_management(item, os.path.basename(item))
layer = arcpy.mapping.Layer(os.path.basename(item))
mxd = arcpy.mapping.MapDocument(map_template)
mxd.description = layer.name
mxd.tags = layer.name
mxd.save()
data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
arcpy.mapping.AddLayer(data_frame, layer)
mxd.save()
create_service(request_folder, mxd, url, username, password, service_name, folder_name)
published += 1
except task_utils.AnalyzeServiceException as ase:
status_writer.send_state(status.STAT_FAILED, _(ase))
errors_reasons[item] = repr(ase)
errors += 1
except requests.RequestException as re:
status_writer.send_state(status.STAT_FAILED, _(re))
errors_reasons[item] = repr(re)
errors += 1
except task_utils.PublishException as pe:
status_writer.send_state(status.STAT_FAILED, _(pe))
errors_reasons[item] = repr(pe)
errors += 1
except arcpy.ExecuteError as ee:
status_writer.send_state(status.STAT_FAILED, _(ee))
errors_reasons[item] = repr(ee)
errors += 1
except Exception as ex:
status_writer.send_state(status.STAT_FAILED, _(ex))
errors_reasons[item] = repr(ex)
errors += 1
finally:
task_utils.report(os.path.join(request['folder'], '__report.json'), published, 0, errors, errors_reasons)
def execute(request):
"""Mosaics input raster datasets into a new raster dataset or mosaic dataset.
:param request: json as a dict.
"""
status_writer = status.Writer()
parameters = request['params']
target_workspace = task_utils.get_parameter_value(parameters, 'target_workspace', 'value')
output_name = task_utils.get_parameter_value(parameters, 'output_dataset_name', 'value')
out_coordinate_system = task_utils.get_parameter_value(parameters, 'output_projection', 'code')
# Advanced options
output_raster_format = task_utils.get_parameter_value(parameters, 'raster_format', 'value')
compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value')
compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value')
arcpy.env.compression = '{0} {1}'.format(compression_method, compression_quality)
if output_raster_format in ('FileGDB', 'MosaicDataset'):
if not os.path.splitext(target_workspace)[1] in ('.gdb', '.mdb', '.sde'):
status_writer.send_state(status.STAT_FAILED, _('Target workspace must be a geodatabase'))
return
task_folder = request['folder']
if not os.path.exists(task_folder):
os.makedirs(task_folder)
clip_area = None
if not output_raster_format == 'MosaicDataset':
# Get the clip region as an extent object.
try:
clip_area_wkt = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt')
if not clip_area_wkt:
clip_area_wkt = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'
if not out_coordinate_system == '0':
clip_area = task_utils.get_clip_region(clip_area_wkt, out_coordinate_system)
else:
clip_area = task_utils.get_clip_region(clip_area_wkt)
except KeyError:
pass
status_writer.send_status(_('Setting the output workspace...'))
if not os.path.exists(target_workspace):
status_writer.send_state(status.STAT_FAILED, _('Target workspace does not exist'))
return
arcpy.env.workspace = target_workspace
status_writer.send_status(_('Starting to process...'))
num_results, response_index = task_utils.get_result_count(parameters)
raster_items = None
if num_results > task_utils.CHUNK_SIZE:
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
for group in groups:
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
input_items = task_utils.get_input_items(results.json()['response']['docs'])
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
raster_items, pixels, bands, skipped = get_items(input_items)
else:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
raster_items, pixels, bands, skipped = get_items(input_items)
if not raster_items:
if skipped == 0:
status_writer.send_state(status.STAT_FAILED, _('Invalid input types'))
skipped_reasons['All Items'] = _('Invalid input types')
task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), num_results, skipped_details=skipped_reasons)
return
else:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons)
return
# Get most common pixel type.
pixel_type = pixel_types[max(set(pixels), key=pixels.count)]
if output_raster_format in ('FileGDB', 'GRID', 'MosaicDataset'):
output_name = arcpy.ValidateTableName(output_name, target_workspace)
else:
output_name = '{0}.{1}'.format(arcpy.ValidateTableName(output_name, target_workspace), output_raster_format.lower())
if arcpy.Exists(os.path.join(target_workspace, output_name)):
status_writer.send_state(status.STAT_FAILED, _('Output dataset already exists.'))
return
if output_raster_format == 'MosaicDataset':
try:
status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters')))
if out_coordinate_system == '0':
out_coordinate_system = raster_items[0]
else:
out_coordinate_system = None
mosaic_ds = arcpy.CreateMosaicDataset_management(target_workspace,
output_name,
out_coordinate_system,
max(bands),
pixel_type)
arcpy.AddRastersToMosaicDataset_management(mosaic_ds, 'Raster Dataset', raster_items)
arcpy.MakeMosaicLayer_management(mosaic_ds, 'mosaic_layer')
layer_object = arcpy.mapping.Layer('mosaic_layer')
task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png'))
except arcpy.ExecuteError:
skipped += 1
skipped_reasons['All Items'] = arcpy.GetMessages(2)
else:
try:
if len(bands) > 1:
status_writer.send_state(status.STAT_FAILED, _('Input rasters must have the same number of bands'))
return
if out_coordinate_system == '0':
out_coordinate_system = None
status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters')))
if clip_area:
ext = '{0} {1} {2} {3}'.format(clip_area.XMin, clip_area.YMin, clip_area.XMax, clip_area.YMax)
tmp_mosaic = arcpy.MosaicToNewRaster_management(
raster_items,
target_workspace,
'tmpMosaic',
out_coordinate_system,
pixel_type,
number_of_bands=bands.keys()[0]
)
status_writer.send_status(_('Clipping...'))
out_mosaic = arcpy.Clip_management(tmp_mosaic, ext, output_name)
arcpy.Delete_management(tmp_mosaic)
else:
out_mosaic = arcpy.MosaicToNewRaster_management(raster_items,
target_workspace,
output_name,
out_coordinate_system,
pixel_type,
number_of_bands=bands.keys()[0],
mosaic_method='BLEND')
arcpy.MakeRasterLayer_management(out_mosaic, 'mosaic_layer')
layer_object = arcpy.mapping.Layer('mosaic_layer')
task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png'))
except arcpy.ExecuteError:
skipped += 1
skipped_reasons['All Items'] = arcpy.GetMessages(2)
# Update state if necessary.
if skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons)
def execute(request):
"""Replace the workspace path for layer files and map document layers.
:param request: json as a dict.
"""
updated = 0
skipped = 0
parameters = request['params']
backup = task_utils.get_parameter_value(parameters, 'create_backup',
'value')
old_data_source = task_utils.get_parameter_value(parameters,
'old_data_source',
'value').lower()
new_data_source = task_utils.get_parameter_value(parameters,
'new_data_source',
'value')
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
if not arcpy.Exists(new_data_source):
status_writer.send_state(
status.STAT_FAILED,
_('{0} does not exist').format(new_data_source))
return
if os.path.splitext(new_data_source)[1] not in ('.gdb', '.mdb', '.sde'):
new_dataset = os.path.basename(new_data_source)
dsc = arcpy.Describe(os.path.dirname(new_data_source))
else:
dsc = arcpy.Describe(new_data_source)
new_dataset = ''
wks_type = 'NONE'
if dsc.dataType == 'FeatureDataset':
new_workspace = dsc.path
wks_type = get_workspace_type(new_workspace)
elif dsc.dataType == 'Workspace':
new_workspace = dsc.catalogPath
wks_type = get_workspace_type(new_workspace)
elif dsc.dataType == 'Folder':
dsc = arcpy.Describe(new_data_source)
new_workspace = dsc.catalogPath
if new_dataset.endswith('.shp'):
wks_type = 'SHAPEFILE_WORKSPACE'
new_dataset = new_dataset.rsplit('.shp')[0]
else:
if arcpy.Describe(new_data_source).dataType == 'RasterDataset':
wks_type = 'RASTER_WORKSPACE'
elif dsc.dataType == 'CadDrawingDataset':
new_workspace = dsc.path
wks_type = 'CAD_WORKSPACE'
else:
new_workspace = os.path.dirname(new_data_source)
num_results, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}'.format(sys.argv[2].split('=')[1], '/select?&wt=json')
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results),
task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']),
task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, num_results),
task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'),
'replace_data_source')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "{0}&rows={1}&start={2}".format(
fl, task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query +
'{0}&ids={1}'.format(fl, ','.join(group)),
verify=verify_ssl,
headers=headers)
else:
results = requests.get(query + "{0}&rows={1}&start={2}".format(
fl, task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
input_items = task_utils.get_input_items(
results.json()['response']['docs'], True)
if not input_items:
input_items = task_utils.get_input_items(
parameters[response_index]['response']['docs'])
result = replace_data_source(input_items, old_data_source,
new_workspace, new_dataset, wks_type,
backup, headers)
updated += result[0]
skipped += result[1]
status_writer.send_percent(
i / num_results, '{0}: {1:.0f}%'.format("Processed",
i / num_results * 100),
'replace_data_source')
# Update state if necessary.
if skipped > 0:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'),
updated,
skipped,
skipped_details=skipped_reasons)
def execute(request):
"""Builds raster pyramids for input raster datasets.
:param request: json as a dict.
"""
processed = 0
skipped = 0
parameters = request['params']
resampling_method = task_utils.get_parameter_value(parameters, 'resampling_method', 'value')
# Advanced options
compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value')
compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value')
# Create the task folder to hold report files.
task_folder = request['folder']
if not os.path.exists(task_folder):
os.makedirs(task_folder)
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
num_results, response_index = task_utils.get_result_count(parameters)
if num_results > task_utils.CHUNK_SIZE:
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'), 'build_raster_pyramids')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
input_items = task_utils.get_input_items(results.json()['response']['docs'])
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
result = build_pyramids(input_items, compression_method, compression_quality, resampling_method)
processed += result[0]
skipped += result[1]
status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'build_raster_pyramids')
else:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
processed, skipped = build_pyramids(input_items, compression_method, compression_quality, resampling_method, True)
# Update state if necessary.
if skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), processed, skipped, skipped_details=skipped_reasons)
def execute(request):
"""Copies files to a target folder.
:param request: json as a dict.
"""
created = 0
skipped = 0
errors = 0
global result_count
parameters = request['params']
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
# meta_folder = task_utils.get_parameter_value(parameters, 'meta_data_folder', 'value')
voyager_server = sys.argv[2].split('=')[1].split('solr')[0][:-1]
url = "{0}/api/rest/system/settings".format(voyager_server)
response = requests.get(url, verify=verify_ssl, headers=headers)
meta_folder = response.json()['folders']['meta']
result_count, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']),
task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'),
'create_layer_files')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query +
'{0}&ids={1}'.format(fl, ','.join(group)),
verify=verify_ssl,
headers=headers)
else:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
docs = results.json()['response']['docs']
# docs = eval(results.read().replace('false', 'False').replace('true', 'True'))['response']['docs']
if not docs:
docs = parameters[response_index]['response']['docs']
input_items = []
for doc in docs:
if 'path' in doc:
input_items.append(
(doc['id'], doc['path'], doc['name'], doc['location']))
result = create_layer_file(input_items, meta_folder, voyager_server,
headers)
created += result[0]
errors += result[1]
skipped += result[2]
try:
shutil.copy2(
os.path.join(os.path.dirname(os.path.dirname(__file__)),
'supportfiles', '_thumb.png'), request['folder'])
except IOError:
pass
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(request['folder'],
'__report.json'), created, skipped, errors,
errors_reasons, skipped_reasons)
def execute(request):
"""Converts each input dataset to kml (.kmz).
:param request: json as a dict.
"""
converted = 0
skipped = 0
errors = 0
global result_count
parameters = request["params"]
out_workspace = os.path.join(request["folder"], "temp")
if not os.path.exists(out_workspace):
os.makedirs(out_workspace)
# Get the boundary box extent for input to KML tools.
extent = ""
try:
try:
ext = task_utils.get_parameter_value(parameters, "processing_extent", "wkt")
if ext:
sr = task_utils.get_spatial_reference("4326")
extent = task_utils.from_wkt(ext, sr)
except KeyError:
ext = task_utils.get_parameter_value(parameters, "processing_extent", "feature")
if ext:
extent = arcpy.Describe(ext).extent
except KeyError:
pass
# Get the output file name.
output_file_name = task_utils.get_parameter_value(parameters, "output_file_name", "value")
if not output_file_name:
output_file_name = "kml_results"
result_count, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = "{0}{1}{2}".format(sys.argv[2].split("=")[1], "/select?&wt=json", fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")
query += fq
elif "ids" in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]["ids"]), task_utils.CHUNK_SIZE, "")
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")
# Begin processing
status_writer.send_percent(0.0, _("Starting to process..."), "convert_to_kml")
headers = {"x-access-token": task_utils.get_security_token(request["owner"])}
for group in groups:
if fq:
results = requests.get(
query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
)
elif "ids" in parameters[response_index]:
results = requests.get(query + "{0}&ids={1}".format(fl, ",".join(group)), headers=headers)
else:
results = requests.get(
query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
)
docs = results.json()["response"]["docs"]
input_items = task_utils.get_input_items(docs)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]["response"]["docs"])
input_rows = collections.defaultdict(list)
for doc in docs:
if "path" not in doc:
input_rows[doc["name"]].append(doc)
if input_rows:
result = convert_to_kml(input_rows, out_workspace, extent)
converted += result[0]
errors += result[1]
skipped += result[2]
if input_items:
result = convert_to_kml(input_items, out_workspace, extent)
converted += result[0]
errors += result[1]
skipped += result[2]
if not input_items and not input_rows:
status_writer.send_state(status.STAT_FAILED, _("No items to process. Check if items exist."))
return
# Zip up kmz files if more than one.
if converted > 1:
status_writer.send_status("Converted: {}".format(converted))
zip_file = task_utils.zip_data(out_workspace, "{0}.zip".format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
shutil.copy2(
os.path.join(os.path.dirname(os.path.dirname(__file__)), "supportfiles", "_thumb.png"), request["folder"]
)
elif converted == 1:
try:
kml_file = glob.glob(os.path.join(out_workspace, "*.kmz"))[0]
tmp_lyr = arcpy.KMLToLayer_conversion(kml_file, out_workspace, "kml_layer")
task_utils.make_thumbnail(tmp_lyr.getOutput(0), os.path.join(request["folder"], "_thumb.png"))
except arcpy.ExecuteError:
pass
shutil.move(kml_file, os.path.join(request["folder"], os.path.basename(kml_file)))
# Update state if necessary.
if skipped > 0 or errors > 0:
status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(errors + skipped))
task_utils.report(
os.path.join(request["folder"], "__report.json"), converted, skipped, errors, errors_reasons, skipped_reasons
)
def execute(request):
"""Exports search results a CSV, shapefile or XML document.
:param request: json as a dict.
"""
# Get SSL trust setting.
verify_ssl = task_utils.get_ssl_mode()
chunk_size = task_utils.CHUNK_SIZE
file_name = task_utils.get_parameter_value(request['params'], 'file_name',
'value')
fields = task_utils.get_parameter_value(request['params'], 'fields',
'value')
out_format = task_utils.get_parameter_value(request['params'],
'output_format', 'value')
if not 'path' in fields and 'path:[absolute]' in fields:
fields.append('path')
if 'geo' in fields:
i_geo = fields.index('geo')
fields.remove('geo')
fields.insert(i_geo, '[geo]')
# Create the temporary workspace.
task_folder = os.path.join(request['folder'], 'temp')
if not os.path.exists(task_folder):
os.makedirs(task_folder)
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
num_results, response_index = task_utils.get_result_count(
request['params'])
if len(sys.argv) == 2:
query = '{0}/solr/v0/select?&wt=json&fl={1}'.format(
'http://localhost:8888', ','.join(fields))
else:
query = '{0}/select?&wt=json&fl={1}'.format(sys.argv[2].split('=')[1],
','.join(fields))
if 'query' in request['params'][response_index]:
# Voyager Search Traditional UI
for p in request['params']:
if 'query' in p:
request_qry = p['query']
break
if 'voyager.list' in request_qry:
query += '&voyager.list={0}'.format(request_qry['voyager.list'])
# Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
if 'fq' in request_qry:
try:
if isinstance(request_qry['fq'], list):
for fq in request_qry['fq']:
try:
query += '&fq={0}'.format(str(fq))
except UnicodeEncodeError:
query += '&fq={0}'.format(str(fq.encode('utf-8')))
else:
query += '&fq={0}'.format(request_qry['fq'])
if '{!expand}' in query:
query = query.replace('{!expand}', '')
if '{!tag' in query:
tag = re.findall('{!(.*?)}', query)
if tag:
tag_str = "{!" + tag[0] + "}"
query = query.replace(tag_str, '')
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['fq']:
query += '&fq={0}'.format(qry).replace("\\", "").replace(
' ', '%20')
if 'q' in request_qry:
try:
query += '&q={0}'.format(request_qry['q'].replace("\\", ""))
query = query.replace(' ', '%20')
except UnicodeEncodeError:
query += '&q={0}'.format(
request_qry['q'].encode('utf-8').replace("\\", ""))
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['q']:
query += '&q={0}'.format(qry).replace("\\", "").replace(
' ', '%20')
if 'place' in request_qry:
try:
query += '&place={0}'.format(request_qry['place'].replace(
"\\", ""))
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['place']:
query += '&place={0}'.format(qry).replace("\\",
"").replace(
' ', '%20')
if 'place.op' in request_qry:
query += '&place.op={0}'.format(request_qry['place.op'])
query += '&rows={0}&start={1}'
exported_cnt = 0.
for i in range(0, num_results, chunk_size):
url = query.replace('{0}', str(chunk_size)).replace('{1}', str(i))
res = requests.get(url, verify=verify_ssl, headers=headers)
jobs = res.json()['response']['docs']
if out_format == 'CSV':
export_to_csv(jobs, file_name, task_folder, fields)
elif out_format == 'XML':
export_to_xml(jobs, file_name, task_folder)
elif out_format == 'SHP':
export_to_shp(jobs, file_name, task_folder)
exported_cnt += chunk_size
if exported_cnt > num_results:
status_writer.send_percent(100, 'exported: 100%',
'export_results')
else:
percent_done = exported_cnt / num_results
status_writer.send_percent(
percent_done,
'{0}: {1:.0f}%'.format("exported", percent_done * 100),
'export_results')
else:
# Voyager Search Portal/Cart UI
ids = []
for p in request['params']:
if 'ids' in p:
ids = p['ids']
break
groups = task_utils.grouper(list(ids), chunk_size, '')
i = 0
for group in groups:
i += len([v for v in group if not v == ''])
results = requests.get(query + '&ids={0}'.format(','.join(group)),
verify=verify_ssl,
headers=headers)
jobs = eval(results.text)['response']['docs']
if out_format == 'CSV':
export_to_csv(jobs, file_name, task_folder, fields)
elif out_format == 'XML':
export_to_xml(jobs, file_name, task_folder)
elif out_format == 'SHP':
export_to_shp(jobs, file_name, task_folder)
percent_done = float(i) / num_results
status_writer.send_percent(
percent_done, '{0}: {1:.0f}%'.format("exported",
percent_done * 100),
'export_results')
# Zip up outputs.
if exported_count == 0:
status_writer.send_state(status.STAT_FAILED)
task_utils.report(os.path.join(request['folder'], '__report.json'),
exported_count, 0, errors_count, errors_reasons)
else:
task_utils.report(os.path.join(request['folder'], '__report.json'),
exported_count, 0, errors_count, errors_reasons)
zip_file = task_utils.zip_data(task_folder,
'{0}.zip'.format(file_name))
shutil.move(
zip_file,
os.path.join(os.path.dirname(task_folder),
os.path.basename(zip_file)))
def execute(request):
"""Writes existing metadata for summary, description and tags.
If overwrite is false, existing metadata is untouched unless any
field is empty or does not exist, then it is created.
:param request: json as a dict.
"""
updated = 0
errors = 0
skipped = 0
global result_count
parameters = request['params']
summary = task_utils.get_parameter_value(parameters, 'summary', 'value')
description = task_utils.get_parameter_value(parameters, 'description', 'value')
tags = task_utils.get_parameter_value(parameters, 'tags', 'value')
data_credits = task_utils.get_parameter_value(parameters, 'credits', 'value')
constraints = task_utils.get_parameter_value(parameters, 'constraints', 'value')
# Handle commas, spaces, and/or new line separators.
tags = [tag for tag in re.split(' |,|\n', tags) if not tag == '']
overwrite = task_utils.get_parameter_value(parameters, 'overwrite', 'value')
if not overwrite:
overwrite = False
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
# Stylesheet
xslt_file = os.path.join(arcpy.GetInstallInfo()['InstallDir'], 'Metadata/Stylesheets/gpTools/exact copy of.xslt')
# Template metadata file.
template_xml = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'metadata_template.xml')
result_count, response_index = task_utils.get_result_count(parameters)
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl + ',links'
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'), 'write_metadata')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
docs = results.json()['response']['docs']
if not docs:
docs = parameters[response_index]['response']['docs']
input_items = []
for doc in docs:
if 'path' in doc and os.path.exists(doc['path']):
if 'links' in doc:
links = eval(doc['links'])
input_items.append((doc['path'], links['links'][0]['link'][0]['id']))
else:
input_items.append((doc['path'], doc['id']))
elif 'absolute_path' in doc and os.path.exists(doc['absolute_path']):
if 'links' in doc:
links = eval(doc['links'])
input_items.append((doc['absolute_path'], links['links'][0]['link'][0]['id']))
else:
input_items.append((doc['absolute_path'], doc['id']))
elif '[downloadURL]' in doc:
if 'links' in doc:
links = eval(doc['links'])
input_items.append((doc['[downloadURL]'], links['links'][0]['link'][0]['id']))
else:
input_items.append((doc['[downloadURL]'], doc['id']))
result = write_metadata(input_items, template_xml, xslt_file, summary, description, tags, data_credits, constraints, overwrite, headers)
updated += result[0]
errors += result[1]
skipped += result[2]
status_writer.send_percent(i / result_count, '{0}: {1:.0f}%'.format("Processed", i / result_count * 100), 'write_metadata')
# Report state.
if skipped > 0 or errors > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
else:
status_writer.send_state(status.STAT_SUCCESS)
task_utils.report(os.path.join(request['folder'], '__report.json'), updated, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Adds a field and calculates it to some value.
:param request: json as a dict.
"""
created = 0
skipped = 0
errors = 0
warnings = 0
global result_count
parameters = request['params']
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
# Parameter values
field_name = task_utils.get_parameter_value(parameters, 'field_name',
'value')
field_type = task_utils.get_parameter_value(parameters, 'field_type',
'value')
field_value = task_utils.get_parameter_value(parameters, 'field_value',
'value')
# Query the index for results in groups of 25.
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
result_count, response_index = task_utils.get_result_count(parameters)
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl + ',links'
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']),
task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'), 'add_field')
for group in groups:
if fq:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query +
'{0}&ids={1}'.format(fl, ','.join(group)),
verify=verify_ssl,
headers=headers)
else:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
docs = results.json()['response']['docs']
input_items = []
for doc in docs:
if 'path' in doc:
if 'links' in doc:
links = eval(doc['links'])
input_items.append((doc['id'], doc['absolute_path'],
links['links'][0]['link'][0]['id']))
else:
input_items.append((doc['id'], doc['absolute_path']))
result = add_field(input_items, field_name, field_type, field_value)
created += result[0]
errors += result[1]
skipped += result[2]
warnings += result[3]
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(request['folder'], '__report.json'),
created, skipped, errors, errors_reasons,
skipped_reasons, warnings, warnings_reasons)
def execute(request):
"""Adds a field and calculates it to some value.
:param request: json as a dict.
"""
created = 0
skipped = 0
errors = 0
warnings = 0
global result_count
parameters = request['params']
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
# Parameter values
field_name = task_utils.get_parameter_value(parameters, 'field_name', 'value')
field_type = task_utils.get_parameter_value(parameters, 'field_type', 'value')
field_value = task_utils.get_parameter_value(parameters, 'field_value', 'value')
# Query the index for results in groups of 25.
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
result_count, response_index = task_utils.get_result_count(parameters)
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl + ',links'
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'), 'add_field')
for group in groups:
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
docs = results.json()['response']['docs']
input_items = []
for doc in docs:
if 'path' in doc:
if 'links' in doc:
links = eval(doc['links'])
input_items.append((doc['id'], doc['path'], links['links'][0]['link'][0]['id']))
else:
input_items.append((doc['id'], doc['path']))
result = add_field(input_items, field_name, field_type, field_value)
created += result[0]
errors += result[1]
skipped += result[2]
warnings += result[3]
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(request['folder'], '__report.json'), created, skipped, errors, errors_reasons, skipped_reasons, warnings, warnings_reasons)
def execute(request):
"""Package inputs to an Esri map or layer package.
:param request: json as a dict.
"""
errors = 0
skipped = 0
layers = []
files = []
app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
parameters = request['params']
out_format = task_utils.get_parameter_value(parameters, 'output_format',
'value')
summary = task_utils.get_parameter_value(parameters, 'summary')
tags = task_utils.get_parameter_value(parameters, 'tags')
output_file_name = task_utils.get_parameter_value(parameters,
'output_file_name')
if not output_file_name:
output_file_name = 'package_results'
# Get the clip region as an extent object.
clip_area = None
try:
clip_area_wkt = task_utils.get_parameter_value(parameters,
'processing_extent',
'wkt')
clip_area = task_utils.get_clip_region(clip_area_wkt)
except (KeyError, ValueError):
pass
out_workspace = os.path.join(request['folder'], 'temp')
if not os.path.exists(out_workspace):
os.makedirs(out_workspace)
num_results, response_index = task_utils.get_result_count(parameters)
# if num_results > task_utils.CHUNK_SIZE:
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results),
task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']),
task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, num_results),
task_utils.CHUNK_SIZE, '')
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
status_writer.send_status(_('Starting to process...'))
for group in groups:
if fq:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query +
'{0}&ids={1}'.format(fl, ','.join(group)),
verify=verify_ssl,
headers=headers)
else:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
input_items = task_utils.get_input_items(
results.json()['response']['docs'])
if not input_items:
input_items = task_utils.get_input_items(
parameters[response_index]['response']['docs'])
layers, files, errors, skipped = get_items(input_items, out_workspace)
# else:
# input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
# layers, files, errors, skipped = get_items(input_items, out_workspace)
if errors == num_results:
status_writer.send_state(status.STAT_FAILED,
_('No results to package'))
return
try:
if out_format == 'MPK':
shutil.copyfile(
os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'),
os.path.join(out_workspace, 'output.mxd'))
mxd = arcpy.mapping.MapDocument(
os.path.join(out_workspace, 'output.mxd'))
if mxd.description == '':
mxd.description = os.path.basename(mxd.filePath)
df = arcpy.mapping.ListDataFrames(mxd)[0]
for layer in layers:
arcpy.mapping.AddLayer(df, layer)
mxd.save()
status_writer.send_status(
_('Generating {0}. Large input {1} will take longer to process.'
.format('MPK', 'results')))
if arcpy.GetInstallInfo()['Version'] == '10.0':
arcpy.PackageMap_management(
mxd.filePath,
os.path.join(os.path.dirname(out_workspace),
'{0}.mpk'.format(output_file_name)),
'PRESERVE',
extent=clip_area)
elif arcpy.GetInstallInfo()['Version'] == '10.1':
arcpy.PackageMap_management(
mxd.filePath,
os.path.join(os.path.dirname(out_workspace),
'{0}.mpk'.format(output_file_name)),
'PRESERVE',
extent=clip_area,
ArcGISRuntime='RUNTIME',
version='10',
additional_files=files,
summary=summary,
tags=tags)
else:
arcpy.PackageMap_management(
mxd.filePath,
os.path.join(os.path.dirname(out_workspace),
'{0}.mpk'.format(output_file_name)),
'PRESERVE',
extent=clip_area,
arcgisruntime='RUNTIME',
version='10',
additional_files=files,
summary=summary,
tags=tags)
# Create a thumbnail size PNG of the mxd.
task_utils.make_thumbnail(
mxd, os.path.join(request['folder'], '_thumb.png'))
else:
status_writer.send_status(
_('Generating {0}. Large input {1} will take longer to process.'
.format('LPK', 'results')))
for layer in layers:
if layer.description == '':
layer.description = layer.name
if arcpy.GetInstallInfo()['Version'] == '10.0':
arcpy.PackageLayer_management(
layers,
os.path.join(os.path.dirname(out_workspace),
'{0}.lpk'.format(output_file_name)),
'PRESERVE',
extent=clip_area,
version='10')
else:
arcpy.PackageLayer_management(
layers,
os.path.join(os.path.dirname(out_workspace),
'{0}.lpk'.format(output_file_name)),
'PRESERVE',
extent=clip_area,
version='10',
additional_files=files,
summary=summary,
tags=tags)
# Create a thumbnail size PNG of the mxd.
task_utils.make_thumbnail(
layers[0], os.path.join(request['folder'], '_thumb.png'))
except (RuntimeError, ValueError, arcpy.ExecuteError) as ex:
status_writer.send_state(status.STAT_FAILED, repr(ex))
return
# Update state if necessary.
if errors > 0 or skipped:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(errors + skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'),
num_results - (skipped + errors), skipped, errors,
errors_reasons, skipped_reasons)
def execute(request):
"""Deletes files.
:param request: json as a dict.
"""
deleted = 0
skipped = 0
parameters = request['params']
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
num_results, response_index = task_utils.get_result_count(parameters)
if num_results > task_utils.CHUNK_SIZE:
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1],
'/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results),
task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(
list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE,
'')
else:
groups = task_utils.grouper(range(0, num_results),
task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'),
'delete_files')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(
task_utils.CHUNK_SIZE, group[0]),
headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(
query + '{0}&ids={1}'.format(fl, ','.join(group)),
headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(
task_utils.CHUNK_SIZE, group[0]),
headers=headers)
input_items = task_utils.get_input_items(
results.json()['response']['docs'], True, True)
if not input_items:
input_items = task_utils.get_input_items(
parameters[response_index]['response']['docs'])
result = delete_files(input_items)
deleted += result[0]
skipped += result[1]
status_writer.send_percent(
i / num_results,
'{0}: {1:.0f}%'.format("Processed",
i / num_results * 100), 'delete_files')
else:
input_items = task_utils.get_input_items(
parameters[response_index]['response']['docs'], True, True)
deleted, skipped = delete_files(input_items, True)
# Update state if necessary.
if skipped > 0:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'),
deleted,
skipped,
skipped_details=skipped_reasons)
def execute(request):
"""Copies files to a target folder.
:param request: json as a dict.
"""
extracted = 0
skipped = 0
errors = 0
global result_count
parameters = request['params']
output_type = task_utils.get_parameter_value(parameters, 'output_format',
'value')
task_folder = os.path.join(request['folder'], 'temp')
if not os.path.exists(task_folder):
os.makedirs(task_folder)
if output_type == 'FGDB':
arcpy.CreateFileGDB_management(task_folder, 'output.gdb')
result_count, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
fl)
# query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']),
task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
status_writer.send_percent(0.0, _('Starting to process...'),
'locate_xt_arcgis_tool')
for group in groups:
if fq:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query +
'{0}&ids={1}'.format(fl, ','.join(group)),
headers=headers)
else:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
headers=headers)
docs = results.json()['response']['docs']
if not docs:
docs = parameters[response_index]['response']['docs']
input_items = task_utils.get_input_items(docs)
if input_items:
result = extract(input_items, output_type, task_folder)
extracted += result[0]
errors += result[1]
skipped += result[2]
else:
status_writer.send_state(
status.STAT_FAILED,
_('No items to process. Check if items exist.'))
return
# Zip up outputs.
zip_file = task_utils.zip_data(task_folder, 'output.zip')
shutil.move(
zip_file,
os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(request['folder'],
'__report.json'), extracted, skipped,
errors, errors_reasons, skipped_reasons)
def execute(request):
"""Copies data to an existing geodatabase or feature dataset.
:param request: json as a dict.
"""
added = 0
errors = 0
skipped = 0
global result_count
parameters = request['params']
# Get the target workspace location.
out_gdb = task_utils.get_parameter_value(parameters, 'target_workspace',
'value')
# Retrieve the coordinate system code.
out_coordinate_system = task_utils.get_parameter_value(
parameters, 'output_projection', 'code')
if not out_coordinate_system == '0': # Same as Input
arcpy.env.outputCoordinateSystem = task_utils.get_spatial_reference(
out_coordinate_system)
task_folder = request['folder']
if not os.path.exists(task_folder):
os.makedirs(task_folder)
# Check if the geodatabase exists or if it is a feature dataset.
is_fds = False
if not os.path.exists(out_gdb):
if out_gdb.endswith('.gdb'):
arcpy.CreateFileGDB_management(os.path.dirname(out_gdb),
os.path.basename(out_gdb))
status_writer.send_status(
_('Created output workspace: {0}').format(out_gdb))
elif out_gdb.endswith('.mdb'):
arcpy.CreatePersonalGDB_management(os.path.dirname(out_gdb),
os.path.basename(out_gdb))
status_writer.send_status(
_('Created output workspace: {0}').format(out_gdb))
elif out_gdb.endswith('.sde'):
status_writer.send_state(status.STAT_FAILED,
_('{0} does not exist').format(out_gdb))
return
else:
# Possible feature dataset.
is_fds = is_feature_dataset(out_gdb)
if not is_fds:
if os.path.dirname(out_gdb).endswith('.gdb'):
if not os.path.exists(os.path.dirname(out_gdb)):
arcpy.CreateFileGDB_management(
os.path.dirname(os.path.dirname(out_gdb)),
os.path.basename(os.path.dirname(out_gdb)))
arcpy.CreateFeatureDataset_management(
os.path.dirname(out_gdb), os.path.basename(out_gdb))
elif os.path.dirname(out_gdb).endswith('.mdb'):
if not os.path.exists(os.path.dirname(out_gdb)):
arcpy.CreatePersonalGDB_management(
os.path.dirname(os.path.dirname(out_gdb)),
os.path.basename(os.path.dirname(out_gdb)))
arcpy.CreateFeatureDataset_management(
os.path.dirname(out_gdb), os.path.basename(out_gdb))
status_writer.send_status(_('Setting the output workspace...'))
arcpy.env.workspace = out_gdb
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
result_count, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
fl)
# query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']),
task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count),
task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'),
'add_to_geodatabase')
for group in groups:
if fq:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query +
'{0}&ids={1}'.format(fl, ','.join(group)),
verify=verify_ssl,
headers=headers)
else:
results = requests.get(
query +
"&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
verify=verify_ssl,
headers=headers)
docs = results.json()['response']['docs']
input_items = task_utils.get_input_items(docs)
if not input_items:
input_items = task_utils.get_input_items(
parameters[response_index]['response']['docs'])
input_rows = collections.defaultdict(list)
for doc in docs:
if 'path' not in doc:
input_rows[doc['title']].append(doc)
if input_rows:
result = add_to_geodatabase(input_rows, out_gdb, is_fds)
added += result[0]
errors += result[1]
skipped += result[2]
if input_items:
result = add_to_geodatabase(input_items, out_gdb, is_fds)
added += result[0]
errors += result[1]
skipped += result[2]
if not input_items and not input_rows:
status_writer.send_state(
status.STAT_FAILED,
_('No items to process. Check if items exist.'))
return
# Update state if necessary.
if skipped > 0 or errors > 0:
status_writer.send_state(
status.STAT_WARNING,
_('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(task_folder, '__report.json'), added,
skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Clips selected search results using the clip geometry.
:param request: json as a dict.
"""
clipped = 0
errors = 0
skipped = 0
global result_count
parameters = request['params']
# Retrieve the clip features.
clip_features = task_utils.get_parameter_value(parameters, 'clip_features', 'value')
# Retrieve the coordinate system code.
out_coordinate_system = int(task_utils.get_parameter_value(parameters, 'output_projection', 'code'))
# Retrieve the output format, create mxd and output file name parameter values.
out_format = task_utils.get_parameter_value(parameters, 'output_format', 'value')
create_mxd = task_utils.get_parameter_value(parameters, 'create_mxd', 'value')
output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
if not output_file_name:
output_file_name = 'clip_results'
# Create the temporary workspace if clip_feature_class:
out_workspace = os.path.join(request['folder'], 'temp')
if not os.path.exists(out_workspace):
os.makedirs(out_workspace)
# Set the output coordinate system.
if not out_coordinate_system == 0: # Same as Input
out_sr = task_utils.get_spatial_reference(out_coordinate_system)
arcpy.env.outputCoordinateSystem = out_sr
# Set the output workspace.
status_writer.send_status(_('Setting the output workspace...'))
if not out_format == 'SHP':
out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0)
arcpy.env.workspace = out_workspace
# Query the index for results in groups of 25.
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
result_count, response_index = task_utils.get_result_count(parameters)
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
# Get the Clip features by id.
id = clip_features['id']
clip_query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', "&fl=id,path,fullpath:[absolute],absolute_path:[absolute],[lyrFile],[geo]&q=id:{0}".format(id))
clip_result = requests.get(clip_query, verify=verify_ssl, headers=headers)
clipper = clip_result.json()['response']['docs'][0]
if 'absolute_path' in clipper and not clipper['absolute_path'].startswith('s3'):
clip_features = clipper['absolute_path']
elif '[lyrFile]' in clipper:
clip_features = clipper['[lyrFile]']
elif '[geo]' in clipper:
clip_features = arcpy.AsShape(clipper['[geo]']).projectAs(arcpy.SpatialReference(4326))
elif 'absolute_path' in clipper and clipper['absolute_path'].startswith('s3'):
base_name = os.path.basename(clipper['path'])
temp_folder = tempfile.mkdtemp()
if '[downloadURL]' in clipper:
download = os.path.join(temp_folder, os.path.basename(clipper['[downloadURL]']))
response = requests.get(clipper['[downloadURL]'], verify=verify_ssl)
with open(download, 'wb') as fp:
fp.write(response.content)
if download.endswith('.zip'):
zip = zipfile.ZipFile(download)
zip.extractall(temp_folder)
clip_features = os.path.join(temp_folder, base_name)
else:
clip_features = download
else:
bbox = clipper['bbox'].split()
extent = arcpy.Extent(*bbox)
pt_array = arcpy.Array([extent.lowerLeft, extent.upperLeft, extent.upperRight, extent.lowerRight])
clip_features = arcpy.Polygon(pt_array, 4326)
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'), 'clip_data')
for group in groups:
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)
docs = results.json()['response']['docs']
input_items = task_utils.get_input_items(docs)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
input_rows = collections.defaultdict(list)
for doc in docs:
if 'path' not in doc:
input_rows[doc['name']].append(doc)
if input_rows:
result = clip_data(input_rows, out_workspace, clip_features, out_format)
clipped += result[0]
errors += result[1]
skipped += result[2]
if input_items:
result = clip_data(input_items, out_workspace, clip_features, out_format)
clipped += result[0]
errors += result[1]
skipped += result[2]
if not input_items and not input_rows:
status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.'))
return
if arcpy.env.workspace.endswith('.gdb'):
out_workspace = os.path.dirname(arcpy.env.workspace)
if clipped > 0:
try:
if out_format == 'MPK':
mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
mxd = task_utils.create_mxd(out_workspace, mxd_template, 'output')
status_writer.send_status(_("Packaging results..."))
task_utils.create_mpk(out_workspace, mxd, files_to_package)
shutil.move(os.path.join(out_workspace, 'output.mpk'),
os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name)))
elif out_format == 'LPK':
status_writer.send_status(_("Packaging results..."))
task_utils.create_lpk(out_workspace, output_file_name, files_to_package)
elif out_format == 'KML':
task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb"))
arcpy.env.workspace = ''
arcpy.RefreshCatalog(os.path.join(out_workspace, "output.gdb"))
try:
arcpy.Delete_management(os.path.join(out_workspace, "output.gdb"))
except arcpy.ExecuteError:
pass
zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
else:
if create_mxd:
mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
task_utils.create_mxd(out_workspace, mxd_template, 'output')
zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
except arcpy.ExecuteError as ee:
status_writer.send_state(status.STAT_FAILED, _(ee))
sys.exit(1)
else:
status_writer.send_state(status.STAT_FAILED, _('No output created. Zero inputs were clipped.'))
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), clipped, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Copies files to a target folder.
:param request: json as a dict.
"""
created = 0
skipped = 0
errors = 0
global result_count
parameters = request['params']
if not os.path.exists(request['folder']):
os.makedirs(request['folder'])
meta_folder = task_utils.get_parameter_value(parameters, 'meta_data_folder', 'value')
result_count, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
status_writer.send_percent(0.0, _('Starting to process...'), 'create_layer_files')
i = 0.
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
docs = results.json()['response']['docs']
# docs = eval(results.read().replace('false', 'False').replace('true', 'True'))['response']['docs']
if not docs:
docs = parameters[response_index]['response']['docs']
input_items = []
for doc in docs:
if 'path' in doc:
input_items.append((doc['id'], doc['path'], doc['name'], doc['location']))
result = create_layer_file(input_items, meta_folder)
created += result[0]
errors += result[1]
skipped += result[2]
try:
shutil.copy2(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', '_thumb.png'), request['folder'])
except IOError:
pass
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(request['folder'], '__report.json'), created, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Exports search results a CSV, shapefile or XML document.
:param request: json as a dict.
"""
chunk_size = task_utils.CHUNK_SIZE
file_name = task_utils.get_parameter_value(request['params'], 'file_name', 'value')
fields = task_utils.get_parameter_value(request['params'], 'fields', 'value')
out_format = task_utils.get_parameter_value(request['params'], 'output_format', 'value')
if not 'path' in fields and 'path:[absolute]' in fields:
fields.append('path')
if 'geo' in fields:
i_geo = fields.index('geo')
fields.remove('geo')
fields.insert(i_geo, '[geo]')
# Create the temporary workspace.
task_folder = os.path.join(request['folder'], 'temp')
if not os.path.exists(task_folder):
os.makedirs(task_folder)
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
num_results, response_index = task_utils.get_result_count(request['params'])
query = '{0}/select?&wt=json&fl={1}'.format(sys.argv[2].split('=')[1], ','.join(fields))
if 'query' in request['params'][response_index]:
# Voyager Search Traditional UI
for p in request['params']:
if 'query' in p:
request_qry = p['query']
break
if 'voyager.list' in request_qry:
query += '&voyager.list={0}'.format(request_qry['voyager.list'])
# Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
if 'fq' in request_qry:
try:
query += '&fq={0}'.format(request_qry['fq'].replace("\\", ""))
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['fq']:
query += '&fq={0}'.format(qry).replace("\\", "").replace(' ', '%20')
if 'q' in request_qry:
try:
query += '&q={0}'.format(request_qry['q'].replace("\\", ""))
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['q']:
query += '&q={0}'.format(qry).replace("\\", "").replace(' ', '%20')
if 'place' in request_qry:
try:
query += '&place={0}'.format(request_qry['place'].replace("\\", ""))
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['place']:
query += '&place={0}'.format(qry).replace("\\", "").replace(' ', '%20')
if 'place.op' in request_qry:
query += '&place.op={0}'.format(request_qry['place.op'])
query += '&rows={0}&start={1}'
exported_cnt = 0.
for i in xrange(0, num_results, chunk_size):
req = urllib2.Request(query.replace('{0}', str(chunk_size)).replace('{1}', str(i)), headers=headers)
for n in urllib2.urlopen(req):
jobs = eval(n.replace('null', '"null"'))['response']['docs']
if out_format == 'CSV':
export_to_csv(jobs, file_name, task_folder, fields)
elif out_format == 'XML':
export_to_xml(jobs, file_name, task_folder)
elif out_format == 'SHP':
export_to_shp(jobs, file_name, task_folder)
exported_cnt += chunk_size
if exported_cnt > num_results:
status_writer.send_percent(100, 'exported: 100%', 'export_results')
else:
percent_done = exported_cnt / num_results
status_writer.send_percent(percent_done, '{0}: {1:.0f}%'.format("exported", percent_done * 100), 'export_results')
else:
# Voyager Search Portal/Cart UI
ids = []
for p in request['params']:
if 'ids' in p:
ids = p['ids']
break
groups = task_utils.grouper(list(ids), chunk_size, '')
i = 0
for group in groups:
i += len([v for v in group if not v == ''])
req = urllib2.Request(query + '&ids={0}'.format(','.join(group)), headers=headers)
results = urllib2.urlopen(req)
jobs = eval(results.read())['response']['docs']
if out_format == 'CSV':
export_to_csv(jobs, file_name, task_folder, fields)
elif out_format == 'XML':
export_to_xml(jobs, file_name, task_folder)
elif out_format == 'SHP':
export_to_shp(jobs, file_name, task_folder)
percent_done = float(i) / num_results
status_writer.send_percent(percent_done, '{0}: {1:.0f}%'.format("exported", percent_done * 100), 'export_results')
# Zip up outputs.
if exported_count == 0:
task_utils.report(os.path.join(request['folder'], '__report.json'), exported_count, 0, errors_count, errors_reasons)
else:
task_utils.report(os.path.join(request['folder'], '__report.json'), exported_count, 0, errors_count, errors_reasons)
zip_file = task_utils.zip_data(task_folder, 'output.zip')
shutil.move(zip_file, os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))
def execute(request):
"""Copies files to a target folder.
:param request: json as a dict.
"""
extracted = 0
skipped = 0
errors = 0
global result_count
parameters = request['params']
output_type = task_utils.get_parameter_value(parameters, 'output_format', 'value')
task_folder = os.path.join(request['folder'], 'temp')
if not os.path.exists(task_folder):
os.makedirs(task_folder)
if output_type == 'FGDB':
arcpy.CreateFileGDB_management(task_folder, 'output.gdb')
result_count, response_index = task_utils.get_result_count(parameters)
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
# query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
status_writer.send_percent(0.0, _('Starting to process...'), 'locate_xt_arcgis_tool')
for group in groups:
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
docs = results.json()['response']['docs']
if not docs:
docs = parameters[response_index]['response']['docs']
input_items = task_utils.get_input_items(docs)
if input_items:
result = extract(input_items, output_type, task_folder)
extracted += result[0]
errors += result[1]
skipped += result[2]
else:
status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.'))
return
# Zip up outputs.
zip_file = task_utils.zip_data(task_folder, 'output.zip')
shutil.move(zip_file, os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
task_utils.report(os.path.join(request['folder'], '__report.json'), extracted, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
"""Clips selected search results using the clip geometry.
:param request: json as a dict.
"""
clipped = 0
errors = 0
skipped = 0
global result_count
parameters = request["params"]
# Retrieve the clip features.
clip_features = task_utils.get_parameter_value(parameters, "clip_features", "value")
# Retrieve the coordinate system code.
out_coordinate_system = int(task_utils.get_parameter_value(parameters, "output_projection", "code"))
# Retrieve the output format, create mxd and output file name parameter values.
out_format = task_utils.get_parameter_value(parameters, "output_format", "value")
create_mxd = task_utils.get_parameter_value(parameters, "create_mxd", "value")
output_file_name = task_utils.get_parameter_value(parameters, "output_file_name", "value")
if not output_file_name:
output_file_name = "clip_results"
# Create the temporary workspace if clip_feature_class:
out_workspace = os.path.join(request["folder"], "temp")
if not os.path.exists(out_workspace):
os.makedirs(out_workspace)
# Set the output coordinate system.
if not out_coordinate_system == 0: # Same as Input
out_sr = task_utils.get_spatial_reference(out_coordinate_system)
arcpy.env.outputCoordinateSystem = out_sr
# Set the output workspace.
status_writer.send_status(_("Setting the output workspace..."))
if not out_format == "SHP":
out_workspace = arcpy.CreateFileGDB_management(out_workspace, "output.gdb").getOutput(0)
arcpy.env.workspace = out_workspace
# Query the index for results in groups of 25.
headers = {"x-access-token": task_utils.get_security_token(request["owner"])}
result_count, response_index = task_utils.get_result_count(parameters)
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
# Get the Clip features by id.
id = clip_features["id"]
clip_query = "{0}{1}{2}".format(
sys.argv[2].split("=")[1], "/select?&wt=json", "&fl=id,path:[absolute],[lyrFile],[geo]&q=id:{0}".format(id)
)
clip_result = requests.get(clip_query, headers=headers)
clipper = clip_result.json()["response"]["docs"][0]
if "path" in clipper:
clip_features = clipper["path"]
elif "[lyrFile]" in clipper:
clip_features = clipper["[lyrFile]"]
elif "[geo]" in clipper:
clip_features = arcpy.AsShape(clipper["[geo]"]).projectAs(arcpy.SpatialReference(4326))
else:
bbox = clipper["bbox"].split()
extent = arcpy.Extent(*bbox)
pt_array = arcpy.Array([extent.lowerLeft, extent.upperLeft, extent.upperRight, extent.lowerRight])
clip_features = arcpy.Polygon(pt_array, 4326)
query = "{0}{1}{2}".format(sys.argv[2].split("=")[1], "/select?&wt=json", fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")
query += fq
elif "ids" in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]["ids"]), task_utils.CHUNK_SIZE, "")
else:
groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")
status_writer.send_percent(0.0, _("Starting to process..."), "clip_data")
for group in groups:
if fq:
results = requests.get(
query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
)
elif "ids" in parameters[response_index]:
results = requests.get(query + "{0}&ids={1}".format(fl, ",".join(group)), headers=headers)
else:
results = requests.get(
query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
)
# docs = eval(results.read().replace('false', 'False').replace('true', 'True').replace('null', 'None'))['response']['docs']
docs = results.json()["response"]["docs"]
input_items = task_utils.get_input_items(docs)
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]["response"]["docs"])
input_rows = collections.defaultdict(list)
for doc in docs:
if "path" not in doc:
input_rows[doc["name"]].append(doc)
if input_rows:
result = clip_data(input_rows, out_workspace, clip_features, out_format)
clipped += result[0]
errors += result[1]
skipped += result[2]
if input_items:
result = clip_data(input_items, out_workspace, clip_features, out_format)
clipped += result[0]
errors += result[1]
skipped += result[2]
if not input_items and not input_rows:
status_writer.send_state(status.STAT_FAILED, _("No items to process. Check if items exist."))
return
if arcpy.env.workspace.endswith(".gdb"):
out_workspace = os.path.dirname(arcpy.env.workspace)
if clipped > 0:
try:
if out_format == "MPK":
mxd_template = os.path.join(
os.path.dirname(os.path.dirname(__file__)), "supportfiles", "MapTemplate.mxd"
)
mxd = task_utils.create_mxd(out_workspace, mxd_template, "output")
status_writer.send_status(_("Packaging results..."))
task_utils.create_mpk(out_workspace, mxd, files_to_package)
shutil.move(
os.path.join(out_workspace, "output.mpk"),
os.path.join(os.path.dirname(out_workspace), "{0}.mpk".format(output_file_name)),
)
elif out_format == "LPK":
status_writer.send_status(_("Packaging results..."))
task_utils.create_lpk(out_workspace, output_file_name, files_to_package)
elif out_format == "KML":
task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb"))
arcpy.env.workspace = ""
arcpy.RefreshCatalog(os.path.join(out_workspace, "output.gdb"))
try:
arcpy.Delete_management(os.path.join(out_workspace, "output.gdb"))
except arcpy.ExecuteError:
pass
zip_file = task_utils.zip_data(out_workspace, "{0}.zip".format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
else:
if create_mxd:
mxd_template = os.path.join(
os.path.dirname(os.path.dirname(__file__)), "supportfiles", "MapTemplate.mxd"
)
task_utils.create_mxd(out_workspace, mxd_template, "output")
zip_file = task_utils.zip_data(out_workspace, "{0}.zip".format(output_file_name))
shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
except arcpy.ExecuteError as ee:
status_writer.send_state(status.STAT_FAILED, _(ee))
sys.exit(1)
else:
status_writer.send_state(status.STAT_FAILED, _("No output created. Zero inputs were clipped."))
# Update state if necessary.
if errors > 0 or skipped > 0:
status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(errors + skipped))
task_utils.report(
os.path.join(request["folder"], "__report.json"), clipped, skipped, errors, errors_reasons, skipped_reasons
)
def execute(request):
"""Builds raster pyramids for input raster datasets.
:param request: json as a dict.
"""
processed = 0
skipped = 0
parameters = request['params']
# Get the extent for for which to use to calculate statistics.
extent = ''
try:
try:
ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt')
if ext:
sr = task_utils.get_spatial_reference("4326")
extent = task_utils.from_wkt(ext, sr)
except KeyError:
ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'feature')
if ext:
extent = arcpy.Describe(ext).extent
except KeyError:
pass
horizontal_skip_factor = task_utils.get_parameter_value(parameters, 'horizontal_skip_factor', 'value')
vertical_skip_factor = task_utils.get_parameter_value(parameters, 'vertical_skip_factor', 'value')
ignore_pixel_values = task_utils.get_parameter_value(parameters, 'ignore_pixel_values', 'value')
# Create the task folder to hold report files.
task_folder = request['folder']
if not os.path.exists(task_folder):
os.makedirs(task_folder)
headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
num_results, response_index = task_utils.get_result_count(parameters)
if num_results > task_utils.CHUNK_SIZE:
# Query the index for results in groups of 25.
query_index = task_utils.QueryIndex(parameters[response_index])
fl = query_index.fl
query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
fq = query_index.get_fq()
if fq:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
query += fq
elif 'ids' in parameters[response_index]:
groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
else:
groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
# Begin processing
status_writer.send_percent(0.0, _('Starting to process...'), 'calculate_raster_statistics')
i = 0.
for group in groups:
i += len(group) - group.count('')
if fq:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
elif 'ids' in parameters[response_index]:
results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
else:
results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
# input_items = task_utils.get_input_items(eval(results.read().replace('false', 'False').replace('true', 'True'))['response']['docs'])
input_items = task_utils.get_input_items(results.json()['response']['docs'])
if not input_items:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
result = calculate_raster_statistics(input_items, extent, horizontal_skip_factor, vertical_skip_factor, ignore_pixel_values)
processed += result[0]
skipped += result[1]
status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'calculate_raster_statistics')
else:
input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
processed, skipped = calculate_raster_statistics(input_items, extent, horizontal_skip_factor,
vertical_skip_factor, ignore_pixel_values, True)
# Update state if necessary.
if skipped > 0:
status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
task_utils.report(os.path.join(request['folder'], '__report.json'), processed, skipped, skipped_details=skipped_reasons)
