import sys
import collections
import shutil
import requests
import tempfile
import zipfile
import arcpy
from utils import status
from utils import task_utils
import warnings
from requests.packages.urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter('ignore', InsecureRequestWarning)
# Get SSL trust setting.
verify_ssl = task_utils.get_ssl_mode()
status_writer = status.Writer()
files_to_package = list()
errors_reasons = {}
skipped_reasons = {}
result_count = 0
processed_count = 0.
# Used when clipping geometry features (globals of these are defined as well).
layer_name = ""
new_fields = []
field_values = []
existing_fields = []
def execute(request):
"""Exports search results a CSV, shapefile or XML document.
:param request: json as a dict.
"""
# Get SSL trust setting.
verify_ssl = task_utils.get_ssl_mode()
chunk_size = task_utils.CHUNK_SIZE
file_name = task_utils.get_parameter_value(request['params'], 'file_name',
'value')
fields = task_utils.get_parameter_value(request['params'], 'fields',
'value')
out_format = task_utils.get_parameter_value(request['params'],
'output_format', 'value')
if not 'path' in fields and 'path:[absolute]' in fields:
fields.append('path')
if 'geo' in fields:
i_geo = fields.index('geo')
fields.remove('geo')
fields.insert(i_geo, '[geo]')
# Create the temporary workspace.
task_folder = os.path.join(request['folder'], 'temp')
if not os.path.exists(task_folder):
os.makedirs(task_folder)
headers = {
'x-access-token': task_utils.get_security_token(request['owner'])
}
num_results, response_index = task_utils.get_result_count(
request['params'])
if len(sys.argv) == 2:
query = '{0}/solr/v0/select?&wt=json&fl={1}'.format(
'http://localhost:8888', ','.join(fields))
else:
query = '{0}/select?&wt=json&fl={1}'.format(sys.argv[2].split('=')[1],
','.join(fields))
if 'query' in request['params'][response_index]:
# Voyager Search Traditional UI
for p in request['params']:
if 'query' in p:
request_qry = p['query']
break
if 'voyager.list' in request_qry:
query += '&voyager.list={0}'.format(request_qry['voyager.list'])
# Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
if 'fq' in request_qry:
try:
if isinstance(request_qry['fq'], list):
for fq in request_qry['fq']:
try:
query += '&fq={0}'.format(str(fq))
except UnicodeEncodeError:
query += '&fq={0}'.format(str(fq.encode('utf-8')))
else:
query += '&fq={0}'.format(request_qry['fq'])
if '{!expand}' in query:
query = query.replace('{!expand}', '')
if '{!tag' in query:
tag = re.findall('{!(.*?)}', query)
if tag:
tag_str = "{!" + tag[0] + "}"
query = query.replace(tag_str, '')
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['fq']:
query += '&fq={0}'.format(qry).replace("\\", "").replace(
' ', '%20')
if 'q' in request_qry:
try:
query += '&q={0}'.format(request_qry['q'].replace("\\", ""))
query = query.replace(' ', '%20')
except UnicodeEncodeError:
query += '&q={0}'.format(
request_qry['q'].encode('utf-8').replace("\\", ""))
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['q']:
query += '&q={0}'.format(qry).replace("\\", "").replace(
' ', '%20')
if 'place' in request_qry:
try:
query += '&place={0}'.format(request_qry['place'].replace(
"\\", ""))
query = query.replace(' ', '%20')
except AttributeError:
for qry in request_qry['place']:
query += '&place={0}'.format(qry).replace("\\",
"").replace(
' ', '%20')
if 'place.op' in request_qry:
query += '&place.op={0}'.format(request_qry['place.op'])
query += '&rows={0}&start={1}'
exported_cnt = 0.
for i in range(0, num_results, chunk_size):
url = query.replace('{0}', str(chunk_size)).replace('{1}', str(i))
res = requests.get(url, verify=verify_ssl, headers=headers)
jobs = res.json()['response']['docs']
if out_format == 'CSV':
export_to_csv(jobs, file_name, task_folder, fields)
elif out_format == 'XML':
export_to_xml(jobs, file_name, task_folder)
elif out_format == 'SHP':
export_to_shp(jobs, file_name, task_folder)
exported_cnt += chunk_size
if exported_cnt > num_results:
status_writer.send_percent(100, 'exported: 100%',
'export_results')
else:
percent_done = exported_cnt / num_results
status_writer.send_percent(
percent_done,
'{0}: {1:.0f}%'.format("exported", percent_done * 100),
'export_results')
else:
# Voyager Search Portal/Cart UI
ids = []
for p in request['params']:
if 'ids' in p:
ids = p['ids']
break
groups = task_utils.grouper(list(ids), chunk_size, '')
i = 0
for group in groups:
i += len([v for v in group if not v == ''])
results = requests.get(query + '&ids={0}'.format(','.join(group)),
verify=verify_ssl,
headers=headers)
jobs = eval(results.text)['response']['docs']
if out_format == 'CSV':
export_to_csv(jobs, file_name, task_folder, fields)
elif out_format == 'XML':
export_to_xml(jobs, file_name, task_folder)
elif out_format == 'SHP':
export_to_shp(jobs, file_name, task_folder)
percent_done = float(i) / num_results
status_writer.send_percent(
percent_done, '{0}: {1:.0f}%'.format("exported",
percent_done * 100),
'export_results')
# Zip up outputs.
if exported_count == 0:
status_writer.send_state(status.STAT_FAILED)
task_utils.report(os.path.join(request['folder'], '__report.json'),
exported_count, 0, errors_count, errors_reasons)
else:
task_utils.report(os.path.join(request['folder'], '__report.json'),
exported_count, 0, errors_count, errors_reasons)
zip_file = task_utils.zip_data(task_folder,
'{0}.zip'.format(file_name))
shutil.move(
zip_file,
os.path.join(os.path.dirname(task_folder),
os.path.basename(zip_file)))