def execute(request): """Move files to a target folder. :param request: json as a dict. """ moved = 0 skipped = 0 errors = 0 new_folder = False parameters = request['params'] target_folder = task_utils.get_parameter_value(parameters, 'target_folder', 'value') flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value') if not os.path.exists(request['folder']): os.makedirs(request['folder']) if target_folder: if not os.path.exists(target_folder): os.makedirs(target_folder) new_folder = True num_results, response_index = task_utils.get_result_count(parameters) task_utils.CHUNK_SIZE = num_results # Query the index for results in groups of 25. headers = {'x-access-token': task_utils.get_security_token(request['owner'])} query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'move_files') i = 0. for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),headers=headers) input_items = task_utils.get_input_items(results.json()['response']['docs'], True, True) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) result = move_files(input_items, target_folder, flatten_results) moved += result[0] errors += result[1] skipped += result[2] status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'move_files') # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(request['folder'], '__report.json'), moved, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Copies files to a target folder. :param request: json as a dict. """ copied = 0 skipped = 0 errors = 0 global result_count parameters = request['params'] target_dirs = '' target_folder = task_utils.get_parameter_value(parameters, 'target_folder', 'value') flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value') if not flatten_results: target_dirs = os.path.splitdrive(target_folder)[1] flatten_results = 'false' if not os.path.exists(request['folder']): os.makedirs(request['folder']) # Query the index for results in groups of 25. result_count, response_index = task_utils.get_result_count(parameters) query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'copy_files') i = 0. headers = {'x-access-token': task_utils.get_security_token(request['owner'])} for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) input_items = task_utils.get_input_items(results.json()['response']['docs'], list_components=True) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) result = copy_files(input_items, target_folder, flatten_results, target_dirs) copied += result[0] errors += result[1] skipped += result[2] # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(request['folder'], '__report.json'), copied, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Copies files to a target folder. :param request: json as a dict. """ copied = 0 skipped = 0 errors = 0 global result_count parameters = request['params'] target_dirs = '' target_folder = task_utils.get_parameter_value(parameters, 'target_folder', 'value') flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value') if not flatten_results: target_dirs = os.path.splitdrive(target_folder)[1] flatten_results = 'false' if not os.path.exists(request['folder']): os.makedirs(request['folder']) # Query the index for results in groups of 25. result_count, response_index = task_utils.get_result_count(parameters) query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'copy_files') i = 0. headers = {'x-access-token': task_utils.get_security_token(request['owner'])} for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) input_items = task_utils.get_input_items(results.json()['response']['docs'], list_components=True) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) result = copy_files(input_items, target_folder, flatten_results, target_dirs) copied += result[0] errors += result[1] skipped += result[2] # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(request['folder'], '__report.json'), copied, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Remove tags. :param request: json as a dict. """ query = '' errors = 0 parameters = request['params'] archive_location = request['folder'] if not os.path.exists(archive_location): os.makedirs(archive_location) # Parameter values snapshot_name = task_utils.get_parameter_value(parameters, 'snapshot_name', 'value') data_folder = task_utils.get_parameter_value(parameters, 'data_folder', 'value') request_owner = request['owner'] result_count, response_index = task_utils.get_result_count(parameters) # Get the query index and query (if any). # query_index = task_utils.QueryIndex(parameters[response_index]) fq = '' if 'fq' in parameters[response_index]['query']: if isinstance(parameters[response_index]['query']['fq'], list): for q in parameters[response_index]['query']['fq']: if '{!tag=' in q: q = q.split('}')[1] fq += q + ' AND ' fq = fq.strip(' AND ') else: # Replace spaces with %20 & remove \\ to avoid HTTP Error 400. fq += '&fq={0}'.format(parameters[response_index]['query']['fq'].replace("\\", "")) fq = fq.replace(' ', '%20') # fq = query_index.get_fq() if fq: query = fq.replace('&fq=', '') if query: result = create_snapshot(snapshot_name, request_owner, query) else: result = create_snapshot(snapshot_name, request_owner) if not result[0]: errors += 1 errors_reasons[snapshot_name] = result[1] # Update state if necessary. if errors > 0: status_writer.send_state(status.STAT_FAILED) else: status_writer.send_status(result[1]) result_name = '' while not result_name: file_path = os.path.join(data_folder, 'backup', '*{0}.zip'.format(snapshot_name)) try: result_name = os.path.basename(glob.glob(file_path)[0]) except IndexError: continue shutil.copyfile(os.path.join(data_folder, 'backup', result_name), os.path.join(archive_location, result_name)) task_utils.report(os.path.join(request['folder'], '__report.json'), 1, 0, errors, errors_details=errors_reasons)
def execute(request): """Delete items. :param request: json as a dict. """ query = '' errors = 0 parameters = request['params'] archive_location = request['folder'] if not os.path.exists(archive_location): os.makedirs(archive_location) # Parameter values delete_thumbs = task_utils.get_parameter_value( parameters, 'delete_thumbnails', 'value') or False delete_metadata = task_utils.get_parameter_value( parameters, 'delete_metadata', 'value') or False delete_layers = task_utils.get_parameter_value(parameters, 'delete_layers', 'value') or False request_owner = request['owner'] result_count, response_index = task_utils.get_result_count(parameters) fq = '' if 'fq' in parameters[response_index]['query']: if isinstance(parameters[response_index]['query']['fq'], list): for q in parameters[response_index]['query']['fq']: if '{!tag=' in q: q = q.split('}')[1] fq += q + ' AND ' fq = fq.strip(' AND ') else: # Replace spaces with %20 & remove \\ to avoid HTTP Error 400. fq = parameters[response_index]['query']['fq'].replace("\\", "") if 'q' in parameters[response_index]['query']: query = parameters[response_index]['query']['q'] result = delete_items(fq, query, delete_thumbs, delete_metadata, delete_layers, request_owner) if not result[0]: errors += 1 errors_reasons['delete_items'] = result[1] # Update state if necessary. if errors > 0: status_writer.send_state(status.STAT_FAILED) else: status_writer.send_status(result[1]) task_utils.report(os.path.join(request['folder'], '__report.json'), 1, 0, errors, errors_details=errors_reasons)
def execute(request): """Delete items. :param request: json as a dict. """ query = '' errors = 0 parameters = request['params'] archive_location = request['folder'] if not os.path.exists(archive_location): os.makedirs(archive_location) # Parameter values delete_thumbs = task_utils.get_parameter_value(parameters, 'delete_thumbnails', 'value') or False delete_metadata = task_utils.get_parameter_value(parameters, 'delete_metadata', 'value') or False delete_layers = task_utils.get_parameter_value(parameters, 'delete_layers', 'value') or False request_owner = request['owner'] result_count, response_index = task_utils.get_result_count(parameters) fq = '' if 'fq' in parameters[response_index]['query']: if isinstance(parameters[response_index]['query']['fq'], list): for q in parameters[response_index]['query']['fq']: if '{!tag=' in q: q = q.split('}')[1] fq += q + ' AND ' fq = fq.strip(' AND ') else: # Replace spaces with %20 & remove \\ to avoid HTTP Error 400. fq = parameters[response_index]['query']['fq'].replace("\\", "") if 'q' in parameters[response_index]['query']: query = parameters[response_index]['query']['q'] result = delete_items(fq, query, delete_thumbs, delete_metadata, delete_layers, request_owner) if not result[0]: errors += 1 errors_reasons['delete_items'] = result[1] # Update state if necessary. if errors > 0: status_writer.send_state(status.STAT_FAILED) else: status_writer.send_status(result[1]) task_utils.report(os.path.join(request['folder'], '__report.json'), 1, 0, errors, errors_details=errors_reasons)
def execute(request): """Creates a GeoPDF. :param request: json as a dict. """ added_to_map = 0 errors = 0 skipped = 0 parameters = request['params'] num_results, response_index = task_utils.get_result_count(parameters) docs = parameters[response_index]['response']['docs'] input_items = task_utils.get_input_items(docs) input_rows = collections.defaultdict(list) for doc in docs: if 'path' not in doc: input_rows[doc['name']].append(doc) if num_results > task_utils.CHUNK_SIZE: status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.') return map_template = task_utils.get_parameter_value(parameters, 'map_template', 'value') base_map = task_utils.get_parameter_value(parameters, 'base_map', 'value') map_title = task_utils.get_parameter_value(parameters, 'map_title', 'value') attribute_setting = task_utils.get_parameter_value(parameters, 'attribute_settings', 'value') author = task_utils.get_parameter_value(parameters, 'map_author', 'value') output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value') if not output_file_name: output_file_name = 'output_pdf' try: map_view = task_utils.get_parameter_value(parameters, 'map_view', 'extent') except KeyError: map_view = None pass temp_folder = os.path.join(request['folder'], 'temp') if not os.path.exists(temp_folder): os.makedirs(temp_folder) if base_map == 'NONE': base_layer = None else: base_layer = arcpy.mapping.Layer( os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'basemaps', '{0}.lyr'.format(base_map))) mxd_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'frame', map_template) mxd = arcpy.mapping.MapDocument(mxd_path) data_frame = arcpy.mapping.ListDataFrames(mxd)[0] layers = [] all_layers = [] if input_rows: for name, rows in input_rows.iteritems(): for row in rows: try: name = arcpy.CreateUniqueName(name, 'in_memory') # Create the geometry. geo_json = row['[geo]'] geom = arcpy.AsShape(geo_json) arcpy.CopyFeatures_management(geom, name) feature_layer = arcpy.MakeFeatureLayer_management( name, os.path.basename(name)) layer_file = arcpy.SaveToLayerFile_management( feature_layer, os.path.join(temp_folder, '{0}.lyr'.format(os.path.basename(name)))) layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) all_layers.append( arcpy.mapping.Layer(layer_file.getOutput(0))) added_to_map += 1 except KeyError: skipped += 1 skipped_reasons[name] = 'No geographic information' continue for i, item in enumerate(input_items, 1): try: # Is the item a mxd data frame. map_frame_name = task_utils.get_data_frame_name(item) if map_frame_name: item = item.split('|')[0].strip() dsc = arcpy.Describe(item) if dsc.dataType == 'Layer': layers.append(arcpy.mapping.Layer(dsc.catalogPath)) elif dsc.dataType == 'FeatureClass' or dsc.dataType == 'ShapeFile': if os.path.basename(item) in [l.name for l in all_layers]: layer_name = '{0}_{1}'.format(os.path.basename(item), i) else: layer_name = os.path.basename(item) feature_layer = arcpy.MakeFeatureLayer_management( item, layer_name) layer_file = arcpy.SaveToLayerFile_management( feature_layer, os.path.join(temp_folder, '{0}.lyr'.format(layer_name))) layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) elif dsc.dataType == 'FeatureDataset': arcpy.env.workspace = item for fc in arcpy.ListFeatureClasses(): layer_file = arcpy.SaveToLayerFile_management( arcpy.MakeFeatureLayer_management( fc, '{0}_{1}'.format(fc, i)), os.path.join(temp_folder, '{0}_{1}.lyr'.format(fc, i))) layer = arcpy.mapping.Layer(layer_file.getOutput(0)) layer.name = fc layers.append(layer) all_layers.append(layer) elif dsc.dataType == 'RasterDataset': if os.path.basename(item) in [l.name for l in all_layers]: layer_name = '{0}_{1}'.format(os.path.basename(item), i) else: layer_name = os.path.basename(item) raster_layer = arcpy.MakeRasterLayer_management( item, layer_name) layer_file = arcpy.SaveToLayerFile_management( raster_layer, os.path.join(temp_folder, '{0}.lyr'.format(layer_name))) layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) elif dsc.catalogPath.endswith('.kml') or dsc.catalogPath.endswith( '.kmz'): if not os.path.splitext(dsc.name)[0] in layers: name = os.path.splitext(dsc.name)[0] else: name = '{0}_{1}'.format(os.path.splitext(dsc.name)[0], i) arcpy.KMLToLayer_conversion(dsc.catalogPath, temp_folder, name) layers.append( arcpy.mapping.Layer( os.path.join(temp_folder, '{0}.lyr'.format(name)))) all_layers.append( arcpy.mapping.Layer( os.path.join(temp_folder, '{0}.lyr'.format(name)))) elif dsc.dataType == 'MapDocument': input_mxd = arcpy.mapping.MapDocument(item) if map_frame_name: df = arcpy.mapping.ListDataFrames(input_mxd, map_frame_name)[0] layers = arcpy.mapping.ListLayers(input_mxd, data_frame=df) else: layers = arcpy.mapping.ListLayers(input_mxd) if layers: for layer in layers: status_writer.send_status( _('Adding layer {0}...').format(layer.name)) arcpy.mapping.AddLayer(data_frame, layer) added_to_map += 1 layers = [] else: status_writer.send_status( _('Invalid input type: {0}').format(item)) skipped_reasons[item] = 'Invalid input type' skipped += 1 except Exception as ex: status_writer.send_status(_('FAIL: {0}').format(repr(ex))) errors += 1 errors_reasons[item] = repr(ex) pass if map_view: extent = map_view.split(' ') new_extent = data_frame.extent new_extent.XMin, new_extent.YMin = float(extent[0]), float(extent[1]) new_extent.XMax, new_extent.YMax = float(extent[2]), float(extent[3]) data_frame.extent = new_extent else: data_frame.zoomToSelectedFeatures() # Update text elements in map template. date_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'date') if date_element: date_element[0].text = 'Date: {0}'.format(task_utils.get_local_date()) title_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'title') if title_element: title_element[0].text = map_title author_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'author') if author_element: author_element[0].text = '{0} {1}'.format(author_element[0].text, author) if map_template in ('ANSI_D_LND.mxd', 'ANSI_E_LND.mxd'): coord_elements = arcpy.mapping.ListLayoutElements( mxd, 'TEXT_ELEMENT', 'x*') coord_elements += arcpy.mapping.ListLayoutElements( mxd, 'TEXT_ELEMENT', 'y*') if coord_elements: for e in coord_elements: new_text = e.text if e.name == 'xmin': dms = task_utils.dd_to_dms(data_frame.extent.XMin) if data_frame.extent.XMin > 0: new_text = new_text.replace('W', 'E') elif e.name == 'xmax': dms = task_utils.dd_to_dms(data_frame.extent.XMax) if data_frame.extent.XMax > 0: new_text = new_text.replace('W', 'E') elif e.name == 'ymin': dms = task_utils.dd_to_dms(data_frame.extent.YMin) if data_frame.extent.YMin < 0: new_text = new_text.replace('N', 'S') elif e.name == 'ymax': if data_frame.extent.YMax < 0: new_text = new_text.replace('N', 'S') dms = task_utils.dd_to_dms(data_frame.extent.YMax) new_text = new_text.replace('d', str(dms[0])) new_text = new_text.replace('m', str(dms[1])) new_text = new_text.replace('s', str(dms[2])) e.text = new_text # Do this now so it does not affect zoom level or extent. if base_layer: status_writer.send_status(_('Adding basemap {0}...').format(base_map)) arcpy.mapping.AddLayer(data_frame, base_layer, 'BOTTOM') if added_to_map > 0: status_writer.send_status(_('Exporting to PDF...')) arcpy.mapping.ExportToPDF(mxd, os.path.join( request['folder'], '{0}.pdf'.format(output_file_name)), layers_attributes=attribute_setting) # Create a thumbnail size PNG of the mxd. task_utils.make_thumbnail( mxd, os.path.join(request['folder'], '_thumb.png'), False) else: status_writer.send_state(status.STAT_FAILED, _('No results can be exported to PDF')) task_utils.report(os.path.join(request['folder'], '__report.json'), added_to_map, skipped, skipped_details=skipped_reasons) return # Update state if necessary. if skipped > 0 or errors > 0: status_writer.send_state( status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(request['folder'], '__report.json'), added_to_map, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Converts each input dataset to kml (.kmz). :param request: json as a dict. """ converted = 0 skipped = 0 errors = 0 global result_count parameters = request['params'] out_workspace = os.path.join(request['folder'], 'temp') if not os.path.exists(out_workspace): os.makedirs(out_workspace) # Get the boundary box extent for input to KML tools. extent = '' try: try: ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt') if ext: sr = task_utils.get_spatial_reference("4326") extent = task_utils.from_wkt(ext, sr) except KeyError: ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'feature') if ext: extent = arcpy.Describe(ext).extent except KeyError: pass # Get the output file name. output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value') if not output_file_name: output_file_name = 'kml_results' result_count, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'convert_to_kml') headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } for group in groups: if fq: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) docs = results.json()['response']['docs'] input_items = task_utils.get_input_items(docs) if not input_items: input_items = task_utils.get_input_items( parameters[response_index]['response']['docs']) input_rows = collections.defaultdict(list) for doc in docs: if 'path' not in doc: input_rows[doc['name']].append(doc) if input_rows: result = convert_to_kml(input_rows, out_workspace, extent) converted += result[0] errors += result[1] skipped += result[2] if input_items: result = convert_to_kml(input_items, out_workspace, extent) converted += result[0] errors += result[1] skipped += result[2] if not input_items and not input_rows: status_writer.send_state( status.STAT_FAILED, _('No items to process. Check if items exist.')) return # Zip up kmz files if more than one. if converted > 1: status_writer.send_status("Converted: {}".format(converted)) zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name)) shutil.move( zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) shutil.copy2( os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', '_thumb.png'), request['folder']) elif converted == 1: try: kml_file = glob.glob(os.path.join(out_workspace, '*.kmz'))[0] tmp_lyr = arcpy.KMLToLayer_conversion(kml_file, out_workspace, 'kml_layer') task_utils.make_thumbnail( tmp_lyr.getOutput(0), os.path.join(request['folder'], '_thumb.png')) except arcpy.ExecuteError: pass shutil.move( kml_file, os.path.join(request['folder'], os.path.basename(kml_file))) # Update state if necessary. if skipped > 0 or errors > 0: status_writer.send_state( status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), converted, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Clips selected search results using the clip geometry. :param request: json as a dict. """ clipped = 0 errors = 0 skipped = 0 global result_count parameters = request['params'] # Retrieve clip geometry. try: clip_area = task_utils.get_parameter_value(parameters, 'clip_geometry', 'wkt') if not clip_area: clip_area = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))' except KeyError: clip_area = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))' # Retrieve the coordinate system code. out_coordinate_system = int(task_utils.get_parameter_value(parameters, 'output_projection', 'code')) # Retrieve the output format, create mxd parameter and output file name values. out_format = task_utils.get_parameter_value(parameters, 'output_format', 'value') create_mxd = task_utils.get_parameter_value(parameters, 'create_mxd', 'value') output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value') if not output_file_name: output_file_name = 'clip_results' # Create the temporary workspace if clip_feature_class: out_workspace = os.path.join(request['folder'], 'temp') if not os.path.exists(out_workspace): os.makedirs(out_workspace) # Set the output coordinate system. if not out_coordinate_system == 0: # Same as Input out_sr = task_utils.get_spatial_reference(out_coordinate_system) arcpy.env.outputCoordinateSystem = out_sr # Create the clip polygon geometry object in WGS84 projection. gcs_sr = task_utils.get_spatial_reference(4326) gcs_clip_poly = task_utils.from_wkt(clip_area, gcs_sr) if not gcs_clip_poly.area > 0: gcs_clip_poly = task_utils.from_wkt('POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))', gcs_sr) # Set the output workspace. status_writer.send_status(_('Setting the output workspace...')) if not out_format == 'SHP': out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0) arcpy.env.workspace = out_workspace # Query the index for results in groups of 25. headers = {'x-access-token': task_utils.get_security_token(request['owner'])} result_count, response_index = task_utils.get_result_count(parameters) query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'clip_data') for group in groups: if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) # docs = eval(results.read().replace('false', 'False').replace('true', 'True').replace('null', 'None'))['response']['docs'] docs = results.json()['response']['docs'] input_items = task_utils.get_input_items(docs) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) input_rows = collections.defaultdict(list) for doc in docs: if 'path' not in doc: input_rows[doc['name']].append(doc) if input_rows: result = clip_data(input_rows, out_workspace, out_coordinate_system, gcs_sr, gcs_clip_poly, out_format) clipped += result[0] errors += result[1] skipped += result[2] if input_items: result = clip_data(input_items, out_workspace, out_coordinate_system, gcs_sr, gcs_clip_poly, out_format) clipped += result[0] errors += result[1] skipped += result[2] if not input_items and not input_rows: status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.')) return if arcpy.env.workspace.endswith('.gdb'): out_workspace = os.path.dirname(arcpy.env.workspace) if clipped > 0: try: if out_format == 'MPK': mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd') mxd = task_utils.create_mxd(out_workspace, mxd_template, 'output') status_writer.send_status(_("Packaging results...")) task_utils.create_mpk(out_workspace, mxd, files_to_package) shutil.move(os.path.join(out_workspace, 'output.mpk'), os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name))) elif out_format == 'LPK': status_writer.send_status(_("Packaging results...")) task_utils.create_lpk(out_workspace,output_file_name, files_to_package) elif out_format == 'KML': task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb")) arcpy.env.workspace = '' try: arcpy.Delete_management(os.path.join(out_workspace, "output.gdb")) except arcpy.ExecuteError: pass zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) else: if create_mxd: mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd') task_utils.create_mxd(out_workspace, mxd_template, 'output') zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) except arcpy.ExecuteError as ee: status_writer.send_state(status.STAT_FAILED, _(ee)) sys.exit(1) else: status_writer.send_state(status.STAT_FAILED, _('No output created. Zero inputs were clipped.')) # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), clipped, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Builds raster pyramids for input raster datasets. :param request: json as a dict. """ processed = 0 skipped = 0 parameters = request['params'] # Get the extent for for which to use to calculate statistics. extent = '' try: try: ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt') if ext: sr = task_utils.get_spatial_reference("4326") extent = task_utils.from_wkt(ext, sr) except KeyError: ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'feature') if ext: extent = arcpy.Describe(ext).extent except KeyError: pass horizontal_skip_factor = task_utils.get_parameter_value(parameters, 'horizontal_skip_factor', 'value') vertical_skip_factor = task_utils.get_parameter_value(parameters, 'vertical_skip_factor', 'value') ignore_pixel_values = task_utils.get_parameter_value(parameters, 'ignore_pixel_values', 'value') # Create the task folder to hold report files. task_folder = request['folder'] if not os.path.exists(task_folder): os.makedirs(task_folder) headers = {'x-access-token': task_utils.get_security_token(request['owner'])} num_results, response_index = task_utils.get_result_count(parameters) if num_results > task_utils.CHUNK_SIZE: # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'calculate_raster_statistics') i = 0. for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) # input_items = task_utils.get_input_items(eval(results.read().replace('false', 'False').replace('true', 'True'))['response']['docs']) input_items = task_utils.get_input_items(results.json()['response']['docs']) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) result = calculate_raster_statistics(input_items, extent, horizontal_skip_factor, vertical_skip_factor, ignore_pixel_values) processed += result[0] skipped += result[1] status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'calculate_raster_statistics') else: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) processed, skipped = calculate_raster_statistics(input_items, extent, horizontal_skip_factor, vertical_skip_factor, ignore_pixel_values, True) # Update state if necessary. if skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), processed, skipped, skipped_details=skipped_reasons)
def execute(request): """Converts each input dataset to kml (.kmz). :param request: json as a dict. """ converted = 0 skipped = 0 errors = 0 global result_count parameters = request['params'] out_workspace = os.path.join(request['folder'], 'temp') if not os.path.exists(out_workspace): os.makedirs(out_workspace) # Get the boundary box extent for input to KML tools. extent = '' try: try: ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt') if ext: sr = task_utils.get_spatial_reference("4326") extent = task_utils.from_wkt(ext, sr) except KeyError: ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'feature') if ext: extent = arcpy.Describe(ext).extent except KeyError: pass # Get the output file name. output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value') if not output_file_name: output_file_name = 'kml_results' result_count, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'convert_to_kml') headers = {'x-access-token': task_utils.get_security_token(request['owner'])} for group in groups: if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) docs = results.json()['response']['docs'] input_items = task_utils.get_input_items(docs) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) input_rows = collections.defaultdict(list) for doc in docs: if 'path' not in doc: input_rows[doc['name']].append(doc) if input_rows: result = convert_to_kml(input_rows, out_workspace, extent) converted += result[0] errors += result[1] skipped += result[2] if input_items: result = convert_to_kml(input_items, out_workspace, extent) converted += result[0] errors += result[1] skipped += result[2] if not input_items and not input_rows: status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.')) return # Zip up kmz files if more than one. if converted > 1: status_writer.send_status("Converted: {}".format(converted)) zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) shutil.copy2(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', '_thumb.png'), request['folder']) elif converted == 1: try: kml_file = glob.glob(os.path.join(out_workspace, '*.kmz'))[0] tmp_lyr = arcpy.KMLToLayer_conversion(kml_file, out_workspace, 'kml_layer') task_utils.make_thumbnail(tmp_lyr.getOutput(0), os.path.join(request['folder'], '_thumb.png')) except arcpy.ExecuteError: pass shutil.move(kml_file, os.path.join(request['folder'], os.path.basename(kml_file))) # Update state if necessary. if skipped > 0 or errors > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), converted, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Builds raster pyramids for input raster datasets. :param request: json as a dict. """ processed = 0 skipped = 0 parameters = request['params'] # Get the extent for for which to use to calculate statistics. extent = '' try: try: ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt') if ext: sr = task_utils.get_spatial_reference("4326") extent = task_utils.from_wkt(ext, sr) except KeyError: ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'feature') if ext: extent = arcpy.Describe(ext).extent except KeyError: pass horizontal_skip_factor = task_utils.get_parameter_value( parameters, 'horizontal_skip_factor', 'value') vertical_skip_factor = task_utils.get_parameter_value( parameters, 'vertical_skip_factor', 'value') ignore_pixel_values = task_utils.get_parameter_value( parameters, 'ignore_pixel_values', 'value') # Create the task folder to hold report files. task_folder = request['folder'] if not os.path.exists(task_folder): os.makedirs(task_folder) headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } num_results, response_index = task_utils.get_result_count(parameters) if num_results > task_utils.CHUNK_SIZE: # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper( list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'calculate_raster_statistics') i = 0. for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "&rows={0}&start={1}".format( task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) elif 'ids' in parameters[response_index]: results = requests.get( query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format( task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) input_items = task_utils.get_input_items( results.json()['response']['docs']) if not input_items: input_items = task_utils.get_input_items( parameters[response_index]['response']['docs']) result = calculate_raster_statistics(input_items, extent, horizontal_skip_factor, vertical_skip_factor, ignore_pixel_values) processed += result[0] skipped += result[1] status_writer.send_percent( i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'calculate_raster_statistics') else: input_items = task_utils.get_input_items( parameters[response_index]['response']['docs']) processed, skipped = calculate_raster_statistics( input_items, extent, horizontal_skip_factor, vertical_skip_factor, ignore_pixel_values, True) # Update state if necessary. if skipped > 0: status_writer.send_state( status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), processed, skipped, skipped_details=skipped_reasons)
def execute(request): """Deletes files. :param request: json as a dict """ errors_reasons = {} errors = 0 published = 0 app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) parameters = request['params'] num_results, response_index = task_utils.get_result_count(parameters) input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) if num_results > task_utils.CHUNK_SIZE: status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.') return url = task_utils.get_parameter_value(parameters, 'url', 'value') username = task_utils.get_parameter_value(parameters, 'username', 'value') password = task_utils.get_parameter_value(parameters, 'password', 'value') service_name = task_utils.get_parameter_value(parameters, 'service_name', 'value') folder_name = task_utils.get_parameter_value(parameters, 'folder_name', 'value') request_folder = os.path.join(request['folder'], 'temp') if not os.path.exists(request_folder): os.makedirs(request_folder) map_template = os.path.join(request_folder, 'output.mxd') shutil.copyfile(os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'), map_template) for item in input_items: try: # Code required because of an Esri bug - cannot describe a map package (raises IOError). if item.endswith('.mpk'): status_writer.send_status(_('Extracting: {0}').format(item)) arcpy.ExtractPackage_management(item, request_folder) pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0]) mxd_file = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.mxd')[0]) mxd = arcpy.mapping.MapDocument(mxd_file) create_service(request_folder, mxd, url, username, password, service_name, folder_name) else: data_type = arcpy.Describe(item).dataType if data_type == 'MapDocument': mxd = arcpy.mapping.MapDocument(item) create_service(request_folder, mxd, url, username, password, service_name, folder_name) elif data_type == 'Layer': if item.endswith('.lpk'): status_writer.send_status(_('Extracting: {0}').format(item)) arcpy.ExtractPackage_management(item, request_folder) pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0]) item = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.lyr')[0]) layer = arcpy.mapping.Layer(item) mxd = arcpy.mapping.MapDocument(map_template) mxd.description = layer.name mxd.tags = layer.name mxd.save() data_frame = arcpy.mapping.ListDataFrames(mxd)[0] arcpy.mapping.AddLayer(data_frame, layer) mxd.save() create_service(request_folder, mxd, url, username, password, service_name, folder_name) elif data_type in ('FeatureClass', 'ShapeFile', 'RasterDataset'): if data_type == 'RasterDataset': arcpy.MakeRasterLayer_management(item, os.path.basename(item)) else: arcpy.MakeFeatureLayer_management(item, os.path.basename(item)) layer = arcpy.mapping.Layer(os.path.basename(item)) mxd = arcpy.mapping.MapDocument(map_template) mxd.description = layer.name mxd.tags = layer.name mxd.save() data_frame = arcpy.mapping.ListDataFrames(mxd)[0] arcpy.mapping.AddLayer(data_frame, layer) mxd.save() create_service(request_folder, mxd, url, username, password, service_name, folder_name) published += 1 except task_utils.AnalyzeServiceException as ase: status_writer.send_state(status.STAT_FAILED, _(ase)) errors_reasons[item] = repr(ase) errors += 1 except requests.RequestException as re: status_writer.send_state(status.STAT_FAILED, _(re)) errors_reasons[item] = repr(re) errors += 1 except task_utils.PublishException as pe: status_writer.send_state(status.STAT_FAILED, _(pe)) errors_reasons[item] = repr(pe) errors += 1 except arcpy.ExecuteError as ee: status_writer.send_state(status.STAT_FAILED, _(ee)) errors_reasons[item] = repr(ee) errors += 1 except Exception as ex: status_writer.send_state(status.STAT_FAILED, _(ex)) errors_reasons[item] = repr(ex) errors += 1 finally: task_utils.report(os.path.join(request['folder'], '__report.json'), published, 0, errors, errors_reasons)
def execute(request): """Replace the workspace path for layer files and map document layers. :param request: json as a dict. """ updated = 0 skipped = 0 parameters = request['params'] backup = task_utils.get_parameter_value(parameters, 'create_backup', 'value') old_data_source = task_utils.get_parameter_value(parameters, 'old_data_source', 'value').lower() new_data_source = task_utils.get_parameter_value(parameters, 'new_data_source', 'value') if not os.path.exists(request['folder']): os.makedirs(request['folder']) if not arcpy.Exists(new_data_source): status_writer.send_state(status.STAT_FAILED, _('{0} does not exist').format(new_data_source)) return if os.path.splitext(new_data_source)[1] not in ('.gdb', '.mdb', '.sde'): new_dataset = os.path.basename(new_data_source) dsc = arcpy.Describe(os.path.dirname(new_data_source)) else: dsc = arcpy.Describe(new_data_source) new_dataset = '' wks_type = 'NONE' if dsc.dataType == 'FeatureDataset': new_workspace = dsc.path wks_type = get_workspace_type(new_workspace) elif dsc.dataType == 'Workspace': new_workspace = dsc.catalogPath wks_type = get_workspace_type(new_workspace) elif dsc.dataType == 'Folder': dsc = arcpy.Describe(new_data_source) new_workspace = dsc.catalogPath if new_dataset.endswith('.shp'): wks_type = 'SHAPEFILE_WORKSPACE' new_dataset = new_dataset.rsplit('.shp')[0] else: if arcpy.Describe(new_data_source).dataType == 'RasterDataset': wks_type = 'RASTER_WORKSPACE' elif dsc.dataType == 'CadDrawingDataset': new_workspace = dsc.path wks_type = 'CAD_WORKSPACE' else: new_workspace = os.path.dirname(new_data_source) num_results, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. headers = {'x-access-token': task_utils.get_security_token(request['owner'])} query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}'.format(sys.argv[2].split('=')[1], '/select?&wt=json') fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'replace_data_source') i = 0. for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "{0}&rows={1}&start={2}".format(fl, task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers= headers) else: results = requests.get(query + "{0}&rows={1}&start={2}".format(fl, task_utils.CHUNK_SIZE, group[0]), headers=headers) input_items = task_utils.get_input_items(results.json()['response']['docs'], True) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) result = replace_data_source(input_items, old_data_source, new_workspace, new_dataset, wks_type, backup, headers) updated += result[0] skipped += result[1] status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'replace_data_source') # Update state if necessary. if skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), updated, skipped, skipped_details=skipped_reasons)
def execute(request): """Builds raster pyramids for input raster datasets. :param request: json as a dict. """ processed = 0 skipped = 0 parameters = request['params'] resampling_method = task_utils.get_parameter_value(parameters, 'resampling_method', 'value') # Advanced options compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value') compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value') # Create the task folder to hold report files. task_folder = request['folder'] if not os.path.exists(task_folder): os.makedirs(task_folder) headers = {'x-access-token': task_utils.get_security_token(request['owner'])} num_results, response_index = task_utils.get_result_count(parameters) if num_results > task_utils.CHUNK_SIZE: # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'build_raster_pyramids') i = 0. for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) input_items = task_utils.get_input_items(results.json()['response']['docs']) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) result = build_pyramids(input_items, compression_method, compression_quality, resampling_method) processed += result[0] skipped += result[1] status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'build_raster_pyramids') else: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) processed, skipped = build_pyramids(input_items, compression_method, compression_quality, resampling_method, True) # Update state if necessary. if skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), processed, skipped, skipped_details=skipped_reasons)
def execute(request): """Deletes files. :param request: json as a dict """ errors_reasons = {} errors = 0 published = 0 app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) parameters = request['params'] num_results, response_index = task_utils.get_result_count(parameters) input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) if num_results > task_utils.CHUNK_SIZE: status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.') return server_conn = task_utils.get_parameter_value(parameters, 'server_connection_path', 'value') service_name = task_utils.get_parameter_value(parameters, 'service_name', 'value') folder_name = task_utils.get_parameter_value(parameters, 'folder_name', 'value') if not server_conn: status_writer.send_state(status.STAT_FAILED, _('A server path is required')) return request_folder = os.path.join(request['folder'], 'temp') if not os.path.exists(request_folder): os.makedirs(request_folder) map_template = os.path.join(request_folder, 'output.mxd') shutil.copyfile(os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'), map_template) for item in input_items: try: # Code required because of an Esri bug - cannot describe a map package (raises IOError). if item.endswith('.mpk'): status_writer.send_status(_('Extracting: {0}').format(item)) arcpy.ExtractPackage_management(item, request_folder) pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0]) mxd_file = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.mxd')[0]) mxd = arcpy.mapping.MapDocument(mxd_file) create_service(request_folder, mxd, server_conn, service_name, folder_name) else: data_type = arcpy.Describe(item).dataType if data_type == 'MapDocument': mxd = arcpy.mapping.MapDocument(item) create_service(request_folder, mxd, server_conn, service_name, folder_name) elif data_type == 'Layer': if item.endswith('.lpk'): status_writer.send_status(_('Extracting: {0}').format(item)) arcpy.ExtractPackage_management(item, request_folder) pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0]) item = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.lyr')[0]) layer = arcpy.mapping.Layer(item) mxd = arcpy.mapping.MapDocument(map_template) mxd.description = layer.name mxd.tags = layer.name mxd.save() data_frame = arcpy.mapping.ListDataFrames(mxd)[0] arcpy.mapping.AddLayer(data_frame, layer) mxd.save() create_service(request_folder, mxd, server_conn, service_name, folder_name) elif data_type in ('FeatureClass', 'ShapeFile', 'RasterDataset'): if data_type == 'RasterDataset': arcpy.MakeRasterLayer_management(item, os.path.basename(item)) else: arcpy.MakeFeatureLayer_management(item, os.path.basename(item)) layer = arcpy.mapping.Layer(os.path.basename(item)) mxd = arcpy.mapping.MapDocument(map_template) mxd.title = layer.name data_frame = arcpy.mapping.ListDataFrames(mxd)[0] arcpy.mapping.AddLayer(data_frame, layer) mxd.save() create_service(request_folder, mxd, server_conn, service_name, folder_name) elif data_type in ('CadDrawingDataset', 'FeatureDataset'): arcpy.env.workspace = item mxd = arcpy.mapping.MapDocument(map_template) data_frame = arcpy.mapping.ListDataFrames(mxd)[0] for fc in arcpy.ListFeatureClasses(): dataset_name = os.path.splitext(os.path.basename(item))[0] l = arcpy.MakeFeatureLayer_management(fc, '{0}_{1}'.format(dataset_name, os.path.basename(fc))) arcpy.mapping.AddLayer(data_frame, l.getOutput(0)) mxd.save() arcpy.ResetEnvironments() create_service(request_folder, mxd, server_conn, service_name, folder_name) published += 1 except task_utils.AnalyzeServiceException as ex: errors += 1 errors_reasons[item] = repr(ex) status_writer.send_state(status.STAT_FAILED) except arcpy.ExecuteError as ex: errors += 1 errors_reasons[item] = repr(ex) status_writer.send_state(status.STAT_FAILED) except Exception as ex: errors += 1 errors_reasons[item] = repr(ex) status_writer.send_state(status.STAT_FAILED) finally: if errors: errors_reasons[item] = repr(ex) status_writer.send_status(_('FAIL: {0}').format(errors_reasons[item])) task_utils.report(os.path.join(request['folder'], '__report.json'), published, 0, errors, errors_reasons)
def execute(request): """Creates a GeoPDF. :param request: json as a dict. """ added_to_map = 0 errors = 0 skipped = 0 parameters = request["params"] num_results, response_index = task_utils.get_result_count(parameters) docs = parameters[response_index]["response"]["docs"] input_items = task_utils.get_input_items(docs) input_rows = collections.defaultdict(list) for doc in docs: if "path" not in doc: input_rows[doc["name"]].append(doc) if num_results > task_utils.CHUNK_SIZE: status_writer.send_state(status.STAT_FAILED, "Reduce results to 25 or less.") return map_template = task_utils.get_parameter_value(parameters, "map_template", "value") base_map = task_utils.get_parameter_value(parameters, "base_map", "value") map_title = task_utils.get_parameter_value(parameters, "map_title", "value") attribute_setting = task_utils.get_parameter_value(parameters, "attribute_settings", "value") author = task_utils.get_parameter_value(parameters, "map_author", "value") output_file_name = task_utils.get_parameter_value(parameters, "output_file_name", "value") if not output_file_name: output_file_name = "output_pdf" try: map_view = task_utils.get_parameter_value(parameters, "map_view", "extent") except KeyError: map_view = None pass temp_folder = os.path.join(request["folder"], "temp") if not os.path.exists(temp_folder): os.makedirs(temp_folder) if base_map == "NONE": base_layer = None else: base_layer = arcpy.mapping.Layer( os.path.join( os.path.dirname(os.path.dirname(__file__)), "supportfiles", "basemaps", "{0}.lyr".format(base_map) ) ) mxd_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "supportfiles", "frame", map_template) mxd = arcpy.mapping.MapDocument(mxd_path) data_frame = arcpy.mapping.ListDataFrames(mxd)[0] layers = [] all_layers = [] if input_rows: for name, rows in input_rows.iteritems(): for row in rows: try: name = arcpy.CreateUniqueName(name, "in_memory") # Create the geometry. geo_json = row["[geo]"] geom = arcpy.AsShape(geo_json) arcpy.CopyFeatures_management(geom, name) feature_layer = arcpy.MakeFeatureLayer_management(name, os.path.basename(name)) layer_file = arcpy.SaveToLayerFile_management( feature_layer, os.path.join(temp_folder, "{0}.lyr".format(os.path.basename(name))) ) layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) added_to_map += 1 except KeyError: skipped += 1 skipped_reasons[name] = "No geographic information" continue for i, item in enumerate(input_items, 1): try: # Is the item a mxd data frame. map_frame_name = task_utils.get_data_frame_name(item) if map_frame_name: item = item.split("|")[0].strip() dsc = arcpy.Describe(item) if dsc.dataType == "Layer": layers.append(arcpy.mapping.Layer(dsc.catalogPath)) elif dsc.dataType == "FeatureClass" or dsc.dataType == "ShapeFile": if os.path.basename(item) in [l.name for l in all_layers]: layer_name = "{0}_{1}".format(os.path.basename(item), i) else: layer_name = os.path.basename(item) feature_layer = arcpy.MakeFeatureLayer_management(item, layer_name) layer_file = arcpy.SaveToLayerFile_management( feature_layer, os.path.join(temp_folder, "{0}.lyr".format(layer_name)) ) layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) elif dsc.dataType == "FeatureDataset": arcpy.env.workspace = item for fc in arcpy.ListFeatureClasses(): layer_file = arcpy.SaveToLayerFile_management( arcpy.MakeFeatureLayer_management(fc, "{0}_{1}".format(fc, i)), os.path.join(temp_folder, "{0}_{1}.lyr".format(fc, i)), ) layer = arcpy.mapping.Layer(layer_file.getOutput(0)) layer.name = fc layers.append(layer) all_layers.append(layer) elif dsc.dataType == "RasterDataset": if os.path.basename(item) in [l.name for l in all_layers]: layer_name = "{0}_{1}".format(os.path.basename(item), i) else: layer_name = os.path.basename(item) raster_layer = arcpy.MakeRasterLayer_management(item, layer_name) layer_file = arcpy.SaveToLayerFile_management( raster_layer, os.path.join(temp_folder, "{0}.lyr".format(layer_name)) ) layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0))) elif dsc.catalogPath.endswith(".kml") or dsc.catalogPath.endswith(".kmz"): if not os.path.splitext(dsc.name)[0] in layers: name = os.path.splitext(dsc.name)[0] else: name = "{0}_{1}".format(os.path.splitext(dsc.name)[0], i) arcpy.KMLToLayer_conversion(dsc.catalogPath, temp_folder, name) layers.append(arcpy.mapping.Layer(os.path.join(temp_folder, "{0}.lyr".format(name)))) all_layers.append(arcpy.mapping.Layer(os.path.join(temp_folder, "{0}.lyr".format(name)))) elif dsc.dataType == "MapDocument": input_mxd = arcpy.mapping.MapDocument(item) if map_frame_name: df = arcpy.mapping.ListDataFrames(input_mxd, map_frame_name)[0] layers = arcpy.mapping.ListLayers(input_mxd, data_frame=df) else: layers = arcpy.mapping.ListLayers(input_mxd) if layers: for layer in layers: status_writer.send_status(_("Adding layer {0}...").format(layer.name)) arcpy.mapping.AddLayer(data_frame, layer) added_to_map += 1 layers = [] else: status_writer.send_status(_("Invalid input type: {0}").format(item)) skipped_reasons[item] = "Invalid input type" skipped += 1 except Exception as ex: status_writer.send_status(_("FAIL: {0}").format(repr(ex))) errors += 1 errors_reasons[item] = repr(ex) pass if map_view: extent = map_view.split(" ") new_extent = data_frame.extent new_extent.XMin, new_extent.YMin = float(extent[0]), float(extent[1]) new_extent.XMax, new_extent.YMax = float(extent[2]), float(extent[3]) data_frame.extent = new_extent else: data_frame.zoomToSelectedFeatures() # Update text elements in map template. date_element = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "date") if date_element: date_element[0].text = "Date: {0}".format(task_utils.get_local_date()) title_element = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "title") if title_element: title_element[0].text = map_title author_element = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "author") if author_element: author_element[0].text = "{0} {1}".format(author_element[0].text, author) if map_template in ("ANSI_D_LND.mxd", "ANSI_E_LND.mxd"): coord_elements = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "x*") coord_elements += arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "y*") if coord_elements: for e in coord_elements: new_text = e.text if e.name == "xmin": dms = task_utils.dd_to_dms(data_frame.extent.XMin) if data_frame.extent.XMin > 0: new_text = new_text.replace("W", "E") elif e.name == "xmax": dms = task_utils.dd_to_dms(data_frame.extent.XMax) if data_frame.extent.XMax > 0: new_text = new_text.replace("W", "E") elif e.name == "ymin": dms = task_utils.dd_to_dms(data_frame.extent.YMin) if data_frame.extent.YMin < 0: new_text = new_text.replace("N", "S") elif e.name == "ymax": if data_frame.extent.YMax < 0: new_text = new_text.replace("N", "S") dms = task_utils.dd_to_dms(data_frame.extent.YMax) new_text = new_text.replace("d", str(dms[0])) new_text = new_text.replace("m", str(dms[1])) new_text = new_text.replace("s", str(dms[2])) e.text = new_text # Do this now so it does not affect zoom level or extent. if base_layer: status_writer.send_status(_("Adding basemap {0}...").format(base_map)) arcpy.mapping.AddLayer(data_frame, base_layer, "BOTTOM") if added_to_map > 0: status_writer.send_status(_("Exporting to PDF...")) arcpy.mapping.ExportToPDF( mxd, os.path.join(request["folder"], "{0}.pdf".format(output_file_name)), layers_attributes=attribute_setting, ) # Create a thumbnail size PNG of the mxd. task_utils.make_thumbnail(mxd, os.path.join(request["folder"], "_thumb.png"), False) else: status_writer.send_state(status.STAT_FAILED, _("No results can be exported to PDF")) task_utils.report( os.path.join(request["folder"], "__report.json"), added_to_map, skipped, skipped_details=skipped_reasons ) return # Update state if necessary. if skipped > 0 or errors > 0: status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(skipped + errors)) task_utils.report( os.path.join(request["folder"], "__report.json"), added_to_map, skipped, errors, errors_reasons, skipped_reasons )
def execute(request): """Clips selected search results using the clip geometry. :param request: json as a dict. """ clipped = 0 errors = 0 skipped = 0 global result_count parameters = request['params'] # Retrieve the clip features. clip_features = task_utils.get_parameter_value(parameters, 'clip_features', 'value') # Retrieve the coordinate system code. out_coordinate_system = int(task_utils.get_parameter_value(parameters, 'output_projection', 'code')) # Retrieve the output format, create mxd and output file name parameter values. out_format = task_utils.get_parameter_value(parameters, 'output_format', 'value') create_mxd = task_utils.get_parameter_value(parameters, 'create_mxd', 'value') output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value') if not output_file_name: output_file_name = 'clip_results' # Create the temporary workspace if clip_feature_class: out_workspace = os.path.join(request['folder'], 'temp') if not os.path.exists(out_workspace): os.makedirs(out_workspace) # Set the output coordinate system. if not out_coordinate_system == 0: # Same as Input out_sr = task_utils.get_spatial_reference(out_coordinate_system) arcpy.env.outputCoordinateSystem = out_sr # Set the output workspace. status_writer.send_status(_('Setting the output workspace...')) if not out_format == 'SHP': out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0) arcpy.env.workspace = out_workspace # Query the index for results in groups of 25. headers = {'x-access-token': task_utils.get_security_token(request['owner'])} result_count, response_index = task_utils.get_result_count(parameters) query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl # Get the Clip features by id. id = clip_features['id'] clip_query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', "&fl=id,path,fullpath:[absolute],absolute_path:[absolute],[lyrFile],[geo]&q=id:{0}".format(id)) clip_result = requests.get(clip_query, headers=headers) clipper = clip_result.json()['response']['docs'][0] if 'absolute_path' in clipper and not clipper['absolute_path'].startswith('s3'): clip_features = clipper['absolute_path'] elif '[lyrFile]' in clipper: clip_features = clipper['[lyrFile]'] elif '[geo]' in clipper: clip_features = arcpy.AsShape(clipper['[geo]']).projectAs(arcpy.SpatialReference(4326)) elif 'absolute_path' in clipper and clipper['absolute_path'].startswith('s3'): base_name = os.path.basename(clipper['path']) temp_folder = tempfile.mkdtemp() if '[downloadURL]' in clipper: download = os.path.join(temp_folder, os.path.basename(clipper['[downloadURL]'])) response = requests.get(clipper['[downloadURL]']) with open(download, 'wb') as fp: fp.write(response.content) if download.endswith('.zip'): zip = zipfile.ZipFile(download) zip.extractall(temp_folder) clip_features = os.path.join(temp_folder, base_name) else: clip_features = download else: bbox = clipper['bbox'].split() extent = arcpy.Extent(*bbox) pt_array = arcpy.Array([extent.lowerLeft, extent.upperLeft, extent.upperRight, extent.lowerRight]) clip_features = arcpy.Polygon(pt_array, 4326) query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'clip_data') for group in groups: if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) docs = results.json()['response']['docs'] input_items = task_utils.get_input_items(docs) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) input_rows = collections.defaultdict(list) for doc in docs: if 'path' not in doc: input_rows[doc['name']].append(doc) if input_rows: result = clip_data(input_rows, out_workspace, clip_features, out_format) clipped += result[0] errors += result[1] skipped += result[2] if input_items: result = clip_data(input_items, out_workspace, clip_features, out_format) clipped += result[0] errors += result[1] skipped += result[2] if not input_items and not input_rows: status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.')) return if arcpy.env.workspace.endswith('.gdb'): out_workspace = os.path.dirname(arcpy.env.workspace) if clipped > 0: try: if out_format == 'MPK': mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd') mxd = task_utils.create_mxd(out_workspace, mxd_template, 'output') status_writer.send_status(_("Packaging results...")) task_utils.create_mpk(out_workspace, mxd, files_to_package) shutil.move(os.path.join(out_workspace, 'output.mpk'), os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name))) elif out_format == 'LPK': status_writer.send_status(_("Packaging results...")) task_utils.create_lpk(out_workspace, output_file_name, files_to_package) elif out_format == 'KML': task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb")) arcpy.env.workspace = '' arcpy.RefreshCatalog(os.path.join(out_workspace, "output.gdb")) try: arcpy.Delete_management(os.path.join(out_workspace, "output.gdb")) except arcpy.ExecuteError: pass zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) else: if create_mxd: mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd') task_utils.create_mxd(out_workspace, mxd_template, 'output') zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) except arcpy.ExecuteError as ee: status_writer.send_state(status.STAT_FAILED, _(ee)) sys.exit(1) else: status_writer.send_state(status.STAT_FAILED, _('No output created. Zero inputs were clipped.')) # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), clipped, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Remove tags. :param request: json as a dict. """ query = '' errors = 0 parameters = request['params'] archive_location = request['folder'] if not os.path.exists(archive_location): os.makedirs(archive_location) # Parameter values search_action = task_utils.get_parameter_value(parameters, 'search_action', 'value') search_name = task_utils.get_parameter_value(parameters, 'saved_searches', 'value') search_name = eval(search_name[0])['text'] groups = task_utils.get_parameter_value(parameters, 'groups', 'value') request_owner = request['owner'] result_count, response_index = task_utils.get_result_count(parameters) fq = '/' if 'fq' in parameters[response_index]['query']: if isinstance(parameters[response_index]['query']['fq'], list): for q in parameters[response_index]['query']['fq']: if '{!tag=' in q: q = q.split('}')[1] if ':' in q: facet = q.split(':')[0] value = q.split(':')[1] if '(' in value: value = value.replace('(', '').replace(')', '') value = urllib.urlencode({'val': value.replace('"', '')}) value = value.split('val=')[1] facet2 = 'f.{0}='.format(facet) q = '{0}{1}'.format(facet2, value) #q.replace(facet + ':', facet2) fq += '{0}/'.format(q).replace('"', '') else: # Replace spaces with %20 & remove \\ to avoid HTTP Error 400. fq += '&fq={0}'.format( parameters[response_index]['query']['fq'].replace("\\", "")) if '{!tag=' in fq: fq = fq.split('}')[1] if ':' in fq: if fq.startswith('/&fq='): fq = fq.replace('/&fq=', '') facet = fq.split(':')[0] value = fq.split(':')[1].replace('(', '').replace(')', '').replace( '"', '') if 'place' not in facet: value = urllib.urlencode({'val': value}).split('val=')[1] facet2 = 'f.{0}='.format(facet) if '(' in value: fq = '' if value.split(' '): for v in value.split(' '): fq += (facet2 + v.replace('(', '').replace(')', '') + '/').replace(':', '') else: value = urllib.urlencode({'val': value}).split('val=')[1] fq = '{0}{1}'.format(facet2, value) if '{! place.op=' in fq: relop = find_between(fq, 'place.op=', '}') fq = fq.replace('}', '').replace('{', '') fq = fq.replace('! place.op={0}'.format(relop), '/place.op={0}/'.format(relop)) fq = fq.replace('place:', 'place=') fq = fq.replace('&fq=', '') hasQ = False if 'q' in parameters[response_index]['query']: query = parameters[response_index]['query']['q'] hasQ = True if fq: query += '/' if fq: if fq.startswith('/place'): query += fq.replace('"', '') elif '!tag' in query and 'OR' in query: # e.g. "path": "/q=id:(92cdd06e01761c4c d9841b2f59b8a326) OR format:(application%2Fvnd.esri.shapefile)" q = query.split('}')[1].replace('))/', '').replace('(', '').replace('(', '') q = urllib.urlencode({'val': q.split(':')[1]}).split('val=')[1] query = query.split(' OR ')[0] + ' OR ' + q else: if fq.startswith('f.//'): fq = fq.replace('f.//', '/').replace('"', '') if ' place.id' in fq: fq = fq.replace(' place.id', '/place.id').replace('"', '') if '{! place.op=' in fq: relop = find_between(fq, 'place.op=', '}') fq = fq.replace('}', '').replace('{', '') fq = fq.replace('! place.op={0}'.format(relop), '/place.op={0}/'.format(relop)).replace( '"', '') query += fq.rstrip('/') query = query.replace('f./', '') query = query.replace('&fq=', '') if search_action == 'Overwrite an existing saved search': delete_result = delete_saved_search(search_name, request_owner) if not delete_result[0]: status_writer.send_state(status.STAT_FAILED, delete_result[1]) return if query: result = create_saved_search(search_name, groups, request_owner, query, hasQ) else: result = create_saved_search(search_name, groups, request_owner, "", hasQ) if not result[0]: errors += 1 errors_reasons[search_name] = result[1] # Update state if necessary. if errors > 0: status_writer.send_state(status.STAT_FAILED, result[1]) else: status_writer.send_status(result[1]) task_utils.report(os.path.join(request['folder'], '__report.json'), 1, 0, errors, errors_details=errors_reasons)
def execute(request): """Copies data to an existing geodatabase or feature dataset. :param request: json as a dict. """ added = 0 errors = 0 skipped = 0 global result_count parameters = request["params"] # Get the target workspace location. out_gdb = task_utils.get_parameter_value(parameters, "target_workspace", "value") # Retrieve the coordinate system code. out_coordinate_system = task_utils.get_parameter_value(parameters, "output_projection", "code") if not out_coordinate_system == "0": # Same as Input arcpy.env.outputCoordinateSystem = task_utils.get_spatial_reference(out_coordinate_system) task_folder = request["folder"] if not os.path.exists(task_folder): os.makedirs(task_folder) # Check if the geodatabase exists or if it is a feature dataset. is_fds = False if not os.path.exists(out_gdb): if out_gdb.endswith(".gdb"): arcpy.CreateFileGDB_management(os.path.dirname(out_gdb), os.path.basename(out_gdb)) status_writer.send_status(_("Created output workspace: {0}").format(out_gdb)) elif out_gdb.endswith(".mdb"): arcpy.CreatePersonalGDB_management(os.path.dirname(out_gdb), os.path.basename(out_gdb)) status_writer.send_status(_("Created output workspace: {0}").format(out_gdb)) elif out_gdb.endswith(".sde"): status_writer.send_state(status.STAT_FAILED, _("{0} does not exist").format(out_gdb)) return else: # Possible feature dataset. is_fds = is_feature_dataset(out_gdb) if not is_fds: if os.path.dirname(out_gdb).endswith(".gdb"): if not os.path.exists(os.path.dirname(out_gdb)): arcpy.CreateFileGDB_management( os.path.dirname(os.path.dirname(out_gdb)), os.path.basename(os.path.dirname(out_gdb)) ) arcpy.CreateFeatureDataset_management(os.path.dirname(out_gdb), os.path.basename(out_gdb)) elif os.path.dirname(out_gdb).endswith(".mdb"): if not os.path.exists(os.path.dirname(out_gdb)): arcpy.CreatePersonalGDB_management( os.path.dirname(os.path.dirname(out_gdb)), os.path.basename(os.path.dirname(out_gdb)) ) arcpy.CreateFeatureDataset_management(os.path.dirname(out_gdb), os.path.basename(out_gdb)) status_writer.send_status(_("Setting the output workspace...")) arcpy.env.workspace = out_gdb headers = {"x-access-token": task_utils.get_security_token(request["owner"])} result_count, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = "{0}{1}{2}".format(sys.argv[2].split("=")[1], "/select?&wt=json", fl) # query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "") query += fq elif "ids" in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]["ids"]), task_utils.CHUNK_SIZE, "") else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "") status_writer.send_percent(0.0, _("Starting to process..."), "add_to_geodatabase") for group in groups: if fq: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers ) elif "ids" in parameters[response_index]: results = requests.get(query + "{0}&ids={1}".format(fl, ",".join(group)), headers=headers) else: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers ) docs = results.json()["response"]["docs"] input_items = task_utils.get_input_items(docs) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]["response"]["docs"]) input_rows = collections.defaultdict(list) for doc in docs: if "path" not in doc: input_rows[doc["title"]].append(doc) if input_rows: result = add_to_geodatabase(input_rows, out_gdb, is_fds) added += result[0] errors += result[1] skipped += result[2] if input_items: result = add_to_geodatabase(input_items, out_gdb, is_fds) added += result[0] errors += result[1] skipped += result[2] if not input_items and not input_rows: status_writer.send_state(status.STAT_FAILED, _("No items to process. Check if items exist.")) return # Update state if necessary. if skipped > 0 or errors > 0: status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(skipped + errors)) task_utils.report( os.path.join(task_folder, "__report.json"), added, skipped, errors, errors_reasons, skipped_reasons )
def execute(request): """Report duplicate items. :param request: json as a dict. """ if not os.path.exists(request['folder']): os.makedirs(request['folder']) date_str = str(datetime.date.today()) report_file = os.path.join(request['folder'], 'Duplicates_{0}.csv'.format(date_str)) request_owner = request['owner'] headers = {'x-access-token': task_utils.get_security_token(request_owner)} fields = list(task_utils.get_parameter_value(request['params'], 'fields', 'value')) required_fields = ("name", "title", "bytes", "id", "format", "absolute_path") [fields.remove(f) for f in required_fields if f in fields] fields_str = "" if fields: fields_str = ",".join(fields) voyager_instance = sys.argv[2].split('=')[1] query = voyager_instance + "/select?q={!func}joindf(md5,md5)&f.md5.facet.mincount=2&f.contentHash.facet.mincount=2&f.schemaHash.facet.mincount=2&sort=md5 desc&start=0&rows=1&fl=id,title,name:[name],format,fullpath:[absolute],absolute_path:[absolute],download:[downloadURL],format_type,bytes,layerURL:[lyrURL],md5,path,name&fq={!frange l=2}{!func}joindf(md5,md5)&wt=json" results = requests.get(query, auth=("admin", "admin"), headers=headers) result_count = results.json()['response']['numFound'] if result_count == 0: status_writer.send_state(status.STAT_WARNING, "No duplicates found.") return duplicates = collections.defaultdict(list) groups = grouper(range(0, result_count), 25, '') url = voyager_instance.split("/solr")[0] req = requests.get("{0}/api/rest/i18n/field/format".format(url), headers=headers) formats = req.json()['VALUE']['format'] for group in groups: query = "%s/select?q={!func}joindf(md5,md5)&f.md5.facet.mincount=2&f.contentHash.facet.mincount=2&f.schemaHash.facet.mincount=2&sort=md5 desc&start=%s&rows=25&fl=id,title,name:[name],format,fullpath:[absolute],absolute_path:[absolute],download:[downloadURL],format_type,bytes,layerURL:[lyrURL],md5,path,name,%s&fq={!frange l=2}{!func}joindf(md5,md5)&wt=json" % (voyager_instance, group[0], fields_str) results = requests.get(query, auth=("admin", "admin"), headers=headers) docs = results.json()['response']['docs'] for doc in docs: file_size = 0 if "bytes" in doc: file_size = float(doc['bytes']) format_type = "" if "format" in doc and not doc["format"] in ("application/vnd.esri.gdb.file.data", "application/vnd.esri.gdb.personal.data"): try: format_type = formats[doc["format"]] except KeyError: format_type = doc["format"] else: continue file_path = "" if "absolute_path" in doc: file_path = doc["absolute_path"] id = "" if "id" in doc: id = doc["id"] name = "" if "name" in doc: name = doc["name"] elif "title" in doc: name = doc["title"] field_dict = {"FILE NAME": name, "FILE SIZE": file_size, "FORMAT": format_type, "ID": id, "FILE PATH": file_path} extra_fields = {} if fields: for fld in fields: if fld in doc: extra_fields[fld.upper()] = doc[fld] field_dict.update(extra_fields) duplicates[doc['md5']].append(field_dict) # Find total number of items in the index. all_query = "%s/select?disp=default&sort=score desc&place.op=within&start=0&fl=id&voyager.config.id=ace4bb77&wt=json" % (voyager_instance) results = requests.get(all_query, auth=("admin", "admin"), headers=headers) index_count = results.json()['response']['numFound'] duplicate_count = 0 total_file_size = 0 # Write all the duplicates to the report file. with open(report_file, "wb") as f: if extra_fields: keys = ["MD5", "FILE NAME", "FILE SIZE", "FORMAT", "ID", "FILE PATH"] + extra_fields.keys() else: keys = ["MD5", "FILE NAME", "FILE SIZE", "FORMAT", "ID", "FILE PATH"] writer = csv.DictWriter(f, fieldnames=keys) writer.writeheader() for md5, values in duplicates.items(): writer.writerow({}) val_count = len(values) if val_count > 1: duplicate_count += val_count - 1 for val in values: total_file_size += val["FILE SIZE"] * (val_count - 1) val["MD5"] = md5 val["FILE SIZE"] = convert_size(val["FILE SIZE"]) writer.writerow(val) # Report a summary to the report file. pct_dups = float(duplicate_count) / index_count with open(report_file, "ab") as f: writer = csv.DictWriter(f, fieldnames=["DUPLICATE COUNT", "INDEX COUNT", "PERCENT DUPLICATES", "TOTAL DUPLICATE FILE SIZE"]) writer.writerow({}) writer.writerow({}) writer.writeheader() writer.writerow({"DUPLICATE COUNT": duplicate_count, "INDEX COUNT": index_count, "PERCENT DUPLICATES": '{:.0f}%'.format(pct_dups * 100), "TOTAL DUPLICATE FILE SIZE": convert_size(total_file_size)}) status_writer.send_status("DUPLICATE COUNT: {0}".format(duplicate_count)) status_writer.send_status("INDEX COUNT: {0}".format(index_count)) status_writer.send_status("PERCENT DUPLICATES: {0}".format('{:.0f}%'.format(pct_dups * 100))) status_writer.send_status("TOTAL DUPLICATE FILE SIZE: {0}".format(convert_size(total_file_size))) status_writer.send_state(status.STAT_SUCCESS)
def execute(request): """Mosaics input raster datasets into a new raster dataset or mosaic dataset. :param request: json as a dict. """ status_writer = status.Writer() parameters = request['params'] target_workspace = task_utils.get_parameter_value(parameters, 'target_workspace', 'value') output_name = task_utils.get_parameter_value(parameters, 'output_dataset_name', 'value') out_coordinate_system = task_utils.get_parameter_value(parameters, 'output_projection', 'code') # Advanced options output_raster_format = task_utils.get_parameter_value(parameters, 'raster_format', 'value') compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value') compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value') arcpy.env.compression = '{0} {1}'.format(compression_method, compression_quality) if output_raster_format in ('FileGDB', 'MosaicDataset'): if not os.path.splitext(target_workspace)[1] in ('.gdb', '.mdb', '.sde'): status_writer.send_state(status.STAT_FAILED, _('Target workspace must be a geodatabase')) return task_folder = request['folder'] if not os.path.exists(task_folder): os.makedirs(task_folder) clip_area = None if not output_raster_format == 'MosaicDataset': # Get the clip region as an extent object. try: clip_area_wkt = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt') if not clip_area_wkt: clip_area_wkt = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))' if not out_coordinate_system == '0': clip_area = task_utils.get_clip_region(clip_area_wkt, out_coordinate_system) else: clip_area = task_utils.get_clip_region(clip_area_wkt) except KeyError: pass status_writer.send_status(_('Setting the output workspace...')) if not os.path.exists(target_workspace): status_writer.send_state(status.STAT_FAILED, _('Target workspace does not exist')) return arcpy.env.workspace = target_workspace status_writer.send_status(_('Starting to process...')) num_results, response_index = task_utils.get_result_count(parameters) raster_items = None if num_results > task_utils.CHUNK_SIZE: # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') headers = {'x-access-token': task_utils.get_security_token(request['owner'])} for group in groups: if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) input_items = task_utils.get_input_items(results.json()['response']['docs']) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) raster_items, pixels, bands, skipped = get_items(input_items) else: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) raster_items, pixels, bands, skipped = get_items(input_items) if not raster_items: if skipped == 0: status_writer.send_state(status.STAT_FAILED, _('Invalid input types')) skipped_reasons['All Items'] = _('Invalid input types') task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), num_results, skipped_details=skipped_reasons) return else: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons) return # Get most common pixel type. pixel_type = pixel_types[max(set(pixels), key=pixels.count)] if output_raster_format in ('FileGDB', 'GRID', 'MosaicDataset'): output_name = arcpy.ValidateTableName(output_name, target_workspace) else: output_name = '{0}.{1}'.format(arcpy.ValidateTableName(output_name, target_workspace), output_raster_format.lower()) if arcpy.Exists(os.path.join(target_workspace, output_name)): status_writer.send_state(status.STAT_FAILED, _('Output dataset already exists.')) return if output_raster_format == 'MosaicDataset': try: status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters'))) if out_coordinate_system == '0': out_coordinate_system = raster_items[0] else: out_coordinate_system = None mosaic_ds = arcpy.CreateMosaicDataset_management(target_workspace, output_name, out_coordinate_system, max(bands), pixel_type) arcpy.AddRastersToMosaicDataset_management(mosaic_ds, 'Raster Dataset', raster_items) arcpy.MakeMosaicLayer_management(mosaic_ds, 'mosaic_layer') layer_object = arcpy.mapping.Layer('mosaic_layer') task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png')) except arcpy.ExecuteError: skipped += 1 skipped_reasons['All Items'] = arcpy.GetMessages(2) else: try: if len(bands) > 1: status_writer.send_state(status.STAT_FAILED, _('Input rasters must have the same number of bands')) return if out_coordinate_system == '0': out_coordinate_system = None status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters'))) if clip_area: ext = '{0} {1} {2} {3}'.format(clip_area.XMin, clip_area.YMin, clip_area.XMax, clip_area.YMax) tmp_mosaic = arcpy.MosaicToNewRaster_management( raster_items, target_workspace, 'tmpMosaic', out_coordinate_system, pixel_type, number_of_bands=bands.keys()[0] ) status_writer.send_status(_('Clipping...')) out_mosaic = arcpy.Clip_management(tmp_mosaic, ext, output_name) arcpy.Delete_management(tmp_mosaic) else: out_mosaic = arcpy.MosaicToNewRaster_management(raster_items, target_workspace, output_name, out_coordinate_system, pixel_type, number_of_bands=bands.keys()[0], mosaic_method='BLEND') arcpy.MakeRasterLayer_management(out_mosaic, 'mosaic_layer') layer_object = arcpy.mapping.Layer('mosaic_layer') task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png')) except arcpy.ExecuteError: skipped += 1 skipped_reasons['All Items'] = arcpy.GetMessages(2) # Update state if necessary. if skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons)
def execute(request): """Copies files to a target folder. :param request: json as a dict. """ extracted = 0 skipped = 0 errors = 0 global result_count parameters = request['params'] output_type = task_utils.get_parameter_value(parameters, 'output_format', 'value') task_folder = os.path.join(request['folder'], 'temp') if not os.path.exists(task_folder): os.makedirs(task_folder) if output_type == 'FGDB': arcpy.CreateFileGDB_management(task_folder, 'output.gdb') result_count, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) # query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') headers = {'x-access-token': task_utils.get_security_token(request['owner'])} status_writer.send_percent(0.0, _('Starting to process...'), 'locate_xt_arcgis_tool') for group in groups: if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) docs = results.json()['response']['docs'] if not docs: docs = parameters[response_index]['response']['docs'] input_items = task_utils.get_input_items(docs) if input_items: result = extract(input_items, output_type, task_folder) extracted += result[0] errors += result[1] skipped += result[2] else: status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.')) return # Zip up outputs. zip_file = task_utils.zip_data(task_folder, 'output.zip') shutil.move(zip_file, os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file))) # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(request['folder'], '__report.json'), extracted, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Replace the workspace path for layer files and map document layers. :param request: json as a dict. """ updated = 0 skipped = 0 parameters = request['params'] backup = task_utils.get_parameter_value(parameters, 'create_backup', 'value') old_data_source = task_utils.get_parameter_value(parameters, 'old_data_source', 'value').lower() new_data_source = task_utils.get_parameter_value(parameters, 'new_data_source', 'value') if not os.path.exists(request['folder']): os.makedirs(request['folder']) if not arcpy.Exists(new_data_source): status_writer.send_state( status.STAT_FAILED, _('{0} does not exist').format(new_data_source)) return if os.path.splitext(new_data_source)[1] not in ('.gdb', '.mdb', '.sde'): new_dataset = os.path.basename(new_data_source) dsc = arcpy.Describe(os.path.dirname(new_data_source)) else: dsc = arcpy.Describe(new_data_source) new_dataset = '' wks_type = 'NONE' if dsc.dataType == 'FeatureDataset': new_workspace = dsc.path wks_type = get_workspace_type(new_workspace) elif dsc.dataType == 'Workspace': new_workspace = dsc.catalogPath wks_type = get_workspace_type(new_workspace) elif dsc.dataType == 'Folder': dsc = arcpy.Describe(new_data_source) new_workspace = dsc.catalogPath if new_dataset.endswith('.shp'): wks_type = 'SHAPEFILE_WORKSPACE' new_dataset = new_dataset.rsplit('.shp')[0] else: if arcpy.Describe(new_data_source).dataType == 'RasterDataset': wks_type = 'RASTER_WORKSPACE' elif dsc.dataType == 'CadDrawingDataset': new_workspace = dsc.path wks_type = 'CAD_WORKSPACE' else: new_workspace = os.path.dirname(new_data_source) num_results, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}'.format(sys.argv[2].split('=')[1], '/select?&wt=json') fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'replace_data_source') i = 0. for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "{0}&rows={1}&start={2}".format( fl, task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers) else: results = requests.get(query + "{0}&rows={1}&start={2}".format( fl, task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) input_items = task_utils.get_input_items( results.json()['response']['docs'], True) if not input_items: input_items = task_utils.get_input_items( parameters[response_index]['response']['docs']) result = replace_data_source(input_items, old_data_source, new_workspace, new_dataset, wks_type, backup, headers) updated += result[0] skipped += result[1] status_writer.send_percent( i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'replace_data_source') # Update state if necessary. if skipped > 0: status_writer.send_state( status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), updated, skipped, skipped_details=skipped_reasons)
def execute(request): """Exports search results a CSV, shapefile or XML document. :param request: json as a dict. """ chunk_size = task_utils.CHUNK_SIZE file_name = task_utils.get_parameter_value(request['params'], 'file_name', 'value') fields = task_utils.get_parameter_value(request['params'], 'fields', 'value') out_format = task_utils.get_parameter_value(request['params'], 'output_format', 'value') if not 'path' in fields and 'path:[absolute]' in fields: fields.append('path') if 'geo' in fields: i_geo = fields.index('geo') fields.remove('geo') fields.insert(i_geo, '[geo]') # Create the temporary workspace. task_folder = os.path.join(request['folder'], 'temp') if not os.path.exists(task_folder): os.makedirs(task_folder) headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } num_results, response_index = task_utils.get_result_count( request['params']) query = '{0}/select?&wt=json&fl={1}'.format(sys.argv[2].split('=')[1], ','.join(fields)) if 'query' in request['params'][response_index]: # Voyager Search Traditional UI for p in request['params']: if 'query' in p: request_qry = p['query'] break if 'voyager.list' in request_qry: query += '&voyager.list={0}'.format(request_qry['voyager.list']) # Replace spaces with %20 & remove \\ to avoid HTTP Error 400. if 'fq' in request_qry: try: query += '&fq={0}'.format(request_qry['fq'].replace("\\", "")) query = query.replace(' ', '%20') except AttributeError: for qry in request_qry['fq']: query += '&fq={0}'.format(qry).replace("\\", "").replace( ' ', '%20') if 'q' in request_qry: try: query += '&q={0}'.format(request_qry['q'].replace("\\", "")) query = query.replace(' ', '%20') except AttributeError: for qry in request_qry['q']: query += '&q={0}'.format(qry).replace("\\", "").replace( ' ', '%20') if 'place' in request_qry: try: query += '&place={0}'.format(request_qry['place'].replace( "\\", "")) query = query.replace(' ', '%20') except AttributeError: for qry in request_qry['place']: query += '&place={0}'.format(qry).replace("\\", "").replace( ' ', '%20') if 'place.op' in request_qry: query += '&place.op={0}'.format(request_qry['place.op']) query += '&rows={0}&start={1}' exported_cnt = 0. for i in xrange(0, num_results, chunk_size): req = urllib2.Request(query.replace('{0}', str(chunk_size)).replace( '{1}', str(i)), headers=headers) for n in urllib2.urlopen(req): jobs = eval(n.replace('null', '"null"'))['response']['docs'] if out_format == 'CSV': export_to_csv(jobs, file_name, task_folder, fields) elif out_format == 'XML': export_to_xml(jobs, file_name, task_folder) elif out_format == 'SHP': export_to_shp(jobs, file_name, task_folder) exported_cnt += chunk_size if exported_cnt > num_results: status_writer.send_percent(100, 'exported: 100%', 'export_results') else: percent_done = exported_cnt / num_results status_writer.send_percent( percent_done, '{0}: {1:.0f}%'.format("exported", percent_done * 100), 'export_results') else: # Voyager Search Portal/Cart UI ids = [] for p in request['params']: if 'ids' in p: ids = p['ids'] break groups = task_utils.grouper(list(ids), chunk_size, '') i = 0 for group in groups: i += len([v for v in group if not v == '']) req = urllib2.Request(query + '&ids={0}'.format(','.join(group)), headers=headers) results = urllib2.urlopen(req) jobs = eval(results.read())['response']['docs'] if out_format == 'CSV': export_to_csv(jobs, file_name, task_folder, fields) elif out_format == 'XML': export_to_xml(jobs, file_name, task_folder) elif out_format == 'SHP': export_to_shp(jobs, file_name, task_folder) percent_done = float(i) / num_results status_writer.send_percent( percent_done, '{0}: {1:.0f}%'.format("exported", percent_done * 100), 'export_results') # Zip up outputs. if exported_count == 0: task_utils.report(os.path.join(request['folder'], '__report.json'), exported_count, 0, errors_count, errors_reasons) else: task_utils.report(os.path.join(request['folder'], '__report.json'), exported_count, 0, errors_count, errors_reasons) zip_file = task_utils.zip_data(task_folder, 'output.zip') shutil.move( zip_file, os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))
def execute(request): """Writes existing metadata for summary, description and tags. If overwrite is false, existing metadata is untouched unless any field is empty or does not exist, then it is created. :param request: json as a dict. """ updated = 0 errors = 0 skipped = 0 global result_count parameters = request['params'] summary = task_utils.get_parameter_value(parameters, 'summary', 'value') description = task_utils.get_parameter_value(parameters, 'description', 'value') tags = task_utils.get_parameter_value(parameters, 'tags', 'value') data_credits = task_utils.get_parameter_value(parameters, 'credits', 'value') constraints = task_utils.get_parameter_value(parameters, 'constraints', 'value') # Handle commas, spaces, and/or new line separators. tags = [tag for tag in re.split(' |,|\n', tags) if not tag == ''] overwrite = task_utils.get_parameter_value(parameters, 'overwrite', 'value') if not overwrite: overwrite = False if not os.path.exists(request['folder']): os.makedirs(request['folder']) # Stylesheet xslt_file = os.path.join( arcpy.GetInstallInfo()['InstallDir'], 'Metadata/Stylesheets/gpTools/exact copy of.xslt') # Template metadata file. template_xml = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'metadata_template.xml') result_count, response_index = task_utils.get_result_count(parameters) headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl + ',links' query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'write_metadata') i = 0. for group in groups: i += len(group) - group.count('') if fq: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) docs = results.json()['response']['docs'] if not docs: docs = parameters[response_index]['response']['docs'] input_items = [] for doc in docs: if 'path' in doc: if 'links' in doc: links = eval(doc['links']) input_items.append( (doc['path'], links['links'][0]['link'][0]['id'])) else: input_items.append((doc['path'], doc['id'])) result = write_metadata(input_items, template_xml, xslt_file, summary, description, tags, data_credits, constraints, overwrite, headers) updated += result[0] errors += result[1] skipped += result[2] status_writer.send_percent( i / result_count, '{0}: {1:.0f}%'.format("Processed", i / result_count * 100), 'write_metadata') # Report state. if skipped > 0 or errors > 0: status_writer.send_state( status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) else: status_writer.send_state(status.STAT_SUCCESS) task_utils.report(os.path.join(request['folder'], '__report.json'), updated, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Zips all input files to output.zip. :param request: json as a dict. """ zipped = 0 skipped = 0 parameters = request['params'] flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value') if not flatten_results: flatten_results = False zip_file_location = request['folder'] if not os.path.exists(zip_file_location): os.makedirs(request['folder']) output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value') if not output_file_name: output_file_name = 'output' zip_file = os.path.join(zip_file_location, '{0}.zip'.format(output_file_name)) zipper = task_utils.ZipFileManager(zip_file, 'w', zipfile.ZIP_DEFLATED) num_results, response_index = task_utils.get_result_count(parameters) headers = {'x-access-token': task_utils.get_security_token(request['owner'])} # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) #query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'zip_files') i = 0. for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) input_items = task_utils.get_input_items(results.json()['response']['docs'], list_components=True) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'], list_components=True) result = zip_files(zipper, input_items, zip_file_location, flatten_results) zipped += result[0] skipped += result[1] status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'zip_files') zipper.close() if zipped == 0: status_writer.send_state(status.STAT_FAILED, _('No results were zipped.')) return # Update state if necessary. if skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), zipped, skipped, skipped_details=skipped_reasons)
def execute(request): """Mosaics input raster datasets into a new raster dataset. :param request: json as a dict. """ parameters = request['params'] out_coordinate_system = task_utils.get_parameter_value(parameters, 'output_projection', 'code') # Advanced options output_raster_format = task_utils.get_parameter_value(parameters, 'raster_format', 'value') compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value') compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value') output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value') if not output_file_name: output_file_name = 'output' arcpy.env.compression = '{0} {1}'.format(compression_method, compression_quality) clip_area = None if not output_raster_format == 'MosaicDataset': # Get the clip region as an extent object. try: clip_area_wkt = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt') if not clip_area_wkt: clip_area_wkt = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))' if not out_coordinate_system == '0': clip_area = task_utils.get_clip_region(clip_area_wkt, out_coordinate_system) else: clip_area = task_utils.get_clip_region(clip_area_wkt) except KeyError: pass status_writer.send_status(_('Setting the output workspace...')) out_workspace = os.path.join(request['folder'], 'temp') if not os.path.exists(out_workspace): os.makedirs(out_workspace) if output_raster_format == 'FileGDB' or output_raster_format == 'MosaicDataset': out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0) arcpy.env.workspace = out_workspace status_writer.send_status(_('Starting to process...')) num_results, response_index = task_utils.get_result_count(parameters) raster_items = None if num_results > task_utils.CHUNK_SIZE: # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') headers = {'x-access-token': task_utils.get_security_token(request['owner'])} for group in groups: if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) input_items = task_utils.get_input_items(results.json()['response']['docs']) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) raster_items, pixels, bands, skipped = get_items(input_items) else: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) raster_items, pixels, bands, skipped = get_items(input_items) if not raster_items: if skipped == 0: status_writer.send_state(status.STAT_FAILED, _('Invalid input types')) skipped_reasons['All Items'] = _('Invalid input types') task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), num_results, skipped_details=skipped_reasons) return else: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons) return # Get most common pixel type. pixel_type = pixel_types[max(set(pixels), key=pixels.count)] if output_raster_format in ('FileGDB', 'GRID', 'MosaicDataset'): output_name = arcpy.ValidateTableName('mosaic', out_workspace) else: output_name = '{0}.{1}'.format(arcpy.ValidateTableName('mosaic', out_workspace)[:9], output_raster_format.lower()) status_writer.send_status(output_name) if output_raster_format == 'MosaicDataset': try: status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters'))) if out_coordinate_system == '0': out_coordinate_system = raster_items[0] else: out_coordinate_system = None mosaic_ds = arcpy.CreateMosaicDataset_management(out_workspace, output_name, out_coordinate_system, max(bands), pixel_type) arcpy.AddRastersToMosaicDataset_management(mosaic_ds, 'Raster Dataset', raster_items) arcpy.MakeMosaicLayer_management(mosaic_ds, 'mosaic_layer') layer_object = arcpy.mapping.Layer('mosaic_layer') task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png')) except arcpy.ExecuteError: status_writer.send_state(status.STAT_FAILED, arcpy.GetMessages(2)) return else: try: if len(bands) > 1: status_writer.send_state(status.STAT_FAILED, _('Input rasters must have the same number of bands')) return status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters'))) if out_coordinate_system == '0': out_coordinate_system = None if clip_area: ext = '{0} {1} {2} {3}'.format(clip_area.XMin, clip_area.YMin, clip_area.XMax, clip_area.YMax) tmp_mosaic = arcpy.MosaicToNewRaster_management( raster_items, out_workspace, 'tm', out_coordinate_system, pixel_type, number_of_bands=bands.keys()[0] ) status_writer.send_status(_('Clipping...')) out_mosaic = arcpy.Clip_management(tmp_mosaic, ext, output_name) arcpy.Delete_management(tmp_mosaic) else: out_mosaic = arcpy.MosaicToNewRaster_management(raster_items, out_workspace, output_name, out_coordinate_system, pixel_type, number_of_bands=bands.keys()[0]) arcpy.MakeRasterLayer_management(out_mosaic, 'mosaic_layer') layer_object = arcpy.mapping.Layer('mosaic_layer') task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png')) except arcpy.ExecuteError: status_writer.send_state(status.STAT_FAILED, arcpy.GetMessages(2)) return if arcpy.env.workspace.endswith('.gdb'): out_workspace = os.path.dirname(arcpy.env.workspace) zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) # Update state if necessary. if skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons)
def execute(request): """Builds raster pyramids for input raster datasets. :param request: json as a dict. """ processed = 0 skipped = 0 parameters = request['params'] resampling_method = task_utils.get_parameter_value(parameters, 'resampling_method', 'value') # Advanced options compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value') compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value') # Create the task folder to hold report files. task_folder = request['folder'] if not os.path.exists(task_folder): os.makedirs(task_folder) headers = {'x-access-token': task_utils.get_security_token(request['owner'])} num_results, response_index = task_utils.get_result_count(parameters) if num_results > task_utils.CHUNK_SIZE: # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'build_raster_pyramids') i = 0. for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) input_items = task_utils.get_input_items(results.json()['response']['docs']) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) result = build_pyramids(input_items, compression_method, compression_quality, resampling_method) processed += result[0] skipped += result[1] status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'build_raster_pyramids') else: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) processed, skipped = build_pyramids(input_items, compression_method, compression_quality, resampling_method, True) # Update state if necessary. if skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), processed, skipped, skipped_details=skipped_reasons)
def execute(request): """Exports search results a CSV, shapefile or XML document. :param request: json as a dict. """ # Get SSL trust setting. verify_ssl = task_utils.get_ssl_mode() chunk_size = task_utils.CHUNK_SIZE file_name = task_utils.get_parameter_value(request['params'], 'file_name', 'value') fields = task_utils.get_parameter_value(request['params'], 'fields', 'value') out_format = task_utils.get_parameter_value(request['params'], 'output_format', 'value') if not 'path' in fields and 'path:[absolute]' in fields: fields.append('path') if 'geo' in fields: i_geo = fields.index('geo') fields.remove('geo') fields.insert(i_geo, '[geo]') # Create the temporary workspace. task_folder = os.path.join(request['folder'], 'temp') if not os.path.exists(task_folder): os.makedirs(task_folder) headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } num_results, response_index = task_utils.get_result_count( request['params']) if len(sys.argv) == 2: query = '{0}/solr/v0/select?&wt=json&fl={1}'.format( 'http://localhost:8888', ','.join(fields)) else: query = '{0}/select?&wt=json&fl={1}'.format(sys.argv[2].split('=')[1], ','.join(fields)) if 'query' in request['params'][response_index]: # Voyager Search Traditional UI for p in request['params']: if 'query' in p: request_qry = p['query'] break if 'voyager.list' in request_qry: query += '&voyager.list={0}'.format(request_qry['voyager.list']) # Replace spaces with %20 & remove \\ to avoid HTTP Error 400. if 'fq' in request_qry: try: if isinstance(request_qry['fq'], list): for fq in request_qry['fq']: try: query += '&fq={0}'.format(str(fq)) except UnicodeEncodeError: query += '&fq={0}'.format(str(fq.encode('utf-8'))) else: query += '&fq={0}'.format(request_qry['fq']) if '{!expand}' in query: query = query.replace('{!expand}', '') if '{!tag' in query: tag = re.findall('{!(.*?)}', query) if tag: tag_str = "{!" + tag[0] + "}" query = query.replace(tag_str, '') query = query.replace(' ', '%20') except AttributeError: for qry in request_qry['fq']: query += '&fq={0}'.format(qry).replace("\\", "").replace( ' ', '%20') if 'q' in request_qry: try: query += '&q={0}'.format(request_qry['q'].replace("\\", "")) query = query.replace(' ', '%20') except UnicodeEncodeError: query += '&q={0}'.format( request_qry['q'].encode('utf-8').replace("\\", "")) query = query.replace(' ', '%20') except AttributeError: for qry in request_qry['q']: query += '&q={0}'.format(qry).replace("\\", "").replace( ' ', '%20') if 'place' in request_qry: try: query += '&place={0}'.format(request_qry['place'].replace( "\\", "")) query = query.replace(' ', '%20') except AttributeError: for qry in request_qry['place']: query += '&place={0}'.format(qry).replace("\\", "").replace( ' ', '%20') if 'place.op' in request_qry: query += '&place.op={0}'.format(request_qry['place.op']) query += '&rows={0}&start={1}' exported_cnt = 0. for i in range(0, num_results, chunk_size): url = query.replace('{0}', str(chunk_size)).replace('{1}', str(i)) res = requests.get(url, verify=verify_ssl, headers=headers) jobs = res.json()['response']['docs'] if out_format == 'CSV': export_to_csv(jobs, file_name, task_folder, fields) elif out_format == 'XML': export_to_xml(jobs, file_name, task_folder) elif out_format == 'SHP': export_to_shp(jobs, file_name, task_folder) exported_cnt += chunk_size if exported_cnt > num_results: status_writer.send_percent(100, 'exported: 100%', 'export_results') else: percent_done = exported_cnt / num_results status_writer.send_percent( percent_done, '{0}: {1:.0f}%'.format("exported", percent_done * 100), 'export_results') else: # Voyager Search Portal/Cart UI ids = [] for p in request['params']: if 'ids' in p: ids = p['ids'] break groups = task_utils.grouper(list(ids), chunk_size, '') i = 0 for group in groups: i += len([v for v in group if not v == '']) results = requests.get(query + '&ids={0}'.format(','.join(group)), verify=verify_ssl, headers=headers) jobs = eval(results.text)['response']['docs'] if out_format == 'CSV': export_to_csv(jobs, file_name, task_folder, fields) elif out_format == 'XML': export_to_xml(jobs, file_name, task_folder) elif out_format == 'SHP': export_to_shp(jobs, file_name, task_folder) percent_done = float(i) / num_results status_writer.send_percent( percent_done, '{0}: {1:.0f}%'.format("exported", percent_done * 100), 'export_results') # Zip up outputs. if exported_count == 0: status_writer.send_state(status.STAT_FAILED) task_utils.report(os.path.join(request['folder'], '__report.json'), exported_count, 0, errors_count, errors_reasons) else: task_utils.report(os.path.join(request['folder'], '__report.json'), exported_count, 0, errors_count, errors_reasons) zip_file = task_utils.zip_data(task_folder, '{0}.zip'.format(file_name)) shutil.move( zip_file, os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))
def execute(request): """Copies files to a target folder. :param request: json as a dict. """ created = 0 skipped = 0 errors = 0 global result_count parameters = request['params'] if not os.path.exists(request['folder']): os.makedirs(request['folder']) meta_folder = task_utils.get_parameter_value(parameters, 'meta_data_folder', 'value') result_count, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'create_layer_files') i = 0. headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } for group in groups: i += len(group) - group.count('') if fq: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) docs = results.json()['response']['docs'] # docs = eval(results.read().replace('false', 'False').replace('true', 'True'))['response']['docs'] if not docs: docs = parameters[response_index]['response']['docs'] input_items = [] for doc in docs: if 'path' in doc: input_items.append( (doc['id'], doc['path'], doc['name'], doc['location'])) result = create_layer_file(input_items, meta_folder) created += result[0] errors += result[1] skipped += result[2] try: shutil.copy2( os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', '_thumb.png'), request['folder']) except IOError: pass # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state( status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(request['folder'], '__report.json'), created, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Adds a field and calculates it to some value. :param request: json as a dict. """ created = 0 skipped = 0 errors = 0 warnings = 0 global result_count parameters = request['params'] if not os.path.exists(request['folder']): os.makedirs(request['folder']) # Parameter values field_name = task_utils.get_parameter_value(parameters, 'field_name', 'value') field_type = task_utils.get_parameter_value(parameters, 'field_type', 'value') field_value = task_utils.get_parameter_value(parameters, 'field_value', 'value') # Query the index for results in groups of 25. headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } result_count, response_index = task_utils.get_result_count(parameters) query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl + ',links' query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'add_field') for group in groups: if fq: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers) else: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) docs = results.json()['response']['docs'] input_items = [] for doc in docs: if 'path' in doc: if 'links' in doc: links = eval(doc['links']) input_items.append((doc['id'], doc['absolute_path'], links['links'][0]['link'][0]['id'])) else: input_items.append((doc['id'], doc['absolute_path'])) result = add_field(input_items, field_name, field_type, field_value) created += result[0] errors += result[1] skipped += result[2] warnings += result[3] # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state( status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(request['folder'], '__report.json'), created, skipped, errors, errors_reasons, skipped_reasons, warnings, warnings_reasons)
def execute(request): """Converts each input dataset to kml (.kmz). :param request: json as a dict. """ converted = 0 skipped = 0 errors = 0 global result_count parameters = request["params"] out_workspace = os.path.join(request["folder"], "temp") if not os.path.exists(out_workspace): os.makedirs(out_workspace) # Get the boundary box extent for input to KML tools. extent = "" try: try: ext = task_utils.get_parameter_value(parameters, "processing_extent", "wkt") if ext: sr = task_utils.get_spatial_reference("4326") extent = task_utils.from_wkt(ext, sr) except KeyError: ext = task_utils.get_parameter_value(parameters, "processing_extent", "feature") if ext: extent = arcpy.Describe(ext).extent except KeyError: pass # Get the output file name. output_file_name = task_utils.get_parameter_value(parameters, "output_file_name", "value") if not output_file_name: output_file_name = "kml_results" result_count, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = "{0}{1}{2}".format(sys.argv[2].split("=")[1], "/select?&wt=json", fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "") query += fq elif "ids" in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]["ids"]), task_utils.CHUNK_SIZE, "") else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "") # Begin processing status_writer.send_percent(0.0, _("Starting to process..."), "convert_to_kml") headers = {"x-access-token": task_utils.get_security_token(request["owner"])} for group in groups: if fq: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers ) elif "ids" in parameters[response_index]: results = requests.get(query + "{0}&ids={1}".format(fl, ",".join(group)), headers=headers) else: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers ) docs = results.json()["response"]["docs"] input_items = task_utils.get_input_items(docs) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]["response"]["docs"]) input_rows = collections.defaultdict(list) for doc in docs: if "path" not in doc: input_rows[doc["name"]].append(doc) if input_rows: result = convert_to_kml(input_rows, out_workspace, extent) converted += result[0] errors += result[1] skipped += result[2] if input_items: result = convert_to_kml(input_items, out_workspace, extent) converted += result[0] errors += result[1] skipped += result[2] if not input_items and not input_rows: status_writer.send_state(status.STAT_FAILED, _("No items to process. Check if items exist.")) return # Zip up kmz files if more than one. if converted > 1: status_writer.send_status("Converted: {}".format(converted)) zip_file = task_utils.zip_data(out_workspace, "{0}.zip".format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) shutil.copy2( os.path.join(os.path.dirname(os.path.dirname(__file__)), "supportfiles", "_thumb.png"), request["folder"] ) elif converted == 1: try: kml_file = glob.glob(os.path.join(out_workspace, "*.kmz"))[0] tmp_lyr = arcpy.KMLToLayer_conversion(kml_file, out_workspace, "kml_layer") task_utils.make_thumbnail(tmp_lyr.getOutput(0), os.path.join(request["folder"], "_thumb.png")) except arcpy.ExecuteError: pass shutil.move(kml_file, os.path.join(request["folder"], os.path.basename(kml_file))) # Update state if necessary. if skipped > 0 or errors > 0: status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(errors + skipped)) task_utils.report( os.path.join(request["folder"], "__report.json"), converted, skipped, errors, errors_reasons, skipped_reasons )
def execute(request): """Package inputs to an Esri map or layer package. :param request: json as a dict. """ errors = 0 skipped = 0 layers = [] files = [] app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) parameters = request['params'] out_format = task_utils.get_parameter_value(parameters, 'output_format', 'value') summary = task_utils.get_parameter_value(parameters, 'summary') tags = task_utils.get_parameter_value(parameters, 'tags') output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name') if not output_file_name: output_file_name = 'package_results' # Get the clip region as an extent object. clip_area = None try: clip_area_wkt = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt') clip_area = task_utils.get_clip_region(clip_area_wkt) except (KeyError, ValueError): pass out_workspace = os.path.join(request['folder'], 'temp') if not os.path.exists(out_workspace): os.makedirs(out_workspace) num_results, response_index = task_utils.get_result_count(parameters) # if num_results > task_utils.CHUNK_SIZE: # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '') headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } status_writer.send_status(_('Starting to process...')) for group in groups: if fq: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers) else: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) input_items = task_utils.get_input_items( results.json()['response']['docs']) if not input_items: input_items = task_utils.get_input_items( parameters[response_index]['response']['docs']) layers, files, errors, skipped = get_items(input_items, out_workspace) # else: # input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) # layers, files, errors, skipped = get_items(input_items, out_workspace) if errors == num_results: status_writer.send_state(status.STAT_FAILED, _('No results to package')) return try: if out_format == 'MPK': shutil.copyfile( os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'), os.path.join(out_workspace, 'output.mxd')) mxd = arcpy.mapping.MapDocument( os.path.join(out_workspace, 'output.mxd')) if mxd.description == '': mxd.description = os.path.basename(mxd.filePath) df = arcpy.mapping.ListDataFrames(mxd)[0] for layer in layers: arcpy.mapping.AddLayer(df, layer) mxd.save() status_writer.send_status( _('Generating {0}. Large input {1} will take longer to process.' .format('MPK', 'results'))) if arcpy.GetInstallInfo()['Version'] == '10.0': arcpy.PackageMap_management( mxd.filePath, os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name)), 'PRESERVE', extent=clip_area) elif arcpy.GetInstallInfo()['Version'] == '10.1': arcpy.PackageMap_management( mxd.filePath, os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name)), 'PRESERVE', extent=clip_area, ArcGISRuntime='RUNTIME', version='10', additional_files=files, summary=summary, tags=tags) else: arcpy.PackageMap_management( mxd.filePath, os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name)), 'PRESERVE', extent=clip_area, arcgisruntime='RUNTIME', version='10', additional_files=files, summary=summary, tags=tags) # Create a thumbnail size PNG of the mxd. task_utils.make_thumbnail( mxd, os.path.join(request['folder'], '_thumb.png')) else: status_writer.send_status( _('Generating {0}. Large input {1} will take longer to process.' .format('LPK', 'results'))) for layer in layers: if layer.description == '': layer.description = layer.name if arcpy.GetInstallInfo()['Version'] == '10.0': arcpy.PackageLayer_management( layers, os.path.join(os.path.dirname(out_workspace), '{0}.lpk'.format(output_file_name)), 'PRESERVE', extent=clip_area, version='10') else: arcpy.PackageLayer_management( layers, os.path.join(os.path.dirname(out_workspace), '{0}.lpk'.format(output_file_name)), 'PRESERVE', extent=clip_area, version='10', additional_files=files, summary=summary, tags=tags) # Create a thumbnail size PNG of the mxd. task_utils.make_thumbnail( layers[0], os.path.join(request['folder'], '_thumb.png')) except (RuntimeError, ValueError, arcpy.ExecuteError) as ex: status_writer.send_state(status.STAT_FAILED, repr(ex)) return # Update state if necessary. if errors > 0 or skipped: status_writer.send_state( status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), num_results - (skipped + errors), skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Writes existing metadata for summary, description and tags. If overwrite is false, existing metadata is untouched unless any field is empty or does not exist, then it is created. :param request: json as a dict. """ updated = 0 errors = 0 skipped = 0 global result_count parameters = request['params'] summary = task_utils.get_parameter_value(parameters, 'summary', 'value') description = task_utils.get_parameter_value(parameters, 'description', 'value') tags = task_utils.get_parameter_value(parameters, 'tags', 'value') data_credits = task_utils.get_parameter_value(parameters, 'credits', 'value') constraints = task_utils.get_parameter_value(parameters, 'constraints', 'value') # Handle commas, spaces, and/or new line separators. tags = [tag for tag in re.split(' |,|\n', tags) if not tag == ''] overwrite = task_utils.get_parameter_value(parameters, 'overwrite', 'value') if not overwrite: overwrite = False if not os.path.exists(request['folder']): os.makedirs(request['folder']) # Stylesheet xslt_file = os.path.join(arcpy.GetInstallInfo()['InstallDir'], 'Metadata/Stylesheets/gpTools/exact copy of.xslt') # Template metadata file. template_xml = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'metadata_template.xml') result_count, response_index = task_utils.get_result_count(parameters) headers = {'x-access-token': task_utils.get_security_token(request['owner'])} # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl + ',links' query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'write_metadata') i = 0. for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) docs = results.json()['response']['docs'] if not docs: docs = parameters[response_index]['response']['docs'] input_items = [] for doc in docs: if 'path' in doc and os.path.exists(doc['path']): if 'links' in doc: links = eval(doc['links']) input_items.append((doc['path'], links['links'][0]['link'][0]['id'])) else: input_items.append((doc['path'], doc['id'])) elif 'absolute_path' in doc and os.path.exists(doc['absolute_path']): if 'links' in doc: links = eval(doc['links']) input_items.append((doc['absolute_path'], links['links'][0]['link'][0]['id'])) else: input_items.append((doc['absolute_path'], doc['id'])) elif '[downloadURL]' in doc: if 'links' in doc: links = eval(doc['links']) input_items.append((doc['[downloadURL]'], links['links'][0]['link'][0]['id'])) else: input_items.append((doc['[downloadURL]'], doc['id'])) result = write_metadata(input_items, template_xml, xslt_file, summary, description, tags, data_credits, constraints, overwrite, headers) updated += result[0] errors += result[1] skipped += result[2] status_writer.send_percent(i / result_count, '{0}: {1:.0f}%'.format("Processed", i / result_count * 100), 'write_metadata') # Report state. if skipped > 0 or errors > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) else: status_writer.send_state(status.STAT_SUCCESS) task_utils.report(os.path.join(request['folder'], '__report.json'), updated, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Copies files to a target folder. :param request: json as a dict. """ extracted = 0 skipped = 0 errors = 0 global result_count parameters = request['params'] output_type = task_utils.get_parameter_value(parameters, 'output_format', 'value') task_folder = os.path.join(request['folder'], 'temp') if not os.path.exists(task_folder): os.makedirs(task_folder) if output_type == 'FGDB': arcpy.CreateFileGDB_management(task_folder, 'output.gdb') result_count, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) # query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } status_writer.send_percent(0.0, _('Starting to process...'), 'locate_xt_arcgis_tool') for group in groups: if fq: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) docs = results.json()['response']['docs'] if not docs: docs = parameters[response_index]['response']['docs'] input_items = task_utils.get_input_items(docs) if input_items: result = extract(input_items, output_type, task_folder) extracted += result[0] errors += result[1] skipped += result[2] else: status_writer.send_state( status.STAT_FAILED, _('No items to process. Check if items exist.')) return # Zip up outputs. zip_file = task_utils.zip_data(task_folder, 'output.zip') shutil.move( zip_file, os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file))) # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state( status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(request['folder'], '__report.json'), extracted, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Adds a field and calculates it to some value. :param request: json as a dict. """ created = 0 skipped = 0 errors = 0 warnings = 0 global result_count parameters = request['params'] if not os.path.exists(request['folder']): os.makedirs(request['folder']) # Parameter values field_name = task_utils.get_parameter_value(parameters, 'field_name', 'value') field_type = task_utils.get_parameter_value(parameters, 'field_type', 'value') field_value = task_utils.get_parameter_value(parameters, 'field_value', 'value') # Query the index for results in groups of 25. headers = {'x-access-token': task_utils.get_security_token(request['owner'])} result_count, response_index = task_utils.get_result_count(parameters) query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl + ',links' query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') # Begin processing status_writer.send_percent(0.0, _('Starting to process...'), 'add_field') for group in groups: if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) docs = results.json()['response']['docs'] input_items = [] for doc in docs: if 'path' in doc: if 'links' in doc: links = eval(doc['links']) input_items.append((doc['id'], doc['path'], links['links'][0]['link'][0]['id'])) else: input_items.append((doc['id'], doc['path'])) result = add_field(input_items, field_name, field_type, field_value) created += result[0] errors += result[1] skipped += result[2] warnings += result[3] # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(request['folder'], '__report.json'), created, skipped, errors, errors_reasons, skipped_reasons, warnings, warnings_reasons)
def execute(request): """Clips selected search results using the clip geometry. :param request: json as a dict. """ clipped = 0 errors = 0 skipped = 0 global result_count parameters = request['params'] # Retrieve the clip features. clip_features = task_utils.get_parameter_value(parameters, 'clip_features', 'value') # Retrieve the coordinate system code. out_coordinate_system = int(task_utils.get_parameter_value(parameters, 'output_projection', 'code')) # Retrieve the output format, create mxd and output file name parameter values. out_format = task_utils.get_parameter_value(parameters, 'output_format', 'value') create_mxd = task_utils.get_parameter_value(parameters, 'create_mxd', 'value') output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value') if not output_file_name: output_file_name = 'clip_results' # Create the temporary workspace if clip_feature_class: out_workspace = os.path.join(request['folder'], 'temp') if not os.path.exists(out_workspace): os.makedirs(out_workspace) # Set the output coordinate system. if not out_coordinate_system == 0: # Same as Input out_sr = task_utils.get_spatial_reference(out_coordinate_system) arcpy.env.outputCoordinateSystem = out_sr # Set the output workspace. status_writer.send_status(_('Setting the output workspace...')) if not out_format == 'SHP': out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0) arcpy.env.workspace = out_workspace # Query the index for results in groups of 25. headers = {'x-access-token': task_utils.get_security_token(request['owner'])} result_count, response_index = task_utils.get_result_count(parameters) query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl # Get the Clip features by id. id = clip_features['id'] clip_query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', "&fl=id,path,fullpath:[absolute],absolute_path:[absolute],[lyrFile],[geo]&q=id:{0}".format(id)) clip_result = requests.get(clip_query, verify=verify_ssl, headers=headers) clipper = clip_result.json()['response']['docs'][0] if 'absolute_path' in clipper and not clipper['absolute_path'].startswith('s3'): clip_features = clipper['absolute_path'] elif '[lyrFile]' in clipper: clip_features = clipper['[lyrFile]'] elif '[geo]' in clipper: clip_features = arcpy.AsShape(clipper['[geo]']).projectAs(arcpy.SpatialReference(4326)) elif 'absolute_path' in clipper and clipper['absolute_path'].startswith('s3'): base_name = os.path.basename(clipper['path']) temp_folder = tempfile.mkdtemp() if '[downloadURL]' in clipper: download = os.path.join(temp_folder, os.path.basename(clipper['[downloadURL]'])) response = requests.get(clipper['[downloadURL]'], verify=verify_ssl) with open(download, 'wb') as fp: fp.write(response.content) if download.endswith('.zip'): zip = zipfile.ZipFile(download) zip.extractall(temp_folder) clip_features = os.path.join(temp_folder, base_name) else: clip_features = download else: bbox = clipper['bbox'].split() extent = arcpy.Extent(*bbox) pt_array = arcpy.Array([extent.lowerLeft, extent.upperLeft, extent.upperRight, extent.lowerRight]) clip_features = arcpy.Polygon(pt_array, 4326) query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'clip_data') for group in groups: if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) docs = results.json()['response']['docs'] input_items = task_utils.get_input_items(docs) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]['response']['docs']) input_rows = collections.defaultdict(list) for doc in docs: if 'path' not in doc: input_rows[doc['name']].append(doc) if input_rows: result = clip_data(input_rows, out_workspace, clip_features, out_format) clipped += result[0] errors += result[1] skipped += result[2] if input_items: result = clip_data(input_items, out_workspace, clip_features, out_format) clipped += result[0] errors += result[1] skipped += result[2] if not input_items and not input_rows: status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.')) return if arcpy.env.workspace.endswith('.gdb'): out_workspace = os.path.dirname(arcpy.env.workspace) if clipped > 0: try: if out_format == 'MPK': mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd') mxd = task_utils.create_mxd(out_workspace, mxd_template, 'output') status_writer.send_status(_("Packaging results...")) task_utils.create_mpk(out_workspace, mxd, files_to_package) shutil.move(os.path.join(out_workspace, 'output.mpk'), os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name))) elif out_format == 'LPK': status_writer.send_status(_("Packaging results...")) task_utils.create_lpk(out_workspace, output_file_name, files_to_package) elif out_format == 'KML': task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb")) arcpy.env.workspace = '' arcpy.RefreshCatalog(os.path.join(out_workspace, "output.gdb")) try: arcpy.Delete_management(os.path.join(out_workspace, "output.gdb")) except arcpy.ExecuteError: pass zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) else: if create_mxd: mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd') task_utils.create_mxd(out_workspace, mxd_template, 'output') zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) except arcpy.ExecuteError as ee: status_writer.send_state(status.STAT_FAILED, _(ee)) sys.exit(1) else: status_writer.send_state(status.STAT_FAILED, _('No output created. Zero inputs were clipped.')) # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped)) task_utils.report(os.path.join(request['folder'], '__report.json'), clipped, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Copies data to an existing geodatabase or feature dataset. :param request: json as a dict. """ added = 0 errors = 0 skipped = 0 global result_count parameters = request['params'] # Get the target workspace location. out_gdb = task_utils.get_parameter_value(parameters, 'target_workspace', 'value') # Retrieve the coordinate system code. out_coordinate_system = task_utils.get_parameter_value( parameters, 'output_projection', 'code') if not out_coordinate_system == '0': # Same as Input arcpy.env.outputCoordinateSystem = task_utils.get_spatial_reference( out_coordinate_system) task_folder = request['folder'] if not os.path.exists(task_folder): os.makedirs(task_folder) # Check if the geodatabase exists or if it is a feature dataset. is_fds = False if not os.path.exists(out_gdb): if out_gdb.endswith('.gdb'): arcpy.CreateFileGDB_management(os.path.dirname(out_gdb), os.path.basename(out_gdb)) status_writer.send_status( _('Created output workspace: {0}').format(out_gdb)) elif out_gdb.endswith('.mdb'): arcpy.CreatePersonalGDB_management(os.path.dirname(out_gdb), os.path.basename(out_gdb)) status_writer.send_status( _('Created output workspace: {0}').format(out_gdb)) elif out_gdb.endswith('.sde'): status_writer.send_state(status.STAT_FAILED, _('{0} does not exist').format(out_gdb)) return else: # Possible feature dataset. is_fds = is_feature_dataset(out_gdb) if not is_fds: if os.path.dirname(out_gdb).endswith('.gdb'): if not os.path.exists(os.path.dirname(out_gdb)): arcpy.CreateFileGDB_management( os.path.dirname(os.path.dirname(out_gdb)), os.path.basename(os.path.dirname(out_gdb))) arcpy.CreateFeatureDataset_management( os.path.dirname(out_gdb), os.path.basename(out_gdb)) elif os.path.dirname(out_gdb).endswith('.mdb'): if not os.path.exists(os.path.dirname(out_gdb)): arcpy.CreatePersonalGDB_management( os.path.dirname(os.path.dirname(out_gdb)), os.path.basename(os.path.dirname(out_gdb))) arcpy.CreateFeatureDataset_management( os.path.dirname(out_gdb), os.path.basename(out_gdb)) status_writer.send_status(_('Setting the output workspace...')) arcpy.env.workspace = out_gdb headers = { 'x-access-token': task_utils.get_security_token(request['owner']) } result_count, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) # query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'add_to_geodatabase') for group in groups: if fq: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers) else: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers) docs = results.json()['response']['docs'] input_items = task_utils.get_input_items(docs) if not input_items: input_items = task_utils.get_input_items( parameters[response_index]['response']['docs']) input_rows = collections.defaultdict(list) for doc in docs: if 'path' not in doc: input_rows[doc['title']].append(doc) if input_rows: result = add_to_geodatabase(input_rows, out_gdb, is_fds) added += result[0] errors += result[1] skipped += result[2] if input_items: result = add_to_geodatabase(input_items, out_gdb, is_fds) added += result[0] errors += result[1] skipped += result[2] if not input_items and not input_rows: status_writer.send_state( status.STAT_FAILED, _('No items to process. Check if items exist.')) return # Update state if necessary. if skipped > 0 or errors > 0: status_writer.send_state( status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(task_folder, '__report.json'), added, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Exports search results a CSV, shapefile or XML document. :param request: json as a dict. """ chunk_size = task_utils.CHUNK_SIZE file_name = task_utils.get_parameter_value(request['params'], 'file_name', 'value') fields = task_utils.get_parameter_value(request['params'], 'fields', 'value') out_format = task_utils.get_parameter_value(request['params'], 'output_format', 'value') if not 'path' in fields and 'path:[absolute]' in fields: fields.append('path') if 'geo' in fields: i_geo = fields.index('geo') fields.remove('geo') fields.insert(i_geo, '[geo]') # Create the temporary workspace. task_folder = os.path.join(request['folder'], 'temp') if not os.path.exists(task_folder): os.makedirs(task_folder) headers = {'x-access-token': task_utils.get_security_token(request['owner'])} num_results, response_index = task_utils.get_result_count(request['params']) query = '{0}/select?&wt=json&fl={1}'.format(sys.argv[2].split('=')[1], ','.join(fields)) if 'query' in request['params'][response_index]: # Voyager Search Traditional UI for p in request['params']: if 'query' in p: request_qry = p['query'] break if 'voyager.list' in request_qry: query += '&voyager.list={0}'.format(request_qry['voyager.list']) # Replace spaces with %20 & remove \\ to avoid HTTP Error 400. if 'fq' in request_qry: try: query += '&fq={0}'.format(request_qry['fq'].replace("\\", "")) query = query.replace(' ', '%20') except AttributeError: for qry in request_qry['fq']: query += '&fq={0}'.format(qry).replace("\\", "").replace(' ', '%20') if 'q' in request_qry: try: query += '&q={0}'.format(request_qry['q'].replace("\\", "")) query = query.replace(' ', '%20') except AttributeError: for qry in request_qry['q']: query += '&q={0}'.format(qry).replace("\\", "").replace(' ', '%20') if 'place' in request_qry: try: query += '&place={0}'.format(request_qry['place'].replace("\\", "")) query = query.replace(' ', '%20') except AttributeError: for qry in request_qry['place']: query += '&place={0}'.format(qry).replace("\\", "").replace(' ', '%20') if 'place.op' in request_qry: query += '&place.op={0}'.format(request_qry['place.op']) query += '&rows={0}&start={1}' exported_cnt = 0. for i in xrange(0, num_results, chunk_size): req = urllib2.Request(query.replace('{0}', str(chunk_size)).replace('{1}', str(i)), headers=headers) for n in urllib2.urlopen(req): jobs = eval(n.replace('null', '"null"'))['response']['docs'] if out_format == 'CSV': export_to_csv(jobs, file_name, task_folder, fields) elif out_format == 'XML': export_to_xml(jobs, file_name, task_folder) elif out_format == 'SHP': export_to_shp(jobs, file_name, task_folder) exported_cnt += chunk_size if exported_cnt > num_results: status_writer.send_percent(100, 'exported: 100%', 'export_results') else: percent_done = exported_cnt / num_results status_writer.send_percent(percent_done, '{0}: {1:.0f}%'.format("exported", percent_done * 100), 'export_results') else: # Voyager Search Portal/Cart UI ids = [] for p in request['params']: if 'ids' in p: ids = p['ids'] break groups = task_utils.grouper(list(ids), chunk_size, '') i = 0 for group in groups: i += len([v for v in group if not v == '']) req = urllib2.Request(query + '&ids={0}'.format(','.join(group)), headers=headers) results = urllib2.urlopen(req) jobs = eval(results.read())['response']['docs'] if out_format == 'CSV': export_to_csv(jobs, file_name, task_folder, fields) elif out_format == 'XML': export_to_xml(jobs, file_name, task_folder) elif out_format == 'SHP': export_to_shp(jobs, file_name, task_folder) percent_done = float(i) / num_results status_writer.send_percent(percent_done, '{0}: {1:.0f}%'.format("exported", percent_done * 100), 'export_results') # Zip up outputs. if exported_count == 0: task_utils.report(os.path.join(request['folder'], '__report.json'), exported_count, 0, errors_count, errors_reasons) else: task_utils.report(os.path.join(request['folder'], '__report.json'), exported_count, 0, errors_count, errors_reasons) zip_file = task_utils.zip_data(task_folder, 'output.zip') shutil.move(zip_file, os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))
def execute(request): """Copies files to a target folder. :param request: json as a dict. """ created = 0 skipped = 0 errors = 0 global result_count parameters = request['params'] if not os.path.exists(request['folder']): os.makedirs(request['folder']) meta_folder = task_utils.get_parameter_value(parameters, 'meta_data_folder', 'value') result_count, response_index = task_utils.get_result_count(parameters) # Query the index for results in groups of 25. query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') query += fq elif 'ids' in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '') else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '') status_writer.send_percent(0.0, _('Starting to process...'), 'create_layer_files') i = 0. headers = {'x-access-token': task_utils.get_security_token(request['owner'])} for group in groups: i += len(group) - group.count('') if fq: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) elif 'ids' in parameters[response_index]: results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers) else: results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers) docs = results.json()['response']['docs'] # docs = eval(results.read().replace('false', 'False').replace('true', 'True'))['response']['docs'] if not docs: docs = parameters[response_index]['response']['docs'] input_items = [] for doc in docs: if 'path' in doc: input_items.append((doc['id'], doc['path'], doc['name'], doc['location'])) result = create_layer_file(input_items, meta_folder) created += result[0] errors += result[1] skipped += result[2] try: shutil.copy2(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', '_thumb.png'), request['folder']) except IOError: pass # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors)) task_utils.report(os.path.join(request['folder'], '__report.json'), created, skipped, errors, errors_reasons, skipped_reasons)
def execute(request): """Clips selected search results using the clip geometry. :param request: json as a dict. """ clipped = 0 errors = 0 skipped = 0 global result_count parameters = request["params"] # Retrieve the clip features. clip_features = task_utils.get_parameter_value(parameters, "clip_features", "value") # Retrieve the coordinate system code. out_coordinate_system = int(task_utils.get_parameter_value(parameters, "output_projection", "code")) # Retrieve the output format, create mxd and output file name parameter values. out_format = task_utils.get_parameter_value(parameters, "output_format", "value") create_mxd = task_utils.get_parameter_value(parameters, "create_mxd", "value") output_file_name = task_utils.get_parameter_value(parameters, "output_file_name", "value") if not output_file_name: output_file_name = "clip_results" # Create the temporary workspace if clip_feature_class: out_workspace = os.path.join(request["folder"], "temp") if not os.path.exists(out_workspace): os.makedirs(out_workspace) # Set the output coordinate system. if not out_coordinate_system == 0: # Same as Input out_sr = task_utils.get_spatial_reference(out_coordinate_system) arcpy.env.outputCoordinateSystem = out_sr # Set the output workspace. status_writer.send_status(_("Setting the output workspace...")) if not out_format == "SHP": out_workspace = arcpy.CreateFileGDB_management(out_workspace, "output.gdb").getOutput(0) arcpy.env.workspace = out_workspace # Query the index for results in groups of 25. headers = {"x-access-token": task_utils.get_security_token(request["owner"])} result_count, response_index = task_utils.get_result_count(parameters) query_index = task_utils.QueryIndex(parameters[response_index]) fl = query_index.fl # Get the Clip features by id. id = clip_features["id"] clip_query = "{0}{1}{2}".format( sys.argv[2].split("=")[1], "/select?&wt=json", "&fl=id,path:[absolute],[lyrFile],[geo]&q=id:{0}".format(id) ) clip_result = requests.get(clip_query, headers=headers) clipper = clip_result.json()["response"]["docs"][0] if "path" in clipper: clip_features = clipper["path"] elif "[lyrFile]" in clipper: clip_features = clipper["[lyrFile]"] elif "[geo]" in clipper: clip_features = arcpy.AsShape(clipper["[geo]"]).projectAs(arcpy.SpatialReference(4326)) else: bbox = clipper["bbox"].split() extent = arcpy.Extent(*bbox) pt_array = arcpy.Array([extent.lowerLeft, extent.upperLeft, extent.upperRight, extent.lowerRight]) clip_features = arcpy.Polygon(pt_array, 4326) query = "{0}{1}{2}".format(sys.argv[2].split("=")[1], "/select?&wt=json", fl) fq = query_index.get_fq() if fq: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "") query += fq elif "ids" in parameters[response_index]: groups = task_utils.grouper(list(parameters[response_index]["ids"]), task_utils.CHUNK_SIZE, "") else: groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "") status_writer.send_percent(0.0, _("Starting to process..."), "clip_data") for group in groups: if fq: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers ) elif "ids" in parameters[response_index]: results = requests.get(query + "{0}&ids={1}".format(fl, ",".join(group)), headers=headers) else: results = requests.get( query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers ) # docs = eval(results.read().replace('false', 'False').replace('true', 'True').replace('null', 'None'))['response']['docs'] docs = results.json()["response"]["docs"] input_items = task_utils.get_input_items(docs) if not input_items: input_items = task_utils.get_input_items(parameters[response_index]["response"]["docs"]) input_rows = collections.defaultdict(list) for doc in docs: if "path" not in doc: input_rows[doc["name"]].append(doc) if input_rows: result = clip_data(input_rows, out_workspace, clip_features, out_format) clipped += result[0] errors += result[1] skipped += result[2] if input_items: result = clip_data(input_items, out_workspace, clip_features, out_format) clipped += result[0] errors += result[1] skipped += result[2] if not input_items and not input_rows: status_writer.send_state(status.STAT_FAILED, _("No items to process. Check if items exist.")) return if arcpy.env.workspace.endswith(".gdb"): out_workspace = os.path.dirname(arcpy.env.workspace) if clipped > 0: try: if out_format == "MPK": mxd_template = os.path.join( os.path.dirname(os.path.dirname(__file__)), "supportfiles", "MapTemplate.mxd" ) mxd = task_utils.create_mxd(out_workspace, mxd_template, "output") status_writer.send_status(_("Packaging results...")) task_utils.create_mpk(out_workspace, mxd, files_to_package) shutil.move( os.path.join(out_workspace, "output.mpk"), os.path.join(os.path.dirname(out_workspace), "{0}.mpk".format(output_file_name)), ) elif out_format == "LPK": status_writer.send_status(_("Packaging results...")) task_utils.create_lpk(out_workspace, output_file_name, files_to_package) elif out_format == "KML": task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb")) arcpy.env.workspace = "" arcpy.RefreshCatalog(os.path.join(out_workspace, "output.gdb")) try: arcpy.Delete_management(os.path.join(out_workspace, "output.gdb")) except arcpy.ExecuteError: pass zip_file = task_utils.zip_data(out_workspace, "{0}.zip".format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) else: if create_mxd: mxd_template = os.path.join( os.path.dirname(os.path.dirname(__file__)), "supportfiles", "MapTemplate.mxd" ) task_utils.create_mxd(out_workspace, mxd_template, "output") zip_file = task_utils.zip_data(out_workspace, "{0}.zip".format(output_file_name)) shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file))) except arcpy.ExecuteError as ee: status_writer.send_state(status.STAT_FAILED, _(ee)) sys.exit(1) else: status_writer.send_state(status.STAT_FAILED, _("No output created. Zero inputs were clipped.")) # Update state if necessary. if errors > 0 or skipped > 0: status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(errors + skipped)) task_utils.report( os.path.join(request["folder"], "__report.json"), clipped, skipped, errors, errors_reasons, skipped_reasons )
def execute(request): """Report duplicate items. :param request: json as a dict. """ if not os.path.exists(request['folder']): os.makedirs(request['folder']) date_str = str(datetime.date.today()) report_file = os.path.join(request['folder'], 'Duplicates_{0}.csv'.format(date_str)) request_owner = request['owner'] headers = {'x-access-token': task_utils.get_security_token(request_owner)} fields = list(task_utils.get_parameter_value(request['params'], 'fields', 'value')) required_fields = ("name", "title", "bytes", "id", "format", "absolute_path", "format_category") [fields.remove(f) for f in required_fields if f in fields] fields_str = "" if fields: fields_str = ",".join(fields) voyager_instance = 'http://localhost:8888/solr/v0' #sys.argv[2].split('=')[1] #voyager_instance = 'http://ec2amaz-7h0t5qu.tnc.sdi.org:8888/solr/v0' query = "%s/select?q={!func}joindf(md5,md5)&f.md5.facet.mincount=2&f.contentHash.facet.mincount=2&f.schemaHash.facet.mincount=2&sort=md5 desc&start=0&rows=1&fl=id,title,name:[name],format,fullpath:[absolute],absolute_path:[absolute],format_type,bytes,md5,path,name&fq={!frange l=2}{!func}joindf(md5,md5)&wt=json" % (voyager_instance) results = requests.get(query, auth=('admin', 'admin')) result_count = results.json()['response']['numFound'] if result_count == 0: status_writer.send_state(status.STAT_WARNING, "No duplicates found.") return duplicates = collections.defaultdict(list) groups = grouper(range(0, result_count), 150, '') url = voyager_instance.split("/solr")[0] req = requests.get("{0}/api/rest/i18n/field/format".format(url), headers=headers) formats = req.json()['VALUE']['format'] processed_count = 0 status_writer.send_status("Generating list of documents with children...") parent_docs = create_listcount_data_in_doc(voyager_instance, headers) increment = task_utils.get_increment(result_count) for group in groups: query = "%s/select?q={!func}joindf(md5,md5)&f.md5.facet.mincount=2&f.contentHash.facet.mincount=2&f.schemaHash.facet.mincount=2&sort=md5 desc&start=%s&rows=150&fl=md5,id,title,name:[name],format,fullpath:[absolute],absolute_path:[absolute],format_type,format_category,bytes,linkcount_data,linkcount_md5,path,name,%s&fq={!frange l=2}{!func}joindf(md5,md5)&wt=json" % (voyager_instance, group[0], fields_str) results = requests.get(query, headers=headers) docs = results.json()['response']['docs'] for doc in docs: file_path = "" if "absolute_path" in doc: file_path = doc["absolute_path"] if os.path.splitext(file_path)[1].lower() in ('.cpg', '.ini'): continue file_size = 0 if "bytes" in doc: file_size = float(doc['bytes']) format_type = "" if "format" in doc and not doc["format"] in ("application/vnd.esri.gdb.file.data", "application/vnd.esri.gdb.personal.data"): try: format_type = formats[doc["format"]] except KeyError: format_type = doc["format"] else: continue format_category = "" if "format_category" in doc: format_category = doc["format_category"] id = "" if "id" in doc: id = doc["id"] name = "" if "name" in doc: name = doc["name"] elif "title" in doc: name = doc["title"] field_dict = {"FILE NAME": name, "FILE SIZE": file_size, "FORMAT CATEGORY": format_category, "FORMAT": format_type, "ID": id, "FILE PATH": file_path} extra_fields = {} if fields: for fld in fields: if fld in doc: extra_fields[fld.upper()] = doc[fld] field_dict.update(extra_fields) duplicates[doc['md5']].append(field_dict) processed_count += len(group) if (processed_count % increment) == 0: status_writer.send_percent(processed_count / float(result_count), 'Grouping duplicates by MD5...', 'report_duplicate_files') # Find total number of items in the index. all_query = "%s/select?&disp=default&sort=score desc&place.op=within&start=0&fl=id&voyager.config.id=ace4bb77&wt=json" % (voyager_instance) results = requests.get(all_query, headers=headers) index_count = results.json()['response']['numFound'] duplicate_count = 0 total_file_size = 0 # Write all the duplicates to the report file. status_writer.send_percent(0, 'Creating the duplicate report and comparing data...', '') processed_count = 0 md5_count = len(duplicates) increment = task_utils.get_increment(md5_count) with open(report_file, "wb") as f: if extra_fields: keys = ["MD5", "FILE NAME", "FILE SIZE", "FORMAT", "FORMAT CATEGORY", "ID", "FILE PATH", "MXD COUNT", "LYR COUNT"] + extra_fields.keys() else: keys = ["MD5", "FILE NAME", "FILE SIZE", "FORMAT", "FORMAT CATEGORY", "ID", "FILE PATH", "MXD COUNT", "LYR COUNT"] writer = csv.DictWriter(f, fieldnames=keys) writer.writeheader() # Loop through each group of MD5 (duplicates). for md5, values in duplicates.items(): try: val_count = len(values) if val_count > 1: # If the files sizes are all 0 bytes, return them as duplicates. file_size = convert_size(values[0]['FILE SIZE']) for z in values: if not convert_size(z['FILE SIZE']) == file_size: values.remove(z) if len(values) == 1: processed_count += 1 continue # Perform data comparison (Feature, Table, File, raster, etc.). If different, don't report and continue. max_severity = compare_data(values) if max_severity > 0: continue writer.writerow({}) duplicate_count += val_count - 1 for val in values: used_in_mxd = 0 used_in_lyr = 0 total_file_size += val["FILE SIZE"] * (val_count - 1) val["MD5"] = md5 val["FILE SIZE"] = convert_size(val["FILE SIZE"]) new_val = {k: change(v, 'utf-8') for (k, v) in val.items()} if new_val["FORMAT CATEGORY"].lower() in ("gis", "cad", "imagery"): for k,v in parent_docs.items(): if new_val['ID'] in v: if k.endswith('.mxd'): used_in_mxd += 1 elif k.endswith('.lyr'): used_in_lyr += 1 new_val['MXD COUNT'] = used_in_mxd val['MXD COUNT'] = used_in_mxd new_val['LYR COUNT'] = used_in_lyr val['LYR COUNT'] = used_in_lyr try: writer.writerow(new_val) except UnicodeEncodeError: try: new_val = {k: change(v, 'latin-1') for (k, v) in new_val.items()} writer.writerow(new_val) except Exception as we: status_writer.send_status('WRITE ERROR: {0}'.format(repr(we))) pass processed_count += 1 except Exception as ex: status_writer.send_status(repr(ex)) processed_count += 1 continue try: if (processed_count % increment) == 0: status_writer.send_percent(processed_count / float(md5_count), 'Creating report and comparing data...', 'report_duplicate_files') except Exception: status_writer.send_status("error reporting progress.") continue try: # Report a summary to the report file. pct_dups = float(duplicate_count) / index_count with open(report_file, "ab") as f: writer = csv.DictWriter(f, fieldnames=["DUPLICATE COUNT", "INDEX COUNT", "PERCENT DUPLICATES", "TOTAL DUPLICATE FILE SIZE"]) writer.writerow({}) writer.writerow({}) writer.writeheader() writer.writerow({"DUPLICATE COUNT": duplicate_count, "INDEX COUNT": index_count, "PERCENT DUPLICATES": '{:.0f}%'.format(pct_dups * 100), "TOTAL DUPLICATE FILE SIZE": convert_size(total_file_size)}) status_writer.send_status("DUPLICATE COUNT: {0}".format(duplicate_count)) status_writer.send_status("INDEX COUNT: {0}".format(index_count)) status_writer.send_status("PERCENT DUPLICATES: {0}".format('{:.0f}%'.format(pct_dups * 100))) status_writer.send_status("TOTAL DUPLICATE FILE SIZE: {0}".format(convert_size(total_file_size))) status_writer.send_state(status.STAT_SUCCESS) except Exception: status_writer.send_status("Error writing summary.")