Ejemplo n.º 1
0
def execute(request):
    """Move files to a target folder.
    :param request: json as a dict.
    """
    moved = 0
    skipped = 0
    errors = 0
    new_folder = False

    parameters = request['params']
    target_folder = task_utils.get_parameter_value(parameters, 'target_folder', 'value')
    flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value')
    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])
    if target_folder:
        if not os.path.exists(target_folder):
            os.makedirs(target_folder)
            new_folder = True

    num_results, response_index = task_utils.get_result_count(parameters)
    task_utils.CHUNK_SIZE = num_results
    # Query the index for results in groups of 25.
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'), 'move_files')
    i = 0.
    for group in groups:
        i += len(group) - group.count('')
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),headers=headers)

        input_items = task_utils.get_input_items(results.json()['response']['docs'], True, True)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])

        result = move_files(input_items, target_folder, flatten_results)
        moved += result[0]
        errors += result[1]
        skipped += result[2]
        status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'move_files')

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'], '__report.json'), moved, skipped, errors, errors_reasons, skipped_reasons)
Ejemplo n.º 2
0
def execute(request):
    """Copies files to a target folder.
    :param request: json as a dict.
    """
    copied = 0
    skipped = 0
    errors = 0
    global result_count
    parameters = request['params']

    target_dirs = ''
    target_folder = task_utils.get_parameter_value(parameters, 'target_folder', 'value')
    flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value')
    if not flatten_results:
        target_dirs = os.path.splitdrive(target_folder)[1]
        flatten_results = 'false'
    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    # Query the index for results in groups of 25.
    result_count, response_index = task_utils.get_result_count(parameters)
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')

    # Begin processing
    status_writer.send_percent(0.0, _('Starting to process...'), 'copy_files')
    i = 0.
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    for group in groups:
        i += len(group) - group.count('')
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

        input_items = task_utils.get_input_items(results.json()['response']['docs'], list_components=True)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
        result = copy_files(input_items, target_folder, flatten_results, target_dirs)
        copied += result[0]
        errors += result[1]
        skipped += result[2]

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'], '__report.json'), copied, skipped, errors, errors_reasons, skipped_reasons)
Ejemplo n.º 3
0
def execute(request):
    """Copies files to a target folder.
    :param request: json as a dict.
    """
    copied = 0
    skipped = 0
    errors = 0
    global result_count
    parameters = request['params']

    target_dirs = ''
    target_folder = task_utils.get_parameter_value(parameters, 'target_folder', 'value')
    flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value')
    if not flatten_results:
        target_dirs = os.path.splitdrive(target_folder)[1]
        flatten_results = 'false'
    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    # Query the index for results in groups of 25.
    result_count, response_index = task_utils.get_result_count(parameters)
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')

    # Begin processing
    status_writer.send_percent(0.0, _('Starting to process...'), 'copy_files')
    i = 0.
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    for group in groups:
        i += len(group) - group.count('')
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)

        input_items = task_utils.get_input_items(results.json()['response']['docs'], list_components=True)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
        result = copy_files(input_items, target_folder, flatten_results, target_dirs)
        copied += result[0]
        errors += result[1]
        skipped += result[2]

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'], '__report.json'), copied, skipped, errors, errors_reasons, skipped_reasons)
Ejemplo n.º 4
0
def execute(request):
    """Remove tags.
    :param request: json as a dict.
    """
    query = ''
    errors = 0
    parameters = request['params']
    archive_location = request['folder']
    if not os.path.exists(archive_location):
        os.makedirs(archive_location)

    # Parameter values
    snapshot_name = task_utils.get_parameter_value(parameters, 'snapshot_name', 'value')
    data_folder = task_utils.get_parameter_value(parameters, 'data_folder', 'value')
    request_owner = request['owner']

    result_count, response_index = task_utils.get_result_count(parameters)
    # Get the query index and query (if any).
    # query_index = task_utils.QueryIndex(parameters[response_index])
    fq = ''
    if 'fq' in parameters[response_index]['query']:
        if isinstance(parameters[response_index]['query']['fq'], list):
            for q in parameters[response_index]['query']['fq']:
                if '{!tag=' in q:
                    q = q.split('}')[1]
                fq += q + ' AND '
            fq = fq.strip(' AND ')
        else:
            # Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
            fq += '&fq={0}'.format(parameters[response_index]['query']['fq'].replace("\\", ""))
            fq = fq.replace(' ', '%20')

    # fq = query_index.get_fq()
    if fq:
        query = fq.replace('&fq=', '')
    if query:
        result = create_snapshot(snapshot_name, request_owner, query)
    else:
        result = create_snapshot(snapshot_name, request_owner)
    if not result[0]:
        errors += 1
        errors_reasons[snapshot_name] = result[1]

    # Update state if necessary.
    if errors > 0:
        status_writer.send_state(status.STAT_FAILED)
    else:
        status_writer.send_status(result[1])
        result_name = ''
        while not result_name:
            file_path = os.path.join(data_folder, 'backup', '*{0}.zip'.format(snapshot_name))
            try:
                result_name = os.path.basename(glob.glob(file_path)[0])
            except IndexError:
                continue
        shutil.copyfile(os.path.join(data_folder, 'backup', result_name), os.path.join(archive_location, result_name))
    task_utils.report(os.path.join(request['folder'], '__report.json'), 1, 0, errors, errors_details=errors_reasons)
Ejemplo n.º 5
0
def execute(request):
    """Delete items.
    :param request: json as a dict.
    """
    query = ''
    errors = 0
    parameters = request['params']
    archive_location = request['folder']
    if not os.path.exists(archive_location):
        os.makedirs(archive_location)

    # Parameter values
    delete_thumbs = task_utils.get_parameter_value(
        parameters, 'delete_thumbnails', 'value') or False
    delete_metadata = task_utils.get_parameter_value(
        parameters, 'delete_metadata', 'value') or False
    delete_layers = task_utils.get_parameter_value(parameters, 'delete_layers',
                                                   'value') or False
    request_owner = request['owner']

    result_count, response_index = task_utils.get_result_count(parameters)
    fq = ''
    if 'fq' in parameters[response_index]['query']:
        if isinstance(parameters[response_index]['query']['fq'], list):
            for q in parameters[response_index]['query']['fq']:
                if '{!tag=' in q:
                    q = q.split('}')[1]
                fq += q + ' AND '
            fq = fq.strip(' AND ')
        else:
            # Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
            fq = parameters[response_index]['query']['fq'].replace("\\", "")

    if 'q' in parameters[response_index]['query']:
        query = parameters[response_index]['query']['q']

    result = delete_items(fq, query, delete_thumbs, delete_metadata,
                          delete_layers, request_owner)

    if not result[0]:
        errors += 1
        errors_reasons['delete_items'] = result[1]

    # Update state if necessary.
    if errors > 0:
        status_writer.send_state(status.STAT_FAILED)
    else:
        status_writer.send_status(result[1])
    task_utils.report(os.path.join(request['folder'], '__report.json'),
                      1,
                      0,
                      errors,
                      errors_details=errors_reasons)
Ejemplo n.º 6
0
def execute(request):
    """Delete items.
    :param request: json as a dict.
    """
    query = ''
    errors = 0
    parameters = request['params']
    archive_location = request['folder']
    if not os.path.exists(archive_location):
        os.makedirs(archive_location)

    # Parameter values
    delete_thumbs = task_utils.get_parameter_value(parameters, 'delete_thumbnails', 'value') or False
    delete_metadata = task_utils.get_parameter_value(parameters, 'delete_metadata', 'value') or False
    delete_layers = task_utils.get_parameter_value(parameters, 'delete_layers', 'value') or False
    request_owner = request['owner']

    result_count, response_index = task_utils.get_result_count(parameters)
    fq = ''
    if 'fq' in parameters[response_index]['query']:
        if isinstance(parameters[response_index]['query']['fq'], list):
            for q in parameters[response_index]['query']['fq']:
                if '{!tag=' in q:
                    q = q.split('}')[1]
                fq += q + ' AND '
            fq = fq.strip(' AND ')
        else:
            # Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
            fq = parameters[response_index]['query']['fq'].replace("\\", "")

    if 'q' in parameters[response_index]['query']:
        query = parameters[response_index]['query']['q']

    result = delete_items(fq, query, delete_thumbs, delete_metadata, delete_layers, request_owner)

    if not result[0]:
        errors += 1
        errors_reasons['delete_items'] = result[1]

    # Update state if necessary.
    if errors > 0:
        status_writer.send_state(status.STAT_FAILED)
    else:
        status_writer.send_status(result[1])
    task_utils.report(os.path.join(request['folder'], '__report.json'), 1, 0, errors, errors_details=errors_reasons)
Ejemplo n.º 7
0
def execute(request):
    """Converts each input dataset to kml (.kmz).
    :param request: json as a dict.
    """
    converted = 0
    skipped = 0
    errors = 0
    global result_count
    parameters = request['params']

    out_workspace = os.path.join(request['folder'], 'temp')
    if not os.path.exists(out_workspace):
        os.makedirs(out_workspace)

    # Get the boundary box extent for input to KML tools.
    extent = ''
    try:
        try:
            ext = task_utils.get_parameter_value(parameters,
                                                 'processing_extent', 'wkt')
            if ext:
                sr = task_utils.get_spatial_reference("4326")
                extent = task_utils.from_wkt(ext, sr)
        except KeyError:
            ext = task_utils.get_parameter_value(parameters,
                                                 'processing_extent',
                                                 'feature')
            if ext:
                extent = arcpy.Describe(ext).extent
    except KeyError:
        pass

    # Get the output file name.
    output_file_name = task_utils.get_parameter_value(parameters,
                                                      'output_file_name',
                                                      'value')
    if not output_file_name:
        output_file_name = 'kml_results'

    result_count, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
                               fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']),
                                    task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')

    # Begin processing
    status_writer.send_percent(0.0, _('Starting to process...'),
                               'convert_to_kml')
    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    for group in groups:
        if fq:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query +
                                   '{0}&ids={1}'.format(fl, ','.join(group)),
                                   headers=headers)
        else:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                headers=headers)

        docs = results.json()['response']['docs']
        input_items = task_utils.get_input_items(docs)
        if not input_items:
            input_items = task_utils.get_input_items(
                parameters[response_index]['response']['docs'])

        input_rows = collections.defaultdict(list)
        for doc in docs:
            if 'path' not in doc:
                input_rows[doc['name']].append(doc)
        if input_rows:
            result = convert_to_kml(input_rows, out_workspace, extent)
            converted += result[0]
            errors += result[1]
            skipped += result[2]

        if input_items:
            result = convert_to_kml(input_items, out_workspace, extent)
            converted += result[0]
            errors += result[1]
            skipped += result[2]

        if not input_items and not input_rows:
            status_writer.send_state(
                status.STAT_FAILED,
                _('No items to process. Check if items exist.'))
            return

    # Zip up kmz files if more than one.
    if converted > 1:
        status_writer.send_status("Converted: {}".format(converted))
        zip_file = task_utils.zip_data(out_workspace,
                                       '{0}.zip'.format(output_file_name))
        shutil.move(
            zip_file,
            os.path.join(os.path.dirname(out_workspace),
                         os.path.basename(zip_file)))
        shutil.copy2(
            os.path.join(os.path.dirname(os.path.dirname(__file__)),
                         'supportfiles', '_thumb.png'), request['folder'])
    elif converted == 1:
        try:
            kml_file = glob.glob(os.path.join(out_workspace, '*.kmz'))[0]
            tmp_lyr = arcpy.KMLToLayer_conversion(kml_file, out_workspace,
                                                  'kml_layer')
            task_utils.make_thumbnail(
                tmp_lyr.getOutput(0),
                os.path.join(request['folder'], '_thumb.png'))
        except arcpy.ExecuteError:
            pass
        shutil.move(
            kml_file,
            os.path.join(request['folder'], os.path.basename(kml_file)))

    # Update state if necessary.
    if skipped > 0 or errors > 0:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(errors + skipped))
    task_utils.report(os.path.join(request['folder'],
                                   '__report.json'), converted, skipped,
                      errors, errors_reasons, skipped_reasons)
Ejemplo n.º 8
0
def execute(request):
    """Mosaics input raster datasets into a new raster dataset.
    :param request: json as a dict.
    """
    parameters = request['params']
    out_coordinate_system = task_utils.get_parameter_value(parameters, 'output_projection', 'code')
    # Advanced options
    output_raster_format = task_utils.get_parameter_value(parameters, 'raster_format', 'value')
    compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value')
    compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value')
    output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
    if not output_file_name:
        output_file_name = 'output'
    arcpy.env.compression = '{0} {1}'.format(compression_method, compression_quality)

    clip_area = None
    if not output_raster_format == 'MosaicDataset':
        # Get the clip region as an extent object.
        try:
            clip_area_wkt = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt')
            if not clip_area_wkt:
                clip_area_wkt = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'
            if not out_coordinate_system == '0':
                clip_area = task_utils.get_clip_region(clip_area_wkt, out_coordinate_system)
            else:
                clip_area = task_utils.get_clip_region(clip_area_wkt)
        except KeyError:
            pass

    status_writer.send_status(_('Setting the output workspace...'))
    out_workspace = os.path.join(request['folder'], 'temp')
    if not os.path.exists(out_workspace):
        os.makedirs(out_workspace)
    if output_raster_format == 'FileGDB' or output_raster_format == 'MosaicDataset':
        out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0)
    arcpy.env.workspace = out_workspace

    status_writer.send_status(_('Starting to process...'))
    num_results, response_index = task_utils.get_result_count(parameters)
    raster_items = None
    if num_results > task_utils.CHUNK_SIZE:
        # Query the index for results in groups of 25.
        query_index = task_utils.QueryIndex(parameters[response_index])
        fl = query_index.fl
        query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
        fq = query_index.get_fq()
        if fq:
            groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
            query += fq
        elif 'ids' in parameters[response_index]:
            groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
        else:
            groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')

        headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
        for group in groups:
            if fq:
                results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
            elif 'ids' in parameters[response_index]:
                results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
            else:
                results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

            input_items = task_utils.get_input_items(results.json()['response']['docs'])
            if not input_items:
                input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
            raster_items, pixels, bands, skipped = get_items(input_items)
    else:
        input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
        raster_items, pixels, bands, skipped = get_items(input_items)

    if not raster_items:
        if skipped == 0:
            status_writer.send_state(status.STAT_FAILED, _('Invalid input types'))
            skipped_reasons['All Items'] = _('Invalid input types')
            task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), num_results, skipped_details=skipped_reasons)
            return
        else:
            status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
            task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons)
            return

    # Get most common pixel type.
    pixel_type = pixel_types[max(set(pixels), key=pixels.count)]
    if output_raster_format in ('FileGDB', 'GRID', 'MosaicDataset'):
        output_name = arcpy.ValidateTableName('mosaic', out_workspace)
    else:
        output_name = '{0}.{1}'.format(arcpy.ValidateTableName('mosaic', out_workspace)[:9], output_raster_format.lower())
        status_writer.send_status(output_name)

    if output_raster_format == 'MosaicDataset':
        try:
            status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters')))
            if out_coordinate_system == '0':
                out_coordinate_system = raster_items[0]
            else:
                out_coordinate_system = None
            mosaic_ds = arcpy.CreateMosaicDataset_management(out_workspace,
                                                             output_name,
                                                             out_coordinate_system,
                                                             max(bands),
                                                             pixel_type)
            arcpy.AddRastersToMosaicDataset_management(mosaic_ds, 'Raster Dataset', raster_items)
            arcpy.MakeMosaicLayer_management(mosaic_ds, 'mosaic_layer')
            layer_object = arcpy.mapping.Layer('mosaic_layer')
            task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png'))
        except arcpy.ExecuteError:
            status_writer.send_state(status.STAT_FAILED, arcpy.GetMessages(2))
            return
    else:
        try:
            if len(bands) > 1:
                status_writer.send_state(status.STAT_FAILED, _('Input rasters must have the same number of bands'))
                return
            status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters')))
            if out_coordinate_system == '0':
                out_coordinate_system = None
            if clip_area:
                ext = '{0} {1} {2} {3}'.format(clip_area.XMin, clip_area.YMin, clip_area.XMax, clip_area.YMax)
                tmp_mosaic = arcpy.MosaicToNewRaster_management(
                    raster_items,
                    out_workspace,
                    'tm',
                    out_coordinate_system,
                    pixel_type,
                    number_of_bands=bands.keys()[0]
                )
                status_writer.send_status(_('Clipping...'))
                out_mosaic = arcpy.Clip_management(tmp_mosaic, ext, output_name)
                arcpy.Delete_management(tmp_mosaic)
            else:
                out_mosaic = arcpy.MosaicToNewRaster_management(raster_items,
                                                                out_workspace,
                                                                output_name,
                                                                out_coordinate_system,
                                                                pixel_type, number_of_bands=bands.keys()[0])
            arcpy.MakeRasterLayer_management(out_mosaic, 'mosaic_layer')
            layer_object = arcpy.mapping.Layer('mosaic_layer')
            task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png'))
        except arcpy.ExecuteError:
            status_writer.send_state(status.STAT_FAILED, arcpy.GetMessages(2))
            return

    if arcpy.env.workspace.endswith('.gdb'):
        out_workspace = os.path.dirname(arcpy.env.workspace)
    zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
    shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))

    # Update state if necessary.
    if skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons)
Ejemplo n.º 9
0
def execute(request):
    """Writes existing metadata for summary, description and tags.
    If overwrite is false, existing metadata is untouched unless any
    field is empty or does not exist, then it is created.

    :param request: json as a dict.
    """
    updated = 0
    errors = 0
    skipped = 0
    global result_count
    parameters = request['params']
    summary = task_utils.get_parameter_value(parameters, 'summary', 'value')
    description = task_utils.get_parameter_value(parameters, 'description',
                                                 'value')
    tags = task_utils.get_parameter_value(parameters, 'tags', 'value')
    data_credits = task_utils.get_parameter_value(parameters, 'credits',
                                                  'value')
    constraints = task_utils.get_parameter_value(parameters, 'constraints',
                                                 'value')

    # Handle commas, spaces, and/or new line separators.
    tags = [tag for tag in re.split(' |,|\n', tags) if not tag == '']
    overwrite = task_utils.get_parameter_value(parameters, 'overwrite',
                                               'value')
    if not overwrite:
        overwrite = False
    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    # Stylesheet
    xslt_file = os.path.join(
        arcpy.GetInstallInfo()['InstallDir'],
        'Metadata/Stylesheets/gpTools/exact copy of.xslt')

    # Template metadata file.
    template_xml = os.path.join(os.path.dirname(os.path.dirname(__file__)),
                                'supportfiles', 'metadata_template.xml')

    result_count, response_index = task_utils.get_result_count(parameters)
    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl + ',links'
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
                               fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']),
                                    task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'),
                               'write_metadata')
    i = 0.
    for group in groups:
        i += len(group) - group.count('')
        if fq:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query +
                                   '{0}&ids={1}'.format(fl, ','.join(group)),
                                   headers=headers)
        else:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                headers=headers)

        docs = results.json()['response']['docs']
        if not docs:
            docs = parameters[response_index]['response']['docs']

        input_items = []
        for doc in docs:
            if 'path' in doc:
                if 'links' in doc:
                    links = eval(doc['links'])
                    input_items.append(
                        (doc['path'], links['links'][0]['link'][0]['id']))
                else:
                    input_items.append((doc['path'], doc['id']))
        result = write_metadata(input_items, template_xml, xslt_file, summary,
                                description, tags, data_credits, constraints,
                                overwrite, headers)
        updated += result[0]
        errors += result[1]
        skipped += result[2]
        status_writer.send_percent(
            i / result_count, '{0}: {1:.0f}%'.format("Processed",
                                                     i / result_count * 100),
            'write_metadata')

    # Report state.
    if skipped > 0 or errors > 0:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(skipped + errors))
    else:
        status_writer.send_state(status.STAT_SUCCESS)
    task_utils.report(os.path.join(request['folder'],
                                   '__report.json'), updated, skipped, errors,
                      errors_reasons, skipped_reasons)
Ejemplo n.º 10
0
def execute(request):
    """Converts each input dataset to kml (.kmz).
    :param request: json as a dict.
    """
    converted = 0
    skipped = 0
    errors = 0
    global result_count
    parameters = request['params']

    out_workspace = os.path.join(request['folder'], 'temp')
    if not os.path.exists(out_workspace):
        os.makedirs(out_workspace)

    # Get the boundary box extent for input to KML tools.
    extent = ''
    try:
        try:
            ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt')
            if ext:
                sr = task_utils.get_spatial_reference("4326")
                extent = task_utils.from_wkt(ext, sr)
        except KeyError:
            ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'feature')
            if ext:
                extent = arcpy.Describe(ext).extent
    except KeyError:
        pass

    # Get the output file name.
    output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
    if not output_file_name:
        output_file_name = 'kml_results'

    result_count, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')

    # Begin processing
    status_writer.send_percent(0.0, _('Starting to process...'), 'convert_to_kml')
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    for group in groups:
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

        docs = results.json()['response']['docs']
        input_items = task_utils.get_input_items(docs)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])

        input_rows = collections.defaultdict(list)
        for doc in docs:
            if 'path' not in doc:
               input_rows[doc['name']].append(doc)
        if input_rows:
            result = convert_to_kml(input_rows, out_workspace, extent)
            converted += result[0]
            errors += result[1]
            skipped += result[2]

        if input_items:
            result = convert_to_kml(input_items, out_workspace, extent)
            converted += result[0]
            errors += result[1]
            skipped += result[2]

        if not input_items and not input_rows:
            status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.'))
            return

    # Zip up kmz files if more than one.
    if converted > 1:
        status_writer.send_status("Converted: {}".format(converted))
        zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
        shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
        shutil.copy2(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', '_thumb.png'), request['folder'])
    elif converted == 1:
        try:
            kml_file = glob.glob(os.path.join(out_workspace, '*.kmz'))[0]
            tmp_lyr = arcpy.KMLToLayer_conversion(kml_file, out_workspace, 'kml_layer')
            task_utils.make_thumbnail(tmp_lyr.getOutput(0), os.path.join(request['folder'], '_thumb.png'))
        except arcpy.ExecuteError:
            pass
        shutil.move(kml_file, os.path.join(request['folder'], os.path.basename(kml_file)))

    # Update state if necessary.
    if skipped > 0 or errors > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), converted, skipped, errors, errors_reasons, skipped_reasons)
Ejemplo n.º 11
0
def execute(request):
    """Zips all input files to output.zip.
    :param request: json as a dict.
    """
    zipped = 0
    skipped = 0
    parameters = request['params']
    flatten_results = task_utils.get_parameter_value(parameters, 'flatten_results', 'value')
    if not flatten_results:
        flatten_results = False
    zip_file_location = request['folder']
    if not os.path.exists(zip_file_location):
        os.makedirs(request['folder'])

    output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
    if not output_file_name:
        output_file_name = 'output'
    zip_file = os.path.join(zip_file_location, '{0}.zip'.format(output_file_name))
    zipper = task_utils.ZipFileManager(zip_file, 'w', zipfile.ZIP_DEFLATED)
    num_results, response_index = task_utils.get_result_count(parameters)
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    #query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'), 'zip_files')
    i = 0.
    for group in groups:
        i += len(group) - group.count('')
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

        input_items = task_utils.get_input_items(results.json()['response']['docs'], list_components=True)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'], list_components=True)
        result = zip_files(zipper, input_items, zip_file_location, flatten_results)
        zipped += result[0]
        skipped += result[1]
        status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'zip_files')

    zipper.close()

    if zipped == 0:
        status_writer.send_state(status.STAT_FAILED, _('No results were zipped.'))
        return

    # Update state if necessary.
    if skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), zipped, skipped, skipped_details=skipped_reasons)
Ejemplo n.º 12
0
def execute(request):
    """Exports search results a CSV, shapefile or XML document.
    :param request: json as a dict.
    """
    chunk_size = task_utils.CHUNK_SIZE
    file_name = task_utils.get_parameter_value(request['params'], 'file_name',
                                               'value')
    fields = task_utils.get_parameter_value(request['params'], 'fields',
                                            'value')
    out_format = task_utils.get_parameter_value(request['params'],
                                                'output_format', 'value')

    if not 'path' in fields and 'path:[absolute]' in fields:
        fields.append('path')

    if 'geo' in fields:
        i_geo = fields.index('geo')
        fields.remove('geo')
        fields.insert(i_geo, '[geo]')

    # Create the temporary workspace.
    task_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)

    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    num_results, response_index = task_utils.get_result_count(
        request['params'])
    query = '{0}/select?&wt=json&fl={1}'.format(sys.argv[2].split('=')[1],
                                                ','.join(fields))
    if 'query' in request['params'][response_index]:
        # Voyager Search Traditional UI
        for p in request['params']:
            if 'query' in p:
                request_qry = p['query']
                break
        if 'voyager.list' in request_qry:
            query += '&voyager.list={0}'.format(request_qry['voyager.list'])

        # Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
        if 'fq' in request_qry:
            try:
                query += '&fq={0}'.format(request_qry['fq'].replace("\\", ""))
                query = query.replace(' ', '%20')
            except AttributeError:
                for qry in request_qry['fq']:
                    query += '&fq={0}'.format(qry).replace("\\", "").replace(
                        ' ', '%20')
        if 'q' in request_qry:
            try:
                query += '&q={0}'.format(request_qry['q'].replace("\\", ""))
                query = query.replace(' ', '%20')
            except AttributeError:
                for qry in request_qry['q']:
                    query += '&q={0}'.format(qry).replace("\\", "").replace(
                        ' ', '%20')
        if 'place' in request_qry:
            try:
                query += '&place={0}'.format(request_qry['place'].replace(
                    "\\", ""))
                query = query.replace(' ', '%20')
            except AttributeError:
                for qry in request_qry['place']:
                    query += '&place={0}'.format(qry).replace("\\",
                                                              "").replace(
                                                                  ' ', '%20')
        if 'place.op' in request_qry:
            query += '&place.op={0}'.format(request_qry['place.op'])

        query += '&rows={0}&start={1}'
        exported_cnt = 0.
        for i in xrange(0, num_results, chunk_size):
            req = urllib2.Request(query.replace('{0}',
                                                str(chunk_size)).replace(
                                                    '{1}', str(i)),
                                  headers=headers)
            for n in urllib2.urlopen(req):
                jobs = eval(n.replace('null', '"null"'))['response']['docs']
                if out_format == 'CSV':
                    export_to_csv(jobs, file_name, task_folder, fields)
                elif out_format == 'XML':
                    export_to_xml(jobs, file_name, task_folder)
                elif out_format == 'SHP':
                    export_to_shp(jobs, file_name, task_folder)
                exported_cnt += chunk_size
                if exported_cnt > num_results:
                    status_writer.send_percent(100, 'exported: 100%',
                                               'export_results')
                else:
                    percent_done = exported_cnt / num_results
                    status_writer.send_percent(
                        percent_done,
                        '{0}: {1:.0f}%'.format("exported", percent_done * 100),
                        'export_results')
    else:
        # Voyager Search Portal/Cart UI
        ids = []
        for p in request['params']:
            if 'ids' in p:
                ids = p['ids']
                break
        groups = task_utils.grouper(list(ids), chunk_size, '')
        i = 0
        for group in groups:
            i += len([v for v in group if not v == ''])
            req = urllib2.Request(query + '&ids={0}'.format(','.join(group)),
                                  headers=headers)
            results = urllib2.urlopen(req)
            jobs = eval(results.read())['response']['docs']
            if out_format == 'CSV':
                export_to_csv(jobs, file_name, task_folder, fields)
            elif out_format == 'XML':
                export_to_xml(jobs, file_name, task_folder)
            elif out_format == 'SHP':
                export_to_shp(jobs, file_name, task_folder)
            percent_done = float(i) / num_results
            status_writer.send_percent(
                percent_done, '{0}: {1:.0f}%'.format("exported",
                                                     percent_done * 100),
                'export_results')

    # Zip up outputs.
    if exported_count == 0:
        task_utils.report(os.path.join(request['folder'], '__report.json'),
                          exported_count, 0, errors_count, errors_reasons)
    else:
        task_utils.report(os.path.join(request['folder'], '__report.json'),
                          exported_count, 0, errors_count, errors_reasons)
    zip_file = task_utils.zip_data(task_folder, 'output.zip')
    shutil.move(
        zip_file,
        os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))
Ejemplo n.º 13
0
def execute(request):
    """Replace the workspace path for layer files and map document layers.
    :param request: json as a dict.
    """
    updated = 0
    skipped = 0
    parameters = request['params']
    backup = task_utils.get_parameter_value(parameters, 'create_backup', 'value')
    old_data_source = task_utils.get_parameter_value(parameters, 'old_data_source', 'value').lower()
    new_data_source = task_utils.get_parameter_value(parameters, 'new_data_source', 'value')

    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    if not arcpy.Exists(new_data_source):
        status_writer.send_state(status.STAT_FAILED, _('{0} does not exist').format(new_data_source))
        return

    if os.path.splitext(new_data_source)[1] not in ('.gdb', '.mdb', '.sde'):
        new_dataset = os.path.basename(new_data_source)
        dsc = arcpy.Describe(os.path.dirname(new_data_source))
    else:
        dsc = arcpy.Describe(new_data_source)
        new_dataset = ''
    wks_type = 'NONE'
    if dsc.dataType == 'FeatureDataset':
        new_workspace = dsc.path
        wks_type = get_workspace_type(new_workspace)
    elif dsc.dataType == 'Workspace':
        new_workspace = dsc.catalogPath
        wks_type = get_workspace_type(new_workspace)
    elif dsc.dataType == 'Folder':
        dsc = arcpy.Describe(new_data_source)
        new_workspace = dsc.catalogPath
        if new_dataset.endswith('.shp'):
            wks_type = 'SHAPEFILE_WORKSPACE'
            new_dataset = new_dataset.rsplit('.shp')[0]
        else:
            if arcpy.Describe(new_data_source).dataType == 'RasterDataset':
                wks_type = 'RASTER_WORKSPACE'
    elif dsc.dataType == 'CadDrawingDataset':
        new_workspace = dsc.path
        wks_type = 'CAD_WORKSPACE'
    else:
        new_workspace = os.path.dirname(new_data_source)

    num_results, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}'.format(sys.argv[2].split('=')[1], '/select?&wt=json')
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'), 'replace_data_source')
    i = 0.
    for group in groups:
        i += len(group) - group.count('')
        if fq:
            results = requests.get(query + "{0}&rows={1}&start={2}".format(fl, task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers= headers)
        else:
            results = requests.get(query + "{0}&rows={1}&start={2}".format(fl, task_utils.CHUNK_SIZE, group[0]), headers=headers)

        input_items = task_utils.get_input_items(results.json()['response']['docs'], True)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
        result = replace_data_source(input_items, old_data_source, new_workspace, new_dataset, wks_type, backup, headers)
        updated += result[0]
        skipped += result[1]
        status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'replace_data_source')

    # Update state if necessary.
    if skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), updated, skipped, skipped_details=skipped_reasons)
Ejemplo n.º 14
0
def execute(request):
    """Creates a GeoPDF.
    :param request: json as a dict.
    """
    added_to_map = 0
    errors = 0
    skipped = 0

    parameters = request["params"]
    num_results, response_index = task_utils.get_result_count(parameters)
    docs = parameters[response_index]["response"]["docs"]
    input_items = task_utils.get_input_items(docs)
    input_rows = collections.defaultdict(list)
    for doc in docs:
        if "path" not in doc:
            input_rows[doc["name"]].append(doc)
    if num_results > task_utils.CHUNK_SIZE:
        status_writer.send_state(status.STAT_FAILED, "Reduce results to 25 or less.")
        return

    map_template = task_utils.get_parameter_value(parameters, "map_template", "value")
    base_map = task_utils.get_parameter_value(parameters, "base_map", "value")
    map_title = task_utils.get_parameter_value(parameters, "map_title", "value")
    attribute_setting = task_utils.get_parameter_value(parameters, "attribute_settings", "value")
    author = task_utils.get_parameter_value(parameters, "map_author", "value")
    output_file_name = task_utils.get_parameter_value(parameters, "output_file_name", "value")
    if not output_file_name:
        output_file_name = "output_pdf"
    try:
        map_view = task_utils.get_parameter_value(parameters, "map_view", "extent")
    except KeyError:
        map_view = None
        pass

    temp_folder = os.path.join(request["folder"], "temp")
    if not os.path.exists(temp_folder):
        os.makedirs(temp_folder)

    if base_map == "NONE":
        base_layer = None
    else:
        base_layer = arcpy.mapping.Layer(
            os.path.join(
                os.path.dirname(os.path.dirname(__file__)), "supportfiles", "basemaps", "{0}.lyr".format(base_map)
            )
        )
    mxd_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "supportfiles", "frame", map_template)
    mxd = arcpy.mapping.MapDocument(mxd_path)
    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]

    layers = []
    all_layers = []

    if input_rows:
        for name, rows in input_rows.iteritems():
            for row in rows:
                try:
                    name = arcpy.CreateUniqueName(name, "in_memory")
                    # Create the geometry.
                    geo_json = row["[geo]"]
                    geom = arcpy.AsShape(geo_json)
                    arcpy.CopyFeatures_management(geom, name)
                    feature_layer = arcpy.MakeFeatureLayer_management(name, os.path.basename(name))
                    layer_file = arcpy.SaveToLayerFile_management(
                        feature_layer, os.path.join(temp_folder, "{0}.lyr".format(os.path.basename(name)))
                    )
                    layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
                    all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
                    added_to_map += 1
                except KeyError:
                    skipped += 1
                    skipped_reasons[name] = "No geographic information"
                    continue

    for i, item in enumerate(input_items, 1):
        try:
            # Is the item a mxd data frame.
            map_frame_name = task_utils.get_data_frame_name(item)
            if map_frame_name:
                item = item.split("|")[0].strip()

            dsc = arcpy.Describe(item)
            if dsc.dataType == "Layer":
                layers.append(arcpy.mapping.Layer(dsc.catalogPath))

            elif dsc.dataType == "FeatureClass" or dsc.dataType == "ShapeFile":
                if os.path.basename(item) in [l.name for l in all_layers]:
                    layer_name = "{0}_{1}".format(os.path.basename(item), i)
                else:
                    layer_name = os.path.basename(item)
                feature_layer = arcpy.MakeFeatureLayer_management(item, layer_name)
                layer_file = arcpy.SaveToLayerFile_management(
                    feature_layer, os.path.join(temp_folder, "{0}.lyr".format(layer_name))
                )
                layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
                all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))

            elif dsc.dataType == "FeatureDataset":
                arcpy.env.workspace = item
                for fc in arcpy.ListFeatureClasses():
                    layer_file = arcpy.SaveToLayerFile_management(
                        arcpy.MakeFeatureLayer_management(fc, "{0}_{1}".format(fc, i)),
                        os.path.join(temp_folder, "{0}_{1}.lyr".format(fc, i)),
                    )
                    layer = arcpy.mapping.Layer(layer_file.getOutput(0))
                    layer.name = fc
                    layers.append(layer)
                    all_layers.append(layer)

            elif dsc.dataType == "RasterDataset":
                if os.path.basename(item) in [l.name for l in all_layers]:
                    layer_name = "{0}_{1}".format(os.path.basename(item), i)
                else:
                    layer_name = os.path.basename(item)
                raster_layer = arcpy.MakeRasterLayer_management(item, layer_name)
                layer_file = arcpy.SaveToLayerFile_management(
                    raster_layer, os.path.join(temp_folder, "{0}.lyr".format(layer_name))
                )
                layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
                all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))

            elif dsc.catalogPath.endswith(".kml") or dsc.catalogPath.endswith(".kmz"):
                if not os.path.splitext(dsc.name)[0] in layers:
                    name = os.path.splitext(dsc.name)[0]
                else:
                    name = "{0}_{1}".format(os.path.splitext(dsc.name)[0], i)
                arcpy.KMLToLayer_conversion(dsc.catalogPath, temp_folder, name)
                layers.append(arcpy.mapping.Layer(os.path.join(temp_folder, "{0}.lyr".format(name))))
                all_layers.append(arcpy.mapping.Layer(os.path.join(temp_folder, "{0}.lyr".format(name))))

            elif dsc.dataType == "MapDocument":
                input_mxd = arcpy.mapping.MapDocument(item)
                if map_frame_name:
                    df = arcpy.mapping.ListDataFrames(input_mxd, map_frame_name)[0]
                    layers = arcpy.mapping.ListLayers(input_mxd, data_frame=df)
                else:
                    layers = arcpy.mapping.ListLayers(input_mxd)

            if layers:
                for layer in layers:
                    status_writer.send_status(_("Adding layer {0}...").format(layer.name))
                    arcpy.mapping.AddLayer(data_frame, layer)
                    added_to_map += 1
                    layers = []
            else:
                status_writer.send_status(_("Invalid input type: {0}").format(item))
                skipped_reasons[item] = "Invalid input type"
                skipped += 1
        except Exception as ex:
            status_writer.send_status(_("FAIL: {0}").format(repr(ex)))
            errors += 1
            errors_reasons[item] = repr(ex)
            pass

    if map_view:
        extent = map_view.split(" ")
        new_extent = data_frame.extent
        new_extent.XMin, new_extent.YMin = float(extent[0]), float(extent[1])
        new_extent.XMax, new_extent.YMax = float(extent[2]), float(extent[3])
        data_frame.extent = new_extent
    else:
        data_frame.zoomToSelectedFeatures()

    # Update text elements in map template.
    date_element = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "date")
    if date_element:
        date_element[0].text = "Date: {0}".format(task_utils.get_local_date())

    title_element = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "title")
    if title_element:
        title_element[0].text = map_title

    author_element = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "author")
    if author_element:
        author_element[0].text = "{0} {1}".format(author_element[0].text, author)

    if map_template in ("ANSI_D_LND.mxd", "ANSI_E_LND.mxd"):
        coord_elements = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "x*")
        coord_elements += arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", "y*")
        if coord_elements:
            for e in coord_elements:
                new_text = e.text
                if e.name == "xmin":
                    dms = task_utils.dd_to_dms(data_frame.extent.XMin)
                    if data_frame.extent.XMin > 0:
                        new_text = new_text.replace("W", "E")
                elif e.name == "xmax":
                    dms = task_utils.dd_to_dms(data_frame.extent.XMax)
                    if data_frame.extent.XMax > 0:
                        new_text = new_text.replace("W", "E")
                elif e.name == "ymin":
                    dms = task_utils.dd_to_dms(data_frame.extent.YMin)
                    if data_frame.extent.YMin < 0:
                        new_text = new_text.replace("N", "S")
                elif e.name == "ymax":
                    if data_frame.extent.YMax < 0:
                        new_text = new_text.replace("N", "S")
                    dms = task_utils.dd_to_dms(data_frame.extent.YMax)

                new_text = new_text.replace("d", str(dms[0]))
                new_text = new_text.replace("m", str(dms[1]))
                new_text = new_text.replace("s", str(dms[2]))
                e.text = new_text

    # Do this now so it does not affect zoom level or extent.
    if base_layer:
        status_writer.send_status(_("Adding basemap {0}...").format(base_map))
        arcpy.mapping.AddLayer(data_frame, base_layer, "BOTTOM")

    if added_to_map > 0:
        status_writer.send_status(_("Exporting to PDF..."))
        arcpy.mapping.ExportToPDF(
            mxd,
            os.path.join(request["folder"], "{0}.pdf".format(output_file_name)),
            layers_attributes=attribute_setting,
        )
        # Create a thumbnail size PNG of the mxd.
        task_utils.make_thumbnail(mxd, os.path.join(request["folder"], "_thumb.png"), False)
    else:
        status_writer.send_state(status.STAT_FAILED, _("No results can be exported to PDF"))
        task_utils.report(
            os.path.join(request["folder"], "__report.json"), added_to_map, skipped, skipped_details=skipped_reasons
        )
        return

    # Update state if necessary.
    if skipped > 0 or errors > 0:
        status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(skipped + errors))
    task_utils.report(
        os.path.join(request["folder"], "__report.json"), added_to_map, skipped, errors, errors_reasons, skipped_reasons
    )
Ejemplo n.º 15
0
def execute(request):
    """Deletes files.
    :param request: json as a dict
    """
    errors_reasons = {}
    errors = 0
    published = 0
    app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    parameters = request['params']
    num_results, response_index = task_utils.get_result_count(parameters)
    input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
    if num_results > task_utils.CHUNK_SIZE:
        status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.')
        return
    server_conn = task_utils.get_parameter_value(parameters, 'server_connection_path', 'value')
    service_name = task_utils.get_parameter_value(parameters, 'service_name', 'value')
    folder_name = task_utils.get_parameter_value(parameters, 'folder_name', 'value')
    if not server_conn:
        status_writer.send_state(status.STAT_FAILED, _('A server path is required'))
        return

    request_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(request_folder):
        os.makedirs(request_folder)

    map_template = os.path.join(request_folder, 'output.mxd')
    shutil.copyfile(os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'), map_template)

    for item in input_items:
        try:
            # Code required because of an Esri bug - cannot describe a map package (raises IOError).
            if item.endswith('.mpk'):
                status_writer.send_status(_('Extracting: {0}').format(item))
                arcpy.ExtractPackage_management(item, request_folder)
                pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
                mxd_file = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.mxd')[0])
                mxd = arcpy.mapping.MapDocument(mxd_file)
                create_service(request_folder, mxd, server_conn, service_name, folder_name)
            else:
                data_type = arcpy.Describe(item).dataType
                if data_type == 'MapDocument':
                    mxd = arcpy.mapping.MapDocument(item)
                    create_service(request_folder, mxd, server_conn, service_name, folder_name)
                elif data_type == 'Layer':
                    if item.endswith('.lpk'):
                        status_writer.send_status(_('Extracting: {0}').format(item))
                        arcpy.ExtractPackage_management(item, request_folder)
                        pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
                        item = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.lyr')[0])
                    layer = arcpy.mapping.Layer(item)
                    mxd = arcpy.mapping.MapDocument(map_template)
                    mxd.description = layer.name
                    mxd.tags = layer.name
                    mxd.save()
                    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
                    arcpy.mapping.AddLayer(data_frame, layer)
                    mxd.save()
                    create_service(request_folder, mxd, server_conn, service_name, folder_name)
                elif data_type in ('FeatureClass', 'ShapeFile', 'RasterDataset'):
                    if data_type == 'RasterDataset':
                        arcpy.MakeRasterLayer_management(item, os.path.basename(item))
                    else:
                        arcpy.MakeFeatureLayer_management(item, os.path.basename(item))
                    layer = arcpy.mapping.Layer(os.path.basename(item))
                    mxd = arcpy.mapping.MapDocument(map_template)
                    mxd.title = layer.name
                    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
                    arcpy.mapping.AddLayer(data_frame, layer)
                    mxd.save()
                    create_service(request_folder, mxd, server_conn, service_name, folder_name)

                elif data_type in ('CadDrawingDataset', 'FeatureDataset'):
                    arcpy.env.workspace = item
                    mxd = arcpy.mapping.MapDocument(map_template)
                    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
                    for fc in arcpy.ListFeatureClasses():
                        dataset_name = os.path.splitext(os.path.basename(item))[0]
                        l = arcpy.MakeFeatureLayer_management(fc, '{0}_{1}'.format(dataset_name, os.path.basename(fc)))
                        arcpy.mapping.AddLayer(data_frame, l.getOutput(0))
                    mxd.save()
                    arcpy.ResetEnvironments()
                    create_service(request_folder, mxd, server_conn, service_name, folder_name)
                published += 1
        except task_utils.AnalyzeServiceException as ex:
            errors += 1
            errors_reasons[item] = repr(ex)
            status_writer.send_state(status.STAT_FAILED)
        except arcpy.ExecuteError as ex:
            errors += 1
            errors_reasons[item] = repr(ex)
            status_writer.send_state(status.STAT_FAILED)
        except Exception as ex:
            errors += 1
            errors_reasons[item] = repr(ex)
            status_writer.send_state(status.STAT_FAILED)
        finally:
            if errors:
                errors_reasons[item] = repr(ex)
                status_writer.send_status(_('FAIL: {0}').format(errors_reasons[item]))
            task_utils.report(os.path.join(request['folder'], '__report.json'), published, 0, errors, errors_reasons)
def execute(request):
    """Clips selected search results using the clip geometry.
    :param request: json as a dict.
    """
    clipped = 0
    errors = 0
    skipped = 0
    global result_count
    parameters = request['params']

    # Retrieve the clip features.
    clip_features = task_utils.get_parameter_value(parameters, 'clip_features', 'value')

    # Retrieve the coordinate system code.
    out_coordinate_system = int(task_utils.get_parameter_value(parameters, 'output_projection', 'code'))

    # Retrieve the output format, create mxd and output file name parameter values.
    out_format = task_utils.get_parameter_value(parameters, 'output_format', 'value')
    create_mxd = task_utils.get_parameter_value(parameters, 'create_mxd', 'value')
    output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
    if not output_file_name:
        output_file_name = 'clip_results'

    # Create the temporary workspace if clip_feature_class:
    out_workspace = os.path.join(request['folder'], 'temp')
    if not os.path.exists(out_workspace):
        os.makedirs(out_workspace)

    # Set the output coordinate system.
    if not out_coordinate_system == 0:  # Same as Input
        out_sr = task_utils.get_spatial_reference(out_coordinate_system)
        arcpy.env.outputCoordinateSystem = out_sr

    # Set the output workspace.
    status_writer.send_status(_('Setting the output workspace...'))
    if not out_format == 'SHP':
        out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0)
    arcpy.env.workspace = out_workspace

    # Query the index for results in groups of 25.
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    result_count, response_index = task_utils.get_result_count(parameters)
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl

    # Get the Clip features by id.
    id = clip_features['id']
    clip_query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', "&fl=id,path,fullpath:[absolute],absolute_path:[absolute],[lyrFile],[geo]&q=id:{0}".format(id))
    clip_result = requests.get(clip_query, headers=headers)
    clipper = clip_result.json()['response']['docs'][0]
    if 'absolute_path' in clipper and not clipper['absolute_path'].startswith('s3'):
        clip_features = clipper['absolute_path']
    elif '[lyrFile]' in clipper:
        clip_features = clipper['[lyrFile]']
    elif '[geo]' in clipper:
        clip_features = arcpy.AsShape(clipper['[geo]']).projectAs(arcpy.SpatialReference(4326))
    elif 'absolute_path' in clipper and clipper['absolute_path'].startswith('s3'):
        base_name = os.path.basename(clipper['path'])
        temp_folder = tempfile.mkdtemp()
        if '[downloadURL]' in clipper:
            download = os.path.join(temp_folder, os.path.basename(clipper['[downloadURL]']))
            response = requests.get(clipper['[downloadURL]'])
            with open(download, 'wb') as fp:
                fp.write(response.content)
            if download.endswith('.zip'):
                zip = zipfile.ZipFile(download)
                zip.extractall(temp_folder)
                clip_features = os.path.join(temp_folder, base_name)
            else:
                clip_features = download
    else:
        bbox = clipper['bbox'].split()
        extent = arcpy.Extent(*bbox)
        pt_array = arcpy.Array([extent.lowerLeft, extent.upperLeft, extent.upperRight, extent.lowerRight])
        clip_features = arcpy.Polygon(pt_array, 4326)

    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'), 'clip_data')
    for group in groups:
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

        docs = results.json()['response']['docs']
        input_items = task_utils.get_input_items(docs)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])

        input_rows = collections.defaultdict(list)
        for doc in docs:
            if 'path' not in doc:
                input_rows[doc['name']].append(doc)
        if input_rows:
            result = clip_data(input_rows, out_workspace, clip_features, out_format)
            clipped += result[0]
            errors += result[1]
            skipped += result[2]

        if input_items:
            result = clip_data(input_items, out_workspace, clip_features, out_format)
            clipped += result[0]
            errors += result[1]
            skipped += result[2]

        if not input_items and not input_rows:
            status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.'))
            return

    if arcpy.env.workspace.endswith('.gdb'):
        out_workspace = os.path.dirname(arcpy.env.workspace)
    if clipped > 0:
        try:
            if out_format == 'MPK':
                mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
                mxd = task_utils.create_mxd(out_workspace, mxd_template, 'output')
                status_writer.send_status(_("Packaging results..."))
                task_utils.create_mpk(out_workspace, mxd, files_to_package)
                shutil.move(os.path.join(out_workspace, 'output.mpk'),
                            os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name)))
            elif out_format == 'LPK':
                status_writer.send_status(_("Packaging results..."))
                task_utils.create_lpk(out_workspace, output_file_name, files_to_package)
            elif out_format == 'KML':
                task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb"))
                arcpy.env.workspace = ''
                arcpy.RefreshCatalog(os.path.join(out_workspace, "output.gdb"))
                try:
                    arcpy.Delete_management(os.path.join(out_workspace, "output.gdb"))
                except arcpy.ExecuteError:
                    pass
                zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
                shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
            else:
                if create_mxd:
                    mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
                    task_utils.create_mxd(out_workspace, mxd_template, 'output')
                zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
                shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
        except arcpy.ExecuteError as ee:
            status_writer.send_state(status.STAT_FAILED, _(ee))
            sys.exit(1)
    else:
        status_writer.send_state(status.STAT_FAILED, _('No output created. Zero inputs were clipped.'))

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), clipped, skipped, errors, errors_reasons, skipped_reasons)
Ejemplo n.º 17
0
def execute(request):
    """Remove tags.
    :param request: json as a dict.
    """
    query = ''
    errors = 0
    parameters = request['params']
    archive_location = request['folder']
    if not os.path.exists(archive_location):
        os.makedirs(archive_location)

    # Parameter values
    search_action = task_utils.get_parameter_value(parameters, 'search_action',
                                                   'value')
    search_name = task_utils.get_parameter_value(parameters, 'saved_searches',
                                                 'value')
    search_name = eval(search_name[0])['text']
    groups = task_utils.get_parameter_value(parameters, 'groups', 'value')
    request_owner = request['owner']

    result_count, response_index = task_utils.get_result_count(parameters)
    fq = '/'
    if 'fq' in parameters[response_index]['query']:
        if isinstance(parameters[response_index]['query']['fq'], list):
            for q in parameters[response_index]['query']['fq']:
                if '{!tag=' in q:
                    q = q.split('}')[1]
                if ':' in q:
                    facet = q.split(':')[0]
                    value = q.split(':')[1]
                    if '(' in value:
                        value = value.replace('(', '').replace(')', '')
                    value = urllib.urlencode({'val': value.replace('"', '')})
                    value = value.split('val=')[1]
                    facet2 = 'f.{0}='.format(facet)
                    q = '{0}{1}'.format(facet2,
                                        value)  #q.replace(facet + ':', facet2)
                fq += '{0}/'.format(q).replace('"', '')
        else:
            # Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
            fq += '&fq={0}'.format(
                parameters[response_index]['query']['fq'].replace("\\", ""))
            if '{!tag=' in fq:
                fq = fq.split('}')[1]
            if ':' in fq:
                if fq.startswith('/&fq='):
                    fq = fq.replace('/&fq=', '')
                facet = fq.split(':')[0]
                value = fq.split(':')[1].replace('(', '').replace(')',
                                                                  '').replace(
                                                                      '"', '')
                if 'place' not in facet:
                    value = urllib.urlencode({'val': value}).split('val=')[1]
                facet2 = 'f.{0}='.format(facet)
                if '(' in value:
                    fq = ''
                    if value.split(' '):
                        for v in value.split(' '):
                            fq += (facet2 +
                                   v.replace('(', '').replace(')', '') +
                                   '/').replace(':', '')
                else:
                    value = urllib.urlencode({'val': value}).split('val=')[1]
                    fq = '{0}{1}'.format(facet2, value)
            if '{! place.op=' in fq:
                relop = find_between(fq, 'place.op=', '}')
                fq = fq.replace('}', '').replace('{', '')
                fq = fq.replace('! place.op={0}'.format(relop),
                                '/place.op={0}/'.format(relop))
                fq = fq.replace('place:', 'place=')
                fq = fq.replace('&fq=', '')

    hasQ = False
    if 'q' in parameters[response_index]['query']:
        query = parameters[response_index]['query']['q']
        hasQ = True
        if fq:
            query += '/'

    if fq:
        if fq.startswith('/place'):
            query += fq.replace('"', '')
        elif '!tag' in query and 'OR' in query:
            # e.g. "path": "/q=id:(92cdd06e01761c4c d9841b2f59b8a326) OR format:(application%2Fvnd.esri.shapefile)"
            q = query.split('}')[1].replace('))/',
                                            '').replace('(',
                                                        '').replace('(', '')
            q = urllib.urlencode({'val': q.split(':')[1]}).split('val=')[1]
            query = query.split(' OR ')[0] + ' OR ' + q
        else:
            if fq.startswith('f.//'):
                fq = fq.replace('f.//', '/').replace('"', '')
            if ' place.id' in fq:
                fq = fq.replace(' place.id', '/place.id').replace('"', '')
            if '{! place.op=' in fq:
                relop = find_between(fq, 'place.op=', '}')
                fq = fq.replace('}', '').replace('{', '')
                fq = fq.replace('! place.op={0}'.format(relop),
                                '/place.op={0}/'.format(relop)).replace(
                                    '"', '')
            query += fq.rstrip('/')
            query = query.replace('f./', '')
        query = query.replace('&fq=', '')

    if search_action == 'Overwrite an existing saved search':
        delete_result = delete_saved_search(search_name, request_owner)
        if not delete_result[0]:
            status_writer.send_state(status.STAT_FAILED, delete_result[1])
            return

    if query:
        result = create_saved_search(search_name, groups, request_owner, query,
                                     hasQ)
    else:
        result = create_saved_search(search_name, groups, request_owner, "",
                                     hasQ)
    if not result[0]:
        errors += 1
        errors_reasons[search_name] = result[1]

    # Update state if necessary.
    if errors > 0:
        status_writer.send_state(status.STAT_FAILED, result[1])
    else:
        status_writer.send_status(result[1])
    task_utils.report(os.path.join(request['folder'], '__report.json'),
                      1,
                      0,
                      errors,
                      errors_details=errors_reasons)
Ejemplo n.º 18
0
def execute(request):
    """Copies data to an existing geodatabase or feature dataset.
    :param request: json as a dict.
    """
    added = 0
    errors = 0
    skipped = 0
    global result_count
    parameters = request["params"]

    # Get the target workspace location.
    out_gdb = task_utils.get_parameter_value(parameters, "target_workspace", "value")

    # Retrieve the coordinate system code.
    out_coordinate_system = task_utils.get_parameter_value(parameters, "output_projection", "code")
    if not out_coordinate_system == "0":  # Same as Input
        arcpy.env.outputCoordinateSystem = task_utils.get_spatial_reference(out_coordinate_system)

    task_folder = request["folder"]
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)

    # Check if the geodatabase exists or if it is a feature dataset.
    is_fds = False
    if not os.path.exists(out_gdb):
        if out_gdb.endswith(".gdb"):
            arcpy.CreateFileGDB_management(os.path.dirname(out_gdb), os.path.basename(out_gdb))
            status_writer.send_status(_("Created output workspace: {0}").format(out_gdb))
        elif out_gdb.endswith(".mdb"):
            arcpy.CreatePersonalGDB_management(os.path.dirname(out_gdb), os.path.basename(out_gdb))
            status_writer.send_status(_("Created output workspace: {0}").format(out_gdb))
        elif out_gdb.endswith(".sde"):
            status_writer.send_state(status.STAT_FAILED, _("{0} does not exist").format(out_gdb))
            return
        else:
            # Possible feature dataset.
            is_fds = is_feature_dataset(out_gdb)
            if not is_fds:
                if os.path.dirname(out_gdb).endswith(".gdb"):
                    if not os.path.exists(os.path.dirname(out_gdb)):
                        arcpy.CreateFileGDB_management(
                            os.path.dirname(os.path.dirname(out_gdb)), os.path.basename(os.path.dirname(out_gdb))
                        )
                    arcpy.CreateFeatureDataset_management(os.path.dirname(out_gdb), os.path.basename(out_gdb))
                elif os.path.dirname(out_gdb).endswith(".mdb"):
                    if not os.path.exists(os.path.dirname(out_gdb)):
                        arcpy.CreatePersonalGDB_management(
                            os.path.dirname(os.path.dirname(out_gdb)), os.path.basename(os.path.dirname(out_gdb))
                        )
                    arcpy.CreateFeatureDataset_management(os.path.dirname(out_gdb), os.path.basename(out_gdb))

    status_writer.send_status(_("Setting the output workspace..."))
    arcpy.env.workspace = out_gdb

    headers = {"x-access-token": task_utils.get_security_token(request["owner"])}
    result_count, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = "{0}{1}{2}".format(sys.argv[2].split("=")[1], "/select?&wt=json", fl)
    # query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")
        query += fq
    elif "ids" in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]["ids"]), task_utils.CHUNK_SIZE, "")
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")

    status_writer.send_percent(0.0, _("Starting to process..."), "add_to_geodatabase")
    for group in groups:
        if fq:
            results = requests.get(
                query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
            )
        elif "ids" in parameters[response_index]:
            results = requests.get(query + "{0}&ids={1}".format(fl, ",".join(group)), headers=headers)
        else:
            results = requests.get(
                query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
            )

        docs = results.json()["response"]["docs"]
        input_items = task_utils.get_input_items(docs)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]["response"]["docs"])
        input_rows = collections.defaultdict(list)
        for doc in docs:
            if "path" not in doc:
                input_rows[doc["title"]].append(doc)
        if input_rows:
            result = add_to_geodatabase(input_rows, out_gdb, is_fds)
            added += result[0]
            errors += result[1]
            skipped += result[2]

        if input_items:
            result = add_to_geodatabase(input_items, out_gdb, is_fds)
            added += result[0]
            errors += result[1]
            skipped += result[2]

        if not input_items and not input_rows:
            status_writer.send_state(status.STAT_FAILED, _("No items to process. Check if items exist."))
            return

    # Update state if necessary.
    if skipped > 0 or errors > 0:
        status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(skipped + errors))
    task_utils.report(
        os.path.join(task_folder, "__report.json"), added, skipped, errors, errors_reasons, skipped_reasons
    )
Ejemplo n.º 19
0
def execute(request):
    """Clips selected search results using the clip geometry.
    :param request: json as a dict.
    """
    clipped = 0
    errors = 0
    skipped = 0
    global result_count
    parameters = request['params']

    # Retrieve clip geometry.
    try:
        clip_area = task_utils.get_parameter_value(parameters, 'clip_geometry', 'wkt')
        if not clip_area:
            clip_area = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'
    except KeyError:
        clip_area = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'

    # Retrieve the coordinate system code.
    out_coordinate_system = int(task_utils.get_parameter_value(parameters, 'output_projection', 'code'))

    # Retrieve the output format, create mxd parameter and output file name values.
    out_format = task_utils.get_parameter_value(parameters, 'output_format', 'value')
    create_mxd = task_utils.get_parameter_value(parameters, 'create_mxd', 'value')
    output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
    if not output_file_name:
        output_file_name = 'clip_results'

    # Create the temporary workspace if clip_feature_class:
    out_workspace = os.path.join(request['folder'], 'temp')
    if not os.path.exists(out_workspace):
        os.makedirs(out_workspace)

    # Set the output coordinate system.
    if not out_coordinate_system == 0:  # Same as Input
        out_sr = task_utils.get_spatial_reference(out_coordinate_system)
        arcpy.env.outputCoordinateSystem = out_sr

    # Create the clip polygon geometry object in WGS84 projection.
    gcs_sr = task_utils.get_spatial_reference(4326)
    gcs_clip_poly = task_utils.from_wkt(clip_area, gcs_sr)
    if not gcs_clip_poly.area > 0:
        gcs_clip_poly = task_utils.from_wkt('POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))', gcs_sr)

    # Set the output workspace.
    status_writer.send_status(_('Setting the output workspace...'))
    if not out_format == 'SHP':
        out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0)
    arcpy.env.workspace = out_workspace

    # Query the index for results in groups of 25.
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    result_count, response_index = task_utils.get_result_count(parameters)
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')

    # Begin processing
    status_writer.send_percent(0.0, _('Starting to process...'), 'clip_data')
    for group in groups:
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

        # docs = eval(results.read().replace('false', 'False').replace('true', 'True').replace('null', 'None'))['response']['docs']
        docs = results.json()['response']['docs']
        input_items = task_utils.get_input_items(docs)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])

        input_rows = collections.defaultdict(list)
        for doc in docs:
            if 'path' not in doc:
               input_rows[doc['name']].append(doc)
        if input_rows:
            result = clip_data(input_rows, out_workspace, out_coordinate_system, gcs_sr, gcs_clip_poly, out_format)
            clipped += result[0]
            errors += result[1]
            skipped += result[2]

        if input_items:
            result = clip_data(input_items, out_workspace, out_coordinate_system, gcs_sr, gcs_clip_poly, out_format)
            clipped += result[0]
            errors += result[1]
            skipped += result[2]

        if not input_items and not input_rows:
            status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.'))
            return

    if arcpy.env.workspace.endswith('.gdb'):
        out_workspace = os.path.dirname(arcpy.env.workspace)
    if clipped > 0:
        try:
            if out_format == 'MPK':
                mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
                mxd = task_utils.create_mxd(out_workspace, mxd_template, 'output')
                status_writer.send_status(_("Packaging results..."))
                task_utils.create_mpk(out_workspace, mxd, files_to_package)
                shutil.move(os.path.join(out_workspace, 'output.mpk'),
                            os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name)))
            elif out_format == 'LPK':
                status_writer.send_status(_("Packaging results..."))
                task_utils.create_lpk(out_workspace,output_file_name, files_to_package)
            elif out_format == 'KML':
                task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb"))
                arcpy.env.workspace = ''
                try:
                    arcpy.Delete_management(os.path.join(out_workspace, "output.gdb"))
                except arcpy.ExecuteError:
                    pass
                zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
                shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
            else:
                if create_mxd:
                    mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
                    task_utils.create_mxd(out_workspace, mxd_template, 'output')
                zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
                shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
        except arcpy.ExecuteError as ee:
            status_writer.send_state(status.STAT_FAILED, _(ee))
            sys.exit(1)
    else:
        status_writer.send_state(status.STAT_FAILED, _('No output created. Zero inputs were clipped.'))

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), clipped, skipped, errors, errors_reasons, skipped_reasons)
Ejemplo n.º 20
0
def execute(request):
    """Creates a GeoPDF.
    :param request: json as a dict.
    """
    added_to_map = 0
    errors = 0
    skipped = 0

    parameters = request['params']
    num_results, response_index = task_utils.get_result_count(parameters)
    docs = parameters[response_index]['response']['docs']
    input_items = task_utils.get_input_items(docs)
    input_rows = collections.defaultdict(list)
    for doc in docs:
        if 'path' not in doc:
            input_rows[doc['name']].append(doc)
    if num_results > task_utils.CHUNK_SIZE:
        status_writer.send_state(status.STAT_FAILED,
                                 'Reduce results to 25 or less.')
        return

    map_template = task_utils.get_parameter_value(parameters, 'map_template',
                                                  'value')
    base_map = task_utils.get_parameter_value(parameters, 'base_map', 'value')
    map_title = task_utils.get_parameter_value(parameters, 'map_title',
                                               'value')
    attribute_setting = task_utils.get_parameter_value(parameters,
                                                       'attribute_settings',
                                                       'value')
    author = task_utils.get_parameter_value(parameters, 'map_author', 'value')
    output_file_name = task_utils.get_parameter_value(parameters,
                                                      'output_file_name',
                                                      'value')
    if not output_file_name:
        output_file_name = 'output_pdf'
    try:
        map_view = task_utils.get_parameter_value(parameters, 'map_view',
                                                  'extent')
    except KeyError:
        map_view = None
        pass

    temp_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(temp_folder):
        os.makedirs(temp_folder)

    if base_map == 'NONE':
        base_layer = None
    else:
        base_layer = arcpy.mapping.Layer(
            os.path.join(os.path.dirname(os.path.dirname(__file__)),
                         'supportfiles', 'basemaps',
                         '{0}.lyr'.format(base_map)))
    mxd_path = os.path.join(os.path.dirname(os.path.dirname(__file__)),
                            'supportfiles', 'frame', map_template)
    mxd = arcpy.mapping.MapDocument(mxd_path)
    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]

    layers = []
    all_layers = []

    if input_rows:
        for name, rows in input_rows.iteritems():
            for row in rows:
                try:
                    name = arcpy.CreateUniqueName(name, 'in_memory')
                    # Create the geometry.
                    geo_json = row['[geo]']
                    geom = arcpy.AsShape(geo_json)
                    arcpy.CopyFeatures_management(geom, name)
                    feature_layer = arcpy.MakeFeatureLayer_management(
                        name, os.path.basename(name))
                    layer_file = arcpy.SaveToLayerFile_management(
                        feature_layer,
                        os.path.join(temp_folder,
                                     '{0}.lyr'.format(os.path.basename(name))))
                    layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
                    all_layers.append(
                        arcpy.mapping.Layer(layer_file.getOutput(0)))
                    added_to_map += 1
                except KeyError:
                    skipped += 1
                    skipped_reasons[name] = 'No geographic information'
                    continue

    for i, item in enumerate(input_items, 1):
        try:
            # Is the item a mxd data frame.
            map_frame_name = task_utils.get_data_frame_name(item)
            if map_frame_name:
                item = item.split('|')[0].strip()

            dsc = arcpy.Describe(item)
            if dsc.dataType == 'Layer':
                layers.append(arcpy.mapping.Layer(dsc.catalogPath))

            elif dsc.dataType == 'FeatureClass' or dsc.dataType == 'ShapeFile':
                if os.path.basename(item) in [l.name for l in all_layers]:
                    layer_name = '{0}_{1}'.format(os.path.basename(item), i)
                else:
                    layer_name = os.path.basename(item)
                feature_layer = arcpy.MakeFeatureLayer_management(
                    item, layer_name)
                layer_file = arcpy.SaveToLayerFile_management(
                    feature_layer,
                    os.path.join(temp_folder, '{0}.lyr'.format(layer_name)))
                layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
                all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))

            elif dsc.dataType == 'FeatureDataset':
                arcpy.env.workspace = item
                for fc in arcpy.ListFeatureClasses():
                    layer_file = arcpy.SaveToLayerFile_management(
                        arcpy.MakeFeatureLayer_management(
                            fc, '{0}_{1}'.format(fc, i)),
                        os.path.join(temp_folder, '{0}_{1}.lyr'.format(fc, i)))
                    layer = arcpy.mapping.Layer(layer_file.getOutput(0))
                    layer.name = fc
                    layers.append(layer)
                    all_layers.append(layer)

            elif dsc.dataType == 'RasterDataset':
                if os.path.basename(item) in [l.name for l in all_layers]:
                    layer_name = '{0}_{1}'.format(os.path.basename(item), i)
                else:
                    layer_name = os.path.basename(item)
                raster_layer = arcpy.MakeRasterLayer_management(
                    item, layer_name)
                layer_file = arcpy.SaveToLayerFile_management(
                    raster_layer,
                    os.path.join(temp_folder, '{0}.lyr'.format(layer_name)))
                layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
                all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))

            elif dsc.catalogPath.endswith('.kml') or dsc.catalogPath.endswith(
                    '.kmz'):
                if not os.path.splitext(dsc.name)[0] in layers:
                    name = os.path.splitext(dsc.name)[0]
                else:
                    name = '{0}_{1}'.format(os.path.splitext(dsc.name)[0], i)
                arcpy.KMLToLayer_conversion(dsc.catalogPath, temp_folder, name)
                layers.append(
                    arcpy.mapping.Layer(
                        os.path.join(temp_folder, '{0}.lyr'.format(name))))
                all_layers.append(
                    arcpy.mapping.Layer(
                        os.path.join(temp_folder, '{0}.lyr'.format(name))))

            elif dsc.dataType == 'MapDocument':
                input_mxd = arcpy.mapping.MapDocument(item)
                if map_frame_name:
                    df = arcpy.mapping.ListDataFrames(input_mxd,
                                                      map_frame_name)[0]
                    layers = arcpy.mapping.ListLayers(input_mxd, data_frame=df)
                else:
                    layers = arcpy.mapping.ListLayers(input_mxd)

            if layers:
                for layer in layers:
                    status_writer.send_status(
                        _('Adding layer {0}...').format(layer.name))
                    arcpy.mapping.AddLayer(data_frame, layer)
                    added_to_map += 1
                    layers = []
            else:
                status_writer.send_status(
                    _('Invalid input type: {0}').format(item))
                skipped_reasons[item] = 'Invalid input type'
                skipped += 1
        except Exception as ex:
            status_writer.send_status(_('FAIL: {0}').format(repr(ex)))
            errors += 1
            errors_reasons[item] = repr(ex)
            pass

    if map_view:
        extent = map_view.split(' ')
        new_extent = data_frame.extent
        new_extent.XMin, new_extent.YMin = float(extent[0]), float(extent[1])
        new_extent.XMax, new_extent.YMax = float(extent[2]), float(extent[3])
        data_frame.extent = new_extent
    else:
        data_frame.zoomToSelectedFeatures()

    # Update text elements in map template.
    date_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT',
                                                    'date')
    if date_element:
        date_element[0].text = 'Date: {0}'.format(task_utils.get_local_date())

    title_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT',
                                                     'title')
    if title_element:
        title_element[0].text = map_title

    author_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT',
                                                      'author')
    if author_element:
        author_element[0].text = '{0} {1}'.format(author_element[0].text,
                                                  author)

    if map_template in ('ANSI_D_LND.mxd', 'ANSI_E_LND.mxd'):
        coord_elements = arcpy.mapping.ListLayoutElements(
            mxd, 'TEXT_ELEMENT', 'x*')
        coord_elements += arcpy.mapping.ListLayoutElements(
            mxd, 'TEXT_ELEMENT', 'y*')
        if coord_elements:
            for e in coord_elements:
                new_text = e.text
                if e.name == 'xmin':
                    dms = task_utils.dd_to_dms(data_frame.extent.XMin)
                    if data_frame.extent.XMin > 0:
                        new_text = new_text.replace('W', 'E')
                elif e.name == 'xmax':
                    dms = task_utils.dd_to_dms(data_frame.extent.XMax)
                    if data_frame.extent.XMax > 0:
                        new_text = new_text.replace('W', 'E')
                elif e.name == 'ymin':
                    dms = task_utils.dd_to_dms(data_frame.extent.YMin)
                    if data_frame.extent.YMin < 0:
                        new_text = new_text.replace('N', 'S')
                elif e.name == 'ymax':
                    if data_frame.extent.YMax < 0:
                        new_text = new_text.replace('N', 'S')
                    dms = task_utils.dd_to_dms(data_frame.extent.YMax)

                new_text = new_text.replace('d', str(dms[0]))
                new_text = new_text.replace('m', str(dms[1]))
                new_text = new_text.replace('s', str(dms[2]))
                e.text = new_text

    # Do this now so it does not affect zoom level or extent.
    if base_layer:
        status_writer.send_status(_('Adding basemap {0}...').format(base_map))
        arcpy.mapping.AddLayer(data_frame, base_layer, 'BOTTOM')

    if added_to_map > 0:
        status_writer.send_status(_('Exporting to PDF...'))
        arcpy.mapping.ExportToPDF(mxd,
                                  os.path.join(
                                      request['folder'],
                                      '{0}.pdf'.format(output_file_name)),
                                  layers_attributes=attribute_setting)
        # Create a thumbnail size PNG of the mxd.
        task_utils.make_thumbnail(
            mxd, os.path.join(request['folder'], '_thumb.png'), False)
    else:
        status_writer.send_state(status.STAT_FAILED,
                                 _('No results can be exported to PDF'))
        task_utils.report(os.path.join(request['folder'], '__report.json'),
                          added_to_map,
                          skipped,
                          skipped_details=skipped_reasons)
        return

    # Update state if necessary.
    if skipped > 0 or errors > 0:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'],
                                   '__report.json'), added_to_map, skipped,
                      errors, errors_reasons, skipped_reasons)
def execute(request):
    """Builds raster pyramids for input raster datasets.
    :param request: json as a dict.
    """
    processed = 0
    skipped = 0
    parameters = request['params']
    resampling_method = task_utils.get_parameter_value(parameters, 'resampling_method', 'value')

    # Advanced options
    compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value')
    compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value')

    # Create the task folder to hold report files.
    task_folder = request['folder']
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)

    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    num_results, response_index = task_utils.get_result_count(parameters)
    if num_results > task_utils.CHUNK_SIZE:
        # Query the index for results in groups of 25.
        query_index = task_utils.QueryIndex(parameters[response_index])
        fl = query_index.fl
        query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
        fq = query_index.get_fq()
        if fq:
            groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
            query += fq
        elif 'ids' in parameters[response_index]:
            groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
        else:
            groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')

        # Begin processing
        status_writer.send_percent(0.0, _('Starting to process...'), 'build_raster_pyramids')
        i = 0.
        for group in groups:
            i += len(group) - group.count('')
            if fq:
                results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)
            elif 'ids' in parameters[response_index]:
                results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers)
            else:
                results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)

            input_items = task_utils.get_input_items(results.json()['response']['docs'])
            if not input_items:
                input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
            result = build_pyramids(input_items, compression_method, compression_quality, resampling_method)
            processed += result[0]
            skipped += result[1]
            status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'build_raster_pyramids')
    else:
        input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
        processed, skipped = build_pyramids(input_items, compression_method, compression_quality, resampling_method, True)

    # Update state if necessary.
    if skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), processed, skipped, skipped_details=skipped_reasons)
Ejemplo n.º 22
0
def execute(request):
    """Builds raster pyramids for input raster datasets.
    :param request: json as a dict.
    """
    processed = 0
    skipped = 0
    parameters = request['params']

    # Get the extent for for which to use to calculate statistics.
    extent = ''
    try:
        try:
            ext = task_utils.get_parameter_value(parameters,
                                                 'processing_extent', 'wkt')
            if ext:
                sr = task_utils.get_spatial_reference("4326")
                extent = task_utils.from_wkt(ext, sr)
        except KeyError:
            ext = task_utils.get_parameter_value(parameters,
                                                 'processing_extent',
                                                 'feature')
            if ext:
                extent = arcpy.Describe(ext).extent
    except KeyError:
        pass

    horizontal_skip_factor = task_utils.get_parameter_value(
        parameters, 'horizontal_skip_factor', 'value')
    vertical_skip_factor = task_utils.get_parameter_value(
        parameters, 'vertical_skip_factor', 'value')
    ignore_pixel_values = task_utils.get_parameter_value(
        parameters, 'ignore_pixel_values', 'value')

    # Create the task folder to hold report files.
    task_folder = request['folder']
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)

    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    num_results, response_index = task_utils.get_result_count(parameters)
    if num_results > task_utils.CHUNK_SIZE:
        # Query the index for results in groups of 25.
        query_index = task_utils.QueryIndex(parameters[response_index])
        fl = query_index.fl
        query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1],
                                   '/select?&wt=json', fl)
        fq = query_index.get_fq()
        if fq:
            groups = task_utils.grouper(range(0, num_results),
                                        task_utils.CHUNK_SIZE, '')
            query += fq
        elif 'ids' in parameters[response_index]:
            groups = task_utils.grouper(
                list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE,
                '')
        else:
            groups = task_utils.grouper(range(0, num_results),
                                        task_utils.CHUNK_SIZE, '')

        # Begin processing
        status_writer.send_percent(0.0, _('Starting to process...'),
                                   'calculate_raster_statistics')
        i = 0.
        for group in groups:
            i += len(group) - group.count('')
            if fq:
                results = requests.get(query + "&rows={0}&start={1}".format(
                    task_utils.CHUNK_SIZE, group[0]),
                                       verify=verify_ssl,
                                       headers=headers)
            elif 'ids' in parameters[response_index]:
                results = requests.get(
                    query + '{0}&ids={1}'.format(fl, ','.join(group)),
                    verify=verify_ssl,
                    headers=headers)
            else:
                results = requests.get(query + "&rows={0}&start={1}".format(
                    task_utils.CHUNK_SIZE, group[0]),
                                       verify=verify_ssl,
                                       headers=headers)

            input_items = task_utils.get_input_items(
                results.json()['response']['docs'])
            if not input_items:
                input_items = task_utils.get_input_items(
                    parameters[response_index]['response']['docs'])

            result = calculate_raster_statistics(input_items, extent,
                                                 horizontal_skip_factor,
                                                 vertical_skip_factor,
                                                 ignore_pixel_values)
            processed += result[0]
            skipped += result[1]
            status_writer.send_percent(
                i / num_results,
                '{0}: {1:.0f}%'.format("Processed", i / num_results * 100),
                'calculate_raster_statistics')
    else:
        input_items = task_utils.get_input_items(
            parameters[response_index]['response']['docs'])
        processed, skipped = calculate_raster_statistics(
            input_items, extent, horizontal_skip_factor, vertical_skip_factor,
            ignore_pixel_values, True)

    # Update state if necessary.
    if skipped > 0:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'),
                      processed,
                      skipped,
                      skipped_details=skipped_reasons)
Ejemplo n.º 23
0
def execute(request):
    """Deletes files.
    :param request: json as a dict
    """
    errors_reasons = {}
    errors = 0
    published = 0
    app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    parameters = request['params']
    num_results, response_index = task_utils.get_result_count(parameters)
    input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
    if num_results > task_utils.CHUNK_SIZE:
        status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.')
        return
    url = task_utils.get_parameter_value(parameters, 'url', 'value')
    username = task_utils.get_parameter_value(parameters, 'username', 'value')
    password = task_utils.get_parameter_value(parameters, 'password', 'value')
    service_name = task_utils.get_parameter_value(parameters, 'service_name', 'value')
    folder_name = task_utils.get_parameter_value(parameters, 'folder_name', 'value')

    request_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(request_folder):
        os.makedirs(request_folder)

    map_template = os.path.join(request_folder, 'output.mxd')
    shutil.copyfile(os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'), map_template)

    for item in input_items:
        try:
            # Code required because of an Esri bug - cannot describe a map package (raises IOError).
            if item.endswith('.mpk'):
                status_writer.send_status(_('Extracting: {0}').format(item))
                arcpy.ExtractPackage_management(item, request_folder)
                pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
                mxd_file = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.mxd')[0])
                mxd = arcpy.mapping.MapDocument(mxd_file)
                create_service(request_folder, mxd, url, username, password, service_name, folder_name)
            else:
                data_type = arcpy.Describe(item).dataType
                if data_type == 'MapDocument':
                    mxd = arcpy.mapping.MapDocument(item)
                    create_service(request_folder, mxd, url, username, password, service_name, folder_name)
                elif data_type == 'Layer':
                    if item.endswith('.lpk'):
                        status_writer.send_status(_('Extracting: {0}').format(item))
                        arcpy.ExtractPackage_management(item, request_folder)
                        pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
                        item = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.lyr')[0])
                    layer = arcpy.mapping.Layer(item)
                    mxd = arcpy.mapping.MapDocument(map_template)
                    mxd.description = layer.name
                    mxd.tags = layer.name
                    mxd.save()
                    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
                    arcpy.mapping.AddLayer(data_frame, layer)
                    mxd.save()
                    create_service(request_folder, mxd, url, username, password,  service_name, folder_name)
                elif data_type in ('FeatureClass', 'ShapeFile', 'RasterDataset'):
                    if data_type == 'RasterDataset':
                        arcpy.MakeRasterLayer_management(item, os.path.basename(item))
                    else:
                        arcpy.MakeFeatureLayer_management(item, os.path.basename(item))
                    layer = arcpy.mapping.Layer(os.path.basename(item))
                    mxd = arcpy.mapping.MapDocument(map_template)
                    mxd.description = layer.name
                    mxd.tags = layer.name
                    mxd.save()
                    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
                    arcpy.mapping.AddLayer(data_frame, layer)
                    mxd.save()
                    create_service(request_folder, mxd, url, username, password, service_name, folder_name)
                published += 1
        except task_utils.AnalyzeServiceException as ase:
            status_writer.send_state(status.STAT_FAILED, _(ase))
            errors_reasons[item] = repr(ase)
            errors += 1
        except requests.RequestException as re:
            status_writer.send_state(status.STAT_FAILED, _(re))
            errors_reasons[item] = repr(re)
            errors += 1
        except task_utils.PublishException as pe:
            status_writer.send_state(status.STAT_FAILED, _(pe))
            errors_reasons[item] = repr(pe)
            errors += 1
        except arcpy.ExecuteError as ee:
            status_writer.send_state(status.STAT_FAILED, _(ee))
            errors_reasons[item] = repr(ee)
            errors += 1
        except Exception as ex:
            status_writer.send_state(status.STAT_FAILED, _(ex))
            errors_reasons[item] = repr(ex)
            errors += 1
        finally:
            task_utils.report(os.path.join(request['folder'], '__report.json'), published, 0, errors, errors_reasons)
Ejemplo n.º 24
0
def execute(request):
    """Deletes files.
    :param request: json as a dict
    """
    errors_reasons = {}
    errors = 0
    published = 0
    app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    parameters = request['params']
    num_results, response_index = task_utils.get_result_count(parameters)
    input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
    if num_results > task_utils.CHUNK_SIZE:
        status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.')
        return
    url = task_utils.get_parameter_value(parameters, 'url', 'value')
    username = task_utils.get_parameter_value(parameters, 'username', 'value')
    password = task_utils.get_parameter_value(parameters, 'password', 'value')
    service_name = task_utils.get_parameter_value(parameters, 'service_name', 'value')
    folder_name = task_utils.get_parameter_value(parameters, 'folder_name', 'value')

    request_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(request_folder):
        os.makedirs(request_folder)

    map_template = os.path.join(request_folder, 'output.mxd')
    shutil.copyfile(os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'), map_template)

    for item in input_items:
        try:
            # Code required because of an Esri bug - cannot describe a map package (raises IOError).
            if item.endswith('.mpk'):
                status_writer.send_status(_('Extracting: {0}').format(item))
                arcpy.ExtractPackage_management(item, request_folder)
                pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
                mxd_file = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.mxd')[0])
                mxd = arcpy.mapping.MapDocument(mxd_file)
                create_service(request_folder, mxd, url, username, password, service_name, folder_name)
            else:
                data_type = arcpy.Describe(item).dataType
                if data_type == 'MapDocument':
                    mxd = arcpy.mapping.MapDocument(item)
                    create_service(request_folder, mxd, url, username, password, service_name, folder_name)
                elif data_type == 'Layer':
                    if item.endswith('.lpk'):
                        status_writer.send_status(_('Extracting: {0}').format(item))
                        arcpy.ExtractPackage_management(item, request_folder)
                        pkg_folder = os.path.join(request_folder, glob.glob1(request_folder, 'v*')[0])
                        item = os.path.join(pkg_folder, glob.glob1(pkg_folder, '*.lyr')[0])
                    layer = arcpy.mapping.Layer(item)
                    mxd = arcpy.mapping.MapDocument(map_template)
                    mxd.description = layer.name
                    mxd.tags = layer.name
                    mxd.save()
                    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
                    arcpy.mapping.AddLayer(data_frame, layer)
                    mxd.save()
                    create_service(request_folder, mxd, url, username, password,  service_name, folder_name)
                elif data_type in ('FeatureClass', 'ShapeFile', 'RasterDataset'):
                    if data_type == 'RasterDataset':
                        arcpy.MakeRasterLayer_management(item, os.path.basename(item))
                    else:
                        arcpy.MakeFeatureLayer_management(item, os.path.basename(item))
                    layer = arcpy.mapping.Layer(os.path.basename(item))
                    mxd = arcpy.mapping.MapDocument(map_template)
                    mxd.description = layer.name
                    mxd.tags = layer.name
                    mxd.save()
                    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]
                    arcpy.mapping.AddLayer(data_frame, layer)
                    mxd.save()
                    create_service(request_folder, mxd, url, username, password, service_name, folder_name)
                published += 1
        except task_utils.AnalyzeServiceException as ase:
            status_writer.send_state(status.STAT_FAILED, _(ase))
            errors_reasons[item] = repr(ase)
            errors += 1
        except requests.RequestException as re:
            status_writer.send_state(status.STAT_FAILED, _(re))
            errors_reasons[item] = repr(re)
            errors += 1
        except task_utils.PublishException as pe:
            status_writer.send_state(status.STAT_FAILED, _(pe))
            errors_reasons[item] = repr(pe)
            errors += 1
        except arcpy.ExecuteError as ee:
            status_writer.send_state(status.STAT_FAILED, _(ee))
            errors_reasons[item] = repr(ee)
            errors += 1
        except Exception as ex:
            status_writer.send_state(status.STAT_FAILED, _(ex))
            errors_reasons[item] = repr(ex)
            errors += 1
        finally:
            task_utils.report(os.path.join(request['folder'], '__report.json'), published, 0, errors, errors_reasons)
Ejemplo n.º 25
0
def execute(request):
    """Mosaics input raster datasets into a new raster dataset or mosaic dataset.
    :param request: json as a dict.
    """
    status_writer = status.Writer()
    parameters = request['params']
    target_workspace = task_utils.get_parameter_value(parameters, 'target_workspace', 'value')
    output_name = task_utils.get_parameter_value(parameters, 'output_dataset_name', 'value')
    out_coordinate_system = task_utils.get_parameter_value(parameters, 'output_projection', 'code')
    # Advanced options
    output_raster_format = task_utils.get_parameter_value(parameters, 'raster_format', 'value')
    compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value')
    compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value')
    arcpy.env.compression = '{0} {1}'.format(compression_method, compression_quality)

    if output_raster_format in ('FileGDB', 'MosaicDataset'):
        if not os.path.splitext(target_workspace)[1] in ('.gdb', '.mdb', '.sde'):
            status_writer.send_state(status.STAT_FAILED, _('Target workspace must be a geodatabase'))
            return

    task_folder = request['folder']
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)

    clip_area = None
    if not output_raster_format == 'MosaicDataset':
        # Get the clip region as an extent object.
        try:
            clip_area_wkt = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt')
            if not clip_area_wkt:
                clip_area_wkt = 'POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'
            if not out_coordinate_system == '0':
                clip_area = task_utils.get_clip_region(clip_area_wkt, out_coordinate_system)
            else:
                clip_area = task_utils.get_clip_region(clip_area_wkt)
        except KeyError:
            pass

    status_writer.send_status(_('Setting the output workspace...'))
    if not os.path.exists(target_workspace):
        status_writer.send_state(status.STAT_FAILED, _('Target workspace does not exist'))
        return
    arcpy.env.workspace = target_workspace

    status_writer.send_status(_('Starting to process...'))
    num_results, response_index = task_utils.get_result_count(parameters)
    raster_items = None
    if num_results > task_utils.CHUNK_SIZE:
        # Query the index for results in groups of 25.
        query_index = task_utils.QueryIndex(parameters[response_index])
        fl = query_index.fl
        query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
        fq = query_index.get_fq()
        if fq:
            groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
            query += fq
        elif 'ids' in parameters[response_index]:
            groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
        else:
            groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')

        headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
        for group in groups:
            if fq:
                results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
            elif 'ids' in parameters[response_index]:
                results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
            else:
                results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

            input_items = task_utils.get_input_items(results.json()['response']['docs'])
            if not input_items:
                input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
            raster_items, pixels, bands, skipped = get_items(input_items)
    else:
        input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
        raster_items, pixels, bands, skipped = get_items(input_items)

    if not raster_items:
        if skipped == 0:
            status_writer.send_state(status.STAT_FAILED, _('Invalid input types'))
            skipped_reasons['All Items'] = _('Invalid input types')
            task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), num_results, skipped_details=skipped_reasons)
            return
        else:
            status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
            task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons)
            return

    # Get most common pixel type.
    pixel_type = pixel_types[max(set(pixels), key=pixels.count)]
    if output_raster_format in ('FileGDB', 'GRID', 'MosaicDataset'):
        output_name = arcpy.ValidateTableName(output_name, target_workspace)
    else:
        output_name = '{0}.{1}'.format(arcpy.ValidateTableName(output_name, target_workspace), output_raster_format.lower())

    if arcpy.Exists(os.path.join(target_workspace, output_name)):
        status_writer.send_state(status.STAT_FAILED, _('Output dataset already exists.'))
        return

    if output_raster_format == 'MosaicDataset':
        try:
            status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters')))
            if out_coordinate_system == '0':
                out_coordinate_system = raster_items[0]
            else:
                out_coordinate_system = None
            mosaic_ds = arcpy.CreateMosaicDataset_management(target_workspace,
                                                             output_name,
                                                             out_coordinate_system,
                                                             max(bands),
                                                             pixel_type)
            arcpy.AddRastersToMosaicDataset_management(mosaic_ds, 'Raster Dataset', raster_items)
            arcpy.MakeMosaicLayer_management(mosaic_ds, 'mosaic_layer')
            layer_object = arcpy.mapping.Layer('mosaic_layer')
            task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png'))
        except arcpy.ExecuteError:
            skipped += 1
            skipped_reasons['All Items'] = arcpy.GetMessages(2)
    else:
        try:
            if len(bands) > 1:
                status_writer.send_state(status.STAT_FAILED, _('Input rasters must have the same number of bands'))
                return
            if out_coordinate_system == '0':
                out_coordinate_system = None
            status_writer.send_status(_('Generating {0}. Large input {1} will take longer to process.'.format('Mosaic', 'rasters')))
            if clip_area:
                ext = '{0} {1} {2} {3}'.format(clip_area.XMin, clip_area.YMin, clip_area.XMax, clip_area.YMax)
                tmp_mosaic = arcpy.MosaicToNewRaster_management(
                    raster_items,
                    target_workspace,
                    'tmpMosaic',
                    out_coordinate_system,
                    pixel_type,
                    number_of_bands=bands.keys()[0]
                )
                status_writer.send_status(_('Clipping...'))
                out_mosaic = arcpy.Clip_management(tmp_mosaic, ext, output_name)
                arcpy.Delete_management(tmp_mosaic)
            else:
                out_mosaic = arcpy.MosaicToNewRaster_management(raster_items,
                                                                target_workspace,
                                                                output_name,
                                                                out_coordinate_system,
                                                                pixel_type,
                                                                number_of_bands=bands.keys()[0],
                                                                mosaic_method='BLEND')
            arcpy.MakeRasterLayer_management(out_mosaic, 'mosaic_layer')
            layer_object = arcpy.mapping.Layer('mosaic_layer')
            task_utils.make_thumbnail(layer_object, os.path.join(request['folder'], '_thumb.png'))
        except arcpy.ExecuteError:
            skipped += 1
            skipped_reasons['All Items'] = arcpy.GetMessages(2)

    # Update state if necessary.
    if skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), len(raster_items), skipped, skipped_details=skipped_reasons)
Ejemplo n.º 26
0
def execute(request):
    """Replace the workspace path for layer files and map document layers.
    :param request: json as a dict.
    """
    updated = 0
    skipped = 0
    parameters = request['params']
    backup = task_utils.get_parameter_value(parameters, 'create_backup',
                                            'value')
    old_data_source = task_utils.get_parameter_value(parameters,
                                                     'old_data_source',
                                                     'value').lower()
    new_data_source = task_utils.get_parameter_value(parameters,
                                                     'new_data_source',
                                                     'value')

    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    if not arcpy.Exists(new_data_source):
        status_writer.send_state(
            status.STAT_FAILED,
            _('{0} does not exist').format(new_data_source))
        return

    if os.path.splitext(new_data_source)[1] not in ('.gdb', '.mdb', '.sde'):
        new_dataset = os.path.basename(new_data_source)
        dsc = arcpy.Describe(os.path.dirname(new_data_source))
    else:
        dsc = arcpy.Describe(new_data_source)
        new_dataset = ''
    wks_type = 'NONE'
    if dsc.dataType == 'FeatureDataset':
        new_workspace = dsc.path
        wks_type = get_workspace_type(new_workspace)
    elif dsc.dataType == 'Workspace':
        new_workspace = dsc.catalogPath
        wks_type = get_workspace_type(new_workspace)
    elif dsc.dataType == 'Folder':
        dsc = arcpy.Describe(new_data_source)
        new_workspace = dsc.catalogPath
        if new_dataset.endswith('.shp'):
            wks_type = 'SHAPEFILE_WORKSPACE'
            new_dataset = new_dataset.rsplit('.shp')[0]
        else:
            if arcpy.Describe(new_data_source).dataType == 'RasterDataset':
                wks_type = 'RASTER_WORKSPACE'
    elif dsc.dataType == 'CadDrawingDataset':
        new_workspace = dsc.path
        wks_type = 'CAD_WORKSPACE'
    else:
        new_workspace = os.path.dirname(new_data_source)

    num_results, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}'.format(sys.argv[2].split('=')[1], '/select?&wt=json')
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, num_results),
                                    task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']),
                                    task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, num_results),
                                    task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'),
                               'replace_data_source')
    i = 0.
    for group in groups:
        i += len(group) - group.count('')
        if fq:
            results = requests.get(query + "{0}&rows={1}&start={2}".format(
                fl, task_utils.CHUNK_SIZE, group[0]),
                                   verify=verify_ssl,
                                   headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query +
                                   '{0}&ids={1}'.format(fl, ','.join(group)),
                                   verify=verify_ssl,
                                   headers=headers)
        else:
            results = requests.get(query + "{0}&rows={1}&start={2}".format(
                fl, task_utils.CHUNK_SIZE, group[0]),
                                   verify=verify_ssl,
                                   headers=headers)

        input_items = task_utils.get_input_items(
            results.json()['response']['docs'], True)
        if not input_items:
            input_items = task_utils.get_input_items(
                parameters[response_index]['response']['docs'])
        result = replace_data_source(input_items, old_data_source,
                                     new_workspace, new_dataset, wks_type,
                                     backup, headers)
        updated += result[0]
        skipped += result[1]
        status_writer.send_percent(
            i / num_results, '{0}: {1:.0f}%'.format("Processed",
                                                    i / num_results * 100),
            'replace_data_source')

    # Update state if necessary.
    if skipped > 0:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'),
                      updated,
                      skipped,
                      skipped_details=skipped_reasons)
def execute(request):
    """Builds raster pyramids for input raster datasets.
    :param request: json as a dict.
    """
    processed = 0
    skipped = 0
    parameters = request['params']
    resampling_method = task_utils.get_parameter_value(parameters, 'resampling_method', 'value')

    # Advanced options
    compression_method = task_utils.get_parameter_value(parameters, 'compression_method', 'value')
    compression_quality = task_utils.get_parameter_value(parameters, 'compression_quality', 'value')

    # Create the task folder to hold report files.
    task_folder = request['folder']
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)

    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    num_results, response_index = task_utils.get_result_count(parameters)
    if num_results > task_utils.CHUNK_SIZE:
        # Query the index for results in groups of 25.
        query_index = task_utils.QueryIndex(parameters[response_index])
        fl = query_index.fl
        query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
        fq = query_index.get_fq()
        if fq:
            groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
            query += fq
        elif 'ids' in parameters[response_index]:
            groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
        else:
            groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')

        # Begin processing
        status_writer.send_percent(0.0, _('Starting to process...'), 'build_raster_pyramids')
        i = 0.
        for group in groups:
            i += len(group) - group.count('')
            if fq:
                results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
            elif 'ids' in parameters[response_index]:
                results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
            else:
                results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

            input_items = task_utils.get_input_items(results.json()['response']['docs'])
            if not input_items:
                input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
            result = build_pyramids(input_items, compression_method, compression_quality, resampling_method)
            processed += result[0]
            skipped += result[1]
            status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'build_raster_pyramids')
    else:
        input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
        processed, skipped = build_pyramids(input_items, compression_method, compression_quality, resampling_method, True)

    # Update state if necessary.
    if skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), processed, skipped, skipped_details=skipped_reasons)
Ejemplo n.º 28
0
def execute(request):
    """Copies files to a target folder.
    :param request: json as a dict.
    """
    created = 0
    skipped = 0
    errors = 0
    global result_count
    parameters = request['params']
    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }

    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    # meta_folder = task_utils.get_parameter_value(parameters, 'meta_data_folder', 'value')
    voyager_server = sys.argv[2].split('=')[1].split('solr')[0][:-1]
    url = "{0}/api/rest/system/settings".format(voyager_server)
    response = requests.get(url, verify=verify_ssl, headers=headers)
    meta_folder = response.json()['folders']['meta']
    result_count, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
                               fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']),
                                    task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'),
                               'create_layer_files')
    i = 0.

    for group in groups:
        i += len(group) - group.count('')
        if fq:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                verify=verify_ssl,
                headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query +
                                   '{0}&ids={1}'.format(fl, ','.join(group)),
                                   verify=verify_ssl,
                                   headers=headers)
        else:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                verify=verify_ssl,
                headers=headers)

        docs = results.json()['response']['docs']
        # docs = eval(results.read().replace('false', 'False').replace('true', 'True'))['response']['docs']
        if not docs:
            docs = parameters[response_index]['response']['docs']
        input_items = []
        for doc in docs:
            if 'path' in doc:
                input_items.append(
                    (doc['id'], doc['path'], doc['name'], doc['location']))
        result = create_layer_file(input_items, meta_folder, voyager_server,
                                   headers)
        created += result[0]
        errors += result[1]
        skipped += result[2]

    try:
        shutil.copy2(
            os.path.join(os.path.dirname(os.path.dirname(__file__)),
                         'supportfiles', '_thumb.png'), request['folder'])
    except IOError:
        pass
    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'],
                                   '__report.json'), created, skipped, errors,
                      errors_reasons, skipped_reasons)
Ejemplo n.º 29
0
def execute(request):
    """Converts each input dataset to kml (.kmz).
    :param request: json as a dict.
    """
    converted = 0
    skipped = 0
    errors = 0
    global result_count
    parameters = request["params"]

    out_workspace = os.path.join(request["folder"], "temp")
    if not os.path.exists(out_workspace):
        os.makedirs(out_workspace)

    # Get the boundary box extent for input to KML tools.
    extent = ""
    try:
        try:
            ext = task_utils.get_parameter_value(parameters, "processing_extent", "wkt")
            if ext:
                sr = task_utils.get_spatial_reference("4326")
                extent = task_utils.from_wkt(ext, sr)
        except KeyError:
            ext = task_utils.get_parameter_value(parameters, "processing_extent", "feature")
            if ext:
                extent = arcpy.Describe(ext).extent
    except KeyError:
        pass

    # Get the output file name.
    output_file_name = task_utils.get_parameter_value(parameters, "output_file_name", "value")
    if not output_file_name:
        output_file_name = "kml_results"

    result_count, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = "{0}{1}{2}".format(sys.argv[2].split("=")[1], "/select?&wt=json", fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")
        query += fq
    elif "ids" in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]["ids"]), task_utils.CHUNK_SIZE, "")
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")

    # Begin processing
    status_writer.send_percent(0.0, _("Starting to process..."), "convert_to_kml")
    headers = {"x-access-token": task_utils.get_security_token(request["owner"])}
    for group in groups:
        if fq:
            results = requests.get(
                query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
            )
        elif "ids" in parameters[response_index]:
            results = requests.get(query + "{0}&ids={1}".format(fl, ",".join(group)), headers=headers)
        else:
            results = requests.get(
                query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
            )

        docs = results.json()["response"]["docs"]
        input_items = task_utils.get_input_items(docs)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]["response"]["docs"])

        input_rows = collections.defaultdict(list)
        for doc in docs:
            if "path" not in doc:
                input_rows[doc["name"]].append(doc)
        if input_rows:
            result = convert_to_kml(input_rows, out_workspace, extent)
            converted += result[0]
            errors += result[1]
            skipped += result[2]

        if input_items:
            result = convert_to_kml(input_items, out_workspace, extent)
            converted += result[0]
            errors += result[1]
            skipped += result[2]

        if not input_items and not input_rows:
            status_writer.send_state(status.STAT_FAILED, _("No items to process. Check if items exist."))
            return

    # Zip up kmz files if more than one.
    if converted > 1:
        status_writer.send_status("Converted: {}".format(converted))
        zip_file = task_utils.zip_data(out_workspace, "{0}.zip".format(output_file_name))
        shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
        shutil.copy2(
            os.path.join(os.path.dirname(os.path.dirname(__file__)), "supportfiles", "_thumb.png"), request["folder"]
        )
    elif converted == 1:
        try:
            kml_file = glob.glob(os.path.join(out_workspace, "*.kmz"))[0]
            tmp_lyr = arcpy.KMLToLayer_conversion(kml_file, out_workspace, "kml_layer")
            task_utils.make_thumbnail(tmp_lyr.getOutput(0), os.path.join(request["folder"], "_thumb.png"))
        except arcpy.ExecuteError:
            pass
        shutil.move(kml_file, os.path.join(request["folder"], os.path.basename(kml_file)))

    # Update state if necessary.
    if skipped > 0 or errors > 0:
        status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(errors + skipped))
    task_utils.report(
        os.path.join(request["folder"], "__report.json"), converted, skipped, errors, errors_reasons, skipped_reasons
    )
Ejemplo n.º 30
0
def execute(request):
    """Exports search results a CSV, shapefile or XML document.
    :param request: json as a dict.
    """

    # Get SSL trust setting.
    verify_ssl = task_utils.get_ssl_mode()

    chunk_size = task_utils.CHUNK_SIZE

    file_name = task_utils.get_parameter_value(request['params'], 'file_name',
                                               'value')
    fields = task_utils.get_parameter_value(request['params'], 'fields',
                                            'value')
    out_format = task_utils.get_parameter_value(request['params'],
                                                'output_format', 'value')

    if not 'path' in fields and 'path:[absolute]' in fields:
        fields.append('path')

    if 'geo' in fields:
        i_geo = fields.index('geo')
        fields.remove('geo')
        fields.insert(i_geo, '[geo]')

    # Create the temporary workspace.
    task_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)

    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    num_results, response_index = task_utils.get_result_count(
        request['params'])

    if len(sys.argv) == 2:
        query = '{0}/solr/v0/select?&wt=json&fl={1}'.format(
            'http://localhost:8888', ','.join(fields))
    else:
        query = '{0}/select?&wt=json&fl={1}'.format(sys.argv[2].split('=')[1],
                                                    ','.join(fields))

    if 'query' in request['params'][response_index]:
        # Voyager Search Traditional UI
        for p in request['params']:
            if 'query' in p:
                request_qry = p['query']
                break
        if 'voyager.list' in request_qry:
            query += '&voyager.list={0}'.format(request_qry['voyager.list'])

        # Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
        if 'fq' in request_qry:
            try:
                if isinstance(request_qry['fq'], list):
                    for fq in request_qry['fq']:
                        try:
                            query += '&fq={0}'.format(str(fq))
                        except UnicodeEncodeError:
                            query += '&fq={0}'.format(str(fq.encode('utf-8')))
                else:
                    query += '&fq={0}'.format(request_qry['fq'])
                if '{!expand}' in query:
                    query = query.replace('{!expand}', '')
                if '{!tag' in query:
                    tag = re.findall('{!(.*?)}', query)
                    if tag:
                        tag_str = "{!" + tag[0] + "}"
                        query = query.replace(tag_str, '')
                query = query.replace(' ', '%20')
            except AttributeError:
                for qry in request_qry['fq']:
                    query += '&fq={0}'.format(qry).replace("\\", "").replace(
                        ' ', '%20')
        if 'q' in request_qry:
            try:
                query += '&q={0}'.format(request_qry['q'].replace("\\", ""))
                query = query.replace(' ', '%20')
            except UnicodeEncodeError:
                query += '&q={0}'.format(
                    request_qry['q'].encode('utf-8').replace("\\", ""))
                query = query.replace(' ', '%20')
            except AttributeError:
                for qry in request_qry['q']:
                    query += '&q={0}'.format(qry).replace("\\", "").replace(
                        ' ', '%20')
        if 'place' in request_qry:
            try:
                query += '&place={0}'.format(request_qry['place'].replace(
                    "\\", ""))
                query = query.replace(' ', '%20')
            except AttributeError:
                for qry in request_qry['place']:
                    query += '&place={0}'.format(qry).replace("\\",
                                                              "").replace(
                                                                  ' ', '%20')
        if 'place.op' in request_qry:
            query += '&place.op={0}'.format(request_qry['place.op'])

        query += '&rows={0}&start={1}'
        exported_cnt = 0.
        for i in range(0, num_results, chunk_size):
            url = query.replace('{0}', str(chunk_size)).replace('{1}', str(i))
            res = requests.get(url, verify=verify_ssl, headers=headers)

            jobs = res.json()['response']['docs']
            if out_format == 'CSV':
                export_to_csv(jobs, file_name, task_folder, fields)
            elif out_format == 'XML':
                export_to_xml(jobs, file_name, task_folder)
            elif out_format == 'SHP':
                export_to_shp(jobs, file_name, task_folder)
            exported_cnt += chunk_size
            if exported_cnt > num_results:
                status_writer.send_percent(100, 'exported: 100%',
                                           'export_results')
            else:
                percent_done = exported_cnt / num_results
                status_writer.send_percent(
                    percent_done,
                    '{0}: {1:.0f}%'.format("exported", percent_done * 100),
                    'export_results')
    else:
        # Voyager Search Portal/Cart UI
        ids = []
        for p in request['params']:
            if 'ids' in p:
                ids = p['ids']
                break
        groups = task_utils.grouper(list(ids), chunk_size, '')
        i = 0
        for group in groups:
            i += len([v for v in group if not v == ''])
            results = requests.get(query + '&ids={0}'.format(','.join(group)),
                                   verify=verify_ssl,
                                   headers=headers)
            jobs = eval(results.text)['response']['docs']
            if out_format == 'CSV':
                export_to_csv(jobs, file_name, task_folder, fields)
            elif out_format == 'XML':
                export_to_xml(jobs, file_name, task_folder)
            elif out_format == 'SHP':
                export_to_shp(jobs, file_name, task_folder)
            percent_done = float(i) / num_results
            status_writer.send_percent(
                percent_done, '{0}: {1:.0f}%'.format("exported",
                                                     percent_done * 100),
                'export_results')

    # Zip up outputs.
    if exported_count == 0:
        status_writer.send_state(status.STAT_FAILED)
        task_utils.report(os.path.join(request['folder'], '__report.json'),
                          exported_count, 0, errors_count, errors_reasons)
    else:
        task_utils.report(os.path.join(request['folder'], '__report.json'),
                          exported_count, 0, errors_count, errors_reasons)
        zip_file = task_utils.zip_data(task_folder,
                                       '{0}.zip'.format(file_name))
        shutil.move(
            zip_file,
            os.path.join(os.path.dirname(task_folder),
                         os.path.basename(zip_file)))
Ejemplo n.º 31
0
def execute(request):
    """Writes existing metadata for summary, description and tags.
    If overwrite is false, existing metadata is untouched unless any
    field is empty or does not exist, then it is created.

    :param request: json as a dict.
    """
    updated = 0
    errors = 0
    skipped = 0
    global result_count
    parameters = request['params']
    summary = task_utils.get_parameter_value(parameters, 'summary', 'value')
    description = task_utils.get_parameter_value(parameters, 'description', 'value')
    tags = task_utils.get_parameter_value(parameters, 'tags', 'value')
    data_credits = task_utils.get_parameter_value(parameters, 'credits', 'value')
    constraints = task_utils.get_parameter_value(parameters, 'constraints', 'value')

    # Handle commas, spaces, and/or new line separators.
    tags = [tag for tag in re.split(' |,|\n', tags) if not tag == '']
    overwrite = task_utils.get_parameter_value(parameters, 'overwrite', 'value')
    if not overwrite:
        overwrite = False
    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    # Stylesheet
    xslt_file = os.path.join(arcpy.GetInstallInfo()['InstallDir'], 'Metadata/Stylesheets/gpTools/exact copy of.xslt')

    # Template metadata file.
    template_xml = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'metadata_template.xml')

    result_count, response_index = task_utils.get_result_count(parameters)
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl + ',links'
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'), 'write_metadata')
    i = 0.
    for group in groups:
        i += len(group) - group.count('')
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

        docs = results.json()['response']['docs']
        if not docs:
            docs = parameters[response_index]['response']['docs']

        input_items = []
        for doc in docs:
            if 'path' in doc and os.path.exists(doc['path']):
                if 'links' in doc:
                    links = eval(doc['links'])
                    input_items.append((doc['path'], links['links'][0]['link'][0]['id']))
                else:
                    input_items.append((doc['path'], doc['id']))
            elif 'absolute_path' in doc and os.path.exists(doc['absolute_path']):
                if 'links' in doc:
                    links = eval(doc['links'])
                    input_items.append((doc['absolute_path'], links['links'][0]['link'][0]['id']))
                else:
                    input_items.append((doc['absolute_path'], doc['id']))
            elif '[downloadURL]' in doc:
                if 'links' in doc:
                    links = eval(doc['links'])
                    input_items.append((doc['[downloadURL]'], links['links'][0]['link'][0]['id']))
                else:
                    input_items.append((doc['[downloadURL]'], doc['id']))

        result = write_metadata(input_items, template_xml, xslt_file, summary, description, tags, data_credits, constraints, overwrite, headers)
        updated += result[0]
        errors += result[1]
        skipped += result[2]
        status_writer.send_percent(i / result_count, '{0}: {1:.0f}%'.format("Processed", i / result_count * 100), 'write_metadata')

    # Report state.
    if skipped > 0 or errors > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
    else:
        status_writer.send_state(status.STAT_SUCCESS)
    task_utils.report(os.path.join(request['folder'], '__report.json'), updated, skipped, errors, errors_reasons, skipped_reasons)
Ejemplo n.º 32
0
def execute(request):
    """Adds a field and calculates it to some value.
    :param request: json as a dict.
    """
    created = 0
    skipped = 0
    errors = 0
    warnings = 0
    global result_count
    parameters = request['params']

    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    # Parameter values
    field_name = task_utils.get_parameter_value(parameters, 'field_name',
                                                'value')
    field_type = task_utils.get_parameter_value(parameters, 'field_type',
                                                'value')
    field_value = task_utils.get_parameter_value(parameters, 'field_value',
                                                 'value')

    # Query the index for results in groups of 25.
    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    result_count, response_index = task_utils.get_result_count(parameters)
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl + ',links'
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
                               fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']),
                                    task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')

    # Begin processing
    status_writer.send_percent(0.0, _('Starting to process...'), 'add_field')
    for group in groups:
        if fq:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                verify=verify_ssl,
                headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query +
                                   '{0}&ids={1}'.format(fl, ','.join(group)),
                                   verify=verify_ssl,
                                   headers=headers)
        else:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                verify=verify_ssl,
                headers=headers)

        docs = results.json()['response']['docs']
        input_items = []
        for doc in docs:
            if 'path' in doc:
                if 'links' in doc:
                    links = eval(doc['links'])
                    input_items.append((doc['id'], doc['absolute_path'],
                                        links['links'][0]['link'][0]['id']))
                else:
                    input_items.append((doc['id'], doc['absolute_path']))

        result = add_field(input_items, field_name, field_type, field_value)
        created += result[0]
        errors += result[1]
        skipped += result[2]
        warnings += result[3]

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'], '__report.json'),
                      created, skipped, errors, errors_reasons,
                      skipped_reasons, warnings, warnings_reasons)
Ejemplo n.º 33
0
def execute(request):
    """Adds a field and calculates it to some value.
    :param request: json as a dict.
    """
    created = 0
    skipped = 0
    errors = 0
    warnings = 0
    global result_count
    parameters = request['params']

    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    # Parameter values
    field_name = task_utils.get_parameter_value(parameters, 'field_name', 'value')
    field_type = task_utils.get_parameter_value(parameters, 'field_type', 'value')
    field_value = task_utils.get_parameter_value(parameters, 'field_value', 'value')

    # Query the index for results in groups of 25.
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    result_count, response_index = task_utils.get_result_count(parameters)
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl + ',links'
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')

    # Begin processing
    status_writer.send_percent(0.0, _('Starting to process...'), 'add_field')
    for group in groups:
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

        docs = results.json()['response']['docs']
        input_items = []
        for doc in docs:
            if 'path' in doc:
                if 'links' in doc:
                    links = eval(doc['links'])
                    input_items.append((doc['id'], doc['path'], links['links'][0]['link'][0]['id']))
                else:
                    input_items.append((doc['id'], doc['path']))

        result = add_field(input_items, field_name, field_type, field_value)
        created += result[0]
        errors += result[1]
        skipped += result[2]
        warnings += result[3]

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'], '__report.json'), created, skipped, errors, errors_reasons, skipped_reasons, warnings, warnings_reasons)
Ejemplo n.º 34
0
def execute(request):
    """Package inputs to an Esri map or layer package.
    :param request: json as a dict.
    """
    errors = 0
    skipped = 0
    layers = []
    files = []

    app_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    parameters = request['params']
    out_format = task_utils.get_parameter_value(parameters, 'output_format',
                                                'value')
    summary = task_utils.get_parameter_value(parameters, 'summary')
    tags = task_utils.get_parameter_value(parameters, 'tags')
    output_file_name = task_utils.get_parameter_value(parameters,
                                                      'output_file_name')
    if not output_file_name:
        output_file_name = 'package_results'

    # Get the clip region as an extent object.
    clip_area = None
    try:
        clip_area_wkt = task_utils.get_parameter_value(parameters,
                                                       'processing_extent',
                                                       'wkt')
        clip_area = task_utils.get_clip_region(clip_area_wkt)
    except (KeyError, ValueError):
        pass

    out_workspace = os.path.join(request['folder'], 'temp')
    if not os.path.exists(out_workspace):
        os.makedirs(out_workspace)

    num_results, response_index = task_utils.get_result_count(parameters)
    # if num_results > task_utils.CHUNK_SIZE:
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
                               fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, num_results),
                                    task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']),
                                    task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, num_results),
                                    task_utils.CHUNK_SIZE, '')

    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    status_writer.send_status(_('Starting to process...'))
    for group in groups:
        if fq:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                verify=verify_ssl,
                headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query +
                                   '{0}&ids={1}'.format(fl, ','.join(group)),
                                   verify=verify_ssl,
                                   headers=headers)
        else:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                verify=verify_ssl,
                headers=headers)

        input_items = task_utils.get_input_items(
            results.json()['response']['docs'])
        if not input_items:
            input_items = task_utils.get_input_items(
                parameters[response_index]['response']['docs'])
        layers, files, errors, skipped = get_items(input_items, out_workspace)
    # else:
    #     input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
    #     layers, files, errors, skipped = get_items(input_items, out_workspace)

    if errors == num_results:
        status_writer.send_state(status.STAT_FAILED,
                                 _('No results to package'))
        return

    try:
        if out_format == 'MPK':
            shutil.copyfile(
                os.path.join(app_folder, 'supportfiles', 'MapTemplate.mxd'),
                os.path.join(out_workspace, 'output.mxd'))
            mxd = arcpy.mapping.MapDocument(
                os.path.join(out_workspace, 'output.mxd'))
            if mxd.description == '':
                mxd.description = os.path.basename(mxd.filePath)
            df = arcpy.mapping.ListDataFrames(mxd)[0]
            for layer in layers:
                arcpy.mapping.AddLayer(df, layer)
            mxd.save()
            status_writer.send_status(
                _('Generating {0}. Large input {1} will take longer to process.'
                  .format('MPK', 'results')))
            if arcpy.GetInstallInfo()['Version'] == '10.0':
                arcpy.PackageMap_management(
                    mxd.filePath,
                    os.path.join(os.path.dirname(out_workspace),
                                 '{0}.mpk'.format(output_file_name)),
                    'PRESERVE',
                    extent=clip_area)
            elif arcpy.GetInstallInfo()['Version'] == '10.1':
                arcpy.PackageMap_management(
                    mxd.filePath,
                    os.path.join(os.path.dirname(out_workspace),
                                 '{0}.mpk'.format(output_file_name)),
                    'PRESERVE',
                    extent=clip_area,
                    ArcGISRuntime='RUNTIME',
                    version='10',
                    additional_files=files,
                    summary=summary,
                    tags=tags)
            else:
                arcpy.PackageMap_management(
                    mxd.filePath,
                    os.path.join(os.path.dirname(out_workspace),
                                 '{0}.mpk'.format(output_file_name)),
                    'PRESERVE',
                    extent=clip_area,
                    arcgisruntime='RUNTIME',
                    version='10',
                    additional_files=files,
                    summary=summary,
                    tags=tags)
            #  Create a thumbnail size PNG of the mxd.
            task_utils.make_thumbnail(
                mxd, os.path.join(request['folder'], '_thumb.png'))
        else:
            status_writer.send_status(
                _('Generating {0}. Large input {1} will take longer to process.'
                  .format('LPK', 'results')))
            for layer in layers:
                if layer.description == '':
                    layer.description = layer.name
            if arcpy.GetInstallInfo()['Version'] == '10.0':
                arcpy.PackageLayer_management(
                    layers,
                    os.path.join(os.path.dirname(out_workspace),
                                 '{0}.lpk'.format(output_file_name)),
                    'PRESERVE',
                    extent=clip_area,
                    version='10')
            else:
                arcpy.PackageLayer_management(
                    layers,
                    os.path.join(os.path.dirname(out_workspace),
                                 '{0}.lpk'.format(output_file_name)),
                    'PRESERVE',
                    extent=clip_area,
                    version='10',
                    additional_files=files,
                    summary=summary,
                    tags=tags)
            #  Create a thumbnail size PNG of the mxd.
            task_utils.make_thumbnail(
                layers[0], os.path.join(request['folder'], '_thumb.png'))
    except (RuntimeError, ValueError, arcpy.ExecuteError) as ex:
        status_writer.send_state(status.STAT_FAILED, repr(ex))
        return

    # Update state if necessary.
    if errors > 0 or skipped:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(errors + skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'),
                      num_results - (skipped + errors), skipped, errors,
                      errors_reasons, skipped_reasons)
Ejemplo n.º 35
0
def execute(request):
    """Deletes files.
    :param request: json as a dict.
    """
    deleted = 0
    skipped = 0

    parameters = request['params']
    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    num_results, response_index = task_utils.get_result_count(parameters)
    if num_results > task_utils.CHUNK_SIZE:
        # Query the index for results in groups of 25.
        query_index = task_utils.QueryIndex(parameters[response_index])
        fl = query_index.fl
        query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1],
                                   '/select?&wt=json', fl)
        fq = query_index.get_fq()
        if fq:
            groups = task_utils.grouper(range(0, num_results),
                                        task_utils.CHUNK_SIZE, '')
            query += fq
        elif 'ids' in parameters[response_index]:
            groups = task_utils.grouper(
                list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE,
                '')
        else:
            groups = task_utils.grouper(range(0, num_results),
                                        task_utils.CHUNK_SIZE, '')

        status_writer.send_percent(0.0, _('Starting to process...'),
                                   'delete_files')
        i = 0.
        for group in groups:
            i += len(group) - group.count('')
            if fq:
                results = requests.get(query + "&rows={0}&start={1}".format(
                    task_utils.CHUNK_SIZE, group[0]),
                                       headers=headers)
            elif 'ids' in parameters[response_index]:
                results = requests.get(
                    query + '{0}&ids={1}'.format(fl, ','.join(group)),
                    headers=headers)
            else:
                results = requests.get(query + "&rows={0}&start={1}".format(
                    task_utils.CHUNK_SIZE, group[0]),
                                       headers=headers)

            input_items = task_utils.get_input_items(
                results.json()['response']['docs'], True, True)
            if not input_items:
                input_items = task_utils.get_input_items(
                    parameters[response_index]['response']['docs'])

            result = delete_files(input_items)
            deleted += result[0]
            skipped += result[1]
            status_writer.send_percent(
                i / num_results,
                '{0}: {1:.0f}%'.format("Processed",
                                       i / num_results * 100), 'delete_files')
    else:
        input_items = task_utils.get_input_items(
            parameters[response_index]['response']['docs'], True, True)
        deleted, skipped = delete_files(input_items, True)

    # Update state if necessary.
    if skipped > 0:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'),
                      deleted,
                      skipped,
                      skipped_details=skipped_reasons)
Ejemplo n.º 36
0
def execute(request):
    """Copies files to a target folder.
    :param request: json as a dict.
    """
    extracted = 0
    skipped = 0
    errors = 0
    global result_count
    parameters = request['params']

    output_type = task_utils.get_parameter_value(parameters, 'output_format',
                                                 'value')
    task_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)
        if output_type == 'FGDB':
            arcpy.CreateFileGDB_management(task_folder, 'output.gdb')

    result_count, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
                               fl)
    # query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']),
                                    task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')

    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    status_writer.send_percent(0.0, _('Starting to process...'),
                               'locate_xt_arcgis_tool')
    for group in groups:
        if fq:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query +
                                   '{0}&ids={1}'.format(fl, ','.join(group)),
                                   headers=headers)
        else:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                headers=headers)

        docs = results.json()['response']['docs']
        if not docs:
            docs = parameters[response_index]['response']['docs']

        input_items = task_utils.get_input_items(docs)
        if input_items:
            result = extract(input_items, output_type, task_folder)
            extracted += result[0]
            errors += result[1]
            skipped += result[2]
        else:
            status_writer.send_state(
                status.STAT_FAILED,
                _('No items to process. Check if items exist.'))
            return

    # Zip up outputs.
    zip_file = task_utils.zip_data(task_folder, 'output.zip')
    shutil.move(
        zip_file,
        os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'],
                                   '__report.json'), extracted, skipped,
                      errors, errors_reasons, skipped_reasons)
Ejemplo n.º 37
0
def execute(request):
    """Copies data to an existing geodatabase or feature dataset.
    :param request: json as a dict.
    """
    added = 0
    errors = 0
    skipped = 0
    global result_count
    parameters = request['params']

    # Get the target workspace location.
    out_gdb = task_utils.get_parameter_value(parameters, 'target_workspace',
                                             'value')

    # Retrieve the coordinate system code.
    out_coordinate_system = task_utils.get_parameter_value(
        parameters, 'output_projection', 'code')
    if not out_coordinate_system == '0':  # Same as Input
        arcpy.env.outputCoordinateSystem = task_utils.get_spatial_reference(
            out_coordinate_system)

    task_folder = request['folder']
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)

    # Check if the geodatabase exists or if it is a feature dataset.
    is_fds = False
    if not os.path.exists(out_gdb):
        if out_gdb.endswith('.gdb'):
            arcpy.CreateFileGDB_management(os.path.dirname(out_gdb),
                                           os.path.basename(out_gdb))
            status_writer.send_status(
                _('Created output workspace: {0}').format(out_gdb))
        elif out_gdb.endswith('.mdb'):
            arcpy.CreatePersonalGDB_management(os.path.dirname(out_gdb),
                                               os.path.basename(out_gdb))
            status_writer.send_status(
                _('Created output workspace: {0}').format(out_gdb))
        elif out_gdb.endswith('.sde'):
            status_writer.send_state(status.STAT_FAILED,
                                     _('{0} does not exist').format(out_gdb))
            return
        else:
            # Possible feature dataset.
            is_fds = is_feature_dataset(out_gdb)
            if not is_fds:
                if os.path.dirname(out_gdb).endswith('.gdb'):
                    if not os.path.exists(os.path.dirname(out_gdb)):
                        arcpy.CreateFileGDB_management(
                            os.path.dirname(os.path.dirname(out_gdb)),
                            os.path.basename(os.path.dirname(out_gdb)))
                    arcpy.CreateFeatureDataset_management(
                        os.path.dirname(out_gdb), os.path.basename(out_gdb))
                elif os.path.dirname(out_gdb).endswith('.mdb'):
                    if not os.path.exists(os.path.dirname(out_gdb)):
                        arcpy.CreatePersonalGDB_management(
                            os.path.dirname(os.path.dirname(out_gdb)),
                            os.path.basename(os.path.dirname(out_gdb)))
                    arcpy.CreateFeatureDataset_management(
                        os.path.dirname(out_gdb), os.path.basename(out_gdb))

    status_writer.send_status(_('Setting the output workspace...'))
    arcpy.env.workspace = out_gdb

    headers = {
        'x-access-token': task_utils.get_security_token(request['owner'])
    }
    result_count, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json',
                               fl)
    # query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']),
                                    task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count),
                                    task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'),
                               'add_to_geodatabase')
    for group in groups:
        if fq:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                verify=verify_ssl,
                headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query +
                                   '{0}&ids={1}'.format(fl, ','.join(group)),
                                   verify=verify_ssl,
                                   headers=headers)
        else:
            results = requests.get(
                query +
                "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]),
                verify=verify_ssl,
                headers=headers)

        docs = results.json()['response']['docs']
        input_items = task_utils.get_input_items(docs)
        if not input_items:
            input_items = task_utils.get_input_items(
                parameters[response_index]['response']['docs'])
        input_rows = collections.defaultdict(list)
        for doc in docs:
            if 'path' not in doc:
                input_rows[doc['title']].append(doc)
        if input_rows:
            result = add_to_geodatabase(input_rows, out_gdb, is_fds)
            added += result[0]
            errors += result[1]
            skipped += result[2]

        if input_items:
            result = add_to_geodatabase(input_items, out_gdb, is_fds)
            added += result[0]
            errors += result[1]
            skipped += result[2]

        if not input_items and not input_rows:
            status_writer.send_state(
                status.STAT_FAILED,
                _('No items to process. Check if items exist.'))
            return

    # Update state if necessary.
    if skipped > 0 or errors > 0:
        status_writer.send_state(
            status.STAT_WARNING,
            _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(task_folder, '__report.json'), added,
                      skipped, errors, errors_reasons, skipped_reasons)
Ejemplo n.º 38
0
def execute(request):
    """Clips selected search results using the clip geometry.
    :param request: json as a dict.
    """
    clipped = 0
    errors = 0
    skipped = 0
    global result_count
    parameters = request['params']

    # Retrieve the clip features.
    clip_features = task_utils.get_parameter_value(parameters, 'clip_features', 'value')

    # Retrieve the coordinate system code.
    out_coordinate_system = int(task_utils.get_parameter_value(parameters, 'output_projection', 'code'))

    # Retrieve the output format, create mxd and output file name parameter values.
    out_format = task_utils.get_parameter_value(parameters, 'output_format', 'value')
    create_mxd = task_utils.get_parameter_value(parameters, 'create_mxd', 'value')
    output_file_name = task_utils.get_parameter_value(parameters, 'output_file_name', 'value')
    if not output_file_name:
        output_file_name = 'clip_results'

    # Create the temporary workspace if clip_feature_class:
    out_workspace = os.path.join(request['folder'], 'temp')
    if not os.path.exists(out_workspace):
        os.makedirs(out_workspace)

    # Set the output coordinate system.
    if not out_coordinate_system == 0:  # Same as Input
        out_sr = task_utils.get_spatial_reference(out_coordinate_system)
        arcpy.env.outputCoordinateSystem = out_sr

    # Set the output workspace.
    status_writer.send_status(_('Setting the output workspace...'))
    if not out_format == 'SHP':
        out_workspace = arcpy.CreateFileGDB_management(out_workspace, 'output.gdb').getOutput(0)
    arcpy.env.workspace = out_workspace

    # Query the index for results in groups of 25.
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    result_count, response_index = task_utils.get_result_count(parameters)
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl

    # Get the Clip features by id.
    id = clip_features['id']
    clip_query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', "&fl=id,path,fullpath:[absolute],absolute_path:[absolute],[lyrFile],[geo]&q=id:{0}".format(id))
    clip_result = requests.get(clip_query, verify=verify_ssl, headers=headers)
    clipper = clip_result.json()['response']['docs'][0]
    if 'absolute_path' in clipper and not clipper['absolute_path'].startswith('s3'):
        clip_features = clipper['absolute_path']
    elif '[lyrFile]' in clipper:
        clip_features = clipper['[lyrFile]']
    elif '[geo]' in clipper:
        clip_features = arcpy.AsShape(clipper['[geo]']).projectAs(arcpy.SpatialReference(4326))
    elif 'absolute_path' in clipper and clipper['absolute_path'].startswith('s3'):
        base_name = os.path.basename(clipper['path'])
        temp_folder = tempfile.mkdtemp()
        if '[downloadURL]' in clipper:
            download = os.path.join(temp_folder, os.path.basename(clipper['[downloadURL]']))
            response = requests.get(clipper['[downloadURL]'], verify=verify_ssl)
            with open(download, 'wb') as fp:
                fp.write(response.content)
            if download.endswith('.zip'):
                zip = zipfile.ZipFile(download)
                zip.extractall(temp_folder)
                clip_features = os.path.join(temp_folder, base_name)
            else:
                clip_features = download
    else:
        bbox = clipper['bbox'].split()
        extent = arcpy.Extent(*bbox)
        pt_array = arcpy.Array([extent.lowerLeft, extent.upperLeft, extent.upperRight, extent.lowerRight])
        clip_features = arcpy.Polygon(pt_array, 4326)

    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'), 'clip_data')
    for group in groups:
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), verify=verify_ssl, headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), verify=verify_ssl, headers=headers)

        docs = results.json()['response']['docs']
        input_items = task_utils.get_input_items(docs)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])

        input_rows = collections.defaultdict(list)
        for doc in docs:
            if 'path' not in doc:
                input_rows[doc['name']].append(doc)
        if input_rows:
            result = clip_data(input_rows, out_workspace, clip_features, out_format)
            clipped += result[0]
            errors += result[1]
            skipped += result[2]

        if input_items:
            result = clip_data(input_items, out_workspace, clip_features, out_format)
            clipped += result[0]
            errors += result[1]
            skipped += result[2]

        if not input_items and not input_rows:
            status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.'))
            return

    if arcpy.env.workspace.endswith('.gdb'):
        out_workspace = os.path.dirname(arcpy.env.workspace)
    if clipped > 0:
        try:
            if out_format == 'MPK':
                mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
                mxd = task_utils.create_mxd(out_workspace, mxd_template, 'output')
                status_writer.send_status(_("Packaging results..."))
                task_utils.create_mpk(out_workspace, mxd, files_to_package)
                shutil.move(os.path.join(out_workspace, 'output.mpk'),
                            os.path.join(os.path.dirname(out_workspace), '{0}.mpk'.format(output_file_name)))
            elif out_format == 'LPK':
                status_writer.send_status(_("Packaging results..."))
                task_utils.create_lpk(out_workspace, output_file_name, files_to_package)
            elif out_format == 'KML':
                task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb"))
                arcpy.env.workspace = ''
                arcpy.RefreshCatalog(os.path.join(out_workspace, "output.gdb"))
                try:
                    arcpy.Delete_management(os.path.join(out_workspace, "output.gdb"))
                except arcpy.ExecuteError:
                    pass
                zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
                shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
            else:
                if create_mxd:
                    mxd_template = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'MapTemplate.mxd')
                    task_utils.create_mxd(out_workspace, mxd_template, 'output')
                zip_file = task_utils.zip_data(out_workspace, '{0}.zip'.format(output_file_name))
                shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
        except arcpy.ExecuteError as ee:
            status_writer.send_state(status.STAT_FAILED, _(ee))
            sys.exit(1)
    else:
        status_writer.send_state(status.STAT_FAILED, _('No output created. Zero inputs were clipped.'))

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(errors + skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), clipped, skipped, errors, errors_reasons, skipped_reasons)
Ejemplo n.º 39
0
def execute(request):
    """Copies files to a target folder.
    :param request: json as a dict.
    """
    created = 0
    skipped = 0
    errors = 0
    global result_count
    parameters = request['params']

    if not os.path.exists(request['folder']):
        os.makedirs(request['folder'])

    meta_folder = task_utils.get_parameter_value(parameters, 'meta_data_folder', 'value')
    result_count, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')

    status_writer.send_percent(0.0, _('Starting to process...'), 'create_layer_files')
    i = 0.
    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    for group in groups:
        i += len(group) - group.count('')
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

        docs = results.json()['response']['docs']
        # docs = eval(results.read().replace('false', 'False').replace('true', 'True'))['response']['docs']
        if not docs:
            docs = parameters[response_index]['response']['docs']
        input_items = []
        for doc in docs:
            if 'path' in doc:
                input_items.append((doc['id'], doc['path'], doc['name'], doc['location']))
        result = create_layer_file(input_items, meta_folder)
        created += result[0]
        errors += result[1]
        skipped += result[2]

    try:
        shutil.copy2(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', '_thumb.png'), request['folder'])
    except IOError:
        pass
    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'], '__report.json'), created, skipped, errors, errors_reasons, skipped_reasons)
Ejemplo n.º 40
0
def execute(request):
    """Exports search results a CSV, shapefile or XML document.
    :param request: json as a dict.
    """
    chunk_size = task_utils.CHUNK_SIZE
    file_name = task_utils.get_parameter_value(request['params'], 'file_name', 'value')
    fields = task_utils.get_parameter_value(request['params'], 'fields', 'value')
    out_format = task_utils.get_parameter_value(request['params'], 'output_format', 'value')

    if not 'path' in fields and 'path:[absolute]' in fields:
        fields.append('path')

    if 'geo' in fields:
        i_geo = fields.index('geo')
        fields.remove('geo')
        fields.insert(i_geo, '[geo]')


    # Create the temporary workspace.
    task_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)

    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    num_results, response_index = task_utils.get_result_count(request['params'])
    query = '{0}/select?&wt=json&fl={1}'.format(sys.argv[2].split('=')[1], ','.join(fields))
    if 'query' in request['params'][response_index]:
        # Voyager Search Traditional UI
        for p in request['params']:
            if 'query' in p:
                request_qry = p['query']
                break
        if 'voyager.list' in request_qry:
            query += '&voyager.list={0}'.format(request_qry['voyager.list'])

        # Replace spaces with %20 & remove \\ to avoid HTTP Error 400.
        if 'fq' in request_qry:
            try:
                query += '&fq={0}'.format(request_qry['fq'].replace("\\", ""))
                query = query.replace(' ', '%20')
            except AttributeError:
                for qry in request_qry['fq']:
                    query += '&fq={0}'.format(qry).replace("\\", "").replace(' ', '%20')
        if 'q' in request_qry:
            try:
                query += '&q={0}'.format(request_qry['q'].replace("\\", ""))
                query = query.replace(' ', '%20')
            except AttributeError:
                for qry in request_qry['q']:
                    query += '&q={0}'.format(qry).replace("\\", "").replace(' ', '%20')
        if 'place' in request_qry:
            try:
                query += '&place={0}'.format(request_qry['place'].replace("\\", ""))
                query = query.replace(' ', '%20')
            except AttributeError:
                for qry in request_qry['place']:
                    query += '&place={0}'.format(qry).replace("\\", "").replace(' ', '%20')
        if 'place.op' in request_qry:
            query += '&place.op={0}'.format(request_qry['place.op'])

        query += '&rows={0}&start={1}'
        exported_cnt = 0.
        for i in xrange(0, num_results, chunk_size):
            req = urllib2.Request(query.replace('{0}', str(chunk_size)).replace('{1}', str(i)), headers=headers)
            for n in urllib2.urlopen(req):
                jobs = eval(n.replace('null', '"null"'))['response']['docs']
                if out_format == 'CSV':
                    export_to_csv(jobs, file_name, task_folder, fields)
                elif out_format == 'XML':
                    export_to_xml(jobs, file_name, task_folder)
                elif out_format == 'SHP':
                    export_to_shp(jobs, file_name, task_folder)
                exported_cnt += chunk_size
                if exported_cnt > num_results:
                    status_writer.send_percent(100, 'exported: 100%', 'export_results')
                else:
                    percent_done = exported_cnt / num_results
                    status_writer.send_percent(percent_done, '{0}: {1:.0f}%'.format("exported", percent_done * 100), 'export_results')
    else:
        # Voyager Search Portal/Cart UI
        ids = []
        for p in request['params']:
            if 'ids' in p:
                ids = p['ids']
                break
        groups = task_utils.grouper(list(ids), chunk_size, '')
        i = 0
        for group in groups:
            i += len([v for v in group if not v == ''])
            req = urllib2.Request(query + '&ids={0}'.format(','.join(group)), headers=headers)
            results = urllib2.urlopen(req)
            jobs = eval(results.read())['response']['docs']
            if out_format == 'CSV':
                export_to_csv(jobs, file_name, task_folder, fields)
            elif out_format == 'XML':
                export_to_xml(jobs, file_name, task_folder)
            elif out_format == 'SHP':
                export_to_shp(jobs, file_name, task_folder)
            percent_done = float(i) / num_results
            status_writer.send_percent(percent_done, '{0}: {1:.0f}%'.format("exported", percent_done * 100), 'export_results')

    # Zip up outputs.
    if exported_count == 0:
        task_utils.report(os.path.join(request['folder'], '__report.json'), exported_count, 0, errors_count, errors_reasons)
    else:
        task_utils.report(os.path.join(request['folder'], '__report.json'), exported_count, 0, errors_count, errors_reasons)
    zip_file = task_utils.zip_data(task_folder, 'output.zip')
    shutil.move(zip_file, os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))
def execute(request):
    """Copies files to a target folder.
    :param request: json as a dict.
    """
    extracted = 0
    skipped = 0
    errors = 0
    global result_count
    parameters = request['params']

    output_type = task_utils.get_parameter_value(parameters, 'output_format', 'value')
    task_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)
        if output_type == 'FGDB':
            arcpy.CreateFileGDB_management(task_folder, 'output.gdb')

    result_count, response_index = task_utils.get_result_count(parameters)
    # Query the index for results in groups of 25.
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl
    query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
    # query = '{0}{1}{2}'.format("http://localhost:8888/solr/v0", '/select?&wt=json', fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')
        query += fq
    elif 'ids' in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, '')

    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    status_writer.send_percent(0.0, _('Starting to process...'), 'locate_xt_arcgis_tool')
    for group in groups:
        if fq:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
        elif 'ids' in parameters[response_index]:
            results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
        else:
            results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

        docs = results.json()['response']['docs']
        if not docs:
            docs = parameters[response_index]['response']['docs']

        input_items = task_utils.get_input_items(docs)
        if input_items:
            result = extract(input_items, output_type, task_folder)
            extracted += result[0]
            errors += result[1]
            skipped += result[2]
        else:
            status_writer.send_state(status.STAT_FAILED, _('No items to process. Check if items exist.'))
            return

    # Zip up outputs.
    zip_file = task_utils.zip_data(task_folder, 'output.zip')
    shutil.move(zip_file, os.path.join(os.path.dirname(task_folder), os.path.basename(zip_file)))

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'], '__report.json'), extracted, skipped, errors, errors_reasons, skipped_reasons)
def execute(request):
    """Clips selected search results using the clip geometry.
    :param request: json as a dict.
    """
    clipped = 0
    errors = 0
    skipped = 0
    global result_count
    parameters = request["params"]

    # Retrieve the clip features.
    clip_features = task_utils.get_parameter_value(parameters, "clip_features", "value")

    # Retrieve the coordinate system code.
    out_coordinate_system = int(task_utils.get_parameter_value(parameters, "output_projection", "code"))

    # Retrieve the output format, create mxd and output file name parameter values.
    out_format = task_utils.get_parameter_value(parameters, "output_format", "value")
    create_mxd = task_utils.get_parameter_value(parameters, "create_mxd", "value")
    output_file_name = task_utils.get_parameter_value(parameters, "output_file_name", "value")
    if not output_file_name:
        output_file_name = "clip_results"

    # Create the temporary workspace if clip_feature_class:
    out_workspace = os.path.join(request["folder"], "temp")
    if not os.path.exists(out_workspace):
        os.makedirs(out_workspace)

    # Set the output coordinate system.
    if not out_coordinate_system == 0:  # Same as Input
        out_sr = task_utils.get_spatial_reference(out_coordinate_system)
        arcpy.env.outputCoordinateSystem = out_sr

    # Set the output workspace.
    status_writer.send_status(_("Setting the output workspace..."))
    if not out_format == "SHP":
        out_workspace = arcpy.CreateFileGDB_management(out_workspace, "output.gdb").getOutput(0)
    arcpy.env.workspace = out_workspace

    # Query the index for results in groups of 25.
    headers = {"x-access-token": task_utils.get_security_token(request["owner"])}
    result_count, response_index = task_utils.get_result_count(parameters)
    query_index = task_utils.QueryIndex(parameters[response_index])
    fl = query_index.fl

    # Get the Clip features by id.
    id = clip_features["id"]
    clip_query = "{0}{1}{2}".format(
        sys.argv[2].split("=")[1], "/select?&wt=json", "&fl=id,path:[absolute],[lyrFile],[geo]&q=id:{0}".format(id)
    )
    clip_result = requests.get(clip_query, headers=headers)
    clipper = clip_result.json()["response"]["docs"][0]
    if "path" in clipper:
        clip_features = clipper["path"]
    elif "[lyrFile]" in clipper:
        clip_features = clipper["[lyrFile]"]
    elif "[geo]" in clipper:
        clip_features = arcpy.AsShape(clipper["[geo]"]).projectAs(arcpy.SpatialReference(4326))
    else:
        bbox = clipper["bbox"].split()
        extent = arcpy.Extent(*bbox)
        pt_array = arcpy.Array([extent.lowerLeft, extent.upperLeft, extent.upperRight, extent.lowerRight])
        clip_features = arcpy.Polygon(pt_array, 4326)

    query = "{0}{1}{2}".format(sys.argv[2].split("=")[1], "/select?&wt=json", fl)
    fq = query_index.get_fq()
    if fq:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")
        query += fq
    elif "ids" in parameters[response_index]:
        groups = task_utils.grouper(list(parameters[response_index]["ids"]), task_utils.CHUNK_SIZE, "")
    else:
        groups = task_utils.grouper(range(0, result_count), task_utils.CHUNK_SIZE, "")

    status_writer.send_percent(0.0, _("Starting to process..."), "clip_data")
    for group in groups:
        if fq:
            results = requests.get(
                query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
            )
        elif "ids" in parameters[response_index]:
            results = requests.get(query + "{0}&ids={1}".format(fl, ",".join(group)), headers=headers)
        else:
            results = requests.get(
                query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers
            )

        # docs = eval(results.read().replace('false', 'False').replace('true', 'True').replace('null', 'None'))['response']['docs']
        docs = results.json()["response"]["docs"]
        input_items = task_utils.get_input_items(docs)
        if not input_items:
            input_items = task_utils.get_input_items(parameters[response_index]["response"]["docs"])

        input_rows = collections.defaultdict(list)
        for doc in docs:
            if "path" not in doc:
                input_rows[doc["name"]].append(doc)
        if input_rows:
            result = clip_data(input_rows, out_workspace, clip_features, out_format)
            clipped += result[0]
            errors += result[1]
            skipped += result[2]

        if input_items:
            result = clip_data(input_items, out_workspace, clip_features, out_format)
            clipped += result[0]
            errors += result[1]
            skipped += result[2]

        if not input_items and not input_rows:
            status_writer.send_state(status.STAT_FAILED, _("No items to process. Check if items exist."))
            return

    if arcpy.env.workspace.endswith(".gdb"):
        out_workspace = os.path.dirname(arcpy.env.workspace)
    if clipped > 0:
        try:
            if out_format == "MPK":
                mxd_template = os.path.join(
                    os.path.dirname(os.path.dirname(__file__)), "supportfiles", "MapTemplate.mxd"
                )
                mxd = task_utils.create_mxd(out_workspace, mxd_template, "output")
                status_writer.send_status(_("Packaging results..."))
                task_utils.create_mpk(out_workspace, mxd, files_to_package)
                shutil.move(
                    os.path.join(out_workspace, "output.mpk"),
                    os.path.join(os.path.dirname(out_workspace), "{0}.mpk".format(output_file_name)),
                )
            elif out_format == "LPK":
                status_writer.send_status(_("Packaging results..."))
                task_utils.create_lpk(out_workspace, output_file_name, files_to_package)
            elif out_format == "KML":
                task_utils.convert_to_kml(os.path.join(out_workspace, "output.gdb"))
                arcpy.env.workspace = ""
                arcpy.RefreshCatalog(os.path.join(out_workspace, "output.gdb"))
                try:
                    arcpy.Delete_management(os.path.join(out_workspace, "output.gdb"))
                except arcpy.ExecuteError:
                    pass
                zip_file = task_utils.zip_data(out_workspace, "{0}.zip".format(output_file_name))
                shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
            else:
                if create_mxd:
                    mxd_template = os.path.join(
                        os.path.dirname(os.path.dirname(__file__)), "supportfiles", "MapTemplate.mxd"
                    )
                    task_utils.create_mxd(out_workspace, mxd_template, "output")
                zip_file = task_utils.zip_data(out_workspace, "{0}.zip".format(output_file_name))
                shutil.move(zip_file, os.path.join(os.path.dirname(out_workspace), os.path.basename(zip_file)))
        except arcpy.ExecuteError as ee:
            status_writer.send_state(status.STAT_FAILED, _(ee))
            sys.exit(1)
    else:
        status_writer.send_state(status.STAT_FAILED, _("No output created. Zero inputs were clipped."))

    # Update state if necessary.
    if errors > 0 or skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _("{0} results could not be processed").format(errors + skipped))
    task_utils.report(
        os.path.join(request["folder"], "__report.json"), clipped, skipped, errors, errors_reasons, skipped_reasons
    )
def execute(request):
    """Builds raster pyramids for input raster datasets.
    :param request: json as a dict.
    """
    processed = 0
    skipped = 0
    parameters = request['params']

    # Get the extent for for which to use to calculate statistics.
    extent = ''
    try:
        try:
            ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'wkt')
            if ext:
                sr = task_utils.get_spatial_reference("4326")
                extent = task_utils.from_wkt(ext, sr)
        except KeyError:
            ext = task_utils.get_parameter_value(parameters, 'processing_extent', 'feature')
            if ext:
                extent = arcpy.Describe(ext).extent
    except KeyError:
        pass

    horizontal_skip_factor = task_utils.get_parameter_value(parameters, 'horizontal_skip_factor', 'value')
    vertical_skip_factor = task_utils.get_parameter_value(parameters, 'vertical_skip_factor', 'value')
    ignore_pixel_values = task_utils.get_parameter_value(parameters, 'ignore_pixel_values', 'value')

    # Create the task folder to hold report files.
    task_folder = request['folder']
    if not os.path.exists(task_folder):
        os.makedirs(task_folder)

    headers = {'x-access-token': task_utils.get_security_token(request['owner'])}
    num_results, response_index = task_utils.get_result_count(parameters)
    if num_results > task_utils.CHUNK_SIZE:
        # Query the index for results in groups of 25.
        query_index = task_utils.QueryIndex(parameters[response_index])
        fl = query_index.fl
        query = '{0}{1}{2}'.format(sys.argv[2].split('=')[1], '/select?&wt=json', fl)
        fq = query_index.get_fq()
        if fq:
            groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')
            query += fq
        elif 'ids' in parameters[response_index]:
            groups = task_utils.grouper(list(parameters[response_index]['ids']), task_utils.CHUNK_SIZE, '')
        else:
            groups = task_utils.grouper(range(0, num_results), task_utils.CHUNK_SIZE, '')

        # Begin processing
        status_writer.send_percent(0.0, _('Starting to process...'), 'calculate_raster_statistics')
        i = 0.
        for group in groups:
            i += len(group) - group.count('')
            if fq:
                results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)
            elif 'ids' in parameters[response_index]:
                results = requests.get(query + '{0}&ids={1}'.format(fl, ','.join(group)), headers=headers)
            else:
                results = requests.get(query + "&rows={0}&start={1}".format(task_utils.CHUNK_SIZE, group[0]), headers=headers)

            # input_items = task_utils.get_input_items(eval(results.read().replace('false', 'False').replace('true', 'True'))['response']['docs'])
            input_items = task_utils.get_input_items(results.json()['response']['docs'])
            if not input_items:
                input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])

            result = calculate_raster_statistics(input_items, extent, horizontal_skip_factor, vertical_skip_factor, ignore_pixel_values)
            processed += result[0]
            skipped += result[1]
            status_writer.send_percent(i / num_results, '{0}: {1:.0f}%'.format("Processed", i / num_results * 100), 'calculate_raster_statistics')
    else:
        input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
        processed, skipped = calculate_raster_statistics(input_items, extent, horizontal_skip_factor,
                                                         vertical_skip_factor, ignore_pixel_values, True)

    # Update state if necessary.
    if skipped > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped))
    task_utils.report(os.path.join(request['folder'], '__report.json'), processed, skipped, skipped_details=skipped_reasons)