def create_and_append(feature_list=None, token=None, portal_url=None, service_url=None, matching=None): token = arcpy.GetSigninToken() portal_url = arcpy.GetActivePortalURL() gis = GIS(portal_url, token=token['token']) layer = FeatureLayer(service_url) features_to_append = [] for feature in feature_list: new_feature = {'attributes': {}, 'geometry': feature['geometry']} #Find fields and it's value for field in matching: new_feature['attributes'][field[0]] = [ x[1] for x in feature['attributes'].items() if x[0] == field[1] ][0] features_to_append.append(new_feature.copy()) if len(features_to_append) > 500: result = layer.edit_features(adds=features_to_append) features_to_append = [] if features_to_append: layer.edit_features(adds=features_to_append)
def delete_rows(service_url, where_clause): deletes = [] lyr = FeatureLayer(service_url) query_result = lyr.query(where=where_clause, return_ids_only=True) deletes = query_result['objectIds'] if deletes: return lyr.edit_features(deletes=str(deletes)) else: return None
def replace_in_ago(**kwargs): from arcgis.features import FeatureLayer gis = GIS("https://detroitmi.maps.arcgis.com", Variable.get('AGO_USER'), Variable.get('AGO_PASS')) item = gis.content.get(kwargs['id']) layer_url = f"{item.url}/0" layer = FeatureLayer(layer_url) if 'conn_id' in kwargs.keys(): hook = PostgresHook(kwargs['conn_id']) else: hook = PostgresHook('etl_postgres') # Here's a query to make an ArcJSON item from each row in the table # Note: only works for point geometries right now. query = f""" SELECT jsonb_build_object( 'geometry', jsonb_build_object( 'x', ST_X(geom), 'y', ST_Y(geom) ), 'attributes', to_jsonb(row) - 'gid' - 'geom' ) FROM (SELECT * FROM {kwargs['table']}) row """ res = hook.get_records(query) payload = [r[0] for r in res] # clear out all the rows in the table layer.manager.truncate() # write all the rows in `res` chunk_size = 1000 print( f"Sending up {len(payload)} features with a batch size of {chunk_size}" ) for i in range(0, len(payload), chunk_size): try: layer.edit_features(adds=payload[i:i + chunk_size]) except: print(f"Errored on {i} - splitting into 2 batches") start = i middle = int(i + (chunk_size / 2)) end = i + chunk_size print(f"start: {start} middle: {middle} end: {end}") layer.edit_features(adds=payload[start:middle]) layer.edit_features(adds=payload[middle:end])
def update_database(current, last): """ Update the big database from the little one if the little one has new data for the big one. current = todays data last = the last row from the big database Returns True if the database was updated """ # If these all match then the latest data # has already been written to the database if ((last.new_cases == current.new_cases) \ and (last.total_cases == current.total_cases) and (last.last_update == current.date)): print("Database is current.") # If you want to test the update, comment this out # and then go in and manually delete the extra row(s). return False print("Appending the new record.") #return False attributes = { "utc_date": datetime.utcnow(), "last_update": current.date, "editor": os.environ.get('USERNAME') or os.environ.get('USER'), "source": "CC", "new_cases" : current.new_cases, "total_cases" : current.total_cases, "new_deaths" : current.new_deaths, "total_deaths" : current.total_deaths, "name": "Clatsop" } gis = GIS(Config.PORTAL_URL, Config.PORTAL_USER, Config.PORTAL_PASSWORD) layer = FeatureLayer(Config.COVID_CASES_URL, gis) f = Feature.from_dict({"attributes": attributes}) fs = FeatureSet(features=[f]) results = layer.edit_features(adds=fs) return True
def main(event, context): # Cityworks settings global baseUrl baseUrl = event["cityworks"]["url"] cwUser = event["cityworks"]["username"] cwPwd = event["cityworks"]["password"] cw_services.url = baseUrl # ArcGIS Online/Portal settings orgUrl = event["arcgis"]["url"] username = event["arcgis"]["username"] password = event["arcgis"]["password"] layers = event["arcgis"]["layers"] tables = event["arcgis"]["tables"] layerfields = event["fields"]["layers"] tablefields = event["fields"]["tables"] fc_flag = event["flag"]["field"] flag_values = [event["flag"]["on"], event["flag"]["off"]] ids = event["fields"]["ids"] probtypes = event["fields"]["type"] if log_to_file: from datetime import datetime as dt id_log = path.join(sys.path[0], "cityworks_log.log") log = open(id_log, "a") log.write("\n\n{}\n".format(dt.now())) try: # Connect to org/portal gis = GIS(orgUrl, username, password) # Get token for CW auth_response = cw_services.authenticate(cwUser, cwPwd) if auth_response['Status'] != 0: raise Exception("Cityworks not authenticated") # get wkid sr = get_wkid() if sr == "error": if log_to_file: log.write("Spatial reference not defined\n") else: print("Spatial reference not defined\n") raise Exception("Spatial reference not defined") # get problem types prob_types = get_problem_types() if prob_types == "error": if log_to_file: log.write("Problem types not defined\n") else: print("Problem types not defined\n") raise Exception("Problem types not defined") for layer in layers: lyr = FeatureLayer(layer, gis=gis) oid_fld = lyr.properties.objectIdField # Get related table URL reltable = "" for relate in lyr.properties.relationships: url_pieces = layer.split("/") url_pieces[-1] = str(relate["relatedTableId"]) table_url = "/".join(url_pieces) if table_url in tables: reltable = table_url break # query reports sql = "{}='{}'".format(fc_flag, flag_values[0]) rows = lyr.query(where=sql, out_sr=sr) updated_rows = [] for row in rows.features: oid = row.attributes[oid_fld] # Submit feature to the Cityworks database requestid = submit_to_cw(row, prob_types, layerfields, oid, probtypes) try: if "WARNING" in requestid: if log_to_file: log.write( "Warning generated while copying record to Cityworks: {}\n" .format(requestid)) else: print( "Warning generated while copying record to Cityworks: {}\n" .format(requestid)) continue else: pass # requestID is str = ok except TypeError: pass # requestID is a number = ok # attachments attachmentmgr = AttachmentManager(lyr) attachments = attachmentmgr.get_list(oid) for attachment in attachments: response = copy_attachment(attachmentmgr, attachment, oid, requestid) if response["Status"] is not 0: if log_to_file: log.write( "Error while copying attachment to Cityworks: {}\n" .format(response["ErrorMessages"])) else: print( "Error while copying attachment to Cityworks: {}\n" .format(response["ErrorMessages"])) # update the record in the service so that it evaluates falsely against sql sql = "{}='{}'".format(oid_fld, oid) row_orig = lyr.query(where=sql).features[0] row_orig.attributes[fc_flag] = flag_values[1] try: row_orig.attributes[ids[1]] = requestid except TypeError: row_orig.attributes[ids[1]] = str(requestid) updated_rows.append(row_orig) # apply edits to updated features if updated_rows: status = lyr.edit_features(updates=updated_rows) if log_to_file: log.write( "Status of updates to ArcGIS layers: {}\n".format( status)) else: print("Status of updates to ArcGIS layers: {}\n".format( status)) rel_records = [] updated_rows = [] # related records rellyr = FeatureLayer(reltable, gis=gis) relname = rellyr.properties['name'] pkey_fld = lyr.properties.relationships[0]["keyField"] fkey_fld = rellyr.properties.relationships[0]["keyField"] sql = "{}='{}'".format(fc_flag, flag_values[0]) rel_records = rellyr.query(where=sql) updated_rows = [] for record in rel_records: rel_oid = record.attributes[oid_fld] parent = get_parent(lyr, pkey_fld, record, fkey_fld) # Upload comment attachments try: attachmentmgr = rellyr.attachments attachments = attachmentmgr.get_list(rel_oid) for attachment in attachments: response = copy_attachment(attachmentmgr, attachment, rel_oid, parent.attributes[ids[1]]) if response["Status"] is not 0: try: error = response["ErrorMessages"] except KeyError: error = response["Message"] msg = "Error copying attachment. Record {} in table {}: {}".format( rel_oid, relname, error) if log_to_file: log.write(msg + '\n') else: print(msg) except RuntimeError: pass # table doesn't support attachments # Process comments response = copy_comments(record, parent, tablefields, ids) if 'error' in response: if log_to_file: log.write('Error accessing comment table {}\n'.format( relname)) else: print( 'Error accessing comment table {}'.format(relname)) break elif response["Status"] is not 0: try: error = response["ErrorMessages"] except KeyError: error = response["Message"] msg = "Error copying record {} from {}: {}".format( rel_oid, relname, error) if log_to_file: log.write(msg + '\n') else: print(msg) continue else: record.attributes[fc_flag] = flag_values[1] try: record.attributes[ids[1]] = parent.attributes[ids[1]] except TypeError: record.attributes[ids[1]] = str( parent.attributes[ids[1]]) updated_rows.append(record) # apply edits to updated records if updated_rows: status = rellyr.edit_features(updates=updated_rows) if log_to_file: log.write( "Status of updates to ArcGIS comments: {}\n".format( status)) else: print("Status of updates to ArcGIS comments: {}\n".format( status)) print("Finished processing: {}".format(lyr.properties["name"])) except Exception as ex: print("error: " + str(ex)) if log_to_file: log.close()
def main(configuration_file): try: with open(configuration_file) as configfile: cfg = json.load(configfile) gis = GIS(cfg['organization url'], cfg['username'], cfg['password']) # Get general id settings global id_settings id_settings = {} for option in cfg['id sequences']: id_settings[option['name']] = { 'interval': int(option['interval']), 'next value': int(option['next value']), 'pattern': option['pattern'] } # Get general moderation settings global modlists modlists = {} subs = cfg['moderation settings']['substitutions'] for modlist in cfg['moderation settings']['lists']: words = [ str(word).upper().strip() for word in modlist['words'].split(',') ] modlists[modlist['filter name']] = build_expression( words, modlist['filter type'], subs) # Get general email settings server = cfg['email settings']['smtp server'] username = cfg['email settings']['smtp username'] password = cfg['email settings']['smtp password'] tls = cfg['email settings']['use tls'] from_address = cfg['email settings']['from address'] if not from_address: from_address = '' reply_to = cfg['email settings']['reply to'] if not reply_to: reply_to = '' global substitutions substitutions = cfg['email settings']['substitutions'] # Process each service for service in cfg['services']: try: lyr = FeatureLayer(service['url'], gis=gis) # GENERATE IDENTIFIERS idseq = service['id sequence'] idfld = service['id field'] if id_settings and idseq and idfld: if idseq in id_settings: new_sequence_value = add_identifiers(lyr, idseq, idfld) id_settings[idseq]['next value'] = new_sequence_value else: _add_message( 'Sequence {} not found in sequence settings'. format(idseq), 'WARNING') # ENRICH REPORTS if service['enrichment']: # reversed, sorted list of enrichment settings enrich_settings = sorted( service['enrichment'], key=lambda k: k['priority']) #, reverse=True) for reflayer in enrich_settings: source_features = FeatureLayer(reflayer['url'], gis) enrich_layer(source_features, lyr, reflayer) # MODERATION if modlists: for query in service['moderation']: if query['list'] in modlists: moderate_features(lyr, query) else: _add_message( 'Moderation list {} not found in moderation settings' .format(modlist), 'WARNING') # SEND EMAILS if service['email']: with EmailServer(server, username, password, tls) as email_server: for message in service['email']: rows = _get_features(lyr, message['sql']) for row in rows: address, subject, body = build_email( row, lyr.properties.fields, message) if address and subject and body: try: email_server.send( from_address=from_address, reply_to=reply_to, to_addresses=[address], subject=subject, email_body=body) row.attributes[message[ 'field']] = message['sent value'] except: _add_message( 'email failed to send for feature {} in layer {}' .format(row.attributes, service['url'])) if rows: results = lyr.edit_features(updates=rows) _report_failures(results) except Exception as ex: _add_message('Failed to process service {}\n{}'.format( service['url'], ex)) except Exception as ex: _add_message( 'Failed. Please verify all configuration values\n{}'.format(ex)) finally: new_sequences = [{ 'name': seq, 'interval': id_settings[seq]['interval'], 'next value': id_settings[seq]['next value'], 'pattern': id_settings[seq]['pattern'] } for seq in id_settings] if not new_sequences == cfg['id sequences']: cfg['id sequences'] = new_sequences try: with open(configuration_file, 'w') as configfile: json.dump(cfg, configfile) except Exception as ex: _add_message( 'Failed to save identifier configuration values.\n{}\nOld values:{}\nNew values:{}' .format(ex, cfg['id sequences'], new_sequences))
def cases_entry_form(): global error portal_url = current_app.config['PORTAL_URL'] portal_user = current_app.config['PORTAL_USER'] portal_password = current_app.config['PORTAL_PASSWORD'] cases_url = current_app.config['COVID_CASES_URL'] form = CasesForm() if form.validate_on_submit(): # We've received input from a form, process it. #session['name'] = form.name.data try: local = parsetime(form.datestamp.data) utc = local2utc(local).strftime(time_format) except Exception as e: print("Time format is confusing to me.", e) error = e return redirect("/fail") try: n = { "attributes": { "utc_date": utc, "last_update": utc, 'name': 'Clatsop', "total_cases": s2i(form.total_cases.data), "new_cases": s2i(form.new_cases.data), "total_negative": s2i(form.negative.data), "total_tests": s2i(form.total_cases.data) + s2i(form.negative.data), "total_deaths": s2i(form.total_deaths.data), "new_deaths": s2i(form.new_deaths.data), "source": VERSION, "editor": "EMD", }, "geometry": county_centroid } except Exception as e: print("Attribute error.", e) error = e return redirect("/fail") # write back to server results = '' try: portal = GIS(portal_url, portal_user, portal_password) layer = FeatureLayer(cases_url) print(layer, n) results = layer.edit_features(adds=[n]) except Exception as e: error = e print("Write failed.", e, results) return redirect("/fail") if not update_cases(layer): return redirect("/fail") return redirect('/thanks/cases') # Create a data entry form try: portal = GIS(portal_url, portal_user, portal_password) df = FeatureLayer(cases_url).query( where="name='Clatsop' AND editor='EMD'", order_by_fields="utc_date DESC").sdf s = df.iloc[0] print(s) except Exception as e: print("Reading old data failed.", e) pass try: # Force the old date into UTC old_date = s['utc_date'].replace(tzinfo=timezone('UTC')) # Show the old date in the local TZ form.old_date = "(previous %s)" % old_date.astimezone( timezone('America/Los_Angeles')).strftime(time_format) except Exception as e: print("Converting old date stamp failed.", e) pass try: form.total_cases.data = s['total_cases'] form.new_cases.data = s['new_cases'] form.negative.data = s['total_negative'] #form.recovered.data = s['total_recovered'] form.total_deaths.data = s['total_deaths'] form.new_deaths.data = s['new_deaths'] #form.editor.data = s['editor'] except Exception as e: print("Filling in form failed.", e) pass now = datetime.now() ds = now.strftime(time_format) form.datestamp.data = ds return render_template('cases.html', form=form)
class NationalTimeSeries(object): def __init__(self, service_url: str, date_field: str = 'Date', region_field: str = 'Region', new_cases_field: str = 'NewCases', new_deaths_field: str = 'NewDeaths', new_recoveries_field: str = 'NewRecoveries', total_cases_field: str = 'TotalCases', total_deaths_field: str = 'TotalDeaths', total_recoveries_field: str = 'TotalRecoveries', active_cases_field: str = 'ActiveCases'): self.layer = FeatureLayer(service_url) self.region_field = region_field self.date_field = date_field self.new_cases_field = new_cases_field self.new_deaths_field = new_deaths_field self.new_recoveries_field = new_recoveries_field self.total_cases_field = total_cases_field self.total_deaths_field = total_deaths_field self.total_recoveries_field = total_recoveries_field self.active_cases_field = active_cases_field def _get_adds(self, target_features: FeatureSet, update_values: dict): # Make a copy so no items are removed from the source update_values item. update_copy = copy.deepcopy(update_values) # remove all items with matching key value. for target_feature in target_features: region = target_feature.attributes[self.region_field] date = datetime.datetime.fromtimestamp( target_feature.attributes[self.date_field] / 1000) key_val = region + date.strftime('%Y%m%d') update_copy.pop(key_val, None) # only new items remain. Create adds for new items. new_features = [] for new_item in update_copy.values(): # Create new item for service adds. Explicitly assigning the values to a new item prevents any possible errors # if the item has additional fields. item = { "attributes": { self.date_field: new_item['date'], self.region_field: new_item['region'], self.new_cases_field: new_item['NewCases'], self.total_cases_field: new_item['TotalCases'], self.new_deaths_field: new_item['NewDeaths'], self.total_deaths_field: new_item['TotalDeaths'], self.new_recoveries_field: new_item['NewRecovered'], self.total_recoveries_field: new_item['TotalRecovered'], self.active_cases_field: new_item['ActiveCases'] }, "geometry": state_coords.get(new_item['region'], { "x": 17100000, "y": -4600000 }) } new_features.append(item) return new_features def update(self, update_values): # get featureset from target service query_result = self.layer.query() target_features = query_result.features # get new values (adds) adds = self._get_adds(target_features=target_features, update_values=update_values) # update feature layer if adds: self.layer.edit_features(adds=adds) return len(adds)
classes = visual_recognition.classify(images_file, parameters=json.dumps({ 'classifier_ids': ['##########'], 'threshold': 0.876 })) #print(json.dumps(classes, indent=2)) data = json.loads(json.dumps(classes, indent=2)) if len(data['images'][0]['classifiers'][0]['classes']) != 0: print(json.dumps(classes, indent=2)) f = open( "./images/" + filename.replace("photo", "coor").replace("jpg", "txt"), "r") d = float(f.read()) * 0.000000301 gis = GIS(username="******", password="******") p = Point({"x": -73.471977 + d, "y": 40.703342}) a = {"pothole_layer": "pothole"} f = Feature(p, a) fs = FeatureSet([f]) lyr = FeatureLayer( "https://services8.arcgis.com/x660UqfqVJlbWB0Y/arcgis/rest/services/pothole_layers/FeatureServer/0", gis=gis) lyr.edit_features(adds=fs) #delete photo and txt
class FeatureServiceHelper(FeatureSourceHelper): def __init__(self, source, id_field): super().__init__(source=source, id_field=id_field) self.layer = FeatureLayer(source) def query(self, fields=None, where_clause=None): if fields is None: fields = '*' elif isinstance(fields, list): fields = ','.join(fields) if where_clause is None: where_clause = '1=1' return self.layer.query(out_fields=fields, where=where_clause) def load_records(self, fields=None, where_clause=None): return self.query(fields=fields, where_clause=where_clause).to_dict('records') @staticmethod def _field_types(query_result): result = {} for field in query_result.fields: result[field['name']] = field['type'] return result def field_names(self): return self.layer.properties.fields def update_records(self, new_data, fields=None, where_clause=None, add_new=False, delete_unmatched=False, rounding=4, case_sensitive=True, shape_field='Shape'): """ Updates the feature class using the new_data, with each record uniquely identified by the self.record_id_field. :param new_data: Dictionary records indexed by id. {record_id: {field_name1: value1, field_name2: value2,...}} :type new_data: dict :param fields: The list of fields to be updated. If an id_field is included, it will not be updated. :type fields: :param where_clause: An optional where clause to filter the updates. :type where_clause: :param add_new: If true, any records found in the new_data that do not have a corresponding record in the feature class will be deleted. :type add_new: bool :param delete_unmatched: If true, any feature class records found that do not have a corresponding record in the new_data will be deleted. :type delete_unmatched: bool :param rounding: decimal rounding to be used when comparing double format numbers. :type rounding: int :param case_sensitive: :type case_sensitive: :param shape_field: The name of the geometry field in the new_data. Set to None if geometry is not to be updated. Default='Shape' :type shape_field: str :return: :rtype: """ self._rounding = rounding self._case_sensitive = case_sensitive # use a local copy of new_data because we will be popping items. my_data = copy.deepcopy(new_data) if fields and self.id_field not in fields: fields.append(self.id_field) query_result = self.query(fields=fields, where_clause=where_clause) field_types = self._field_types(query_result) updates = [] deletes = [] for row in query_result: id_value = row.attributes[self.id_field] new_row = my_data.pop(id_value, None) if new_row: if self.update_row(row=row, field_types=field_types, new_values=new_row, shape_field=shape_field): updates.append(row) elif delete_unmatched: deletes.append(row.attributes[query_result.object_id_field_name]) adds = [] if add_new: # any remaining data_models items are new records for id_value, new_item in my_data.items(): new_geometry = new_item.pop(shape_field, None) row = self.generate_new_row(new_item, new_geometry) adds.append(row) return self.update_layer(adds=adds, deletes=None, updates=updates) def update_layer(self, adds=None, deletes=None, updates=None, chunk_size=1000): """ Performs updates on an arcgis feature service in manageable chunks. Updates are performed using multiple calls to the service where needed. Large sets of updates are broken into smaller calls to avoid timeouts and other data size related issues. Updates are executed in the following order: - Deletes - Adds - Updates If no elements are submitted for Adds, Deletes or Updates, then that stage of the process is skipped. :param adds: The list of add items to be added. :type adds: :param deletes: :type deletes: :param updates: :type updates: :param chunk_size: :type chunk_size: :return: :rtype: """ logging.info('Updating: ' + self.source) result = {'adds': 0, 'deletes': 0, 'updates': 0} # perform updates in order deletes, adds, updates to support models where: # - updates are performed by deleting old records and replacing with new (remove old items before adding new) # - items can be added and updated in same cycles (ensure adds are in place before updates are applied) if deletes: for chunk in self._list_chunks(master_list=deletes, chunk_size=chunk_size): logging.info('Applying {} Deletes'.format(len(chunk))) self.layer.edit_features(deletes=str(chunk)) result['deletes'] = len(deletes) logging.info('Total Deletes: {}'.format(result['deletes'])) if adds: for chunk in self._list_chunks(master_list=adds, chunk_size=chunk_size): logging.info('Applying {} Adds'.format(len(chunk))) self.layer.edit_features(adds=chunk) result['adds'] = len(adds) logging.info('Total Adds: {}'.format(result['adds'])) if updates: for chunk in self._list_chunks(master_list=updates, chunk_size=chunk_size): logging.info('Applying {} Updates'.format(len(chunk))) self.layer.edit_features(updates=chunk) result['updates'] = len(updates) logging.info('Total Updates: {}'.format(result['updates'])) return result @staticmethod def update_date_field(row: Feature, field_name: str, new_value: datetime.date): current_date_value = datetime_utils.to_datetime(row.attributes[field_name]) new_date_value = datetime_utils.to_datetime(new_value) if current_date_value == new_date_value: # if the values are the same, return False. return False # The values are different, Update the row. row.attributes[field_name] = new_date_value return True @staticmethod def update_int_field(row: Feature, field_name: str, new_value: int = None): if row.attributes[field_name] != new_value: row.attributes[field_name] = new_value return True return False def update_str_field(self, row: Feature, field_name: str, new_value: str = None): current_value = row.attributes[field_name] # if both are equal (str=str or None=None) return False. if current_value == new_value: return False # if (str and None) or (None and str) if not (current_value and new_value): row.attributes[field_name] = new_value return True # both values are non-identical strings. # if the test is not case sensitive and both UC strings match, no update needed if not self._case_sensitive and current_value.upper() == new_value.upper(): return False # the strings are non-equivalent. Update. row.attributes[field_name] = new_value return True def update_float_field(self, row: Feature, field_name: str, new_value: float = None): current_value = row.attributes[field_name] if current_value: current_value = round(current_value, self._rounding) # round non zero, non-null values. if new_value: test_value = round(new_value, self._rounding) # round non zero, non-null values. else: test_value = new_value if current_value == test_value: return False row.attributes[field_name] = new_value return True def update_field(self, row: Feature, field_name: str, field_type: str, new_value): ignore_types = ['esriFieldTypeOID', 'esriFieldTypeGeometry', 'esriFieldTypeBlob', 'esriFieldTypeRaster'] if field_type in ['esriFieldTypeSmallInteger', 'esriFieldTypeInteger']: return self.update_int_field(row, field_name=field_name, new_value=new_value) elif field_type in ['esriFieldTypeSingle', 'esriFieldTypeDouble']: return self.update_float_field(row=row, field_name=field_name, new_value=new_value) elif field_type == 'esriFieldTypeString': return self.update_str_field(row=row, field_name=field_name, new_value=new_value) elif field_type == 'esriFieldTypeDate': return self.update_date_field(row=row, field_name=field_name, new_value=new_value) elif field_type in ignore_types: return False raise ValueError('Unhandled field type: ' + field_type) def update_row(self, row: Feature, field_types, new_values, shape_field=None): update_required = False for field_name in row.fields: if field_name != self.id_field and field_name in new_values: field_type = field_types[field_name] new_value = new_values[field_name] if self.update_field(row=row, field_name=field_name, field_type=field_type, new_value=new_value): update_required = True #if shape_field: TODO implement feature updates in a way that handles minor locational variations (nm) # # new_wkt = self.to_wkt(new_values[shape_field]) # current_wkt = self.to_wkt(row.geometry) # if current_wkt != new_wkt: # row.geometry = self.to_geometry(new_values[shape_field]) # update_required = True return update_required @ staticmethod def to_wkt(source): if source is None: return None if isinstance(source, Geometry): return source.WKT if isinstance(source, arcpy.Geometry): return source.WKT geom = Geometry(source) return geom.WKT @staticmethod def to_geometry(source): if source is None: return None if isinstance(source, Geometry): return source if isinstance(source, arcpy.Geometry): return Geometry(source.JSON) return Geometry(source) @staticmethod def generate_new_row(new_values, geometry=None): attributes = {} for field_name, value in new_values.items(): attributes[field_name] = value geometry_value = geometry if geometry_value and isinstance(geometry_value, arcpy.Geometry): geometry_value = json.loads(geometry_value.JSON) return {"attributes": attributes, "geometry": geometry_value} @staticmethod def _list_chunks(master_list, chunk_size): """ Yield successive chunk-sized chunks from master_list. A utility function to support other methods in this module. """ for i in range(0, len(master_list), chunk_size): yield master_list[i:i + chunk_size]
for att in results: attachmentids[att['PARENTOBJECTID']].append(fcURL + "/{}/attachments/{}".format(att['PARENTOBJECTID'], att['ID'])) #Remove all attachments but the one in the middle of each house for key, values in attachmentids.items(): if values: midVal = values[int(len(values) / 2)] attachmentids[key] = midVal features = feature_layer.query(where=whereClause, return_geometry=False) featuresDict = [feature for feature in features if feature.get_value(flOID) in attachmentids] count = 0 arcpy.SetProgressor("step", "Analyzing Photos", 0, len(attachmentids) * len(predictProjects) ,1) for key, value in sorted(attachmentids.items()): #url = feature_layer.url + '/{0}/attachments/{1}'.format(key, value) feature = featuresDict[count] count += 1 for project in predictProjects: arcpy.SetProgressorLabel("Detecting Category '{}' in Feature {} of {}".format(project["name"],count,len(attachmentids))) results = predictor.classify_image_url_with_no_store(project["projectID"], project["currentIteration"].publish_name, url=value + tokenStr) for prediction in results.predictions: if prediction.tag_name in categoryList: feature.set_value(prediction.tag_name.lower() + "_prb", prediction.probability * 100) arcpy.SetProgressorPosition() feature_layer.edit_features(updates=[feature])
""" A quick script to clean up duplicate postcodes introduced during development of UpdateTotalCasesByPostcode.py. """ from arcgis.gis import GIS from arcgis.features import FeatureLayer target = None# r'https://services9.arcgis.com/7eQuDPPaB6g9029K/arcgis/rest/services/Total_Cases_By_Postcode/FeatureServer/0' gis = GIS(profile='agol_graphc') target_layer = FeatureLayer(target) query_result = target_layer.query() postcode_features = query_result.features known_postcodes = [] duplicates = [] for postcode_feature in postcode_features: postcode = postcode_feature.attributes['PostCode'] if postcode in known_postcodes: duplicates.append(postcode_feature.attributes['OBJECTID']) else: known_postcodes.append(postcode) result = target_layer.edit_features(deletes=str(duplicates)) print(result)
class TotalCovid19CasesByPostcode(object): """ A data access class to support the corresponding feature service layer. - Each postcode is represented by a single record. There should be no duplicate postcodes in this feature layer. """ def __init__(self, service_url=default_service_url): self.layer = FeatureLayer(service_url) self.postcode_field = 'PostCode' self.total_cases_field = 'TotalCases' self.date_of_last_case_field = 'DateOfLastCase' self.days_since_last_case_field = 'DaysSinceLastCase' def query(self, where_clause): return self.layer.query(where=where_clause) def update(self, updates_items=None): """ Updates the feature service: - recalculates the date_of_last_case values and updates as required. - if update_items are submitted, updates or adds the submitted rows. Rows are identified by postcode. Rows are only added if the postcode does not already exist in the table. Rows are never deleted by this process. In order to prevent accidental deletions they must be performed as a separate task. Existing XYs are not updated, submitted xys are used if rows are added. Use 'xy': None if location is not known for new records :param updates_items: A dictionary of named row values indexed by postcode. {postcode: { 'postcode': str, 'total_cases': int, 'date_of_last_case': datetime, 'xy': { 'x': float, 'y': float} } } :type updates_items: dict :return: :rtype: """ updates = [] if not updates_items: updates_items = {} # identify updates to be performed for existing rows. query_result = self.layer.query() for row in query_result: update_required = False poa = row.attributes[self.postcode_field] item = updates_items.pop(poa, None) if item: if row.attributes[ self.total_cases_field] != item['total_cases']: row.attributes[ self.total_cases_field] = item['total_cases'] update_required = True if row.attributes[self.date_of_last_case_field] != item[ 'date_of_last_case']: row.attributes[self.date_of_last_case_field] = item[ 'date_of_last_case'] update_required = True report_age = self.calc_days_since( row.attributes[self.date_of_last_case_field]) if report_age != row.attributes[self.days_since_last_case_field]: row.attributes[self.days_since_last_case_field] = report_age update_required = True if update_required: updates.append(row) # any remaining data_models items are new records adds = [] new_items = updates_items.values() if new_items: for new_item in new_items: days_since_last_case = self.calc_days_since( new_item['date_of_last_case']) row = { "attributes": { self.postcode_field: new_item['postcode'], self.total_cases_field: new_item['total_cases'], self.date_of_last_case_field: new_item['date_of_last_case'], self.days_since_last_case_field: days_since_last_case }, "geometry": new_item['xy'] } adds.append(row) logging.info('Updating: {}'.format(self.layer.url)) logging.info('ADDS: {}'.format(len(adds))) logging.info('UPDATES: {}'.format(len(updates))) if updates or adds: return self.layer.edit_features(updates=updates, adds=adds) else: return None @staticmethod def calc_days_since(reference_date): if reference_date: return (datetime.datetime.now() - reference_date).days else: return None
def main(event, context): import sys # Cityworks settings global baseUrl baseUrl = event["cityworks"]["url"] cwUser = event["cityworks"]["username"] cwPwd = event["cityworks"]["password"] timezone = event["cityworks"].get("timezone", "") isCWOL = event["cityworks"].get("isCWOL", False) # ArcGIS Online/Portal settings orgUrl = event["arcgis"]["url"] username = event["arcgis"]["username"] password = event["arcgis"]["password"] layers = event["arcgis"]["layers"] tables = event["arcgis"]["tables"] layerfields = event["fields"]["layers"] tablefields = event["fields"]["tables"] fc_flag = event["flag"]["field"] flag_values = [event["flag"]["on"], event["flag"]["off"]] ids = event["fields"]["ids"] probtypes = event["fields"]["type"] opendate = event["fields"].get("opendate", "") if log_to_file: from datetime import datetime as dt id_log = path.join(sys.path[0], "cityworks_log.log") log = open(id_log, "a") log.write("\n{} ".format(dt.now())) log.write("Sending reports to: {}\n".format(baseUrl)) else: print("Sending reports to: {}".format(baseUrl)) try: # Connect to org/portal gis = GIS(orgUrl, username, password) # Get token for CW status = get_cw_token(cwUser, cwPwd, isCWOL) if "error" in status: if log_to_file: log.write("Failed to get Cityworks token. {}\n".format(status)) else: print("Failed to get Cityworks token. {}".format(status)) raise Exception( "Failed to get Cityworks token. {}".format(status)) # get wkid sr = get_wkid() if sr == "error": if log_to_file: log.write("Spatial reference not defined\n") else: print("Spatial reference not defined") raise Exception("Spatial reference not defined") # get problem types prob_types = get_problem_types() if prob_types == "error": if log_to_file: log.write("Problem types not defined\n") else: print("Problem types not defined") raise Exception("Problem types not defined") for layer in layers: lyr = FeatureLayer(layer, gis=gis) oid_fld = lyr.properties.objectIdField lyrname = lyr.properties["name"] # Get related table URL reltable = "" try: for relate in lyr.properties.relationships: url_pieces = layer.split("/") url_pieces[-1] = str(relate["relatedTableId"]) table_url = "/".join(url_pieces) if table_url in tables: reltable = table_url break # if related tables aren't being used except AttributeError: pass # query reports sql = "{}='{}'".format(fc_flag, flag_values[0]) rows = lyr.query(where=sql, out_sr=sr) updated_rows = [] for row in rows.features: try: oid = row.attributes[oid_fld] # Submit feature to the Cityworks database request = submit_to_cw(row, prob_types, layerfields, oid, probtypes) try: reqid = request["RequestId"] initDate = int( parse(request[opendate[0]]).replace( tzinfo=gettz(timezone)).timestamp() * 1000) if opendate else "" except TypeError: if "WARNING" in request: msg = "Warning generated while copying ObjectID:{} from layer {} to Cityworks: {}".format( oid, lyrname, request) if log_to_file: log.write(msg + '\n') else: print(msg) continue elif 'error' in request: msg = "Error generated while copying ObjectID:{} from layer {} to Cityworks: {}".format( oid, lyrname, request) if log_to_file: log.write(msg + '\n') else: print(msg) continue else: msg = "Uncaught response generated while copying ObjectID:{} from layer {} to Cityworks: {}".format( oid, lyrname, request) if log_to_file: log.write(msg + '\n') else: print(msg) continue # update the record in the service so that it evaluates falsely against sql sql = "{}='{}'".format(oid_fld, oid) row_orig = lyr.query(where=sql).features[0] row_orig.attributes[fc_flag] = flag_values[1] if opendate: row_orig.attributes[opendate[1]] = initDate try: row_orig.attributes[ids[1]] = reqid except TypeError: row_orig.attributes[ids[1]] = str(reqid) # apply edits to updated row status = lyr.edit_features(updates=[row_orig]) if log_to_file: log.write( "Status of updates to {}, ObjectID:{} {}\n".format( lyr.properties["name"], oid, status)) else: print("Status of updates to {}, ObjectID:{} {}".format( lyr.properties["name"], oid, status)) # attachments try: attachmentmgr = lyr.attachments attachments = attachmentmgr.get_list(oid) for attachment in attachments: response = copy_attachment(attachmentmgr, attachment, oid, reqid) if response["Status"] is not 0: try: error = response["ErrorMessages"] except KeyError: error = response["Message"] msg = "Error copying attachment from feature {} in layer {}: {}".format( oid, lyrname, error) if log_to_file: log.write(msg + '\n') else: print(msg) except RuntimeError: pass # feature layer doesn't support attachments # any other error in row execution, move on to next row except Exception as e: if log_to_file: log.write(str(e) + '\n') else: print(str(e)) continue # end of row execution # end of features execution # related records rel_records = [] #if comments tables aren't used, script will crash here try: if len(lyr.properties.relationships) > 0: # related records rellyr = FeatureLayer(reltable, gis=gis) relname = rellyr.properties["name"] pkey_fld = lyr.properties.relationships[0]["keyField"] fkey_fld = rellyr.properties.relationships[0]["keyField"] sql = "{}='{}'".format(fc_flag, flag_values[0]) rel_records = rellyr.query(where=sql) # if related tables aren't being used except AttributeError: pass except KeyError: relname = "Comments" updated_rows = [] for record in rel_records: try: rel_oid = record.attributes[oid_fld] parent = get_parent(lyr, pkey_fld, record, fkey_fld) # Process comments response = copy_comments(record, parent, tablefields, ids) if 'error' in response: if log_to_file: log.write( 'Error accessing comment table {}\n'.format( relname)) else: print('Error accessing comment table {}'.format( relname)) break elif response["Status"] is not 0: try: error = response["ErrorMessages"] except KeyError: error = response["Message"] msg = "Error copying record {} from {}: {}".format( rel_oid, relname, error) if log_to_file: log.write(msg + '\n') else: print(msg) continue else: record.attributes[fc_flag] = flag_values[1] try: record.attributes[ids[1]] = parent.attributes[ ids[1]] except TypeError: record.attributes[ids[1]] = str( parent.attributes[ids[1]]) # apply edits to updated record status = rellyr.edit_features(updates=[record]) if log_to_file: log.write( "Status of updates to {}, ObjectID:{} comments: {}\n" .format(relname, rel_oid, status)) else: print( "Status of updates to {}, ObjectID:{} comments: {}" .format(relname, rel_oid, status)) # Upload comment attachments try: attachmentmgr = rellyr.attachments attachments = attachmentmgr.get_list(rel_oid) for attachment in attachments: response = copy_attachment( attachmentmgr, attachment, rel_oid, parent.attributes[ids[1]]) if response["Status"] is not 0: try: error = response["ErrorMessages"] except KeyError: error = response["Message"] msg = "Error copying attachment. Record {} in table {}: {}".format( rel_oid, relname, error) if log_to_file: log.write(msg + '\n') else: print(msg) except RuntimeError: pass # table doesn't support attachments # any other uncaught Exception in related record export, move on to next row except Exception as e: if log_to_file: log.write(str(e) + '\n') else: print(str(e)) continue print("Finished processing: {}".format(lyrname)) except BaseException as ex: exc_tb = sys.exc_info()[2] exc_typ = sys.exc_info()[0] print('error: {} {}, Line {}'.format(exc_typ, str(ex), exc_tb.tb_lineno)) if log_to_file: log.write('error: {} {}, Line {}'.format(exc_typ, str(ex), exc_tb.tb_lineno)) except: exc_tb = sys.exc_info()[2] exc_typ = sys.exc_info()[0] print('error: {}, Line {}'.format(exc_typ, exc_tb.tb_lineno)) if log_to_file: log.write('error: {}, Line {}'.format(exc_typ, exc_tb.tb_lineno)) finally: if log_to_file: log.close()
class DailyCovid19TestingByPostcode(object): def __init__(self, service_url=default_service_url): self.layer = FeatureLayer(service_url) self.postcode_field = 'Postcode' self.date_field = 'Date' self.date_code_field = 'DateCode' self.tests_field = 'Tests' self.total_tests_field = 'TotalTests' self.date_code_format = '%Y%m%d' def query(self, where_clause): return self.layer.query(where=where_clause) def update(self, rows, allow_deletes=False): """ updates the feature service by updating or adding the submitted rows. Rows are identified by date_code and postcode. Date values are derived from the date_code values. Existing XYs are not updated, submitted xys are used if rows are added. :param rows: [{'postcode': str, 'date_code': str, 'tests': int, 'total_tests': int, 'xy': {'x': float, 'y': float}}] :type rows: [dict] :param allow_deletes: If set to True any records not found in the source rows will be deleted. Default=False :type allow_deletes: bool :return: :rtype: """ # build lookup from submitted rows: items = {} for row in rows: key = '{}_{}'.format(row['postcode'], row['date_code']) items[key] = row deletes = [] updates = [] query_result = self.layer.query() for row in query_result: poa = row.attributes[self.postcode_field] date_code = row.attributes[self.date_code_field] key = '{}_{}'.format(poa, date_code) if allow_deletes and key not in items: deletes.append( row.attributes[query_result.object_id_field_name]) item = items.pop(key, None) if item: update_required = False if row.attributes[self.tests_field] != item['tests']: row.attributes[self.tests_field] = item['tests'] update_required = True if row.attributes[ self.total_tests_field] != item['total_tests']: row.attributes[ self.total_tests_field] = item['total_tests'] update_required = True if update_required: updates.append(row) # any remaining data_models items are new records adds = [] new_items = items.values() if new_items: for new_item in new_items: date = datetime.datetime.strptime(new_item['date_code'], '%Y%m%d') row = { "attributes": { self.date_field: date, self.postcode_field: new_item['postcode'], self.date_code_field: new_item['date_code'], self.total_tests_field: new_item['total_tests'], self.tests_field: new_item['tests'] }, "geometry": new_item['xy'] } adds.append(row) logging.info('Updating: {}'.format(self.layer.url)) logging.info('ADDS: {}'.format(len(adds))) logging.info('DELETES: {}'.format(len(deletes))) logging.info('UPDATES: {}'.format(len(updates))) if updates or adds or deletes: return self.layer.edit_features(updates=updates, adds=adds, deletes=str(deletes)) else: return None def format_date_string(self, date_string, in_format): """ Reformats the submitted date or datetime string to the correct format for the date_code field :type date_string: str :param in_format: the format string of the submitted date_string. eg: '%Y-%m-%d' :type in_format: str :return: The reformatted date_string :rtype: str """ if in_format == self.date_code_format: return date_string dt = datetime.datetime.strptime(date_string, in_format) return dt.strftime(self.date_code_format)
def main(): # Create log file with open(path.join(sys.path[0], 'attr_log.log'), 'a') as log: log.write('\n{}\n'.format(dt.now())) # connect to org/portal if username: gis = GIS(orgURL, username, password) else: gis = GIS(orgURL) for service in services: try: # Connect to source and target layers fl_source = FeatureLayer(service['source url'], gis) fl_target = FeatureLayer(service['target url'], gis) # get field map fields = [[key, service['fields'][key]] for key in service['fields'].keys()] # Get source rows to copy rows = fl_source.query(service['query']) adds = [] updates = [] for row in rows: # Build dictionary of attributes & geometry in schema of target layer # Default status and priority values can be overwritten if those fields are mapped to reporter layer attributes = {'status': 0, 'priority': 0} for field in fields: attributes[field[1]] = row.attributes[field[0]] new_request = {'attributes': attributes, 'geometry': {'x': row.geometry['x'], 'y': row.geometry['y']}} adds.append(new_request) # update row to indicate record has been copied if service['update field']: row.attributes[service['update field']] = service['update value'] updates.append(row) # add records to target layer if adds: add_result = fl_target.edit_features(adds=adds) for result in add_result['updateResults']: if not result['success']: raise Exception('error {}: {}'.format(result['error']['code'], result['error']['description'])) # update records: if updates: update_result = fl_source.edit_features(updates=updates) for result in update_result['updateResults']: if not result['success']: raise Exception('error {}: {}'.format(result['error']['code'], result['error']['description'])) except Exception as ex: msg = 'Failed to copy feature from layer {}'.format(service['url']) print(ex) print(msg) log.write('{}\n{}\n'.format(msg, ex))
class TotalCasesByPostcodeFeatureLayer(object): def __init__(self, service_url: str, postcode_field: str = 'PostCode', total_cases_field: str = 'TotalCases', date_of_last_case_field: str = 'DateOfLastCase', days_since_last_case_field: str = 'DaysSinceLastCase'): self.layer = FeatureLayer(service_url) self.postcode_field = postcode_field self.totalCases_field = total_cases_field self.dateOfLastCase_field = date_of_last_case_field self.daysSinceLastCase_field = days_since_last_case_field def _get_updates(self, postcode_features: FeatureSet, update_values: dict): """ :param postcode_features: The postcodes featureset to be updated :type postcode_features: FeatureSet :param update_values: The current count values to be applied. :type update_values: :return: :rtype: """ updates = [] no_change = 0 for postcode_feature in postcode_features: postcode = postcode_feature.attributes[self.postcode_field] updated_counts = update_values.pop(postcode, None) if updated_counts: total_cases = updated_counts['Total Cases'] date_last = updated_counts['DateOfLastCase'] days_since = updated_counts['DaysSinceLastCase'] else: total_cases = None date_last = None days_since = None current_cases = postcode_feature.attributes[self.totalCases_field] current_days_since = postcode_feature.attributes[ self.daysSinceLastCase_field] if current_cases != total_cases or days_since != current_days_since: postcode_feature.attributes[ self.totalCases_field] = total_cases postcode_feature.attributes[ self.dateOfLastCase_field] = date_last postcode_feature.attributes[ self.daysSinceLastCase_field] = days_since updates.append(postcode_feature) else: no_change += 1 return updates, no_change def _get_adds(self, update_values): new_postcodes = [] for new_item in update_values: # Create new item for service adds. Explicitly assigning the values to a new item prevents any possible errors # if the item has additional fields. item = { "attributes": { self.postcode_field: new_item['Postcode'], self.totalCases_field: new_item['Total Cases'], self.dateOfLastCase_field: new_item['DateOfLastCase'], self.daysSinceLastCase_field: new_item['DaysSinceLastCase'] }, "geometry": { "x": 17100000, "y": -4600000 } } new_postcodes.append(item) return new_postcodes def update_from_case_list(self, case_list: CaseList): """ Updates the layer using the values in the case_list parameter. If a postcode in the service is not found in the case_list then the value for that postcode is set to None. Postcodes are never deleted from the service. If postcodes are found in the case_list that are not in the table, then those postcodes are added to the service. This function does not alter the case_list object. :param case_list: The case list to be used to update the feature layer. :type case_list: CaseList :return: :rtype: UpdateResultCounts """ # make a local copy of postcode_counts so we don't inadvertently change the source dictionary for other uses. update_values = case_list.counts_by_postcode() # get featureset from target service query_result = self.layer.query() postcode_features = query_result.features updates, no_change = self._get_updates(postcode_features, update_values) adds = self._get_adds(update_values) if updates or adds: self.layer.edit_features(updates=updates, adds=adds) return UpdateResultCounts(adds=len(adds), updates=len(updates), unchanged=no_change) def get_geometry_lookups(self, where_clause: str = None): """ Gets a dictionary of {key: geometry pairs} :param where_clause: :type where_clause: :return: :rtype: """ result = {} if where_clause: query_result = self.layer.query(out_fields=[self.postcode_field], where=where_clause) else: query_result = self.layer.query(out_fields=[self.postcode_field]) postcode_features = query_result.features for postcode_feature in postcode_features: postcode = postcode_feature.attributes[self.postcode_field] geometry = postcode_feature.geometry result[postcode] = geometry return result
from arcgis.gis import GIS from arcgis.features import FeatureLayer lyr_url = r'https://services9.arcgis.com/7eQuDPPaB6g9029K/arcgis/rest/services/COVID_19_Alerts_Editable/FeatureServer/0' str_field = 'Dates' date_field = 'DateValue' profile = 'agol_graphc' where_clause = '1=1' gis = GIS(profile='agol_graphc') layer = FeatureLayer(url=lyr_url) fields = [str_field, date_field] test_dt = datetime.datetime.now() query_result = layer.query(out_fields=",".join(fields), where=where_clause) updates = [] for row in query_result: date_str = row.attributes[str_field] date_val = parser.parse(date_str).replace(hour=12) if date_val > test_dt: date_val = date_val + relativedelta(years=-1) date_ms = date_val.timestamp() * 1000 if row.attributes[date_field] != date_ms: row.attributes[date_field] = date_ms updates.append(row) if updates: pprint(layer.edit_features(updates=updates)) else: print('No Updates Required')
class IndividualCasesByPostcodeFeatureLayer(object): def __init__(self, service_url: str, date_string_field='DateString', postcode_field='PostCode', report_date_field='ReportDate', report_age_field='ReportAge', oid_field='OBJECTID'): self.layer = FeatureLayer(service_url) self.oid_field = oid_field self.dateString_field = date_string_field self.postcode_field = postcode_field self.reportAge_field = report_age_field self.reportDate_field = report_date_field @staticmethod def _get_source_values(case_list, postcode_centroids, default_xy=(17100000, -4600000)): """ Generates the update values by combining counts by date and postcode from the case list and postcode centroids from the geometry source. :param case_list: The CaseList containing the cases to be used to update the FeatureLayer :type case_list: CaseList :param postcode_centroids: A lookup of postcode centroid geometries :type postcode_centroids: dict :param default_xy: Optional - The xy tuple to be used if no matching postcode geometry is found. Default is (17100000, -4600000), a WGS84-WMAS coordinate that falls of the NSW/VIC coast. :type default_xy: tuple (float, float) :return: {yyyymmddpppp: {'date_string': str, 'date': datetime, 'postcode': int, 'report_age': int, 'count': int, 'xy': (float, float)} :rtype: dict """ result = case_list.counts_by_date_and_postcode() # add the coordinate values to each item. for item in result.values(): postcode_centroid = postcode_centroids.get(item['postcode'], None) if postcode_centroid: item['xy'] = postcode_centroid else: item['xy'] = {'x': default_xy[0], 'y': default_xy[1]} return result def update(self, case_list, postcode_centroids, default_xy=(17100000, -4600000)): source_values = self._get_source_values(case_list, postcode_centroids, default_xy) # get featureset from target service query_result = self.layer.query() postcode_features = query_result.features adds = [] deletes = [] updates = [] unchanged = 0 for postcode_feature in postcode_features: postcode = postcode_feature.attributes[self.postcode_field] datestring = postcode_feature.attributes[self.dateString_field] key_value = datestring + postcode item = source_values.get(key_value, None) if item: if item['count'] == 1: """ Remove the item from source if all source items have been matched. Additional items in the feature layer with the same key will be deleted.""" source_values.pop(key_value) item['count'] -= 1 if item['report_age'] != postcode_feature.attributes[ self.reportAge_field]: """If the report age values are not equal, updates are required. We do not need to compare postcode or date values because they are defined at record creation and do not change. A source record where the postcode or date changes will result in a deletion of the current record and a creation of a new record with the new date/postcode key.""" postcode_feature.attributes[ self.reportAge_field] = item['report_age'] updates.append(postcode_feature) else: unchanged += 1 else: # if no match found in source, add the OID to the deletes deletes.append(postcode_feature.attributes[self.oid_field]) # any items left in the source_values represent new records. Append them to the Adds list. for source_value in source_values.values(): item = { "attributes": { self.postcode_field: source_value['postcode'], self.dateString_field: source_value['date_string'], self.reportDate_field: source_value['date'], self.reportAge_field: source_value['report_age'] }, "geometry": source_value['xy'] } for i in range(source_value['count']): adds.append(item) if updates or adds or deletes: self.layer.edit_features(updates=updates, adds=adds, deletes=str(deletes)) return UpdateResultCounts(adds=len(adds), deletes=len(deletes), updates=len(updates), unchanged=unchanged)
import random gis = GIS("https://www.arcgis.com", "devlaahernandezgo", "fQ2HFMfk5ya7E9J") url = 'https://services5.arcgis.com/otu0qUsUpyUjfoF3/arcgis/rest/services/informacion_lugar/FeatureServer/0' layer = FeatureLayer(url) for x in range(0, 9): body = { "geometry": { "objectId": 2, "x": -74.07, "y": 4.7 - random.random(), "spatialReference": { "wkid": 4326 } }, "attributes": { "edificio": "Aqui porfin", "nombre": "prueba", "numero_interno": "001", "maxima_ocupacion": "18", "tasa": random.randrange(0, 18), "tiempo_promedio": "30", "indice_bioseguro": "70" } } layer.edit_features(adds=[body]) print(layer.query().sdf)
class NationalTimeSeries(object): def __init__(self, service_url: str, date_field: str = 'Date', region_field: str = 'Region', new_cases_field: str = 'NewCases', new_deaths_field: str = 'NewDeaths', new_recoveries_field: str = 'NewRecoveries', total_cases_field: str = 'TotalCases', total_deaths_field: str = 'TotalDeaths', total_recoveries_field: str = 'TotalRecoveries', active_cases_field: str = 'ActiveCases'): self.layer = FeatureLayer(service_url) self.region_field = region_field self.date_field = date_field self.new_cases_field = new_cases_field self.new_deaths_field = new_deaths_field self.new_recoveries_field = new_recoveries_field self.total_cases_field = total_cases_field self.total_deaths_field = total_deaths_field self.total_recoveries_field = total_recoveries_field self.active_cases_field = active_cases_field @staticmethod def _update_attribute(target_feature, attribute_name, new_value): if target_feature.attributes[attribute_name] == new_value: return False else: target_feature.attributes[attribute_name] = new_value return True def _get_updates(self, target_features: FeatureSet, update_values: dict): """ :param target_features: The featureset to be updated :type target_features: FeatureSet :param update_values: The current count values to be applied. :type update_values: :return: :rtype: """ updates = [] no_change = 0 for target_feature in target_features: region = target_feature.attributes[self.region_field] date = datetime.datetime.fromtimestamp(target_feature.attributes[self.date_field]/1000) key_val = region + date.strftime('%Y%m%d') updated_counts = update_values.pop(key_val, None) if updated_counts: if (self._update_attribute(target_feature, self.total_cases_field, updated_counts['TotalCases']) or self._update_attribute(target_feature, self.new_cases_field, updated_counts['NewCases']) or self._update_attribute(target_feature, self.total_deaths_field, updated_counts['TotalDeaths']) or self._update_attribute(target_feature, self.new_deaths_field, updated_counts['NewDeaths']) or self._update_attribute(target_feature, self.total_recoveries_field, updated_counts['TotalRecovered']) or self._update_attribute(target_feature, self.new_recoveries_field, updated_counts['NewRecovered']) or self._update_attribute(target_feature, self.active_cases_field, updated_counts['ActiveCases']) ): updates.append(target_feature) else: no_change += 1 return updates, no_change def _get_adds(self, data): new_features = [] for new_item in data.values(): # Create new item for service adds. Explicitly assigning the values to a new item prevents any possible errors # if the item has additional fields. item = {"attributes": {self.date_field: new_item['date'], self.region_field: new_item['region'], self.new_cases_field: new_item['NewCases'], self.total_cases_field: new_item['TotalCases'], self.new_deaths_field: new_item['NewDeaths'], self.total_deaths_field: new_item['TotalDeaths'], self.new_recoveries_field: new_item['NewRecovered'], self.total_recoveries_field: new_item['TotalRecovered'], self.active_cases_field: new_item['ActiveCases']}, "geometry": state_coords.get(new_item['region'], {"x": 17100000, "y": -4600000})} new_features.append(item) return new_features def update(self, update_values): # get featureset from target service query_result = self.layer.query() target_features = query_result.features updates, no_change = self._get_updates(target_features, update_values) adds = self._get_adds(update_values) if updates or adds: self.layer.edit_features(updates=updates, adds=adds) return UpdateResultCounts(adds=len(adds), updates=len(updates), unchanged=no_change)