def create_and_append(feature_list=None,
token=None,
portal_url=None,
service_url=None,
matching=None):
token = arcpy.GetSigninToken()
portal_url = arcpy.GetActivePortalURL()
gis = GIS(portal_url, token=token['token'])
layer = FeatureLayer(service_url)
features_to_append = []
for feature in feature_list:
new_feature = {'attributes': {}, 'geometry': feature['geometry']}
#Find fields and it's value
for field in matching:
new_feature['attributes'][field[0]] = [
x[1] for x in feature['attributes'].items() if x[0] == field[1]
][0]
features_to_append.append(new_feature.copy())
if len(features_to_append) > 500:
result = layer.edit_features(adds=features_to_append)
features_to_append = []
if features_to_append:
layer.edit_features(adds=features_to_append)
def delete_rows(service_url, where_clause):
deletes = []
lyr = FeatureLayer(service_url)
query_result = lyr.query(where=where_clause, return_ids_only=True)
deletes = query_result['objectIds']
if deletes:
return lyr.edit_features(deletes=str(deletes))
else:
return None
def replace_in_ago(**kwargs):
from arcgis.features import FeatureLayer
gis = GIS("https://detroitmi.maps.arcgis.com", Variable.get('AGO_USER'),
Variable.get('AGO_PASS'))
item = gis.content.get(kwargs['id'])
layer_url = f"{item.url}/0"
layer = FeatureLayer(layer_url)
if 'conn_id' in kwargs.keys():
hook = PostgresHook(kwargs['conn_id'])
else:
hook = PostgresHook('etl_postgres')
# Here's a query to make an ArcJSON item from each row in the table
# Note: only works for point geometries right now.
query = f"""
SELECT jsonb_build_object(
'geometry', jsonb_build_object(
'x', ST_X(geom),
'y', ST_Y(geom)
),
'attributes', to_jsonb(row) - 'gid' - 'geom'
) FROM (SELECT * FROM {kwargs['table']}) row
"""
res = hook.get_records(query)
payload = [r[0] for r in res]
# clear out all the rows in the table
layer.manager.truncate()
# write all the rows in `res`
chunk_size = 1000
print(
f"Sending up {len(payload)} features with a batch size of {chunk_size}"
)
for i in range(0, len(payload), chunk_size):
try:
layer.edit_features(adds=payload[i:i + chunk_size])
except:
print(f"Errored on {i} - splitting into 2 batches")
start = i
middle = int(i + (chunk_size / 2))
end = i + chunk_size
print(f"start: {start} middle: {middle} end: {end}")
layer.edit_features(adds=payload[start:middle])
layer.edit_features(adds=payload[middle:end])
def update_database(current, last):
"""
Update the big database from the little one if the little one has new data for the big one.
current = todays data
last = the last row from the big database
Returns True if the database was updated
"""
# If these all match then the latest data
# has already been written to the database
if ((last.new_cases == current.new_cases) \
and (last.total_cases == current.total_cases)
and (last.last_update == current.date)):
print("Database is current.")
# If you want to test the update, comment this out
# and then go in and manually delete the extra row(s).
return False
print("Appending the new record.")
#return False
attributes = {
"utc_date": datetime.utcnow(),
"last_update": current.date,
"editor": os.environ.get('USERNAME') or os.environ.get('USER'),
"source": "CC",
"new_cases" : current.new_cases,
"total_cases" : current.total_cases,
"new_deaths" : current.new_deaths,
"total_deaths" : current.total_deaths,
"name": "Clatsop"
}
gis = GIS(Config.PORTAL_URL, Config.PORTAL_USER, Config.PORTAL_PASSWORD)
layer = FeatureLayer(Config.COVID_CASES_URL, gis)
f = Feature.from_dict({"attributes": attributes})
fs = FeatureSet(features=[f])
results = layer.edit_features(adds=fs)
return True
def main(event, context):
# Cityworks settings
global baseUrl
baseUrl = event["cityworks"]["url"]
cwUser = event["cityworks"]["username"]
cwPwd = event["cityworks"]["password"]
cw_services.url = baseUrl
# ArcGIS Online/Portal settings
orgUrl = event["arcgis"]["url"]
username = event["arcgis"]["username"]
password = event["arcgis"]["password"]
layers = event["arcgis"]["layers"]
tables = event["arcgis"]["tables"]
layerfields = event["fields"]["layers"]
tablefields = event["fields"]["tables"]
fc_flag = event["flag"]["field"]
flag_values = [event["flag"]["on"], event["flag"]["off"]]
ids = event["fields"]["ids"]
probtypes = event["fields"]["type"]
if log_to_file:
from datetime import datetime as dt
id_log = path.join(sys.path[0], "cityworks_log.log")
log = open(id_log, "a")
log.write("\n\n{}\n".format(dt.now()))
try:
# Connect to org/portal
gis = GIS(orgUrl, username, password)
# Get token for CW
auth_response = cw_services.authenticate(cwUser, cwPwd)
if auth_response['Status'] != 0:
raise Exception("Cityworks not authenticated")
# get wkid
sr = get_wkid()
if sr == "error":
if log_to_file:
log.write("Spatial reference not defined\n")
else:
print("Spatial reference not defined\n")
raise Exception("Spatial reference not defined")
# get problem types
prob_types = get_problem_types()
if prob_types == "error":
if log_to_file:
log.write("Problem types not defined\n")
else:
print("Problem types not defined\n")
raise Exception("Problem types not defined")
for layer in layers:
lyr = FeatureLayer(layer, gis=gis)
oid_fld = lyr.properties.objectIdField
# Get related table URL
reltable = ""
for relate in lyr.properties.relationships:
url_pieces = layer.split("/")
url_pieces[-1] = str(relate["relatedTableId"])
table_url = "/".join(url_pieces)
if table_url in tables:
reltable = table_url
break
# query reports
sql = "{}='{}'".format(fc_flag, flag_values[0])
rows = lyr.query(where=sql, out_sr=sr)
updated_rows = []
for row in rows.features:
oid = row.attributes[oid_fld]
# Submit feature to the Cityworks database
requestid = submit_to_cw(row, prob_types, layerfields, oid,
probtypes)
try:
if "WARNING" in requestid:
if log_to_file:
log.write(
"Warning generated while copying record to Cityworks: {}\n"
.format(requestid))
else:
print(
"Warning generated while copying record to Cityworks: {}\n"
.format(requestid))
continue
else:
pass # requestID is str = ok
except TypeError:
pass # requestID is a number = ok
# attachments
attachmentmgr = AttachmentManager(lyr)
attachments = attachmentmgr.get_list(oid)
for attachment in attachments:
response = copy_attachment(attachmentmgr, attachment, oid,
requestid)
if response["Status"] is not 0:
if log_to_file:
log.write(
"Error while copying attachment to Cityworks: {}\n"
.format(response["ErrorMessages"]))
else:
print(
"Error while copying attachment to Cityworks: {}\n"
.format(response["ErrorMessages"]))
# update the record in the service so that it evaluates falsely against sql
sql = "{}='{}'".format(oid_fld, oid)
row_orig = lyr.query(where=sql).features[0]
row_orig.attributes[fc_flag] = flag_values[1]
try:
row_orig.attributes[ids[1]] = requestid
except TypeError:
row_orig.attributes[ids[1]] = str(requestid)
updated_rows.append(row_orig)
# apply edits to updated features
if updated_rows:
status = lyr.edit_features(updates=updated_rows)
if log_to_file:
log.write(
"Status of updates to ArcGIS layers: {}\n".format(
status))
else:
print("Status of updates to ArcGIS layers: {}\n".format(
status))
rel_records = []
updated_rows = []
# related records
rellyr = FeatureLayer(reltable, gis=gis)
relname = rellyr.properties['name']
pkey_fld = lyr.properties.relationships[0]["keyField"]
fkey_fld = rellyr.properties.relationships[0]["keyField"]
sql = "{}='{}'".format(fc_flag, flag_values[0])
rel_records = rellyr.query(where=sql)
updated_rows = []
for record in rel_records:
rel_oid = record.attributes[oid_fld]
parent = get_parent(lyr, pkey_fld, record, fkey_fld)
# Upload comment attachments
try:
attachmentmgr = rellyr.attachments
attachments = attachmentmgr.get_list(rel_oid)
for attachment in attachments:
response = copy_attachment(attachmentmgr, attachment,
rel_oid,
parent.attributes[ids[1]])
if response["Status"] is not 0:
try:
error = response["ErrorMessages"]
except KeyError:
error = response["Message"]
msg = "Error copying attachment. Record {} in table {}: {}".format(
rel_oid, relname, error)
if log_to_file:
log.write(msg + '\n')
else:
print(msg)
except RuntimeError:
pass # table doesn't support attachments
# Process comments
response = copy_comments(record, parent, tablefields, ids)
if 'error' in response:
if log_to_file:
log.write('Error accessing comment table {}\n'.format(
relname))
else:
print(
'Error accessing comment table {}'.format(relname))
break
elif response["Status"] is not 0:
try:
error = response["ErrorMessages"]
except KeyError:
error = response["Message"]
msg = "Error copying record {} from {}: {}".format(
rel_oid, relname, error)
if log_to_file:
log.write(msg + '\n')
else:
print(msg)
continue
else:
record.attributes[fc_flag] = flag_values[1]
try:
record.attributes[ids[1]] = parent.attributes[ids[1]]
except TypeError:
record.attributes[ids[1]] = str(
parent.attributes[ids[1]])
updated_rows.append(record)
# apply edits to updated records
if updated_rows:
status = rellyr.edit_features(updates=updated_rows)
if log_to_file:
log.write(
"Status of updates to ArcGIS comments: {}\n".format(
status))
else:
print("Status of updates to ArcGIS comments: {}\n".format(
status))
print("Finished processing: {}".format(lyr.properties["name"]))
except Exception as ex:
print("error: " + str(ex))
if log_to_file:
log.close()
def main(configuration_file):
try:
with open(configuration_file) as configfile:
cfg = json.load(configfile)
gis = GIS(cfg['organization url'], cfg['username'], cfg['password'])
# Get general id settings
global id_settings
id_settings = {}
for option in cfg['id sequences']:
id_settings[option['name']] = {
'interval': int(option['interval']),
'next value': int(option['next value']),
'pattern': option['pattern']
}
# Get general moderation settings
global modlists
modlists = {}
subs = cfg['moderation settings']['substitutions']
for modlist in cfg['moderation settings']['lists']:
words = [
str(word).upper().strip()
for word in modlist['words'].split(',')
]
modlists[modlist['filter name']] = build_expression(
words, modlist['filter type'], subs)
# Get general email settings
server = cfg['email settings']['smtp server']
username = cfg['email settings']['smtp username']
password = cfg['email settings']['smtp password']
tls = cfg['email settings']['use tls']
from_address = cfg['email settings']['from address']
if not from_address:
from_address = ''
reply_to = cfg['email settings']['reply to']
if not reply_to:
reply_to = ''
global substitutions
substitutions = cfg['email settings']['substitutions']
# Process each service
for service in cfg['services']:
try:
lyr = FeatureLayer(service['url'], gis=gis)
# GENERATE IDENTIFIERS
idseq = service['id sequence']
idfld = service['id field']
if id_settings and idseq and idfld:
if idseq in id_settings:
new_sequence_value = add_identifiers(lyr, idseq, idfld)
id_settings[idseq]['next value'] = new_sequence_value
else:
_add_message(
'Sequence {} not found in sequence settings'.
format(idseq), 'WARNING')
# ENRICH REPORTS
if service['enrichment']:
# reversed, sorted list of enrichment settings
enrich_settings = sorted(
service['enrichment'],
key=lambda k: k['priority']) #, reverse=True)
for reflayer in enrich_settings:
source_features = FeatureLayer(reflayer['url'], gis)
enrich_layer(source_features, lyr, reflayer)
# MODERATION
if modlists:
for query in service['moderation']:
if query['list'] in modlists:
moderate_features(lyr, query)
else:
_add_message(
'Moderation list {} not found in moderation settings'
.format(modlist), 'WARNING')
# SEND EMAILS
if service['email']:
with EmailServer(server, username, password,
tls) as email_server:
for message in service['email']:
rows = _get_features(lyr, message['sql'])
for row in rows:
address, subject, body = build_email(
row, lyr.properties.fields, message)
if address and subject and body:
try:
email_server.send(
from_address=from_address,
reply_to=reply_to,
to_addresses=[address],
subject=subject,
email_body=body)
row.attributes[message[
'field']] = message['sent value']
except:
_add_message(
'email failed to send for feature {} in layer {}'
.format(row.attributes,
service['url']))
if rows:
results = lyr.edit_features(updates=rows)
_report_failures(results)
except Exception as ex:
_add_message('Failed to process service {}\n{}'.format(
service['url'], ex))
except Exception as ex:
_add_message(
'Failed. Please verify all configuration values\n{}'.format(ex))
finally:
new_sequences = [{
'name': seq,
'interval': id_settings[seq]['interval'],
'next value': id_settings[seq]['next value'],
'pattern': id_settings[seq]['pattern']
} for seq in id_settings]
if not new_sequences == cfg['id sequences']:
cfg['id sequences'] = new_sequences
try:
with open(configuration_file, 'w') as configfile:
json.dump(cfg, configfile)
except Exception as ex:
_add_message(
'Failed to save identifier configuration values.\n{}\nOld values:{}\nNew values:{}'
.format(ex, cfg['id sequences'], new_sequences))
def cases_entry_form():
global error
portal_url = current_app.config['PORTAL_URL']
portal_user = current_app.config['PORTAL_USER']
portal_password = current_app.config['PORTAL_PASSWORD']
cases_url = current_app.config['COVID_CASES_URL']
form = CasesForm()
if form.validate_on_submit():
# We've received input from a form, process it.
#session['name'] = form.name.data
try:
local = parsetime(form.datestamp.data)
utc = local2utc(local).strftime(time_format)
except Exception as e:
print("Time format is confusing to me.", e)
error = e
return redirect("/fail")
try:
n = {
"attributes": {
"utc_date":
utc,
"last_update":
utc,
'name':
'Clatsop',
"total_cases":
s2i(form.total_cases.data),
"new_cases":
s2i(form.new_cases.data),
"total_negative":
s2i(form.negative.data),
"total_tests":
s2i(form.total_cases.data) + s2i(form.negative.data),
"total_deaths":
s2i(form.total_deaths.data),
"new_deaths":
s2i(form.new_deaths.data),
"source":
VERSION,
"editor":
"EMD",
},
"geometry": county_centroid
}
except Exception as e:
print("Attribute error.", e)
error = e
return redirect("/fail")
# write back to server
results = ''
try:
portal = GIS(portal_url, portal_user, portal_password)
layer = FeatureLayer(cases_url)
print(layer, n)
results = layer.edit_features(adds=[n])
except Exception as e:
error = e
print("Write failed.", e, results)
return redirect("/fail")
if not update_cases(layer):
return redirect("/fail")
return redirect('/thanks/cases')
# Create a data entry form
try:
portal = GIS(portal_url, portal_user, portal_password)
df = FeatureLayer(cases_url).query(
where="name='Clatsop' AND editor='EMD'",
order_by_fields="utc_date DESC").sdf
s = df.iloc[0]
print(s)
except Exception as e:
print("Reading old data failed.", e)
pass
try:
# Force the old date into UTC
old_date = s['utc_date'].replace(tzinfo=timezone('UTC'))
# Show the old date in the local TZ
form.old_date = "(previous %s)" % old_date.astimezone(
timezone('America/Los_Angeles')).strftime(time_format)
except Exception as e:
print("Converting old date stamp failed.", e)
pass
try:
form.total_cases.data = s['total_cases']
form.new_cases.data = s['new_cases']
form.negative.data = s['total_negative']
#form.recovered.data = s['total_recovered']
form.total_deaths.data = s['total_deaths']
form.new_deaths.data = s['new_deaths']
#form.editor.data = s['editor']
except Exception as e:
print("Filling in form failed.", e)
pass
now = datetime.now()
ds = now.strftime(time_format)
form.datestamp.data = ds
return render_template('cases.html', form=form)
class NationalTimeSeries(object):
def __init__(self,
service_url: str,
date_field: str = 'Date',
region_field: str = 'Region',
new_cases_field: str = 'NewCases',
new_deaths_field: str = 'NewDeaths',
new_recoveries_field: str = 'NewRecoveries',
total_cases_field: str = 'TotalCases',
total_deaths_field: str = 'TotalDeaths',
total_recoveries_field: str = 'TotalRecoveries',
active_cases_field: str = 'ActiveCases'):
self.layer = FeatureLayer(service_url)
self.region_field = region_field
self.date_field = date_field
self.new_cases_field = new_cases_field
self.new_deaths_field = new_deaths_field
self.new_recoveries_field = new_recoveries_field
self.total_cases_field = total_cases_field
self.total_deaths_field = total_deaths_field
self.total_recoveries_field = total_recoveries_field
self.active_cases_field = active_cases_field
def _get_adds(self, target_features: FeatureSet, update_values: dict):
# Make a copy so no items are removed from the source update_values item.
update_copy = copy.deepcopy(update_values)
# remove all items with matching key value.
for target_feature in target_features:
region = target_feature.attributes[self.region_field]
date = datetime.datetime.fromtimestamp(
target_feature.attributes[self.date_field] / 1000)
key_val = region + date.strftime('%Y%m%d')
update_copy.pop(key_val, None)
# only new items remain. Create adds for new items.
new_features = []
for new_item in update_copy.values():
# Create new item for service adds. Explicitly assigning the values to a new item prevents any possible errors
# if the item has additional fields.
item = {
"attributes": {
self.date_field: new_item['date'],
self.region_field: new_item['region'],
self.new_cases_field: new_item['NewCases'],
self.total_cases_field: new_item['TotalCases'],
self.new_deaths_field: new_item['NewDeaths'],
self.total_deaths_field: new_item['TotalDeaths'],
self.new_recoveries_field: new_item['NewRecovered'],
self.total_recoveries_field: new_item['TotalRecovered'],
self.active_cases_field: new_item['ActiveCases']
},
"geometry":
state_coords.get(new_item['region'], {
"x": 17100000,
"y": -4600000
})
}
new_features.append(item)
return new_features
def update(self, update_values):
# get featureset from target service
query_result = self.layer.query()
target_features = query_result.features
# get new values (adds)
adds = self._get_adds(target_features=target_features,
update_values=update_values)
# update feature layer
if adds:
self.layer.edit_features(adds=adds)
return len(adds)
classes = visual_recognition.classify(images_file,
parameters=json.dumps({
'classifier_ids':
['##########'],
'threshold':
0.876
}))
#print(json.dumps(classes, indent=2))
data = json.loads(json.dumps(classes, indent=2))
if len(data['images'][0]['classifiers'][0]['classes']) != 0:
print(json.dumps(classes, indent=2))
f = open(
"./images/" +
filename.replace("photo", "coor").replace("jpg", "txt"), "r")
d = float(f.read()) * 0.000000301
gis = GIS(username="******", password="******")
p = Point({"x": -73.471977 + d, "y": 40.703342})
a = {"pothole_layer": "pothole"}
f = Feature(p, a)
fs = FeatureSet([f])
lyr = FeatureLayer(
"https://services8.arcgis.com/x660UqfqVJlbWB0Y/arcgis/rest/services/pothole_layers/FeatureServer/0",
gis=gis)
lyr.edit_features(adds=fs)
#delete photo and txt
class FeatureServiceHelper(FeatureSourceHelper):
def __init__(self, source, id_field):
super().__init__(source=source, id_field=id_field)
self.layer = FeatureLayer(source)
def query(self, fields=None, where_clause=None):
if fields is None:
fields = '*'
elif isinstance(fields, list):
fields = ','.join(fields)
if where_clause is None:
where_clause = '1=1'
return self.layer.query(out_fields=fields, where=where_clause)
def load_records(self, fields=None, where_clause=None):
return self.query(fields=fields, where_clause=where_clause).to_dict('records')
@staticmethod
def _field_types(query_result):
result = {}
for field in query_result.fields:
result[field['name']] = field['type']
return result
def field_names(self):
return self.layer.properties.fields
def update_records(self, new_data, fields=None, where_clause=None, add_new=False, delete_unmatched=False,
rounding=4, case_sensitive=True, shape_field='Shape'):
"""
Updates the feature class using the new_data, with each record uniquely identified by the self.record_id_field.
:param new_data: Dictionary records indexed by id. {record_id: {field_name1: value1, field_name2: value2,...}}
:type new_data: dict
:param fields: The list of fields to be updated. If an id_field is included, it will not be updated.
:type fields:
:param where_clause: An optional where clause to filter the updates.
:type where_clause:
:param add_new: If true, any records found in the new_data that do not have a corresponding record in the feature class will be deleted.
:type add_new: bool
:param delete_unmatched: If true, any feature class records found that do not have a corresponding record in the new_data will be deleted.
:type delete_unmatched: bool
:param rounding: decimal rounding to be used when comparing double format numbers.
:type rounding: int
:param case_sensitive:
:type case_sensitive:
:param shape_field: The name of the geometry field in the new_data. Set to None if geometry is not to be updated. Default='Shape'
:type shape_field: str
:return:
:rtype:
"""
self._rounding = rounding
self._case_sensitive = case_sensitive
# use a local copy of new_data because we will be popping items.
my_data = copy.deepcopy(new_data)
if fields and self.id_field not in fields:
fields.append(self.id_field)
query_result = self.query(fields=fields, where_clause=where_clause)
field_types = self._field_types(query_result)
updates = []
deletes = []
for row in query_result:
id_value = row.attributes[self.id_field]
new_row = my_data.pop(id_value, None)
if new_row:
if self.update_row(row=row, field_types=field_types, new_values=new_row, shape_field=shape_field):
updates.append(row)
elif delete_unmatched:
deletes.append(row.attributes[query_result.object_id_field_name])
adds = []
if add_new:
# any remaining data_models items are new records
for id_value, new_item in my_data.items():
new_geometry = new_item.pop(shape_field, None)
row = self.generate_new_row(new_item, new_geometry)
adds.append(row)
return self.update_layer(adds=adds, deletes=None, updates=updates)
def update_layer(self, adds=None, deletes=None, updates=None, chunk_size=1000):
"""
Performs updates on an arcgis feature service in manageable chunks.
Updates are performed using multiple calls to the service where needed. Large sets of updates are broken into
smaller calls to avoid timeouts and other data size related issues. Updates are executed in the following order:
- Deletes
- Adds
- Updates
If no elements are submitted for Adds, Deletes or Updates, then that stage of the process is skipped.
:param adds: The list of add items to be added.
:type adds:
:param deletes:
:type deletes:
:param updates:
:type updates:
:param chunk_size:
:type chunk_size:
:return:
:rtype:
"""
logging.info('Updating: ' + self.source)
result = {'adds': 0, 'deletes': 0, 'updates': 0}
# perform updates in order deletes, adds, updates to support models where:
# - updates are performed by deleting old records and replacing with new (remove old items before adding new)
# - items can be added and updated in same cycles (ensure adds are in place before updates are applied)
if deletes:
for chunk in self._list_chunks(master_list=deletes, chunk_size=chunk_size):
logging.info('Applying {} Deletes'.format(len(chunk)))
self.layer.edit_features(deletes=str(chunk))
result['deletes'] = len(deletes)
logging.info('Total Deletes: {}'.format(result['deletes']))
if adds:
for chunk in self._list_chunks(master_list=adds, chunk_size=chunk_size):
logging.info('Applying {} Adds'.format(len(chunk)))
self.layer.edit_features(adds=chunk)
result['adds'] = len(adds)
logging.info('Total Adds: {}'.format(result['adds']))
if updates:
for chunk in self._list_chunks(master_list=updates, chunk_size=chunk_size):
logging.info('Applying {} Updates'.format(len(chunk)))
self.layer.edit_features(updates=chunk)
result['updates'] = len(updates)
logging.info('Total Updates: {}'.format(result['updates']))
return result
@staticmethod
def update_date_field(row: Feature, field_name: str, new_value: datetime.date):
current_date_value = datetime_utils.to_datetime(row.attributes[field_name])
new_date_value = datetime_utils.to_datetime(new_value)
if current_date_value == new_date_value:
# if the values are the same, return False.
return False
# The values are different, Update the row.
row.attributes[field_name] = new_date_value
return True
@staticmethod
def update_int_field(row: Feature, field_name: str, new_value: int = None):
if row.attributes[field_name] != new_value:
row.attributes[field_name] = new_value
return True
return False
def update_str_field(self, row: Feature, field_name: str, new_value: str = None):
current_value = row.attributes[field_name]
# if both are equal (str=str or None=None) return False.
if current_value == new_value:
return False
# if (str and None) or (None and str)
if not (current_value and new_value):
row.attributes[field_name] = new_value
return True
# both values are non-identical strings.
# if the test is not case sensitive and both UC strings match, no update needed
if not self._case_sensitive and current_value.upper() == new_value.upper():
return False
# the strings are non-equivalent. Update.
row.attributes[field_name] = new_value
return True
def update_float_field(self, row: Feature, field_name: str, new_value: float = None):
current_value = row.attributes[field_name]
if current_value:
current_value = round(current_value, self._rounding) # round non zero, non-null values.
if new_value:
test_value = round(new_value, self._rounding) # round non zero, non-null values.
else:
test_value = new_value
if current_value == test_value:
return False
row.attributes[field_name] = new_value
return True
def update_field(self, row: Feature, field_name: str, field_type: str, new_value):
ignore_types = ['esriFieldTypeOID', 'esriFieldTypeGeometry', 'esriFieldTypeBlob', 'esriFieldTypeRaster']
if field_type in ['esriFieldTypeSmallInteger', 'esriFieldTypeInteger']:
return self.update_int_field(row, field_name=field_name, new_value=new_value)
elif field_type in ['esriFieldTypeSingle', 'esriFieldTypeDouble']:
return self.update_float_field(row=row, field_name=field_name, new_value=new_value)
elif field_type == 'esriFieldTypeString':
return self.update_str_field(row=row, field_name=field_name, new_value=new_value)
elif field_type == 'esriFieldTypeDate':
return self.update_date_field(row=row, field_name=field_name, new_value=new_value)
elif field_type in ignore_types:
return False
raise ValueError('Unhandled field type: ' + field_type)
def update_row(self, row: Feature, field_types, new_values, shape_field=None):
update_required = False
for field_name in row.fields:
if field_name != self.id_field and field_name in new_values:
field_type = field_types[field_name]
new_value = new_values[field_name]
if self.update_field(row=row, field_name=field_name, field_type=field_type, new_value=new_value):
update_required = True
#if shape_field: TODO implement feature updates in a way that handles minor locational variations (nm)
#
# new_wkt = self.to_wkt(new_values[shape_field])
# current_wkt = self.to_wkt(row.geometry)
# if current_wkt != new_wkt:
# row.geometry = self.to_geometry(new_values[shape_field])
# update_required = True
return update_required
@ staticmethod
def to_wkt(source):
if source is None:
return None
if isinstance(source, Geometry):
return source.WKT
if isinstance(source, arcpy.Geometry):
return source.WKT
geom = Geometry(source)
return geom.WKT
@staticmethod
def to_geometry(source):
if source is None:
return None
if isinstance(source, Geometry):
return source
if isinstance(source, arcpy.Geometry):
return Geometry(source.JSON)
return Geometry(source)
@staticmethod
def generate_new_row(new_values, geometry=None):
attributes = {}
for field_name, value in new_values.items():
attributes[field_name] = value
geometry_value = geometry
if geometry_value and isinstance(geometry_value, arcpy.Geometry):
geometry_value = json.loads(geometry_value.JSON)
return {"attributes": attributes,
"geometry": geometry_value}
@staticmethod
def _list_chunks(master_list, chunk_size):
"""
Yield successive chunk-sized chunks from master_list.
A utility function to support other methods in this module.
"""
for i in range(0, len(master_list), chunk_size):
yield master_list[i:i + chunk_size]
for att in results:
attachmentids[att['PARENTOBJECTID']].append(fcURL + "/{}/attachments/{}".format(att['PARENTOBJECTID'], att['ID']))
#Remove all attachments but the one in the middle of each house
for key, values in attachmentids.items():
if values:
midVal = values[int(len(values) / 2)]
attachmentids[key] = midVal
features = feature_layer.query(where=whereClause, return_geometry=False)
featuresDict = [feature for feature in features if feature.get_value(flOID) in attachmentids]
count = 0
arcpy.SetProgressor("step", "Analyzing Photos", 0, len(attachmentids) * len(predictProjects) ,1)
for key, value in sorted(attachmentids.items()):
#url = feature_layer.url + '/{0}/attachments/{1}'.format(key, value)
feature = featuresDict[count]
count += 1
for project in predictProjects:
arcpy.SetProgressorLabel("Detecting Category '{}' in Feature {} of {}".format(project["name"],count,len(attachmentids)))
results = predictor.classify_image_url_with_no_store(project["projectID"], project["currentIteration"].publish_name, url=value + tokenStr)
for prediction in results.predictions:
if prediction.tag_name in categoryList:
feature.set_value(prediction.tag_name.lower() + "_prb", prediction.probability * 100)
arcpy.SetProgressorPosition()
feature_layer.edit_features(updates=[feature])
"""
A quick script to clean up duplicate postcodes introduced during development of UpdateTotalCasesByPostcode.py.
"""
from arcgis.gis import GIS
from arcgis.features import FeatureLayer
target = None# r'https://services9.arcgis.com/7eQuDPPaB6g9029K/arcgis/rest/services/Total_Cases_By_Postcode/FeatureServer/0'
gis = GIS(profile='agol_graphc')
target_layer = FeatureLayer(target)
query_result = target_layer.query()
postcode_features = query_result.features
known_postcodes = []
duplicates = []
for postcode_feature in postcode_features:
postcode = postcode_feature.attributes['PostCode']
if postcode in known_postcodes:
duplicates.append(postcode_feature.attributes['OBJECTID'])
else:
known_postcodes.append(postcode)
result = target_layer.edit_features(deletes=str(duplicates))
print(result)
class TotalCovid19CasesByPostcode(object):
"""
A data access class to support the corresponding feature service layer.
- Each postcode is represented by a single record. There should be no duplicate postcodes in this feature layer.
"""
def __init__(self, service_url=default_service_url):
self.layer = FeatureLayer(service_url)
self.postcode_field = 'PostCode'
self.total_cases_field = 'TotalCases'
self.date_of_last_case_field = 'DateOfLastCase'
self.days_since_last_case_field = 'DaysSinceLastCase'
def query(self, where_clause):
return self.layer.query(where=where_clause)
def update(self, updates_items=None):
"""
Updates the feature service:
- recalculates the date_of_last_case values and updates as required.
- if update_items are submitted, updates or adds the submitted rows.
Rows are identified by postcode.
Rows are only added if the postcode does not already exist in the table.
Rows are never deleted by this process. In order to prevent accidental deletions they must be performed as a separate task.
Existing XYs are not updated, submitted xys are used if rows are added.
Use 'xy': None if location is not known for new records
:param updates_items: A dictionary of named row values indexed by postcode.
{postcode: {
'postcode': str,
'total_cases': int,
'date_of_last_case': datetime,
'xy': {
'x': float,
'y': float}
}
}
:type updates_items: dict
:return:
:rtype:
"""
updates = []
if not updates_items:
updates_items = {}
# identify updates to be performed for existing rows.
query_result = self.layer.query()
for row in query_result:
update_required = False
poa = row.attributes[self.postcode_field]
item = updates_items.pop(poa, None)
if item:
if row.attributes[
self.total_cases_field] != item['total_cases']:
row.attributes[
self.total_cases_field] = item['total_cases']
update_required = True
if row.attributes[self.date_of_last_case_field] != item[
'date_of_last_case']:
row.attributes[self.date_of_last_case_field] = item[
'date_of_last_case']
update_required = True
report_age = self.calc_days_since(
row.attributes[self.date_of_last_case_field])
if report_age != row.attributes[self.days_since_last_case_field]:
row.attributes[self.days_since_last_case_field] = report_age
update_required = True
if update_required:
updates.append(row)
# any remaining data_models items are new records
adds = []
new_items = updates_items.values()
if new_items:
for new_item in new_items:
days_since_last_case = self.calc_days_since(
new_item['date_of_last_case'])
row = {
"attributes": {
self.postcode_field: new_item['postcode'],
self.total_cases_field: new_item['total_cases'],
self.date_of_last_case_field:
new_item['date_of_last_case'],
self.days_since_last_case_field: days_since_last_case
},
"geometry": new_item['xy']
}
adds.append(row)
logging.info('Updating: {}'.format(self.layer.url))
logging.info('ADDS: {}'.format(len(adds)))
logging.info('UPDATES: {}'.format(len(updates)))
if updates or adds:
return self.layer.edit_features(updates=updates, adds=adds)
else:
return None
@staticmethod
def calc_days_since(reference_date):
if reference_date:
return (datetime.datetime.now() - reference_date).days
else:
return None
def main(event, context):
import sys
# Cityworks settings
global baseUrl
baseUrl = event["cityworks"]["url"]
cwUser = event["cityworks"]["username"]
cwPwd = event["cityworks"]["password"]
timezone = event["cityworks"].get("timezone", "")
isCWOL = event["cityworks"].get("isCWOL", False)
# ArcGIS Online/Portal settings
orgUrl = event["arcgis"]["url"]
username = event["arcgis"]["username"]
password = event["arcgis"]["password"]
layers = event["arcgis"]["layers"]
tables = event["arcgis"]["tables"]
layerfields = event["fields"]["layers"]
tablefields = event["fields"]["tables"]
fc_flag = event["flag"]["field"]
flag_values = [event["flag"]["on"], event["flag"]["off"]]
ids = event["fields"]["ids"]
probtypes = event["fields"]["type"]
opendate = event["fields"].get("opendate", "")
if log_to_file:
from datetime import datetime as dt
id_log = path.join(sys.path[0], "cityworks_log.log")
log = open(id_log, "a")
log.write("\n{} ".format(dt.now()))
log.write("Sending reports to: {}\n".format(baseUrl))
else:
print("Sending reports to: {}".format(baseUrl))
try:
# Connect to org/portal
gis = GIS(orgUrl, username, password)
# Get token for CW
status = get_cw_token(cwUser, cwPwd, isCWOL)
if "error" in status:
if log_to_file:
log.write("Failed to get Cityworks token. {}\n".format(status))
else:
print("Failed to get Cityworks token. {}".format(status))
raise Exception(
"Failed to get Cityworks token. {}".format(status))
# get wkid
sr = get_wkid()
if sr == "error":
if log_to_file:
log.write("Spatial reference not defined\n")
else:
print("Spatial reference not defined")
raise Exception("Spatial reference not defined")
# get problem types
prob_types = get_problem_types()
if prob_types == "error":
if log_to_file:
log.write("Problem types not defined\n")
else:
print("Problem types not defined")
raise Exception("Problem types not defined")
for layer in layers:
lyr = FeatureLayer(layer, gis=gis)
oid_fld = lyr.properties.objectIdField
lyrname = lyr.properties["name"]
# Get related table URL
reltable = ""
try:
for relate in lyr.properties.relationships:
url_pieces = layer.split("/")
url_pieces[-1] = str(relate["relatedTableId"])
table_url = "/".join(url_pieces)
if table_url in tables:
reltable = table_url
break
# if related tables aren't being used
except AttributeError:
pass
# query reports
sql = "{}='{}'".format(fc_flag, flag_values[0])
rows = lyr.query(where=sql, out_sr=sr)
updated_rows = []
for row in rows.features:
try:
oid = row.attributes[oid_fld]
# Submit feature to the Cityworks database
request = submit_to_cw(row, prob_types, layerfields, oid,
probtypes)
try:
reqid = request["RequestId"]
initDate = int(
parse(request[opendate[0]]).replace(
tzinfo=gettz(timezone)).timestamp() *
1000) if opendate else ""
except TypeError:
if "WARNING" in request:
msg = "Warning generated while copying ObjectID:{} from layer {} to Cityworks: {}".format(
oid, lyrname, request)
if log_to_file:
log.write(msg + '\n')
else:
print(msg)
continue
elif 'error' in request:
msg = "Error generated while copying ObjectID:{} from layer {} to Cityworks: {}".format(
oid, lyrname, request)
if log_to_file:
log.write(msg + '\n')
else:
print(msg)
continue
else:
msg = "Uncaught response generated while copying ObjectID:{} from layer {} to Cityworks: {}".format(
oid, lyrname, request)
if log_to_file:
log.write(msg + '\n')
else:
print(msg)
continue
# update the record in the service so that it evaluates falsely against sql
sql = "{}='{}'".format(oid_fld, oid)
row_orig = lyr.query(where=sql).features[0]
row_orig.attributes[fc_flag] = flag_values[1]
if opendate:
row_orig.attributes[opendate[1]] = initDate
try:
row_orig.attributes[ids[1]] = reqid
except TypeError:
row_orig.attributes[ids[1]] = str(reqid)
# apply edits to updated row
status = lyr.edit_features(updates=[row_orig])
if log_to_file:
log.write(
"Status of updates to {}, ObjectID:{} {}\n".format(
lyr.properties["name"], oid, status))
else:
print("Status of updates to {}, ObjectID:{} {}".format(
lyr.properties["name"], oid, status))
# attachments
try:
attachmentmgr = lyr.attachments
attachments = attachmentmgr.get_list(oid)
for attachment in attachments:
response = copy_attachment(attachmentmgr,
attachment, oid, reqid)
if response["Status"] is not 0:
try:
error = response["ErrorMessages"]
except KeyError:
error = response["Message"]
msg = "Error copying attachment from feature {} in layer {}: {}".format(
oid, lyrname, error)
if log_to_file:
log.write(msg + '\n')
else:
print(msg)
except RuntimeError:
pass # feature layer doesn't support attachments
# any other error in row execution, move on to next row
except Exception as e:
if log_to_file:
log.write(str(e) + '\n')
else:
print(str(e))
continue
# end of row execution
# end of features execution
# related records
rel_records = []
#if comments tables aren't used, script will crash here
try:
if len(lyr.properties.relationships) > 0:
# related records
rellyr = FeatureLayer(reltable, gis=gis)
relname = rellyr.properties["name"]
pkey_fld = lyr.properties.relationships[0]["keyField"]
fkey_fld = rellyr.properties.relationships[0]["keyField"]
sql = "{}='{}'".format(fc_flag, flag_values[0])
rel_records = rellyr.query(where=sql)
# if related tables aren't being used
except AttributeError:
pass
except KeyError:
relname = "Comments"
updated_rows = []
for record in rel_records:
try:
rel_oid = record.attributes[oid_fld]
parent = get_parent(lyr, pkey_fld, record, fkey_fld)
# Process comments
response = copy_comments(record, parent, tablefields, ids)
if 'error' in response:
if log_to_file:
log.write(
'Error accessing comment table {}\n'.format(
relname))
else:
print('Error accessing comment table {}'.format(
relname))
break
elif response["Status"] is not 0:
try:
error = response["ErrorMessages"]
except KeyError:
error = response["Message"]
msg = "Error copying record {} from {}: {}".format(
rel_oid, relname, error)
if log_to_file:
log.write(msg + '\n')
else:
print(msg)
continue
else:
record.attributes[fc_flag] = flag_values[1]
try:
record.attributes[ids[1]] = parent.attributes[
ids[1]]
except TypeError:
record.attributes[ids[1]] = str(
parent.attributes[ids[1]])
# apply edits to updated record
status = rellyr.edit_features(updates=[record])
if log_to_file:
log.write(
"Status of updates to {}, ObjectID:{} comments: {}\n"
.format(relname, rel_oid, status))
else:
print(
"Status of updates to {}, ObjectID:{} comments: {}"
.format(relname, rel_oid, status))
# Upload comment attachments
try:
attachmentmgr = rellyr.attachments
attachments = attachmentmgr.get_list(rel_oid)
for attachment in attachments:
response = copy_attachment(
attachmentmgr, attachment, rel_oid,
parent.attributes[ids[1]])
if response["Status"] is not 0:
try:
error = response["ErrorMessages"]
except KeyError:
error = response["Message"]
msg = "Error copying attachment. Record {} in table {}: {}".format(
rel_oid, relname, error)
if log_to_file:
log.write(msg + '\n')
else:
print(msg)
except RuntimeError:
pass # table doesn't support attachments
# any other uncaught Exception in related record export, move on to next row
except Exception as e:
if log_to_file:
log.write(str(e) + '\n')
else:
print(str(e))
continue
print("Finished processing: {}".format(lyrname))
except BaseException as ex:
exc_tb = sys.exc_info()[2]
exc_typ = sys.exc_info()[0]
print('error: {} {}, Line {}'.format(exc_typ, str(ex),
exc_tb.tb_lineno))
if log_to_file:
log.write('error: {} {}, Line {}'.format(exc_typ, str(ex),
exc_tb.tb_lineno))
except:
exc_tb = sys.exc_info()[2]
exc_typ = sys.exc_info()[0]
print('error: {}, Line {}'.format(exc_typ, exc_tb.tb_lineno))
if log_to_file:
log.write('error: {}, Line {}'.format(exc_typ, exc_tb.tb_lineno))
finally:
if log_to_file:
log.close()
class DailyCovid19TestingByPostcode(object):
def __init__(self, service_url=default_service_url):
self.layer = FeatureLayer(service_url)
self.postcode_field = 'Postcode'
self.date_field = 'Date'
self.date_code_field = 'DateCode'
self.tests_field = 'Tests'
self.total_tests_field = 'TotalTests'
self.date_code_format = '%Y%m%d'
def query(self, where_clause):
return self.layer.query(where=where_clause)
def update(self, rows, allow_deletes=False):
"""
updates the feature service by updating or adding the submitted rows. Rows are identified by date_code and postcode.
Date values are derived from the date_code values.
Existing XYs are not updated, submitted xys are used if rows are added.
:param rows: [{'postcode': str, 'date_code': str, 'tests': int, 'total_tests': int, 'xy': {'x': float, 'y': float}}]
:type rows: [dict]
:param allow_deletes: If set to True any records not found in the source rows will be deleted. Default=False
:type allow_deletes: bool
:return:
:rtype:
"""
# build lookup from submitted rows:
items = {}
for row in rows:
key = '{}_{}'.format(row['postcode'], row['date_code'])
items[key] = row
deletes = []
updates = []
query_result = self.layer.query()
for row in query_result:
poa = row.attributes[self.postcode_field]
date_code = row.attributes[self.date_code_field]
key = '{}_{}'.format(poa, date_code)
if allow_deletes and key not in items:
deletes.append(
row.attributes[query_result.object_id_field_name])
item = items.pop(key, None)
if item:
update_required = False
if row.attributes[self.tests_field] != item['tests']:
row.attributes[self.tests_field] = item['tests']
update_required = True
if row.attributes[
self.total_tests_field] != item['total_tests']:
row.attributes[
self.total_tests_field] = item['total_tests']
update_required = True
if update_required:
updates.append(row)
# any remaining data_models items are new records
adds = []
new_items = items.values()
if new_items:
for new_item in new_items:
date = datetime.datetime.strptime(new_item['date_code'],
'%Y%m%d')
row = {
"attributes": {
self.date_field: date,
self.postcode_field: new_item['postcode'],
self.date_code_field: new_item['date_code'],
self.total_tests_field: new_item['total_tests'],
self.tests_field: new_item['tests']
},
"geometry": new_item['xy']
}
adds.append(row)
logging.info('Updating: {}'.format(self.layer.url))
logging.info('ADDS: {}'.format(len(adds)))
logging.info('DELETES: {}'.format(len(deletes)))
logging.info('UPDATES: {}'.format(len(updates)))
if updates or adds or deletes:
return self.layer.edit_features(updates=updates,
adds=adds,
deletes=str(deletes))
else:
return None
def format_date_string(self, date_string, in_format):
"""
Reformats the submitted date or datetime string to the correct format for the date_code field
:type date_string: str
:param in_format: the format string of the submitted date_string. eg: '%Y-%m-%d'
:type in_format: str
:return: The reformatted date_string
:rtype: str
"""
if in_format == self.date_code_format:
return date_string
dt = datetime.datetime.strptime(date_string, in_format)
return dt.strftime(self.date_code_format)
def main():
# Create log file
with open(path.join(sys.path[0], 'attr_log.log'), 'a') as log:
log.write('\n{}\n'.format(dt.now()))
# connect to org/portal
if username:
gis = GIS(orgURL, username, password)
else:
gis = GIS(orgURL)
for service in services:
try:
# Connect to source and target layers
fl_source = FeatureLayer(service['source url'], gis)
fl_target = FeatureLayer(service['target url'], gis)
# get field map
fields = [[key, service['fields'][key]] for key in service['fields'].keys()]
# Get source rows to copy
rows = fl_source.query(service['query'])
adds = []
updates = []
for row in rows:
# Build dictionary of attributes & geometry in schema of target layer
# Default status and priority values can be overwritten if those fields are mapped to reporter layer
attributes = {'status': 0,
'priority': 0}
for field in fields:
attributes[field[1]] = row.attributes[field[0]]
new_request = {'attributes': attributes,
'geometry': {'x': row.geometry['x'],
'y': row.geometry['y']}}
adds.append(new_request)
# update row to indicate record has been copied
if service['update field']:
row.attributes[service['update field']] = service['update value']
updates.append(row)
# add records to target layer
if adds:
add_result = fl_target.edit_features(adds=adds)
for result in add_result['updateResults']:
if not result['success']:
raise Exception('error {}: {}'.format(result['error']['code'],
result['error']['description']))
# update records:
if updates:
update_result = fl_source.edit_features(updates=updates)
for result in update_result['updateResults']:
if not result['success']:
raise Exception('error {}: {}'.format(result['error']['code'],
result['error']['description']))
except Exception as ex:
msg = 'Failed to copy feature from layer {}'.format(service['url'])
print(ex)
print(msg)
log.write('{}\n{}\n'.format(msg, ex))
class TotalCasesByPostcodeFeatureLayer(object):
def __init__(self,
service_url: str,
postcode_field: str = 'PostCode',
total_cases_field: str = 'TotalCases',
date_of_last_case_field: str = 'DateOfLastCase',
days_since_last_case_field: str = 'DaysSinceLastCase'):
self.layer = FeatureLayer(service_url)
self.postcode_field = postcode_field
self.totalCases_field = total_cases_field
self.dateOfLastCase_field = date_of_last_case_field
self.daysSinceLastCase_field = days_since_last_case_field
def _get_updates(self, postcode_features: FeatureSet, update_values: dict):
"""
:param postcode_features: The postcodes featureset to be updated
:type postcode_features: FeatureSet
:param update_values: The current count values to be applied.
:type update_values:
:return:
:rtype:
"""
updates = []
no_change = 0
for postcode_feature in postcode_features:
postcode = postcode_feature.attributes[self.postcode_field]
updated_counts = update_values.pop(postcode, None)
if updated_counts:
total_cases = updated_counts['Total Cases']
date_last = updated_counts['DateOfLastCase']
days_since = updated_counts['DaysSinceLastCase']
else:
total_cases = None
date_last = None
days_since = None
current_cases = postcode_feature.attributes[self.totalCases_field]
current_days_since = postcode_feature.attributes[
self.daysSinceLastCase_field]
if current_cases != total_cases or days_since != current_days_since:
postcode_feature.attributes[
self.totalCases_field] = total_cases
postcode_feature.attributes[
self.dateOfLastCase_field] = date_last
postcode_feature.attributes[
self.daysSinceLastCase_field] = days_since
updates.append(postcode_feature)
else:
no_change += 1
return updates, no_change
def _get_adds(self, update_values):
new_postcodes = []
for new_item in update_values:
# Create new item for service adds. Explicitly assigning the values to a new item prevents any possible errors
# if the item has additional fields.
item = {
"attributes": {
self.postcode_field: new_item['Postcode'],
self.totalCases_field: new_item['Total Cases'],
self.dateOfLastCase_field: new_item['DateOfLastCase'],
self.daysSinceLastCase_field: new_item['DaysSinceLastCase']
},
"geometry": {
"x": 17100000,
"y": -4600000
}
}
new_postcodes.append(item)
return new_postcodes
def update_from_case_list(self, case_list: CaseList):
"""
Updates the layer using the values in the case_list parameter.
If a postcode in the service is not found in the case_list then the value for that postcode is set to None.
Postcodes are never deleted from the service.
If postcodes are found in the case_list that are not in the table, then those postcodes are added to the service.
This function does not alter the case_list object.
:param case_list: The case list to be used to update the feature layer.
:type case_list: CaseList
:return:
:rtype: UpdateResultCounts
"""
# make a local copy of postcode_counts so we don't inadvertently change the source dictionary for other uses.
update_values = case_list.counts_by_postcode()
# get featureset from target service
query_result = self.layer.query()
postcode_features = query_result.features
updates, no_change = self._get_updates(postcode_features,
update_values)
adds = self._get_adds(update_values)
if updates or adds:
self.layer.edit_features(updates=updates, adds=adds)
return UpdateResultCounts(adds=len(adds),
updates=len(updates),
unchanged=no_change)
def get_geometry_lookups(self, where_clause: str = None):
"""
Gets a dictionary of {key: geometry pairs}
:param where_clause:
:type where_clause:
:return:
:rtype:
"""
result = {}
if where_clause:
query_result = self.layer.query(out_fields=[self.postcode_field],
where=where_clause)
else:
query_result = self.layer.query(out_fields=[self.postcode_field])
postcode_features = query_result.features
for postcode_feature in postcode_features:
postcode = postcode_feature.attributes[self.postcode_field]
geometry = postcode_feature.geometry
result[postcode] = geometry
return result
from arcgis.gis import GIS
from arcgis.features import FeatureLayer
lyr_url = r'https://services9.arcgis.com/7eQuDPPaB6g9029K/arcgis/rest/services/COVID_19_Alerts_Editable/FeatureServer/0'
str_field = 'Dates'
date_field = 'DateValue'
profile = 'agol_graphc'
where_clause = '1=1'
gis = GIS(profile='agol_graphc')
layer = FeatureLayer(url=lyr_url)
fields = [str_field, date_field]
test_dt = datetime.datetime.now()
query_result = layer.query(out_fields=",".join(fields), where=where_clause)
updates = []
for row in query_result:
date_str = row.attributes[str_field]
date_val = parser.parse(date_str).replace(hour=12)
if date_val > test_dt:
date_val = date_val + relativedelta(years=-1)
date_ms = date_val.timestamp() * 1000
if row.attributes[date_field] != date_ms:
row.attributes[date_field] = date_ms
updates.append(row)
if updates:
pprint(layer.edit_features(updates=updates))
else:
print('No Updates Required')
class IndividualCasesByPostcodeFeatureLayer(object):
def __init__(self,
service_url: str,
date_string_field='DateString',
postcode_field='PostCode',
report_date_field='ReportDate',
report_age_field='ReportAge',
oid_field='OBJECTID'):
self.layer = FeatureLayer(service_url)
self.oid_field = oid_field
self.dateString_field = date_string_field
self.postcode_field = postcode_field
self.reportAge_field = report_age_field
self.reportDate_field = report_date_field
@staticmethod
def _get_source_values(case_list,
postcode_centroids,
default_xy=(17100000, -4600000)):
"""
Generates the update values by combining counts by date and postcode from the case list
and postcode centroids from the geometry source.
:param case_list: The CaseList containing the cases to be used to update the FeatureLayer
:type case_list: CaseList
:param postcode_centroids: A lookup of postcode centroid geometries
:type postcode_centroids: dict
:param default_xy: Optional - The xy tuple to be used if no matching postcode geometry is found.
Default is (17100000, -4600000), a WGS84-WMAS coordinate that falls of the NSW/VIC coast.
:type default_xy: tuple (float, float)
:return: {yyyymmddpppp: {'date_string': str, 'date': datetime, 'postcode': int, 'report_age': int, 'count': int, 'xy': (float, float)}
:rtype: dict
"""
result = case_list.counts_by_date_and_postcode()
# add the coordinate values to each item.
for item in result.values():
postcode_centroid = postcode_centroids.get(item['postcode'], None)
if postcode_centroid:
item['xy'] = postcode_centroid
else:
item['xy'] = {'x': default_xy[0], 'y': default_xy[1]}
return result
def update(self,
case_list,
postcode_centroids,
default_xy=(17100000, -4600000)):
source_values = self._get_source_values(case_list, postcode_centroids,
default_xy)
# get featureset from target service
query_result = self.layer.query()
postcode_features = query_result.features
adds = []
deletes = []
updates = []
unchanged = 0
for postcode_feature in postcode_features:
postcode = postcode_feature.attributes[self.postcode_field]
datestring = postcode_feature.attributes[self.dateString_field]
key_value = datestring + postcode
item = source_values.get(key_value, None)
if item:
if item['count'] == 1:
""" Remove the item from source if all source items have been matched.
Additional items in the feature layer with the same key will be deleted."""
source_values.pop(key_value)
item['count'] -= 1
if item['report_age'] != postcode_feature.attributes[
self.reportAge_field]:
"""If the report age values are not equal, updates are required.
We do not need to compare postcode or date values because they are defined
at record creation and do not change. A source record where the postcode or date changes
will result in a deletion of the current record and a creation of a new record with the
new date/postcode key."""
postcode_feature.attributes[
self.reportAge_field] = item['report_age']
updates.append(postcode_feature)
else:
unchanged += 1
else: # if no match found in source, add the OID to the deletes
deletes.append(postcode_feature.attributes[self.oid_field])
# any items left in the source_values represent new records. Append them to the Adds list.
for source_value in source_values.values():
item = {
"attributes": {
self.postcode_field: source_value['postcode'],
self.dateString_field: source_value['date_string'],
self.reportDate_field: source_value['date'],
self.reportAge_field: source_value['report_age']
},
"geometry": source_value['xy']
}
for i in range(source_value['count']):
adds.append(item)
if updates or adds or deletes:
self.layer.edit_features(updates=updates,
adds=adds,
deletes=str(deletes))
return UpdateResultCounts(adds=len(adds),
deletes=len(deletes),
updates=len(updates),
unchanged=unchanged)
import random
gis = GIS("https://www.arcgis.com", "devlaahernandezgo", "fQ2HFMfk5ya7E9J")
url = 'https://services5.arcgis.com/otu0qUsUpyUjfoF3/arcgis/rest/services/informacion_lugar/FeatureServer/0'
layer = FeatureLayer(url)
for x in range(0, 9):
body = {
"geometry": {
"objectId": 2,
"x": -74.07,
"y": 4.7 - random.random(),
"spatialReference": {
"wkid": 4326
}
},
"attributes": {
"edificio": "Aqui porfin",
"nombre": "prueba",
"numero_interno": "001",
"maxima_ocupacion": "18",
"tasa": random.randrange(0, 18),
"tiempo_promedio": "30",
"indice_bioseguro": "70"
}
}
layer.edit_features(adds=[body])
print(layer.query().sdf)
class NationalTimeSeries(object):
def __init__(self, service_url: str,
date_field: str = 'Date',
region_field: str = 'Region',
new_cases_field: str = 'NewCases',
new_deaths_field: str = 'NewDeaths',
new_recoveries_field: str = 'NewRecoveries',
total_cases_field: str = 'TotalCases',
total_deaths_field: str = 'TotalDeaths',
total_recoveries_field: str = 'TotalRecoveries',
active_cases_field: str = 'ActiveCases'):
self.layer = FeatureLayer(service_url)
self.region_field = region_field
self.date_field = date_field
self.new_cases_field = new_cases_field
self.new_deaths_field = new_deaths_field
self.new_recoveries_field = new_recoveries_field
self.total_cases_field = total_cases_field
self.total_deaths_field = total_deaths_field
self.total_recoveries_field = total_recoveries_field
self.active_cases_field = active_cases_field
@staticmethod
def _update_attribute(target_feature, attribute_name, new_value):
if target_feature.attributes[attribute_name] == new_value:
return False
else:
target_feature.attributes[attribute_name] = new_value
return True
def _get_updates(self, target_features: FeatureSet, update_values: dict):
"""
:param target_features: The featureset to be updated
:type target_features: FeatureSet
:param update_values: The current count values to be applied.
:type update_values:
:return:
:rtype:
"""
updates = []
no_change = 0
for target_feature in target_features:
region = target_feature.attributes[self.region_field]
date = datetime.datetime.fromtimestamp(target_feature.attributes[self.date_field]/1000)
key_val = region + date.strftime('%Y%m%d')
updated_counts = update_values.pop(key_val, None)
if updated_counts:
if (self._update_attribute(target_feature, self.total_cases_field, updated_counts['TotalCases']) or
self._update_attribute(target_feature, self.new_cases_field, updated_counts['NewCases']) or
self._update_attribute(target_feature, self.total_deaths_field, updated_counts['TotalDeaths']) or
self._update_attribute(target_feature, self.new_deaths_field, updated_counts['NewDeaths']) or
self._update_attribute(target_feature, self.total_recoveries_field, updated_counts['TotalRecovered']) or
self._update_attribute(target_feature, self.new_recoveries_field, updated_counts['NewRecovered']) or
self._update_attribute(target_feature, self.active_cases_field, updated_counts['ActiveCases'])
):
updates.append(target_feature)
else:
no_change += 1
return updates, no_change
def _get_adds(self, data):
new_features = []
for new_item in data.values():
# Create new item for service adds. Explicitly assigning the values to a new item prevents any possible errors
# if the item has additional fields.
item = {"attributes":
{self.date_field: new_item['date'],
self.region_field: new_item['region'],
self.new_cases_field: new_item['NewCases'],
self.total_cases_field: new_item['TotalCases'],
self.new_deaths_field: new_item['NewDeaths'],
self.total_deaths_field: new_item['TotalDeaths'],
self.new_recoveries_field: new_item['NewRecovered'],
self.total_recoveries_field: new_item['TotalRecovered'],
self.active_cases_field: new_item['ActiveCases']},
"geometry": state_coords.get(new_item['region'], {"x": 17100000, "y": -4600000})}
new_features.append(item)
return new_features
def update(self, update_values):
# get featureset from target service
query_result = self.layer.query()
target_features = query_result.features
updates, no_change = self._get_updates(target_features, update_values)
adds = self._get_adds(update_values)
if updates or adds:
self.layer.edit_features(updates=updates, adds=adds)
return UpdateResultCounts(adds=len(adds),
updates=len(updates),
unchanged=no_change)
