def create_and_append(feature_list=None,
token=None,
portal_url=None,
service_url=None,
matching=None):
token = arcpy.GetSigninToken()
portal_url = arcpy.GetActivePortalURL()
gis = GIS(portal_url, token=token['token'])
layer = FeatureLayer(service_url)
features_to_append = []
for feature in feature_list:
new_feature = {'attributes': {}, 'geometry': feature['geometry']}
#Find fields and it's value
for field in matching:
new_feature['attributes'][field[0]] = [
x[1] for x in feature['attributes'].items() if x[0] == field[1]
][0]
features_to_append.append(new_feature.copy())
if len(features_to_append) > 500:
result = layer.edit_features(adds=features_to_append)
features_to_append = []
if features_to_append:
layer.edit_features(adds=features_to_append)
def thanks(df=None):
portal_url = current_app.config['PORTAL_URL']
portal_user = current_app.config['PORTAL_USER']
portal_password = current_app.config['PORTAL_PASSWORD']
cases_url = current_app.config['COVID_CASES_URL']
ppe_url = current_app.config['PPE_INVENTORY_URL']
# Show the data that was just entered.
portal = GIS(portal_url, portal_user, portal_password)
if df == 'cases':
# Generate new CSV files while we're at it.
# In production they will be written to the "capacity" webserver
# In a test environment they end up in the local folder.
results_df = FeatureLayer(cases_url).query(
where="editor='EMD'",
order_by_fields="utc_date DESC",
return_all_records=False,
result_record_count=1,
return_geometry=False).sdf
elif df == 'ppe':
results_df = FeatureLayer(ppe_url).query(
where="facility='Clatsop'",
order_by_fields="utc_date DESC",
return_all_records=False,
result_record_count=1,
return_geometry=False).sdf
else:
results_df = pd.DataFrame()
return render_template('thanks.html', df=results_df)
def __init__(self, service_url=default_service_url):
self.layer = FeatureLayer(service_url)
self.postcode_field = 'Postcode'
self.date_field = 'Date'
self.date_code_field = 'DateCode'
self.tests_field = 'Tests'
self.total_tests_field = 'TotalTests'
self.date_code_format = '%Y%m%d'
def delete_rows(service_url, where_clause):
deletes = []
lyr = FeatureLayer(service_url)
query_result = lyr.query(where=where_clause, return_ids_only=True)
deletes = query_result['objectIds']
if deletes:
return lyr.edit_features(deletes=str(deletes))
else:
return None
def __init__(self, service_url=default_service_url):
self.layer = FeatureLayer(service_url)
self.lga_code_field = 'LGA_CODE'
self.lga_name_field = 'LGA_NAME'
self.lga_version_field = 'LGA_Version'
self.date_field = 'Date'
self.date_code_field = 'DateCode'
self.cases_field = 'Cases'
self.total_cases_field = 'TotalCases'
self.date_code_format = '%Y%m%d'
def run(self):
layer = FeatureLayer("https://services-eu1.arcgis.com/CZ1GXX3MIjSRSHoC/ArcGIS/rest/services/Covid19_Impfmeldungen_%c3%96ffentlich/FeatureServer/0")
start = time.time()
data = layer.query(order_by_fields='Einrichtung, Meldedatum, Impfungen_proTyp')
print('> Queried data in %.1fs' % (time.time() - start))
if len(data) == 0:
raise Exception('Queried data is empty')
features_filtered = list(filter_duplicate_days(data.features))
apply_manual_fixes(features_filtered)
check_cumulative_plausability(features_filtered)
rows_nach_einrichtung = list(map(map_nach_einrichtung, features_filtered))
rows_nach_einrichtung_file = os.path.join('corona-impfungen', 'arcgisImpfungenNachEinrichtung.csv')
rows_nach_einrichtung_diff = self.get_csv_diff(rows_nach_einrichtung_file, rows_nach_einrichtung)
if len(rows_nach_einrichtung_diff) > 0:
self.write_csv_rows(rows_nach_einrichtung_file, rows_nach_einrichtung)
if self.telegram_bot != None and self.telegram_chat_id != None:
data = ''.join(rows_nach_einrichtung_diff)
self.telegram_bot.send_message(
self.telegram_chat_id,
'```\n' + (data[:4080] if len(data) > 4080 else data) + '```',
parse_mode = "Markdown"
)
rows_nach_geschlecht = list(itertools.chain.from_iterable(map(map_nach_geschlecht, features_filtered)))
rows_nach_geschlecht_file = os.path.join('corona-impfungen', 'arcgisImpfungenNachGeschlecht.csv')
rows_nach_geschlecht_diff = self.get_csv_diff(rows_nach_geschlecht_file, rows_nach_geschlecht)
if len(rows_nach_geschlecht_diff) > 0:
self.write_csv_rows(rows_nach_geschlecht_file, rows_nach_geschlecht)
if self.telegram_bot != None and self.telegram_chat_id != None:
data = ''.join(rows_nach_geschlecht_diff)
self.telegram_bot.send_message(
self.telegram_chat_id,
'```\n' + (data[:4080] if len(data) > 4080 else data) + '```',
parse_mode = "Markdown"
)
rows_nach_alter = list(itertools.chain.from_iterable(map(map_nach_alter, features_filtered)))
rows_nach_alter_file = os.path.join('corona-impfungen', 'arcgisImpfungenNachAlter.csv')
rows_nach_alter_diff = self.get_csv_diff(rows_nach_alter_file, rows_nach_alter)
if len(rows_nach_alter_diff) > 0:
self.write_csv_rows(rows_nach_alter_file, rows_nach_alter)
if self.telegram_bot != None and self.telegram_chat_id != None:
data = ''.join(rows_nach_alter_diff)
self.telegram_bot.send_message(
self.telegram_chat_id,
'```\n' + (data[:4080] if len(data) > 4080 else data) + '```',
parse_mode = "Markdown"
)
def __init__(self,
service_url: str,
postcode_field: str = 'PostCode',
total_cases_field: str = 'TotalCases',
date_of_last_case_field: str = 'DateOfLastCase',
days_since_last_case_field: str = 'DaysSinceLastCase'):
self.layer = FeatureLayer(service_url)
self.postcode_field = postcode_field
self.totalCases_field = total_cases_field
self.dateOfLastCase_field = date_of_last_case_field
self.daysSinceLastCase_field = days_since_last_case_field
def run(self):
csv_filename = os.path.join('corona-fallzahlen',
'arcgisInzidenzGemeinden.csv')
current_rows = self.read_csv_rows(csv_filename)
layer = FeatureLayer(
"https://services-eu1.arcgis.com/CZ1GXX3MIjSRSHoC/ArcGIS/rest/services/EBE_Gemeinden_Inzidenztabelle_3/FeatureServer/0"
)
start = time.time()
data = layer.query(order_by_fields='Ort, Datum_Meldung')
print('> Queried data in %.1fs' % (time.time() - start))
if len(data) == 0:
raise Exception('Queried data is empty')
if len(data) < len(current_rows) * (1 / 1.5):
raise Exception(
'Queried data has much less items (%d) than current data (%d)'
% (len(data), len(current_rows)))
if len(data) > len(current_rows) * 1.5:
raise Exception(
'Queried data has much more items (%d) than current data (%d)'
% (len(data), len(current_rows)))
rows = list(
map(
lambda x: {
'datum':
datetime.utcfromtimestamp(x.attributes['Datum_Meldung'] /
1000).strftime('%Y-%m-%d'),
'ort':
x.attributes['Ort'],
'neuPositiv':
str(x.attributes['positiv_neu']),
'inzidenz7tage':
str(round(x.attributes['inzidenz_letzte7Tage'], 2)),
}, data.features))
csv_diff = self.get_csv_diff(csv_filename, rows)
if len(csv_diff) == 0:
return
if self.telegram_bot != None and self.telegram_chat_id != None:
data = ''.join(csv_diff)
self.telegram_bot.send_message(
self.telegram_chat_id,
'```\n' + (data[:4080] if len(data) > 4080 else data) + '```',
parse_mode="Markdown")
self.write_csv_rows(csv_filename, rows)
def __init__(self,
service_url: str,
date_string_field='DateString',
postcode_field='PostCode',
report_date_field='ReportDate',
report_age_field='ReportAge',
oid_field='OBJECTID'):
self.layer = FeatureLayer(service_url)
self.oid_field = oid_field
self.dateString_field = date_string_field
self.postcode_field = postcode_field
self.reportAge_field = report_age_field
self.reportDate_field = report_date_field
def main(arguments):
# initialize logger
logger = initialize_logging(arguments.log_file)
# Create the GIS
logger.info("Authenticating...")
# First step is to get authenticate and get a valid token
gis = GIS(arguments.org_url,
username=arguments.username,
password=arguments.password,
verify_cert=not arguments.skip_ssl_verification)
if not gis.properties.isPortal:
logger.error("This script only works with ArcGIS Enterprise")
sys.exit(0)
logger.info("Getting location tracking service")
try:
tracks_layer = gis.admin.location_tracking.tracks_layer
except Exception as e:
logger.info(e)
logger.info(
"Getting location tracking service failed - check that you are an admin and that location tracking is enabled for your organization"
)
sys.exit(0)
logger.info("Getting polygon layer")
try:
layer = FeatureLayer(url=args.layer_url, gis=gis)
_ = layer._lyr_json
except Exception as e:
logger.info(e)
logger.info(
"Layer could not be found based on given input. Please check your parameters again. Exiting the script"
)
sys.exit(0)
features = layer.query(where=args.where, out_sr=3857).features
if len(features) > 0:
geometries = [feature.geometry for feature in features]
logger.info("Unifying geometry data")
union_geometry = geometry.union(spatial_ref=3857,
geometries=geometries,
gis=gis)
if args.symmetric_difference:
union_geometry['rings'] = form_donut(union_geometry['rings'])
intersect_filter = geometry.filters.intersects(union_geometry, sr=3857)
logger.info("Querying features")
x = tracks_layer.delete_features(geometry_filter=intersect_filter)
logger.info("Deleting features")
logger.info("Deleted: " + str(len(x['deleteResults'])) + " tracks")
logger.info("Completed!")
def main(arguments):
# initialize logger
logger = initialize_logging(arguments.log_file)
# Create the GIS
logger.info("Authenticating...")
# First step is to get authenticate and get a valid token
gis = GIS(arguments.org_url,
username=arguments.username,
password=arguments.password,
verify_cert=not arguments.skip_ssl_verification)
# Get the feature layer
if gis.content.get(arguments.item_id):
logger.info("Getting feature layer")
item = gis.content.get(arguments.item_id)
mirror_layer = item.layers[0]
if arguments.lkl_layer_url:
lkl_layer = FeatureLayer(url=arguments.lkl_layer_url)
else:
logger.info("Please pass an LKL layer url!")
sys.exit(0)
# Query LKL and mirror layer
lkl_fset = lkl_layer.query('1=1', out_sr=3857)
if len(lkl_fset) == 0:
logger.info("No LKLs in your layer yet!")
sys.exit(0)
mirror_fset = mirror_layer.query('1=1', out_sr=3857)
add_features = []
update_features = []
logger.info("Iterating through current LKL data")
for feature in lkl_fset:
for mirror_feature in mirror_fset:
# use "in" instead of == comparison due to the potential for brackets to be in the GUID field
if mirror_feature.attributes[return_field_name(
mirror_layer, "global_id")].lower(
) in feature.attributes["globalid"].lower():
update_features.append(feature)
break
else:
add_features.append(feature)
logger.info("Posting updated data to mirrored layer")
mirror_layer.edit_features(adds=add_features,
updates=update_features,
use_global_ids=True)
logger.info("Completed!")
else:
logger.info("Item not found")
def temporal_accuracy(c_features, curr_url, output_workspace, output_features,
years, curr_gis):
import zipfile
import arcpy
fl = FeatureLayer(url=curr_url, gis=curr_gis)
item = curr_gis.content.get(fl.properties.serviceItemId)
export_item = item.export(export_format='File Geodatabase',
title='CURRENCY')
result = export_item.download(save_path=output_workspace)
folder = os.path.dirname(result)
with zipfile.ZipFile(result, "r") as zip_ref:
zip_ref.extractall(folder)
gdbs = []
for file in zip_ref.namelist():
gdbs.append(os.path.split(file)[0])
gdb = os.path.join(folder, most_common(gdbs))
arcpy.env.workspace = gdb
fc = arcpy.ListFeatureClasses()
feature_class = os.path.join(gdb, fc[0])
temp_acc_calc = ta.TemporalAccuracy(c_features, feature_class,
output_features, years)
temp_acc_calc.create_temporal_accuracy()
def get_features_from_feature_server(url, query):
"""
Given a url to a Feature Server, return a list
of Features (for example, parking lots that are not full)
:param url: url for Feature Server
:param query: query to select features
example: {'where': '1=1', 'out_sr': '4326'}
:return: list of all features returned from the query
"""
features = []
f = FeatureLayer(url=url)
feature_set = f.query(**query)
for feature in feature_set:
features.append(feature.as_dict)
return features
def row_in_feature_layer(row: pd.Series, feature_layer: FeatureLayer) -> bool:
# Null check
if pd.isna(row['pin_longitude']) or pd.isna(row['pin_latitude']):
return False
# Construct a point at the row's coordinates
pin = Point({"x": row['pin_longitude'], "y": row['pin_latitude']})
# construct a geometry filter to check if each point is in a disputed area
pin_filter = intersects(pin)
continue_query = True
retries = 0
MAX_RETRIES = 9
# Default to setting in_disputed_area = True to ensure we never show pins in disputed area
in_disputed_area = True
# Make query to determine whether or not the pin is in the disputed area
# If the query times out, retry with exponential backoff
while continue_query:
try:
in_disputed_area = len(feature_layer.query(geometry_filter=pin_filter).features) > 0
continue_query = False
except Exception as e:
# send slack message if we exceed retry count
if retries > MAX_RETRIES:
body = f'Unable to check if the record with ID {row["source_id"]} is in a disputed region.'
send_slack_message(body, channel='#dev-logging-etl')
continue_query = False
else:
sleep(1.5**(retries))
retries += 1
return in_disputed_area
def layer(self):
"""returns the Parcel Layer for the service"""
if "controllerDatasetLayers" in self._flc.properties and \
"parcelLayerId" in self._flc.properties.controllerDatasetLayers:
url = "%s/%s" % (self._flc.url,
self._flc.properties.controllerDatasetLayers.parcelLayerId)
return FeatureLayer(url=url, gis=self._gis)
return None
def update_database(current, last):
"""
Update the big database from the little one if the little one has new data for the big one.
current = todays data
last = the last row from the big database
Returns True if the database was updated
"""
# If these all match then the latest data
# has already been written to the database
if ((last.new_cases == current.new_cases) \
and (last.total_cases == current.total_cases)
and (last.last_update == current.date)):
print("Database is current.")
# If you want to test the update, comment this out
# and then go in and manually delete the extra row(s).
return False
print("Appending the new record.")
#return False
attributes = {
"utc_date": datetime.utcnow(),
"last_update": current.date,
"editor": os.environ.get('USERNAME') or os.environ.get('USER'),
"source": "CC",
"new_cases" : current.new_cases,
"total_cases" : current.total_cases,
"new_deaths" : current.new_deaths,
"total_deaths" : current.total_deaths,
"name": "Clatsop"
}
gis = GIS(Config.PORTAL_URL, Config.PORTAL_USER, Config.PORTAL_PASSWORD)
layer = FeatureLayer(Config.COVID_CASES_URL, gis)
f = Feature.from_dict({"attributes": attributes})
fs = FeatureSet(features=[f])
results = layer.edit_features(adds=fs)
return True
def get_features_from_feature_server(url, query):
"""
Given a url to a City of Boston Feature Server, return a list
of Features (for example, parking lots that are not full)
:param url: url for Feature Server
:param query: a JSON object (example: { 'where': '1=1', 'out_sr': '4326' })
:return: list of all features returned from the query
"""
logger.debug('url received: ' + url + ', query received: ' + str(query))
features = []
f = FeatureLayer(url=url)
feature_set = f.query(**query)
for feature in feature_set:
features.append(feature.as_dict)
return features
def get_features_from_feature_server(url, query):
"""
Given a url to a City of Boston Feature Server, return a list
of Features (for example, parking lots that are not full)
:param url: url for Feature Server
:param query: query to select features (example: "Spaces > 0")
:return: list of all features returned from the query
"""
logger.debug('url received: ' + url + ', query received: ' + query)
features = []
f = FeatureLayer(url = url)
feature_set = f.query(where = query)
for feature in feature_set:
features.append(feature.as_dict)
return features
def replace_in_ago(**kwargs):
from arcgis.features import FeatureLayer
gis = GIS("https://detroitmi.maps.arcgis.com", Variable.get('AGO_USER'),
Variable.get('AGO_PASS'))
item = gis.content.get(kwargs['id'])
layer_url = f"{item.url}/0"
layer = FeatureLayer(layer_url)
if 'conn_id' in kwargs.keys():
hook = PostgresHook(kwargs['conn_id'])
else:
hook = PostgresHook('etl_postgres')
# Here's a query to make an ArcJSON item from each row in the table
# Note: only works for point geometries right now.
query = f"""
SELECT jsonb_build_object(
'geometry', jsonb_build_object(
'x', ST_X(geom),
'y', ST_Y(geom)
),
'attributes', to_jsonb(row) - 'gid' - 'geom'
) FROM (SELECT * FROM {kwargs['table']}) row
"""
res = hook.get_records(query)
payload = [r[0] for r in res]
# clear out all the rows in the table
layer.manager.truncate()
# write all the rows in `res`
chunk_size = 1000
print(
f"Sending up {len(payload)} features with a batch size of {chunk_size}"
)
for i in range(0, len(payload), chunk_size):
try:
layer.edit_features(adds=payload[i:i + chunk_size])
except:
print(f"Errored on {i} - splitting into 2 batches")
start = i
middle = int(i + (chunk_size / 2))
end = i + chunk_size
print(f"start: {start} middle: {middle} end: {end}")
layer.edit_features(adds=payload[start:middle])
layer.edit_features(adds=payload[middle:end])
def main(arguments):
# initialize logger
logger = initialize_logging(arguments.log_file)
# Create the GIS
logger.info("Authenticating...")
# First step is to get authenticate and get a valid token
gis = GIS(arguments.org_url,
username=arguments.username,
password=arguments.password,
verify_cert=not arguments.skip_ssl_verification)
# Get the feature layer
logger.info("Getting feature layer")
layer = FeatureLayer(arguments.layer_url)
logger.info("Getting tracks layer")
if arguments.tracks_layer_url:
tracks_layer = FeatureLayer(url=arguments.tracks_layer_url)
else:
try:
tracks_layer = gis.admin.location_tracking.tracks_layer
except Exception as e:
logger.info(e)
logger.info(
"Getting location tracking service failed - check that you are an admin and that location tracking is enabled for your organization"
)
sys.exit(0)
# Return invalid work orders
workers = arguments.workers.replace(" ", "").split(",")
invalid_work_orders = get_invalid_work_orders(layer, arguments.field_name,
arguments.time_tolerance,
arguments.distance_tolerance,
arguments.min_accuracy,
workers, tracks_layer,
logger)
if len(invalid_work_orders) == 0:
logger.info("No features found that match the criteria you've set")
else:
for work_order in invalid_work_orders:
logger.info(
f"The user {work_order[0]} who last edited the feature with OBJECTID {work_order[1]} was potentially "
f"not within the distance tolerance when updating the field {arguments.field_name}"
)
def read_local_cases_df():
gis = GIS(Config.PORTAL_URL, Config.PORTAL_USER, Config.PORTAL_PASSWORD)
layer = FeatureLayer(Config.COVID_CASES_URL, gis)
# Be careful, Esri 'where' clauses are very broken.
# It would be elegant if they worked but, they don't.
#sdf = layer.query(where="name=='Clatsop'", out_fields="*").sdf
sdf = pd.DataFrame.spatial.from_layer(layer)
df = sdf[(sdf['name'] == 'Clatsop') & (sdf['source'] != 'worldometer') &
(sdf['source'] != 'OHA')]
#print(df)
return df
def compute_pin_info(df: pd.DataFrame, geo_dfs: dict) -> pd.DataFrame:
# Get record coordinates
df['pin_latitude'], df['pin_longitude'] = \
zip(*df.apply(lambda record: get_record_coordinates(record, geo_dfs), axis=1))
# Populate in_disputed_area col
WHO_FL_URL = "https://services.arcgis.com/5T5nSi527N4F7luB/arcgis/rest/services/DISPUTED_AREAS_mask/FeatureServer/0"
# Create feature layer object
disputed_areas_fl = FeatureLayer(WHO_FL_URL)
# apply row_in_disputed_area across the whole df
df['in_disputed_area'] = df.apply(lambda row: row_in_feature_layer(row, disputed_areas_fl), axis=1)
return df
def __all_events(self):
"""
Fetches all events for particular hub.
"""
events = []
_events_layer = self._gis.content.search(query="typekeywords:hubEventsLayer", max_items=5000)[0]
_events_layer_url = _events_layer.url + '/0'
_events_data = FeatureLayer(_events_layer_url).query().features
for event in _events_data:
events.append(Event(self._gis, event))
return events
def __init__(self, service_url: str,
date_field: str = 'Date',
region_field: str = 'Region',
new_cases_field: str = 'NewCases',
new_deaths_field: str = 'NewDeaths',
new_recoveries_field: str = 'NewRecoveries',
total_cases_field: str = 'TotalCases',
total_deaths_field: str = 'TotalDeaths',
total_recoveries_field: str = 'TotalRecoveries',
active_cases_field: str = 'ActiveCases'):
self.layer = FeatureLayer(service_url)
self.region_field = region_field
self.date_field = date_field
self.new_cases_field = new_cases_field
self.new_deaths_field = new_deaths_field
self.new_recoveries_field = new_recoveries_field
self.total_cases_field = total_cases_field
self.total_deaths_field = total_deaths_field
self.total_recoveries_field = total_recoveries_field
self.active_cases_field = active_cases_field
def run(self):
csv_filename = os.path.join('corona-impfungen', 'arcgisImpfungen.csv')
current_rows = self.read_csv_rows(csv_filename)
layer = FeatureLayer(
"https://services-eu1.arcgis.com/CZ1GXX3MIjSRSHoC/ArcGIS/rest/services/EBE_Gesamtsummen_Impfmeldungen_Öffentlich/FeatureServer/0"
)
start = time.time()
data = layer.query(order_by_fields='Meldedatum')
print('> Queried data in %.1fs' % (time.time() - start))
if len(data) == 0:
raise Exception('Queried data is empty')
if len(data) < len(current_rows) * (1 / 1.5):
raise Exception(
'Queried data has much less items (%d) than current data (%d)'
% (len(data), len(current_rows)))
if len(data) > len(current_rows) * 1.5:
raise Exception(
'Queried data has much more items (%d) than current data (%d)'
% (len(data), len(current_rows)))
rows = list(map(feature_to_row, data.features))
csv_diff = self.get_csv_diff(csv_filename, rows)
if len(csv_diff) == 0:
return
if self.telegram_bot != None and self.telegram_chat_id != None:
data = ''.join(csv_diff)
self.telegram_bot.send_message(
self.telegram_chat_id,
'```\n' + (data[:4080] if len(data) > 4080 else data) + '```',
parse_mode="Markdown")
self.write_csv_rows(csv_filename, rows)
def append_geo_json(geo_json):
gis = GIS("https://www.arcgis.com", username="", password="")
crime_properties = {
'title': 'Crime data',
'tags': 'crimes, open data, devlabs',
'type': 'GeoJson'
}
search_result = gis.content.search(query="", item_type="Feature Layer")
crime_data_item = search_result[0]
crime_data_feature_layer = FeatureLayer.fromitem(crime_data_item,
layer_id=0)
new_crime_set = FeatureSet.from_geojson(geo_json)
crime_data_feature_layer.edit_features(adds=new_crime_set)
def update(config):
lyr = FeatureLayer(url=config['FeatureLyr'])
lyr_properties = lyr.properties
lyr_fields = lyr_properties._mapping['fields']
entity_type = config['EntityType']
uid_field = get_uid_field(entity_type)
for field_dict in lyr_fields:
field_name_upper = field_dict['name'].upper()
if uid_field == field_name_upper:
entity_uid_field_dict = field_dict
if entity_uid_field_dict == None:
return "Configured EntityUid field is not in provided Feature Layer"
else:
query(config, lyr, entity_uid_field_dict)
def execute(self, parameters, messages):
"""The source code of the tool."""
arcpy.SetProgressor("default",
message="Accesing to a destinational resouse")
# Acessing outpud data
token = arcpy.GetSigninToken()
portal_url = arcpy.GetActivePortalURL()
gis = GIS(portal_url, token=token['token'])
layer = FeatureLayer(parameters[0].valueAsText)
arcpy.SetProgressorLabel("Prepearing input data")
#Prepearing input data
feature_set = arcpy.FeatureSet(parameters[1].valueAsText)
feature_set_dict = json.loads(feature_set.JSON)
# Matching parameter
matching = parameters[2].value
# Split features by number of threads
list_of_lists = chunkIt(feature_set_dict['features'],
parameters[3].value)
# List of threads
threads = []
arcpy.SetProgressorLabel("Starting threads")
# Starting threads
for feature_list in list_of_lists:
threads.append(
Thread(target=create_and_append,
args=[
feature_list,
arcpy.GetSigninToken(), portal_url,
parameters[0].valueAsText, matching
]))
threads[-1].start()
# Joining all threads
arcpy.SetProgressorLabel("Executing appendence")
for thread in threads:
thread.join()
return
def main(arguments):
# initialize logger
logger = initialize_logging(arguments.log_file)
# Create the GIS
logger.info("Authenticating...")
# First step is to get authenticate and get a valid token
gis = GIS(arguments.org_url,
username=arguments.username,
password=arguments.password,
verify_cert=not arguments.skip_ssl_verification)
if args.item_id is None and args.layer_url is None:
raise Exception("Must pass either item id or layer url")
if args.item_id:
item = gis.content.get(args.item_id)
if item:
layers = item.layers + item.tables
else:
raise Exception("Bad item id, please check again")
else:
layers = [FeatureLayer(url=args.layer_url, gis=gis)]
logger.info("Downloading attachments")
for layer in layers:
try:
found_attach = layer.attachments.search(args.where)
for attach in found_attach:
try:
layer.attachments.download(attachment_id=attach['ID'], oid=attach['PARENTOBJECTID'], save_path=args.out_folder)
except Exception:
try:
print(f"Failed to download attachment {attach['NAME']} from object id {attach['PARENTOBJECTID']}'")
except Exception as e:
print(e)
except Exception:
pass
logger.info("Completed successfully!")
def process_by_metadata(gis):
return_all_records = False
look_back_days = config.look_back_days
dates = csl.get_dates_in_range(look_back_days)
where_clause = csl.form_query_string(dates)
grid_fl = FeatureLayer(url=config.grid_url)
grid_sdf = grid_fl.query(return_all_records=return_all_records,
where=where_clause).df
geometry = grid_sdf.geometry
sr = {'wkid': 4326}
sp_rel = "esriSpatialRelIntersects"
for idx, row in enumerate(grid_sdf.iterrows()):
geom = row[1].SHAPE
new_geom = Geometry({
"rings":
[[[geom.extent.upperRight.X - .1, geom.extent.lowerLeft.Y + .1],
[geom.extent.lowerLeft.X + .1, geom.extent.lowerLeft.Y + .1],
[geom.extent.lowerLeft.X + .1, geom.extent.upperRight.Y - .1],
[geom.extent.upperRight.X - .1, geom.extent.upperRight.Y - .1],
[geom.extent.upperRight.X - .1, geom.extent.lowerLeft.Y + .1]]],
"spatialReference": {
"wkid": 4326
}
})
grid_filter = filters._filter(new_geom, sr, sp_rel)
sp_filter = filters._filter(geom, sr, sp_rel)
data_fl = FeatureLayer(url=config.features_url)
#out_fields=in_fields,
data_sdf = data_fl.query(geometry_filter=sp_filter,
return_geometry=True,
return_all_records=return_all_records).df
print('Processing Completeness')
#bounding_box = '(37.708132, -122.513617, 37.832132, -122.349607)'
bounding_box = '(' + \
str(geom.extent.lowerLeft.Y) + ',' + \
str(geom.extent.lowerLeft.X) + ',' + \
str(geom.extent.upperRight.Y) + ',' + \
str(geom.extent.upperRight.X) + ')'
osm_sdf = runner.gen_osm_sdf('line',
bounding_box,
osm_tag='highway',
present=True)
completeness_sdf, completeness_fl = comp.completeness(
gis, osm_sdf, data_sdf, config.completeness_url, grid_filter, geom)
print(completeness_sdf)
#update_features(them_acc_sdf, them_acc_fl)
print('Completeness Updated')
print('Processing Logical Consistency')
lc_sdf, lc_fl = lc.logical_consisitency(
gis, config.template_fc, config.template_gdb,
config.attr_check_file, config.attr_check_tab, data_sdf,
config.features_url, config.logical_consistency_url, grid_filter,
geom, config.attr_error_field_count, config.attr_error_field_def)
print(lc_sdf)
update_features(lc_sdf, lc_fl)
print('Logical Consistency Updated.')
print('Processing temporal currency')
tc_sdf, tc_fl = tc.temporal_currency(gis, data_sdf,
config.currency_url, grid_filter,
geom, config.currency_field)
print(tc_sdf)
#update_features(tc_sdf, tc_fl)
print('Temporal Currency Updated')
print('Processing source lineage')
sl_sdf, sl_fl = sl.source_lineage(gis, data_sdf,
config.source_lineage_url,
grid_filter, geom,
config.search_field,
config.value_field)
print(sl_sdf)
#update_features(sl_sdf, sl_fl)
print('Source Lineage Updated')
print('Processing Positional Accuracy')
pa_sdf, pa_fl = pa.positional_accuracy(gis, data_sdf,
config.positional_acc_url,
grid_filter, geom,
config.positional_acc_field)
print(pa_sdf)
#update_features(pa_sdf, pa_fl)
print('Positional Accuracy Updated')
print('Processing Thematic Accuracy')
them_acc_sdf, them_acc_fl = them_acc.thematic_accuracy(
gis, data_sdf, config.thematic_url, grid_filter, geom,
config.thematic_acc_field)
print(them_acc_sdf)
#update_features(them_acc_sdf, them_acc_fl)
print('Theamatic Accuracy Updated')
return
