def match_spatial_reference(self, match_geometry):
"""
Match the spatial reference of the calling geometry to another geometry.
Typically used to get data into single spatial reference for subsequent
analysis.
:param match_geometry: Required arcgis.geometry.Geometry
Another target geometry to ensure the source spatial reference matches to.
:return: arcgis.geometry.Geometry
Same geometry projected to new coordinate system.
"""
# pull the wkid off the inputs
wkid_in = self.spatial_reference['wkid']
wkid_out = match_geometry.spatial_reference['wkid']
# if the spatial references match, don't do anything
if wkid_in == wkid_out:
return self
# get the best applicable transformation, if needed
transformation_list = find_transformation(wkid_in, wkid_out)['transformations']
# if a transformation IS needed, project using it
if len(transformation_list):
out_geom = project([self], wkid_in, wkid_out, transformation_list[0])[0]
# if a transformation IS NOT needed, project without
else:
out_geom = project([self], wkid_in, wkid_out)[0]
# add the spatial reference to the geometry, since it does not have it in the response
out_geom.spatial_reference = match_geometry.spatial_reference
return out_geom
def latlon_to_swiss_batch(latlonPoints):
"""
Takes a list of [lat, lon] (y, x) point sand returns aa list of [E, N] (x, y) Lv03 points.
"""
latlonpts = [arcgeo.Point({"x" : latlonPoint[1], "y" : latlonPoint[0]}) for latlonPoint in latlonPoints]
result_list = arcgeo.project(latlonpts, '4326', '21781')
return [[result['x'], result['y']] for result in result_list]
def latlon_to_swiss(latlonPoint):
"""
Takes a [lat, lon] (y, x) point and returns an [E, N] (x, y) Lv03 point.
"""
result_list = arcgeo.project([arcgeo.Point({"x" : latlonPoint[1], "y" : latlonPoint[0]})], '4326', '21781')
result = result_list[0]
return [result['x'], result['y']]
def swiss_to_latlon_batch(swissPoints):
"""
Takes a list of [E, N] (x, y) Lv03 points and returns a list of [lat, lon] (y, x) points.
"""
chpts = [arcgeo.Point({"x" : swissPoint[0], "y" : swissPoint[1]}) for swissPoint in swissPoints]
result_list = arcgeo.project(chpts, '21781', '4326')
return [[result['y'], result['x']] for result in result_list]
def web_mercator_to_geographic(x, y):
input_geometry = {'y': float(y), 'x': float(x)}
output_geometry = geometry.project(geometries=[input_geometry],
in_sr=3857,
out_sr=4326,
gis=geoplatform)
return output_geometry[0]
def geographic_to_web_mercator(lon, lat):
input_geometry = {'y': float(lat), 'x': float(lon)}
output_geometry = geometry.project(geometries=[input_geometry],
in_sr=4326,
out_sr=3857,
gis=geoplatform)
return output_geometry[0]
def project_all_geometries(nexrad_data):
geometries = [{'y': i[7], 'x': i[8]} for i in nexrad_data]
# ArcGIS Online seems to be expecting coordinates in WebMercator
projected_geometries = geometry.project(geometries=geometries,
in_sr=4326,
out_sr=3857,
gis=geoplatform)
return projected_geometries
def convert_with_GIS(input_geometry):
gis = arcgis.GIS()
time_start = time.time()
output_geometry = arcgeo.project(geometries=input_geometry,
in_sr=3857,
out_sr=4326)
time_end = time.time()
print(output_geometry)
print("Entire operation took {} seconds.".format(time_end - time_start))
def build_feats(df):
# get a template feature object
inc_fset = inc_layer.query()
template_feature = deepcopy(inc_fset.features[0])
features_to_be_added = []
for row in df.iterrows():
new_feature = deepcopy(template_feature)
# print("Creating " + row[1]['name'])
#get geometries in the destination coordinate system
input_geometry = {
'y': float(row[1]['latitude']),
'x': float(row[1]['longitude'])
}
output_geometry = geometry.project(
geometries=[input_geometry],
in_sr=4326,
out_sr=inc_fset.spatial_reference['latestWkid'],
gis=gis)
# assign the new incident values
new_feature.geometry = output_geometry[0]
new_feature.attributes['incidentNo'] = row[1]['incidentNo']
new_feature.attributes['lastUpdateDateTime'] = row[1]['lastUpdatedDt']
new_feature.attributes['originDateTime'] = row[1]['originDateTime']
new_feature.attributes['incidentType'] = row[1]['incidentType']
new_feature.attributes['incidentLocation'] = row[1]['incidentLocation']
new_feature.attributes['incidentStatus'] = row[1]['incidentStatus']
new_feature.attributes['incidentSize'] = row[1]['incidentSize']
new_feature.attributes['name'] = row[1]['name']
new_feature.attributes['territory'] = row[1]['territory']
new_feature.attributes['resourceCount'] = row[1]['resourceCount']
new_feature.attributes['eventCode'] = row[1]['eventCode']
new_feature.attributes['fireDistrict'] = row[1]['fireDistrict']
new_feature.attributes['municipality'] = row[1]['municipality']
new_feature.attributes['category1'] = row[1]['category1']
new_feature.attributes['category2'] = row[1]['fireDistrict']
new_feature.attributes['fireDistrict'] = row[1]['category2']
new_feature.attributes['agency'] = row[1]['agency']
new_feature.attributes['originStatus'] = row[1]['originStatus']
new_feature.attributes['type'] = row[1]['type']
#new_feature.attributes['lastUpdatedDt'] = int(row[1]['lastUpdatedDt'])
new_feature.attributes['lastUpdatedDtStr'] = row[1]['lastUpdatedDtStr']
new_feature.attributes['originDateTimeStr'] = row[1][
'originDateTimeStr']
features_to_be_added.append(new_feature)
return features_to_be_added
def project_as(self, output_spatial_reference):
"""
Project the geometry to another spatial reference - automatically
applying a transformation if necessary.
:param output_spatial_reference: Required - SpatialReference
Spatial reference object defining the output spatial reference.
:return: Geometry object in new spatial reference.
"""
# get the wkid for the input and output
if type(output_spatial_reference) == int:
wkid_out = output_spatial_reference
elif type(output_spatial_reference) == SpatialReference:
wkid_out = SpatialReference['wkid']
else:
raise Exception('Valid output spatial reference must be provided.')
wkid_in = self.spatial_reference['wkid']
# if the spatial references match, don't do anything
if wkid_in == wkid_out:
return self
# get the best applicable transformation, if needed
transformation_list = find_transformation(wkid_in, wkid_out)['transformations']
# if a transformation IS needed, project using it
if len(transformation_list):
out_geom = project([self], wkid_in, wkid_out, transformation_list[0])[0]
# if a transformation IS NOT needed, project without
else:
out_geom = project([self], wkid_in, wkid_out)[0]
# add the spatial reference to the geometry, since it does not have it in the response
out_geom.spatial_reference = SpatialReference(wkid=wkid_out)
return out_geom
def process_hfl(event_layer, gis):
# Create Name with Time to Avoid Duplicate Errors When Publishing from CSV
output_name = f'GDELT_Extract_{round(time.time())}'
csv_out = f'{os.path.dirname(mod_gdelt.root_path)}/scratch/{output_name}.csv'
try:
# Wait for Newly Published Service to be Ready
if not wait_for_service(event_layer):
raise Exception('Could Not Contact Service That Prompted Route')
else:
print(f'Processing: {event_layer.properties.name}')
# Collect Lastest GDELT Data as a Spatial Data Frame
gdelt_df = Extractor().get_v2()
# Dissolve Trigger Event Layer into Single Geometry & Project to Decimal Degrees
layer_df = dissolve_boundaries(event_layer).query().sdf
layer_df.SHAPE = project(layer_df.SHAPE.to_list(), '3857', '4326', gis=gis)
# Determine Intersections
gdelt_df.SHAPE = intersect('4326', gdelt_df.SHAPE.to_list(), layer_df.SHAPE.to_list()[0])
gdelt_df['Valid'] = gdelt_df.SHAPE.apply(lambda x: x.get('x') != 'NaN')
gdelt_df = gdelt_df[gdelt_df['Valid']]
# Create Name with Time to Avoid Duplicate Errors When Publishing from CSV
output_name = f'GDELT_Extract_{round(time.time())}'
# Export Data Frame to Local CSV
gdelt_df.to_csv(csv_out, index=False)
# Publish Hosted Feature Layer from CSV
print('Publishing GDELT Data')
prp = {'type': 'CSV', 'title': output_name}
itm = gis.content.add(item_properties=prp, data=csv_out)
res = itm.publish(publish_parameters={
'name': f'GDELT_{round(time.time())}',
'longitudeFieldName': 'ActionGeo_Long',
'latitudeFieldName': 'ActionGeo_Lat',
'locationType': 'coordinates',
'type': 'csv'
})
print(f'GDELT Data Published: {res}')
return res
finally:
if os.path.exists(csv_out):
os.unlink(csv_out)
def find_objects(model, tiles, extent, crop=(0, 0, 0, 0), cycle=1):
xmin = extent['xmin'] + crop[0]
ymin = extent['ymin'] + crop[1]
xmax = extent['xmax'] - crop[2]
ymax = extent['ymax'] - crop[3]
img_normed = norm(tiles)
clas, bbox = predict_(model, img_normed.transpose(0, 3, 1, 2))
preds = {}
for idx in range(bbox.size()[0]):
preds[idx] = get_nms_preds(clas, bbox, idx, anchors)
w, h = 90, 90
pools = []
numboxes = bbox.size()[0]
side = math.sqrt(numboxes)
for idx in range(numboxes):
i, j = idx // side, idx % side
x = xmin + (j) * w
y = ymax - (i + 1) * h
for pred in preds[idx]:
objx = x + ((pred['x1'] + pred['x2']) / 2) * w
objy = y + h - ((pred['y1'] + pred['y2']) / 2) * h
pools.append({
'x': objx,
'y': objy,
'category': pred['category'],
'score': pred['score']
})
if len(pools) > 0:
result = project(geometries=pools, in_sr=3857, out_sr=4326)
for i, p in enumerate(pools):
p['SHAPE'] = dict(result[i])
p['cycle'] = cycle
return pools
def find_objects(model, tiles, extent, crop=(0, 0, 0, 0), cycle=1):
xmin = extent['xmin'] + crop[0]
ymin = extent['ymin'] + crop[1]
xmax = extent['xmax'] - crop[2]
ymax = extent['ymax'] - crop[3]
img_normed = norm(tiles)
clas, bbox = predict_(model, img_normed.transpose(0, 3, 1, 2))
preds = { }
for idx in range(bbox.size()[0]):
preds[idx] = get_nms_preds(clas, bbox, idx, anchors)
w, h = 90, 90
pools = []
numboxes = bbox.size()[0]
side = math.sqrt(numboxes)
for idx in range(numboxes):
i, j = idx//side, idx%side
x = xmin + (j)*w
y = ymax - (i+1)*h
for pred in preds[idx]:
objx = x + ((pred['x1'] + pred['x2']) / 2)*w
objy = y + h - ((pred['y1'] + pred['y2']) / 2)*h
pools.append({'x': objx, 'y': objy, 'category': pred['category'], 'score': pred['score']})
if len(pools) > 0:
result = project(geometries=pools, in_sr=3857, out_sr=4326)
for i, p in enumerate(pools):
p['SHAPE'] = dict(result[i])
p['cycle'] = cycle
return pools
def project_geom(self, geoms, in_sr, our_sr):
return project(geoms, in_sr, our_sr)
"outStatisticFieldName": "SUM_ELDERLY"
}, {
"statisticType": "sum",
"onStatisticField": "E_SNGPNT",
"outStatisticFieldName": "SUM_SINGLE"
}, {
"statisticType": "sum",
"onStatisticField": "E_NOVEH",
"outStatisticFieldName": "SUM_NOVEH"
}]
data.delete_features(where='1=1')
for feature in moderate.features:
weather_geom = feature.geometry
weather = [weather_geom]
weather_proj = project(geometries=weather,
in_sr=102100,
out_sr=4269,
transform_forward=True)
svi_weather_selection = svi.query(where="1=1",
out_statistics=stats,
geometry_filter=intersects(
weather_proj[0]))
statistics = svi_weather_selection.features[0]
proj = str(weather_proj[0])
proj = '{"rings"' + proj[8:]
statistics = str(statistics)
statistics = statistics[0:-1]
statistics = "" + statistics + ', "geometry": ' + proj + '}'
statistics = json.loads(statistics)
data.edit_features(adds=[statistics])
for feature in severe.features:
weather_geom = feature.geometry
toroletype, uniquets)
linkobjs.append(linkobj)
header = ["RDFID", "ENTITY", "TIMEST", "RDFLINKS", "TYPE", "SUBTYPE", "ORGDOC", "UNIQUEID", "LINKDETAILS", 'Long', 'Lat']
rows = []
filePath = web_markUp + file
for r in rdfobjsGeo:
features_to_be_added = []
new_feature = deepcopy(geo_template_feature)
input_geometry = {'y':r.lat,
'x':r.long}
output_geometry = geometry.project(geometries = [input_geometry],
in_sr=geo_fset.spatial_reference['latestWkid'],
out_sr=geo_fset.spatial_reference['latestWkid'],
gis=gis)
new_feature.geometry = output_geometry[0]
new_feature.attributes['rdfid'] = r.id
new_feature.attributes['rdfvalue'] = r.value
new_feature.attributes['timest'] = r.timest
# need less than 23 links or fc won't accept list (255 char max)
if len(r.links) > 23:
new_feature.attributes['rdflinks'] = r.links[0:22]
else:
new_feature.attributes['rdflinks'] = r.links
new_feature.attributes['type'] = r.type
new_feature.attributes['subtype'] = r.subtype
new_feature.attributes['orgdoc'] = r.orgdoc
updatefeatures = []
all_features = flquery.features
# for loop to prepare updated geometries and attributes for each of the updated features
# geometry module used to project coordinates form geographic to projected coordinate system
for fid in overlap_rows['FID']:
# get the feature to be updated
original_feature = [f for f in all_features if f.attributes['FID'] == fid][0]
ftbu = deepcopy(original_feature)
# get the matching row from csv
matching_row = updates_df_2.where(updates_df_2.FID == fid).dropna()
# get geometries in the destination coordinate system
input_geometry = {'y':float(matching_row['']),
'x':float(matching_row[''])}
output_geometry = geometry.project(geometries = [input_geometry],
in_sr = 4326,
out_sr = flquery.spatial_reference['latestWkid'],
gis = gis)
# assign the updated values
ftbu.geometry = output_geometry[0]
ftbu.attributes[''] = matching_row[''].values[0]
ftbu.attributes[''] = int(matching_row[''])
# add this to the list of features to be updated
updatefeatures.append(ftbu)
# call the edit_features() method of FeatureLayer object and pass features to the updates parameter
if len(updatefeatures) > 0:
fl.edit_features(updates = updatefeatures)
###########################################################
## SECTION 4: CLEAN UP DATA WITHIN SQL AND LOCAL MACHINE ##
###########################################################
def updating_feature():
overlap_rows = pandas.merge(left=query.df,
right=local,
how='inner',
on='forecast_date2')
print(overlap_rows
) # связывание онлайн и локальной таблицы по "дата прогноза 2"
features_for_update = [
] # объявление списка для наполнения обновляемыми строками
all_features = query.features #создание шаблона обновляемой строки
i = 0 #
for forecast_date2 in overlap_rows['forecast_date2']: #
features_for_update = []
original_feature = [
f for f in all_features
if f.attributes['forecast_date2'] == forecast_date2
][0] #Получение строки которую надо обновить
feature_to_be_updated = deepcopy(original_feature) #
matching_row = local.where(
local.forecast_date2 == forecast_date2).dropna(
) #Вытаскивание из фрейма совпадающей строки
feature_to_be_updated.attributes['temp'] = float(matching_row['temp'])
feature_to_be_updated.attributes['temp_min'] = int(
matching_row['temp_min']
) #Подмена значений в шаблонй строке на новую
feature_to_be_updated.attributes['temp_max'] = int(
matching_row['temp_max'])
feature_to_be_updated.attributes['request_date'] = str(
matching_row['request_date'].values[0])
feature_to_be_updated.attributes['pressure'] = int(
matching_row['pressure']) #
feature_to_be_updated.attributes['pressure_g_lvl'] = int(
matching_row['pressure_g_lvl']) #
feature_to_be_updated.attributes['wind_speed'] = float(
matching_row['wind_speed'])
feature_to_be_updated.attributes['wind_degree'] = str(
(matching_row['wind_degree'].values[0])) #
feature_to_be_updated.attributes['clouds'] = int(
matching_row['clouds']) #
feature_to_be_updated.attributes['weather_description'] = str(
(matching_row['weather_description'].values[0]))
feature_to_be_updated.attributes['rain'] = float(
matching_row['rain']) #
feature_to_be_updated.attributes['snow'] = float(
matching_row['snow']) #
feature_to_be_updated.attributes['forecast_date'] = str(
matching_row['forecast_date'].values[0])
features_for_update.append(
feature_to_be_updated) #создание списка обновляемых строк
i = i + 1
print(i)
print(features_for_update)
print('====================================')
weatherD.edit_features(updates=features_for_update) #применение обновлений
new_rows = local[~local['forecast_date2'].
isin(overlap_rows['forecast_date2'])]
print(new_rows) # идентификация строк отсутсвующих в онлайне
features_to_be_added = [] #объявление списка строк, которые надо добавить
template_feature = deepcopy(features_for_update[0]) #копирование шаблона
i = 0
for row in new_rows.iterrows(
): #для каждой строки в фрейме создание строки
i = i + 1
features_to_be_added = []
new_feature = deepcopy(template_feature)
print(
'creating {}'.format(i)
) #создание геометрии для новой строки, если залить без этой части, отображаться новые точки не будут
input_geometry = {'y': float(row[1]['lat']), 'x': float(row[1]['lon'])}
output_geometry = geometry.project(
geometries=[input_geometry],
in_sr=4326,
out_sr=
4326, #тут значение 4326 можно заменить на строку query.spatial_reference['latestWkid'],
gis=gis
) #в таком случае, он перестроит геометрию под онлайн слой, если она отличается от геометрии проекта, но если слой изначально пуст, то выйдет на ошибку
new_feature.geometry = output_geometry[0] #запись геометрии в строку
new_feature.attributes['lat'] = float(
row[1]['lat']
) #Подстановка новых значений в отправляемую строку, поля широты и долготы в таком случае не особо актуальны, поскольку геометрия уже задана
new_feature.attributes['lon'] = float(row[1]['lon'])
new_feature.attributes['city_id'] = int(row[1]['city_id'])
new_feature.attributes['temp'] = float(row[1]['temp'])
new_feature.attributes['temp_min'] = int(row[1]['temp_min'])
new_feature.attributes['temp_max'] = int(row[1]['temp_max'])
new_feature.attributes['pressure'] = int(row[1]['pressure'])
new_feature.attributes['pressure_s_lvl'] = int(
row[1]['pressure_s_lvl'])
new_feature.attributes['pressure_g_lvl'] = int(
row[1]['pressure_g_lvl'])
new_feature.attributes['wind_speed'] = float(row[1]['wind_speed'])
new_feature.attributes['forecast_date2'] = str(
row[1]['forecast_date2'])
new_feature.attributes['clouds'] = int(row[1]['clouds'])
new_feature.attributes['request_date'] = str(row[1]['request_date'])
new_feature.attributes['forecast_date'] = str(
row[1]['forecast_date']
) # время в данном случае отправляется в виде строки, и в онлайне автоматом разбивается на датувремя с учётом часовых поясов
new_feature.attributes['rain'] = float(row[1]['rain'])
new_feature.attributes['snow'] = float(row[1]['snow'])
new_feature.attributes['wind_degree'] = str(row[1]['wind_degree'])
new_feature.attributes['name'] = str(row[1]['name'])
features_to_be_added.append(
new_feature) #наполнение списка строками, которые нужно отправить
print(features_to_be_added)
print("============================================================")
weatherD.edit_features(adds=features_to_be_added) # отправка
