def main():
# Specify the source organization containing the items to clone.
# If the item to be cloned is shared with everyone and is on arcgis online you don't need to change anything below.
# If the item is on a local portal or not shared you need to provide the url to the portal and the username and password.
source = gis.GIS()
# Specify the target portal, provide the url and the username/password to the organization or portal you want to clone the items to.
# To use the active portal in Pro set the first parameter to 'pro'
target = gis.GIS('http://www.arcgis.com/', '', '')
# Provide the list of item's ids to clone.
# If it is an application or map, you don't need to provide the item id's of the hosted feature layers used by the map or the map used by the application.
# These item dependencies will automatically be cloned as well.
item_ids = [
'c43607465cdb11e7907ba6006ad3dba0', 'fb1942765cfb11e7907ba6006ad3dba0'
]
# Optionally specify whether the data from the original feature service or feature collection should be copied to the cloned item.
# The default is False
clone_items.COPY_DATA = False
# Optionally specify whether to search the org to see if the item has already been cloned, and if so use it instead of cloning a duplicate.
# This is common if you have multiple web maps that share the same service. Rather than creating the service again it will find the already cloned version.
# If multiple versions of the item have been cloned to the org, the most recently created item will be used.
# The default is True
clone_items.SEARCH_ORG_FOR_EXISTING_ITEMS = True
# Optionally specify whether the organization's default basemap should be used when creating new web maps.
# The default is False
clone_items.USE_DEFAULT_BASEMAP = False
# Optionally specify whether gps metadata fields used by collector should be added to any new feature services.
# The default is False
clone_items.ADD_GPS_METADATA_FIELDS = False
# Optionally provide an extent to set the output extent of the cloned items.
# A string representing the desired extent for new items. The string should be formatted as 'XMin, YMin, XMax, YMax' and the coordinates should be in WGS84.
# Example '-180, -90, 180, 90'. If not provided the new item will have the same extent as the original item.
# The default is None
clone_items.ITEM_EXTENT = None
# Optionally provide a well known id, for example 3857 for Web Mercator, to set the output spatial reference of any cloned feature services.
# This parameter requires the arcpy module. If not provided or if the arcpy module is not available the service will have the same spatial reference as the original service.
# The default is None
clone_items.SPATIAL_REFERENCE = None
created_items = []
for item_id in item_ids:
# Get the item
item = source.content.get(item_id)
print('Cloning {0}'.format(item['title']))
# Specify the name of the folder to clone the items to. If a folder by the name doesn't already exist a new folder will be created.
folder_name = "Output"
# Clone the item to the target portal. The function will return all the new items that were created during the cloning.
created_items += clone_items.clone(target, item, folder_name,
created_items)
def agoSignIn(cred_json):
sites = readConfig(cred_json)
for site in sites:
if site['name'].lower() == 'bc maphub':
params = site['params']
ago = gis.GIS(params['mapurl'], params['usr'], params['password'])
return ago
def selectproject():
arc = gis.GIS(username=session.get('arcuser'),
password=session.get('arcsenha'))
mytoken = session.get('mytoken')
listaprojetos = session.get('listaprojetos')
form = ProjectForm()
form.selecionaprojeto.choices = listaprojetos
if form.validate_on_submit():
items = arc.content.search(
query="NOT title: %stakeholder% AND NOT title: %fieldworker% AND "
+ "owner:" + arc.users.me.username + " AND Survey",
item_type="Feature Layer",
max_items=500)
session['projectname'] = str(form.selecionaprojeto.data)
flash(
f'Projeto ' + session.get('projectname') +
' selecionado com sucesso!', 'success')
return redirect(url_for('selectcategory', mytoken=session['mytoken']))
else:
flash(
'Projeto não pode ser selecionado. Verifique se há registros na camada.',
'danger')
return render_template('selectprojecto.html',
title='SelectProject',
mytoken=mytoken,
form=form)
def validateinventsys():
form = LoginFormInventsys()
if form.validate_on_submit():
arc = gis.GIS(username=form.username.data, password=form.password.data)
mytoken = generate_random_string()
session['mytoken'] = mytoken
if arc:
session['arcuser'] = str(form.username.data)
session['arcsenha'] = str(form.password.data)
items = arc.content.search(
query=
"NOT title: %stakeholder% AND NOT title: %fieldworker% AND " +
"owner:" + arc.users.me.username + " AND Survey",
item_type="Feature Layer",
max_items=500)
listaprojetos = []
for item in items:
listaprojetos.append(item.title)
session['listaprojetos'] = listaprojetos
flash(f'Login realizado com sucesso para {form.username.data}!',
'success')
return redirect(url_for('selectproject', mytoken=mytoken))
else:
flash('Login não realizado. Verifique o usuário e a senha.',
'danger')
return render_template('logininventsys.html',
title='LoginInventsys',
form=form)
def convert_with_GIS(input_geometry):
gis = arcgis.GIS()
time_start = time.time()
output_geometry = arcgeo.project(geometries=input_geometry,
in_sr=3857,
out_sr=4326)
time_end = time.time()
print(output_geometry)
print("Entire operation took {} seconds.".format(time_end - time_start))
def connect(instance, username=None, password=None):
try:
instance_item = Portal.objects.get(alias=instance)
url = instance_item.url
token_expiration = instance_item.token_expiration
if username and password:
target = gis.GIS(url, username, password)
elif token_expiration is None or token_expiration < datetime.datetime.now() or instance_item.portal_type == "agol":
instance_username = instance_item.username
instance_password = instance_item.password
target = gis.GIS(url, instance_username, instance_password)
instance_item.token = target._con.token
instance_item.token_expiration = datetime.datetime.now() + datetime.timedelta(
minutes=target._con._expiration)
instance_item.save()
else:
instance_token = instance_item.token
target = gis.GIS(url, token=instance_token)
return target
except Exception as e:
return False
def try_connection(instance):
try:
url = instance['url']
instance_username = instance['username']
instance_password = instance['password']
target = gis.GIS(url, instance_username, instance_password)
if target.properties.user.username == instance_username:
return True
except:
message_type = "error"
result = ""
return False
if creDateElement is not None:
deleteElem(metaTag["createdate"])
tree.write(xmlfile)
else:
print("# XML file not found, remove item from the list")
itemlist.pop(counter)
xmlpathlist.pop(counter)
print("\n\n****************************************")
print("Enter AGOL Username and Password")
# agol gis container
agousername = input("Username: "******"Password: "******" ")
for counter2, item in enumerate(itemlist):
# print(item, xmlpathlist[counter2])
itemIDtemp = item
agolitem = gis.Item(gisContainer, itemid=itemIDtemp)
print("\n## {} of {}: uploading metadata of {}".format(
counter2 + 1, len(itemlist), agolitem.title))
itemProperties = {'typeKeywords': 'Metadata'}
# xml file upload
agolitem.update(item_properties=itemProperties,
metadata=xmlpathlist[counter2])
#Script to back up feature layer as csv
from datetime import date
from arcgis import gis, features
import getpass
today = date.today().strftime('%Y%m%d')
username = '******'
password = getpass.getpass()
print('Connecting to AGOL')
#Connect to AGOL org
gis = gis.GIS('https://learngis2.maps.arcgis.com', username, password)
#Set to feature service URL
#*****************************************************************************************************************************************************
f_lyr_URL = "https://services3.arcgis.com/U26uBjSD32d7xvm2/arcgis/rest/services/SF_GIS_Data/FeatureServer/0"
#*****************************************************************************************************************************************************
print('Creating feature layer and spatial dataframe')
#Create feature layer of feature services
f_lyr = features.FeatureLayer(f_lyr_URL, gis)
#Create spatial dataframes
f_lyr_sdf = features.GeoAccessor.from_layer(f_lyr)
print('Exporting spatial dataframe as a csv file')
#Export spatial dataframes as csv files
except:
print("failed to parse configuration")
else:
return d
logging.debug("Config Loaded")
ts = makeTimeStamp()
print("Time Stamp: {}".format(ts))
sites = readConfig(secrets)
for site in sites:
if site['name'].lower() == 'bc maphub':
params = site['params']
mh = gis.GIS(params['mapurl'], params['usr'], params['password'])
contents = mh.content.search(query="owner:{}".format(params['usr']))
items = []
for item in contents:
#print (f"Name:{item['name']} Id: {item['id']}")
items.append(item['id'])
usr_groups = mh.groups.search(query="owner:{}".format(params['usr']))
group_of_interest = mh.groups.search(query='title: "{}"'.format(grp_title))
list_by_owner = False
list_by_group_of_interest = False
if list_by_owner == True:
for group in usr_groups:
print("* {}:\n {}".format(group.title, group.get_members()))
from arcgis import gis, geocoding
from airflow.models import Variable
from airflow.hooks.postgres_hook import PostgresHook
ago = gis.GIS("https://detroitmi.maps.arcgis.com", Variable.get('AGO_USER'),
Variable.get('AGO_PASS'))
geocoders = {
'composite': geocoding.get_geocoders(ago)[0],
'address': geocoding.get_geocoders(ago)[1],
'centerline': geocoding.get_geocoders(ago)[2]
}
def geocode_rows(**kwargs):
from arcgis import geocoding
# Expected **kwargs:
# table: name of table to geocode
# address_column: name of address column or expression
# geometry_column: name of geometry column
# parcel_column: name of parcel column
# where: a Postgres clause to restrict which rows get geocoded
# geocoder: enum ['composite', 'address', 'centerline']. default to composite
hook = PostgresHook('etl_postgres')
# default where clause
where = kwargs['where'] if 'where' in kwargs.keys() else "1 = 1"
geocoder = kwargs['geocoder'] if 'geocoder' in kwargs.keys(
) else 'composite'
from arcgis import gis, geocoding
from os import environ as env
from .utils import connect_to_pg
import sqlalchemy
ago = gis.GIS("https://detroitmi.maps.arcgis.com", env['AGO_USER'],
env['AGO_PASS'])
geocoders = {
'composite': geocoding.get_geocoders(ago)[0],
'address': geocoding.get_geocoders(ago)[1],
'centerline': geocoding.get_geocoders(ago)[2]
}
class GeocodeTable(object):
def __init__(self,
table,
addr_col='address',
geom_col='geom',
parcel_col=None,
where_clause="1=1",
geocoder='composite'):
self.table = table
self.addr_col = addr_col
self.geom_col = geom_col
self.parcel_col = parcel_col
self.where_clause = where_clause
self.geocoder = geocoders[geocoder]
def geocode_rows(self):
def loginpostgis():
mytoken = session.get('mytoken')
projectname = session.get('projectname')
datainicio = session.get('inicio')
datafim = session.get('fim')
arc = gis.GIS(username=session.get('arcuser'),
password=session.get('arcsenha'))
items = arc.content.search(
query="NOT title: %stakeholder% AND NOT title: %fieldworker% AND " +
"owner:" + arc.users.me.username + " AND Survey",
item_type="Feature Layer",
max_items=500)
item_to_add = [
temp_item for temp_item in items
if temp_item.title == session.get('projectname')
]
project = item_to_add[0].layers[0].properties['serviceItemId']
session['project'] = project
if item_to_add[0].layers[0].properties[
'geometryType'] == 'esriGeometryPoint':
registrosbruto = pd.DataFrame.spatial.from_layer(
item_to_add[0].layers[0])
registros = []
for i in range(0, len(registrosbruto)):
ano = int(str(registrosbruto.iloc[i]['CreationDate'])[0:4])
mes = int(str(registrosbruto.iloc[i]['CreationDate'])[5:7])
dia = int(str(registrosbruto.iloc[i]['CreationDate'])[8:10])
dataobjeto = datetime.date(ano, mes, dia)
if dataobjeto < datafim:
registros.append(registrosbruto.iloc[i])
ano = datafim.year
mes = datafim.month
if mes < 10:
dataref = str(ano) + '0' + str(mes)
else:
dataref = str(ano) + str(mes)
session['dataref'] = dataref
form = LoginFormPostgis()
if form.validate_on_submit():
session['hostinput'] = str(form.hostinput.data)
session['dbnameinput'] = str(form.dbnameinput.data)
session['userinput'] = str(form.userinput.data)
session['senhainput'] = str(form.senhainput.data)
#Define our connection string
conn_string = "host=" + str(
session.get('hostinput')) + " dbname=" + str(
session.get('dbnameinput')) + " user="******" password="******"""CREATE EXTENSION IF NOT EXISTS postgis;
DROP TABLE IF EXISTS {}"""
nometabela = dataref + '_' + projectid
createdbgenerica = """CREATE UNLOGGED TABLE IF NOT EXISTS {}(
id integer PRIMARY KEY,
created_at DATE,
updated_at DATE,
latitude real,
longitude real,
geom geometry(Point, 4326)
);"""
cur.execute(
sql.SQL(dropdbgenerica).format(sql.Identifier(nometabela)))
cur.execute(
sql.SQL(createdbgenerica).format(sql.Identifier(nometabela)))
tabelagerada = dataref + '_' + projectid
conn.commit()
for item in registros:
genericfields = [
str(item['objectid']),
str(item['CreationDate']),
str(item['EditDate']),
float(item['SHAPE']['y']),
float(item['SHAPE']['x'])
]
my_data = [field for field in genericfields]
cur.execute(
sql.SQL(
"INSERT INTO {} VALUES (%s, %s, %s, %s, %s)").format(
sql.Identifier(nometabela)), tuple(my_data))
conn.commit()
nomeindex = tabelagerada + 'index'
cur.execute(
sql.SQL(
"UPDATE {} SET geom = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326); CREATE INDEX {} ON {} USING GIST(geom)"
).format(sql.Identifier(tabelagerada),
sql.Identifier(nomeindex),
sql.Identifier(tabelagerada)))
conn.commit()
dbsegmentos = """DROP TABLE IF EXISTS {};
CREATE UNLOGGED TABLE IF NOT EXISTS {}(
nome TEXT PRIMARY KEY,
geom geometry(MultiPolygon, 4326)
);"""
segmentnome = "egrfauna_segmentos"
cur.execute(
sql.SQL(dbsegmentos).format(sql.Identifier(segmentnome),
sql.Identifier(segmentnome)))
conn.commit()
segmentos = requests.get(
'https://raw.githubusercontent.com/guilhermeiablo/survey2infoambiente/master/dados/ERS_segmentos_rodoviarios.geojson'
)
for feature in segmentos.json()['features']:
geom = (json.dumps(feature['geometry']))
nome = feature['properties']['nome']
cur.execute(
sql.SQL(
"INSERT INTO {} (nome, geom) VALUES (%s, ST_SetSRID(ST_GeomFromGeoJSON(%s), 4326));"
).format(sql.Identifier(segmentnome)), (nome, geom))
cur.execute(
sql.SQL(
"CREATE INDEX sp_index_segmentos ON {} USING GIST(geom)").
format(sql.Identifier(segmentnome)))
conn.commit()
intersecta = '''DROP TABLE IF EXISTS {nome0}; SELECT {nome1}.*, {nome2}.nome INTO {nome0} FROM {nome2} INNER JOIN {nome1} ON ST_Intersects({nome2}.geom, {nome1}.geom) AND {nome2}.nome=%s;'''
for feature in segmentos.json()['features']:
nomedosegmento = feature['properties']['nome']
if projectid == '10762':
nomecompleto = str(feature['properties']['nome'] +
'_PMF_' + tabelagerada)
else:
nomecompleto = str(feature['properties']['nome'] + '_' +
tabelagerada)
cur.execute(
sql.SQL(intersecta).format(
nome0=sql.Identifier(nomecompleto),
nome1=sql.Identifier(tabelagerada),
nome2=sql.Identifier(segmentnome)), [
nomedosegmento,
])
conn.commit()
session['tabelagerada'] = str(tabelagerada)
return redirect(
url_for('logingeoserver',
mytoken=session['mytoken'],
project=session['project']))
else:
flash('Erro ao conectar a base de dados. Tente novamente.',
'danger')
return render_template('loginpostgis.html',
title='LoginPostgis',
form=form,
mytoken=mytoken,
project=session['project'])
