def __init__(self, id, api='https://api.openstreetmap.org'):
self.id = id
logger.debug('Using api={}'.format(api))
if api:
self.osmapi = OsmApi(api=api)
else:
self.osmapi = None
self.meta = None
self.changes = None
# History for modified/deleted elements, inner dicts indexed by object id
self.history_one_version_back = True
self.hist = {'node': {}, 'way':{}, 'relation':{}}
# Summary of elemets, created, modified, deleted. '_' versions are summarized across all object types
self.summary = {'create' : { 'node': 0, 'way':0, 'relation':0, 'relation_tags':{}},
'modify' : { 'node': 0, 'way':0, 'relation':0, 'relation_tags':{}},
'delete' : { 'node': 0, 'way':0, 'relation':0, 'relation_tags':{}},
'_create':0, '_modify':0, '_delete':0}
# Tag changes
self.tagdiff = self.getEmptyDiffDict()
# Tags unchanged, i.e. mostly ID if object geometrically changed
self.tags = {}
# Textual diff description
self.diffs = {}
# Simple (no tags) nodes
self.simple_nodes = {'create':0, 'modify':0, 'delete':0}
self.other_users = None
self.mileage = None
self.apidebug = False
self.datadebug = False
def getWay(id):
#time is started
start_time = time.time()
MyApi = OsmApi()
#print("The Way are : ", MyApi.WayGet(id,WayVersion=-1))
ways = MyApi.WayGet(id, WayVersion=-1)
#print("Time taken in the Process --- %s seconds ---" % (time.time() - start_time))
return ways
def getNode(id, name):
#time is started
start_time = time.time()
MyApi = OsmApi()
#print("The node is : ", MyApi.NodeGet(id))
node = MyApi.NodeGet(id)
waysDraw(node['lat'], node['lon'], name) #this node goes to draw ways
#print("Time taken in the Process --- %s seconds ---" % (time.time() - start_time))
return
def __init__(self):
osm_user = BaseConfig.OSM_USER
osm_pass = BaseConfig.OSM_PASSWORD
osm_api = BaseConfig.OSM_API_URL
self.osm = OsmApi(api=osm_api,
appid='Kort',
username=osm_user,
password=osm_pass)
self.kort_api = kort_api.KortApi()
def getData(id, geom_type='nod'):
osm_data = None
api = OsmApi()
if geom_type == 'nod':
osm_data = api.NodeGet(int(id))
elif geom_type == 'way':
osm_data = api.WayGet(int(id))
elif geom_type == 'rel':
osm_data = api.RelationGet(int(id))
return osm_data
def main():
global myOSMAPI
config = configparser.ConfigParser()
config.read('config.ini')
myOSMAPI = OsmApi(username=config.get('OSM', 'OPENSTREETMAP_USERNAME'),
password=config.get('OSM', 'OPENSTREETMAP_PASSWORD'))
myOSMAPI.ChangesetCreate(
{u"comment": u"district wikidata and wikipedia tags"})
i = 0
#reads list of relations that need to be modified
txt = open("tmplist", 'r')
for place in txt.readlines():
try:
i += 1
#get relation details
relJSON = myOSMAPI.RelationGet(place)
name = relJSON["tag"]["name"]
print name
#get wikipedia article name
qid = open("qidandname", 'r')
sch = wikipedia.search(name + '_district', results=1)
article = sch[0]
print article
#get wikidata qid
n = wptools.page(sch[0].encode('ascii', 'ignore'), silent=True)
qid = n.get_wikidata().wikibase
print qid
#make changes in osm
newrelJSON = relJSON
#newrelJSON["tag"]["wikidata"] = qid
newrelJSON["tag"]["wikipedia"] = "en:" + article
if (i >= 10):
myOSMAPI.ChangesetClose()
break
except:
print place + " failed"
def __init__(self, config):
osm_user = config.get('Osm', 'username')
osm_pass = config.get('Osm', 'password')
osm_api = config.get('Osm', 'api')
osm_app = config.get('Osm', 'appid')
self.osm = OsmApi(
api=osm_api,
appid=osm_app,
username=osm_user,
password=osm_pass
)
self.kort_api = kort_api.KortApi(config)
def Map(min_lon, min_lat, max_lon, max_lat, area):
global mini_lon
global mini_lat
global maxi_lon
global maxi_lat
mini_lon = min_lon
mini_lat = min_lat
maxi_lon = max_lon
maxi_lat = max_lat
#time is started
start_time = time.time()
MyApi = OsmApi()
result = MyApi.Map(min_lon, min_lat, max_lon,
max_lat) #get dataset from OSM website
#create new figure for map
fig, ax = plt.subplots() #add labels, title and axes ticks
ax.ticklabel_format(useOffset=False)
ax.set_ylabel('Longitude')
ax.set_xlabel('Latitude')
ax.set_title(area)
for rs in result:
if (rs['type'] == "node"): #to print the places
if (len(rs['data']['tag']) != 0): #check place is not empty
try: #try if a place has no name
key = rs['data']['tag'].keys()
detail = ""
for k in key:
detail = detail + k + " - " + rs['data']['tag'][
k] + "\n" #collect all information of particular node
#print (detail)
placesDraw(lat=rs['data']['lat'],
long=rs['data']['lon'],
name=detail)
except:
print("error")
elif (rs['type'] == "way"): #to print the ways
if (len(rs['data']['nd']) != 0):
key = rs['data']['tag'].keys()
detail = ""
for k in key:
detail = detail + k + " - " + rs['data']['tag'][
k] + "\n" #collect all information of particular way
#print (detail)
global f
f = 0 # a new way is started
for nd in rs['data']['nd']:
getNode(nd, detail)
draw_graphs(min_lat, max_lat, min_lon, max_lon)
return result
def getNodeHistory(id) :
global tagfirst
global tagsecond
global tagthird
global tagfourth
global tagfifth
global tagsixth
global nodefirst
global nodesecond
global nodethird
global nodefourth
global nodefifth
global nodesixth
#time is started
start_time = time.time()
MyApi=OsmApi()
nodeHistory=MyApi.NodeHistory(id)
print(nodeHistory)
for version in nodeHistory.keys():
user = nodeHistory[version]['user']
version = int(nodeHistory[version]['version'])
id=nodeHistory[version]['id']
if(version == 1):
if(len(nodeHistory[version]['tag'])!= 0):
tagfirst +=len(nodeHistory[version]['tag'])
nodefirst +=1
if(version == 2):
if(len(nodeHistory[version]['tag'])!= 0):
tagsecond +=len(nodeHistory[version]['tag'])
nodesecond +=1
if(version == 3):
if(len(nodeHistory[version]['tag'])!= 0):
tagthird +=len(nodeHistory[version]['tag'])
nodethird +=1
if(version == 4):
if(len(nodeHistory[version]['tag'])!= 0):
tagfourth +=len(nodeHistory[version]['tag'])
nodefourth +=1
if(version == 5):
if(len(nodeHistory[version]['tag'])!= 0):
tagfifth +=len(nodeHistory[version]['tag'])
nodefifth +=1
if(version == 6):
if(len(nodeHistory[version]['tag'])!= 0):
tagsixth +=len(nodeHistory[version]['tag'])
nodesixth +=1
graph()
return
def getChangset(id):
#time is started
start_time = time.time()
MyApi = OsmApi()
#print("Changset is :",MyApi.ChangesetGet(id))
#print("Time taken in the Process --- %s seconds ---" % (time.time() - start_time))
return
def setUp(self):
self.api_base = "http://api06.dev.openstreetmap.org"
self.api = OsmApi(
api=self.api_base
)
self.maxDiff = None
print(self._testMethodName)
print(self.api)
def osm_tenant(self, **kw):
bus_ids = http.request.params['bus_ids']
name = http.request.params['name']
citizen = http.request.params['citizen']
rent = http.request.params['rent']
vrn = http.request.params['vrn']
assess = http.request.params['assess']
branch = http.request.params['branch']
tax = http.request.params['tax']
tin = http.request.params['tin']
efd = http.request.params['efd']
valued = http.request.params['valued']
r = requests.get('http://www.openstreetmap.org/api/0.6/way/%s/full' %
(bus_ids))
response = BeautifulSoup(r.text)
nodes = response.osm.findAll("node")
lat = 0
lon = 0
for x in nodes:
lat = lat + float(x['lat'])
lon = lon + float(x['lon'])
lat = lat / len(nodes)
lon = lon / len(nodes)
MyApi = OsmApi(username=u"Wahab Ali Malik", password=u"Loveodoo")
MyApi.ChangesetCreate(
{u"comment": u"Adding One of the owner of building"})
print MyApi.NodeCreate({
u"lon": lon,
u"lat": lat,
u"tag": {
u"tenant": u"%s" % (name),
u"vrn": u"%s" % (vrn),
u"assess": u"%s" % (assess),
u"branch": u"%s" % (branch),
u"tin": u"%s" % (tin),
u"efd": u"%s" % (efd),
u"citizen": u"%s" % (citizen),
u"valued": u"%s" % (valued),
u"rent": u"%s" % (rent),
u"tax": u"%s" % (tax),
}
})
def main():
global myOSMAPI
config = configparser.ConfigParser()
config.read('config.ini')
myOSMAPI = OsmApi(username=config.get('OSM', 'OPENSTREETMAP_USERNAME'),
password=config.get('OSM', 'OPENSTREETMAP_PASSWORD'))
if (config.get('DEFAULT', 'UPDATE_TYPE') == 'Node'):
UpdateNodes()
if (config.get('DEFAULT', 'UPDATE_TYPE') == 'Way'):
UpdateWays()
def test_NodeCreate_changesetauto(self):
# setup mock
self.api = OsmApi(api="api06.dev.openstreetmap.org",
changesetauto=True)
self._http_mock(filenames=[
'test_NodeCreate_changesetauto.xml',
'test_ChangesetUpload_create_node.xml',
'test_ChangesetClose.xml',
])
test_node = {
'lat': 47.123,
'lon': 8.555,
'tag': {
'amenity': 'place_of_worship',
'religion': 'pastafarian'
}
}
self.assertIsNone(self.api.NodeCreate(test_node))
def getRelation(id):
#time is started
start_time = time.time()
MyApi = OsmApi()
rs = MyApi.RelationGet(id, RelationVersion=-1)
#print("The relations are : ", rs)
key = rs['tag'].keys()
if (len(rs['member']) != 0):
for member in rs['member']:
if (member['type'] == "way"):
resultway = getWay(member['ref'])
nodes = resultway['nd']
keys = resultway['tag'].keys()
detail = ""
for k in keys:
try:
detail = detail + k + " - " + resultway['tag'][
k] + "\n" #collect all information of particular way
except:
print("error")
#print (detail)
global f
f = 0 # a new way is started
for nd in nodes:
getNode(nd, detail)
if (member['type'] == "node"):
resultnode = MyApi.NodeGet(memeber['ref'])
keynode = resultnode['tag'].keys()
detail = ""
for k in keynode:
detail = detail + k + " - " + resultnode['tag'][
k] + "\n" #collect all information of particular node
#print (detail)
placesDraw(lat=resultnode['lat'],
long=resultnode['lon'],
name=detail)
#print("Time taken in the Process --- %s seconds ---" % (time.time() - start_time))
return rs
def sync(self):
LIMIT = 0 # 每一種同步動作的次數限制，除錯用
# 注意!! 實驗階段指向測試伺服器
#host = 'api06.dev.openstreetmap.org'
host = 'api.openstreetmap.org'
user = '******'
pwf = '%s/.osmpass' % os.environ['HOME']
# 注意!! 有中文的地方需要使用 unicode 字串型態
chset = {
u'comment': u'自動同步 (%s)' % self.source_name,
u'created_by': u'小璋流同步機器人 (osmapi/0.6.0)',
u'bot': u'yes'
}
api = OsmApi(api=host, username=user, passwordfile=pwf)
api.ChangesetCreate(chset)
# 新增/修改/刪除
actions = ['create', 'update', 'delete']
points_map = {
'create': self.points_new,
'update': self.points_changed,
'delete': self.points_disappeared
}
for act in actions:
points = points_map[act]
count = 0
failed = 0
total = len(points)
for p in points:
node = self.toNode(p, api)
if node is not None:
try:
if act == 'create': api.NodeCreate(node)
if act == 'update': api.NodeUpdate(node)
if act == 'delete': api.NodeDelete(node)
except:
# TODO: 加入 logger 機制
failed = failed + 1
else:
# TODO: 加入 logger 機制
failed = failed + 1
# 計數/次數限制/摘要顯示
count = count + 1
print('\r%s: %d/%d (失敗: %d)' % (act, count, total, failed)),
sys.stdout.flush()
if count == LIMIT: break
print('')
return api.ChangesetClose()
def _session_mock(self, auth=False, filenames=None, status=200):
response_mock = mock.Mock()
response_mock.status_code = status
return_values = self._return_values(filenames)
print(filenames)
print(return_values)
assert len(return_values) < 2
if return_values:
response_mock.content = return_values[0]
self.session_mock = mock.Mock()
self.session_mock.request = mock.Mock(return_value=response_mock)
if auth:
self.api = OsmApi(api=self.api_base,
username='******',
password='******',
session=self.session_mock)
else:
self.api = OsmApi(api=self.api_base, session=self.session_mock)
self.api._get_http_session = mock.Mock(return_value=self.session_mock)
self.api._session._sleep = mock.Mock()
async def main():
osm = OsmApi(username=config.get("OSM_SUBMIT_USER"), password=config.get("OSM_SUBMIT_PASSWORD"))
print("Connecting...")
await database.connect()
print("Updating materialised view...")
await database.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY solartag.candidates")
print("Fetching changes...")
while True:
res = await database.fetch_all("""SELECT results.osm_id, array_agg(results.user_id) AS users,
solartag.int_decision_value(module_count) AS modules
FROM solartag.results, solartag.candidates
WHERE results.osm_id = candidates.osm_id
GROUP BY results.osm_id
HAVING solartag.int_decision(module_count) = true
AND solartag.int_decision_value(module_count) IS NOT NULL
ORDER BY results.osm_id DESC
LIMIT 100""")
users = set()
for row in res:
users |= set(row['users'])
if len(list(res)) < 10:
print("Fewer than 10 new changes, not submitting.")
return
print(f"Submitting changeset of {len(list(res))} objects...")
print("Creating changeset...")
osm.ChangesetCreate({"contributor_ids": ";".join(map(str, users)),
"source": "https://solartagger.russss.dev",
"comment": "Add generator:solar:modules tag",
"bot": "yes",
"imagery_used": "Bing"
})
for row in res:
node = osm.NodeGet(row['osm_id'])
if 'generator:solar:modules' in node['tag']:
print("Tag already exists for node", node['id'])
continue
node['tag']['generator:solar:modules'] = str(row['modules'])
osm.NodeUpdate(node)
osm.ChangesetClose()
print("Done")
class OsmFix(object):
def __init__(self, config):
osm_user = config.get('Osm', 'username')
osm_pass = config.get('Osm', 'password')
osm_api = config.get('Osm', 'api')
osm_app = config.get('Osm', 'appid')
self.osm = OsmApi(
api=osm_api,
appid=osm_app,
username=osm_user,
password=osm_pass
)
self.kort_api = kort_api.KortApi(config)
def get_for_type(self, type, id):
"""
Returns the 'getter' of the requested OSM type
"""
if type == 'node':
return self.osm.NodeGet(id)
if type == 'way':
return self.osm.WayGet(id)
if type == 'relation':
return self.osm.RelationGet(id)
def update_for_type(self, type, new_values):
"""
Returns the 'update' method of the requested OSM type
"""
if type == 'node':
return self.osm.NodeUpdate(new_values)
if type == 'way':
return self.osm.WayUpdate(new_values)
if type == 'relation':
return self.osm.RelationUpdate(new_values)
def apply_kort_fix(self, limit=1, dry=False):
try:
for kort_fix in self.kort_api.read_fix(limit):
try:
log.debug("---- Fix from Kort: ----")
log.debug("%s" % pprint.pformat(kort_fix))
osm_entity = self.get_for_type(
kort_fix['osm_type'],
kort_fix['osm_id']
)
if not osm_entity:
raise OsmEntityNotFoundError("OSM entity not found")
log.debug("---- OSM type before fix ----")
log.debug("%s" % pprint.pformat(osm_entity['tag']))
error_type = errortypes.Error(
kort_fix['error_type'],
osm_entity
)
fixed = error_type.apply_fix(kort_fix)
fixed_osm_entity, description = fixed
log.debug("---- OSM type after fix ----")
log.debug("%s" % pprint.pformat(fixed_osm_entity['tag']))
except (errortypes.ErrorTypeError,
OsmEntityNotFoundError,
ValueError), e:
log.warning(
"The fix could not be applied: %s, fix: %s"
% (str(e), kort_fix)
)
fixed_osm_entity = None
if not dry:
if fixed_osm_entity is not None:
comment = self.gen_changelog_comment(
kort_fix,
description
)
self.submit_entity(
kort_fix['osm_type'],
fixed_osm_entity,
comment
)
self.kort_api.mark_fix(kort_fix['fix_id'])
except Exception, e:
log.exception("Failed to apply fix of Kort to OpenStreetMap")
user = '******'
pwf = '%s/.osmpass' % os.environ['HOME']
# 注意!! 有中文的地方需要使用 unicode 字串型態
chset = {
u'comment': u'自動同步 U-bike 租借站',
u'created_by': u'小璋流同步機器人 (osmapi/0.6.0)'
}
node = {
u'id': 4299484763,
u'lat': 25.10,
u'lon': 121.0,
u'version': 1,
}
api = OsmApi(api=host, username=user, passwordfile=pwf)
api.ChangesetCreate(chset)
n = api.NodeGet(4299484799)
n = api.NodeDelete(n)
print(n)
#n = api.NodeUpdate(node)
#n = api.NodeDelete(node)
cid = api.ChangesetClose()
# 更新結果摘要
print('Changeset 編號: %d' % cid)
print('URL: http://api06.dev.openstreetmap.org/changeset/%d' % cid)
print('Node 編號: %s' % n['id'])
print('URL: http://api06.dev.openstreetmap.org/api/0.6/node/%s' % n['id'])
'tag']:
response += "\xF0\x9F\x93\x9E " + osm_data[
'tag']['phone'] + "\n"
response += "http://www.osm.org/?minlat={0}&maxlat={1}&minlon={2}&maxlon={3}&mlat={4}&mlon={5}\n".format(
result['boundingbox'][0],
result['boundingbox'][1],
result['boundingbox'][2],
result['boundingbox'][3], result['lat'],
result['lon'])
elif re.match("/search.*", message) is not None:
response = "Please indicate what are you searching"
else:
response = "Use /search <search term> command to indicate what you are searching"
usr_id = query["message"]["chat"]["id"]
bot.sendMessage(usr_id,
response,
disable_web_page_preview='true')
last_id = query["update_id"]
f = open("last.id", "w")
f.write(str(last_id))
f.close()
sc.enter(15, 1, attend, (sc, ))
api = OsmApi()
nom = nominatim.Nominatim()
bot = OSMbot("token")
s = sched.scheduler(time.time, time.sleep)
s.enter(1, 1, attend, (s, ))
s.run()
def main():
logging.info('Starting {0} ...'.format(__program__))
db = POIBase('{}://{}:{}@{}:{}/{}'.format(
config.get_database_type(), config.get_database_writer_username(),
config.get_database_writer_password(),
config.get_database_writer_host(), config.get_database_writer_port(),
config.get_database_poi_database()))
import_basic_data(db.session)
import_poi_data(db.session)
logging.info('Loading data from database ...')
if not os.path.exists(config.get_directory_output()):
os.makedirs(config.get_directory_output())
# Build Dataframe from our POI database
addr_data = db.query_all_gpd('poi_address')
addr_data[['poi_addr_city',
'poi_postcode']] = addr_data[['poi_addr_city', 'poi_postcode'
]].fillna('0').astype(int)
comm_data = db.query_all_pd('poi_common')
logging.info('Exporting CSV files ...')
# And merge and them into one Dataframe and save it to a CSV file
save_csv_file(config.get_directory_output(), 'poi_common.csv', comm_data,
'poi_common')
logging.info('Merging dataframes ...')
data = pd.merge(addr_data,
comm_data,
left_on='poi_common_id',
right_on='pc_id',
how='inner')
save_csv_file(config.get_directory_output(), 'poi_address.csv', data,
'poi_address')
# Generating CSV files group by poi_code
poi_codes = data.poi_code.unique()
# Adding additional empty fields
data['osm_id'] = None
data['node'] = None
data['osm_version'] = None
data['osm_changeset'] = None
data['osm_timestamp'] = None
data['osm_live_tags'] = None
logging.info('Saving separated files ...')
for c in poi_codes:
save_csv_file(config.get_directory_output(),
'poi_address_{}.csv'.format(c), data[data.poi_code == c],
'poi_address')
with open(
os.path.join(config.get_directory_output(),
'poi_address_{}.osm'.format(c)), 'wb') as oxf:
oxf.write(generate_osm_xml(data[data.poi_code == c]))
with open(os.path.join(config.get_directory_output(), 'poi_address.osm'),
'wb') as oxf:
oxf.write(generate_osm_xml(data))
logging.info('Merging with OSM datasets ...')
counter = 0
data['osm_nodes'] = None
osm_live_query = OsmApi()
for i, row in data.iterrows():
common_row = comm_data.loc[comm_data['pc_id'] == row['poi_common_id']]
osm_query = (db.query_osm_shop_poi_gpd_with_metadata(
row['poi_lon'], row['poi_lat'], common_row['poi_type'].item()))
if osm_query is not None:
# Collect additional OSM metadata. Note: this needs style change during osm2pgsql
osm_id = osm_query['osm_id'].values[0]
osm_node = osm_query['node'].values[0]
data.loc[[i], 'osm_id'] = osm_id
data.loc[[i], 'node'] = osm_node
data.loc[[i], 'osm_version'] = osm_query['osm_version'].values[0]
data.loc[[i],
'osm_changeset'] = osm_query['osm_changeset'].values[0]
osm_timestamp = pd.to_datetime(
str((osm_query['osm_timestamp'].values[0])))
data.loc[[i], 'osm_timestamp'] = '{:{dfmt}T{tfmt}Z}'.format(
osm_timestamp, dfmt='%Y-%m-%d', tfmt='%H:%M%S')
# For OSM way also query node points
if osm_node == False:
logging.info(
'This is an OSM way looking for id {} nodes.'.format(
osm_id))
# Add list of nodes to the dataframe
nodes = db.query_ways_nodes(osm_id)
data.at[i, 'osm_nodes'] = nodes
try:
if osm_node == False:
data.at[i, 'osm_live_tags'] = osm_live_query.WayGet(
osm_id)['tag']
else:
data.at[i, 'osm_live_tags'] = osm_live_query.NodeGet(
osm_id)['tag']
except Exception as err:
logging.warning(
'There was an error during OSM request: {}.'.format(err))
print(data.loc[[i], 'osm_live_tags'])
counter += 1
for c in poi_codes:
save_csv_file(config.get_directory_output(),
'poi_address_merge_{}.csv'.format(c),
data[data.poi_code == c], 'poi_address')
with open(
os.path.join(config.get_directory_output(),
'poi_address_merge_{}.osm'.format(c)),
'wb') as oxf:
oxf.write(generate_osm_xml(data[data.poi_code == c]))
with open(
os.path.join(config.get_directory_output(),
'poi_address_merge.osm'), 'wb') as oxf:
oxf.write(generate_osm_xml(data))
logging.info('{} objects found in OSM dataset.'.format(counter))
def __init__(self): self.db,self.curs=openDB() self.api=OsmApi(passwordfile='.pwdFile')
def prueba_osmapi():
MyApi = OsmApi()
print MyApi.NodeGet(123)
def zoosmarazzi(conf, inputs, outputs):
""" Function for ZOO for import vector data on OSM DB
All parameters are passed from ZOO request
"""
# the json string
inputName = inputs["inputvector"]["value"]
# the username of openstreetmap
usern = inputs["username"]["value"]
# the password of user in openstreetmap
passw = inputs["password"]["value"]
# the comment of changset
change = inputs["changeset"]["value"]
# try to decode json
try:
js = json.loads(inputName)
except:
conf["lenv"]["message"] = "error_json"
return 4
# set the changeset comment
if change:
comment = 'auto import of features using geopaparazzi and geopaposm.\n %s' % change
else:
comment = 'auto import of features using geopaparazzi and geopaposm.'
# connect to api
osmapi = OsmApi(api="api06.dev.openstreetmap.org",
username=unicode(usern), password=unicode(passw),
appid=unicode('Z0OSM 0.2'), changesetauto=True,
changesetautotags={unicode('comment'): unicode(comment)})
# set to zero the number of feature
nfeatures = 0
nfeaturesload = 0
# temporary id for osmapi
iD = -1
# for each feature in json string create the node definition
for j in js:
values = j['forms'][0]['formitems']
tags = {}
# set temporary id
nodeDef = {unicode('id'): iD}
# for each value put it to tags or coordinate
for v in values:
if v['value'] != '':
if v['type'] == 'boolean' and v['value'] == 'false':
continue
elif v['type'] == 'boolean' and v['value'] == 'true':
tags[v['key']] = 'yes'
elif v['key'] == 'LONGITUDE':
nodeDef[unicode('lon')] = v['value']
elif v['key'] == 'LATITUDE':
nodeDef[unicode('lat')] = v['value']
else:
tags[v['key']] = v['value']
nodeDef[unicode('tag')] = tags
if len(list(set([unicode('tag'), unicode('lat'), unicode('lon')]
).intersection(set(nodeDef.keys())))) == 3:
# create geometry point with ogr to check if it's present
# a similar point in a buffer
geom = ogr.Geometry(ogr.wkbPoint)
geom.AddPoint_2D(float(nodeDef[unicode('lon')]),
float(nodeDef[unicode('lat')]))
# get osm data near the feature (look function getMapData)
osmData = getMapData(geom, osmapi)
# from osm data extract only point
pointData = getPointData(osmData)
# some control to search if feature already exist on osm db
if checkNewPoint(geom, tags, pointData):
osmapi.NodeCreate(nodeDef)
nfeaturesload += 1
nfeatures += 1
iD -= 1
# try to load data or return an error
try:
osmapi.flush()
except:
conf["lenv"]["message"] = "error_osm"
return 4
# return the string of the number of feature loaded
npoints = abs(iD) - 1
if nfeaturesload == nfeatures:
output = 'features_imported'
else:
output = 'features_imported_%i_%i' % (nfeaturesload, nfeatures)
outputs["output"]["value"] = output
return 3
def getosm(address='Rennes', latlon=0, dist_m=400, cart=False):
""" get osm region from osmapi
Parameters
----------
address : string
latlon : tuple or 0
dist_m : float
cart : boolean
Notes
-----
if latlon tuple is precised it has priority over the string
"""
level_height = 3.45
rad_to_deg = (180 / np.pi)
deg_to_rad = (np.pi / 180)
if latlon == 0:
place = geo.google(address)
try:
lat, lon = place.latlng
except:
print place
else:
lat = latlon[0]
lon = latlon[1]
r_earth = 6370e3
alpha = (dist_m / r_earth) * rad_to_deg
Osm = OsmApi()
#
# Get Map around the specified coordinates
#
osmmap = Osm.Map(lon - alpha, lat - alpha, lon + alpha, lat + alpha)
#print(osmmap)
nodes = Nodes()
nodes.clean()
nodes.readmap(osmmap)
coords = Coords()
coords.clean()
coords.from_nodes(nodes)
m = coords.cartesian(cart=cart)
ways = Ways()
ways.clean()
ways.readmap(osmmap, coords)
# list of nodes involved in buildings
lnodes_id = []
for iw in ways.w:
lnodes_id += ways.w[iw][0]
# list of all nodes of coords
lnodes_id = np.unique(np.array(lnodes_id))
lnodes_full = np.unique(np.array(coords.latlon.keys()))
mask = np.in1d(lnodes_full, lnodes_id, invert=True)
# nodes not involved in buildings
lexcluded = lnodes_full[mask]
coords.filter(lexcluded)
dpoly = {}
for iw in ways.w:
ways.way[iw].tags = {}
# material
if ways.w[iw][1].has_key('material'):
ways.way[iw].tags['name'] = ways.w[iw][1]['material']
elif ways.w[iw][1].has_key('building:material'):
ways.way[iw].tags['name'] = ways.w[iw][1]['building:material']
else:
ways.way[iw].tags['name'] = 'WALL'
# height
if ways.w[iw][1].has_key('height'):
ways.way[iw].tags['z'] = (0, eval(ways.w[iw][1]['height']))
elif ways.w[iw][1].has_key('building:height'):
ways.way[iw].tags['z'] = (0,
eval(ways.w[iw][1]['building:height']))
elif ways.w[iw][1].has_key('building:levels'):
nb_levels = eval(ways.w[iw][1]['building:levels'])
if type(nb_levels) != int:
try:
nb_levels = max(nb_levels)
except:
nb_levels = 2
ways.way[iw].tags['z'] = (0, nb_levels * level_height)
elif ways.w[iw][1].has_key('levels'):
nb_levels = eval(ways.w[iw][1]['levels'])
if type(nb_levels) != int:
try:
nb_levels = max(nb_levels)
except:
nb_levels = 2
ways.way[iw].tags['z'] = (0, nb_levels * level_height)
else:
ways.way[iw].tags['z'] = (0, 12)
ptpoly = [coords.xy[x] for x in ways.w[iw][0]]
dpoly[iw] = geu.Polygon(ptpoly, vnodes=ways.w[iw][0])
dpoly[iw].coorddeter()
return coords, nodes, ways, dpoly, m
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#Python script to download from OSM the created bus routes in Yogyakarta and create a GTFS file
from osmapi import OsmApi
MyApi = OsmApi()
#Fixes Routes of TransJogja
#TODO: create a web javascript framework to let anybody create the GTFS trough web.
routes = [
5332612, 5334914, 1913445, 1761302, 5334915, 5334916, 5334918, 5334917
]
platforms_id = {}
unique_platforms_id = set()
routes_info = {}
for route_id in routes:
platforms_id[route_id] = []
print 'Getting route ', route_id
relation = MyApi.RelationGet(route_id)
routes_info[route_id] = relation['tag']
for a in relation['member']:
if a['role'] == 'platform':
platforms_id[route_id].append(a['ref'])
unique_platforms_id.add(a['ref'])
print platforms_id[route_id]
import transitfeed
schedule = transitfeed.Schedule()
schedule.AddAgency("Transjogja", "http://iflyagency.com",
"Indonesia/Yogyakarta")
#print "Got character", repr(c)
return c
except IOError:
pass
finally:
termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)
fcntl.fcntl(fd, fcntl.F_SETFL, oldflags)
#---
with open('../dolmens/dolmensC.json', 'r') as f:
dolmens_dict = json.load(f)
# creem la sessió, doncs només vull fer un changeset per tots els canvis
MyApi = OsmApi(passwordfile="/home/joan/projectes/OSM/.password")
changeset_comment = '{"comment": "Inserció i actualització dòlmens Catalunya (22-02-2020)"}'
changeset_comment_json = json.loads(changeset_comment)
MyApi.ChangesetCreate(changeset_comment_json)
num_item = 1
for dolmen in dolmens_dict:
name = dolmen['name']
lat = float(dolmen['lat'])
lon = float(dolmen['lon'])
hi_es = dolmen['hi_es']
megalith_type = dolmen['megalith_type']
wikipedia = dolmen['wikipedia']
wikidata = dolmen['wikidata']
source = dolmen['source']
munipnumbers.append(row[0])
#cursor.execute("select person from osmimportresp where kommunenummer=\""+str(munipnumber)+"\" and person != 'rubund' and deleted != 1;")
#rows = cursor.fetchall()
#if(len(rows) > 0):
# print ("Someone else is responsible for this one. Not importing...")
# sys.exit()
#else:
# print ("Nobody is responsible for this one")
print(munipnumbers)
#sys.exit(0)
#api = OsmApi(api="api06.dev.openstreetmap.org", username="", password="", changesetauto=True, changesetautotags={"source":"Kartverket"})
#api = OsmApi(api="api06.dev.openstreetmap.org", username="******", passwordfile="./password.txt")
if not dryrun:
api = OsmApi(username=mypasswords.osmuser, password=mypasswords.osm)
#api.NodeGet(123)
mycomment = u"addr node update Norway"
#mycomment=u"addr node import municipality number "+munipnumberpadded+", Norway"
#api.ChangesetCreate({"comment":u"addr node import "+str(sys.argv[2].decode('utf-8')), "source":"Kartverket", "source:date":"2014-08-24"})
if not dryrun:
api.ChangesetCreate({
"comment": mycomment,
"source": "Kartverket",
"source:date": "2017-06-26"
})
for munipnumber in munipnumbers:
print("\nProcessing %d..." % (munipnumber))
munipnumberpadded = "%04d" % (int(munipnumber))
def getosm(**kwargs):
""" get osm region from osmapi
Parameters
----------
address : string
latlon : tuple or 0
dist_m : float
bcart : boolean
level_height : float
typical level height for deriving building height from # levels
typical_height : float
typical height for building when no information
Returns
-------
coords
nodes
ways
dpoly
m
latlon : tuple or 0
Notes
-----
There are 3 ways to read an Open Street Map structure
1 - From an osm file ex : filename = 'B11.osm'
2 - From an osm (lat,lon) string or tuple of float
3 - From an osm address string
if latlon tuple is precised, it has priority over the address string
"""
filename = kwargs.pop('filename', '')
bcart = kwargs.pop('cart', False)
typ = kwargs.pop('typ', 'indoor')
level_height = kwargs.pop('level_height', 3.45)
typical_height = kwargs.pop('typical_height', 10)
if filename == '':
address = kwargs.pop('address', 'Rennes')
latlon = kwargs.pop('latlon', 0)
dist_m = kwargs.pop('dist_m', 400)
rad_to_deg = (180 / np.pi)
if latlon == 0:
place = geo.osm(address)
try:
lat, lon = place.latlng
except:
print(place)
else:
lat = latlon[0]
lon = latlon[1]
r_earth = 6370e3
alpha = (dist_m / r_earth) * rad_to_deg
#
# get map from OsmApi (Database query)
#
Osm = OsmApi()
osmmap = Osm.Map(lon - alpha, lat - alpha, lon + alpha, lat + alpha)
else:
#
# get map from osm (xml) file
# type : 'node'
# 'ways'
#
latlon = 0
e = xml.parse(filename).getroot()
osmmap = []
lnode_key = ['id', 'lat', 'lon']
lway_key = ['id']
for i in e:
d = {}
d['type'] = i.tag
d['data'] = i.attrib
#print(i.tag)
if d['type'] == 'node':
for k in lnode_key:
try:
d['data'][k] = eval(d['data'][k])
except:
pass
if k == 'id':
if not 'action' in d['data']:
d['data'][k] = -d['data'][k]
d['data']['tag'] = {}
elif d['type'] == 'way':
lk = i.getchildren()
nd = []
tag = {}
for k in lk:
if k.tag == 'nd':
nd.append(eval(k.get('ref')))
if k.tag == 'tag':
tag[k.get('k')] = k.get('v')
d['data']['nd'] = nd
d['data']['tag'] = tag
# for k in lway_key:
# lk = k.get_children()
# print(lk)
# d['data'][k]=eval(d['data'][k])
# d['data']['visible']=eval(d['data']['visible'])
osmmap.append(d)
nodes = Nodes()
nodes.clean()
nodes.readmap(osmmap)
coords = Coords()
coords.clean()
coords.from_nodes(nodes)
m = coords.cartesian(cart=bcart)
ways = Ways()
ways.clean()
lat = coords.latlon[list(coords.latlon.keys())[0]][0]
lon = coords.latlon[list(coords.latlon.keys())[0]][1]
if typ == 'indoor':
ways.readmap1(osmmap, coords)
else:
ways.readmap2(osmmap, coords)
# list of nodes involved in buildings
lnodes_id = []
for iw in ways.w:
lnodes_id += ways.w[iw][0]
# list of all nodes of coords
lnodes_id = np.unique(np.array(lnodes_id))
lnodes_full = np.unique(np.array(list(coords.latlon.keys())))
mask = np.in1d(lnodes_full, lnodes_id, invert=True)
# nodes not involved in buildings
#if typ != 'indoor':
# lexcluded = lnodes_full[mask]
# coords.filter(lexcluded)
# dpoly = {}
# for iw in ways.w:
# # ways.way[iw].tags = {}
# # # material
# # if 'material' in ways.w[iw][1]:
# # ways.way[iw].tags['name']=ways.w[iw][1]['material']
# # elif 'building:material' in ways.w[iw][1]:
# # ways.way[iw].tags['name']=ways.w[iw][1]['building:material']
# # else:
# # ways.way[iw].tags['name']='WALL'
# # # min_height
# # if 'building:min_height' in ways.w[iw][1]:
# # min_height = eval(ways.w[iw][1]['building:min_height'])
# # else:
# # min_height = 0
# # # height
# # if 'height' in ways.w[iw][1]:
# # ways.way[iw].tags['z'] = (min_height, eval(ways.w[iw][1]['height']))
# # elif 'building:height' in ways.w[iw][1]:
# # ways.way[iw].tags['z'] = (min_height, eval(ways.w[iw][1]['building:height']))
# # elif 'building:levels' in ways.w[iw][1]:
# # nb_levels = eval(ways.w[iw][1]['building:levels'])
# # if type(nb_levels)!=int:
# # try:
# # nb_levels = max(nb_levels)
# # except:
# # nb_levels=2
# # ways.way[iw].tags['z']=(min_height,nb_levels*level_height)
# # elif 'levels' in ways.w[iw][1]:
# # nb_levels = eval(ways.w[iw][1]['levels'])
# # if type(nb_levels)!=int:
# # try:
# # nb_levels=max(nb_levels)
# # except:
# # nb_levels=2
# # ways.way[iw].tags['z'] = (min_height,nb_levels*level_height)
# # else:
# # ways.way[iw].tags['z'] = (0,typical_height)
# ptpoly = [coords.xy[x] for x in ways.w[iw][0]]
# dpoly[iw] = geu.Polygon(ptpoly,vnodes=ways.w[iw][0])
# dpoly[iw].coorddeter()
#return coords,nodes,ways,dpoly,m
return coords, nodes, ways, m, (lat, lon)
class Changeset(object):
def __init__(self, id, api='https://api.openstreetmap.org'):
self.id = id
logger.debug('Using api={}'.format(api))
if api:
self.osmapi = OsmApi(api=api)
else:
self.osmapi = None
self.meta = None
self.changes = None
# History for modified/deleted elements, inner dicts indexed by object id
self.history_one_version_back = True
self.hist = {'node': {}, 'way':{}, 'relation':{}}
# Summary of elemets, created, modified, deleted. '_' versions are summarized across all object types
self.summary = {'create' : { 'node': 0, 'way':0, 'relation':0, 'relation_tags':{}},
'modify' : { 'node': 0, 'way':0, 'relation':0, 'relation_tags':{}},
'delete' : { 'node': 0, 'way':0, 'relation':0, 'relation_tags':{}},
'_create':0, '_modify':0, '_delete':0}
# Tag changes
self.tagdiff = self.getEmptyDiffDict()
# Tags unchanged, i.e. mostly ID if object geometrically changed
self.tags = {}
# Textual diff description
self.diffs = {}
# Simple (no tags) nodes
self.simple_nodes = {'create':0, 'modify':0, 'delete':0}
self.other_users = None
self.mileage = None
self.apidebug = False
self.datadebug = False
@staticmethod
def get_timestamp(meta, typeof=None, include_discussion=False):
if include_discussion and 'comments_count' in meta and int(meta['comments_count'])>0:
typeof = 'comment'
cset_ts = reduce(lambda c1,c2: c1 if c1['date']>c2['date'] else c2, meta['discussion'])['date']
else:
if not typeof:
if 'closed_at' in meta.keys():
typeof = 'closed_at'
else:
typeof = 'created_at'
cset_ts = meta[typeof]
if type(cset_ts) is datetime.datetime:
# Some osmapi's pass datetime's here without tz instead of a unicode string?
timestamp = cset_ts.replace(tzinfo=pytz.utc)
else:
timestamp = diff.OsmDiffApi.timetxt2datetime(cset_ts)
return (typeof, timestamp)
def printSummary(self):
s = self.summary
print '{} elements created: Nodes: {}, ways:{} Relations:{}'.format(s['_create'], s['create']['node'], s['create']['way'], s['create']['relation'])
print '{} elements modified: Nodes: {}, ways:{} Relations:{}'.format(s['_modify'], s['modify']['node'], s['modify']['way'], s['modify']['relation'])
print '{} elements deleted: Nodes: {}, ways:{} Relations:{}'.format(s['_delete'], s['delete']['node'], s['delete']['way'], s['delete']['relation'])
print 'Simple nodes: {}'.format(pprint.pformat(self.simple_nodes))
if (self.tagdiff['create'] or self.tagdiff['modify'] or self.tagdiff['delete']):
print 'Tag change stats: {}'.format(pprint.pformat(self.tagdiff))
else:
print 'No tags changed'
if self.other_users:
print 'Modifies objects previously edited by: {}'.format(pprint.pformat(self.other_users))
if self.mileage:
print 'Mileage (ways): {} meters'.format(int(self.mileage['_all_create']-self.mileage['_all_delete']))
print 'Mileage (navigable): {} meters'.format(int(self.mileage['_navigable_create']-self.mileage['_navigable_delete']))
for nav_cat in self.mileage['by_type'].keys():
for nav_type in self.mileage['by_type'][nav_cat].keys():
print 'Mileage ({}={}): {} meters'.format(nav_cat, nav_type,
int(self.mileage['by_type'][nav_cat][nav_type]))
def printDiffs(self):
self.buildDiffList()
pprint.pprint(self.diffs)
def _pluS(self, num):
'''Return plural s'''
if num==1:
return ''
return 's'
def buildDiffList(self, maxtime=None):
logger.debug('Start building diff list')
self.startProcessing(maxtime)
self.diffs = self.getEmptyObjDict()
for modif in self.changes:
logger.debug('Processing modif: {}'.format(modif))
self.checkProcessingLimits()
etype = modif['type']
data = modif['data']
id = data['id']
version = data['version']
action = modif['action']
diff = self.getTagDiff(etype, id, version)
label = self.getLabel(etype, id, version)
#logger.debug('-- {} {} {} --'.format(action, etype, id))
notes = []
prev_authors = []
entry = (action, label, diff, notes, prev_authors)
self.diffs[etype][str(id)] = entry
if action == 'modify':
old = self.old(etype,id,version-1)
if etype=='way':
nd_ops = self.diffStat(old['nd'], data['nd'])
if nd_ops or diff:
if nd_ops:
if nd_ops[0]:
notes.append(u'added {} node{}'.format(nd_ops[0], self._pluS(nd_ops[0])))
if nd_ops[1]:
notes.append(u'removed {} node{}'.format(nd_ops[1], self._pluS(nd_ops[1])))
if old['uid'] != data['uid']:
prev_authors.append(old['user'])
if etype=='relation':
# member is list of dict's: {u'role': u'', u'ref': 1234, u'type': u'way'}
ombr = [x['ref'] for x in old['member']]
nmbr = [x['ref'] for x in data['member']]
m_ops = self.diffStat(ombr, nmbr)
if m_ops or diff:
if m_ops:
if m_ops[0]:
notes.append(u'added {} member{}'.format(m_ops[0], self._pluS(m_ops[0])))
if m_ops[1]:
notes.append(u'deleted {} member{}'.format(m_ops[1], self._pluS(m_ops[1])))
if old['uid'] != data['uid']:
prev_authors.append(old['user'])
if not m_ops and ombr!=nmbr:
notes.append(u'Reordered members')
# TODO: Handle relation role changes (e.g. inner to outer)
# TODO: Show relation as modified if member changes (e.g. way has added a node)
return self.diffs
def diffStat(self, a, b):
''' Given two lists of ids, return tuple with (added, removed) '''
aa = set(a)
bb = set(b)
d1 = aa-bb
d2 = bb-aa
if not d1 and not d2:
return None
return (len(d2), len(d1))
def downloadMeta(self, set_tz=True):
if not self.meta:
if self.apidebug:
logger.debug('osmapi.ChangesetGet({}, include_discussion=True)'.format(self.id))
self.meta = self.osmapi.ChangesetGet(self.id, include_discussion=True)
if set_tz:
for ts in ['created_at', 'closed_at']:
if ts in self.meta:
self.meta[ts] = self.meta[ts].replace(tzinfo=pytz.utc)
if 'discussion' in self.meta:
for disc in self.meta['discussion']:
disc['date'] = disc['date'].replace(tzinfo=pytz.utc)
if self.datadebug:
logger.debug(u'meta({})={}'.format(self.id, self.meta))
def downloadData(self):
if not self.changes:
if self.apidebug:
logger.debug('osmapi.ChangesetDownload({})'.format(self.id))
self.changes = self.osmapi.ChangesetDownload(self.id)
if self.datadebug:
logger.debug(u'changes({})={}'.format(self.id, self.changes))
def _downloadGeometry(self, overpass_api='https://overpass-api.de/api'):
# https://overpass-api.de/api/interpreter?data=[adiff:"2016-07-02T22:23:17Z","2016-07-02T22:23:19Z"];(node(bbox)(changed);way(bbox)(changed););out meta geom(bbox);&bbox=11.4019207,55.8270254,11.4030363,55.8297091
opened = self.get_timestamp(self.meta, 'created_at')[1] - datetime.timedelta(seconds=1)
closed = self.get_timestamp(self.meta, 'closed_at')[1] + datetime.timedelta(seconds=1)
tfmt = '%Y-%m-%dT%h:%M:%sZ'
url = overpass_api+'/interpreter?data=[adiff:"'+ \
opened.strftime(tfmt) + '","' + closed.strftime(tfmt) + \
'"];(node(bbox)(changed);way(bbox)(changed););out meta geom(bbox);&bbox=' + \
'{},{},{},{}'.format(self.meta['min_lon'], self.meta['min_lat'], self.meta['max_lon'], self.meta['max_lat'])
#r = requests.get(url, stream=True, headers={'Connection':'close'})
r = requests.get(url)
if r.status_code!=200:
raise Exception('Overpass error:{}:{}:{}'.format(r.status_code,r.text,url))
#r.raw.decode_content = True
def downloadGeometry(self, maxtime=None, way_nodes=True):
self.startProcessing(maxtime)
for mod in self.changes:
self.checkProcessingLimits()
etype = mod['type']
data = mod['data']
eid = data['id']
version = data['version']
action = mod['action']
if action == 'create':
self.hist[etype][eid] = {1: data}
else:
self.hist[etype][eid] = {version: data}
self.old(etype, eid, version-1)
if etype == 'way' and action != 'delete':
for nid in data['nd']:
self.old('node', nid, data['timestamp'])
def startProcessing(self, maxtime=None):
self.max_processing_time = maxtime
self.processing_start = time.time()
def checkProcessingLimits(self):
if self.max_processing_time:
used = time.time()-self.processing_start
logger.debug('Used {:.2f}s of {}s to process history'.format(used, self.max_processing_time))
if used > self.max_processing_time:
logger.warning('Timeout: Used {:.2f}s of processing time'.format(used))
raise Timeout
def unload(self):
ch = self.changes
self.changes = None
del ch
hist = self.hist
self.hist = None
del hist
def wayIsNavigable(self, tags):
navigable = ['highway', 'cycleway', 'busway']
return set(navigable) & set(tags)
def buildSummary(self, mileage=True, maxtime=None):
logger.debug('Start building change summary')
self.startProcessing(maxtime)
self.other_users = {}
self.mileage = {'_navigable_create':0, '_navigable_delete':0, '_all_create':0, '_all_delete':0, 'by_type': {}}
for modif in self.changes:
logger.debug('Processing modif: {}'.format(modif))
self.checkProcessingLimits()
etype = modif['type']
data = modif['data']
eid = data['id']
version = data['version']
action = modif['action']
self.summary['_'+action] += 1
self.summary[action][etype] += 1
diff = self.getTagDiff(etype, eid, version)
if diff:
self.addDiffDicts(self.tagdiff, diff)
self.tags = self.getTags(etype, eid, version, self.tags)
if etype=='node':
if action == 'delete':
old = self.old(etype,eid,version-1)
if not diff and ('tag' not in old.keys() or not old['tag']):
self.simple_nodes[action] += 1
else:
if not diff and ('tag' not in data.keys() or not data['tag']):
self.simple_nodes[action] += 1
# For modify and delete we summarize affected users
if action != 'create':
old = self.old(etype,eid,version-1)
old_uid = old['uid']
if old_uid != data['uid']:
old_uid = str(old_uid)
if not old_uid in self.other_users.keys():
if old['user']:
usr = old['user']
else:
usr = '******'
self.other_users[old_uid] = {'user':usr, 'edits':0}
self.other_users[old_uid]['edits'] = +1
# FIXME: Since we are only summing created/deleted ways, we ignore edited mileage
if (action == 'create' or action == 'delete') and etype=='way':
if action == 'create':
# If created, we take the latest node version - in special
# cases where a node is edited multiple times in the same
# diff, this might not be correct
nv = -1
nd = data['nd']
tags = data['tag']
else:
# For deleted ways, we take the previous version
nv = old['timestamp']
nd = old['nd']
tags = old['tag']
d = 0
flon = flat = None
for nid in nd:
n = self.old('node', nid, nv)
#logger.debug('({}, {})'.format(n['lon'], n['lat']))
if flon:
d += geotools.haversine(flon, flat, n['lon'], n['lat'])
flon, flat = (n['lon'], n['lat'])
if action == 'delete':
d = -d
self.mileage['_all_'+action] += d
navigable = self.wayIsNavigable(tags)
if navigable:
self.mileage['_navigable_'+action] += d
nav_cat = navigable.pop()
nav_type = tags[nav_cat]
if not nav_cat in self.mileage['by_type'].keys():
self.mileage['by_type'][nav_cat] = {}
if not nav_type in self.mileage['by_type'][nav_cat].keys():
self.mileage['by_type'][nav_cat][nav_type] = 0
self.mileage['by_type'][nav_cat][nav_type] += d
#else:
# # Buildings, natural objects etc
# logger.debug('*** Not navigable way ({}) mileage: {} {} {}'.format(tags, d, self.mileage, navigable))
def getEmptyDiffDict(self):
return {'create':{}, 'delete':{}, 'modify':{}}
def getEmptyObjDict(self):
return {'node':{}, 'way':{}, 'relation':{}}
def addDiffDicts(self, into, src):
for ac in src.keys():
for k,v in src[ac].iteritems():
into[ac][k] = into[ac].get(k, 0)+v
def getTagDiff(self, etype, eid, version):
''' Compute tag diffence between 'version' and previous version '''
diff = self.getEmptyDiffDict()
curr = self.old(etype,eid,version)
ntags = curr['tag']
if version > 1:
old = self.old(etype,eid,version-1)
otags = old['tag']
else:
old = None
otags = {}
#logger.debug('Tags curr:{}'.format(ntags))
#logger.debug('Tags old:{}'.format(otags))
for t in ntags.keys():
if t in otags:
if ntags[t]!=otags[t]:
k = u'{}={} --> {}={}'.format(t, otags[t], t, ntags[t])
diff['modify'][k] = diff['modify'].get(k, 0)+1
else:
k = u'{}={}'.format(t, ntags[t])
diff['create'][k] = diff['create'].get(k, 0)+1
for t in otags.keys():
if not t in ntags:
k = u'{}={}'.format(t, otags[t])
diff['delete'][k] = diff['delete'].get(k, 0)+1
if not diff['create'] and not diff['delete'] and not diff['modify']:
return None
return diff
def getTags(self, etype, eid, version, curr_tags=None):
'''Compute unmodified tags, i.e. tags on objects changed geometrically, but
where tags are identical between 'version' and previous version'''
if not curr_tags:
tags = {}
else:
tags = curr_tags
curr = self.old(etype,eid,version)
ntags = curr['tag']
if version > 1:
old = self.old(etype,eid,version-1)
otags = old['tag']
else:
old = None
otags = {}
#logger.debug('Tags curr:{}'.format(ntags))
#logger.debug('Tags old:{}'.format(otags))
for t in ntags.keys():
if t in otags:
if ntags[t]==otags[t]:
k = u'{}={}'.format(t, ntags[t])
tags[k] = tags.get(k, 0)+1
return tags
def getLabel(self, etype, eid, version):
e = self.old(etype,eid,version)
if 'tag' in e.keys():
tag = e['tag']
if 'name' in tag.keys():
label = u'name={}'.format(tag['name'])
else:
label = u'{}<{}>'.format(etype.capitalize(), eid)
keytags = ['highway', 'amenity', 'man_made', 'leisure', 'historic', 'landuse', 'type']
for kt in keytags:
if kt in tag.keys():
return u'{}={}, {}'.format(kt, tag[kt], label)
return u'{}<{}>'.format(etype, eid)
# Note: Deleted objects only have history
def getElement(self, modif):
etype = modif['type']
data = modif['data']
eid = data['id']
version = data['version']
action = modif['action']
if action == 'create':
self.hist[etype][eid] = {1: data}
else:
e = self.old(etype, eid, version-1)
def getElementHistory(self, etype, eid, version):
logger.debug('GetElementHistory({} idw {} version {})'.format(etype, eid, version));
hv = None
if self.history_one_version_back or version<4:
if not eid in self.hist[etype].keys():
self.hist[etype][eid] = {}
if self.apidebug:
logger.debug('cset {} -> osmapi.{}Get({},ver={})'.format(self.id, etype.capitalize(), eid, version))
if etype == 'node':
hv = self.osmapi.NodeGet(eid, NodeVersion=version)
elif etype == 'way':
hv = self.osmapi.WayGet(eid, WayVersion=version)
elif etype == 'relation':
hv = self.osmapi.RelationGet(eid, RelationVersion=version)
if hv:
self.hist[etype][eid][version] = hv
else:
# Possibly deleted element, fall-through
logger.warning('Failed to get element history by version: {} id {} version {}'.format(etype, eid, version))
if not hv:
if self.apidebug:
logger.debug('cset {} -> osmapi.{}History({})'.format(self.id, etype.capitalize(), eid))
if etype == 'node':
h = self.osmapi.NodeHistory(eid)
elif etype == 'way':
h = self.osmapi.WayHistory(eid)
elif etype == 'relation':
h = self.osmapi.RelationHistory(eid)
self.hist[etype][eid] = h
logger.debug('{} id {} history: {}'.format(etype, eid, h))
def old(self, etype, eid, version, only_visible=True):
logger.debug('Get old {} id {} version {}'.format(etype, eid, version))
if not isinstance(version, int):
'''Support timestamp versioning. Ways and relations refer un-versioned
nodes/ways/relations, i.e. the only way to find the relevant node
postion when the node was e.g. edited is to lookup using the
timestamp. Using the latest version of the node is node correct
if the node was moved subsequently..
'''
ts = diff.OsmDiffApi.timetxt2datetime(version)
# First look if we have version very close in time
if eid in self.hist[etype]:
for v in self.hist[etype][eid].keys():
e = self.hist[etype][eid][v]
diffsec = abs((ts-e['timestamp']).total_seconds())
# If timestamp difference is less than two seconds, return the element we have
# This will cover e.g. newly created nodes+ways
if diffsec<2:
return e
if not self.osmapi:
# If we have no api, return the newest version
v = max(self.hist[etype][eid].keys())
e = self.hist[etype][eid][v]
if only_visible and not e['visible']:
e = self.hist[etype][eid][v-1]
return e
# Lookup the old node
if self.apidebug:
logger.debug('cset {} -> osmapi.{}History({})'.format(self.id, etype.capitalize(), eid))
if etype == 'node':
self.hist[etype][eid] = self.osmapi.NodeHistory(eid)
elif etype == 'way':
self.hist[etype][eid] = self.osmapi.WayHistory(eid)
elif etype == 'relation':
self.hist[etype][eid] = self.osmapi.RelationHistory(eid)
k = self.hist[etype][eid].keys()
k.sort(reverse=True)
version = 1 # Default, if timestamps does not work - should never be needed
for v in k:
e = self.hist[etype][eid][v]
ets = diff.OsmDiffApi.timetxt2datetime(e['timestamp'])
if ets<=ts:
version = e['version']
break;
if version==-1:
# Latest version we already have
if eid in self.hist[etype].keys():
ks = self.hist[etype][eid].keys()
ks.sort()
version = ks[-1]
logger.debug('version -1 changed to {} (ks={})'.format(version, ks))
for v in range(version, 1, -1):
if not only_visible or self.hist[etype][eid][version]['visible']:
return self.hist[etype][eid][version]
logger.debug('Did not find existing history on {} id {} version {}'.format(etype, eid, version))
if (not eid in self.hist[etype].keys()) or (not version in self.hist[etype][eid].keys()):
logger.debug('Do not have element {} id {} version {}'.format(etype, eid, version))
if not eid in self.hist[etype].keys():
logger.debug('Id {} not in {} keys'.format(eid, etype))
elif not version in self.hist[etype][eid].keys():
logger.debug('Version {} not in {}/{} keys'.format(version, etype, eid))
self.getElementHistory(etype, eid, version)
ks = self.hist[etype][eid].keys()
ks.sort()
version = ks[-1]
logger.debug('version -1 changed to {} (ks={})'.format(version, ks))
logger.debug('{} id {} version {}: {}'.format(etype, eid, version, self.hist[etype][eid]))
elem = self.hist[etype][eid][version]
if only_visible and not elem['visible']:
if version > 1:
# Deleted and then reverted elements will not have lat/lons on the old version
logger.debug('Non-visible element found, trying {} id {} version {}'.format(etype, eid, version-1))
elem = self.old(etype, eid, version-1, only_visible)
else:
logger.error('Non-visible element found: {} id {} version {}'.format(etype, eid, version))
if not ('uid' in elem.keys() and 'user' in elem.keys()):
logger.warning('*** Warning, old element type={} id={} v={} elem={}'.format(etype, eid, version, elem))
# API-QUIRK (Anonymous edits, discontinued April 2009): Not all old
# elements have uid. See
# e.g. 'http://www.openstreetmap.org/api/0.6/way/8599635/history'
# Also, 'created_by' does not seem like a complete substitute
#if hasattr(elem, 'create_by'):
# user = '******'.format(elem['create_by'])
#else:
user = None
# Insert pseudo-values
if not 'uid' in elem.keys():
elem['uid'] = 0
if not 'user' in elem.keys():
elem['user'] = user
return elem
# def getReferencedElements(self):
# ''' Get elements referenced by changeset but not directly modified. '''
# for id,w in self.elems['way'].iteritems():
# self.getWay(id, w)
# for id,r in self.elems['relation'].iteritems():
# self.getRelation(id, r)
# def getNode(self, id, data=None):
# if not data:
# if self.apidebug:
# logger.debug('osmapi.NodeGet({})'.format(id))
# data = self.osmapi.NodeGet(id)
# if not data: # Deleted, get history
# self.hist['node'][id] = self.osmapi.NodeHistory(id)
# else:
# self.elems['node'][id] = data
# def getWay(self, id, data=None):
# if not data:
# if self.apidebug:
# logger.debug('osmapi.WayGet({})'.format(id))
# data = self.osmapi.WayGet(id)
# if not data: # Deleted, get history
# self.hist['way'][id] = self.osmapi.WayHistory(id)
# else:
# # Api has limitations on how many elements we can request in one multi-request
# # probably a char limitation, not number of ids
# api_max = 100
# all_nds = data['nd']
# for l in [all_nds[x:x+api_max] for x in xrange(0, len(all_nds), api_max)]:
# # We dont know node version - if node has been deleted we are in trouble
# if self.apidebug:
# logger.debug('osmapi.NodesGet({})'.format(l))
# nds = self.osmapi.NodesGet(l)
# for nd in nds.keys():
# self.getNode(nd, nds[nd])
# def getRelation(self, id, data=None):
# if not data:
# if self.apidebug:
# logger.debug('osmapi.RelationGet({})'.format(id))
# data = self.osmapi.RelationGet(id)
# if not data: # Deleted, get history
# self.hist['relation'][id] = self.osmapi.RelationHistory(id)
# else:
# for mbr in data['member']:
# ref = mbr['ref']
# etype = mbr['type']
# # We dont know version - if way/node has been deleted we are in trouble
# if etype == 'node':
# self.getNode(ref)
# elif etype == 'way':
# self.getWay(ref)
# elif etype == 'relation':
# self.getRelation(ref)
def isInside(self, area, load_way_nodes=True):
'''Return true if there are node edits in changeset and one or more nodes are within area'''
hasnodes = False
for modif in self.changes:
etype = modif['type']
data = modif['data']
action = modif['action']
if etype=='node':
hasnodes = True
if action!='delete':
if area.contains(data['lon'], data['lat']):
return True
else:
# Deleted node do not have lat/lon
id = data['id']
version = data['version']
n = self.old(etype,id,version-1)
if area.contains(n['lon'], n['lat']):
return True
if hasnodes:
# Changesset has node edits, but none inside area i.e. most likely
# not within area. We could have way/relation changes inside area,
# which we will miss (FIXME).
return False
else:
# FIXME: We really do not know because only tags/members on/off
# ways/relations where changes. Maybe download way/relation nodes
# to detect where edit where
if load_way_nodes:
for modif in self.changes:
etype = modif['type']
data = modif['data']
#action = modif['action']
if etype=='way':
nd = data['nd']
for nid in nd:
n = self.old('node', nid, data['timestamp'])
if area.contains(n['lon'], n['lat']):
return True
return False
else:
# If we do not load nodes, we assume there are changed within area
return True
def getGeoJsonDiff(self, include_modified_ways=True):
#self.getReferencedElements()
g = gj.GeoJson()
c_create = '009a00' # Green
c_delete = 'ff2200' # Red
c_old = 'ffff60' # Yellow
c_mod = '66aacc' # Light blue
for modif in self.changes:
etype = modif['type']
n = modif['data']
id = n['id']
version = n['version']
action = modif['action']
#diff = self.getTagDiff(etype, id, version)
f = None
if action=='delete':
e = self.old(etype,id,version-1)
else:
e = self.old(etype,id,version)
if etype=='node':
if action=='modify':
oe = self.old(etype,id,version-1)
logger.debug('Modify node {} version={} e={}, oe={}'.format(id, version, e, oe))
l = g.addLineString()
g.addLineStringPoint(l, e['lon'], e['lat'])
g.addLineStringPoint(l, oe['lon'], oe['lat'])
g.addColour(l, c_old)
g.addProperty(l, 'popupContent', 'Node moved')
g.addProperty(l, 'action', action)
g.addProperty(l, 'type', etype)
#f = g.addPoint(oe['lon'], oe['lat'])
#g.addColour(f, c_old)
f = g.addPoint(e['lon'], e['lat'])
else:
f = g.addPoint(e['lon'], e['lat'])
if etype=='way' and (include_modified_ways or action!='modify'):
f = g.addLineString()
nd = e['nd']
for nid in nd:
# Using timestamp here means we draw the old way. If
# existing points have been moved and new ones added, we
# will draw the old way but show points as being moved.
n = self.old('node', nid, e['timestamp'])
g.addLineStringPoint(f, n['lon'], n['lat'])
if f:
# Popup text
txt = ''
e = self.old(etype,id,version)
tag = e['tag']
g.addProperty(f, 'action', action)
g.addProperty(f, 'type', etype)
g.addProperty(f, 'id', id)
if action=='delete':
g.addColour(f, c_delete)
g.addProperty(f, 'tag', {version: tag})
elif action=='create':
g.addColour(f, c_create)
g.addProperty(f, 'tag', {version: tag})
else:
g.addColour(f, c_mod)
oe = self.old(etype,id,version-1)
g.addProperty(f, 'tag', {version: tag, version-1: oe['tag']})
if self.diffs:
diff = self.diffs[etype][str(id)]
if diff:
if action!='create': # Dont show tags twice
d = diff[2]
if d:
tags = 0
repl = {'create': 'Added tags:', 'modify': 'Modified tags:', 'delete': 'Removed tags:'}
for k in ['create', 'modify', 'delete']:
if len(d[k].keys()) > 0:
txt = self.joinTxt(txt, repl[k])
for kk in d[k].keys():
txt = self.joinTxt(txt, kk)
tags += 1
if tags > 0:
txt = self.joinTxt(txt, '', new_ph=True)
notes = diff[3]
if notes:
for n in notes:
txt = self.joinTxt(txt, n)
txt = self.joinTxt(txt, '', new_ph=True)
usr = diff[4]
if usr:
txt = self.joinTxt(txt, u'Affects edits by:', new_ph=True)
for u in usr:
if not u:
u = '(Anonymous)'
txt = self.joinTxt(txt, u)
if txt != '':
txt = self.joinTxt(txt, '')
g.addProperty(f, 'popupContent', txt)
return g.getData()
def joinTxt(self, t1, t2, new_ph=False, pstart='<p>', pend='</p>'):
if (t1=='' or t1.endswith(pend)) and t2!='':
t1 += pstart+t2.capitalize()
else:
if t2=='':
t1+=pend
else:
if new_ph:
if t1!='':
t1 += '.'+pend
t1 += pstart+t2.capitalize()
else:
if not t1.endswith(':'):
t1 += ', '
t1 += t2
return t1
def get_elem_elem(self, obj, elem):
'''Find elements within elements'''
els = elem.split('.')[1:]
try:
for e in els:
logger.debug(u'get e={} obj={}'.format(e,obj))
if type(obj) is dict:
obj = obj[e]
else:
obj = getattr(obj, e)
logger.debug(u'new obj={}'.format(obj))
return obj
except (AttributeError, KeyError):
return None
def regex_test(self, regex_filter):
'''Check if changeset matches regexp. Input regex_filter is a list of
dicts. For each dict in list, check if all dict elements match, if a
full match is found, return True. I.e. match is OR between list of
dicts and AND between elements in each dict.
'''
logger.debug('Cset check regex filter: {}'.format(regex_filter))
for rf in regex_filter:
matchcnt = 0
for k,v in rf.iteritems():
logger.debug(u"Evaluating: '{}'='{}'".format(k,v))
if k.startswith('.changes'):
if self.regex_test_changes(k, v):
logger.debug("Match found")
matchcnt += 1
else:
e = self.get_elem_elem(self, k)
logger.debug(u"regex: field '{}'='{}', regex '{}'".format(k,e,v))
if e:
m = re.match(v, e)
if m:
logger.debug(u"Match found on '{}'".format(e))
matchcnt += 1
if matchcnt == len(rf.keys()):
logger.debug(u"Found '{}' matches of: '{}'".format(matchcnt, rf))
return True
logger.debug(u"No match: '{}' (found {} of {})".format(rf, matchcnt, len(rf.keys())))
return False
def regex_test_changes(self, k, v):
'''Regex test on changeset changes. Format is:
.changes[.action][.element-type].elements
where optional '.action' is either '.modify', '.create' or '.delete' and optional
'.element-type' is either '.node', '.way' or '.relation'
Examples: '.changes.modify.node.tag.name'
'.changes.node.tag.name'
'''
# FIXME: This code only looks at the new values (e.g. tags on new
# version). We need to investigate old version also to detect e.g. deleted tags
if not self.changes:
return False
action = None
elemtype = None
rg = k.split('.')[2:]
if rg[0] in ['modify', 'create', 'delete']:
action = rg.pop(0)
if rg[0] in ['node', 'way', 'relation']:
elemtype = rg.pop(0)
logger.debug("Action: '{}', element type '{}'".format(action, elemtype))
for modif in self.changes:
if action and action!=modif['action']:
continue
if elemtype and elemtype!=modif['type']:
continue
data = modif['data']
logger.debug('Modif {}'.format(data))
field = '.'+'.'.join(rg)
e = self.get_elem_elem(data, field)
logger.debug("regex: field '{}'='{}', regex '{}'".format(field,e,v))
if e:
return re.match(v, e)
return False
def build_labels(self, label_rules):
'''Build list of labels based on regex and area check. Note that both regex and
area check can be defined with and AND rule between then, i.e. both
must match if both are defined.
'''
labels = []
for dd in label_rules:
if dd['label'] in labels:
logger.debug('Label already set: {}'.format(dd['label']))
continue # duplicate label
match = True
if 'regex' in dd:
logger.debug('regex test, rule={}'.format(dd))
if self.regex_test(dd['regex']):
logger.debug('Regex test OK')
else:
match = False
if 'area_file' in dd:
logger.debug('area test, rule={}'.format(dd))
area = poly.Poly()
if 'OSMTRACKER_REGION' in os.environ:
area_file = os.environ['OSMTRACKER_REGION']
else:
area_file = dd['area_file']
area.load(area_file)
logger.debug("Loaded area polygon from '{}' with {} points".format(area_file, len(area)))
if ('area_check_type' not in dd or dd['area_check_type']=='cset-bbox') and area.contains_chgset(self.meta):
logger.debug('Area test OK, changeset bbox')
elif set(['min_lon', 'min_lat', 'max_lon', 'max_lat']).issubset(self.meta.keys()) and ('area_check_type' in dd and dd['area_check_type']=='cset-center') and area.contains((float(self.meta['min_lon'])+float(self.meta['max_lon']))/2,
(float(self.meta['min_lat'])+float(self.meta['max_lat']))/2):
logger.debug('Area test OK, changeset center')
else:
match = False
if match:
logger.debug("Adding label '{}'".format(dd['label']))
labels.append(dd['label'])
return labels
def data_export(self):
return {'state': {},
'summary': self.summary,
'tags': self.tags,
'tagdiff': self.tagdiff,
'simple_nodes': self.simple_nodes,
'diffs': self.diffs,
'other_users': self.other_users,
'mileage_m': self.mileage,
'geometry': self.hist,
'changes': self.changes}
def data_import(self, data):
self.summary = data['summary']
self.tags = data['tags']
self.tagdiff = data['tagdiff']
self.simple_nodes = data['simple_nodes']
self.diffs = data['diffs']
self.other_users = data['other_users']
self.mileage = data['mileage_m']
self.changes = data['changes']
self.hist = {}
# Exporting to JSON causes int keys to be converted to strings
for etype in data['geometry'].keys():
self.hist[etype] = {}
for eid in data['geometry'][etype].keys():
self.hist[etype][long(eid)] = {}
for v in data['geometry'][etype][eid].keys():
self.hist[etype][long(eid)][long(v)] = data['geometry'][etype][eid][v]
