def sync(self):
LIMIT = 0 # 每一種同步動作的次數限制，除錯用
# 注意!! 實驗階段指向測試伺服器
#host = 'api06.dev.openstreetmap.org'
host = 'api.openstreetmap.org'
user = '******'
pwf = '%s/.osmpass' % os.environ['HOME']
# 注意!! 有中文的地方需要使用 unicode 字串型態
chset = {
u'comment': u'自動同步 (%s)' % self.source_name,
u'created_by': u'小璋流同步機器人 (osmapi/0.6.0)',
u'bot': u'yes'
}
api = OsmApi(api=host, username=user, passwordfile=pwf)
api.ChangesetCreate(chset)
# 新增/修改/刪除
actions = ['create', 'update', 'delete']
points_map = {
'create': self.points_new,
'update': self.points_changed,
'delete': self.points_disappeared
}
for act in actions:
points = points_map[act]
count = 0
failed = 0
total = len(points)
for p in points:
node = self.toNode(p, api)
if node is not None:
try:
if act == 'create': api.NodeCreate(node)
if act == 'update': api.NodeUpdate(node)
if act == 'delete': api.NodeDelete(node)
except:
# TODO: 加入 logger 機制
failed = failed + 1
else:
# TODO: 加入 logger 機制
failed = failed + 1
# 計數/次數限制/摘要顯示
count = count + 1
print('\r%s: %d/%d (失敗: %d)' % (act, count, total, failed)),
sys.stdout.flush()
if count == LIMIT: break
print('')
return api.ChangesetClose()
async def main():
osm = OsmApi(username=config.get("OSM_SUBMIT_USER"), password=config.get("OSM_SUBMIT_PASSWORD"))
print("Connecting...")
await database.connect()
print("Updating materialised view...")
await database.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY solartag.candidates")
print("Fetching changes...")
while True:
res = await database.fetch_all("""SELECT results.osm_id, array_agg(results.user_id) AS users,
solartag.int_decision_value(module_count) AS modules
FROM solartag.results, solartag.candidates
WHERE results.osm_id = candidates.osm_id
GROUP BY results.osm_id
HAVING solartag.int_decision(module_count) = true
AND solartag.int_decision_value(module_count) IS NOT NULL
ORDER BY results.osm_id DESC
LIMIT 100""")
users = set()
for row in res:
users |= set(row['users'])
if len(list(res)) < 10:
print("Fewer than 10 new changes, not submitting.")
return
print(f"Submitting changeset of {len(list(res))} objects...")
print("Creating changeset...")
osm.ChangesetCreate({"contributor_ids": ";".join(map(str, users)),
"source": "https://solartagger.russss.dev",
"comment": "Add generator:solar:modules tag",
"bot": "yes",
"imagery_used": "Bing"
})
for row in res:
node = osm.NodeGet(row['osm_id'])
if 'generator:solar:modules' in node['tag']:
print("Tag already exists for node", node['id'])
continue
node['tag']['generator:solar:modules'] = str(row['modules'])
osm.NodeUpdate(node)
osm.ChangesetClose()
print("Done")
if ("name" not in tags.keys() and name != ""):
tags[u"name"] = name
tags[u"historic"] = u"archaeological_site"
tags[u"site_type"] = "megalith"
tags[u"megalith_type"] = megalith_type
#if (tags[u"wikipedia"]):
if ("wikipedia" not in tags.keys() and wikipedia != ""):
tags[u"wikipedia"] = wikipedia
if ("wikidata" not in tags.keys() and wikidata != ""):
tags[u"wikidata"] = wikidata
if ("source" not in tags.keys() and source != ""):
tags[u"source"] = source
if ("alt_name" not in tags.keys() and alt_name != ""):
tags[u"alt_name"] = alt_name
print(MyApi.NodeUpdate(node))
print("dolmen/menhir actualitzat\n")
num_item = num_item + 1
else:
print('No volem actualitzar\n')
if (num == 0):
print("\n\nINSERCIÓ\n========")
print('Vols inserir (y/n)')
tecla = press_key()
print(tecla)
if (tecla == 'y'):
#primer les dades que segur que hi són:
data = '{"lat":%f, "lon":%f, "tag": { "historic":"archaeological_site","site_type":"megalith", "name":"%s", "megalith_type":"%s"' % (
class OsmFix(object):
def __init__(self, config):
osm_user = config.get('Osm', 'username')
osm_pass = config.get('Osm', 'password')
osm_api = config.get('Osm', 'api')
osm_app = config.get('Osm', 'appid')
self.osm = OsmApi(
api=osm_api,
appid=osm_app,
username=osm_user,
password=osm_pass
)
self.kort_api = kort_api.KortApi(config)
def get_for_type(self, type, id):
"""
Returns the 'getter' of the requested OSM type
"""
if type == 'node':
return self.osm.NodeGet(id)
if type == 'way':
return self.osm.WayGet(id)
if type == 'relation':
return self.osm.RelationGet(id)
def update_for_type(self, type, new_values):
"""
Returns the 'update' method of the requested OSM type
"""
if type == 'node':
return self.osm.NodeUpdate(new_values)
if type == 'way':
return self.osm.WayUpdate(new_values)
if type == 'relation':
return self.osm.RelationUpdate(new_values)
def apply_kort_fix(self, limit=1, dry=False):
try:
for kort_fix in self.kort_api.read_fix(limit):
try:
log.debug("---- Fix from Kort: ----")
log.debug("%s" % pprint.pformat(kort_fix))
osm_entity = self.get_for_type(
kort_fix['osm_type'],
kort_fix['osm_id']
)
if not osm_entity:
raise OsmEntityNotFoundError("OSM entity not found")
log.debug("---- OSM type before fix ----")
log.debug("%s" % pprint.pformat(osm_entity['tag']))
error_type = errortypes.Error(
kort_fix['error_type'],
osm_entity
)
fixed = error_type.apply_fix(kort_fix)
fixed_osm_entity, description = fixed
log.debug("---- OSM type after fix ----")
log.debug("%s" % pprint.pformat(fixed_osm_entity['tag']))
except (errortypes.ErrorTypeError,
OsmEntityNotFoundError,
ValueError), e:
log.warning(
"The fix could not be applied: %s, fix: %s"
% (str(e), kort_fix)
)
fixed_osm_entity = None
if not dry:
if fixed_osm_entity is not None:
comment = self.gen_changelog_comment(
kort_fix,
description
)
self.submit_entity(
kort_fix['osm_type'],
fixed_osm_entity,
comment
)
self.kort_api.mark_fix(kort_fix['fix_id'])
except Exception, e:
log.exception("Failed to apply fix of Kort to OpenStreetMap")
else:
if node.tagName == "node":
editnode = {
"id": -counter,
"version": osm_version,
"lat": latitude,
"lon": longitude,
"tag": jtags
}
counter = counter + 1
#print (str(latitude)+""+str(longitude)+""+street+" "+housenumber)
if action == "modify":
print("Modifying")
if node.tagName == "node":
if not dryrun:
api.NodeUpdate(editnode)
else:
if not dryrun:
api.WayUpdate(editnode)
elif action == "delete":
print("Deleting")
if node.tagName == "node":
if not dryrun:
api.NodeDelete(editnode)
else:
if node.tagName == "node":
print("Creating")
if not dryrun:
api.NodeCreate(editnode)
print(editnode)
atleastone = True
#Initialise osmapi - Replace the value in username with your username.
#Replace the value in password with your password
myOSMAPI = OsmApi(username="******", password="******")
#retrieve the result nodes using overpass for the Rajajinagar bounding box
resultNodes = overpyAPI.query(
"[out:json];node(12.9936,77.549,12.99651,77.55526);out;")
#Iterate through all the nodes in the result
for i in resultNodes.nodes:
#filter nodes having tags
if (i.tags):
tag = i.tags
#if Kannada tag(Or any other Indic Tag) is present, filter once more
if ("name:kn" in tag.keys()):
#transliterate value in tag to required Indic language
tag["name:ta"] = kannadaToTamizhTransliterator.transliterate_indic_indic(
tag["name:kn"], "kn_IN", "ta_IN")
#store the result in a new JSON
nodeJSON = {}
nodeJSON["id"] = i.id
nodeJSON["lat"] = i.lat
nodeJSON["lon"] = i.lon
nodeJSON["version"] = 8
nodeJSON["tag"] = i.tags
#update Node according to osmapi guidelines -1) Create changeset 2) Update node 3)Close changeset
myOSMAPI.ChangesetCreate(
{u"comment": u"Automating Transliteration"})
myOSMAPI.NodeUpdate(nodeJSON)
myOSMAPI.ChangesetClose()
class TestOsmApiNode(osmapi_tests.TestOsmApi):
def test_NodeGet(self):
self._conn_mock()
result = self.api.NodeGet(123)
args, kwargs = self.api._conn.putrequest.call_args
self.assertEquals(args[0], 'GET')
self.assertEquals(args[1], '/api/0.6/node/123')
self.assertEquals(
result, {
'id': 123,
'changeset': 15293,
'uid': 605,
'timestamp': datetime.datetime(2012, 4, 18, 11, 14, 26),
'lat': 51.8753146,
'lon': -1.4857118,
'visible': True,
'version': 8,
'user': '******',
'tag': {
'amenity': 'school',
'foo': 'bar',
'name': 'Berolina & Schule'
},
})
def test_NodeGet_with_version(self):
self._conn_mock()
result = self.api.NodeGet(123, NodeVersion=2)
args, kwargs = self.api._conn.putrequest.call_args
self.assertEquals(args[0], 'GET')
self.assertEquals(args[1], '/api/0.6/node/123/2')
self.assertEquals(
result, {
'id': 123,
'changeset': 4152,
'uid': 605,
'timestamp': datetime.datetime(2011, 4, 18, 11, 14, 26),
'lat': 51.8753146,
'lon': -1.4857118,
'visible': True,
'version': 2,
'user': '******',
'tag': {
'amenity': 'school',
},
})
def test_NodeCreate_changesetauto(self):
# setup mock
self.api = OsmApi(api="api06.dev.openstreetmap.org",
changesetauto=True)
self._conn_mock(auth=True,
filenames=[
'test_NodeCreate_changesetauto.xml',
'test_ChangesetUpload_create_node.xml',
'test_ChangesetClose.xml',
])
test_node = {
'lat': 47.123,
'lon': 8.555,
'tag': {
'amenity': 'place_of_worship',
'religion': 'pastafarian'
}
}
self.assertIsNone(self.api.NodeCreate(test_node))
def test_NodeCreate(self):
self._conn_mock(auth=True)
# setup mock
self.api.ChangesetCreate = mock.Mock(return_value=1111)
self.api._CurrentChangesetId = 1111
test_node = {
'lat': 47.287,
'lon': 8.765,
'tag': {
'amenity': 'place_of_worship',
'religion': 'pastafarian'
}
}
cs = self.api.ChangesetCreate({'comment': 'This is a test dataset'})
self.assertEquals(cs, 1111)
result = self.api.NodeCreate(test_node)
args, kwargs = self.api._conn.putrequest.call_args
self.assertEquals(args[0], 'PUT')
self.assertEquals(args[1], '/api/0.6/node/create')
self.assertEquals(result['id'], 9876)
self.assertEquals(result['lat'], test_node['lat'])
self.assertEquals(result['lon'], test_node['lon'])
self.assertEquals(result['tag'], test_node['tag'])
def test_NodeCreate_wo_changeset(self):
test_node = {
'lat': 47.287,
'lon': 8.765,
'tag': {
'amenity': 'place_of_worship',
'religion': 'pastafarian'
}
}
with self.assertRaisesRegexp(Exception, 'need to open a changeset'):
self.api.NodeCreate(test_node)
def test_NodeCreate_wo_auth(self):
self._conn_mock()
# setup mock
self.api.ChangesetCreate = mock.Mock(return_value=1111)
self.api._CurrentChangesetId = 1111
test_node = {
'lat': 47.287,
'lon': 8.765,
'tag': {
'amenity': 'place_of_worship',
'religion': 'pastafarian'
}
}
with self.assertRaisesRegexp(UsernamePasswordMissingError,
'Username/Password missing'):
self.api.NodeCreate(test_node)
def test_NodeUpdate(self):
self._conn_mock(auth=True)
# setup mock
self.api.ChangesetCreate = mock.Mock(return_value=1111)
self.api._CurrentChangesetId = 1111
test_node = {
'id': 7676,
'lat': 47.287,
'lon': 8.765,
'tag': {
'amenity': 'place_of_worship',
'name': 'christian'
}
}
cs = self.api.ChangesetCreate({'comment': 'This is a test dataset'})
self.assertEquals(cs, 1111)
result = self.api.NodeUpdate(test_node)
args, kwargs = self.api._conn.putrequest.call_args
self.assertEquals(args[0], 'PUT')
self.assertEquals(args[1], '/api/0.6/node/7676')
self.assertEquals(result['id'], 7676)
self.assertEquals(result['version'], 3)
self.assertEquals(result['lat'], test_node['lat'])
self.assertEquals(result['lon'], test_node['lon'])
self.assertEquals(result['tag'], test_node['tag'])
def test_NodeDelete(self):
self._conn_mock(auth=True)
# setup mock
self.api.ChangesetCreate = mock.Mock(return_value=1111)
self.api._CurrentChangesetId = 1111
test_node = {'id': 7676}
cs = self.api.ChangesetCreate({'comment': 'This is a test dataset'})
self.assertEquals(cs, 1111)
result = self.api.NodeDelete(test_node)
args, kwargs = self.api._conn.putrequest.call_args
self.assertEquals(args[0], 'DELETE')
self.assertEquals(args[1], '/api/0.6/node/7676')
self.assertEquals(result['id'], 7676)
self.assertEquals(result['version'], 4)
def test_NodeHistory(self):
self._conn_mock()
result = self.api.NodeHistory(123)
args, kwargs = self.api._conn.putrequest.call_args
self.assertEquals(args[0], 'GET')
self.assertEquals(args[1], '/api/0.6/node/123/history')
self.assertEquals(len(result), 8)
self.assertEquals(result[4]['id'], 123)
self.assertEquals(result[4]['version'], 4)
self.assertEquals(result[4]['lat'], 51.8753146)
self.assertEquals(result[4]['lon'], -1.4857118)
self.assertEquals(result[4]['tag'], {
'empty': '',
'foo': 'bar',
})
def test_NodeWays(self):
self._conn_mock()
result = self.api.NodeWays(234)
args, kwargs = self.api._conn.putrequest.call_args
self.assertEquals(args[0], 'GET')
self.assertEquals(args[1], '/api/0.6/node/234/ways')
self.assertEquals(len(result), 1)
self.assertEquals(result[0]['id'], 60)
self.assertEquals(result[0]['changeset'], 61)
self.assertEquals(result[0]['tag'], {
'highway': 'path',
'name': 'Dog walking path',
})
def test_NodeRelations(self):
self._conn_mock()
result = self.api.NodeRelations(4295668179)
args, kwargs = self.api._conn.putrequest.call_args
self.assertEquals(args[0], 'GET')
self.assertEquals(args[1], '/api/0.6/node/4295668179/relations')
self.assertEquals(len(result), 1)
self.assertEquals(result[0]['id'], 4294968148)
self.assertEquals(result[0]['changeset'], 23123)
self.assertEquals(result[0]['member'][1], {
'role': 'point',
'ref': 4295668179,
'type': 'node',
})
self.assertEquals(result[0]['tag'], {
'type': 'fancy',
})
def test_NodesGet(self):
self._conn_mock()
result = self.api.NodesGet([123, 345])
args, kwargs = self.api._conn.putrequest.call_args
self.assertEquals(args[0], 'GET')
self.assertEquals(args[1], '/api/0.6/nodes?nodes=123,345')
self.assertEquals(len(result), 2)
self.assertEquals(
result[123], {
'id': 123,
'changeset': 15293,
'uid': 605,
'timestamp': datetime.datetime(2012, 4, 18, 11, 14, 26),
'lat': 51.8753146,
'lon': -1.4857118,
'visible': True,
'version': 8,
'user': '******',
'tag': {
'amenity': 'school',
'foo': 'bar',
'name': 'Berolina & Schule'
},
})
self.assertEquals(
result[345], {
'id': 345,
'changeset': 244,
'timestamp': datetime.datetime(2009, 9, 12, 3, 22, 59),
'uid': 1,
'visible': False,
'version': 2,
'user': '******',
'tag': {},
})
class OsmFix(object):
def __init__(self):
osm_user = BaseConfig.OSM_USER
osm_pass = BaseConfig.OSM_PASSWORD
osm_api = BaseConfig.OSM_API_URL
self.osm = OsmApi(api=osm_api,
appid='Kort',
username=osm_user,
password=osm_pass)
self.kort_api = kort_api.KortApi()
def get_for_type(self, type, id):
"""
Returns the 'getter' of the requested OSM type
"""
if type == 'node':
return self.osm.NodeGet(id)
if type == 'way':
return self.osm.WayGet(id)
if type == 'relation':
return self.osm.RelationGet(id)
def update_for_type(self, type, new_values):
"""
Returns the 'update' method of the requested OSM type
"""
if type == 'node':
return self.osm.NodeUpdate(new_values)
if type == 'way':
return self.osm.WayUpdate(new_values)
if type == 'relation':
return self.osm.RelationUpdate(new_values)
def apply_kort_fix(self, limit=1, dry=False):
try:
for kort_fix in self.kort_api.read_fix():
try:
log.debug("---- Fix from Kort: ----")
log.debug("%s" % pprint.pformat(kort_fix))
osm_entity = self.get_for_type(kort_fix['osm_type'],
kort_fix['osm_id'])
if not osm_entity:
raise OsmEntityNotFoundError("OSM entity not found")
log.debug("---- OSM type before fix ----")
log.debug("%s" % pprint.pformat(osm_entity['tag']))
error_type = errortypes.Error(kort_fix['error_type'],
osm_entity)
fixed = error_type.apply_fix(kort_fix)
fixed_osm_entity, description = fixed
log.debug("---- OSM type after fix ----")
log.debug("%s" % pprint.pformat(fixed_osm_entity['tag']))
except (errortypes.ErrorTypeError, OsmEntityNotFoundError,
ValueError) as e:
log.warning("The fix could not be applied: %s, fix: %s" %
(str(e), kort_fix))
fixed_osm_entity = None
if not dry:
if fixed_osm_entity is not None:
comment = self.gen_changelog_comment(
kort_fix, description)
self.submit_entity(kort_fix['osm_type'],
fixed_osm_entity, comment, kort_fix)
self.kort_api.mark_fix(kort_fix['fix_id'])
except Exception as e:
log.exception("Failed to apply fix of Kort to OpenStreetMap")
def gen_changelog_comment(self, kort_fix, change_description):
comment = (
u"Change from kort, user: %s (id: %s), "
u"fix id: %s, error: %s (source: %s), "
u"description: %s, "
u"see this users profile for more information: "
u"http://www.openstreetmap.org/user/kort-to-osm" %
(kort_fix['username'], kort_fix['user_id'], kort_fix['fix_id'],
kort_fix['error_type'], kort_fix['source'], change_description))
return comment
def submit_entity(self, type, entity, comment, kort_fix):
"""
Submits an OSM entity (node, way, relation) to OSM
"""
self.osm.ChangesetCreate({
"comment": comment[:255],
"mechanical": "yes",
"kort:username": kort_fix['username'],
"kort:user_id": str(kort_fix['user_id']),
"kort:fix_id": str(kort_fix['fix_id']),
"kort:error_type": kort_fix['error_type'],
"kort:error_source": kort_fix['source']
})
changeset = self.update_for_type(type, entity)
log.info("%s" % pprint.pformat(changeset))
self.osm.ChangesetClose()