def import_data(self, metadata, output):
"Imports places data from OpenStreetMap"
old_etag = metadata.get('etag', '')
request = AnyMethodRequest(self._url, method='HEAD')
response = urllib2.urlopen(request)
new_etag = response.headers['ETag'][1:-1]
if False and new_etag == old_etag:
output.write('OSM data not updated. Not updating.\n')
return
p = subprocess.Popen([self.SHELL_CMD % self._url],
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
close_fds=True)
parser = make_parser()
parser.setContentHandler(
OSMHandler(self._get_source(), self._get_entity_types(),
self._find_types, output, self._lat_north,
self._lat_south, self._lon_west, self._lon_east))
parser.parse(p.stdout)
self.disambiguate_titles(self._get_source())
return {
'etag': new_etag,
}
def resize_external_image(url, width, timeout=5):
ei, created = ExternalImage.objects.get_or_create(url=url)
request = AnyMethodRequest(url, method='HEAD')
try:
try:
response = urllib2.urlopen(request, timeout=timeout)
except TypeError:
import socket
old_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(timeout)
response = urllib2.urlopen(request)
socket.setdefaulttimeout(old_timeout)
except urllib2.HTTPError as e:
if e.code in (405, 501):
try:
response = urllib2.urlopen(url, timeout=timeout)
except (urllib2.HTTPError, urllib2.URLError,
httplib.HTTPException):
return None
else:
return None
except (urllib2.URLError, httplib.HTTPException):
return None
# Check whether the image has changed since last we looked at it
if response.headers.get('ETag',
ei.etag) != ei.etag or response.headers.get(
'Last-Modified', True) != ei.last_modified:
# Can't use the shorter EIS.objects.filter(...).delete() as that
# doesn't call the delete() method on individual objects, resulting
# in the old images not being deleted.
for eis in ExternalImageSized.objects.filter(external_image=ei):
eis.delete()
ei.etag = response.headers.get('Etag')
ei.last_modified = response.headers.get('Last-Modified')
ei.save()
try:
eis, created = ExternalImageSized.objects.get_or_create(
external_image=ei, width=width)
except ExternalImageSized.MultipleObjectsReturned:
for eis in ExternalImageSized.objects.filter(external_image=ei,
width=width):
eis.delete()
eis, created = ExternalImageSized.objects.get_or_create(
external_image=ei, width=width)
if not os.path.exists(eis.get_filename()):
eis.delete()
eis, created = ExternalImageSized.objects.get_or_create(
external_image=ei, width=width)
return eis
def handle_POST(cls, request, context, ptype):
data = simplejson.loads(request.raw_post_data)
auth = 'Basic ' + base64.b64encode(':'.join((data['username'], data['password'])))
request_data = '<osm><changeset>'
request_data += ' <tag k="created_by" v="Mobile Oxford"/>'
request_data += ' <tag k="comment" v="%s"/>' % escape(data['comment'])
request_data += '</changeset></osm>'
request = AnyMethodRequest(cls.api_url+'changeset/create',
data=request_data,
method='PUT')
request.headers['Authorization'] = auth
try:
response = urllib2.urlopen(request)
except Exception, e:
return HttpResponse(simplejson.dumps({
'status': 'There was an error opening the changeset; please check your username and password: %s' % e.read(),
'changes': {},
}))
def import_data(self, metadata, output):
"Imports places data from OpenStreetMap"
old_etag = metadata.get('etag', '')
request = AnyMethodRequest(self._url, method='HEAD')
response = urllib2.urlopen(request)
new_etag = response.headers['ETag'][1:-1]
self.output = output
if not settings.DEBUG and new_etag == old_etag:
output.write('OSM data not updated. Not updating.\n')
return
parser = make_parser(['xml.sax.xmlreader.IncrementalParser'])
parser.setContentHandler(
OSMHandler(self._get_source(),
self._get_entity_types(),
lambda tags, type_list=None: self._find_types(
tags, self._osm_tags
if type_list is None else type_list),
output,
self._lat_north,
self._lat_south,
self._lon_west,
self._lon_east))
# Parse in 8k chunks
osm = urllib2.urlopen(self._url)
buffer = osm.read(8192)
bunzip = bz2.BZ2Decompressor()
while buffer:
parser.feed(bunzip.decompress(buffer))
buffer = osm.read(8192)
parser.close()
for lang_code, lang_name in settings.LANGUAGES:
with override(lang_code):
self.disambiguate_titles(self._get_source())
return {
'etag': new_etag,
}
}))
diffResult = ET.parse(response)
updated_entities = {}
for c in diffResult.getroot():
id = '%s%s' % (c.tag[0].upper(), c.attrib['old_id'])
metadata = data['changes'][id]
metadata['version'] = c.attrib['new_version']
updated_entities[id] = metadata
entity = Entity.objects.get(osm_id=id)
entity.metadata['attrs']['version'] = metadata['version']
entity.metadata['osm']['tags'] = metadata['tags']
entity.save()
request = AnyMethodRequest(cls.api_url+'changeset/%s/close'%changeset,
method='PUT')
request.headers['Authorization'] = auth
try:
response = urllib2.urlopen(request)
except Exception, e:
return HttpResponse(simplejson.dumps({
'status': 'There was an error closing the changeset: %s' % e.read(),
'changes': {},
}))
return HttpResponse(simplejson.dumps({
'status': 'Upload complete: <a href="%schangeset/%s">changeset</a>' % (cls.api_url, changeset),
'changes': updated_entities,
}))
def get_osmchange(cls, data, changeset):