def test_fromstring_kml_document(self):
"Tests the parsing of an valid KML string"
test_kml = '<kml xmlns="http://www.opengis.net/kml/2.2"/>'
tree = fromstring(test_kml, schema=Schema("ogckml22.xsd"))
self.assertEquals(etree.tostring(tree), test_kml)
tree = fromstring(test_kml)
self.assertEquals(etree.tostring(tree), test_kml)
def load_sat_from_net():
"""
Program to load the satllite track prediction from the internet
Checks at the avdc website
"""
from datetime import datetime, timedelta
from urllib2 import urlopen
from pykml import parser
today = datetime.now().strftime('%Y%m%d')
site = 'http://avdc.gsfc.nasa.gov/download_2.php?site=98675770&id=25&go=download&path=%2FSubsatellite%2Fkml&file=A-Train_subsatellite_prediction_' + today + 'T000000Z.kml'
print 'Satellite tracks url: %s' % site
try:
response = urlopen(site)
print 'Getting the kml prediction file from avdc.gsfc.nasa.gov'
r = response.read()
kml = parser.fromstring(r)
except:
print 'Problem with day, trying previous day...'
try:
yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y%m%d')
site = 'http://avdc.gsfc.nasa.gov/download_2.php?site=98675770&id=25&go=download&path=%2FSubsatellite%2Fkml&file=A-Train_subsatellite_prediction_' + yesterday + 'T000000Z.kml'
response = urlopen(site)
print 'Getting the kml prediction file from avdc.gsfc.nasa.gov'
r = response.read()
kml = parser.fromstring(r)
except:
import tkMessageBox
tkMessageBox.showerror(
'No sat',
'There was an error communicating with avdc.gsfc.nasa.gov')
return None
print 'Kml file read...'
return kml
def load_sat_from_net():
"""
Program to load the satllite track prediction from the internet
Checks at the avdc website
"""
from datetime import datetime,timedelta
from urllib2 import urlopen
from pykml import parser
today = datetime.now().strftime('%Y%m%d')
site = 'http://avdc.gsfc.nasa.gov/download_2.php?site=98675770&id=25&go=download&path=%2FSubsatellite%2Fkml&file=A-Train_subsatellite_prediction_'+today+'T000000Z.kml'
print 'Satellite tracks url: %s' %site
try:
response = urlopen(site)
print 'Getting the kml prediction file from avdc.gsfc.nasa.gov'
r = response.read()
kml = parser.fromstring(r)
except:
print 'Problem with day, trying previous day...'
try:
yesterday = (datetime.now()-timedelta(days=1)).strftime('%Y%m%d')
site = 'http://avdc.gsfc.nasa.gov/download_2.php?site=98675770&id=25&go=download&path=%2FSubsatellite%2Fkml&file=A-Train_subsatellite_prediction_'+yesterday+'T000000Z.kml'
response = urlopen(site)
print 'Getting the kml prediction file from avdc.gsfc.nasa.gov'
r = response.read()
kml = parser.fromstring(r)
except:
import tkMessageBox
tkMessageBox.showerror('No sat','There was an error communicating with avdc.gsfc.nasa.gov')
return None
print 'Kml file read...'
return kml
def test_fromstring_kml_document(self):
"Tests the parsing of an valid KML string"
test_kml = '<kml xmlns="http://www.opengis.net/kml/2.2"/>'
tree = fromstring(test_kml, schema=Schema("ogckml22.xsd"))
self.assertEquals(etree.tostring(tree), test_kml)
tree = fromstring(test_kml)
self.assertEquals(etree.tostring(tree), test_kml)
def test_fromstring_kml_document(self):
"""Tests the parsing of an valid KML string"""
test_kml = b'<kml xmlns="http://www.opengis.net/kml/2.2"/>'
tree = fromstring(test_kml, schema=Schema('ogckml22.xsd'))
self.assertEqual(etree.tostring(tree, encoding='ascii'), test_kml)
tree = fromstring(test_kml)
self.assertEqual(etree.tostring(tree, encoding='ascii'), test_kml)
def parse_london_metro(basepath):
with open(basepath + 'stations.kml', 'rb') as x:
xml = etree.parse(x)
k = parser.fromstring(etree.tostring(xml))
places = (k.findall('.//{http://www.opengis.net/kml/2.2}Placemark'))
stations = []
count = 0
for p in places:
station = {'mode': 'metro'}
station['city'] = 'London'
station['name'] = p.name.text.strip()
coords = [
float(c.strip()) for c in p.Point.coordinates.text.split(',')
]
# London inserts a trailing 0 coordinate too? Why!?
coords.pop()
loc = {'type': 'Point'}
loc['coordinates'] = coords
station['loc'] = loc
stations.append(station)
return stations
def parse_madrid_metro(basepath):
files = ['Metro.kml', 'MetroLigero.kml']
places = []
for f in files:
with open(basepath + f, 'rb') as x:
xml = etree.parse(x)
k = parser.fromstring(etree.tostring(xml))
places.extend(
k.findall('.//{http://www.opengis.net/kml/2.2}Placemark'))
stations = []
count = 0
for p in places:
station = {'mode': 'metro'}
station['city'] = 'Madrid'
station['name'] = p.name.text
coords = [
float(c.strip()) for c in p.Point.coordinates.text.split(',')
]
loc = {'type': 'Point'}
loc['coordinates'] = coords
station['loc'] = loc
stations.append(station)
return stations
def parse(input_file):
with input_file.open("r") as f:
contents = f.read().encode("UTF-8")
root = parser.fromstring(contents)
locations = []
for folder in root.Document.Folder:
for place in folder.Placemark:
location = {"_folder_name": folder.name.text}
for item in place.getchildren():
key = etree.QName(item).localname
if key in element_parsers:
location[key] = element_parsers[key](item)
else:
# if key starts with an uppercase letter, there's probably
# a element to be parsed more that just '.text'
if key[0].isupper():
logger.warn(
f"WARNING: {key} should probably have a parser")
location[key] = item.text
if "description" in location:
location["description"] = [
x.strip() for x in location["description"].split("<br>")
]
locations.append(location)
return locations
def get_sonde():
sonde_data = {}
sonde_data["valid"] = False
sonde_data["lat"] = "0.00000"
sonde_data["lon"] = "0.00000"
sonde_data["alt"] = "00000"
sonde_data["id"] = "Unknown"
sonde_data["temp"] = "0.0"
sonde_data["freq"] = "XX.XXMHz"
sm_f = urllib2.urlopen(sm_url)
sm_data = sm_f.read()
if len(sm_data) == 0:
print "No data from SM."
else:
try:
root = parser.fromstring(sm_data)
if (root.Document.Placemark[0].description == "Sonde"):
# We have something...
sm_name = str(root.Document.Placemark[0].name).split(" ")
sm_coords = str(
root.Document.Placemark[0].Point.coordinates).split(",")
sonde_data["freq"] = sm_name[2][1:-1]
sonde_data["lon"] = sm_coords[0]
sonde_data["lat"] = sm_coords[1]
sonde_data["alt"] = sm_coords[2]
sonde_data["id"] = sm_name[1]
sonde_data["temp"] = sm_name[-1] # Last field is temp.
sonde_data["valid"] = True
return sonde_data
else:
print "Fail"
except:
print "Something didn't parse right"
return sonde_data
def __init__(self, isc_coords_file, fdsn_inventory, our_picks,
event_folder, output_path):
self.event_folder = event_folder
self.output_path = output_path
self.comm = MPI.COMM_WORLD
self.nproc = self.comm.Get_size()
self.rank = self.comm.Get_rank()
self.counter = 0
self.isc_coords_file = isc_coords_file
k = parser.fromstring(open(self.isc_coords_file).read())
ns = k.nsmap.values()[0]
elist = k.findall('.//{%s}Placemark' % (ns))
self.isc_coords_dict = defaultdict(list)
for e in elist:
self.isc_coords_dict[e['name']] = map(float, (str(
e['Point']['coordinates'])).split(','))
# end for
# retrieve list of all event xml files
self.event_folder = event_folder
# retrieve list of all csv files
self.csv_files = sorted(recursive_glob(self.event_folder, '*.csv'))
self.fdsn_inventory = fdsn_inventory
self.our_picks = our_picks
self._load_events()
def kmlBigFileRead(self):
# # 初始化生成shape文件的类
# self.newMap = CreateMapFeature(self.outputPath)
# 获取kml文件的 声明前缀
self.kmlPrefix = self.getKmlPrefix()
# 获取kml文件中的表头
self.kmlTableTitle = self.getTableTitle()
# 循环读取 kml文件 每次读取 self.readBlockSize 大小
while not self.isEnd:
xmlTag = self.kmlFileBlockRead(self.readBlockSize,
self.startPostion, '</Placemark>')
self.startPostion = xmlTag['offset']
self.isEnd = xmlTag['isEnd']
xmlTagStr = self.tagComplement(
xmlTag['content']).encode(encoding='utf-8')
kmlTag = parser.fromstring(xmlTagStr)
placemarkTags = kmlTag.findall(
'.//{http://www.opengis.net/kml/2.2}Placemark')
featureDatas = self.getTab(placemarkTags)
# 生成 shape文件 字段名及类型 # 如: (("index", (4, 254)), ("name", (4, 254)), ("lon", 2), ("lat", 2))
fieldList = [(title[:8], (4, 254)) for title in self.kmlTableTitle]
# 添加到shape文件
self.addFeatureToFile(fieldList, featureDatas)
def getTableTitle(self):
"""
:param placemarks: 为 placemarks 标签的列表
:return: [属性 point坐标 polygon坐标列表] 的 列表
"""
# 读取文件 的第一个 <Placemark> 标签的内容
placemark = self.kmlFileBlockRead(32,
startPost=0,
tagStr='</Placemark>')
# 检查 kml文件是否完整
kmlStr = self.tagComplement(
placemark['content']).encode(encoding='utf-8')
# 将kml字符串生成 kml 对象
kml = parser.fromstring(kmlStr)
placemarkTag = kml.findall(
'.//{http://www.opengis.net/kml/2.2}Placemark')[0]
# 查找key的正则表达式
keyReStr = r'class="atr-name">(.+?)</span>'
attribStr = placemarkTag.description.text
# keysReObj = re.finditer(keyReStr, attribStr)
keyReObj = re.finditer(keyReStr, attribStr)
keyList = []
for keys in keyReObj:
key = keys.group().replace('class="atr-name">',
'').replace('</span>', '')
keyList.append(key)
# 保存 表头
# self.kmlTableTitle = list(keyList)
return list(keyList)
def parse_valencia_bus(basepath):
with open(basepath + 'Emt_paradas.KML', 'rb') as f:
xml = etree.parse(f)
k = parser.fromstring(etree.tostring(xml))
places = k.findall('.//{http://www.opengis.net/kml/2.2}Placemark')
stations = []
count = 0
for p in places:
station = { 'mode': 'bus' }
station['city'] = 'Valencia'
data = p.findall('.//{http://www.opengis.net/kml/2.2}Data')
for d in data:
if d.attrib['name'] == 'numportal':
station['name'] = str(d.value)
coords = [float(c.strip()) for c in p.Point.coordinates.text.split(',')]
loc = { 'type' : 'Point' }
loc['coordinates'] = coords
station['loc'] = loc
stations.append(station)
return stations
def getCountryNodes(Placemarks, hostname):
""" extrahiert für in der Karte verwendete Länder die Country-Placemarks (Polygone) aus countryKMLPath aus, passt sie an und liefert sie als Liste zurück """
countryNames = countrySet(Placemarks)
fileobj = open(countryKMLPath, "r")
countryText = fileobj.read()
fileobj.close()
del fileobj
countryNodeList = []
countryRoot = parser.fromstring(countryText)
countryPlacemarks = countryRoot.Document.findall(
"{http://www.opengis.net/kml/2.2}Placemark")
for countryPlacemark in countryPlacemarks:
if countryPlacemark.name.text in countryNames:
point = countryPlacemark.MultiGeometry.find(
"{http://www.opengis.net/kml/2.2}Point")
countryPlacemark.MultiGeometry.remove(point)
newStyleUrl = KML.styleUrl(hostname + "/UserMap/" + "styles.kml" +
"#country")
countryPlacemark.append(newStyleUrl)
typeNode = KML.type("country")
countryPlacemark.append(typeNode)
countryNodeList.append(countryPlacemark)
return countryNodeList
def parse_madrid_metro(basepath):
files = ['Metro.kml', 'MetroLigero.kml']
places = []
for f in files:
with open(basepath + f, 'rb') as x:
xml = etree.parse(x)
k = parser.fromstring(etree.tostring(xml))
places.extend(k.findall('.//{http://www.opengis.net/kml/2.2}Placemark'))
stations = []
count = 0
for p in places:
station = { 'mode' : 'metro' }
station['city'] = 'Madrid'
station['name'] = p.name.text
coords = [float(c.strip()) for c in p.Point.coordinates.text.split(',')]
loc = { 'type' : 'Point' }
loc['coordinates'] = coords
station['loc'] = loc
stations.append(station)
return stations
def stop_read(filename):
kmz = ZipFile(filename, 'r')
files = kmz.namelist()
# there must be just one file
kmltext = kmz.open(files[0], 'r').read().decode('iso-8859-1')
kmltext = kmltext.replace(' encoding="UTF-8"', '')
k = parser.fromstring(kmltext)
compania = k.Document.Folder[0]
compania_nombre = compania.name
coche = compania.Folder[0]
coche_nombre = coche.name
puntos = recuperar_paradas(coche)
resultado = {
'compania': compania_nombre,
'coche': coche_nombre,
'puntos': puntos
}
return resultado
def test_set_max_decimal_places_track(self):
"""Tests setting the number of decimal places for track data"""
from pykml.helpers import set_max_decimal_places
test_kml = ('<?xml version="1.0" encoding="UTF-8"?>'
'<kml xmlns="http://www.opengis.net/kml/2.2" '
'xmlns:gx="http://www.google.com/kml/ext/2.2">'
'<Folder>'
'<Placemark>'
'<gx:Track>'
'<when>2010-05-28T02:02:09Z</when>'
'<when>2010-05-28T02:02:35Z</when>'
'<when>2010-05-28T02:02:44Z</when>'
'<gx:coord>-122.111111 37.111111 151.333333</gx:coord>'
'<gx:coord>-122.222222 37.222222 152.222222</gx:coord>'
'<gx:coord>-122.333333 37.333333 153.333333</gx:coord>'
'</gx:Track>'
'</Placemark>'
'</Folder>'
'</kml>')
doc = fromstring(test_kml, schema=Schema("kml22gx.xsd"))
set_max_decimal_places(doc,
max_decimals={
'longitude': 3,
'latitude': 2,
'altitude': 1,
})
coords_list = doc.findall(
".//{http://www.google.com/kml/ext/2.2}coord")
#import ipdb; ipdb.set_trace()
self.assertEquals(coords_list[0], '-122.111 37.11 151.3')
def parse_bcn_metro(basepath):
files = ['TMB_EST.kml', 'TRAM_EST.kml']
places = []
for f in files:
with open(basepath + f, 'rb') as x:
xml = etree.parse(x)
k = parser.fromstring(etree.tostring(xml))
places.extend(k.findall('.//{http://www.opengis.net/kml/2.2}Placemark'))
stations = []
count = 0
for p in places:
station = { 'mode' : 'metro' }
station['city'] = 'Barcelona'
station['name'] = p.name.text
coords = [float(c.strip()) for c in p.Point.coordinates.text.split(',')]
# BCN inserts a trailing 0 coordinate? Why!?
coords.pop()
loc = { 'type' : 'Point' }
loc['coordinates'] = coords
station['loc'] = loc
stations.append(station)
return stations
def convertFolder(self, inputPath, outputName, outputDir):
kmzPaths = self.getKMZs(inputPath)
colors = self.getColors()
root = KML.kml(KML.Document())
folders = []
for index, kmzPath in enumerate(kmzPaths):
folder = KML.Folder()
folder.append(KML.name(os.path.basename(kmzPath)))
color = colors[index % len(colors)]
folder.append(self.getStyle(color))
kml_str = self.getKmlStrFromPath(os.path.join(inputPath, kmzPath))
if kml_str is None:
return
root = parser.fromstring(kml_str)
for style in root.Document.Style:
root.Document.remove(style)
for placemark in root.Document.Placemark:
placemark.Point.altitudeMode = 'absolute'
lon, lat, alt = str(placemark.Point.coordinates).split(',')
placemark.styleUrl = "#{}".format(color)
folder.append(placemark)
folders.append(folder)
root.Document.remove(root.Document.name)
root.Document.append(KML.name(outputName))
for folder in folders:
root.Document.append(folder)
outputPath = os.path.join(outputDir, outputName + ".kmz")
kml_str = '<?xml version="1.0" encoding="UTF-8"?>\n'
kml_str += etree.tostring(root, pretty_print=True).decode('UTF-8')
with ZipFile(outputPath, 'w') as zip:
zip.writestr('doc.kml', kml_str)
def retrieve_sites_info_waverider_kml(kml_url=WAVERIDER_KML_URL):
"""
downloads a kml from dept_of_transport WA. retrieve information to create a dictionary of site info(lat, lon, data url ...
:param kml_url: string url kml to parse
:return: dictionary
"""
logger.info('Parsing {url}'.format(url=kml_url))
try:
fileobject = requests.get(kml_url).content
except:
logger.error('{url} not reachable. Retry'.format(url=kml_url))
raise requests.ConnectionError
root = kml_parser.fromstring(fileobject)
# this kml has two 'sub-folders'. One for current, one for historical data.
current_data = root.Document.Folder[0]
historic_data = root.Document.Folder[1]
current_site_info = placemark_info_folder(current_data)
historic_site_info = placemark_info_folder(historic_data)
# merging dicts
sites_info = current_site_info.copy()
sites_info.update(historic_site_info)
return sites_info
def stop_read(filename):
kmz = ZipFile(filename, 'r')
files = kmz.namelist()
# there must be just one file
kmltext = kmz.open(files[0], 'r').read().decode('iso-8859-1')
kmltext = kmltext.replace(' encoding="UTF-8"', '')
k = parser.fromstring(kmltext)
compania = k.Document.Folder[0]
compania_nombre = compania.name
coche = compania.Folder[0]
coche_nombre = coche.name
puntos = recuperar_paradas(coche)
resultado = {
'compania': compania_nombre,
'coche': coche_nombre,
'puntos': puntos
}
return resultado
def parse_london_metro(basepath):
with open(basepath + 'stations.kml', 'rb') as x:
xml = etree.parse(x)
k = parser.fromstring(etree.tostring(xml))
places = (k.findall('.//{http://www.opengis.net/kml/2.2}Placemark'))
stations = []
count = 0
for p in places:
station = { 'mode' : 'metro' }
station['city'] = 'London'
station['name'] = p.name.text.strip()
coords = [float(c.strip()) for c in p.Point.coordinates.text.split(',')]
# London inserts a trailing 0 coordinate too? Why!?
coords.pop()
loc = { 'type' : 'Point' }
loc['coordinates'] = coords
station['loc'] = loc
stations.append(station)
return stations
def parse_bcn_train(basepath):
files = ['FGC_EST.kml', 'RENFE_EST.kml']
places = []
for f in files:
with open(basepath + f, 'rb') as x:
xml = etree.parse(x)
k = parser.fromstring(etree.tostring(xml))
places.extend(k.findall('.//{http://www.opengis.net/kml/2.2}Placemark'))
stations = []
count = 0
for p in places:
station = { 'mode' : 'train' }
station['city'] = 'Barcelona'
station['name'] = p.name.text
coords = [float(c.strip()) for c in p.Point.coordinates.text.split(',')]
# BCN inserts a trailing 0 coordinate? Why!?
coords.pop()
loc = { 'type' : 'Point' }
loc['coordinates'] = coords
station['loc'] = loc
stations.append(station)
return stations
def extract_tracks(f):
from pykml import parser
root = parser.fromstring(f.read())
tracks = find_tracks(root.Document.Placemark)
return tracks
def loadKML( self, fileName ):
"""
return a pykml object that is saved in the kml file fileName
"""
with open( fileName, 'rb' ) as f:
kml_str = f.read()
return parser.fromstring( kml_str )
def load_grid_from_file(self, kml_file):
""" parse kml file and add grid to grid_cache """
file = open(kml_file, "r")
data = str.encode(file.read())
root = parser.fromstring(data)
pms = root.Document.Folder.Placemark
for pm in pms:
id = str(pm.ExtendedData.SchemaData.SimpleData[0])
print(id)
coords = None
try:
coords = pm.MultiGeometry.Polygon.outerBoundaryIs.LinearRing.coordinates
except:
try:
coords = pm.Polygon.outerBoundaryIs.LinearRing.coordinates
except:
raise
self.grid_cache[id] = Grid(id, parse_coords(str(coords)))
print(len(self.grid_cache.keys()))
def import_cord():
file = parser.fromstring(open('C:\\Users\\Prathvi\\Desktop\\prj\\repo\\prathvi\\output_pics\\85.kml', 'r').read())
cords = [[0.0 for x in range(num)] for y in range(cor)] # create empty 4*3 matrix
for x in range(cor):
for y in range(num):
cords[x][y]=float(str(file.Document.Placemark.Point[x].coordinates).split(',')[y]);
return cords
def retrieve_sites_info_awac_kml(kml_url=AWAC_KML_URL):
"""
downloads a kml from dept_of_transport WA. retrieve informations to create a dictionary of site info(lat, lon, data url ...
:param kml_url: string url kml to parse
:return: dictionary
"""
logger.info('Parsing {url}'.format(url=kml_url))
try:
fileobject = requests.get(kml_url).content
except:
logger.error('{url} not reachable. Retry'.format(url=kml_url))
raise requests.ConnectionError
root = kml_parser.fromstring(fileobject)
doc = root.Document.Folder
sites_info = dict()
for pm in doc.Placemark:
coordinates = pm.Point.coordinates.pyval
latitude = float(coordinates.split(',')[1])
longitude = float(coordinates.split(',')[0])
water_depth = float(coordinates.split(',')[2])
description = pm.description.text
snippet = pm.snippet.pyval
time_start = snippet.split(' - ')[0]
time_end = snippet.split(' - ')[1]
time_start = datetime.datetime.strptime(time_start, '%Y-%m-%d')
time_end = datetime.datetime.strptime(time_end, '%Y-%m-%d')
name = pm.name.text
soup = BeautifulSoup(description)
text_zip_url = soup.findAll('a',
attrs={
'href':
re.compile("^http(s|)://.*_Text.zip")
})[0].attrMap['href']
m = re.search('<b>AWAC LOCATION ID:</b>(.*)<br>', description)
site_code = m.group(1).lstrip()
logger.info('{site} available for download'.format(site=site_code))
site_info = {
'site_name': name,
'lat_lon': [latitude, longitude],
'timezone': 8,
'latitude': latitude,
'longitude': longitude,
'water_depth': water_depth,
'time_start': time_start,
'time_end': time_end,
'text_zip_url': text_zip_url,
'site_code': site_code
}
sites_info[site_code] = site_info
return sites_info
def parseKml(kmlFilePath):
""" liest *.kml-Datei aus und liefert geparsten KML-tree zurück """
fileobj = open(kmlFilePath + ".kml", "r")
KMLText = fileobj.read()
fileobj.close()
del fileobj
return parser.fromstring(KMLText)
def load_kml_string(kml_string, schema_type = "kml22gx.xsd"):
schema_gx = Schema(schema_type)
doc = parser.fromstring(kml_string, schema_gx)
if (schema_gx.validate(doc)):
return doc
else:
print "Invalid kml format for string passed in"
return None
def load_sat_from_file(filename):
"""
Program to load the satellite track prediction from a saved file
"""
from pykml import parser
f = open(filename, 'r')
r = f.read()
kml = parser.fromstring(r)
return kml
def _lines_from_kml(self, tracks_file):
lines = []
contents = tracks_file.tracks_file.read().split["\n"]
for content in contents:
root = parser.fromstring(content)
lines = self._load_kml_str(root)
return lines
def _kml2polygon(kml_file):
polystring = parser.fromstring(
kml_file
).Document.Placemark.MultiGeometry.Polygon.outerBoundaryIs.LinearRing.coordinates.text
#polystring.split(" ")
polygon = Polygon(
np.array([p.split(",")
for p in polystring.split(" ")])[:, :2].astype(float))
return polygon
def load_sat_from_file(filename):
"""
Program to load the satellite track prediction from a saved file
"""
from pykml import parser
f = open(filename,'r')
r = f.read()
kml = parser.fromstring(r)
return kml
def download_and_parse_data(self):
"""
Downloads the data using the get_no_fly_zones class and then parses it usign the general method
Input: none
Output: bool from parsing
"""
downloader_module = get_no_fly_zones(self.debug)
downloader_module.download_zones()
file = parser.fromstring(downloader_module.get_data())
return self.parse_data(file)
def openKML(self,kmlfile):
if os.path.splitext(kmlfile)[1]=='.kmz':
kmlfile =zipfile.ZipFile(kmlfile)
kmlfilename =kmlfile.namelist()[0]
coordfile =kmlfile.read(kmlfilename)
else:
coordfile =open(kmlfile).read()
self.kmlfile =coordfile
self.doc =parser.fromstring(coordfile)
def openKML(self, kmlfile):
if os.path.splitext(kmlfile)[1] == '.kmz':
kmlfile = zipfile.ZipFile(kmlfile)
kmlfilename = kmlfile.namelist()[0]
coordfile = kmlfile.read(kmlfilename)
else:
coordfile = open(kmlfile).read()
self.kmlfile = coordfile
self.doc = parser.fromstring(coordfile)
def import_cord(file_name):
file = parser.fromstring(open(file_name, 'r').read())
cords = [[0.0 for x in range(num)]
for y in range(cor)] # create empty 4*3 matrix
for x in range(cor):
for y in range(num):
cords[x][y] = float(
str(file.Document.Placemark.Point[x].coordinates).split(',')
[y])
return cords
def test_fromstring_invalid_kml_document(self):
"Tests the parsing of an invalid KML string"
test_kml = '<bad_element />'
try:
tree = fromstring(test_kml, schema=Schema("ogckml22.xsd"))
self.assertTrue(False)
except etree.XMLSyntaxError:
self.assertTrue(True)
except:
self.assertTrue(False)
def test_fromstring_invalid_kml_document(self):
"Tests the parsing of an invalid KML string"
test_kml = '<bad_element />'
try:
tree = fromstring(test_kml, schema=Schema("ogckml22.xsd"))
self.assertTrue(False)
except etree.XMLSyntaxError:
self.assertTrue(True)
except:
self.assertTrue(False)
def __get_all_coords(data):
doc = parser.fromstring(data)
coords = {}
for folder in doc.Document.getchildren():
try:
dataPoints = folder.Placemark.LineString.coordinates.text.split(" ")
section_id = folder.Placemark.name.text
coords[section_id] = [[float(i) for i in x.split(",")[:2]] for x in dataPoints]
except (Exception):
continue
return coords
def __get_all_coords(data):
doc = parser.fromstring(data)
coords = {}
for folder in doc.Document.getchildren():
try:
dataPoints = folder.Placemark.LineString.coordinates.text.split(" ")
section_id = folder.Placemark.name.text
coords[section_id] = [[float(i) for i in x.split(",")[:2]] for x in dataPoints]
except (Exception):
continue
return coords
def __init__(self, docFile, city_id, chk_delete, action_title):
root = parser.fromstring(docFile)
self.city_id = int(city_id)
self.chk_delete = chk_delete
self.folders = []
self.action_title = action_title
try:
self.folders = root.Document.Folder
except:
self.folders = root.Folder.Document.Folder
self.parent_model = get_model('master', action_title)
def _lines_from_kmz(self, tracks_file):
lines = []
zf = zipfile.ZipFile(tracks_file.tracks_file)
contents = []
for info in zf.infolist():
contents.append(zf.read(info.filename))
for content in contents:
root = parser.fromstring(content)
lines = self._load_kml_str(root)
return lines
def validate_kml(filepath, schema = "https://developers.google.com/kml/schema/kml21.xsd"):
"""
Validates kml located at filepath.
TODO: Even imported kml files do not validate against default xml
"""
schema_gomaps = Schema(schema)
data = ""
with open(filepath, "r") as temp:
data = temp.read()
assert(data != "")
doc = parser.fromstring(data)
return schema_ogc.assertValid(doc)
def __init__(self, docFile):
self.root = parser.fromstring(docFile.read())
self.components = []
self.folders = None
self.elements = {}
self.data = {}
self.city_id = 0
self.parser_component = {
"admin": self.parse_admin,
"electoral": self.parse_electoral,
"slum": self.parse_slum
}
def retrieve_sites_info_awac_kml(kml_url=AWAC_KML_URL):
"""
downloads a kml from dept_of_transport WA. retrieve informations to create a dictionary of site info(lat, lon, data url ...
:param kml_url: string url kml to parse
:return: dictionary
"""
logger.info('Parsing {url}'.format(url=kml_url))
try:
fileobject = requests.get(kml_url).content
except:
logger.error('{url} not reachable. Retry'.format(url=kml_url))
raise requests.ConnectionError
root = kml_parser.fromstring(fileobject)
doc = root.Document.Folder
sites_info = dict()
for pm in doc.Placemark:
coordinates = pm.Point.coordinates.pyval
latitude = float(coordinates.split(',')[1])
longitude = float(coordinates.split(',')[0])
water_depth = float(coordinates.split(',')[2])
description = pm.description.text
snippet = pm.snippet.pyval
time_start = snippet.split(' - ')[0]
time_end = snippet.split(' - ')[1]
time_start = datetime.datetime.strptime(time_start, '%Y-%m-%d')
time_end = datetime.datetime.strptime(time_end, '%Y-%m-%d')
name = pm.name.text
soup = BeautifulSoup(description)
text_zip_url = soup.findAll('a', attrs={'href': re.compile("^http(s|)://.*_Text.zip")})[0].attrMap['href']
m = re.search('<b>AWAC LOCATION ID:</b>(.*)<br>', description)
site_code = m.group(1).lstrip()
logger.info('{site} available for download'.format(site=site_code))
site_info = {'site_name': name,
'lat_lon': [latitude, longitude],
'timezone': 8,
'latitude': latitude,
'longitude': longitude,
'water_depth': water_depth,
'time_start': time_start,
'time_end': time_end,
'text_zip_url': text_zip_url,
'site_code': site_code}
sites_info[site_code] = site_info
return sites_info
def handle_uploaded_kmlfile(filename,objectid):
from pykml import parser
from pykml.util import to_wkt_list
# check if the file is too large
if filename.size >= filename.DEFAULT_CHUNK_SIZE:
raise IOError
# parse the incoming file
doc = parser.fromstring(filename.read())
# look for geometries
wkt_list = [{'geom':wkt} for wkt in to_wkt_list(doc)]
return(wkt_list)
def load_route():
try:
data = json.loads(request.data)
url = data['url']
except Exception as e:
logging.error(0)
abort(400)
obj = urllib2.urlopen(url)
str = obj.read()
kml_str = ""
for line in iter(str.splitlines()):
if not 'atom:link' in line:
kml_str += line
kml_str += '\n'
Point.objects(type='route').delete()
root = parser.fromstring(kml_str)
pointid = 1000
for placemark in root.Document.Folder.Placemark:
coordinates = placemark.MultiGeometry.Point.coordinates.text.split(',')
try:
point = Point(
title=placemark.name.text,
type='route',
latitude=float(coordinates[1]),
longitude=float(coordinates[0]),
pointid=pointid,
timestamp=datetime.now()
)
except TypeError:
abort(500)
except Exception as e:
logging.error(0)
abort(500)
try:
point.save()
except TypeError:
abort(400)
except BadRequest as e:
logging.error(e)
abort(400)
except Exception as e:
logging.error(e)
abort(500)
pointid += 1
return Response(json.dumps({'status': 'ok'}), status=200, mimetype='application/json')
def parse_map_user(user):
kml = os.path.join(os.getcwd(), 'data.kml')
with open(kml) as f:
doc = parser.fromstring(f.read())
coords = doc.Document.Folder.Placemark.LineString.coordinates[0].text.strip().split()
points = [(x.split(',')[0], x.split(',')[1]) for x in coords]
for p in points:
d = Data(0, 0, 0, p[1], p[0], datetime.now(), user)
db.session.add(d)
db.session.commit()
def kml_to_geoJSON():
"""Converts all kml files to geoJSON formated json files"""
folders = os.listdir('data/' + my.DATA_FOLDER + '/kml/')
if '.DS_Store' in folders:
folders.remove('.DS_Store')
#print folders
for folder in folders:
print '\n' + folder + '\n'
path = 'data/' + my.DATA_FOLDER + 'kml/' + folder + '/'
json_path = 'data/' + my.DATA_FOLDER + 'json/' + \
folder.replace(' Division Gangs', '') + '/'
if not os.path.exists(json_path):
os.makedirs(json_path)
files = os.listdir(path)
if '.DS_Store' in files:
files.remove('.DS_Store')
files_ = []
for file in files:
file = file.split('.')
files_.append(file[0])
files_ = list(set(files_))
#pprint(files_)
for file in files_:
kml = path + file + '.kml'
json = json_path + file + '.json'
#print json
if 'Crips' in file or 'Bloods' in file:
print file
raw = open(kml, 'rb').read()
root = parser.fromstring(raw)
try:
cord = root.Document.Folder.Placemark.Polygon.outerBoundaryIs.LinearRing.coordinates
except:
cord = root.Document.Folder.Placemark.MultiGeometry.Polygon.outerBoundaryIs.LinearRing.coordinates
cord = str(cord)
cord = cord.split(' ')
pol = []
for ll in cord:
ll = ll.split(',')
lng = float(ll[0])
lat = float(ll[1])
pol.append([lat, lng])
geo = {
"type" : "Polygon",
"coordinates" : [pol]
}
open(json, 'wb').write(anyjson.dumps(geo))
def load_route():
try:
data = json.loads(request.data)
url = data['url']
except Exception as e:
logging.error(e.args[0])
abort(400)
obj = urllib2.urlopen(url)
str = obj.read()
kml_str = ""
for line in iter(str.splitlines()):
if not 'atom:link' in line:
kml_str+=line
kml_str+='\n'
Point.delete_all('route')
root = parser.fromstring(kml_str)
pointid = 1000
for placemark in root.Document.Folder.Placemark:
coordinates = placemark.MultiGeometry.Point.coordinates.text.split(',')
try:
point = Point(
title = placemark.name.text,
type = 'route',
latitude = float(coordinates[1]),
longitude = float(coordinates[0]),
pointid = pointid,
timestamp = datetime.now()
)
except TypeError:
abort(500)
except Exception as e:
logging.error(e.args[0])
abort(500)
try:
point.put()
except CapabilityDisabledError:
logging.error(u'App Engine Datastore is currently in read-only mode.')
abort(500)
except Exception as e:
logging.error(e.args[0])
abort(500)
pointid += 1
return list_point('route')
def get_geometry_in_kml(self, full=True, placemark=False):
geometry = self.geometry
if geometry:
if full:
line_color, geometry_color = convert_color_hex_to_kml(self.color)
id = 'geo_object_%s' % self.id
kml = KML.kml(KML.Document(
KML.Style(
KML.PolyStyle(KML.color(geometry_color)),
KML.LineStyle(KML.color(line_color), KML.width(2)),
id=id
),
KML.Placemark(KML.name(self.title), KML.styleUrl('#' + id), parser.fromstring(self.geometry.kml))
))
geometry = etree.tostring(kml, pretty_print=True)
elif placemark:
kml = KML.Placemark(KML.name(self.title), parser.fromstring(self.geometry.kml))
geometry = etree.tostring(kml, pretty_print=True)
else:
geometry = self.geometry.kml
return geometry
def get_kml_by_title(request):
if request.GET or request.is_ajax():
query = request.GET['title']
result = nominatim.geocode(query=query, geometry='kml')
polygon = result.raw['geokml'] if result else None
if polygon:
polygon = result.raw['geokml']
kml = KML.kml(KML.Document(KML.Placemark(KML.name(query), parser.fromstring(polygon))))
polygon = etree.tostring(kml)
response = get_file_response(polygon, query)
else:
response = HttpResponseBadRequest('Not found')
else:
response = HttpResponseBadRequest('Request is not GET')
return response
def route_from_file(self, filepath, max_vertices=100):
contents = []
if filepath.endswith(".kmz"):
contents = self._load_kmz_content(filepath)
if filepath.endswith(".kml"):
contents = self._load_kml_content(filepath)
geo_line = []
for content in contents:
root = parser.fromstring(content)
line_tuples = self._load_kml_str(root)
nth_vertex = max(1, int(len(line_tuples) / max_vertices))
for (lat, lng, alt) in line_tuples[0::nth_vertex]:
geo_line.append((lat, lng))
line = LineString(geo_line)
lines = MultiLineString([line])
name = filepath.split("/")[-1].split(".")[0]
return self.create(lines=lines, name=name)
def assertKmlValid(self, response):
self.assertEqual(response.status_code, 200)
self.assert_(response['Content-Type'].startswith('application/vnd.google-earth.kml+xml'))
if 0 and pykml:
# real validation against KML schema -- disabled for now because it seems to reject
# some valid kml files.
doc = kmlparser.fromstring(response.content)
schema = kmlparser.Schema("kml22gx.xsd")
try:
schema.assertValid(doc)
except lxml.etree.DocumentInvalid:
fd, path = tempfile.mkstemp('-TestGeocamTrackViews.kml')
with os.fdopen(fd, 'w') as f:
f.write(response.content)
logging.warning('kml contents written to %s for debugging', path)
raise
else:
# superficial check that it looks like KML
self.assert_(response.content.startswith('<?xml'))
self.assert_('<kml' in response.content)
self.assert_('</kml>' in response.content)
def get_kml_for_queryset(queryset, filename=_('Geographical objects')):
"""
:param queryset: GeoObject queryset
:return: KML file from GeoObject queryset
"""
kml = KML.kml(KML.Document(KML.open(1)))
geo_objects = queryset.values_list('color', 'title', 'lat', 'lon', 'geometry', 'short_description')
colors = set(items[0] for items in sorted(geo_objects))
colors_dict = {color: convert_color_hex_to_kml(color) for color in colors}
for line_color, polygon_color in colors_dict.values():
kml.Document.append(KML.Style(
KML.PolyStyle(KML.color(polygon_color)),
KML.LineStyle(KML.color(line_color), KML.width(2)),
id=line_color,
))
print(line_color, polygon_color)
for (color, title, lat, lon, polygon, short_description) in geo_objects:
style_id = colors_dict[color][0]
fld = KML.Folder(
KML.name(title),
KML.Placemark(
KML.name(title),
KML.styleUrl('#' + style_id),
KML.description(short_description),
KML.Point(KML.coordinates("{},{},0".format(lon, lat)))
)
)
if polygon:
polygon = parser.fromstring(polygon.kml)
fld.append(KML.Placemark(KML.name(title), KML.styleUrl('#' + style_id),
KML.description(short_description), polygon))
kml.Document.append(fld)
kml_str = '<?xml version="1.0" encoding="UTF-8"?>\n' + etree.tostring(kml, pretty_print=True).decode()
return get_file_response(kml_str, filename)
def test_set_max_decimal_places_track(self):
"""Tests setting the number of decimal places for track data"""
from pykml.helpers import set_max_decimal_places
test_kml = (
'<?xml version="1.0" encoding="UTF-8"?>'
'<kml xmlns="http://www.opengis.net/kml/2.2" '
'xmlns:gx="http://www.google.com/kml/ext/2.2">'
'<Folder>'
'<Placemark>'
'<gx:Track>'
'<when>2010-05-28T02:02:09Z</when>'
'<when>2010-05-28T02:02:35Z</when>'
'<when>2010-05-28T02:02:44Z</when>'
'<gx:coord>-122.111111 37.111111 151.333333</gx:coord>'
'<gx:coord>-122.222222 37.222222 152.222222</gx:coord>'
'<gx:coord>-122.333333 37.333333 153.333333</gx:coord>'
'</gx:Track>'
'</Placemark>'
'</Folder>'
'</kml>'
)
doc = fromstring(test_kml, schema=Schema("kml22gx.xsd"))
set_max_decimal_places(
doc,
max_decimals={
'longitude': 3,
'latitude': 2,
'altitude': 1,
}
)
coords_list = doc.findall(".//{http://www.google.com/kml/ext/2.2}coord")
#import ipdb; ipdb.set_trace()
self.assertEquals(
coords_list[0],
'-122.111 37.11 151.3'
)
def procesar_kml(self):
tup_colectivo = self.env['tup.colectivo']
tup_recorrido = self.env['tup.recorrido']
for obj in self:
archivo = base64.b64decode(obj.archivo_kml_zip)
if archivo:
file_like_object = io.BytesIO(archivo)
zf = zipfile.ZipFile(file_like_object, 'a')
for filename in zf.namelist():
data = zf.read(filename)
try:
k = parser.fromstring(data)
recorrido = str(k.Document.Placemark.name)
if recorrido:
linea = recorrido.replace(" Ida", "").replace(" Vuelta", "")
if linea:
kml_data = base64.b64encode(data)
tup_col_id = tup_colectivo.search([('name','=', linea)])
if not tup_col_id:
tup_col_id = tup_colectivo.create({'name':linea})
tup_rec_id = tup_recorrido.search([('name','=', linea)])
if not tup_rec_id:
tup_rec_id = tup_recorrido.create({'name':linea, 'colectivo_id':tup_col_id.id})
tup_col_id.write({'recorrido_id':tup_rec_id.id})
if "Ida" in recorrido:
tup_rec_id.write({'kml_ida':kml_data})
elif "Vuelta" in recorrido:
tup_rec_id.write({'kml_vuelta':kml_data})
except Exception, e:
print e
print "No se pudo importar: ",filename
pass
tup_colectivo.search([]).procesar_kml()
def test_fromstring_invalid_kml_document(self):
"""Tests the parsing of an invalid KML string"""
test_kml = b'<bad_element />'
with self.assertRaises(etree.XMLSyntaxError):
# tree =
fromstring(test_kml, schema=Schema('ogckml22.xsd'))
