def createCache(server, capabilitiesXML, coverageXML):
#import xml.etree.ElementTree as ET
#from xml.etree.ElementTree import XMLParser
from lxml import etree as ET
import json
# Save out the xml file for later
utils.saveFile(
SERVERCACHEPATH + server['name'] + '-GetCapabilities' +
FILEEXTENSIONXML, capabilitiesXML)
utils.saveFile(
SERVERCACHEPATH + server['name'] + '-DescribeCoverage' +
FILEEXTENSIONXML, coverageXML)
print 'Creating caches...'
subMasterCache = {}
subMasterCache['server'] = {}
#parse = XMLParser( encoding="UTF-8" )
# Parse the GetCapabilities XML
#root = ET.XML(capabilitiesXML, parser=parse)
root = ET.fromstring(removeNonUTF8(capabilitiesXML))
# Parse the DescribeCoverage XML
coverageRoot = ET.fromstring(removeNonUTF8(coverageXML))
if root.find('./%sCapability/%sLayer/%sLayer' %
(WMS_NAMESPACE, WMS_NAMESPACE, WMS_NAMESPACE)) == None:
dirtyCaches.append(server)
return
for service in root.findall('./%sService' % (WMS_NAMESPACE)):
serverTitle = service.find('./%sTitle' % (WMS_NAMESPACE)).text
serverAbstract = service.find(
'./%sAbstract' % (WMS_NAMESPACE)).text if service.find(
'./%sAbstract' % (WMS_NAMESPACE)) is not None else None
for product in root.findall('./%sCapability/%sLayer/%sLayer' %
(WMS_NAMESPACE, WMS_NAMESPACE, WMS_NAMESPACE)):
sensorName = product.find('./%sTitle' % (WMS_NAMESPACE)).text
if utils.blackfilter(sensorName, productBlackList):
sensorName = utils.replaceAll(sensorName, {
' ': '_',
'(': '_',
')': '_',
'/': '_'
})
print sensorName
layers = []
for layer in product.findall('./%sLayer' % (WMS_NAMESPACE)):
name = layer.find('./%sName' % (WMS_NAMESPACE)).text
title = layer.find('./%sTitle' % (WMS_NAMESPACE)).text
abstract = layer.find('./%sAbstract' % (WMS_NAMESPACE)).text
temporal = False
if name not in server['indicators']:
print "NOTICE: Indicator '" + name + "' found on WMS server but not in local config file, ignoring."
continue
#Find the CoverageOffering from DescribeCoverage
coverage = findCoverageNode(coverageRoot, name)
if coverage == None:
print serverTitle + " " + name + " could not be found in DescribeCoverage. Not including."
continue
offsetVectorsArray = coverage.findall('.//%soffsetVector' %
(GML_NAMESPACE))
offsetVectors = []
for i in range(0, len(offsetVectorsArray)):
offsetVectors.append(
float(offsetVectorsArray[i].text.split(" ")[i]))
exGeographicBoundingBox = {
"WestBoundLongitude":
layer.find(
'./%sEX_GeographicBoundingBox/%swestBoundLongitude' %
(WMS_NAMESPACE, WMS_NAMESPACE)).text,
"EastBoundLongitude":
layer.find(
'./%sEX_GeographicBoundingBox/%seastBoundLongitude' %
(WMS_NAMESPACE, WMS_NAMESPACE)).text,
"SouthBoundLatitude":
layer.find(
'./%sEX_GeographicBoundingBox/%ssouthBoundLatitude' %
(WMS_NAMESPACE, WMS_NAMESPACE)).text,
"NorthBoundLatitude":
layer.find(
'./%sEX_GeographicBoundingBox/%snorthBoundLatitude' %
(WMS_NAMESPACE, WMS_NAMESPACE)).text
}
boundingBox = {
"CRS":
layer.find('./%sBoundingBox' % (WMS_NAMESPACE)).get('CRS'),
"MinX":
layer.find('./%sBoundingBox' %
(WMS_NAMESPACE)).get('minx'),
"MaxX":
layer.find('./%sBoundingBox' %
(WMS_NAMESPACE)).get('maxx'),
"MinY":
layer.find('./%sBoundingBox' %
(WMS_NAMESPACE)).get('miny'),
"MaxY":
layer.find('./%sBoundingBox' % (WMS_NAMESPACE)).get('maxy')
}
dimensions = createDimensionsArray(layer, server)
temporal = dimensions['temporal']
styles = createStylesArray(layer)
if server['options']['providerShortTag'] not in providers:
raise Exception("Provider shortTag " +
server['options']['providerShortTag'] +
" was not in the 'providers.py' file")
# Get the default details for the provider
providerDetails = providers[server['options']
['providerShortTag']]
if (layerHasMoreInfo(server['options']['providerShortTag'])):
moreProviderInfo = True
else:
moreProviderInfo = False
if 'providerDetails' in server['indicators'][name]:
# Overwrite any details with the indicator specific details
for i in server['indicators'][name]['providerDetails']:
providerDetails[i] = server['indicators'][name][
'providerDetails'][i]
#import pprint
#pprint.pprint(server['indicators'][name])
#print '-'*40
if utils.blackfilter(name, layerBlackList):
if layerHasMoreInfo(
server['indicators'][name]['niceName']):
moreIndicatorInfo = True
else:
moreIndicatorInfo = False
masterLayer = {
"Name": name,
"Title": title,
"Abstract": abstract,
"FirstDate": dimensions['firstDate'],
"LastDate": dimensions['lastDate'],
"OffsetVectors": offsetVectors,
"ProviderDetails": providerDetails,
"EX_GeographicBoundingBox": exGeographicBoundingBox,
"MoreIndicatorInfo": moreIndicatorInfo,
"MoreProviderInfo": moreProviderInfo
}
if name in server['indicators']:
masterLayer['tags'] = server['indicators'][name]
# Data to be sent in the mastercache
layers.append(masterLayer)
# Data to be saved out
layer = { #"Name": name,
#"wmsURL": server['wmsURL'],
#"wcsURL": server['wcsURL'],
#"Title": title,
#"Abstract": abstract,
"FirstDate": dimensions['firstDate'],
"LastDate": dimensions['lastDate'],
"OffsetVectors": offsetVectors,
#"EX_GeographicBoundingBox": exGeographicBoundingBox,
"BoundingBox": boundingBox,
"Dimensions": dimensions['dimensions'],
"Styles": styles
}
cleanServerName = server['name'].replace('/', '-')
cleanLayerName = name.replace('/', '-')
# Save out layer cache
utils.saveFile(
LAYERCACHEPATH + cleanServerName + "_" +
cleanLayerName + FILEEXTENSIONJSON, json.dumps(layer))
subMasterCache['server'][sensorName] = layers
subMasterCache['options'] = server['options']
subMasterCache['wmsURL'] = server['services']['wms']['url']
subMasterCache['wcsURL'] = server['services']['wcs']['url']
subMasterCache['serverName'] = server['name']
print 'Cache creation complete...'
# Return and save out the cache for this server
return utils.saveFile(SERVERCACHEPATH + server['name'] + FILEEXTENSIONJSON,
json.dumps(subMasterCache))
def createCache(server, xml):
import json
import urllib
print 'Creating caches...'
subMasterCache = {}
subMasterCache['layers'] = []
tags = None
cleanServerName = server['name'].replace('/', '-')
cleanLayerName = server['name']
if server['params'] and server['params']['TypeName']:
cleanLayerName = utils.replaceAll(server['params']['TypeName'], {':': '-', '\\': '-'})
if server['name'] in extraInfo:
tags = extraInfo[server['name']]
# Layer iter
if 'passthrough' in server['options'] and server['options']['passthrough']:
if server['params']:
encodedParams = urllib.urlencode(server['params'])
subMasterCache['url'] = server['url'] + encodedParams
else:
subMasterCache['url'] = server['url']
layer = {
'name': cleanLayerName,
'options': server['options']
}
if tags:
layer['tags'] = tags
subMasterCache['layers'].append(layer)
elif xml != None:
# Save out the xml file for later
utils.saveFile(SERVERCACHEPATH + server['name'] + FILEEXTENSIONXML, xml)
times = processTimes(server, xml)
layer = {
'name': cleanLayerName,
'options': server['options'],
'times': times
}
if tags:
layer['tags'] = tags
# Save out layer cache
utils.saveFile(LAYERCACHEPATH + cleanServerName + "_" + cleanLayerName + FILEEXTENSIONJSON, json.dumps(layer))
subMasterCache['layers'].append(layer)
subMasterCache['url'] = server['url']
subMasterCache['serverName'] = server['name']
print 'Cache creation complete...'
# Return and save out the cache for this server
return utils.saveFile(SERVERCACHEPATH + server['name'] + FILEEXTENSIONJSON, json.dumps(subMasterCache))
def createCache(server, xml):
import json
import urllib
print 'Creating caches...'
subMasterCache = {}
subMasterCache['layers'] = []
tags = None
cleanServerName = server['name'].replace('/', '-')
cleanLayerName = server['name']
if server['params'] and server['params']['TypeName']:
cleanLayerName = utils.replaceAll(server['params']['TypeName'], {
':': '-',
'\\': '-'
})
if server['name'] in extraInfo:
tags = extraInfo[server['name']]
# Layer iter
if 'passthrough' in server['options'] and server['options']['passthrough']:
if server['params']:
encodedParams = urllib.urlencode(server['params'])
subMasterCache['url'] = server['url'] + encodedParams
else:
subMasterCache['url'] = server['url']
layer = {'name': cleanLayerName, 'options': server['options']}
if tags:
layer['tags'] = tags
subMasterCache['layers'].append(layer)
elif xml != None:
# Save out the xml file for later
utils.saveFile(SERVERCACHEPATH + server['name'] + FILEEXTENSIONXML,
xml)
times = processTimes(server, xml)
layer = {
'name': cleanLayerName,
'options': server['options'],
'times': times
}
if tags:
layer['tags'] = tags
# Save out layer cache
utils.saveFile(
LAYERCACHEPATH + cleanServerName + "_" + cleanLayerName +
FILEEXTENSIONJSON, json.dumps(layer))
subMasterCache['layers'].append(layer)
subMasterCache['url'] = server['url']
subMasterCache['serverName'] = server['name']
print 'Cache creation complete...'
# Return and save out the cache for this server
return utils.saveFile(SERVERCACHEPATH + server['name'] + FILEEXTENSIONJSON,
json.dumps(subMasterCache))
def createCache(server, xml):
import xml.etree.ElementTree as ET
import json
# Save out the xml file for later
utils.saveFile(SERVERCACHEPATH + server['name'] + FILEEXTENSIONXML, xml)
print 'Creating caches...'
subMasterCache = {}
subMasterCache['server'] = {}
#ET.register_namespace(NAMESPACE, NAMESPACE)
root = ET.fromstring(xml)
if root.find('./%sCapability/%sLayer/%sLayer' % (NAMESPACE,NAMESPACE,NAMESPACE)) == None:
dirtyCaches.append(server)
return
for service in root.findall('./%sService' % (NAMESPACE)):
serverTitle = service.find('./%sTitle' % (NAMESPACE)).text
serverAbstract = service.find('./%sAbstract' % (NAMESPACE)).text if service.find('./%sAbstract' % (NAMESPACE)) is not None else None
for product in root.findall('./%sCapability/%sLayer/%sLayer' % (NAMESPACE,NAMESPACE,NAMESPACE)):
sensorName = product.find('./%sTitle' % (NAMESPACE)).text
if utils.blackfilter(sensorName, productBlackList):
sensorName = utils.replaceAll(sensorName, {' ':'_', '(':'_', ')':'_', '/':'_'})
print sensorName
layers = []
for layer in product.findall('./%sLayer' % (NAMESPACE)):
name = layer.find('./%sName' % (NAMESPACE)).text
title = layer.find('./%sTitle' % (NAMESPACE)).text
abstract = layer.find('./%sAbstract' % (NAMESPACE)).text
temporal = False
exGeographicBoundingBox = {"WestBoundLongitude": layer.find('./%sEX_GeographicBoundingBox/%swestBoundLongitude' % (NAMESPACE,NAMESPACE)).text,
"EastBoundLongitude": layer.find('./%sEX_GeographicBoundingBox/%seastBoundLongitude' % (NAMESPACE,NAMESPACE)).text,
"SouthBoundLatitude": layer.find('./%sEX_GeographicBoundingBox/%ssouthBoundLatitude' % (NAMESPACE,NAMESPACE)).text,
"NorthBoundLatitude": layer.find('./%sEX_GeographicBoundingBox/%snorthBoundLatitude' % (NAMESPACE,NAMESPACE)).text}
boundingBox = {"CRS": layer.find('./%sBoundingBox' % (NAMESPACE)).get('CRS'),
"MinX": layer.find('./%sBoundingBox' % (NAMESPACE)).get('minx'),
"MaxX": layer.find('./%sBoundingBox' % (NAMESPACE)).get('maxx'),
"MinY": layer.find('./%sBoundingBox' % (NAMESPACE)).get('miny'),
"MaxY": layer.find('./%sBoundingBox' % (NAMESPACE)).get('maxy')}
dimensions = createDimensionsArray(layer, server)
temporal = dimensions['temporal']
styles = createStylesArray(layer)
if utils.blackfilter(name, layerBlackList):
masterLayer = {"Name": name,
"Title": title,
"Abstract": abstract,
"FirstDate": dimensions['firstDate'],
"LastDate": dimensions['lastDate'],
"EX_GeographicBoundingBox": exGeographicBoundingBox }
if server['name'] in extraInfo:
if name in extraInfo[server['name']]:
masterLayer['tags'] = extraInfo[server['name']][name]
# Data to be sent in the mastercache
layers.append(masterLayer)
# Data to be saved out
layer = {#"Name": name,
#"wmsURL": server['wmsURL'],
#"wcsURL": server['wcsURL'],
#"Title": title,
#"Abstract": abstract,
"FirstDate": dimensions['firstDate'],
"LastDate": dimensions['lastDate'],
#"EX_GeographicBoundingBox": exGeographicBoundingBox,
"BoundingBox": boundingBox,
"Dimensions": dimensions['dimensions'],
"Styles": styles }
cleanServerName = server['name'].replace('/', '-')
cleanLayerName = name.replace('/', '-')
# Save out layer cache
utils.saveFile(LAYERCACHEPATH + cleanServerName + "_" + cleanLayerName + FILEEXTENSIONJSON, json.dumps(layer))
subMasterCache['server'][sensorName] = layers
subMasterCache['options'] = server['options']
subMasterCache['wmsURL'] = server['url']
subMasterCache['wcsURL'] = server['wcsurl']
subMasterCache['serverName'] = server['name']
print 'Cache creation complete...'
# Return and save out the cache for this server
return utils.saveFile(SERVERCACHEPATH + server['name'] + FILEEXTENSIONJSON, json.dumps(subMasterCache))
def createCache(server, capabilitiesXML, coverageXML):
#import xml.etree.ElementTree as ET
#from xml.etree.ElementTree import XMLParser
from lxml import etree as ET
import json
# Save out the xml file for later
utils.saveFile(SERVERCACHEPATH + server['name'] + '-GetCapabilities' + FILEEXTENSIONXML, capabilitiesXML)
utils.saveFile(SERVERCACHEPATH + server['name'] + '-DescribeCoverage' + FILEEXTENSIONXML, coverageXML)
print 'Creating caches...'
subMasterCache = {}
subMasterCache['server'] = {}
#parse = XMLParser( encoding="UTF-8" )
# Parse the GetCapabilities XML
#root = ET.XML(capabilitiesXML, parser=parse)
root = ET.fromstring( removeNonUTF8(capabilitiesXML) )
# Parse the DescribeCoverage XML
coverageRoot = ET.fromstring( removeNonUTF8(coverageXML) )
if root.find('./%sCapability/%sLayer/%sLayer' % (WMS_NAMESPACE,WMS_NAMESPACE,WMS_NAMESPACE)) == None:
dirtyCaches.append(server)
return
for service in root.findall('./%sService' % (WMS_NAMESPACE)):
serverTitle = service.find('./%sTitle' % (WMS_NAMESPACE)).text
serverAbstract = service.find('./%sAbstract' % (WMS_NAMESPACE)).text if service.find('./%sAbstract' % (WMS_NAMESPACE)) is not None else None
for product in root.findall('./%sCapability/%sLayer/%sLayer' % (WMS_NAMESPACE,WMS_NAMESPACE,WMS_NAMESPACE)):
sensorName = product.find('./%sTitle' % (WMS_NAMESPACE)).text
if utils.blackfilter(sensorName, productBlackList):
sensorName = utils.replaceAll(sensorName, {' ':'_', '(':'_', ')':'_', '/':'_'})
print sensorName
layers = []
for layer in product.findall('./%sLayer' % (WMS_NAMESPACE)):
name = layer.find('./%sName' % (WMS_NAMESPACE)).text
title = layer.find('./%sTitle' % (WMS_NAMESPACE)).text
abstract = layer.find('./%sAbstract' % (WMS_NAMESPACE)).text
temporal = False
if name not in server['indicators']:
print "NOTICE: Indicator '" + name + "' found on WMS server but not in local config file, ignoring."
continue
#Find the CoverageOffering from DescribeCoverage
coverage = findCoverageNode( coverageRoot, name )
if coverage == None:
print serverTitle + " " + name + " could not be found in DescribeCoverage. Not including."
continue
offsetVectorsArray = coverage.findall( './/%soffsetVector' % (GML_NAMESPACE) )
offsetVectors = []
for i in range( 0 , len( offsetVectorsArray )):
offsetVectors.append(float(offsetVectorsArray[i].text.split(" ")[i]))
exGeographicBoundingBox = {"WestBoundLongitude": layer.find('./%sEX_GeographicBoundingBox/%swestBoundLongitude' % (WMS_NAMESPACE,WMS_NAMESPACE)).text,
"EastBoundLongitude": layer.find('./%sEX_GeographicBoundingBox/%seastBoundLongitude' % (WMS_NAMESPACE,WMS_NAMESPACE)).text,
"SouthBoundLatitude": layer.find('./%sEX_GeographicBoundingBox/%ssouthBoundLatitude' % (WMS_NAMESPACE,WMS_NAMESPACE)).text,
"NorthBoundLatitude": layer.find('./%sEX_GeographicBoundingBox/%snorthBoundLatitude' % (WMS_NAMESPACE,WMS_NAMESPACE)).text}
boundingBox = {"CRS": layer.find('./%sBoundingBox' % (WMS_NAMESPACE)).get('CRS'),
"MinX": layer.find('./%sBoundingBox' % (WMS_NAMESPACE)).get('minx'),
"MaxX": layer.find('./%sBoundingBox' % (WMS_NAMESPACE)).get('maxx'),
"MinY": layer.find('./%sBoundingBox' % (WMS_NAMESPACE)).get('miny'),
"MaxY": layer.find('./%sBoundingBox' % (WMS_NAMESPACE)).get('maxy')}
dimensions = createDimensionsArray(layer, server)
temporal = dimensions['temporal']
styles = createStylesArray(layer)
if server['options']['providerShortTag'] not in providers:
raise Exception("Provider shortTag " + server['options']['providerShortTag'] + " was not in the 'providers.py' file")
# Get the default details for the provider
providerDetails = providers[ server['options']['providerShortTag'] ]
if (layerHasMoreInfo(server['options']['providerShortTag'])):
moreProviderInfo = True
else:
moreProviderInfo = False
if 'providerDetails' in server['indicators'][name]:
# Overwrite any details with the indicator specific details
for i in server['indicators'][name]['providerDetails']:
providerDetails[ i ] = server['indicators'][name]['providerDetails'][ i ]
#import pprint
#pprint.pprint(server['indicators'][name])
#print '-'*40
if utils.blackfilter(name, layerBlackList):
if layerHasMoreInfo(server['indicators'][name]['niceName']):
moreIndicatorInfo = True
else:
moreIndicatorInfo = False
masterLayer = {"Name": name,
"Title": title,
"Abstract": abstract,
"FirstDate": dimensions['firstDate'],
"LastDate": dimensions['lastDate'],
"OffsetVectors": offsetVectors,
"ProviderDetails": providerDetails,
"EX_GeographicBoundingBox": exGeographicBoundingBox,
"MoreIndicatorInfo" : moreIndicatorInfo,
"MoreProviderInfo" : moreProviderInfo }
if name in server['indicators']:
masterLayer['tags'] = server['indicators'][name]
# Data to be sent in the mastercache
layers.append(masterLayer)
# Data to be saved out
layer = {#"Name": name,
#"wmsURL": server['wmsURL'],
#"wcsURL": server['wcsURL'],
#"Title": title,
#"Abstract": abstract,
"FirstDate": dimensions['firstDate'],
"LastDate": dimensions['lastDate'],
"OffsetVectors": offsetVectors,
#"EX_GeographicBoundingBox": exGeographicBoundingBox,
"BoundingBox": boundingBox,
"Dimensions": dimensions['dimensions'],
"Styles": styles }
cleanServerName = server['name'].replace('/', '-')
cleanLayerName = name.replace('/', '-')
# Save out layer cache
utils.saveFile(LAYERCACHEPATH + cleanServerName + "_" + cleanLayerName + FILEEXTENSIONJSON, json.dumps(layer))
subMasterCache['server'][sensorName] = layers
subMasterCache['options'] = server['options']
subMasterCache['wmsURL'] = server['services']['wms']['url']
subMasterCache['wcsURL'] = server['services']['wcs']['url']
subMasterCache['serverName'] = server['name']
print 'Cache creation complete...'
# Return and save out the cache for this server
return utils.saveFile(SERVERCACHEPATH + server['name'] + FILEEXTENSIONJSON, json.dumps(subMasterCache))
mac_last_group = hex(int(ipv4_last_byte))[2:]
lxc_mac_address = lxc_mac_prefix + mac_last_group
lxc_mac_nat_address = lxc_mac_nat_prefix + mac_last_group
lxc_ipv4 = lxc_ipv4_prefix + str(ipv4_last_byte)
lxc_ipv4_with_mask = lxc_ipv4 + "/" + lxc_ipv4_mask
lxc_config_dir = "/var/lib/lxc/" + container_name
if os.path.isdir(lxc_config_dir):
shutil.rmtree(lxc_config_dir)
os.mkdir(lxc_config_dir)
config_path = lxc_config_dir + '/config'
shutil.copyfile(lxc_config_template, config_path)
utils.replaceAll(config_path, '{{CONTAINER_NAME}}', container_name)
utils.replaceAll(config_path, '{{IPV4}}', lxc_ipv4)
utils.replaceAll(config_path, '{{NET_IF}}', lxc_net_if)
utils.replaceAll(config_path, '{{MAC_ADDRESS}}', lxc_mac_address)
utils.replaceAll(config_path, '{{MAC_NAT_ADDRESS}}', lxc_mac_nat_address)
utils.replaceAll(config_path, '{{ARCH}}', lxc_arch)
config_network = lxc_path + '/etc/network/interfaces'
shutil.copyfile(lxc_network_template, config_network)
hosts.write(lxc_ipv4 + " " + lxc_hostname + " # Auto-generated on %s.\n" % date.today())
hosts.close()
lxc_hostname_file = open(lxc_path + "/etc/hostname", "w")
lxc_hostname_file.write(container_name + "\n")
lxc_hostname_file.close()
import utils
m = utils.opener.raw("input/04.txt")[:-1]
rt = [(" ", ","), ("\n", ","), (":", "':'"), (",", "','")]
pp = [eval("{'" + utils.replaceAll(p, rt) + "'}")
for p in m.split("\n\n")] # list of dicts representing passports
btwn = lambda a, f, b: a <= int(f) <= b
s1 = 0
s2 = 0
for p in pp:
if (len(p) == 8) or (len(p) == 7 and "cid" not in p):
s1 += 1
byr, iyr, eyr, hgt, hcl, ecl, pid = p["byr"], p["iyr"], p["eyr"], p[
"hgt"], p["hcl"], p["ecl"], p["pid"]
if (btwn(1920, byr, 2002) and btwn(2010, iyr, 2020)
and btwn(2020, eyr, 2030)) and (
(hgt.endswith("cm") and btwn(150, hgt[:-2], 193)) or
(hgt.endswith("in") and btwn(59, hgt[:-2], 76))
) and (len(hcl) == 7 and hcl[0] == "#" and all(
[x in utils.hexanums for x in hcl[1:]])) and (ecl in {
"amb", "blu", "brn", "gry", "grn", "hzl", "oth"
}) and (len(pid) == 9
and all([x in utils.nums for x in p["pid"]])):
s2 += 1
print("1:", s1)
print("2:", s2)