def test_model(drop=False):
"""Read and write some key value pairs"""
doc = load_metadata()
make_container(doc, drop)
weather_data = doc.root.DataServices[
'WeatherSchema.CambridgeWeather.DataPoints']
weather_notes = doc.root.DataServices[
'WeatherSchema.CambridgeWeather.Notes']
if drop:
load_data(weather_data, SAMPLE_DIR)
load_notes(weather_notes, 'weathernotes.txt', weather_data)
with weather_data.open() as collection:
collection.set_orderby(
core.CommonExpression.orderby_from_str('WindSpeedMax desc'))
collection.set_page(30)
for e in collection.iterpage():
note = e['Note'].get_entity()
if e['WindSpeedMax'] and e['Pressure']:
output(
"%s: Pressure %imb, max wind speed %0.1f knots "
"(%0.1f mph); %s" % (
to_text(e['TimePoint'].value), e['Pressure'].value,
e['WindSpeedMax'].value,
e['WindSpeedMax'].value * 1.15078,
note['Details'] if note is not None else ""))
def test_model():
"""Read and write some key value pairs"""
doc = load_metadata()
InMemoryEntityContainer(doc.root.DataServices['MemCacheSchema.MemCache'])
mem_cache = doc.root.DataServices['MemCacheSchema.MemCache.KeyValuePairs']
test_data(mem_cache)
with mem_cache.open() as collection:
for e in collection.itervalues():
output("%s: %s (expires %s)\n" %
(e['Key'].value, e['Value'].value, str(e['Expires'].value)))
def test_model():
"""Read and write some key value pairs"""
doc = load_metadata()
InMemoryEntityContainer(doc.root.DataServices['MemCacheSchema.MemCache'])
mem_cache = doc.root.DataServices['MemCacheSchema.MemCache.KeyValuePairs']
test_data(mem_cache)
with mem_cache.open() as collection:
for e in collection.itervalues():
output("%s: %s (expires %s)\n" %
(e['Key'].value, e['Value'].value, str(e['Expires'].value)))
def load_notes(weather_notes, file_name, weather_data):
with open(file_name, 'r') as f:
id = 1
with weather_notes.open() as collection:
with weather_data.open() as data:
while True:
line = f.readline()
if len(line) == 0:
break
elif line[0] == '#':
continue
note_words = line.split()
if note_words:
note = collection.new_entity()
note['ID'].set_from_value(id)
start = iso.TimePoint(
date=iso.Date.from_str(note_words[0]),
time=iso.Time(hour=0, minute=0, second=0))
note['StartDate'].set_from_value(start)
end = iso.TimePoint(
date=iso.Date.from_str(
note_words[1]).offset(days=1),
time=iso.Time(hour=0, minute=0, second=0))
note['EndDate'].set_from_value(end)
note['Details'].set_from_value(
' '.join(note_words[2:]))
collection.insert_entity(note)
# now find the data points that match
data.set_filter(
core.CommonExpression.from_str(
"TimePoint ge datetime'%s' and "
"TimePoint lt datetime'%s'" %
(to_text(start), to_text(end))))
for data_point in data.values():
# use values, not itervalues to avoid this bug
# in Python 2.7 http://bugs.python.org/issue10513
data_point['Note'].bind_entity(note)
data.update_entity(data_point)
id = id + 1
with weather_notes.open() as collection:
collection.set_orderby(
core.CommonExpression.orderby_from_str('StartDate desc'))
for e in collection.itervalues():
with e['DataPoints'].open() as affectedData:
output(
"%s-%s: %s (%i data points affected)" %
(to_text(e['StartDate'].value),
to_text(e['EndDate'].value),
e['Details'].value, len(affectedData)))
def test_output(self):
txt_out = io.StringIO()
save_stdout = sys.stdout
try:
sys.stdout = txt_out
py2.output(py2.ul("Going to the\nCaf\xe9"))
finally:
sys.stdout = save_stdout
self.assertTrue(txt_out.getvalue() == py2.ul("Going to the\nCaf\xe9"))
bin_out = io.BytesIO()
try:
sys.stdout = bin_out
py2.output(py2.ul("Going to the\nCaf\xe9"))
finally:
sys.stdout = save_stdout
self.assertTrue(bin_out.getvalue() == b"Going to the\nCaf\xc3\xa9")
def test_output(self):
txt_out = io.StringIO()
save_stdout = sys.stdout
try:
sys.stdout = txt_out
py2.output(py2.ul("Going to the\nCaf\xe9"))
finally:
sys.stdout = save_stdout
self.assertTrue(txt_out.getvalue() == py2.ul("Going to the\nCaf\xe9"))
bin_out = io.BytesIO()
try:
sys.stdout = bin_out
py2.output(py2.ul("Going to the\nCaf\xe9"))
finally:
sys.stdout = save_stdout
self.assertTrue(bin_out.getvalue() == b"Going to the\nCaf\xc3\xa9")
def main(user, password, product_list):
metadata = URI.from_path('metadata.xml')
service = URI.from_path('scihub.copernicus.eu.xml')
credentials = BasicCredentials()
credentials.userid = user
credentials.password = password
credentials.protectionSpace = URI.from_octets(SERVICE).get_canonical_root()
# the full link of odata is https://scihub.copernicus.eu/apihub/odata/v1
# this is for the authentication
c = Client(ca_certs=CERTIFICATE)
c.add_credentials(credentials)
c.load_service(service_root=service, metadata=metadata)
with c.feeds['Products'].open() as products:
for pid in product_list:
p = products[pid]
name = p['Name'].value
size = p['ContentLength'].value
output("Product: %s [%i]\n" % (name, size))
with open('%s.zip' % name, 'wb') as f:
products.read_stream(p.key(), f)
if not product_list:
i = 0
for p in products.itervalues():
name = p['Name'].value
type = p['ContentType'].value
size = p['ContentLength'].value
output("%s\n" % str(p.get_location()))
output(" %s %s[%i]\n" % (name, type, size))
i += 1
if i > MAX_LIST:
break
def main(user, password, product_list):
metadata = URI.from_path('metadata.xml')
service = URI.from_path('scihub.copernicus.eu.xml')
credentials = BasicCredentials()
credentials.userid = user
credentials.password = password
credentials.protectionSpace = URI.from_octets(SERVICE).get_canonical_root()
# the full link of odata is https://scihub.copernicus.eu/apihub/odata/v1
# this is for the authentication
c = Client(ca_certs=CERTIFICATE)
c.add_credentials(credentials)
c.load_service(service_root=service, metadata=metadata)
with c.feeds['Products'].open() as products:
for pid in product_list:
p = products[pid]
name = p['Name'].value
size = p['ContentLength'].value
output("Product: %s [%i]\n" % (name, size))
with open('%s.zip' % name, 'wb') as f:
products.read_stream(p.key(), f)
if not product_list:
i = 0
for p in products.itervalues():
name = p['Name'].value
type = p['ContentType'].value
size = p['ContentLength'].value
output("%s\n" % str(p.get_location()))
output(" %s %s[%i]\n" % (name, type, size))
i += 1
if i > MAX_LIST:
break
#! /usr/bin/env python
import base64
import os
from pyslet.py2 import output
if __name__ == '__main__':
password = base64.encodestring(os.urandom(16)).strip(b"\r\n=")
python_password = repr(password.decode('ascii'))
output("DB_PASSWORD = %s\n" % python_password)
#! /usr/bin/env python
from pyslet.py2 import output
import weather_config as config
if __name__ == '__main__':
password = config.DB_PASSWORD
sql_password = password.replace("'", "''")
output("DROP DATABASE weather;\n")
output("CREATE DATABASE weather;\n")
output("GRANT SELECT,INSERT,UPDATE,DELETE,CREATE,CREATE TEMPORARY TABLES,")
output("DROP,INDEX,ALTER ON weather.* TO weather@localhost ")
output("IDENTIFIED BY '%s';\n" % sql_password)