def test_remote_library_partitions(self):
l = self.get_library()
r = l.put(self.bundle)
r = l.get(self.bundle.identity.name)
self.assertEquals(self.bundle.identity.name, r.bundle.identity.name)
for partition in self.bundle.partitions:
r = l.put(partition)
# Get the partition with a name
r = l.get(partition.identity.name)
self.assertTrue(r is not False)
self.assertEquals(partition.identity.name, r.partition.identity.name)
self.assertEquals(self.bundle.identity.name, r.bundle.identity.name)
# Copy all of the newly added files to the server.
l.push()
l2 = get_library('clean')
l2.purge()
r = l2.get('b1DxuZ001')
self.assertTrue(r is not None and r is not False)
self.assertEquals(r.partition.identity.id_,'b1DxuZ001' )
self.assertTrue(os.path.exists(r.partition.database.path))
def demo3(self):
import databundles
import databundles.library as dl
import databundles.geo as dg
from matplotlib import pyplot as plt
import numpy as np
l = dl.get_library()
aa = dg.get_analysis_area(l, geoid='CG0666000')
r = l.find(dl.QueryCommand().identity(id='a2z2HM').partition(table='incidents',space=aa.geoid)).pop()
p = l.get(r.partition).partition
a = aa.new_array()
k = dg.ConstantKernel(9)
print aa
k.apply_add(a, dg.Point(400,1919))
k.apply_add(a, dg.Point(400,1920))
k.apply_add(a, dg.Point(400,1921))
k.apply_add(a, dg.Point(400,1922))
k.apply_add(a, dg.Point(400,1923))
for row in p.query("select date, time, cellx, celly from incidents"):
p = dg.Point(row['cellx'],row['celly'])
k.apply_add(a, p)
a /= np.max(a)
print np.sum(a)
img = plt.imshow(a, interpolation='nearest')
img.set_cmap('spectral_r')
plt.colorbar()
plt.show()
def get_library(self):
"""Clear out the database before the test run"""
l = get_library(self.client_rc, 'client')
l.database.clean()
l.logger.setLevel(logging.DEBUG)
return l
def setUp(self):
"""Setup a library that will download bundles from production, freshly for every run"""
import testbundle.bundle
from databundles.util import rm_rf
self.bundle_dir = os.path.dirname(testbundle.bundle.__file__)
self.rc = get_runconfig((os.path.join(self.bundle_dir,'client-test-config.yaml'),
os.path.join(self.bundle_dir,'bundle.yaml'))
)
self.copy_or_build_bundle()
self.bundle = Bundle()
#print "Deleting: {}".format(self.rc.filesystem.root_dir)
#rm_rf(self.rc.filesystem.root_dir)
self.library = get_library(self.rc, 'production', reset = True)
def _get_sub_aas(self):
import yaml
import databundles.geo as dg
import databundles.library as dl
aa = dg.get_analysis_area(dl.get_library(), geoid=self.config.build.aa_geoid)
with open(self.filesystem.path('extracts','envelopes.yaml')) as f:
envelopes = yaml.load(f)
aas = []
for r in envelopes:
saa = aa.get_aa_from_envelope(r['env'], '', '')
aas.append( (r['area'], saa))
aas = sorted(aas, cmp=lambda x,y: cmp(x[0], y[0]), reverse=True)
return [aa[1] for aa in aas]
def build_aa_map(self):
'''
'''
import databundles.library as dl
import databundles.geo as dg
import random
from numpy import ma
rs = 3
# make a helper to store files in the extracts directory
ed = lambda f: self.filesystem.path('extracts','subs',f+'.tiff')
l = dl.get_library()
aa = dg.get_analysis_area(l, geoid=self.config.build.aa_geoid)
r = l.find(dl.QueryCommand().identity(id='a2z2HM').partition(table='incidents',space=aa.geoid)).pop()
source_partition = l.get(r.partition).partition
k = dg.GaussianKernel(33,11)
sub_aas = [self.get_sub_aas()[1]]
top_a = aa.new_array()
for i, sub_aa in enumerate(sub_aas):
where = sub_aa.is_in_ll_query()
sub_a = sub_aa.new_array()
trans = sub_aa.get_translator()
q = "select * from incidents WHERE {}".format(where)
for row in source_partition.query(q):
p = trans(row['lon'],row['lat'])
k.apply_add(sub_a, p)
print row
sub_aa.write_geotiff(ed(str(i)), dg.std_norm(ma.masked_equal(sub_a,0)))
sub_aa.write_geotiff(ed(str(i)), sub_a)
return True
def test_dump(self):
import time
import logging
l = get_library(self.server_rc, name='default-remote', reset = True)
l.clean()
self.start_server()
l.run_dumper_thread()
l.run_dumper_thread()
self.assertFalse(l.database.needs_dump())
l.put(self.bundle)
self.assertTrue(l.database.needs_dump())
l.run_dumper_thread()
time.sleep(6)
self.assertFalse(l.database.needs_dump())
l.run_dumper_thread()
l.put(self.bundle)
l.run_dumper_thread()
time.sleep(7)
print l.database.needs_dump()
self.assertFalse(l.database.needs_dump())
self.assertEquals(self.bundle.identity.name, l.get(self.bundle.identity.name).identity.name)
l.clean()
self.assertEqual(None, l.get(self.bundle.identity.name))
l.restore()
self.assertEquals(self.bundle.identity.name, l.get(self.bundle.identity.name).identity.name)
def get_library(self, name='default'):
"""Clear out the database before the test run"""
return get_library(self.rc, name=name, reset = True)
def get_library(self):
"""Clear out the database before the test run"""
return get_library(self.rc, reset = True)