def test_protect_existing(self):
db_name = os.path.join(self.test_data_dir, 'test_tsdb')
with self.assertRaises(AlreadyExistsError) as context:
create(db_name)
self.assertEqual(str(context.exception),
"PhilDB database already exists at: {0}".format(db_name))
def test_protect_existing(self):
db_name = os.path.join(self.test_data_dir, 'test_tsdb')
with self.assertRaises(AlreadyExistsError) as context:
create(db_name)
self.assertEqual(str(context.exception),
"PhilDB database already exists at: {0}".format(db_name))
def test_add_ts_entry(self):
create(self.temp_dir)
db = PhilDB(self.temp_dir)
db.add_timeseries('410730')
conn = sqlite3.connect(db._PhilDB__meta_data_db())
c = conn.cursor()
c.execute("SELECT * FROM timeseries;")
pk, primary_id = c.fetchone();
self.assertEqual(primary_id, '410730')
def test_add_ts_entry(self):
create(self.temp_dir)
db = PhilDB(self.temp_dir)
db.add_timeseries('410730')
conn = sqlite3.connect(db._PhilDB__meta_data_db())
c = conn.cursor()
c.execute("SELECT * FROM timeseries;")
pk, primary_id = c.fetchone()
self.assertEqual(primary_id, '410730')
def test_add_measurand_entry(self):
create(self.temp_dir)
db = PhilDB(self.temp_dir)
db.add_measurand('P', 'PRECIPITATION', 'Precipitation')
conn = sqlite3.connect(db._PhilDB__meta_data_db())
c = conn.cursor()
c.execute("SELECT * FROM measurand;")
pk, measurand_short_id, measurand_long_id, measurand_description = c.fetchone();
self.assertEqual(measurand_short_id, 'P')
self.assertEqual(measurand_long_id, 'PRECIPITATION')
self.assertEqual(measurand_description, 'Precipitation')
def __init__(self, dbfile, *args, **kwargs):
super(PhilDBSink, self).__init__(*args, **kwargs)
try:
create(dbfile)
except AlreadyExistsError:
pass # Database already exists, so no creation required.
self.db = PhilDB(dbfile)
self.last_known_freq = None
try:
self.db.add_source('SENSOR', 'Data from hardware sensor')
except DuplicateError:
pass # DuplicateError means the source already existed
def test_add_measurand_entry(self):
create(self.temp_dir)
db = PhilDB(self.temp_dir)
db.add_measurand('P', 'PRECIPITATION', 'Precipitation')
conn = sqlite3.connect(db._PhilDB__meta_data_db())
c = conn.cursor()
c.execute("SELECT * FROM measurand;")
pk, measurand_short_id, measurand_long_id, measurand_description = c.fetchone(
)
self.assertEqual(measurand_short_id, 'P')
self.assertEqual(measurand_long_id, 'PRECIPITATION')
self.assertEqual(measurand_description, 'Precipitation')
def write_phildb(file_list, results_file, first_run = False):
if first_run:
create('hrs_phildb')
db = PhilDB('hrs_phildb')
if first_run:
db.add_measurand('Q', 'STREAMFLOW', 'Streamflow')
db.add_source('BOM_HRS', 'Bureau of Meteorology; Hydrological Reference Stations dataset.')
write_times = []
for filename in file_list:
print("Processing file: ", filename, '...')
station_id = os.path.basename(filename).split('_')[0]
print("Using station ID: ", station_id, '...')
streamflow = pd.read_csv(filename, parse_dates=True, index_col=0, header = None)
if first_run:
db.add_timeseries(station_id)
db.add_timeseries_instance(station_id, freq, '', measurand = 'Q', source = 'BOM_HRS')
start = time.time()
db.write(station_id, freq, streamflow, measurand = 'Q', source = 'BOM_HRS')
write_times.append(time.time() - start)
np.savetxt(results_file, np.array(write_times))
def test_create_existing_dir(self):
db_name = os.path.join(self.temp_dir)
create(db_name)
db = PhilDB(db_name)
self.assertEqual(os.path.exists(db._PhilDB__meta_data_db()), True)
def test_create_new(self):
db_name = os.path.join(self.temp_dir, 'new_project')
create(db_name)
db = PhilDB(db_name)
self.assertEqual(os.path.exists(db._PhilDB__meta_data_db()), True)
import shutil
import sys
import datetime
import pandas as pd
from phildb.database import PhilDB
from phildb.create import create
test_tsdb_path = os.path.join(os.path.dirname(__file__), 'test_tsdb')
try:
shutil.rmtree(test_tsdb_path)
except OSError as e:
if e.errno != 2: # Code 2: No such file or directory.
raise
create(test_tsdb_path)
db = PhilDB(test_tsdb_path)
db.add_measurand('Q', 'STREAMFLOW', 'Streamflow')
db.add_source('DATA_SOURCE', '')
db.add_timeseries('410730')
db.add_timeseries_instance('410730',
'D',
'',
measurand='Q',
source='DATA_SOURCE')
db.write('410730',
'D',
pd.Series(index=[
datetime.date(2014, 1, 1),
import shutil
import sys
import datetime
import pandas as pd
from phildb.database import PhilDB
from phildb.create import create
test_tsdb_path = os.path.join(os.path.dirname(__file__), 'test_tsdb')
try:
shutil.rmtree(test_tsdb_path)
except OSError as e:
if e.errno != 2: # Code 2: No such file or directory.
raise
create(test_tsdb_path)
db = PhilDB(test_tsdb_path)
db.add_measurand('Q', 'STREAMFLOW', 'Streamflow')
db.add_source('DATA_SOURCE', '')
db.add_timeseries('410730')
db.add_timeseries_instance('410730', 'D', '', measurand = 'Q', source = 'DATA_SOURCE')
db.write('410730', 'D', pd.Series( index = [datetime.date(2014, 1, 1),
datetime.date(2014, 1, 2),
datetime.date(2014, 1, 3)],
data = [1,2,3]),
source = 'DATA_SOURCE',
measurand = 'Q'
)
def test_create_existing_dir(self):
db_name = os.path.join(self.temp_dir)
create(db_name)
db = PhilDB(db_name)
self.assertEqual(os.path.exists(db._PhilDB__meta_data_db()), True)
def test_create_new(self):
db_name = os.path.join(self.temp_dir, 'new_project')
create(db_name)
db = PhilDB(db_name)
self.assertEqual(os.path.exists(db._PhilDB__meta_data_db()), True)
from datetime import date
from phildb.create import create
from phildb.database import PhilDB
create('pypi_downloads')
from count import write_downloads
db = PhilDB('pypi_downloads')
db.add_source('pypi', 'pypi.python.org')
db.add_measurand('last_day', 'last_day', 'Downloads in the last day')
db.add_measurand('last_week', 'last_week', 'Downloads in the last week')
db.add_measurand('last_month', 'last_month', 'Downloads in the last month')
db.add_measurand('total', 'total', 'Total downloads')
# Write some download information I had manually collected over the last few days
write_downloads(
{
'info': {
'name': 'PhilDB',
'downloads': {'last_day': 6, 'last_month': 572, 'last_week': 74}
}
}, date(2015, 11, 12)
)
write_downloads(
{
'info': {
'name': 'PhilDB',
'downloads': {'last_day': 20, 'last_month': 596, 'last_week': 92}
}