def test_write_points_from_dataframe_with_float_nan(self):
now = pd.Timestamp('1970-01-01 00:00+00:00')
dataframe = pd.DataFrame(data=[[1, float("NaN"), 1.0], [2, 2, 2.0]],
index=[now, now + timedelta(hours=1)],
columns=["column_one", "column_two",
"column_three"])
points = [
{
"points": [
[1, None, 1.0, 0],
[2, 2, 2.0, 3600]
],
"name": "foo",
"columns": ["column_one", "column_two", "column_three", "time"]
}
]
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe})
self.assertListEqual(json.loads(m.last_request.body), points)
def test_write_points_from_dataframe_with_numeric_column_names(self):
"""Test write points from dataframe with numeric columns."""
now = pd.Timestamp('1970-01-01 00:00+00:00')
# df with numeric column names
dataframe = pd.DataFrame(data=[["1", 1, 1.0], ["2", 2, 2.0]],
index=[now, now + timedelta(hours=1)])
points = [
{
"points": [
["1", 1, 1.0, 0],
["2", 2, 2.0, 3600]
],
"name": "foo",
"columns": ['0', '1', '2', "time"]
}
]
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe})
self.assertListEqual(json.loads(m.last_request.body), points)
def test_write_points_from_dataframe_with_period_index(self):
dataframe = pd.DataFrame(data=[["1", 1, 1.0], ["2", 2, 2.0]],
index=[pd.Period('1970-01-01'),
pd.Period('1970-01-02')],
columns=["column_one", "column_two",
"column_three"])
points = [
{
"points": [
["1", 1, 1.0, 0],
["2", 2, 2.0, 86400]
],
"name": "foo",
"columns": ["column_one", "column_two", "column_three", "time"]
}
]
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe})
self.assertListEqual(json.loads(m.last_request.body), points)
def test_write_points_from_dataframe_fails_with_series(self):
now = pd.Timestamp('1970-01-01 00:00+00:00')
dataframe = pd.Series(data=[1.0, 2.0],
index=[now, now + timedelta(hours=1)])
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe})
def test_write_points_from_dataframe_fails_without_time_index(self):
dataframe = pd.DataFrame(data=[["1", 1, 1.0], ["2", 2, 2.0]],
columns=["column_one", "column_two",
"column_three"])
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe})
def test_write_points_from_dataframe_fails_without_time_index(self):
"""Test write points from dataframe that fails without time index."""
dataframe = pd.DataFrame(data=[["1", 1, 1.0], ["2", 2, 2.0]],
columns=["column_one", "column_two",
"column_three"])
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe})
def test_write_points_from_dataframe_fails_with_series(self):
"""Test failed write points from dataframe with series."""
now = pd.Timestamp('1970-01-01 00:00+00:00')
dataframe = pd.Series(data=[1.0, 2.0],
index=[now, now + timedelta(hours=1)])
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe})
def test_write_points_from_dataframe_with_numeric_column_names(self):
now = pd.Timestamp('1970-01-01 00:00+00:00')
# df with numeric column names
dataframe = pd.DataFrame(data=[["1", 1, 1.0], ["2", 2, 2.0]],
index=[now, now + timedelta(hours=1)])
points = [{
"points": [["1", 1, 1.0, 0], ["2", 2, 2.0, 3600]],
"name": "foo",
"columns": ['0', '1', '2', "time"]
}]
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe})
self.assertListEqual(json.loads(m.last_request.body), points)
def test_write_points_from_dataframe_in_batches(self):
now = pd.Timestamp('1970-01-01 00:00+00:00')
dataframe = pd.DataFrame(data=[["1", 1, 1.0], ["2", 2, 2.0]],
index=[now, now + timedelta(hours=1)],
columns=["column_one", "column_two",
"column_three"])
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
self.assertTrue(cli.write_points({"foo": dataframe}, batch_size=1))
def test_write_points_from_dataframe_with_period_index(self):
dataframe = pd.DataFrame(
data=[["1", 1, 1.0], ["2", 2, 2.0]],
index=[pd.Period('1970-01-01'),
pd.Period('1970-01-02')],
columns=["column_one", "column_two", "column_three"])
points = [{
"points": [["1", 1, 1.0, 0], ["2", 2, 2.0, 86400]],
"name":
"foo",
"columns": ["column_one", "column_two", "column_three", "time"]
}]
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe})
self.assertListEqual(json.loads(m.last_request.body), points)
def test_write_points_from_dataframe_in_batches(self):
now = pd.Timestamp('1970-01-01 00:00+00:00')
dataframe = pd.DataFrame(
data=[["1", 1, 1.0], ["2", 2, 2.0]],
index=[now, now + timedelta(hours=1)],
columns=["column_one", "column_two", "column_three"])
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
assert cli.write_points({"foo": dataframe}, batch_size=1) is True
def test_write_points_from_dataframe_with_float_nan(self):
now = pd.Timestamp('1970-01-01 00:00+00:00')
dataframe = pd.DataFrame(
data=[[1, float("NaN"), 1.0], [2, 2, 2.0]],
index=[now, now + timedelta(hours=1)],
columns=["column_one", "column_two", "column_three"])
points = [{
"points": [[1, None, 1.0, 0], [2, 2, 2.0, 3600]],
"name":
"foo",
"columns": ["column_one", "column_two", "column_three", "time"]
}]
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe})
self.assertListEqual(json.loads(m.last_request.body), points)
def test_write_points_from_dataframe_with_time_precision(self):
now = pd.Timestamp('1970-01-01 00:00+00:00')
dataframe = pd.DataFrame(
data=[["1", 1, 1.0], ["2", 2, 2.0]],
index=[now, now + timedelta(hours=1)],
columns=["column_one", "column_two", "column_three"])
points = [{
"points": [["1", 1, 1.0, 0], ["2", 2, 2.0, 3600]],
"name":
"foo",
"columns": ["column_one", "column_two", "column_three", "time"]
}]
points_ms = copy.deepcopy(points)
points_ms[0]["points"][1][-1] = 3600 * 1000
points_us = copy.deepcopy(points)
points_us[0]["points"][1][-1] = 3600 * 1000000
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe}, time_precision='s')
self.assertListEqual(json.loads(m.last_request.body), points)
cli.write_points({"foo": dataframe}, time_precision='m')
self.assertListEqual(json.loads(m.last_request.body), points_ms)
cli.write_points({"foo": dataframe}, time_precision='u')
self.assertListEqual(json.loads(m.last_request.body), points_us)
def test_write_points_from_dataframe_with_time_precision(self):
"""Test write points from dataframe with time precision."""
now = pd.Timestamp('1970-01-01 00:00+00:00')
dataframe = pd.DataFrame(data=[["1", 1, 1.0], ["2", 2, 2.0]],
index=[now, now + timedelta(hours=1)],
columns=["column_one", "column_two",
"column_three"])
points = [
{
"points": [
["1", 1, 1.0, 0],
["2", 2, 2.0, 3600]
],
"name": "foo",
"columns": ["column_one", "column_two", "column_three", "time"]
}
]
points_ms = copy.deepcopy(points)
points_ms[0]["points"][1][-1] = 3600 * 1000
points_us = copy.deepcopy(points)
points_us[0]["points"][1][-1] = 3600 * 1000000
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/db/db/series")
cli = DataFrameClient(database='db')
cli.write_points({"foo": dataframe}, time_precision='s')
self.assertListEqual(json.loads(m.last_request.body), points)
cli.write_points({"foo": dataframe}, time_precision='m')
self.assertListEqual(json.loads(m.last_request.body), points_ms)
cli.write_points({"foo": dataframe}, time_precision='u')
self.assertListEqual(json.loads(m.last_request.body), points_us)
]
df = DataFrameClient('localhost',8086,'root','root')
if({'name':'Econ'} not in df.get_list_database()):
df.create_database('Econ')
df.switch_database('Econ')
#add all items in interested list
start = timeit.default_timer()
for item in interest_list:
if item[0] in df.get_list_series():
df.delete_series(item[0])
results = influx_Fred.interpret(item[1])
results = results.replace(to_replace='NaN',value='.')
results = DataFrame({'value':results['value']})
df.write_points({item[0]:results})
print item
print 'total time in seconds: %.2f' % (timeit.default_timer() - start)
#add all items in processed list
start = timeit.default_timer()
influx_Econ = InfluxDB(db_name='Econ')
for item in processed_list:
if item[0] in df.get_list_series():
df.delete_series(item[0])
results = influx_Econ.interpret(item[1])
results = results.replace(to_replace='NaN',value='.')
results = DataFrame({'value':results['value']})
df.write_points({item[0]:results})
print item
print 'total time in seconds: %.2f' % (timeit.default_timer() - start)
'''
Created on Jul 2, 2015
@author: shaunz
'''
from influxdb.influxdb08 import DataFrameClient
from pandas import DataFrame
from FredAPI import FredLink
from FredLink.FredTicker import Fred_ticker_list
import timeit
fred = FredLink()
df = DataFrameClient('localhost', 8086, 'root', 'root')
if({'name':'FRED'} not in df.get_list_database()):
df.create_database('FRED')
df.switch_database('FRED')
for series in df.get_list_series():
df.delete_series(series)
start = timeit.default_timer()
for item in Fred_ticker_list:
results = fred.get_series(item[1])
results = results.replace(to_replace='NaN',value='.')
data = DataFrame({'value': results})
print item
df.write_points({item[0]:data})
print 'total time in seconds: %.2f' % (timeit.default_timer() - start)
