def setUp(self, update, record):
self.print_outlet = PrintOutlet()
self.update = update
self.record = record
record.payload = 'test'
record.metadata.return_value = {}
record.__repr__ = lambda x: 'TestRecord(test)'
update.__repr__ = lambda x: 'TestUpdate()'
def run(self):
# planner = APSPlanner()
planner = SchedulePlanner(refresh_interval=0.5)
http_inlet = HttpInlet('https://jsonplaceholder.typicode.com/todos/1',
metadata={CsvOutlet.FILE_MODE: 'a'})
file_inlet = FileInlet('output_03.csv', read_mode=FileInletMode.LINE)
http_inlet2 = HttpInlet(
'https://postman-echo.com/get?foo1=bar1&foo2=bar2',
metadata={
'MONGODB_COLLECTION': 'test_collection2',
'csv_file': 'output_02.csv'
})
print_outlet = PrintOutlet(only_payload=True)
mongo_outlet = MongoOutlet('databay', 'test_collection')
csv_outlet = CsvOutlet('output_03.csv')
planner.add_links(
Link([file_inlet], [print_outlet], timedelta(seconds=0.5)))
planner.add_links(
Link([http_inlet, http_inlet, http_inlet], [csv_outlet],
timedelta(seconds=2)))
# planner.add_links(Link([http_inlet], [mongo_outlet], timedelta(seconds=1), name='first'))
# planner.add_links(Link([http_inlet2, http_inlet2, http_inlet2], [mongo_outlet], timedelta(seconds=5), name='second'))
# planner.add_links(Link([], [], timedelta(seconds=1.5)))
# planner.add_links(Link([alphavantage_inlet], [mongo_outlet], timedelta(seconds=5)))
# planner.add_links(Link([iex_inlet], [mongo_outlet], timedelta(seconds=5)))
planner.start()
class TestPrintOutlet(TestCase):
@patch(fqname(Record), spec=Record)
@patch(fqname(Update), spec=Update)
def setUp(self, update, record):
self.print_outlet = PrintOutlet()
self.update = update
self.record = record
record.payload = 'test'
record.metadata.return_value = {}
record.__repr__ = lambda x: 'TestRecord(test)'
update.__repr__ = lambda x: 'TestUpdate()'
@patch('sys.stdout', new_callable=io.StringIO)
def test_push(self, stdout):
asyncio.run(self.print_outlet._push([self.record], self.update))
self.assertEqual(stdout.getvalue(), 'TestUpdate() TestRecord(test)\n')
@patch('sys.stdout', new_callable=io.StringIO)
def test_push_only_payload(self, stdout):
self.print_outlet.only_payload = True
asyncio.run(self.print_outlet._push([self.record], self.update))
self.assertEqual(stdout.getvalue(), 'TestUpdate() test\n')
@patch('sys.stdout', new_callable=io.StringIO)
def test_push_skip_update(self, stdout):
self.print_outlet.skip_update = True
asyncio.run(self.print_outlet._push([self.record], self.update))
self.assertEqual(stdout.getvalue(), 'TestRecord(test)\n')
from databay.inlets import HttpInlet
from databay.outlets import PrintOutlet
from databay import Link
from databay.planners import ApsPlanner
from databay.outlets.file_outlet import FileOutlet
# filter
class BitcoinInlet(HttpInlet):
async def pull(self, update):
response = await super().pull(update)
return response.get('USD').get('last')
# produce
# stock_inlet = HttpInlet('https://blockchain.info/ticker')
stock_inlet = BitcoinInlet('https://blockchain.info/ticker')
# consume
print_outlet = PrintOutlet(True, True)
file_outlet = FileOutlet('bitcoin_price_1s.txt')
# transfer
link = Link(stock_inlet, [print_outlet, file_outlet],
interval=datetime.timedelta(seconds=1))
planner = ApsPlanner(link)
planner.start()
from databay import Link
from databay.outlets import PrintOutlet
from databay.planners import SchedulePlanner
from datetime import timedelta
from databay import Inlet
import random
class RandomIntInlet(Inlet):
def pull(self, update):
return random.randint(0, 100)
random_int_inlet = RandomIntInlet()
print_outlet = PrintOutlet(only_payload=True)
link = Link(random_int_inlet,
print_outlet,
interval=timedelta(seconds=5),
name='random_ints')
planner = SchedulePlanner(link)
planner.start()
import urllib.request
class WeatherInlet(Inlet):
def __init__(self, api_key: str, city_name: str, *args, **kwargs):
super().__init__(*args, **kwargs)
self.api_key = api_key
self.city_name = city_name
def pull(self, update) -> List[Record]:
url = f'https://api.openweathermap.org/data/2.5/weather?' \
f'q={self.city_name}&' \
f'appid={self.api_key}'
contents = urllib.request.urlopen(url).read().decode('utf8')
formatted = json.loads(contents)
return formatted['weather'][0]['description']
api_key = os.environ.get('OPEN_WEATHER_MAP_API_KEY')
weather_inlet = WeatherInlet(api_key, 'Bangkok')
link = Link(weather_inlet,
PrintOutlet(only_payload=True),
interval=timedelta(seconds=2),
name='bangkok_weather')
planner = APSPlanner(link)
planner.start()