def both_external(db):
external1 = databot.Bot('sqlite:///:memory:')
external1.define('p1').append([(1, 'a'), (2, 'b'), (3, 'c')])
external2 = databot.Bot('sqlite:///:memory:')
external2.define('p2')
internal = db.Bot().main(argv=['-v0', 'run'])
internal.define('p1', external1.engine)
internal.define('p2', external2.engine)
return internal
def runbot(pipeline):
"""Automatically sets database to Sqlite file that matches script path.
Example how script path will be converted to sqlite uri:
bots/vlkk/vardai.py -> sqlite:///data/vlkk/vardai.db
"""
dburi = 'sqlite:///%s' % find_data_dir(sys.argv[0])
return databot.Bot(dburi).main(pipeline)
def getbot(path):
db_path = pathlib.Path('data', '%s.db' % path)
if db_path.exists():
module = get_bot_module(path)
dburi = 'sqlite:///data/%s.db' % path
bot = databot.Bot(dburi)
if hasattr(module, 'define'):
module.define(bot)
return bot
else:
raise ValueError("Path '%s' does not exists." % db_path)
def test_missing_pipe_name(db):
external = databot.Bot('sqlite:///:memory:')
external.define('p1').append([(1, 'a'), (2, 'b'), (3, 'c')])
bot = db.Bot().main(argv=['-v0', 'run'])
bot.define('pp', external.engine)
bot.define('p2')
pp, p2 = bot.pipe('pp'), bot.pipe('p2')
assert p2(pp).count() is 0
def main(argv=None, output=sys.stdout):
argv = argv or sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument(
'db', help='path to sqlite datbase or database connection string')
args = parser.parse_args(argv[:1])
bot = databot.Bot(args.db, output=output)
pipeline = {
'pipes': [databot.define(pipe.pipe) for pipe in get_pipe_tables(bot)],
'tasks': [],
}
bot.main(pipeline, argv=argv[1:])
def test_external_write(db):
external = databot.Bot('sqlite:///:memory:')
external.define('p1')
bot = db.Bot().main(argv=['-v0', 'run'])
bot.define('p1', external.engine)
bot.define('p2').append([(1, 'a'), (2, 'b'), (3, 'c')])
p1, p2 = bot.pipe('p1'), bot.pipe('p2')
p1(p2).call(handler)
assert list(p1.items()) == [(1, 'A'), (2, 'B'), (3, 'C')]
assert list(p2.items()) == [(1, 'a'), (2, 'b'), (3, 'c')]
assert list(external.pipe('p1').items()) == [(1, 'A'), (2, 'B'), (3, 'C')]
def test_main(tmpdir):
maindb = str(tmpdir / 'main.db')
bot = databot.Bot(maindb)
bot.define('p1').append([(1, 'a')])
bot.define('p2', str(tmpdir / 'external.db')).append([(2, 'b')])
output = io.StringIO()
main([maindb], output)
assert output.getvalue().splitlines() == [
' id rows source',
' errors left target',
'=================================',
' 1 1 p1',
'---------------------------------',
]
def test_error_limit(limit, errors, left, raises):
def errorif(*keys):
def handler(row):
if row.key in keys:
raise ValueError('Error for key: %r.' % row.key)
else:
yield row.key, row.value.upper()
return handler
bot = databot.Bot()
t1 = bot.define('t1').append([(1, 'a'), (2, 'b'), (3, 'c')])
t2 = bot.define('t2')
if raises:
with pytest.raises(ValueError):
t2(t1).call(errorif(1, 2), error_limit=limit)
else:
t2(t1).call(errorif(1, 2), error_limit=limit)
assert t2(t1).errors.count() == errors
assert t2(t1).count() == left
def Bot(self):
output = io.StringIO()
return databot.Bot(self.engine, output=output, models=self.models)
def bot():
return databot.Bot('sqlite:///:memory:', output=io.StringIO())
('form > table.info_table xpath:tr[contains(th/text(), "Narių balsavimo rezultatai")] '
'css:td.long table.info_table xpath:tr[contains(td/text(), "Už")] css:td[3]:text?'
),
'susilaike':
('form > table.info_table xpath:tr[contains(th/text(), "Narių balsavimo rezultatai")] '
'css:td.long table.info_table xpath:tr[contains(td/text(), "Susilaikė")] css:td[3]:text?'
)
},
'bendru_sutarimu':
('form > table.info_table xpath:tr[contains(th/text(), "Narių balsavimo rezultatai")] '
'css:td.long table[2] xpath:tr[contains(td/text(), "Bendru sutarimu")] css:td:text?'
),
'balsai': [
'#voteprotocol xpath:tr[count(td)>0]', {
'person': 'td[1]:text',
'date': 'td[2]:content',
'value': 'td[3]:text',
}
],
'nedalyvavo':
('form > table.info_table xpath:tr[contains(th/text(), "Nedalyvavo balsavime")] css:td.long:content?'
),
})
bot.compact()
if __name__ == '__main__':
databot.Bot('sqlite:///data/vtaryba.db').argparse(sys.argv[1:], define,
run)