def main():
if len(sys.argv) < 4:
print("Please provide: db_name num_runs output.csv query")
return
global con
global c
global db
db = sys.argv[1]
numRuns = int(sys.argv[2])
output = sys.argv[3]
query = int(sys.argv[4])
try:
if db == "duck":
con = duckdb.connect(":memory:")
c = con.cursor()
elif db == "monet":
con = monetdblite.make_connection(":memory:")
c = con.cursor()
elif db == "sqlite":
con = sqlite3.connect(":memory:")
c = con.cursor()
elif db == "pandas":
c = lambda q: sqldf(q, globals())
pandasqlSetup()
runTest(output, numRuns, query)
return
elif db == "sqliteopt":
con = sqlite3.connect(":memory:")
c = con.cursor()
c.execute("PRAGMA synchronous = NORMAL")
db = "sqlite"
else:
print("Not a recognized database")
return
except RuntimeError:
print("Error creating database.")
return
try:
testCon()
except RuntimeError:
print("Tests are failing.")
closeProgram()
try:
createTables()
importData()
except RuntimeError:
print("Error importing data or creating tables.")
closeProgram()
runTest(output, numRuns, query)
def monetdblite_empty_cursor(request, tmp_path):
test_dbfarm = tmp_path.resolve().as_posix()
def finalizer():
monetdblite.shutdown()
if tmp_path.is_dir():
shutil.rmtree(test_dbfarm)
request.addfinalizer(finalizer)
connection = monetdblite.make_connection(test_dbfarm)
cursor = connection.cursor()
return cursor
def monetdblite_cursor_autocommit(request, tmp_path):
test_dbfarm = tmp_path.resolve().as_posix()
def finalizer():
monetdblite.shutdown()
if tmp_path.is_dir():
shutil.rmtree(test_dbfarm)
request.addfinalizer(finalizer)
connection = monetdblite.make_connection(test_dbfarm)
connection.set_autocommit(True)
cursor = connection.cursor()
return (cursor, connection, test_dbfarm)
def test_many_shutdowns(self, monetdblite_cursor_autocommit):
(cursor, connection, dbfarm) = monetdblite_cursor_autocommit
for i in range(10):
cursor.transaction()
cursor.execute('CREATE TABLE integers (i INTEGER)')
cursor.executemany('INSERT INTO integers VALUES (%s)', [[x] for x in range(10)])
cursor.execute('SELECT MIN(i * 3 + 5) FROM integers')
result = cursor.fetchall()
assert result == [[5]], "Incorrect result returned"
connection.close()
connection = monetdblite.make_connection(dbfarm)
connection.set_autocommit(True)
cursor = connection.cursor()
def monetdblite_cursor(request, tmp_path):
test_dbfarm = tmp_path.resolve().as_posix()
def finalizer():
monetdblite.shutdown()
if tmp_path.is_dir():
shutil.rmtree(test_dbfarm)
request.addfinalizer(finalizer)
connection = monetdblite.make_connection(test_dbfarm)
cursor = connection.cursor()
cursor.create('integers', {'i': numpy.arange(10)})
cursor.execute('INSERT INTO integers VALUES(NULL)')
return cursor
def test_transaction_aborted_on_shutdown(self, monetdblite_cursor_autocommit):
(cursor, connection, dbfarm) = monetdblite_cursor_autocommit
cursor.transaction()
cursor.execute('CREATE TABLE integers (i INTEGER)')
cursor.executemany('INSERT INTO integers VALUES (%s)', [[x] for x in range(3)])
cursor.execute('SELECT * FROM integers')
result = cursor.fetchall()
assert result == [[0], [1], [2]], "Incorrect result returned"
connection.close()
connection = monetdblite.make_connection(dbfarm)
cursor = connection.cursor()
if not PY26:
with pytest.raises(monetdblite.DatabaseError):
cursor.execute('SELECT * FROM integers')
def test_commited_on_restart(self, monetdblite_cursor_autocommit):
(cursor, connection, dbfarm) = monetdblite_cursor_autocommit
cursor.transaction()
cursor.execute('CREATE TABLE integers (i INTEGER)')
cursor.executemany('INSERT INTO integers VALUES (%s)', [[x] for x in range(3)])
cursor.execute('SELECT * FROM integers')
result = cursor.fetchall()
assert result == [[0], [1], [2]], "Incorrect result returned"
cursor.commit()
connection.close()
connection = monetdblite.make_connection(dbfarm)
cursor = connection.cursor()
cursor.execute('SELECT * FROM integers')
assert result == [[0], [1], [2]], "Incorrect result returned"
def setUp(self) -> None:
self.db = monetdblite.make_connection(':memory:')
self.dao = MonetDAO(self.db, self.SCHEMA)
self.dao.create_schema()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('-v',
'--version',
action='version',
version=f'Ballcone v{__version__}')
parser.add_argument('-sh',
'--syslog-host',
default='127.0.0.1',
help='syslog host to bind')
parser.add_argument('-sp',
'--syslog-port',
default=65140,
type=int,
help='syslog UDP port to bind')
parser.add_argument('-wh',
'--web-host',
default='127.0.0.1',
help='Web interface host to bind')
parser.add_argument('-wp',
'--web-port',
default=8080,
type=int,
help='Web interface TCP port to bind')
parser.add_argument('-m',
'--monetdb',
default='monetdb',
help='Path to MonetDB database')
parser.add_argument('-ms',
'--monetdb-schema',
default='ballcone',
help='MonetDB schema name')
parser.add_argument('-p',
'--period',
default=5,
type=int,
help='Persistence period, in seconds')
parser.add_argument('-t',
'--top-limit',
default=5,
type=int,
help='Limit for top-n queries')
args = parser.parse_args()
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
connection = monetdblite.make_connection(args.monetdb)
dao = MonetDAO(connection, args.monetdb_schema)
if not dao.schema_exists():
dao.create_schema()
geoip = geolite2.reader()
ballcone = Ballcone(dao, geoip, args.top_limit, args.period)
asyncio.ensure_future(ballcone.persist_timer())
loop = asyncio.get_event_loop()
syslog = loop.create_datagram_endpoint(lambda: SyslogProtocol(ballcone),
local_addr=(args.syslog_host,
args.syslog_port))
loop.run_until_complete(syslog)
# PyInstaller
if getattr(sys, 'frozen', False):
jinja2_loader = cast(
jinja2.BaseLoader,
jinja2.FileSystemLoader(
os.path.join(getattr(sys, '_MEIPASS'), 'templates')))
else:
jinja2_loader = cast(jinja2.BaseLoader,
jinja2.PackageLoader('ballcone'))
app = web.Application()
aiohttp_jinja2.setup(app, loader=jinja2_loader)
handler = WebBallcone(ballcone)
app.router.add_get('/', handler.root, name='root')
app.router.add_get('/services', handler.services, name='services')
app.router.add_get('/services/{service}', handler.service, name='service')
app.router.add_get('/services/{service}/average/{field}',
handler.average_or_count,
name='average')
app.router.add_get('/services/{service}/count/{field}',
handler.average_or_count,
name='count')
app.router.add_get('/services/{service}/count_group/{group}',
handler.count_group,
name='count_group')
app.router.add_get('/sql', handler.sql, name='sql')
app.router.add_post('/sql', handler.sql, name='sql')
app.router.add_get('/nginx', handler.nginx, name='nginx')
web.run_app(app, host=args.web_host, port=args.web_port)
try:
loop.run_forever()
finally:
with suppress(RuntimeError):
for task in asyncio.all_tasks():
task.cancel()
with suppress(asyncio.CancelledError):
loop.run_until_complete(task)
geoip.close()
try:
ballcone.persist()
finally:
connection.close()
conn = sqlite3.connect(tag_sqldb)
c=conn.cursor()
t1 = time.time()
# stags = c.execute("select mbid, weight, mbid_type from tags where weight=74 or weight= 59 or weight between 1 and 22").
stags = c.execute("select mbid, weight, mbid_type from tags").fetchall()
t2 = time.time()
with open('/home/johannes/Dropbox/gsss/thesis/anls/try1/add_data/dump.csv', 'a') as fo:
wr = csv.writer(fo)
wr.writerows(stags)
import monetdblite
conn = monetdblite.connect('/home/johannes/Dropbox/gsss/thesis/anls/try1/add_data/monet2')
conn = monetdblite.make_connection('/home/johannes/Dropbox/gsss/thesis/anls/try1/add_data/monet2')
c2 = conn.cursor()
c2.execute('create table tags (mbid string)')
c2.execute("alter table tags add weight INTEGER")
c2.execute("alter table tags add mbid_type string")
c2.execute("insert into tags (mbid, weight, mbid_type) values ('eee', 33, 'iii')")
c2.execute('select * from tags where weight between 14 and 74')
t1 = time.time()
ddx=c2.fetchall()
t2 = time.time()
def _connect(self):
self._connection = monetdblite.make_connection(self._dbpath, True)
def __init__(self):
self.con = monetdblite.make_connection("tempDb")