def test_fetch_multiple_pages_yesdb(self, http_mocker):
http_mocker.get(self.test_re, text=self.mock_response)
initdb_args = Postgresql.DEFAULT_SETTINGS['initdb_args']
initdb_args = ' '.join([initdb_args, '-E UTF-8'])
db = Postgresql(initdb_args=initdb_args)
fl = self.get_fl(override_config={
'lister': {
'cls': 'local',
'args': {
'db': db.url()
}
}
})
self.init_db(db, fl.MODEL)
self.disable_storage_and_scheduler(fl)
fl.run(min_bound=self.first_index)
self.assertEqual(fl.db_last_index(), self.last_index)
partitions = fl.db_partition_indices(5)
self.assertGreater(len(partitions), 0)
for k in partitions:
self.assertLessEqual(len(k), 5)
self.assertGreater(len(k), 0)
class TestDataStoreInitialisePostgres(TestCase):
def setUp(self):
self.postgres = None
try:
self.postgres = Postgresql(database="test",
host="localhost",
user="******",
port=55527)
except RuntimeError:
print("PostgreSQL database couldn't be created! "
"Test is skipping.")
def tearDown(self):
try:
self.postgres.stop()
except AttributeError:
return
def test_postgres_initialise(self):
"""Test whether schemas created successfully on PostgresSQL"""
if self.postgres is None:
self.skipTest("Postgres is not available. Test is skipping")
data_store_postgres = DataStore(
db_username="******",
db_password="",
db_host="localhost",
db_port=55527,
db_name="test",
)
# inspector makes it possible to load lists of schema, table, column
# information for the sqlalchemy engine
inspector = inspect(data_store_postgres.engine)
table_names = inspector.get_table_names()
schema_names = inspector.get_schema_names()
# there must be no table, and no schema for datastore at the beginning
self.assertEqual(len(table_names), 0)
self.assertNotIn("datastore_schema", schema_names)
# creating database from schema
data_store_postgres.initialise()
inspector = inspect(data_store_postgres.engine)
table_names = inspector.get_table_names()
schema_names = inspector.get_schema_names()
# 11 tables must be created. A few of them tested
self.assertEqual(len(table_names), 11)
self.assertIn("Entry", table_names)
self.assertIn("Platforms", table_names)
self.assertIn("State", table_names)
self.assertIn("Datafiles", table_names)
self.assertIn("Nationalities", table_names)
# datastore_schema must be created
self.assertIn("datastore_schema", schema_names)
def app(tmpdir_factory):
"""Method to create an app for testing."""
# need to import this late as it might have side effects
from app import app as app_
from app import celery
# need to save old configurations of the app
# to restore them later upon tear down
old_url_map = copy.copy(app_.url_map)
old_view_functions = copy.copy(app_.view_functions)
app_.testing = True
app_.debug = False
old_config = copy.copy(app_.config)
# set task always eager to true in celery configurations in order to run celery tasks synchronously
app_.config['task_always_eager'] = True
celery.conf.update(app_.config)
# update configuration file path
global_config = yaml.load(
open(
path.abspath(path.dirname(__file__) +
"/unittest_data/config.yml")))
app_.config['system_config'] = global_config
# update conditions file path
conditions = yaml.load(
open(
path.abspath(
path.dirname(__file__) + "/unittest_data/conditions.yml")))
app_.config['conditions'] = conditions
config = configparser.ConfigParser()
config.read(
path.abspath(path.dirname(__file__) + "/unittest_data/config.ini"))
app_.config['dev_config'] = config
temp_tasks = tmpdir_factory.mktemp('tasks')
temp_reports = tmpdir_factory.mktemp('reports')
# update upload directories path
app_.config['dev_config']['UPLOADS']['task_dir'] = str(
temp_tasks) # path to task_ids file upload
app_.config['dev_config']['UPLOADS']['report_dir'] = str(
temp_reports) # path to non compliant report upload
# initialize temp database and yield app
postgresql = Postgresql()
app_.config['SQLALCHEMY_DATABASE_URI'] = postgresql.url()
yield app_
# restore old configs after successful session
app_.url_map = old_url_map
app_.view_functions = old_view_functions
app_.config = old_config
shutil.rmtree(path=str(temp_tasks))
shutil.rmtree(path=str(temp_reports))
postgresql.stop()
class TestDatabaseReflection(TestCase):
def setUp(self):
self.postgresql = Postgresql()
self.engine = create_engine(self.postgresql.url())
def tearDown(self):
self.postgresql.stop()
def test_split_table(self):
assert dbreflect.split_table("staging.incidents") == ("staging",
"incidents")
assert dbreflect.split_table("incidents") == (None, "incidents")
with self.assertRaises(ValueError):
dbreflect.split_table("blah.staging.incidents")
def test_table_object(self):
assert isinstance(dbreflect.table_object("incidents", self.engine),
Table)
def test_reflected_table(self):
self.engine.execute("create table incidents (col1 varchar)")
assert dbreflect.reflected_table("incidents", self.engine).exists()
def test_table_exists(self):
self.engine.execute("create table incidents (col1 varchar)")
assert dbreflect.table_exists("incidents", self.engine)
assert not dbreflect.table_exists("compliments", self.engine)
def test_table_has_data(self):
self.engine.execute("create table incidents (col1 varchar)")
self.engine.execute("create table compliments (col1 varchar)")
self.engine.execute("insert into compliments values ('good job')")
assert dbreflect.table_has_data("compliments", self.engine)
assert not dbreflect.table_has_data("incidents", self.engine)
def test_table_has_duplicates(self):
self.engine.execute("create table events (col1 int, col2 int)")
assert not dbreflect.table_has_duplicates("events", ['col1', 'col2'],
self.engine)
self.engine.execute("insert into events values (1,2)")
self.engine.execute("insert into events values (1,3)")
assert dbreflect.table_has_duplicates("events", ['col1'], self.engine)
assert not dbreflect.table_has_duplicates("events", ['col1', 'col2'],
self.engine)
self.engine.execute("insert into events values (1,2)")
assert dbreflect.table_has_duplicates("events", ['col1', 'col2'],
self.engine)
def test_table_has_column(self):
self.engine.execute("create table incidents (col1 varchar)")
assert dbreflect.table_has_column("incidents", "col1", self.engine)
assert not dbreflect.table_has_column("incidents", "col2", self.engine)
def test_column_type(self):
self.engine.execute("create table incidents (col1 varchar)")
assert dbreflect.column_type("incidents", "col1",
self.engine) == VARCHAR
def setUp(self):
self.postgres = None
try:
self.postgres = Postgresql(database="test",
host="localhost",
user="******",
port=55527)
except RuntimeError:
print("PostgreSQL database couldn't be created! "
"Test is skipping.")
def dsn():
postgres = Postgresql()
scheduler = new_test_scheduler(postgres.url())
scheduler.setup_database()
scheduler.commit()
yield postgres.url()
scheduler.close()
postgres.stop()
def app():
"""api fixture"""
from flask_bp.api import app as app_
app_.testing = True
app_.debug = False
# initialize a temp db instance
postgresql = Postgresql()
app_.config['SQLALCHEMY_DATABASE_URI'] = postgresql.url()
yield app_
postgresql.stop()
class SqlLayer(PloneSandboxLayer):
default_bases = (PLONE_FIXTURE,)
class Session(dict):
def set(self, key, value):
self[key] = value
def start_postgres(self):
self.postgres = Postgresql()
return self.postgres.url().replace(
'postgresql://', 'postgresql+psycopg2://')
def stop_postgres(self):
self.postgres.stop()
def init_config(self, dsn):
config = getConfiguration()
if not hasattr(config, 'product_config'):
config.product_config = {}
config.product_config['seantis.reservation'] = dict(dsn=dsn)
setConfiguration(config)
def setUpZope(self, app, configurationContext):
# Set up sessioning objects
app.REQUEST['SESSION'] = self.Session()
ZopeTestCase.utils.setupCoreSessions(app)
self.init_config(dsn=self.start_postgres())
import seantis.reservation
xmlconfig.file(
'configure.zcml',
seantis.reservation,
context=configurationContext
)
self.loadZCML(package=seantis.reservation)
def setUpPloneSite(self, portal):
quickInstallProduct(portal, 'plone.app.dexterity')
quickInstallProduct(portal, 'seantis.reservation')
applyProfile(portal, 'seantis.reservation:default')
def tearDownZope(self, app):
z2.uninstallProduct(app, 'seantis.reservation')
self.stop_postgres()
def app(tmpdir_factory):
"""Method to create an app for testing."""
# need to import this late as it might have side effects
from app import app as app_, db
# need to save old configurations of the app
# to restore them later upon tear down
old_url_map = copy.copy(app_.url_map)
old_view_functions = copy.copy(app_.view_functions)
app_.testing = True
app_.debug = False
old_config = copy.copy(app_.config)
# initialize temp database and yield app
with Postgresql() as postgresql:
app_.config['SQLALCHEMY_DATABASE_URI'] = postgresql.url()
temp_download = tmpdir_factory.mktemp('downloads')
temp_uploads = tmpdir_factory.mktemp('uploads')
app_.config['DPS_UPLOADS'] = str(temp_uploads)
app_.config['DPS_DOWNLOADS'] = str(temp_download)
yield app_
# restore old configs after successful session
app_.url_map = old_url_map
app_.view_functions = old_view_functions
app_.config = old_config
shutil.rmtree(str(temp_download))
shutil.rmtree(str(temp_uploads))
postgresql.stop()
def sql_sc(sc):
with Postgresql() as postgresql:
conn = psycopg2.connect(**postgresql.dsn())
cur = conn.cursor()
cur.execute(
'CREATE TABLE test (id serial PRIMARY KEY, a integer, b integer, c text, d varchar(255), e boolean, f float, grp integer)'
)
cur.execute(
"INSERT INTO test (a, b, c, d, e, f, grp) VALUES (10, 0, 'hi', 'hello', true, 2.0, 0)"
)
cur.execute(
"INSERT INTO test (a, b, c, d, e, f, grp) VALUES (20, 0, 'hi', 'hello', true, 2.0, 0)"
)
cur.execute(
"INSERT INTO test (a, b, c, d, e, f, grp) VALUES (30, 0, 'hi', 'hello', true, 2.0, 1)"
)
cur.execute('CREATE TABLE jobs (id serial PRIMARY KEY, name text)')
cur.execute(
'CREATE TABLE test2 (id serial PRIMARY KEY, b integer, s text)')
conn.commit()
sql_params = postgresql.dsn()
sql_config = protobufs.SQLConfig(hostaddr=sql_params['host'],
port=sql_params['port'],
dbname=sql_params['database'],
user=sql_params['user'],
adapter='postgres')
yield sc, SQLStorage(config=sql_config, job_table='jobs'), cur
cur.close()
conn.close()
class TestConfig(object):
DEBUG = True
TESTING = True
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_DATABASE_URI = Postgresql().url()
ENV = 'test'
TESTING = True
ADMIN = os.getenv('ADMIN')
def postgres():
"""
Postgres fixture - starts a postgres instance inside a temp directory
and closes it after tests are done
"""
with Postgresql() as postgresql:
yield postgresql
postgresql.stop()
class TestDatabaseReflection(TestCase):
def setUp(self):
self.postgresql = Postgresql()
self.engine = create_engine(self.postgresql.url())
def tearDown(self):
self.postgresql.stop()
def test_split_table(self):
assert dbreflect.split_table('staging.incidents') == ('staging',
'incidents')
assert dbreflect.split_table('incidents') == (None, 'incidents')
with self.assertRaises(ValueError):
dbreflect.split_table('blah.staging.incidents')
def test_table_object(self):
assert isinstance(dbreflect.table_object('incidents', self.engine),
Table)
def test_reflected_table(self):
self.engine.execute('create table incidents (col1 varchar)')
assert dbreflect.reflected_table('incidents', self.engine).exists()
def test_table_exists(self):
self.engine.execute('create table incidents (col1 varchar)')
assert dbreflect.table_exists('incidents', self.engine)
assert not dbreflect.table_exists('compliments', self.engine)
def test_table_has_data(self):
self.engine.execute('create table incidents (col1 varchar)')
self.engine.execute('create table compliments (col1 varchar)')
self.engine.execute('insert into compliments values (\'good job\')')
assert dbreflect.table_has_data('compliments', self.engine)
assert not dbreflect.table_has_data('incidents', self.engine)
def test_table_has_column(self):
self.engine.execute('create table incidents (col1 varchar)')
assert dbreflect.table_has_column('incidents', 'col1', self.engine)
assert not dbreflect.table_has_column('incidents', 'col2', self.engine)
def test_column_type(self):
self.engine.execute('create table incidents (col1 varchar)')
assert dbreflect.column_type('incidents', 'col1',
self.engine) == VARCHAR
def app():
_app = create_app()
with Postgresql() as postgresql:
_app.config['SQLALCHEMY_DATABASE_URI'] = postgresql.url()
ctx = _app.app_context()
ctx.push()
yield _app
ctx.pop()
def postgres():
try:
import psycopg2
except ImportError:
yield None # PostgreSQL database adapter is unavailable on this system
else:
try:
with Postgresql() as pg:
yield pg
except FileNotFoundError:
yield None # PostgreSQL is unavailable on this system
def app():
_app = flask_app
_app.config.from_object(__name__ + '.TestConfig')
with Postgresql() as postgresql:
_app.config['SQLALCHEMY_DATABASE_URI'] = postgresql.url()
ctx = _app.app_context()
ctx.push()
yield _app
ctx.pop()
def app():
with Postgresql() as postgresql:
_app = create_app(
config_object=TestingConfig,
settings_override={'SQLALCHEMY_DATABASE_URI': postgresql.url()})
ctx = _app.app_context()
ctx.push()
yield _app
ctx.pop()
def test_db(app):
"""Context manager to provide a URL for the test database. If one is
configured, then that is used. Otherwise one is created using
testing.postgresql."""
test_db_url = app.config.get('SQLALCHEMY_DATABASE_URI', None)
if test_db_url:
# Test database configured manually, use that
yield test_db_url
else:
# No test database configured, create one using testing.postgresql
with Postgresql() as postgresql:
yield postgresql.url()
def app(tmpdir_factory):
"""Method to create an app for testing."""
# need to import this late as it might have side effects
from app import app as app_
# need to save old configurations of the app
# to restore them later upon tear down
old_url_map = copy.copy(app_.url_map)
old_view_functions = copy.copy(app_.view_functions)
app_.testing = True
app_.debug = False
old_config = copy.copy(app_.config)
# update configuration file path
global_config = yaml.safe_load(
open(path.abspath(path.dirname(__file__) + "/testdata/config.yml")))
app_.config['system_config'] = global_config
# update configuration file path
config = configparser.ConfigParser()
config.read(
path.abspath(path.dirname(__file__) + "/testdata/config_test.ini"))
temp_lists = tmpdir_factory.mktemp('uploads')
# update upload directories path
app_.config['system_config'] = config
app_.config['system_config']['UPLOADS']['list_dir'] = str(temp_lists)
# initialize temp database and yield app
postgresql = Postgresql()
app_.config['SQLALCHEMY_DATABASE_URI'] = postgresql.url()
yield app_
# restore old configs after successful session
app_.url_map = old_url_map
app_.view_functions = old_view_functions
app_.config = old_config
shutil.rmtree(path=str(temp_lists))
postgresql.stop()
def _app(request):
app = create_app('test')
app.config['DEBUG'] = True
app.config['TESTING'] = True
with Postgresql() as postgresql:
app.config['SQLALCHEMY_DATABASE_URI'] = postgresql.url()
with app.app_context():
db.init_app(app)
db.create_all()
populate(db)
yield app
def app(mocked_config, tmpdir_factory):
"""Method to create an app for testing."""
# need to import this late as it might have side effects
from app import app as app_
# need to save old configurations of the app
# to restore them later upon tear down
old_url_map = copy.copy(app_.url_map)
old_view_functions = copy.copy(app_.view_functions)
app_.testing = True
app_.debug = False
old_config = copy.copy(app_.config)
# initialize temp database and yield app
postgresql = Postgresql()
# monkey patch temp dirs
temp_lists = tmpdir_factory.mktemp('lists')
mocked_config['lists']['path'] = str(temp_lists)
temp_uploads = tmpdir_factory.mktemp('uploads')
mocked_config['global']['upload_directory'] = str(temp_uploads)
app_.config['drs_config'] = mocked_config
app_.config['SQLALCHEMY_DATABASE_URI'] = postgresql.url()
app_.config['DRS_UPLOADS'] = str(temp_uploads)
app_.config['DRS_LISTS'] = str(temp_lists)
app_.config['CORE_BASE_URL'] = mocked_config['global']['core_api_v2']
app_.config['DVS_BASE_URL'] = mocked_config['global']['dvs_api_v1']
yield app_
# restore old configs after successful session
app_.url_map = old_url_map
app_.view_functions = old_view_functions
app_.config = old_config
shutil.rmtree(str(temp_lists))
shutil.rmtree(str(temp_uploads))
postgresql.stop()
def app():
_app = create_app()
with Postgresql() as postgresql:
_app.config.update({
"TESTING": True,
"SQLALCHEMY_DATABASE_URI": postgresql.url(),
"SECURITY_PASSWORD_HASH": "plaintext",
"SECURITY_PASSWORD_SCHEMES": ['plaintext'],
"SECURITY_HASHING_SCHEMES": ["plaintext"],
"SECURITY_DEPRECATED_HASHING_SCHEMES": [],
})
ctx = _app.app_context()
ctx.push()
yield _app
ctx.pop()
from app.db.tables import Base
from sqlalchemy import create_engine
from contextlib2 import contextmanager
from sqlalchemy.orm import sessionmaker
from testing.postgresql import Postgresql
from app.db.crud import (get_near_users, add_user, get_all_users, delete_user,
get_user_by_id, is_authenticated_user,
get_user_by_email, update_categories_to_user)
# Tests logger.
logger = logging.getLogger()
logging.basicConfig(level=logging.DEBUG)
# Startup test db.
fake = Faker()
temp_test_db = Postgresql()
engine = create_engine(temp_test_db.url(), echo=True)
MockedSession = sessionmaker(bind=engine)
logger.info("Tests DB was created: %s", temp_test_db.url())
@contextmanager
def mocked_transaction():
s = MockedSession()
try:
yield s
except Exception:
s.rollback()
raise
def start_postgres(self):
self.postgres = Postgresql()
return self.postgres.url().replace(
'postgresql://', 'postgresql+psycopg2://')
scheduler.rollback()
raise
return json.dumps(
{'status': 'success', 'message': 'A reservation has been made'}
)
@app.route('/')
def index():
return render_template('index.html')
if __name__ == '__main__':
postgresql = Postgresql()
try:
context = libres.registry.register_context('flask-exmaple')
context.set_setting('dsn', postgresql.url())
scheduler = libres.new_scheduler(
context, 'Test Scheduler', timezone='Europe/Zurich'
)
scheduler.setup_database()
scheduler.commit()
app.run(debug=True)
finally:
postgresql.stop()
import os
import pytest
from dj_database_url import parse
from django.conf import settings
from testing.postgresql import Postgresql
postgres = os.environ.get("POSTGRESQL_PATH")
initdb = os.environ.get("INITDB_PATH")
_POSTGRESQL = Postgresql(postgres=postgres, initdb=initdb)
@pytest.hookimpl(tryfirst=True)
def pytest_load_initial_conftests(early_config, parser, args):
os.environ["DJANGO_SETTINGS_MODULE"] = early_config.getini(
"DJANGO_SETTINGS_MODULE")
settings.DATABASES["default"] = parse(_POSTGRESQL.url())
settings.DATABASES["dashboard"] = parse(_POSTGRESQL.url())
def pytest_unconfigure(config):
_POSTGRESQL.stop()
def setUp(self):
self.postgresql = Postgresql()
self.engine = create_engine(self.postgresql.url())
def postgresql(request):
psql = Postgresql()
yield psql
psql.stop()
import os
import pytest
from dj_database_url import parse
from django.conf import settings
from testing.postgresql import Postgresql
_POSTGRESQL = Postgresql()
@pytest.hookimpl(tryfirst=True)
def pytest_load_initial_conftests(early_config, parser, args):
os.environ["DJANGO_SETTINGS_MODULE"] = early_config.getini("DJANGO_SETTINGS_MODULE")
settings.DATABASES["default"] = parse(_POSTGRESQL.url())
settings.DATABASES["dashboard"] = parse(_POSTGRESQL.url())
def pytest_unconfigure(config):
_POSTGRESQL.stop()
def postgres():
with Postgresql() as pg:
yield pg
def postgres_db():
with Postgresql() as postgresql:
yield create_engine(postgresql.url())