def add_arguments(self, parser):
parser.add_argument("--only", choices=list(MATERIALIZED_VIEWS.keys()))
parser.add_argument(
"--leave-sql",
action="store_true",
help="Leave the generated SQL files instead of cleaning them after script completion.",
)
parser.add_argument(
"--leave-old",
action="store_true",
help="Leave the old materialized views instead of dropping them from the DB.",
)
parser.add_argument(
"--temp-dir",
type=Path,
help="Choose a non-default directory to store materialized view SQL files.",
default=DEFAULT_MATIVEW_DIR,
)
parser.add_argument(
"--temp-chunked-dir",
type=Path,
help="Choose a non-default directory to store materialized view SQL files.",
default=DEFAULT_CHUNKED_MATIVEW_DIR,
)
parser.add_argument(
"--dependencies", action="store_true", help="Run the SQL dependencies before the materialized view SQL."
)
parser.add_argument(
"--chunk-count", default=10, help="Number of chunks to split chunked matviews into", type=int
)
def mock_matviews_qs(monkeypatch):
"""Mocks all matvies to a single mock queryset"""
mock_qs = MockSet() # mock queryset
for k, v in MATERIALIZED_VIEWS.items():
if k not in ["tas_autocomplete_matview"]:
monkeypatch.setattr(
"usaspending_api.awards.models_matviews.{}.objects".format(
v["model"].__name__), mock_qs)
yield mock_qs
mock_qs.delete()
def test_matview_sql_generator(
convert_traditional_views_to_materialized_views):
"""
The goal of this test is to ensure that we can successfully create materialized views using our homegrown
materialized view generator. We will not test the validity of the data contained therein, just that we
can create the materialized views without issue.
"""
# Run through all of the materialized views and perform a simple count query.
for matview in MATERIALIZED_VIEWS.values():
# This will fail if the materialized view doesn't exist for whatever reason which is what we want.
matview["model"].objects.count()
def add_arguments(self, parser):
parser.add_argument(
"--only",
choices=list(MATERIALIZED_VIEWS.keys()) + ["none"],
help=
"If matviews are listed with this option, only those matviews will be run. 'none' will result in no matviews being run",
)
parser.add_argument(
"--leave-sql",
action="store_true",
help=
"Leave the generated SQL files instead of cleaning them after script completion.",
)
parser.add_argument(
"--leave-old",
action="store_true",
help=
"Leave the old materialized views instead of dropping them from the DB.",
)
parser.add_argument(
"--temp-dir",
type=Path,
help=
"Choose a non-default directory to store materialized view SQL files.",
default=DEFAULT_MATIVEW_DIR,
)
parser.add_argument(
"--temp-chunked-dir",
type=Path,
help=
"Choose a non-default directory to store materialized view SQL files.",
default=DEFAULT_CHUNKED_MATIVEW_DIR,
)
parser.add_argument(
"--dependencies",
action="store_true",
help="Run the SQL dependencies before the materialized view SQL.")
parser.add_argument(
"--chunk-count",
default=10,
help="Number of chunks to split chunked matviews into",
type=int)
parser.add_argument(
"--include-chunked-matviews",
action="store_true",
help=
"Chunked Transaction Search matviews will be refreshed and inserted into table",
)
parser.add_argument("--index-concurrency",
default=20,
help="Number of indexes to be created at once",
type=int)
def add_arguments(self, parser):
parser.add_argument("--only", choices=list(MATERIALIZED_VIEWS.keys()))
parser.add_argument(
"--leave-sql",
action="store_true",
help="Leave the generated SQL files instead of cleaning them after script completion.",
)
parser.add_argument(
"--temp-dir",
type=Path,
help="Choose a non-default directory to store materialized view SQL files.",
default=DEFAULT_MATIVEW_DIR,
)
from django.conf import settings
from django.db import connection
from fiscalyear import datetime
from usaspending_api.common.matview_manager import (
OVERLAY_VIEWS,
DEPENDENCY_FILEPATH,
MATERIALIZED_VIEWS,
MATVIEW_GENERATOR_FILE,
DEFAULT_MATIVEW_DIR,
)
from usaspending_api.references.models import Agency
logger = logging.getLogger(__name__)
TEMP_SQL_FILES = [DEFAULT_MATIVEW_DIR / val["sql_filename"] for val in MATERIALIZED_VIEWS.values()]
def read_text_file(filepath):
with open(filepath, "r") as plaintext_file:
file_content_str = plaintext_file.read()
return file_content_str
def convert_string_to_datetime(input: str) -> datetime.datetime:
"""Parse a string into a datetime object"""
return parser.parse(input)
def convert_string_to_date(input: str) -> datetime.date:
"""Parse a string into a date object"""
from django.db import connection
from fiscalyear import FiscalDateTime, FiscalQuarter, datetime, FiscalDate
from usaspending_api.common.exceptions import InvalidParameterException
from usaspending_api.common.matview_manager import (
OVERLAY_VIEWS,
DEPENDENCY_FILEPATH,
MATERIALIZED_VIEWS,
MATVIEW_GENERATOR_FILE,
DEFAULT_MATIVEW_DIR,
)
from usaspending_api.references.models import Agency
logger = logging.getLogger(__name__)
TEMP_SQL_FILES = [
DEFAULT_MATIVEW_DIR / val["sql_filename"]
for val in MATERIALIZED_VIEWS.values()
]
def read_text_file(filepath):
with open(filepath, "r") as plaintext_file:
file_content_str = plaintext_file.read()
return file_content_str
def validate_date(date):
if not isinstance(date, (datetime.datetime, datetime.date)):
raise TypeError("Incorrect parameter type provided")
if not (date.day or date.month or date.year):
raise Exception("Malformed date object provided")