def build(db_url: str):
slack.sendmsg('Rebuilding Who Owns What tables...')
cosmetic_dataset_name = 'wow'
tables = [
TableInfo(name=name, dataset=cosmetic_dataset_name)
for name in parse_created_tables_in_dir(WOW_SQL_DIR, WOW_SCRIPTS)
]
with psycopg2.connect(db_url) as conn:
install_db_extensions(conn)
temp_schema = create_temp_schema_name(cosmetic_dataset_name)
with create_and_enter_temporary_schema(conn, temp_schema):
run_wow_sql(conn)
ensure_schema_exists(conn, WOW_SCHEMA)
with save_and_reapply_permissions(conn, tables, WOW_SCHEMA):
drop_tables_if_they_exist(conn, tables, WOW_SCHEMA)
change_table_schemas(conn, tables, temp_schema, WOW_SCHEMA)
# The WoW tables are now ready, but the functions defined by WoW were
# in the temporary schema that just got destroyed. Let's re-run only
# the function-creating SQL in the WoW schema now.
#
# Note this means that any client which uses the functions will need
# to set their search_path to "{WOW_SCHEMA}, public" or else the function
# may not be found or might even crash!
print(
f"Re-running CREATE FUNCTION statements in the {WOW_SCHEMA} schema..."
)
sql = get_all_create_function_sql(WOW_SQL_DIR, WOW_SCRIPTS)
run_sql_if_nonempty(
conn, sql, initial_sql=f'SET search_path TO {WOW_SCHEMA}, public')
slack.sendmsg('Finished rebuilding Who Owns What tables.')
def error_handling(dataset: str):
init_rollbar()
try:
yield
except Exception as e:
if ROLLBAR_ACCESS_TOKEN:
rollbar.report_exc_info(extra_data={"dataset": dataset})
slack.sendmsg(
f"Alas, an error occurred when loading the dataset `{dataset}`.",
stdout=not isinstance(e, CommandError),
)
if isinstance(e, CommandError):
print(e.message)
sys.exit(1)
else:
raise
def load_dataset(
dataset: str, config: Config = Config(), force_check_urls: bool = False
):
"""
Load the given dataset using the given configuration.
Note that `force_check_urls` is only used by the test suite. This is a
bad code smell, but unfortunately it was the easiest way to test the URL-checking
functionality with test data.
"""
if dataset == "wow":
import wowutil
wowutil.build(config.database_url)
return
tables = get_tables_for_dataset(dataset)
ds = Dataset(dataset, args=config.nycdb_args)
ds = reset_files_if_test(ds, config)
ds.setup_db()
conn = ds.db.conn
dbhash = get_dbhash(conn)
modtracker = UrlModTracker(get_urls_for_dataset(dataset), dbhash)
check_urls = (not config.use_test_data) or force_check_urls
if check_urls and not modtracker.did_any_urls_change():
slack.sendmsg(
f"The dataset `{dataset}` has not changed since we last retrieved it."
)
return
slack.sendmsg(f"Downloading the dataset `{dataset}`...")
ds.download_files()
slack.sendmsg(
f"Downloaded the dataset `{dataset}`. Loading it into the database..."
)
temp_schema = create_temp_schema_name(dataset)
with create_and_enter_temporary_schema(conn, temp_schema):
ds.db_import()
with save_and_reapply_permissions(conn, tables, "public"):
drop_tables_if_they_exist(conn, tables, "public")
change_table_schemas(conn, tables, temp_schema, "public")
# The dataset's tables are ready, but any functions defined by the
# dataset's custom SQL were in the temporary schema that just got
# destroyed. Let's re-run only the function-creating SQL for the
# dataset now, in the public schema so that clients can use it.
run_sql_if_nonempty(conn, get_all_create_function_sql_for_dataset(dataset))
modtracker.update_lastmods()
slack.sendmsg(f"Finished loading the dataset `{dataset}` into the database.")
print("Success!")
def main(argv: List[str]=sys.argv):
sanity_check()
NYCDB_DATA_DIR.mkdir(parents=True, exist_ok=True)
dataset = os.environ.get('DATASET', '')
if len(argv) > 1:
dataset = argv[1]
if not dataset:
print(f"Usage: {argv[0]} <dataset>")
print(f"Alternatively, set the DATASET environment variable.")
sys.exit(1)
try:
load_dataset(dataset)
except Exception as e:
slack.sendmsg(f"Alas, an error occurred when loading the dataset `{dataset}`.")
raise e
def update_landlord_search_index(conn):
app_id = os.environ.get("ALGOLIA_APP_ID", None)
api_key = os.environ.get("ALGOLIA_API_KEY", None)
if not app_id or not api_key:
slack.sendmsg("Connection to Algolia not configured. Skipping...")
return
# Initialize the Algolia Searchclient
# www.algolia.com/doc/api-client/getting-started/instantiate-client-index/?client=python
client = SearchClient.create(app_id, api_key)
indices_info_resp = client.list_indices()
index_info: Dict = next(
(item for item in indices_info_resp["items"] if item["name"] == "wow_landlords")
)
index_last_updated = datetime.strptime(
index_info["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ"
)
print("Our Algolia landlord search index was last updated on: ", index_last_updated)
hpd_regs_last_updated = get_hpd_last_updated_date(conn)
print("HPD Registrations tables were last updated on: ", hpd_regs_last_updated)
if hpd_regs_last_updated < index_last_updated:
slack.sendmsg(
"No new HPD Registration data to add to Algolia search index. Skipping..."
)
return
with conn.cursor() as cur:
slack.sendmsg("Rebuilding Algolia landlord index...")
cur.execute(f"SET search_path TO {WOW_SCHEMA}, public")
conn.commit()
import portfoliograph.landlord_index
portfoliograph.landlord_index.update_landlord_search_index(
conn, app_id, api_key
)
slack.sendmsg("Finished rebuilding Algolia landlord search index.")
def test_sendmsg_returns_false_when_post_to_webhook_fails(
requests_mock, monkeypatch):
monkeypatch.setattr(slack, 'SLACK_WEBHOOK_URL', 'http://boop')
requests_mock.post(slack.SLACK_WEBHOOK_URL, status_code=500)
assert slack.sendmsg('hi') is False
def test_sendmsg_returns_false_when_settings_are_not_defined(monkeypatch):
with patch.object(slack.logger, 'debug') as m:
monkeypatch.setattr(slack, 'SLACK_WEBHOOK_URL', '')
assert slack.sendmsg('hi') is False
m.assert_called_with(
'SLACK_WEBHOOK_URL is empty; not sending message.')
def test_text_is_unescaped_if_specified(self):
with patch.object(slack, 'send_payload') as m:
slack.sendmsg('bop < <', is_safe=True)
m.assert_called_with({'text': 'bop < <'})
def test_text_is_escaped_by_default(self):
with patch.object(slack, 'send_payload') as m:
slack.sendmsg('bop < <')
m.assert_called_with({'text': 'bop < <'})
def test_sendmsg_returns_true_on_success(requests_mock, monkeypatch):
monkeypatch.setattr(slack, 'SLACK_WEBHOOK_URL', 'http://boop')
requests_mock.post(slack.SLACK_WEBHOOK_URL, status_code=200)
assert slack.sendmsg('hi') is True
def test_sendmsg_returns_false_when_settings_are_not_defined(monkeypatch):
with patch.object(slack.logger, "debug") as m:
monkeypatch.setattr(slack, "SLACK_WEBHOOK_URL", "")
assert slack.sendmsg("hi") is False
m.assert_called_with(
"SLACK_WEBHOOK_URL is empty; not sending message.")
def test_text_is_unescaped_if_specified(self):
with patch.object(slack, "send_payload") as m:
slack.sendmsg("bop < <", is_safe=True)
m.assert_called_with({"text": "bop < <"})
def test_text_is_escaped_by_default(self):
with patch.object(slack, "send_payload") as m:
slack.sendmsg("bop < <")
m.assert_called_with({"text": "bop < <"})
