def build(db_url: str): slack.sendmsg('Rebuilding Who Owns What tables...') cosmetic_dataset_name = 'wow' tables = [ TableInfo(name=name, dataset=cosmetic_dataset_name) for name in parse_created_tables_in_dir(WOW_SQL_DIR, WOW_SCRIPTS) ] with psycopg2.connect(db_url) as conn: install_db_extensions(conn) temp_schema = create_temp_schema_name(cosmetic_dataset_name) with create_and_enter_temporary_schema(conn, temp_schema): run_wow_sql(conn) ensure_schema_exists(conn, WOW_SCHEMA) with save_and_reapply_permissions(conn, tables, WOW_SCHEMA): drop_tables_if_they_exist(conn, tables, WOW_SCHEMA) change_table_schemas(conn, tables, temp_schema, WOW_SCHEMA) # The WoW tables are now ready, but the functions defined by WoW were # in the temporary schema that just got destroyed. Let's re-run only # the function-creating SQL in the WoW schema now. # # Note this means that any client which uses the functions will need # to set their search_path to "{WOW_SCHEMA}, public" or else the function # may not be found or might even crash! print( f"Re-running CREATE FUNCTION statements in the {WOW_SCHEMA} schema..." ) sql = get_all_create_function_sql(WOW_SQL_DIR, WOW_SCRIPTS) run_sql_if_nonempty( conn, sql, initial_sql=f'SET search_path TO {WOW_SCHEMA}, public') slack.sendmsg('Finished rebuilding Who Owns What tables.')
def error_handling(dataset: str): init_rollbar() try: yield except Exception as e: if ROLLBAR_ACCESS_TOKEN: rollbar.report_exc_info(extra_data={"dataset": dataset}) slack.sendmsg( f"Alas, an error occurred when loading the dataset `{dataset}`.", stdout=not isinstance(e, CommandError), ) if isinstance(e, CommandError): print(e.message) sys.exit(1) else: raise
def load_dataset( dataset: str, config: Config = Config(), force_check_urls: bool = False ): """ Load the given dataset using the given configuration. Note that `force_check_urls` is only used by the test suite. This is a bad code smell, but unfortunately it was the easiest way to test the URL-checking functionality with test data. """ if dataset == "wow": import wowutil wowutil.build(config.database_url) return tables = get_tables_for_dataset(dataset) ds = Dataset(dataset, args=config.nycdb_args) ds = reset_files_if_test(ds, config) ds.setup_db() conn = ds.db.conn dbhash = get_dbhash(conn) modtracker = UrlModTracker(get_urls_for_dataset(dataset), dbhash) check_urls = (not config.use_test_data) or force_check_urls if check_urls and not modtracker.did_any_urls_change(): slack.sendmsg( f"The dataset `{dataset}` has not changed since we last retrieved it." ) return slack.sendmsg(f"Downloading the dataset `{dataset}`...") ds.download_files() slack.sendmsg( f"Downloaded the dataset `{dataset}`. Loading it into the database..." ) temp_schema = create_temp_schema_name(dataset) with create_and_enter_temporary_schema(conn, temp_schema): ds.db_import() with save_and_reapply_permissions(conn, tables, "public"): drop_tables_if_they_exist(conn, tables, "public") change_table_schemas(conn, tables, temp_schema, "public") # The dataset's tables are ready, but any functions defined by the # dataset's custom SQL were in the temporary schema that just got # destroyed. Let's re-run only the function-creating SQL for the # dataset now, in the public schema so that clients can use it. run_sql_if_nonempty(conn, get_all_create_function_sql_for_dataset(dataset)) modtracker.update_lastmods() slack.sendmsg(f"Finished loading the dataset `{dataset}` into the database.") print("Success!")
def main(argv: List[str]=sys.argv): sanity_check() NYCDB_DATA_DIR.mkdir(parents=True, exist_ok=True) dataset = os.environ.get('DATASET', '') if len(argv) > 1: dataset = argv[1] if not dataset: print(f"Usage: {argv[0]} <dataset>") print(f"Alternatively, set the DATASET environment variable.") sys.exit(1) try: load_dataset(dataset) except Exception as e: slack.sendmsg(f"Alas, an error occurred when loading the dataset `{dataset}`.") raise e
def update_landlord_search_index(conn): app_id = os.environ.get("ALGOLIA_APP_ID", None) api_key = os.environ.get("ALGOLIA_API_KEY", None) if not app_id or not api_key: slack.sendmsg("Connection to Algolia not configured. Skipping...") return # Initialize the Algolia Searchclient # www.algolia.com/doc/api-client/getting-started/instantiate-client-index/?client=python client = SearchClient.create(app_id, api_key) indices_info_resp = client.list_indices() index_info: Dict = next( (item for item in indices_info_resp["items"] if item["name"] == "wow_landlords") ) index_last_updated = datetime.strptime( index_info["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ" ) print("Our Algolia landlord search index was last updated on: ", index_last_updated) hpd_regs_last_updated = get_hpd_last_updated_date(conn) print("HPD Registrations tables were last updated on: ", hpd_regs_last_updated) if hpd_regs_last_updated < index_last_updated: slack.sendmsg( "No new HPD Registration data to add to Algolia search index. Skipping..." ) return with conn.cursor() as cur: slack.sendmsg("Rebuilding Algolia landlord index...") cur.execute(f"SET search_path TO {WOW_SCHEMA}, public") conn.commit() import portfoliograph.landlord_index portfoliograph.landlord_index.update_landlord_search_index( conn, app_id, api_key ) slack.sendmsg("Finished rebuilding Algolia landlord search index.")
def test_sendmsg_returns_false_when_post_to_webhook_fails( requests_mock, monkeypatch): monkeypatch.setattr(slack, 'SLACK_WEBHOOK_URL', 'http://boop') requests_mock.post(slack.SLACK_WEBHOOK_URL, status_code=500) assert slack.sendmsg('hi') is False
def test_sendmsg_returns_false_when_settings_are_not_defined(monkeypatch): with patch.object(slack.logger, 'debug') as m: monkeypatch.setattr(slack, 'SLACK_WEBHOOK_URL', '') assert slack.sendmsg('hi') is False m.assert_called_with( 'SLACK_WEBHOOK_URL is empty; not sending message.')
def test_text_is_unescaped_if_specified(self): with patch.object(slack, 'send_payload') as m: slack.sendmsg('bop < <', is_safe=True) m.assert_called_with({'text': 'bop < <'})
def test_text_is_escaped_by_default(self): with patch.object(slack, 'send_payload') as m: slack.sendmsg('bop < <') m.assert_called_with({'text': 'bop < <'})
def test_sendmsg_returns_true_on_success(requests_mock, monkeypatch): monkeypatch.setattr(slack, 'SLACK_WEBHOOK_URL', 'http://boop') requests_mock.post(slack.SLACK_WEBHOOK_URL, status_code=200) assert slack.sendmsg('hi') is True
def test_sendmsg_returns_false_when_settings_are_not_defined(monkeypatch): with patch.object(slack.logger, "debug") as m: monkeypatch.setattr(slack, "SLACK_WEBHOOK_URL", "") assert slack.sendmsg("hi") is False m.assert_called_with( "SLACK_WEBHOOK_URL is empty; not sending message.")
def test_text_is_unescaped_if_specified(self): with patch.object(slack, "send_payload") as m: slack.sendmsg("bop < <", is_safe=True) m.assert_called_with({"text": "bop < <"})
def test_text_is_escaped_by_default(self): with patch.object(slack, "send_payload") as m: slack.sendmsg("bop < <") m.assert_called_with({"text": "bop < <"})