def _extract_data(self, shot_type: str) -> pd.DataFrame:
ts = TableauScraper()
ts.loads(self.url)
# set type of metric and demographic (initiated is fetched by default)
book = ts.getWorkbook()
# get all possible paramaters
params = pd.DataFrame(book.getParameters())
shot_vals = params.loc[params["column"] == "View By", "values"].iloc[0]
demo_vals = params.loc[params["column"] == "Key Metrics",
"values"].iloc[0]
assert self.demographic_col_name in demo_vals
book.setParameter("Key Metrics", self.demographic_col_name)
if shot_type == "complete":
print(f"set parameter to: {shot_vals[1]}")
book.setParameter("View By", shot_vals[1])
parts = []
ws = ts.getWorksheet("New Map")
counties = ws.getSelectableValues("county")
for county in counties:
print("working on county: ", county)
wb = ws.select("county", county)
try:
df = wb.getWorksheet("New Demographics").data.assign(
location_name=county)
except TypeError:
print(f"Could not fetch data for {county}, skipping...")
df = df.rename(
columns={"SUM(Chosen Dose Status Metric)-alias": shot_type})
parts.append(df)
ws = ts.getWorksheet("New Map")
return pd.concat(parts)
def cli(db_path, tableau_views):
"""Fetch data from Tableau into a SQLite database
Pass this command a SQLite databsae file and one or more
Tableau views, where a Tableau view looks like this:
OregonCOVID-19VaccineProviderEnrollment/COVID-19VaccineProviderEnrollment
For example:
tableau-to-sqlite tableau.db OregonCOVID-19VaccineProviderEnrollment/COVID-19VaccineProviderEnrollment
You can also pass a full URL to a Tableau dashboard, for example:
tableau-to-sqlite tableau.db https://results.mo.gov/t/COVID19/views/VaccinationsDashboard/Vaccinations
"""
for view in tableau_views:
if not (view.startswith("http://") or view.startswith("https://")):
url = "https://public.tableau.com/views/{}".format(view)
else:
url = view
ts = TableauScraper()
ts.loads(url)
workboop = ts.getWorkbook()
conn = sqlite3.connect(str(db_path))
for worksheet in workboop.worksheets:
worksheet.data.to_sql(fix_name(worksheet.name), conn)
def request_tableau_scraper(query):
ts = TableauScraper()
ts.loads(query.url)
dashboard = ts.getDashboard()
dfs = []
prefixes = [] if not query.params else query.params.get('worksheet', [])
for ws in dashboard.worksheets:
if prefixes is None:
dfs.append(ws.data)
elif any([ws.name.startswith(n) for n in prefixes]):
dfs.append(ws.data)
return dfs
def _parse_data(self, url: str) -> pd.DataFrame:
"""Parse data from url"""
t_scraper = TableauScraper()
t_scraper.loads(url)
# Get the metrics
count = self._parse_metrics(t_scraper)
# Get the date
date = self._parse_date(t_scraper)
df = pd.DataFrame({
"Date": [date],
"Cumulative total": [count],
})
return df
def _get_county(variable: str, county: str):
engine = TableauScraper()
engine.loads(
url="https://www.nh.gov/t/DHHS/views/VaccineOperationalDashboard/VaccineDashboard"
)
engine.getWorkbook().setParameter("ShowBy", "Race/Ethnicity")
engine.getWorkbook().setParameter("Metric", variable)
raw_data = []
worksheet = engine.getWorksheet("Count and Prop: Map (no R/E with town&rphn)")
workbook = worksheet.select("CMN + Town + RPHN", county)
raw_data.append(
workbook.getWorksheet("Count: Bar Chart").data.assign(location_name=county)
)
data = pd.concat(raw_data)
return data
def fetch(self):
engine = TableauScraper()
engine.loads(self.fetch_url)
engine = engine.getWorksheet("Vaccine Map By SC residents PEOPLE")
filters = engine.getFilters()
counties = [
t["values"] for t in filters
if t["column"] == "Recipient County for maps"
][0]
data = []
for county in counties:
for race in self.races:
# set the filter functions to select specific county and race
workbook = engine.setFilter("Recipient County for maps",
county)
workbook = workbook.getWorksheet("Final Age xSex x Race REC")
workbook = workbook.setFilter("Assigned Race", race)
county_data = workbook.getWorksheet(
"Final Age xSex x Race REC").data
data.append(county_data.assign(location_name=county))
return pd.concat(data)
def fetch(self):
scraper_instance = TableauScraper()
scraper_instance.loads(self.fetch_url)
workbook = scraper_instance.getWorkbook()
return workbook.getWorksheet("County Map").data
def fetch(self):
engine = TableauScraper()
engine.loads(self.fetch_url)
return engine.getWorksheet(" County using admin county")