def _extract_data(self, shot_type: str) -> pd.DataFrame:
ts = TableauScraper()
ts.loads(self.url)
# set type of metric and demographic (initiated is fetched by default)
book = ts.getWorkbook()
# get all possible paramaters
params = pd.DataFrame(book.getParameters())
shot_vals = params.loc[params["column"] == "View By", "values"].iloc[0]
demo_vals = params.loc[params["column"] == "Key Metrics",
"values"].iloc[0]
assert self.demographic_col_name in demo_vals
book.setParameter("Key Metrics", self.demographic_col_name)
if shot_type == "complete":
print(f"set parameter to: {shot_vals[1]}")
book.setParameter("View By", shot_vals[1])
parts = []
ws = ts.getWorksheet("New Map")
counties = ws.getSelectableValues("county")
for county in counties:
print("working on county: ", county)
wb = ws.select("county", county)
try:
df = wb.getWorksheet("New Demographics").data.assign(
location_name=county)
except TypeError:
print(f"Could not fetch data for {county}, skipping...")
df = df.rename(
columns={"SUM(Chosen Dose Status Metric)-alias": shot_type})
parts.append(df)
ws = ts.getWorksheet("New Map")
return pd.concat(parts)
def main():
logging.basicConfig(filename='app.log', filemode='a', format='%(asctime)s - %(message)s', level=logging.INFO)
url="https://analytics.la.gov/t/DOE/views/DigitalLearningModel/SchoolLevelModeofLearning/[email protected]/29ea44c5-4c98-44e0-87dc-2c2b907e7ac3?%3Adisplay_count=n&%3AshowVizHome=n&%3Aorigin=viz_share_link&%3AisGuestRedirectFromVizportal=y&%3Aembed=y"
ts = TS()
ts.loads(url)
logging.info("Received LA Data", exc_info=False);
ws = ts.getWorksheet("Site Map (2)")
ws.data.to_csv("out/LA_" + datetime.now().strftime('%Y%m%d') + ".csv")
logging.info("Wrote LA Data", exc_info=False);
def main():
logging.basicConfig(filename='app.log', filemode='a', format='%(asctime)s - %(message)s', level=logging.INFO)
url="https://public.tableau.com/views/COVIDPlanningTool-Embed700px/SimpleDashboard?:embed=y&:showVizHome=no&:host_url=https%3A%2F%2Fpublic.tableau.com%2F&:embed_code_version=3&:tabs=no&:toolbar=yes&:animate_transition=yes&:display_static_image=no&:display_spinner=no&:display_overlay=yes&:display_count=yes&:language=en&publish=yes&:loadOrderID=0"
ts = TS()
ts.loads(url)
logging.info("Received Hawaii Data", exc_info=False);
ws = ts.getWorksheet("CDC Map")
ws.data.to_csv("HI_" + datetime.now().strftime('%Y%m%d') + ".csv")
logging.info("Wrote Hawaii Data", exc_info=False);
def test_TableauScraper_getWorksheet(mocker: MockerFixture) -> None:
mocker.patch("tableauscraper.api.getTableauViz",
return_value=tableauVizHtmlResponse)
mocker.patch("tableauscraper.api.getTableauData",
return_value=tableauDataResponse)
ts = TS()
ts.loads(fakeUri)
tableauDataFrame = ts.getWorksheet("[WORKSHEET1]")
assert type(tableauDataFrame) is TableauWorksheet
assert tableauDataFrame.name == "[WORKSHEET1]"
assert tableauDataFrame.data.shape[0] == 4
assert tableauDataFrame.data.shape[1] == 2
def main():
logging.basicConfig(filename='app.log',
filemode='a',
format='%(asctime)s - %(message)s',
level=logging.INFO)
url = "https://results.mo.gov/t/DESE/views/LearningMethods-Opening-Enrollment-Public/LearningMethods?:showAppBanner=false&:display_count=n&:showVizHome=n&:origin=viz_share_link%22%20title%3D%22open%20Enrollment%20Public&:isGuestRedirectFromVizportal=y&:embed=y&:toolbar=no"
ts = TS()
ts.loads(url)
logging.info("Received MO Data", exc_info=False)
ws = ts.getWorksheet("MOmap-AllCategory")
ws.data.to_csv("out/MO_" + datetime.now().strftime('%Y%m%d') + ".csv")
logging.info("Wrote MO Data", exc_info=False)
def _get_county(variable: str, county: str):
engine = TableauScraper()
engine.loads(
url="https://www.nh.gov/t/DHHS/views/VaccineOperationalDashboard/VaccineDashboard"
)
engine.getWorkbook().setParameter("ShowBy", "Race/Ethnicity")
engine.getWorkbook().setParameter("Metric", variable)
raw_data = []
worksheet = engine.getWorksheet("Count and Prop: Map (no R/E with town&rphn)")
workbook = worksheet.select("CMN + Town + RPHN", county)
raw_data.append(
workbook.getWorksheet("Count: Bar Chart").data.assign(location_name=county)
)
data = pd.concat(raw_data)
return data
def fetch(self):
engine = TableauScraper()
engine.loads(self.fetch_url)
engine = engine.getWorksheet("Vaccine Map By SC residents PEOPLE")
filters = engine.getFilters()
counties = [
t["values"] for t in filters
if t["column"] == "Recipient County for maps"
][0]
data = []
for county in counties:
for race in self.races:
# set the filter functions to select specific county and race
workbook = engine.setFilter("Recipient County for maps",
county)
workbook = workbook.getWorksheet("Final Age xSex x Race REC")
workbook = workbook.setFilter("Assigned Race", race)
county_data = workbook.getWorksheet(
"Final Age xSex x Race REC").data
data.append(county_data.assign(location_name=county))
return pd.concat(data)
#Some pandas display options
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
pd.set_option('display.width', None)
#Export CSV Files -- This is the file the Cleaned Data is exported to
EXPORT_CSV_FILE = "Cruisedata.csv"
#Setting up the TableauScraper
url = "https://cruisemarketwatch.com/market-share/"
ts = TS()
ts.loads(url)
#Getting the worksheet labelled "Details"
spreadSheet = ts.getWorksheet("Details")
#Listing the individual columns to a CSV -- this'll make sense in a minute
spreadSheet.data[[
'Brand-value', 'Measure Names-alias', 'Measure Values-alias'
]].to_csv("./SpreadsheetInfoFirst.csv", header=False, index=False)
#Split the three columns each into seperate lists -- this only works if they're in a CSV already, so it should make sense now
brandValueColumn = spreadSheet.data.loc[:, 'Brand-value']
measureNameAliasColumn = spreadSheet.data.loc[:, 'Measure Names-alias']
measureValueAliasColumn = spreadSheet.data.loc[:, 'Measure Values-alias']
#Arrays to use as columns from the data pulled from the Measure Values-alias list
revenueValueColumn = []
revenuePercentValueColumn = []
totalPassengersValueColumn = []
from tableauscraper import TableauScraper as TS
url = "https://public.tableau.com/views/PlayerStats-Top5Leagues20192020/OnePlayerSummary"
ts = TS()
ts.loads(url)
# show selectable columns
columns = ts.getWorksheet("ATT MID CREATIVE COMP").getSelectableColumns()
print(columns)
# show values by column name
values = ts.getWorksheet("ATT MID CREATIVE COMP").getValues("ATTR(Player)")
print(values)
# select that value
dashboard = ts.getWorksheet("ATT MID CREATIVE COMP").select(
"ATTR(Player)", "Vinicius Júnior")
# display worksheets
for t in dashboard.worksheets:
print(t.data)
from tableauscraper import TableauScraper as TS
url = "https://public.tableau.com/views/PlayerStats-Top5Leagues20192020/OnePlayerSummary"
ts = TS()
ts.loads(url)
ws = ts.getWorksheet("ATT MID CREATIVE COMP")
# show selectable values
selections = ws.getSelectableItems()
print(selections)
# select that value
dashboard = ws.select("ATTR(Player)", "Vinicius Júnior")
# display worksheets
for t in dashboard.worksheets:
print(t.data)
def fetch(self):
engine = TableauScraper()
engine.loads(self.fetch_url)
return engine.getWorksheet(" County using admin county")
def _parse_date(self, t_scraper: TableauScraper) -> str:
"""Parse date from TableauScraper"""
date = t_scraper.getWorksheet("COVID-19 | Prensa.Titulo").data.iat[0,
0]
return clean_date(date, "%d/%m/%Y")
def _parse_metrics(self, t_scraper: TableauScraper) -> int:
"""Parse metrics from TableauScraper"""
count = int(
t_scraper.getWorksheet("Resumen").data.loc[
0, "SUM(Cantidad Pruebas)-alias"])
return clean_count(count)
def _parse_data(self, url: str) -> pd.DataFrame:
"""Parse metrics from source"""
# Get raw dataframe
ts = TS()
ts.loads(url)
return ts.getWorksheet("D_Vac_Stack").data