def download_url_report(self, report_url, temp_folder_name): """ Downloads a MealTime report. Args: report_url (string): Information pertaining to the path and query string for the report whose access is desired. Any filtering that can be done with a stateful URL should be included. temp_folder_name (string): The name of the folder in which this specific report's download files should be stored. Returns: A Pandas DataFrame of the report contents. """ csv_download_folder_path = self.temp_folder_path + '/' + temp_folder_name # set up the driver for execution self.driver = DriverBuilder().get_driver(csv_download_folder_path, self.headless) #self.driver = configure_selenium_chrome(csv_download_folder_path) self._login() # get the report url self.driver.get(interpret_report_url(self.base_url, report_url)) # select the download format (csv) and execute export_format_select = Select(self.driver.find_element_by_id('ctl00_ctl00_MainContent_reportViewer_ctl01_ctl05_ctl00')) try: export_format_select.select_by_value('CSV') dl_type = 'csv' except NoSuchElementException: export_format_select.select_by_value('EXCELNoHeader') dl_type = 'xls' self.driver.find_element_by_id('ctl00_ctl00_MainContent_reportViewer_ctl01_ctl05_ctl01').click() # wait until file has downloaded to close the browser. We can do this # because we delete the file before we return it, so the temp dir should # always be empty when this command is run # TODO add a try/except block here wait_for_any_file_in_folder(csv_download_folder_path, dl_type) # remove the header rows #xlrd.open_workbook(utils.get_most_recent_file_in_dir(csv_download_folder_path), formatting_info=False) if dl_type == 'csv': report_df = pd.read_csv(get_most_recent_file_in_dir(csv_download_folder_path), header=2) else: report_df = pd.read_excel(get_most_recent_file_in_dir(csv_download_folder_path), header=3) # delete any files in the mealtime temp folder; we don't need them now # TODO: move this out of this function. It should happen as cleanup once # the whole DAG has completed delete_folder_contents(csv_download_folder_path) self.driver.close() # if the dataframe is empty (the report had no data), raise an error if report_df.shape[0] == 0: raise ValueError('No data in report for user {} at url: {}'.format(self.username, interpret_report_url(self.base_url, report_url))) return report_df
def download_url_report(self, report_url, write_to_disk=None, **kwargs): """ Downloads a Summit Learning report at a URL that triggers a CSV download Args: report_url (string): Information pertaining to the path and query string for the report whose access is desired. Any filtering that can be done with a stateful URL should be included. write_to_disk (string): The path for a directory to store the downloaded file. If nothing is provided, the file will be stored in a temporary directory and deleted at the end of this function. **kwargs: additional arguments to pass to Pandas read_excel or read_csv (depending on the report_url) Returns: A Pandas DataFrame of the report contents. """ report_download_url = interpret_report_url(self.base_url, report_url) if write_to_disk: csv_download_folder_path = write_to_disk else: csv_download_folder_path = mkdtemp() self.driver = DriverBuilder().get_driver(csv_download_folder_path, self.headless) self._login() self.log.debug( 'Getting report page at: {}'.format(report_download_url)) self.driver.get(report_download_url) self.log.debug('Starting download of: '.format(report_download_url)) wait_for_any_file_in_folder(csv_download_folder_path, "csv") self.log.debug('Download Finished.') df_report = pd.read_csv( get_most_recent_file_in_dir(csv_download_folder_path), **kwargs) # if the dataframe is empty (the report had no data), raise an error if df_report.shape[0] == 0: raise NoDataError( 'No data in report for user {} at url: {}'.format( self.username, interpret_report_url(self.base_url, report_url))) self.driver.close() if not write_to_disk: shutil.rmtree(csv_download_folder_path) return df_report
def download_csv_custom_report(self, report_name, school_year, download_if_generating=False, pandas_read_csv_kwargs={}): """Download a SchoolMint Custom Report that downloads as a single CSV file""" temp_folder_name = report_name.replace(" ", "_").lower() csv_download_folder_path = self.temp_folder_path + '/' + temp_folder_name driver = self._download_custom_report(report_name, school_year, csv_download_folder_path, download_if_generating) # wait until file has downloaded to close the browser. We can do this # because we delete the file before we return it, so the temp dir should # always be empty when this command is run wait_for_any_file_in_folder(csv_download_folder_path, "csv") report_df = pd.read_csv( get_most_recent_file_in_dir(csv_download_folder_path), encoding=SCHOOLMINT_DEFAULT_EXPORT_ENCODING, **pandas_read_csv_kwargs) # delete any files in the mealtime temp folder; we don't need them now # TODO: move this out of this function. It should happen as cleanup once # the whole DAG has completed delete_folder_contents(csv_download_folder_path) # close the driver for this task driver.close() # if the dataframe is empty (the report had no data), raise an error if report_df.shape[0] == 0: raise NoDataError( 'No data for user {} in Custom Report: {}'.format( self.username, report_name)) return report_df
def download_url_report(self, report_url, write_to_disk=None, **kwargs): """ Downloads a Lexia report at a URL for a page with an 'export' button. Args: report_url (string): Information pertaining to the path and query string for the report whose access is desired. Any filtering that can be done with a stateful URL should be included. write_to_disk (string): The path for a directory to store the downloaded file. If nothing is provided, the file will be stored in a temporary directory and deleted at the end of this function. **kwargs: additional arguments to pass to Pandas read_excel or read_csv (depending on the report_url) Returns: A Pandas DataFrame of the report contents. """ report_download_url = interpret_report_url(self.base_url, report_url) # if user is trying to download a manage tab report (for convenience) if '/mylexiaweb/app/index.html#/groups/' in report_download_url: return self.download_manage_tab_report(report_url, write_to_disk, **kwargs) if write_to_disk: csv_download_folder_path = write_to_disk else: csv_download_folder_path = mkdtemp() self.driver = DriverBuilder().get_driver(csv_download_folder_path, self.headless) self._login() self.log.debug( 'Getting report page at: {}'.format(report_download_url)) self.driver.get(report_download_url) # find and click the download button elem = WebDriverWait(self.driver, self.wait_time).until( EC.presence_of_element_located( (By.XPATH, "//button[contains(text(), 'Export')]"))) self.log.debug('Starting download of: '.format(report_download_url)) elem.click() wait_for_any_file_in_folder(csv_download_folder_path, "xlsx") self.log.debug('Downloada Finished.') df_report = pd.read_excel( get_most_recent_file_in_dir(csv_download_folder_path), **kwargs) # if the dataframe is empty (the report had no data), raise an error if df_report.shape[0] == 0: raise ValueError('No data in report for user {} at url: {}'.format( self.username, interpret_report_url(self.base_url, report_url))) self.driver.close() if not write_to_disk: shutil.rmtree(csv_download_folder_path) return df_report
def download_manage_tab_report(self, report_url, write_to_disk=None, **kwargs): """ Downloads a Lexia report from the 'Manage' tab. Args: report_url (string): Information pertaining to the path and query string for the report whose access is desired. Any filtering that can be done with a stateful URL should be included. write_to_disk (string): The path for a directory to store the downloaded file. If nothing is provided, the file will be stored in a temporary directory and deleted at the end of this function. **kwargs: additional arguments to pass to Pandas read_csv Returns: A Pandas DataFrame of the report contents. """ if write_to_disk: csv_download_folder_path = write_to_disk else: csv_download_folder_path = mkdtemp() self.driver = DriverBuilder().get_driver(csv_download_folder_path, self.headless) self._login() report_download_url = interpret_report_url(self.base_url, report_url) self.log.debug( 'Getting report page at: {}'.format(report_download_url)) self.driver.get(report_download_url) # select all users and find the download button def check_for_export_button_enabled(driver, elem_select_all_locator, elem_export_locator): elem_select_all = driver.find_element(*elem_select_all_locator) if not elem_select_all.is_enabled(): return False elem_select_all.click() if not elem_select_all.is_selected(): return False elem_export = driver.find_element(*elem_export_locator) if elem_export.is_enabled() and elem_export.is_displayed(): return elem_export else: return False # have to use a lambda because until expects a callable elem_export = WebDriverWait( self.driver, self.wait_time).until(lambda x: check_for_export_button_enabled( self.driver, (By.NAME, "lexia-select-all"), (By.XPATH, "//button[contains(text(), 'Export')]"))) self.log.debug('Starting download of: '.format(report_download_url)) elem_export.click() wait_for_any_file_in_folder(csv_download_folder_path, "xls") self.log.debug('Download Finished.') df_report = pd.read_csv( get_most_recent_file_in_dir(csv_download_folder_path), sep='\t', **kwargs) # if the dataframe is empty (the report had no data), raise an error if df_report.shape[0] == 0: raise ValueError('No data in report for user {} at url: {}'.format( self.username, interpret_report_url(self.base_url, report_url))) self.driver.close() if not write_to_disk: shutil.rmtree(csv_download_folder_path) return df_report
def download_zip_custom_report(self, report_name, school_year, download_folder_path=None, download_if_generating=False, unzip=True, pandas_read_csv_kwargs={}): """ Downloads a SchoolMint Custom Report that downloads as a zipped set of CSVs :param report_name: The name of the report exactly as it is shown in the SchoolMint UI :param school_year: The year in SchoolMint. Should be formatted as shown in the UI (e.g. '2018-2019') :param download_folder_path: The path to where you want to store the zip file. :param download_if_generating: Whether or not to download a custom report if the report is currently generating. :param unzip: Boolean. If True, not only downloads the file, but also unzips it and returns each csv in a Pandas Dataframe in a dictionary. :param pandas_read_csv_kwargs: Additional keyward arguments to pass to Panda's read_csv function. :return: None or a dictionary of Pandas DataFrames representing each of the CSVs in the zipped file. """ # create a folder for this specific run run_time = datetime.datetime.utcnow() if not download_folder_path: download_folder_path = self.temp_folder_path download_dir_final = "{}/{}-{}-{}".format(download_folder_path, report_name, run_time.strftime('%Y%m%d'), run_time.strftime('%H%M%S')) driver = self._download_custom_report(report_name, school_year, download_dir_final, download_if_generating) # wait until file has downloaded to close the browser. We can do this # because we delete the file before we return it, so the temp dir should # always be empty when this command is run # TODO add a try/except block here wait_for_any_file_in_folder(download_dir_final, "zip") driver.close() if unzip: # unzip the files file_path = max(glob.iglob(download_dir_final + '/*.zip'), key=os.path.getctime) ZipfileLongPaths(file_path).extractall(download_dir_final) dfs = dict() # iterate through the unzipped files and load them into dataframes for csv_filepath in glob.iglob(download_dir_final + '/*.csv'): csv_filename = os.path.basename(csv_filepath) #print(csv_filename) # find the files that start with a number, these are the custom forms files if re.match("^(\d+)", csv_filename): num_beg = re.match("^(\d+)", csv_filename).group(0) words = re.findall("[A-Za-z]+", csv_filename) dict_key = csv_filename # "{}_{}".format(num_beg, '_'.join(words[0:3])).lower() dfs[dict_key] = pd.read_csv( csv_filepath, encoding=SCHOOLMINT_DEFAULT_EXPORT_ENCODING, skiprows=[0, 2], **pandas_read_csv_kwargs) # otherwise it is the info file that comes along with the zip export (application-data-export, etc.) else: words = re.findall("[A-Za-z]+", csv_filename) dict_key = csv_filename # "{}".format('_'.join(words[0:3])).lower() dfs[dict_key] = pd.read_csv( csv_filepath, encoding=SCHOOLMINT_DEFAULT_EXPORT_ENCODING, **pandas_read_csv_kwargs) return dfs
def download_url_report(self, report_url, school_year, temp_folder_name=None, pandas_read_csv_kwargs={}): """ Downloads a SchoolMint data-stream-table report. Args: report_url (string): Information pertaining to the path and query string for the report whose access is desired. Any filtering that can be done with a stateful URL should be included. school_year (string): The SchoolMint school year to download from (e.g. '2018-2019') temp_folder_name (string): The name for a sub-directory in which the files from the browser will be temporarily stored. If this directory does not exist, it will be created. NOTE: This sub-directory will be pandas_read_csv_kwargs: additional arguments to pass to Pandas read_csv Returns: A Pandas DataFrame of the report contents. """ if temp_folder_name: csv_download_folder_path = self.temp_folder_path + '/' + temp_folder_name else: csv_download_folder_path = mkdtemp(dir=self.temp_folder_path) # set up the driver for execution self.driver = DriverBuilder().get_driver(csv_download_folder_path, self.headless) self._login() self._set_year(school_year, self.driver) # get the report url self.driver.get(interpret_report_url(self.base_url, report_url)) self.__remove_walk_me_and_support() # wait until we have rows in the stream data table before starting to # look for results elem = WebDriverWait(self.driver, self.wait_time).until( EC.presence_of_element_located( (By.XPATH, "//*[@id='stream-table']/tbody/tr[1]/td[1]"))) if not self.check_school_year(school_year): raise ReportNotFound( "Wrong school detected prior to clicking generate.") self.log.debug('Waiting for report-data-summary to load') # wait until the stream table is fully loaded before downloading prev_data_summary_elem = self.driver.find_element_by_id( 'report-data-summary').text # print(prev_data_summary_elem) time.sleep(1) # we use the following count as a proxy for time elapsed, so we can # use the class's wait_time as the number of retries count = 0 while True: # check id=report-data-summary report_data_summary_elem = self.driver.find_element_by_id( 'report-data-summary').text # if it matches, wait a little longer and double deck that it hasn't changed if prev_data_summary_elem == report_data_summary_elem: time.sleep(3) count += 3 report_data_summary_elem = self.driver.find_element_by_id( 'report-data-summary').text if prev_data_summary_elem == report_data_summary_elem: break prev_data_summary_elem = report_data_summary_elem time.sleep(1) count += 1 if count >= self.wait_time: raise TimeoutError( 'SchoolMint Report Data never did not fully load within %d' % self.wait_time) # click the button to download the report self.log.debug('Starting download...') elem = self.driver.find_element_by_class_name("export-table") elem.click() # wait until file has downloaded to close the browser. We can do this # because we delete the file before we return it, so the temp dir should # always be empty when this command is run wait_for_any_file_in_folder(csv_download_folder_path, "csv") self.log.debug('Download finished.') report_df = pd.read_csv( get_most_recent_file_in_dir(csv_download_folder_path), encoding=SCHOOLMINT_DEFAULT_EXPORT_ENCODING, **pandas_read_csv_kwargs) # TODO: move this out of this function. It should happen as cleanup once # the whole DAG has completed #delete_folder_contents(csv_download_folder_path) shutil.rmtree(csv_download_folder_path) # close the driver for this task self.driver.close() # if the dataframe is empty (the report had no data), raise an error if report_df.shape[0] == 0: #delete_folder_contents(csv_download_folder_path) shutil.rmtree(csv_download_folder_path) raise ValueError('No data in report for user {} at url: {}'.format( self.username, interpret_report_url(self.base_url, report_url))) return report_df
def download_site_data_download( self, dl_heading, site_id, academic_year, report_generation_wait=REPORT_GENERATION_WAIT, write_to_disk=None, **kwargs): if write_to_disk: csv_download_folder_path = write_to_disk else: csv_download_folder_path = mkdtemp() self.driver = DriverBuilder().get_driver(csv_download_folder_path, self.headless) self._login() dl_page_url = "{base_url}/sites/{site_id}/data_downloads/".format( base_url=self.base_url, site_id=site_id) self.driver.get(dl_page_url) self._set_dl_academic_year(academic_year) if not self.check_dl_academic_year(academic_year): raise ValueError("Academic Year not correctly set") # start the CSV generation process download_button_xpath = "//h3[contains(text(), '{dl_heading}')]/parent::div/parent::div//a[contains(text(), '{button_text}')]" # try to find the "Download CSV" button - old version of the interface old_interface = False try: elem = self.driver.find_element_by_xpath( download_button_xpath.format(dl_heading=dl_heading, button_text='Download CSV')) old_interface = True self.log.info("'Download CSV' interface detected.") elem.click() # if it's not there, it may have changed to a "Refresh" button except NoSuchElementException as e: pass # try to find the "Generate CSV" button - new version of the interface if not old_interface: gen_button_xpath = "//h3[contains(text(), '{dl_heading}')]/parent::div/parent::div//button[contains(text(), '{button_text}')]" try: elem = self.driver.find_element_by_xpath( gen_button_xpath.format(dl_heading=dl_heading, button_text='Generate CSV')) self.log.info("'Generate CSV' interface detected.") elem.click() # if it's not there, it may have changed to a "Refresh" button except NoSuchElementException as e: try: elem = self.driver.find_element_by_xpath( gen_button_xpath.format(dl_heading=dl_heading, button_text='Download')) except NoSuchElementException as e: elem = self.driver.find_element_by_xpath( gen_button_xpath.format(dl_heading=dl_heading, button_text='Refresh')) elem.click() # wait for the refresh command to be issued time.sleep(1) # wait for the report to be available and download it self.log.info( 'Starting download of report "{}" for site_id "{}"'.format( dl_heading, site_id)) dl_button_xpath = "//h3[contains(text(), '{dl_heading}')]/parent::div/parent::div//a[contains(text(), 'Download')]" try: elem = WebDriverWait(self.driver, report_generation_wait).until( EC.presence_of_element_located( (By.XPATH, dl_button_xpath.format(dl_heading=dl_heading)))) elem.click() # if the download is not ready, refresh the page and try one more time except TimeoutException: self.driver.refresh() elem = WebDriverWait(self.driver, report_generation_wait).until( EC.presence_of_element_located( (By.XPATH, dl_button_xpath.format(dl_heading=dl_heading)))) elem.click() wait_for_any_file_in_folder(csv_download_folder_path, "csv") self.log.debug('Download Finished.') df_report = pd.read_csv( get_most_recent_file_in_dir(csv_download_folder_path), **kwargs) # if the dataframe is empty (the report had no data), raise an error if df_report.shape[0] == 0: raise NoDataError('No data in report "{}" for site_id "{}"'.format( dl_heading, site_id)) self.driver.close() if not write_to_disk: shutil.rmtree(csv_download_folder_path) return df_report
def download_url_report(self, report_url, temp_folder_name): """ Downloads an Informed K12 report. Args: report_url (string): Information pertaining to the path and query string for the report whose access is desired. Any filtering that can be done with a stateful URL should be included. temp_folder_name (string): The name of the folder in which this specific report's download files should be stored. Returns: A Pandas DataFrame of the report contents. """ count = 0 while True: try: # WebDriverException - except csv_download_folder_path = self.temp_folder_path + '/' + temp_folder_name # set up the driver for execution self.driver = configure_selenium_chrome( csv_download_folder_path) self._login() time.sleep(2) #self.driver.get(self.base_url) # get the report url self.driver.get(interpret_report_url(self.base_url, report_url)) # select all responses # get the report url #self.driver.get(interpret_report_url(self.base_url, report_url)) # check to see if there are no submissions. If so, abort by exception try: self.driver.find_element_by_xpath( "//h2[contains(text(), 'No submissions')]") self.driver.close() raise ValueError( 'No data in report for user {} at url: {}'.format( self.username, interpret_report_url(self.base_url, report_url))) except NoSuchElementException: # We actually don't want to find this. pass # wait until we have rows in the responses data table before starting to # look for results try: elem = WebDriverWait(self.driver, self.wait_time).until( EC.presence_of_element_located(( By.XPATH, "//*[@class='responses-table']/table/thead/tr[1]/*[@class='checkboxes']/input" ))) except TimeoutException: raise # select all elem.click() # check to see if a new link populates to 'select all filtered submissions" (happens if more than 50 submissions) try: elem = self.driver.find_element_by_xpath( "//*[@class='responses-bulk-actions']/*[@class='select-link']" ) elem.click() except NoSuchElementException(): pass # click download elem = self.driver.find_element_by_xpath( "//*[contains(text(), 'Download') and @class='hidden-xs']") elem.click() # click 'As a spreadsheet' elem = self.driver.find_element_by_xpath( "//*[@class='dropdown-menu dropdown-menu-right']//*[contains(text(), 'As a spreadsheet')]" ) elem.click() # activate the menu that allows 'select all' try: # the following elem selection fails b/c is moves, so we time.sleep to let it load first time.sleep(0.5) elem = WebDriverWait(self.driver, self.wait_time).until( EC.visibility_of_element_located(( By.XPATH, "//*[@class='dropdown-toggle']/*[contains(text(), 'columns')]/i" ))) elem.click() except TimeoutException: # TODO raise # click on 'select all' elem = self.driver.find_element_by_xpath( "//*[@class='dropdown-menu dropdown-menu-right']//*[contains(text(), 'Select all')]" ) elem.click() # wait a moment for the info to populate time.sleep(2) # click download # elem = self.driver.find_element_by_xpath( # "//*[@class='btn btn-primary' and contains(text(), 'Download')]") # elem.click() # # time.sleep(1) # try: # elem = self.driver.find_element_by_xpath( # "//*[@class='btn btn-primary' and contains(text(), 'Download')]") # elem.click() # except WebDriverException: # pass c = 0 while True: try: elem = self.driver.find_element_by_xpath( "//*[@class='btn btn-primary' and contains(text(), 'Download')]" ) elem.click() except NoSuchElementException: if c >= 9: raise time.sleep(1) c += 1 continue break # wait until file has downloaded to close the browser. We can do this # because we delete the file before we return it, so the temp dir should # always be empty when this command is run # TODO add a try/except block here wait_for_any_file_in_folder(csv_download_folder_path, 'csv') report_df = pd.read_csv( get_most_recent_file_in_dir(csv_download_folder_path)) # delete any files in the mealtime temp folder; we don't need them now # TODO: move this out of this function. It should happen as cleanup once # the whole DAG has completed delete_folder_contents(csv_download_folder_path) self.driver.close() except WebDriverException: if count >= 9: raise count += 1 self.driver.close() continue break return report_df
def download_data_shared_with_application(self, application_page_url, collection, write_to_disk=None, **kwargs): """ Downloads the students shared with a particular application through Clever. :param application_page_url: The url for the main Clever management page for a particular application. For example, for My Lexia, this would be https://schools.clever.com/applications/lexia-mylexia :param collection: A string of 'schools', 'students', 'sections', 'teachers', 'schooladmins' that indicates which shared data to download :param write_to_disk: A path to a directory where the downloaded CSV should be saved. If nothing is passed, it will not be saved and only a Pandas DataFrame will be returned. :param kwargs: Additional keyword arguments to be passed to the Pandas read_csv function. :return: A Pandas DataFrame of the indicated collection download. """ collection = collection.lower().replace(' ', '') if collection not in ['schools', 'students', 'sections', 'teachers', 'schooladmins']: raise ReportNotFound( ( "Argument for collection '{collection}' is not a valid. Please choose from: " "'schools', 'students', 'sections', 'teachers', 'schooladmins'." ).format(collection=collection) ) report_access_page_url = interpret_report_url(self.base_url, application_page_url) if write_to_disk: csv_download_folder_path = write_to_disk else: csv_download_folder_path = mkdtemp() self.driver = DriverBuilder().get_driver(csv_download_folder_path, self.headless) self._login() self.log.debug('Getting report access page at: {}'.format(report_access_page_url)) self.driver.get(report_access_page_url) # find and click the download button based on the collection desired elem = WebDriverWait(self.driver, self.wait_time).until( EC.presence_of_element_located( (By.XPATH, "//a[contains(@href, '{collection}.csv')]".format(collection=collection)) ) ) self.log.info('Starting download of: {} - {}'.format(report_access_page_url, collection)) elem.click() wait_for_any_file_in_folder(csv_download_folder_path, "csv") self.log.info('Download Finished.') df_report = pd.read_csv(get_most_recent_file_in_dir(csv_download_folder_path), **kwargs) # if the dataframe is empty (the report had no data), raise an error if df_report.shape[0] == 0 and collection != 'schooladmins': raise ValueError('No data in report for user {} at url: {}'.format( self.username, interpret_report_url(self.base_url, application_page_url))) elif df_report.shape[0] == 0: warnings.warn("The 'schooladmins' collection has no data. Ensure that no school admins are shared.") self.driver.close() if not write_to_disk: shutil.rmtree(csv_download_folder_path) return df_report