def test_initialize_identity_file(monkeypatch, tmpdir): """ This tests creates a new identities.json file in a temporary directory, adds an identity to it, re-reads it from disk and extracts user and entered password. """ identity_file = Path(tmpdir / "identities.json") monkeypatch.setattr("getpass.getpass", lambda: "abcd") monkeypatch.setattr("pansat.download.accounts._IDENTITY_FILE", identity_file) monkeypatch.setattr("pansat.download.accounts._PANSAT_SECRET", None) accs.parse_identity_file() assert (tmpdir / "identities.json").exists() accs.add_identity("provider", "user_name") user_name, password = accs.get_identity("provider") assert user_name == "user_name" assert password == "abcd" accs.add_identity("provider", "other_name") user_name, password = accs.get_identity("provider") assert user_name == "other_name" assert password == "abcd" accs.delete_identity("provider") with pytest.raises(accs.MissingProviderError): accs.get_identity("provider")
def test_parse_data_provider_failure(monkeypatch): """ This test reads the identity file from the ``test_data`` folder and request login information for an unknown provider. This test asserts that the expected exception is thrown. """ accs.authenticate() with pytest.raises(accs.MissingProviderError): accs.get_identity("Unknown")
def _ftp_listing_to_list(self, path, item_type=int): """ Retrieve directory content from ftp listing as list. Args: path(str): The path from which to retrieve the ftp listing. t(type): Type constructor to apply to the elements of the listing. To retrieve a list of strings use t = str. Return: A list containing the content of the ftp directory. """ if not path in self.cache: with FTP(NOAAProvider.base_url) as ftp: user, password = get_identity("NOAAProvider") ftp.login(user=user, passwd=password) try: ftp.cwd(path) except: raise Exception( "Can't find product folder " + path + "on the NOAA server. Are you sure this is the right path?" ) listing = ftp.nlst() listing = [item_type(l) for l in listing] self.cache[path] = listing return self.cache[path]
def test_parse_data_provider_failure(monkeypatch): """ This test reads the identity file from the ``test_data`` folder and request login information for an unknown provider. This test asserts that the expected exception is thrown. """ test_data = Path(PurePath(__file__).parent / "test_data" / "identities.json") import pansat.download.accounts as accs monkeypatch.setattr("pansat.download.accounts._IDENTITY_FILE", test_data) monkeypatch.setattr("pansat.download.accounts._PANSAT_SECRET", None) accs.parse_identity_file() accs.authenticate() with pytest.raises(accs.MissingProviderError): accs.get_identity("Unknown")
def download(self, start, end, destination): """ This method downloads data for a given time range from the respective data provider. Args: start(``int``): start year end(``int``): end year destination(``str`` or ``pathlib.Path``): path to directory where the downloaded files should be stored. """ # get file list files = self.get_file_names(self.product.variable, start, end) ftp = ftplib.FTP(self.base_url) user, password = get_identity("NOAAProvider") ftp.login(user=user, passwd=password) ftp.cwd(self.product_path) output_files = [] for filename in files: output = Path(str(destination)) / str(filename) output_files.append(str(output)) with open(str(output), "wb") as fp: ftp.retrbinary("RETR " + filename, fp.write) ftp.quit() return output_files
def __ftp_listing_to_list__(self, path, t=int): """ Retrieve directory content from ftp listing as list. Args: path(str): The path from which to retrieve the ftp listing. t(type): Type constructor to apply to the elements of the listing. To retrieve a list of strings use t = str. Return: A list containing the content of the ftp directory. """ if not path in self.cache: with FTP(IcareProvider.base_url) as ftp: identity = get_identity("Icare") ftp.login(user=identity["user"], passwd=identity["password"]) try: ftp.cwd(path) except: raise Exception( "Can't find product folder " + path + "on the ICARE ftp server.. Are you sure this is" "a ICARE multi sensor product?") listing = ftp.nlst() listing = [t(l) for l in listing] self.cache[path] = listing return self.cache[path]
def test_parse_identity(monkeypatch): """ This test reads the identity file from the ``test_data`` folder and tries to authenticates with the password read from the PANSAT_PASSWORD environment variable. """ accs.authenticate() login = accs.get_identity("Icare") assert login
def test_parse_identity(monkeypatch): """ This test reads the identity file from the ``test_data`` folder and tries to authenticates with the password read from the PANSAT_PASSWORD environment variable. """ test_data = Path(PurePath(__file__).parent / "test_data" / "identities.json") import pansat.download.accounts as accs monkeypatch.setattr("pansat.download.accounts._IDENTITY_FILE", test_data) monkeypatch.setattr("pansat.download.accounts._PANSAT_SECRET", None) accs.parse_identity_file() accs.authenticate() login = accs.get_identity("Icare") assert login
def test_add_parse_get_identity(monkeypatch, tmpdir): """ This tests creates a new identities.json file in a temporary directory, adds an identity to it, re-reads it from disk and extracts user and entered password. """ monkeypatch.setattr("getpass.getpass", lambda: "abcd") monkeypatch.setattr("appdirs.user_config_dir", lambda x, y: tmpdir) import pansat.download.accounts as accs assert (tmpdir / "identities.json").exists() accs.add_identity("provider", "user_name") user_name, password = accs.get_identity("provider") assert user_name == "user_name" assert password == "abcd"
def _download_with_redirect(self, url, destination): """ Handles download from GES DISC server with redirect. Arguments: url(``str``): The URL of the file to retrieve. destination(``str``): Destination to store the data. """ auth = accounts.get_identity("GES DISC") response = requests.get(url, auth=auth) url = response.url response = requests.get(url, auth=auth) with open(destination, "wb") as f: for chunk in response: f.write(chunk)
def download_file(self, filename, destination): """ Download file from data provider. Args: filename(``str``): The name of the file to download. destination(``str`` or ``pathlib.Path``): path to directory where the downloaded files should be stored. """ date = self.product.filename_to_date(filename) path = "/".join([self.product_path, str(date.year), date.strftime("%Y_%m_%d")]) user, password = get_identity("Icare") with FTP(self.base_url) as ftp: ftp.login(user=user, passwd=password) ftp.cwd(path) with open(destination, "wb") as file: ftp.retrbinary("RETR " + filename, file.write)
def download(self, filename, dest): """ Download file with given name and store to location. Args: filename(``str``): The name of the file dest(``dest``): The path to which to store the file. """ date = self.product.name_to_date(filename) path = os.path.join(self.product_path, str(date.year), date.strftime("%Y_%m_%d")) identity = get_identity("Icare") with FTP(self.base_url) as ftp: ftp.login(user=identity["user"], passwd=identity["password"]) ftp.cwd(path) with open(dest, "wb") as file: ftp.retrbinary("RETR " + filename, file.write)
def _create_cds_api_rc(): """ Context manager to create a temporary file with the Copernicus CDS login credentials obtained from the Pansat account manager. """ _, path = tempfile.mkstemp() url, key = get_identity("Copernicus") # Write key to file. file = open(path, "w") file.write(f"url: {url}\n") file.write(f"key: {key}\n") file.close() LOGGER.info("Creating file: %s", open(path).read()) os.environ["CDSAPI_RC"] = path try: yield path finally: os.environ.pop("CDSAPI_RC") Path(path).unlink()
def get_files_by_day(self, year, day): """ Return list of available files for a given day of a year. Args: year(``int``): The year for which to look up the files. day(``int``): The Julian day for which to look up the files. Return: A list of strings containing the filename that are available for the given day. """ day = str(day) day = "0" * (3 - len(day)) + day request_string = self._request_string.format(year=year, day=day, filename="") auth = accounts.get_identity("GES DISC") response = requests.get(request_string, auth=auth) files = list(set(GesdiscProvider.file_pattern.findall(response.text))) return [f[1:-1] for f in files]
def test_add_identity_file(monkeypatch, tmpdir): """ This file tests adding a new account to an existing identities file to ensure that the identities file is decrypted before an account is added to it. """ identity_file = Path(tmpdir / "identities.json") shutil.copyfile(accs._IDENTITY_FILE, identity_file) monkeypatch.setattr("getpass.getpass", lambda: "abcd") monkeypatch.setattr("pansat.download.accounts._IDENTITY_FILE", identity_file) monkeypatch.setattr("pansat.download.accounts._PANSAT_SECRET", None) accs.parse_identity_file() assert (tmpdir / "identities.json").exists() accs.add_identity("provider", "user_name") user_name, password = accs.get_identity("provider") assert user_name == "user_name" assert password == "abcd"
def download_file(self, filename, destination): """ Download file from data provider. Args: filename(``str``): The name of the file to download. destination(``str`` or ``pathlib.Path``): path to directory where the downloaded files should be stored. """ t = self.product.filename_to_date(filename) year = t.year day = t.strftime("%j") day = "0" * (3 - len(day)) + day request_string = self._request_string.format( year=year, day=day, filename=filename ) auth = accounts.get_identity("GES DISC") r = requests.get(request_string, auth=auth) with open(destination, "wb") as f: for chunk in r: f.write(chunk)
def __init__(self, product): """ Create a new product instance. Arguments: product_path(str): The path of the product. This should point to the folder that bears the product name and contains the directory tree which contains the data files sorted by date. name_to_date(function): Funtion to convert filename to datetime object. """ super().__init__() self.product = product name, password = get_identity("GeoservicesMeteofrance") request = ( GeoservicesProvider.base_url + f"/GetAPIKey?username={name}&password={password}" ) c = http.client.HTTPSConnection("geoservices.meteofrance.fr") c.request("GET", request) r = c.getresponse().read().decode() root = ElementTree.fromstring(r) self.token = root.text
def download_url(cls, url, destination): auth = accounts.get_identity("GES DISC") r = requests.get(url, auth=auth) with open(destination, "wb") as f: for chunk in r: f.write(chunk)
def setup_cds_identity(monkeypatch): url, key = accs.get_identity("Copernicus") read_config = lambda x: {"url": url, "key": key} monkeypatch.setattr("cdsapi.api.read_config", read_config)