def test_initialize_identity_file(monkeypatch, tmpdir):
"""
This tests creates a new identities.json file in a temporary directory,
adds an identity to it, re-reads it from disk and extracts user and
entered password.
"""
identity_file = Path(tmpdir / "identities.json")
monkeypatch.setattr("getpass.getpass", lambda: "abcd")
monkeypatch.setattr("pansat.download.accounts._IDENTITY_FILE",
identity_file)
monkeypatch.setattr("pansat.download.accounts._PANSAT_SECRET", None)
accs.parse_identity_file()
assert (tmpdir / "identities.json").exists()
accs.add_identity("provider", "user_name")
user_name, password = accs.get_identity("provider")
assert user_name == "user_name"
assert password == "abcd"
accs.add_identity("provider", "other_name")
user_name, password = accs.get_identity("provider")
assert user_name == "other_name"
assert password == "abcd"
accs.delete_identity("provider")
with pytest.raises(accs.MissingProviderError):
accs.get_identity("provider")
def test_parse_data_provider_failure(monkeypatch):
"""
This test reads the identity file from the ``test_data`` folder and
request login information for an unknown provider. This test asserts
that the expected exception is thrown.
"""
accs.authenticate()
with pytest.raises(accs.MissingProviderError):
accs.get_identity("Unknown")
def _ftp_listing_to_list(self, path, item_type=int):
"""
Retrieve directory content from ftp listing as list.
Args:
path(str): The path from which to retrieve the ftp listing.
t(type): Type constructor to apply to the elements of the
listing. To retrieve a list of strings use t = str.
Return:
A list containing the content of the ftp directory.
"""
if not path in self.cache:
with FTP(NOAAProvider.base_url) as ftp:
user, password = get_identity("NOAAProvider")
ftp.login(user=user, passwd=password)
try:
ftp.cwd(path)
except:
raise Exception(
"Can't find product folder " + path +
"on the NOAA server. Are you sure this is the right path?"
)
listing = ftp.nlst()
listing = [item_type(l) for l in listing]
self.cache[path] = listing
return self.cache[path]
def test_parse_data_provider_failure(monkeypatch):
"""
This test reads the identity file from the ``test_data`` folder and
request login information for an unknown provider. This test asserts
that the expected exception is thrown.
"""
test_data = Path(PurePath(__file__).parent / "test_data" / "identities.json")
import pansat.download.accounts as accs
monkeypatch.setattr("pansat.download.accounts._IDENTITY_FILE", test_data)
monkeypatch.setattr("pansat.download.accounts._PANSAT_SECRET", None)
accs.parse_identity_file()
accs.authenticate()
with pytest.raises(accs.MissingProviderError):
accs.get_identity("Unknown")
def download(self, start, end, destination):
"""
This method downloads data for a given time range from the respective
data provider.
Args:
start(``int``): start year
end(``int``): end year
destination(``str`` or ``pathlib.Path``): path to directory where
the downloaded files should be stored.
"""
# get file list
files = self.get_file_names(self.product.variable, start, end)
ftp = ftplib.FTP(self.base_url)
user, password = get_identity("NOAAProvider")
ftp.login(user=user, passwd=password)
ftp.cwd(self.product_path)
output_files = []
for filename in files:
output = Path(str(destination)) / str(filename)
output_files.append(str(output))
with open(str(output), "wb") as fp:
ftp.retrbinary("RETR " + filename, fp.write)
ftp.quit()
return output_files
def __ftp_listing_to_list__(self, path, t=int):
"""
Retrieve directory content from ftp listing as list.
Args:
path(str): The path from which to retrieve the ftp listing.
t(type): Type constructor to apply to the elements of the
listing. To retrieve a list of strings use t = str.
Return:
A list containing the content of the ftp directory.
"""
if not path in self.cache:
with FTP(IcareProvider.base_url) as ftp:
identity = get_identity("Icare")
ftp.login(user=identity["user"], passwd=identity["password"])
try:
ftp.cwd(path)
except:
raise Exception(
"Can't find product folder " + path +
"on the ICARE ftp server.. Are you sure this is"
"a ICARE multi sensor product?")
listing = ftp.nlst()
listing = [t(l) for l in listing]
self.cache[path] = listing
return self.cache[path]
def test_parse_identity(monkeypatch):
"""
This test reads the identity file from the ``test_data`` folder and
tries to authenticates with the password read from the PANSAT_PASSWORD
environment variable.
"""
accs.authenticate()
login = accs.get_identity("Icare")
assert login
def test_parse_identity(monkeypatch):
"""
This test reads the identity file from the ``test_data`` folder and
tries to authenticates with the password read from the PANSAT_PASSWORD
environment variable.
"""
test_data = Path(PurePath(__file__).parent / "test_data" / "identities.json")
import pansat.download.accounts as accs
monkeypatch.setattr("pansat.download.accounts._IDENTITY_FILE", test_data)
monkeypatch.setattr("pansat.download.accounts._PANSAT_SECRET", None)
accs.parse_identity_file()
accs.authenticate()
login = accs.get_identity("Icare")
assert login
def test_add_parse_get_identity(monkeypatch, tmpdir):
"""
This tests creates a new identities.json file in a temporary directory,
adds an identity to it, re-reads it from disk and extracts user and
entered password.
"""
monkeypatch.setattr("getpass.getpass", lambda: "abcd")
monkeypatch.setattr("appdirs.user_config_dir", lambda x, y: tmpdir)
import pansat.download.accounts as accs
assert (tmpdir / "identities.json").exists()
accs.add_identity("provider", "user_name")
user_name, password = accs.get_identity("provider")
assert user_name == "user_name"
assert password == "abcd"
def _download_with_redirect(self, url, destination):
"""
Handles download from GES DISC server with redirect.
Arguments:
url(``str``): The URL of the file to retrieve.
destination(``str``): Destination to store the data.
"""
auth = accounts.get_identity("GES DISC")
response = requests.get(url, auth=auth)
url = response.url
response = requests.get(url, auth=auth)
with open(destination, "wb") as f:
for chunk in response:
f.write(chunk)
def download_file(self, filename, destination):
"""
Download file from data provider.
Args:
filename(``str``): The name of the file to download.
destination(``str`` or ``pathlib.Path``): path to directory where
the downloaded files should be stored.
"""
date = self.product.filename_to_date(filename)
path = "/".join([self.product_path, str(date.year), date.strftime("%Y_%m_%d")])
user, password = get_identity("Icare")
with FTP(self.base_url) as ftp:
ftp.login(user=user, passwd=password)
ftp.cwd(path)
with open(destination, "wb") as file:
ftp.retrbinary("RETR " + filename, file.write)
def download(self, filename, dest):
"""
Download file with given name and store to location.
Args:
filename(``str``): The name of the file
dest(``dest``): The path to which to store the file.
"""
date = self.product.name_to_date(filename)
path = os.path.join(self.product_path, str(date.year),
date.strftime("%Y_%m_%d"))
identity = get_identity("Icare")
with FTP(self.base_url) as ftp:
ftp.login(user=identity["user"], passwd=identity["password"])
ftp.cwd(path)
with open(dest, "wb") as file:
ftp.retrbinary("RETR " + filename, file.write)
def _create_cds_api_rc():
"""
Context manager to create a temporary file with the Copernicus CDS login
credentials obtained from the Pansat account manager.
"""
_, path = tempfile.mkstemp()
url, key = get_identity("Copernicus")
# Write key to file.
file = open(path, "w")
file.write(f"url: {url}\n")
file.write(f"key: {key}\n")
file.close()
LOGGER.info("Creating file: %s", open(path).read())
os.environ["CDSAPI_RC"] = path
try:
yield path
finally:
os.environ.pop("CDSAPI_RC")
Path(path).unlink()
def get_files_by_day(self, year, day):
"""
Return list of available files for a given day of a year.
Args:
year(``int``): The year for which to look up the files.
day(``int``): The Julian day for which to look up the files.
Return:
A list of strings containing the filename that are available
for the given day.
"""
day = str(day)
day = "0" * (3 - len(day)) + day
request_string = self._request_string.format(year=year,
day=day,
filename="")
auth = accounts.get_identity("GES DISC")
response = requests.get(request_string, auth=auth)
files = list(set(GesdiscProvider.file_pattern.findall(response.text)))
return [f[1:-1] for f in files]
def test_add_identity_file(monkeypatch, tmpdir):
"""
This file tests adding a new account to an existing identities file to ensure
that the identities file is decrypted before an account is added to it.
"""
identity_file = Path(tmpdir / "identities.json")
shutil.copyfile(accs._IDENTITY_FILE, identity_file)
monkeypatch.setattr("getpass.getpass", lambda: "abcd")
monkeypatch.setattr("pansat.download.accounts._IDENTITY_FILE",
identity_file)
monkeypatch.setattr("pansat.download.accounts._PANSAT_SECRET", None)
accs.parse_identity_file()
assert (tmpdir / "identities.json").exists()
accs.add_identity("provider", "user_name")
user_name, password = accs.get_identity("provider")
assert user_name == "user_name"
assert password == "abcd"
def download_file(self, filename, destination):
"""
Download file from data provider.
Args:
filename(``str``): The name of the file to download.
destination(``str`` or ``pathlib.Path``): path to directory where
the downloaded files should be stored.
"""
t = self.product.filename_to_date(filename)
year = t.year
day = t.strftime("%j")
day = "0" * (3 - len(day)) + day
request_string = self._request_string.format(
year=year, day=day, filename=filename
)
auth = accounts.get_identity("GES DISC")
r = requests.get(request_string, auth=auth)
with open(destination, "wb") as f:
for chunk in r:
f.write(chunk)
def __init__(self, product):
"""
Create a new product instance.
Arguments:
product_path(str): The path of the product. This should point to
the folder that bears the product name and contains the directory
tree which contains the data files sorted by date.
name_to_date(function): Funtion to convert filename to datetime object.
"""
super().__init__()
self.product = product
name, password = get_identity("GeoservicesMeteofrance")
request = (
GeoservicesProvider.base_url
+ f"/GetAPIKey?username={name}&password={password}"
)
c = http.client.HTTPSConnection("geoservices.meteofrance.fr")
c.request("GET", request)
r = c.getresponse().read().decode()
root = ElementTree.fromstring(r)
self.token = root.text
def download_url(cls, url, destination):
auth = accounts.get_identity("GES DISC")
r = requests.get(url, auth=auth)
with open(destination, "wb") as f:
for chunk in r:
f.write(chunk)
def setup_cds_identity(monkeypatch):
url, key = accs.get_identity("Copernicus")
read_config = lambda x: {"url": url, "key": key}
monkeypatch.setattr("cdsapi.api.read_config", read_config)
