def test_constructor_invalid(self):
"""Invalid constructor 100B"""
with self.assertRaises(TypeError):
CatalogManager(123)
with self.assertRaises(ValueError):
CatalogManager(" ")
class FTPClient:
def __init__(self):
self.config = ConfigManager('cfg.ini')
self.ftp = FTP(self.config.getFtpHostname())
self.ftp.login(self.config.getFtpUsername(),
self.config.getFtpPassword())
self.catalog = CatalogManager()
def listProductFiles(self, product):
flist = {}
ylist = []
if product == 'daily':
pDir = DAILY_FTP_ROOT
elif product == '5day':
pDir = FIVEDAILY_FTP_ROOT
elif product == 'monthly':
pDir = MONTHLY_FTP_ROOT
else:
raise Exception('Unknown product')
self.ftp.cwd(pDir)
self.ftp.retrlines('NLST', ylist.append)
for yearDir in ylist:
flist[yearDir] = []
self.ftp.cwd(pDir + yearDir + '/')
self.ftp.retrlines('NLST', flist[yearDir].append)
res = {}
for y in flist.keys():
for f in flist[y]:
if not self.catalog.exists(product, y + '/' + f):
m = re.search('OCx-([0-9]{6,8})-fv', f)
res[m.group(1)] = y + '/' + f
return res
def getFile(self, product, srcFile, target):
if product == 'daily':
pDir = DAILY_FTP_ROOT
elif product == '5day':
pDir = FIVEDAILY_FTP_ROOT
elif product == 'monthly':
pDir = MONTHLY_FTP_ROOT
else:
raise Exception('Unknown product')
self.ftp.cwd(pDir)
if not os.path.exists(os.path.dirname(target)):
os.makedirs(os.path.dirname(target))
with open(target, 'wb') as t:
self.ftp.retrbinary('RETR ' + srcFile, t.write)
def setUp(self):
"""Creates catalog manager object for testing"""
test_db.bind([Books, Multimedia])
test_db.create_tables([Books, Multimedia])
self.catalog_manager1 = CatalogManager("Catalog1")
self.valid_multimedia = Multimedia(isbn=1234567891023, author="Eric N", publisher="BE Publishing", title="1000 night at school",
genre="romantic", pub_date=datetime.date(2020, 1, 30), length="01:32:24", sub_type="CD", type_="multimedia")
self.valid_borrowed_multimedia = Multimedia(isbn=1234567891024, author="Eric K", publisher="KE Publishing", title="BCIT First look",
genre="informative", pub_date=datetime.date(2018, 2, 13), length="01:32:24", sub_type="Bluray", type_="multimedia")
self.valid_borrowed_multimedia.borrow(datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d"))
self.valid_overdue_multimedia = Multimedia(isbn=1234567891025, author="Eric W", publisher="CA Publishing", title="Fairies",
genre="romantic", pub_date=datetime.date(1998, 1, 30), length="01:32:24", sub_type="VHS", type_="multimedia")
self.valid_overdue_multimedia.borrow("1999-02-03")
self.valid_Books = Books(isbn=1234567891026, author="Eric N", publisher="BE Publishing", title="1000 night at school",
genre="romantic", pub_date=datetime.date(2020, 1, 30), length=324, sub_type="hardcover", type_="books")
self.valid_borrowed_Books = Books(isbn=1234567891027, author="Eric K", publisher="KE Publishing", title="BCIT First look",
genre="informative", pub_date=datetime.date(2018, 2, 13), length=134, sub_type="softcover", type_="books")
self.valid_borrowed_Books.borrow(datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d"))
self.valid_overdue_Books = Books(isbn=1234567891028, author="Eric W", publisher="CA Publishing", title="Fairies",
genre="romantic", pub_date=datetime.date(1998, 1, 30), length=688, sub_type="softcover", type_="books")
self.valid_overdue_Books.borrow("1999-02-03")
def create_list(self, runDate, productList, outputListFile, seedDate,
esaCredentials, dbConnectionString):
lastIngestionDate = None
if seedDate == constants.DEFAULT_DATE:
lastIngestionDate = self.__get_last_ingestion_date(productList)
# If latest record is older than 3 days, fail
if lastIngestionDate is None:
raise Exception("Unable to determine last ingestion date")
if (runDate - lastIngestionDate).days > 3:
raise Exception("Last ingestion date older then 3 days")
else:
lastIngestionDate = seedDate
if lastIngestionDate is None:
raise Exception("Unable to determine last ingestion date")
page = 0
pages = 1
searchUrl = self.__get_search_url(lastIngestionDate, page)
rawProductsData = self.__get_xml_data(searchUrl, esaCredentials)
pages = self.__get_pages(rawProductsData)
while page <= (pages - 1):
self.__add_products_to_list(rawProductsData, productList)
page = page + 1
searchUrl = self.__get_search_url(lastIngestionDate, page)
rawProductsData = self.__get_xml_data(searchUrl, esaCredentials)
if rawProductsData == None:
break
# remove duplicate products
# remove products that are already in the catalog
with CatalogManager(dbConnectionString) as cat:
productList["products"] = (seq(
productList["products"]).distinct_by(
lambda x: x["uniqueId"]).filter(lambda x: cat.exists(x[
"uniqueId"]) != True)).to_list()
outputListFile.write(json.dumps(productList))
class FolderClient:
catalog = CatalogManager()
def listProductFiles(self, product, folder):
files = [
f for f in os.listdir(folder)
if (os.path.isfile(os.path.join(folder, f))
and fnmatch.fnmatch(f, '*.nc'))
]
res = {}
for f in files:
m = re.search('OCx_QAA-([0-9]{4})-fv', f)
y = m.group(1)
if not self.catalog.exists(product, y + '/' + f):
res[y] = folder + '/' + f
return res
"""
This python file contains a RESTful API for the catalog_manager
"""
# ACIT 2515 - Assignment 2
# library_api.py
# Group 19
from flask import jsonify, make_response, Flask, request
from catalog_manager import CatalogManager
import json
app = Flask(__name__)
catalog_manager = CatalogManager("BC Library")
@app.route("/catalogmanager/catalog", methods=["POST"])
def create_item():
""" Create an item using POSTed json """
content = json.loads(request.data.decode())
try:
catalog_manager.add_item_from_json(content)
except ValueError as err:
print(err)
return make_response(f"{err}", 400)
except:
return make_response("Item is in invalid format, no item created.",
400)
return make_response(f"{content['isbn']}", 200)
def __init__(self):
self.config = ConfigManager('cfg.ini')
self.ftp = FTP(self.config.getFtpHostname())
self.ftp.login(self.config.getFtpUsername(),
self.config.getFtpPassword())
self.catalog = CatalogManager()
class TestCatalogManager(TestCase):
"""TestCatalogManager class"""
def setUp(self):
"""Creates catalog manager object for testing"""
test_db.bind([Books, Multimedia])
test_db.create_tables([Books, Multimedia])
self.catalog_manager1 = CatalogManager("Catalog1")
self.valid_multimedia = Multimedia(isbn=1234567891023, author="Eric N", publisher="BE Publishing", title="1000 night at school",
genre="romantic", pub_date=datetime.date(2020, 1, 30), length="01:32:24", sub_type="CD", type_="multimedia")
self.valid_borrowed_multimedia = Multimedia(isbn=1234567891024, author="Eric K", publisher="KE Publishing", title="BCIT First look",
genre="informative", pub_date=datetime.date(2018, 2, 13), length="01:32:24", sub_type="Bluray", type_="multimedia")
self.valid_borrowed_multimedia.borrow(datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d"))
self.valid_overdue_multimedia = Multimedia(isbn=1234567891025, author="Eric W", publisher="CA Publishing", title="Fairies",
genre="romantic", pub_date=datetime.date(1998, 1, 30), length="01:32:24", sub_type="VHS", type_="multimedia")
self.valid_overdue_multimedia.borrow("1999-02-03")
self.valid_Books = Books(isbn=1234567891026, author="Eric N", publisher="BE Publishing", title="1000 night at school",
genre="romantic", pub_date=datetime.date(2020, 1, 30), length=324, sub_type="hardcover", type_="books")
self.valid_borrowed_Books = Books(isbn=1234567891027, author="Eric K", publisher="KE Publishing", title="BCIT First look",
genre="informative", pub_date=datetime.date(2018, 2, 13), length=134, sub_type="softcover", type_="books")
self.valid_borrowed_Books.borrow(datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d"))
self.valid_overdue_Books = Books(isbn=1234567891028, author="Eric W", publisher="CA Publishing", title="Fairies",
genre="romantic", pub_date=datetime.date(1998, 1, 30), length=688, sub_type="softcover", type_="books")
self.valid_overdue_Books.borrow("1999-02-03")
def tearDown(self) -> None:
test_db.drop_tables([Books, Multimedia])
test_db.close()
def test_constructor(self):
"""Valid constructor 010A"""
self.assertIsInstance(self.catalog_manager1, CatalogManager)
def test_constructor_invalid(self):
"""Invalid constructor 100B"""
with self.assertRaises(TypeError):
CatalogManager(123)
with self.assertRaises(ValueError):
CatalogManager(" ")
def test_add_item(self):
"""Valid item addition to catalog manager 020A """
self.catalog_manager1.add_item(self.valid_Books)
self.assertIn(self.valid_Books, self.catalog_manager1.get_all_items())
self.catalog_manager1.add_item(self.valid_multimedia)
self.assertIn(self.valid_multimedia, self.catalog_manager1.get_all_items())
def test_add_item_invalid(self):
"""Invalid item addition to catalog manager 020B """
with self.assertRaises(ValueError):
self.catalog_manager1.add_item(123)
def test_add_item_invalid_duplicated(self):
""" Invalid item addition to catalog manager (same isbn) 020C """
self.catalog_manager1.add_item(self.valid_Books)
with self.assertRaises(ValueError):
self.catalog_manager1.add_item(self.valid_Books)
def test_get_item_by_isbn(self):
""" Get an item with a valid isbn 030A """
self.catalog_manager1.add_item(self.valid_Books)
self.assertEqual(self.catalog_manager1.get_item_by_isbn(1234567891026), self.valid_Books)
def test_get_item_by_isbn_invalid(self):
""" Get item with an invalid isbn 030B """
self.catalog_manager1.add_item(self.valid_Books)
with self.assertRaises(TypeError):
self.catalog_manager1.get_item_by_isbn("1234567891023")
with self.assertRaises(ValueError):
self.catalog_manager1.get_item_by_isbn(123456789)
def test_get_item_by_isbn_nonexist(self):
""" Get item with an nonexist isbn 030C """
self.assertIsNone(self.catalog_manager1.get_item_by_isbn(1234567891099))
def test_get_items_by_type(self):
""" Get all items with type 040A """
self.catalog_manager1.add_item(self.valid_Books)
self.catalog_manager1.add_item(self.valid_borrowed_Books)
self.catalog_manager1.add_item(self.valid_overdue_Books)
self.catalog_manager1.add_item(self.valid_multimedia)
self.catalog_manager1.add_item(self.valid_borrowed_multimedia)
self.catalog_manager1.add_item(self.valid_overdue_multimedia)
for item in self.catalog_manager1.get_items_by_type("book"):
self.assertIsInstance(item, Books)
for item in self.catalog_manager1.get_items_by_type("multimedia"):
self.assertIsInstance(item, Multimedia)
def test_get_items_by_type_invalid(self):
""" Get all item with invalid type 040B """
with self.assertRaises(ValueError):
self.catalog_manager1.get_items_by_type(" ")
with self.assertRaises(TypeError):
self.catalog_manager1.get_items_by_type(321)
def test_get_all_items(self):
""" Get all items in the catalog manager 050A """
self.catalog_manager1.add_item(self.valid_Books)
self.catalog_manager1.add_item(self.valid_borrowed_Books)
self.catalog_manager1.add_item(self.valid_overdue_Books)
self.catalog_manager1.add_item(self.valid_multimedia)
self.catalog_manager1.add_item(self.valid_borrowed_multimedia)
self.catalog_manager1.add_item(self.valid_overdue_multimedia)
all_items = self.catalog_manager1.get_all_items()
self.assertEqual(len(all_items), 6)
def test_delete_item_by_isbn(self):
"""Valid item removal from catalog manager 060A """
self.catalog_manager1.add_item(self.valid_Books)
self.assertIn(self.valid_Books, self.catalog_manager1.get_all_items())
self.catalog_manager1.delete_item_by_isbn(1234567891026)
self.assertNotIn(self.valid_Books, self.catalog_manager1.get_all_items())
def test_delete_item_by_isbn_invalid(self):
"""Invalid item removal from catalog manager 060B """
self.catalog_manager1.add_item(self.valid_Books)
with self.assertRaises(TypeError):
self.catalog_manager1.delete_item_by_isbn("1234567891026")
with self.assertRaises(ValueError):
self.catalog_manager1.delete_item_by_isbn(123456789102)
def test_delete_item_by_isbn_invalid_not_exist(self):
"""Invalid item removal from catalog manager 060C """
self.assertIsNone(self.catalog_manager1.delete_item_by_isbn(1234567891046))
def test_get_borrowed_items(self):
""" Get All Borrowed items 070A """
self.catalog_manager1.add_item(self.valid_Books)
self.catalog_manager1.add_item(self.valid_borrowed_Books)
self.catalog_manager1.add_item(self.valid_overdue_Books)
self.catalog_manager1.add_item(self.valid_multimedia)
self.catalog_manager1.add_item(self.valid_borrowed_multimedia)
self.catalog_manager1.add_item(self.valid_overdue_multimedia)
for item in [self.valid_borrowed_Books,
self.valid_overdue_Books,
self.valid_borrowed_multimedia,
self.valid_overdue_multimedia]:
self.assertIn(item, self.catalog_manager1.get_borrow_items())
def test_get_overdue_items(self):
""" Get all overdue items 080A """
self.catalog_manager1.add_item(self.valid_Books)
self.catalog_manager1.add_item(self.valid_borrowed_Books)
self.catalog_manager1.add_item(self.valid_overdue_Books)
self.catalog_manager1.add_item(self.valid_multimedia)
self.catalog_manager1.add_item(self.valid_borrowed_multimedia)
self.catalog_manager1.add_item(self.valid_overdue_multimedia)
for item in [self.valid_overdue_Books, self.valid_overdue_multimedia]:
self.assertIn(item, self.catalog_manager1.get_overdue_items())
def test_get_overdue_fees(self):
""" Get the total amount of overdue fee 090A """
self.catalog_manager1.add_item(self.valid_Books)
self.catalog_manager1.add_item(self.valid_borrowed_Books)
self.catalog_manager1.add_item(self.valid_multimedia)
self.catalog_manager1.add_item(self.valid_borrowed_multimedia)
# 4611.0 is the initial overdue fees
self.assertEqual(self.catalog_manager1.get_overdue_fees(), 0)
self.catalog_manager1.add_item(self.valid_overdue_Books)
self.catalog_manager1.add_item(self.valid_overdue_multimedia)
self.assertEqual(self.catalog_manager1.get_overdue_fees(),
(datetime.date.today() - datetime.date(1999, 2, 3) - datetime.timedelta(4 * 7)).days * 0.3 * 2)
def test_to_dict(self):
""" Get a list of dictionary of the catalog """
self.assertEqual(self.catalog_manager1.to_dict(), [])
self.catalog_manager1.add_item(self.valid_Books)
self.catalog_manager1.add_item(self.valid_borrowed_Books)
self.catalog_manager1.add_item(self.valid_multimedia)
self.catalog_manager1.add_item(self.valid_borrowed_multimedia)
self.catalog_manager1.add_item(self.valid_overdue_Books)
self.catalog_manager1.add_item(self.valid_overdue_multimedia)
for i in [self.valid_Books, self.valid_borrowed_Books, self.valid_multimedia, self.valid_borrowed_multimedia,
self.valid_overdue_Books, self.valid_overdue_multimedia]:
temp_dict = i.to_dict()
# Only length become string
temp_dict['length'] = str(temp_dict['length'])
self.assertIn(temp_dict, self.catalog_manager1.to_dict())
def test_add_json_books(self):
""" Input a valid book in json format """
self.catalog_manager1.add_item_from_json(
{
"author": "Eric K",
"borrow_date": "2020-04-10",
"genre": "informative",
"is_borrowed": True,
"isbn": 1234567891027,
"length": "134",
"pub_date": "2018-02-13",
"publisher": "KE Publishing",
"sub_type": "softcover",
"title": "BCIT First look",
"type_": "books"
}
)
self.assertIn(self.valid_borrowed_Books, self.catalog_manager1.get_all_items())
def test_add_json_multimedia(self):
""" Input a valid multimedia in json format """
self.catalog_manager1.add_item_from_json(
{
"author": "Eric K",
"borrow_date": "2020-04-10",
"genre": "informative",
"is_borrowed": True,
"isbn": 1234567891024,
"length": "01:32:24",
"pub_date": "2018-02-13",
"publisher": "KE Publishing",
"sub_type": "Bluray",
"title": "BCIT First look",
"type_": "multimedia"
}
)
self.assertIn(self.valid_borrowed_multimedia, self.catalog_manager1.get_all_items())
def test_get_stats(self):
""" Get a CatalogState object 100A """
self.assertIsInstance(self.catalog_manager1.get_stats(), CatalogStats)
self.catalog_manager1.add_item(self.valid_Books)
self.catalog_manager1.add_item(self.valid_borrowed_Books)
self.catalog_manager1.add_item(self.valid_overdue_Books)
self.catalog_manager1.add_item(self.valid_multimedia)
self.catalog_manager1.add_item(self.valid_borrowed_multimedia)
self.catalog_manager1.add_item(self.valid_overdue_multimedia)
self.assertIsInstance(self.catalog_manager1.get_stats(), CatalogStats)
def download_products(self, productListFile, runDate, awsAccessKeyId,
awsSecretKey, dbConnectionString):
productList = json.load(productListFile)
downloadedProductCount = 0
errorCount = 0
tempPath = self.__createTempPath(runDate)
with CatalogManager(dbConnectionString) as cat:
for product in productList["products"]:
# download product
productZipFile = None
try:
productZipFile = self.__download_product(product, tempPath)
self.logger.info("Downloaded product %s", product["title"])
except Exception as e:
self.logger.warn(
"Failed to download product %s with error %s ",
product["title"], e)
continue
if productZipFile is None and not self.debug:
continue
# verify product
if not self.debug:
verified = self.__verify_zip_file(productZipFile)
self.logger.info("Verified product %s", product["title"])
if not verified:
self.logger.warn(
"Failed to download product %s with error invalid zip file",
product["title"])
continue
# transfer to s3
try:
product["location"] = self.__copy_product_to_s3(
productZipFile, product["title"], awsAccessKeyId,
awsSecretKey)
self.logger.info(
"Coppied product %s to S3 bucket, removing temp file",
product["title"])
except Exception as e:
self.logger.warn(
"Failed to copy product %s to S3 with error %s",
product["title"], e)
continue
if not self.debug:
os.remove(productZipFile)
# add metadata to catalog
if not self.debug:
cat.addProduct(product)
else:
self.logger.info("DEBUG: Add product to catalog %s",
product["title"])
downloadedProductCount = downloadedProductCount + 1
return downloadedProductCount
class ProcessNetCDFFile(luigi.Task):
runDate = luigi.DateParameter(default=datetime.datetime.now())
product = luigi.Parameter()
srcFile = luigi.Parameter()
fileDate = luigi.Parameter()
ftp = FTPClient()
catalog = CatalogManager()
config = ConfigManager('cfg.ini')
metadata = NetCDFMetadata()
def run(self):
ncFile = os.path.join(
os.path.join(FILE_ROOT, self.runDate.strftime("%Y-%m-%d")),
self.product + '-' + self.fileDate + '.nc')
tiffFile = os.path.join(
os.path.join(FILE_ROOT, self.runDate.strftime("%Y-%m-%d")),
'UK-' + self.product + '-' + self.fileDate + '.tif')
s3DstFile = os.path.join(
os.path.join(
os.path.join(os.path.join('meo-ap/chlor_a', self.product),
self.fileDate[:4]), self.fileDate[4:6]),
'UK-' + self.product + '-' + self.fileDate + '.tif')
httpLoc = 'http://jncc-data.s3-eu-west-1.amazonaws.com/' + s3DstFile
# Get NetCDF file from FTP site
print('Retrieving ' + self.srcFile)
self.ftp.getFile(self.product, self.srcFile, ncFile)
# Get metadata from NetCDF (also acts as a validation check)
tc = self.metadata.getTimeCoverage(ncFile)
# Use GDAL to translate NetCDF to GeoTIFF and trim file to UK waters
os.system('gdal_translate NETCDF:' + ncFile +
':chlor_a -projwin -24 63 6 48 ' + tiffFile)
# Push file to S3
s3Helper.copy_file_to_s3(self.config.getAmazonKeyId(),
self.config.getAmazonKeySecret(),
self.config.getAmazonRegion(),
self.config.getAmazonBucketName(), tiffFile,
s3DstFile, True, None)
# Add entry to catalog
self.catalog.addEntry(
self.product, 'Chlorophyll-a Concentration for UK Waters - ' +
self.product + ' - ' + self.fileDate, self.srcFile, httpLoc,
tc['start'][:8], tc['end'][:8],
datetime.datetime.now().strftime("%Y-%m-%d"))
# Clean up intermediate files so we don't flood out /tmp
os.remove(ncFile)
os.remove(tiffFile)
return
def output(self):
filePath = os.path.join(
os.path.join(
os.path.join(os.path.join(S3_FILE_ROOT, self.product),
self.fileDate[:4]), self.fileDate[4:6]),
'UK-' + self.product + '-' + self.fileDate + '.tif')
return S3Target(filePath)
from catalog_manager import CatalogManager
from multimedia import Multimedia
from books import Books
import datetime
if __name__ == "__main__":
catalog_manager1 = CatalogManager("Catalog1")
valid_multimedia = Multimedia(isbn=1234567891023,
author="Eric N",
publisher="BE Publishing",
title="1000 night at school",
genre="romantic",
pub_date=datetime.date(2020, 1, 30),
length="01:32:24",
sub_type="CD",
type_="multimedia")
valid_borrowed_multimedia = Multimedia(isbn=1234567891024,
author="Eric K",
publisher="KE Publishing",
title="BCIT First look",
genre="informative",
pub_date=datetime.date(2018, 2, 13),
length="01:32:24",
sub_type="Bluray",
type_="multimedia")
valid_borrowed_multimedia.borrow(
datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d"))
valid_overdue_multimedia = Multimedia(isbn=1234567891025,
author="Eric W",
publisher="CA Publishing",
title="Fairies",