def test_constructor_invalid(self):
"""Invalid constructor 100B"""
with self.assertRaises(TypeError):
CatalogManager(123)
with self.assertRaises(ValueError):
CatalogManager(" ")
def create_list(self, runDate, productList, outputListFile, seedDate,
esaCredentials, dbConnectionString):
lastIngestionDate = None
if seedDate == constants.DEFAULT_DATE:
lastIngestionDate = self.__get_last_ingestion_date(productList)
# If latest record is older than 3 days, fail
if lastIngestionDate is None:
raise Exception("Unable to determine last ingestion date")
if (runDate - lastIngestionDate).days > 3:
raise Exception("Last ingestion date older then 3 days")
else:
lastIngestionDate = seedDate
if lastIngestionDate is None:
raise Exception("Unable to determine last ingestion date")
page = 0
pages = 1
searchUrl = self.__get_search_url(lastIngestionDate, page)
rawProductsData = self.__get_xml_data(searchUrl, esaCredentials)
pages = self.__get_pages(rawProductsData)
while page <= (pages - 1):
self.__add_products_to_list(rawProductsData, productList)
page = page + 1
searchUrl = self.__get_search_url(lastIngestionDate, page)
rawProductsData = self.__get_xml_data(searchUrl, esaCredentials)
if rawProductsData == None:
break
# remove duplicate products
# remove products that are already in the catalog
with CatalogManager(dbConnectionString) as cat:
productList["products"] = (seq(
productList["products"]).distinct_by(
lambda x: x["uniqueId"]).filter(lambda x: cat.exists(x[
"uniqueId"]) != True)).to_list()
outputListFile.write(json.dumps(productList))
class FolderClient:
catalog = CatalogManager()
def listProductFiles(self, product, folder):
files = [
f for f in os.listdir(folder)
if (os.path.isfile(os.path.join(folder, f))
and fnmatch.fnmatch(f, '*.nc'))
]
res = {}
for f in files:
m = re.search('OCx_QAA-([0-9]{4})-fv', f)
y = m.group(1)
if not self.catalog.exists(product, y + '/' + f):
res[y] = folder + '/' + f
return res
def setUp(self):
"""Creates catalog manager object for testing"""
test_db.bind([Books, Multimedia])
test_db.create_tables([Books, Multimedia])
self.catalog_manager1 = CatalogManager("Catalog1")
self.valid_multimedia = Multimedia(isbn=1234567891023, author="Eric N", publisher="BE Publishing", title="1000 night at school",
genre="romantic", pub_date=datetime.date(2020, 1, 30), length="01:32:24", sub_type="CD", type_="multimedia")
self.valid_borrowed_multimedia = Multimedia(isbn=1234567891024, author="Eric K", publisher="KE Publishing", title="BCIT First look",
genre="informative", pub_date=datetime.date(2018, 2, 13), length="01:32:24", sub_type="Bluray", type_="multimedia")
self.valid_borrowed_multimedia.borrow(datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d"))
self.valid_overdue_multimedia = Multimedia(isbn=1234567891025, author="Eric W", publisher="CA Publishing", title="Fairies",
genre="romantic", pub_date=datetime.date(1998, 1, 30), length="01:32:24", sub_type="VHS", type_="multimedia")
self.valid_overdue_multimedia.borrow("1999-02-03")
self.valid_Books = Books(isbn=1234567891026, author="Eric N", publisher="BE Publishing", title="1000 night at school",
genre="romantic", pub_date=datetime.date(2020, 1, 30), length=324, sub_type="hardcover", type_="books")
self.valid_borrowed_Books = Books(isbn=1234567891027, author="Eric K", publisher="KE Publishing", title="BCIT First look",
genre="informative", pub_date=datetime.date(2018, 2, 13), length=134, sub_type="softcover", type_="books")
self.valid_borrowed_Books.borrow(datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d"))
self.valid_overdue_Books = Books(isbn=1234567891028, author="Eric W", publisher="CA Publishing", title="Fairies",
genre="romantic", pub_date=datetime.date(1998, 1, 30), length=688, sub_type="softcover", type_="books")
self.valid_overdue_Books.borrow("1999-02-03")
"""
This python file contains a RESTful API for the catalog_manager
"""
# ACIT 2515 - Assignment 2
# library_api.py
# Group 19
from flask import jsonify, make_response, Flask, request
from catalog_manager import CatalogManager
import json
app = Flask(__name__)
catalog_manager = CatalogManager("BC Library")
@app.route("/catalogmanager/catalog", methods=["POST"])
def create_item():
""" Create an item using POSTed json """
content = json.loads(request.data.decode())
try:
catalog_manager.add_item_from_json(content)
except ValueError as err:
print(err)
return make_response(f"{err}", 400)
except:
return make_response("Item is in invalid format, no item created.",
400)
return make_response(f"{content['isbn']}", 200)
def __init__(self):
self.config = ConfigManager('cfg.ini')
self.ftp = FTP(self.config.getFtpHostname())
self.ftp.login(self.config.getFtpUsername(),
self.config.getFtpPassword())
self.catalog = CatalogManager()
def download_products(self, productListFile, runDate, awsAccessKeyId,
awsSecretKey, dbConnectionString):
productList = json.load(productListFile)
downloadedProductCount = 0
errorCount = 0
tempPath = self.__createTempPath(runDate)
with CatalogManager(dbConnectionString) as cat:
for product in productList["products"]:
# download product
productZipFile = None
try:
productZipFile = self.__download_product(product, tempPath)
self.logger.info("Downloaded product %s", product["title"])
except Exception as e:
self.logger.warn(
"Failed to download product %s with error %s ",
product["title"], e)
continue
if productZipFile is None and not self.debug:
continue
# verify product
if not self.debug:
verified = self.__verify_zip_file(productZipFile)
self.logger.info("Verified product %s", product["title"])
if not verified:
self.logger.warn(
"Failed to download product %s with error invalid zip file",
product["title"])
continue
# transfer to s3
try:
product["location"] = self.__copy_product_to_s3(
productZipFile, product["title"], awsAccessKeyId,
awsSecretKey)
self.logger.info(
"Coppied product %s to S3 bucket, removing temp file",
product["title"])
except Exception as e:
self.logger.warn(
"Failed to copy product %s to S3 with error %s",
product["title"], e)
continue
if not self.debug:
os.remove(productZipFile)
# add metadata to catalog
if not self.debug:
cat.addProduct(product)
else:
self.logger.info("DEBUG: Add product to catalog %s",
product["title"])
downloadedProductCount = downloadedProductCount + 1
return downloadedProductCount
class ProcessNetCDFFile(luigi.Task):
runDate = luigi.DateParameter(default=datetime.datetime.now())
product = luigi.Parameter()
srcFile = luigi.Parameter()
fileDate = luigi.Parameter()
ftp = FTPClient()
catalog = CatalogManager()
config = ConfigManager('cfg.ini')
metadata = NetCDFMetadata()
def run(self):
ncFile = os.path.join(
os.path.join(FILE_ROOT, self.runDate.strftime("%Y-%m-%d")),
self.product + '-' + self.fileDate + '.nc')
tiffFile = os.path.join(
os.path.join(FILE_ROOT, self.runDate.strftime("%Y-%m-%d")),
'UK-' + self.product + '-' + self.fileDate + '.tif')
s3DstFile = os.path.join(
os.path.join(
os.path.join(os.path.join('meo-ap/chlor_a', self.product),
self.fileDate[:4]), self.fileDate[4:6]),
'UK-' + self.product + '-' + self.fileDate + '.tif')
httpLoc = 'http://jncc-data.s3-eu-west-1.amazonaws.com/' + s3DstFile
# Get NetCDF file from FTP site
print('Retrieving ' + self.srcFile)
self.ftp.getFile(self.product, self.srcFile, ncFile)
# Get metadata from NetCDF (also acts as a validation check)
tc = self.metadata.getTimeCoverage(ncFile)
# Use GDAL to translate NetCDF to GeoTIFF and trim file to UK waters
os.system('gdal_translate NETCDF:' + ncFile +
':chlor_a -projwin -24 63 6 48 ' + tiffFile)
# Push file to S3
s3Helper.copy_file_to_s3(self.config.getAmazonKeyId(),
self.config.getAmazonKeySecret(),
self.config.getAmazonRegion(),
self.config.getAmazonBucketName(), tiffFile,
s3DstFile, True, None)
# Add entry to catalog
self.catalog.addEntry(
self.product, 'Chlorophyll-a Concentration for UK Waters - ' +
self.product + ' - ' + self.fileDate, self.srcFile, httpLoc,
tc['start'][:8], tc['end'][:8],
datetime.datetime.now().strftime("%Y-%m-%d"))
# Clean up intermediate files so we don't flood out /tmp
os.remove(ncFile)
os.remove(tiffFile)
return
def output(self):
filePath = os.path.join(
os.path.join(
os.path.join(os.path.join(S3_FILE_ROOT, self.product),
self.fileDate[:4]), self.fileDate[4:6]),
'UK-' + self.product + '-' + self.fileDate + '.tif')
return S3Target(filePath)
from catalog_manager import CatalogManager
from multimedia import Multimedia
from books import Books
import datetime
if __name__ == "__main__":
catalog_manager1 = CatalogManager("Catalog1")
valid_multimedia = Multimedia(isbn=1234567891023,
author="Eric N",
publisher="BE Publishing",
title="1000 night at school",
genre="romantic",
pub_date=datetime.date(2020, 1, 30),
length="01:32:24",
sub_type="CD",
type_="multimedia")
valid_borrowed_multimedia = Multimedia(isbn=1234567891024,
author="Eric K",
publisher="KE Publishing",
title="BCIT First look",
genre="informative",
pub_date=datetime.date(2018, 2, 13),
length="01:32:24",
sub_type="Bluray",
type_="multimedia")
valid_borrowed_multimedia.borrow(
datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d"))
valid_overdue_multimedia = Multimedia(isbn=1234567891025,
author="Eric W",
publisher="CA Publishing",
title="Fairies",