TEST_GETFEATURE_OLD_PRODUCTS_JSON = f.read()
with open(GETFEATURE_EMPTY_FILE) as f:
TEST_GETFEATURE_EMPTY_JSON = f.read()
# Create collection of input files for the products
# These will be uploaded to the mocked equivalent of S3 (where the real input files will be)
features = json.loads(TEST_GETFEATURE_JSON)['features']
INPUT_FILE_COLLECTION = PipelineFileCollection()
for f in features:
pf = PipelineFile(
os.path.join(TEST_ROOT, os.path.basename(f['properties']['url'])),
dest_path=f['properties']['url']
)
pf.publish_type = PipelineFilePublishType.UPLOAD_ONLY
INPUT_FILE_COLLECTION.add(pf)
class TestMooringsProductsHandler(HandlerTestCase):
def setUp(self):
self.handler_class = MooringsProductsHandler
upload_broker = get_storage_broker(self.config.pipeline_config['global']['upload_uri'])
upload_broker.upload(INPUT_FILE_COLLECTION)
super().setUp()
@patch('aodncore.util.wfs.WebFeatureService')
def test_all_products(self, mock_webfeatureservice):
mock_webfeatureservice().getfeature().getvalue.side_effect = [TEST_GETFEATURE_JSON,
TEST_GETFEATURE_OLD_PRODUCTS_JSON]
handler = self.run_handler(GOOD_MANIFEST)
def get_previous_version(self, previous_file_list, path, input_file_name):
"""
Find previous version of each incoming file based on its type/extension and
add them to the filecollection with the correct publish type
extension can be: .inf', '.nc.png','.pitch.csv','.roll.csv',.gps.csv'
inputs: previous_file_list : dictionary containing file listing(full path) and metadata from destination
input_file : file basename
path : full destination path
"""
if not previous_file_list:
return
files_to_delete = PipelineFileCollection()
try:
extension = ALLOWED_CONTENT_EXTENSIONS.match(
input_file_name).groupdict()['extension']
except KeyError:
raise ValueError(
"unable to determine extension from file name {infile}".format(
infile=input_file_name))
# get list of previous files basename to search through
basenames = {os.path.basename(f) for f in previous_file_list}
this_extension_pattern = re.compile(
r".*\.{ext}$".format(ext=extension))
if input_file_name not in basenames:
previous_file = [
f for f in previous_file_list
if this_extension_pattern.match(f)
]
if extension == 'nc':
if len(previous_file) != 1:
raise ValueError(
"Expected exactly 1 previous versions of the netcdf file, found {n}. Aborting "
.format(n=len(previous_file)))
else:
# if uploaded file name has the same name published file => no action, file will be overwritten, otherwise
# sort file per wildcard and work out which one to delete (
# check previous file widcard :
# can be '.inf', '.nc.png','.pitch.csv','.roll.csv',.gps.csv'
if len(previous_file) > 1:
raise ValueError(
"Found more than one previous versions of the extension '{ext}'. Aborting"
.format(ext=extension))
elif len(previous_file) == 0:
return
prev_file = previous_file[0]
dest_path = os.path.join(path, os.path.basename(prev_file))
self.logger.info(
"adding deletion of previous file '{dest_path}'".format(
dest_path=dest_path))
file_to_delete = PipelineFile(prev_file,
is_deletion=True,
dest_path=dest_path)
if extension == 'nc':
file_to_delete.publish_type = PipelineFilePublishType.DELETE_UNHARVEST
else:
file_to_delete.publish_type = PipelineFilePublishType.DELETE_ONLY
files_to_delete.add(file_to_delete)
return files_to_delete