def test_retry_other():
f = FailRepeatedly(10, NotMyProblem)
ecreceive.retry_n(f,
interval=0.01,
exceptions=(MyProblem, ),
warning=1,
error=2,
give_up=3)
def test_retry_indefinitely():
f = FailRepeatedly(10)
ecreceive.retry_n(f,
interval=0.01,
exceptions=(MyProblem, ),
warning=1,
error=2,
give_up=-1)
assert f.count == 11
def clean(self):
"""!
@brief Clean up old files in destination directory.
"""
logging.info('Cleaning up old files in destination directory %s',
self.kwargs['destination_directory'])
# Instantiate API client
api = productstatus.api.Api(
self.kwargs['productstatus_url'],
username=self.kwargs['productstatus_username'],
api_key=self.kwargs['productstatus_api_key'],
verify_ssl=self.kwargs['productstatus_verify_ssl'],
)
# Get a queryset of all expired files
logging.info('Fetching a list of all expired DataInstance resources in my storage...')
now = datetime.datetime.now().replace(tzinfo=dateutil.tz.tzutc())
datainstances = api.datainstance.objects.filter(
data__productinstance__product__source=api.institution[self.kwargs['productstatus_source']],
servicebackend=api.servicebackend[self.kwargs['productstatus_service_backend']],
expires__lte=now,
deleted=False,
).order_by('-expires')
# Loop through and delete expired files
index = 0
total = datainstances.count()
logging.info('Found a total of %d expired resources, will now delete them.', total)
for datainstance in datainstances:
logging.info('[%d/%d] %s, URL %s',
index + 1,
total,
datainstance,
datainstance.url)
path = datainstance.url.replace(self.kwargs['base_url'], '').lstrip('/')
path = os.path.join(self.kwargs['destination_directory'], path)
try:
dataset = ecreceive.dataset.Dataset(path)
dataset.delete()
logging.info("Registering deletion with Productstatus...")
datainstance.deleted = True
ecreceive.retry_n(
datainstance.save,
exceptions=(
productstatus.exceptions.ServiceUnavailableException,
ecreceive.exceptions.ECReceiveProductstatusException
)
)
index += 1
except Exception, e:
logging.critical("Error when deleting file '%s': %s", path, e)
raise