def do_resubmit(self, session, minutes=60):
"""
Resubmit a FAILAED job and store a new entry in the database
"""
allow_resubmit_before = datetime.now() + timedelta(
minutes=-int(minutes))
#allow_delete_before = datetime.now() + timedelta(minutes=-life_in_minutes)
records = (session.query(Transfer).filter_by(is_invalid=0).filter_by(
src_host=self.host).filter_by(status='FAILED').filter_by(
resubmit_id=None).order_by(Transfer.updated_at.desc()).filter(
Transfer.updated_at < allow_resubmit_before).limit(1))
resubmit_count = 0
if records.count() > 0:
for record in records:
session.expunge_all()
self.logger.info('Resubmitting: ' + str(record.id))
source = SourceController(self.host, record.getSourcePath())
#Protecting by uncontrolled deleted sources
if source.exists():
try:
self.resubmit(session, source, record) #No overwrite
resubmit_count += 1
except Exception as ex:
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
self.logger.error(
'Error resubmitting status of file: ' +
record.getSourcePath() + ' exception message: ' +
ex.message + ' message: ' + message)
continue
else:
#Invalidating
self.logger.error(
'Error File doesnt exist on the source host: ' +
record.getSourcePath())
try:
record.is_invalid = 1
record.comment = 'File doesnt exist on the source host'
session.merge(record)
session.commit()
except Exception as ex:
session.rollback()
self.logger.error(
'Error cannot invalidate the log on the Database: '
+ record.getSourcePath() + ' exception message: ' +
ex.message)
continue
# release the connection to the pool
session.close()
else:
self.logger.info('No record to resubmit!')
return resubmit_count
records = (session.query(Transfer).filter_by(is_invalid=0).filter_by(
status='SUBMITTED').filter_by(resubmit_id=None).filter_by(
src_host=host).filter(
Transfer.updated_at < allow_resubmit_before).order_by(
Transfer.transfer_attemp.desc()).limit(10))
resubmit_count = 0
suitable_for_resubmission_count = 0
if records.count() > 0:
castor = Gfal2Controller()
for record in records:
source_path = record.src_path + record.file_name
source = SourceController(host, source_path)
local_exist = source.exists()
remote_exist = castor.file_exist(record.getDestinationPath())
if not remote_exist and local_exist:
suitable_for_resubmission_count += 1
#if remote_exist:
# print "Exist"
## if remote_size == 0:
## size_zero_count += 1
## if database_size == local_size:
## suitable_for_overwrite_count += 1
## else:
## size_non_zero_count += 1
## if database_size == local_size and database_size == local_size:
## fail_fetch_status_count += 1
## else:
def do_overwrite(self, session, amount=1):
"""
Resubmit with overwrite true in case the remote file size is 0
This function use a wrapper around Castor to understand if the remote
file exist and the size is 0
Check for transfer status
"""
records = (
session.query(Transfer).filter_by(is_invalid=0).filter_by(
src_host=self.host).filter_by(status='FAILED').filter_by(
resubmit_id=None).filter(
Transfer.transfer_attemp >= 3).order_by(
Transfer.transfer_attemp.desc())
#.order_by(Transfer.updated_at.desc())
#.filter(Transfer.updated_at < allow_resubmit_before)
.limit(amount))
overwrite_count = 0
if records.count() > 0:
for record in records:
#stackoverflow.com/questions/8253978/sqlalchemy-get-object-not-bound-to-a-session
#If you want a bunch of objects produced by querying a session to be usable outside
# the scope of the session, you need to expunge them for the session.
session.expunge_all()
source_path = record.getSourcePath()
source = SourceController(self.host, source_path)
#Protecting by uncontrolled deleted sources
if not source.exists():
#Invalidating
self.logger.error(
'Error File doesnt exist on the source host')
try:
record.is_invalid = 1
record.comment = 'File doesnt exist on the source host'
session.merge(record)
session.commit()
except Exception as exception:
session.rollback()
self.logger.error(
'Failed to commit on the database: ' +
'Exception message' + exception.message)
continue
dst = record.getDestinationPath()
if not self.castor.file_exist(dst):
self.logger.error(
'File doesnt exist on the remote host with why I am overwriting? '
+ dst)
continue
remote_size = self.castor.get_size(dst)
database_size = record.file_size
local_size = source.get_size()
if self.castor.is_size_zero(dst):
if database_size == local_size:
# Cases where the file has 0 size on castor
self.logger.info('File suitable for overwrite: ' + dst)
try:
self.resubmit(session,
source,
record,
overwrite=True) #overwrite
overwrite_count += 1
except Exception as exception:
self.logger.error(
'Failed resubmission with overwrite file: ' +
dst + 'exception message: ' +
exception.message)
continue
else:
self.logger.error('File not suitable for overwrite:' +
dst)
continue
else:
# Cases where the file has the same size on database mergers and castor
# it must be a checksum error
all_sizes_are_equal = local_size == database_size and \
database_size == remote_size
if all_sizes_are_equal:
self.logger.info('File suitable for overwrite: ' +
dst + ' All sizes match size: ' +
str(remote_size))
try:
self.resubmit(session,
source,
record,
overwrite=True) #overwrite
overwrite_count += 1
except Exception as exception:
self.logger.error(
'Failed resubmission with overwrite file: ' +
dst + 'exception message: ' +
exception.message)
continue
else:
## In those cases file has been recreated with the same name..
self.logger.error('Sizes mismatch ' + dst)
else:
self.logger.info('No record to overwrite!')
# release the connection to the pool
session.close()
return overwrite_count