def testDestination(host, file_name, expected_result):
source_dir = '/merger/cdr'
new_source = SourceController(host, source_dir + '/' + file_name)
if new_source.get_destination() != expected_result:
print file_name
print(" : {}".format(new_source.get_destination()))
print ' Should be ' + expected_result
else:
print 'Test OK!'
def do_resubmit(self, session, minutes=60):
"""
Resubmit a FAILAED job and store a new entry in the database
"""
allow_resubmit_before = datetime.now() + timedelta(
minutes=-int(minutes))
#allow_delete_before = datetime.now() + timedelta(minutes=-life_in_minutes)
records = (session.query(Transfer).filter_by(is_invalid=0).filter_by(
src_host=self.host).filter_by(status='FAILED').filter_by(
resubmit_id=None).order_by(Transfer.updated_at.desc()).filter(
Transfer.updated_at < allow_resubmit_before).limit(1))
resubmit_count = 0
if records.count() > 0:
for record in records:
session.expunge_all()
self.logger.info('Resubmitting: ' + str(record.id))
source = SourceController(self.host, record.getSourcePath())
#Protecting by uncontrolled deleted sources
if source.exists():
try:
self.resubmit(session, source, record) #No overwrite
resubmit_count += 1
except Exception as ex:
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
self.logger.error(
'Error resubmitting status of file: ' +
record.getSourcePath() + ' exception message: ' +
ex.message + ' message: ' + message)
continue
else:
#Invalidating
self.logger.error(
'Error File doesnt exist on the source host: ' +
record.getSourcePath())
try:
record.is_invalid = 1
record.comment = 'File doesnt exist on the source host'
session.merge(record)
session.commit()
except Exception as ex:
session.rollback()
self.logger.error(
'Error cannot invalidate the log on the Database: '
+ record.getSourcePath() + ' exception message: ' +
ex.message)
continue
# release the connection to the pool
session.close()
else:
self.logger.info('No record to resubmit!')
return resubmit_count
def get_new_files(self, source_dir):
"""
Return a list of new files found in the source dir
"""
new_sources = []
amount = 0
source_dir = source_dir + '/'
files = filter(os.path.isfile, glob.glob(source_dir + "*"))
files.sort(key=lambda x: os.path.getmtime(x))
for new_file in files:
self.logger.info('Checking file: ' + new_file)
new_source = SourceController(self.host, new_file)
if new_source.get_owner() == 'na62cdr' and new_source.get_group(
) == 'root':
new_sources.append(new_source)
# Limiting the amount of concurrent submission
amount += 1
if amount > 100:
break
return new_sources
def delete_files(self, session, life_in_minutes):
"""
Check for files to delete and deletes them
"""
allow_delete_before = datetime.now() + timedelta(
minutes=-life_in_minutes)
self.logger.info('Delete Files before: ' + str(allow_delete_before))
records = (session.query(Transfer).filter_by(is_invalid=0).filter_by(
src_host=self.host).filter_by(status='FINISHED').filter_by(
is_src_deleted=0).filter_by(is_on_tape=1).order_by(
Transfer.created_at.asc()).filter(
Transfer.created_at < allow_delete_before).limit(100))
count_delete = 0
if records.count() > 0:
for record in records:
self.logger.info("Deleting Source: " + record.getSourcePath())
new_source = SourceController(self.host,
record.getSourcePath())
new_source.delete()
try:
record.is_src_deleted = 1
record.deleted_at = datetime.now()
session.merge(record)
session.commit()
self.logger.info("Commit on the DB %s is_source_deleted",
record.job_id)
count_delete += 1
except Exception as exception:
session.rollback()
self.logger.error(
"Failed Commit on the DB %s is_source_deleted exception\
message %s", record.job_id, exception.message)
else:
self.logger.info("No files to delete")
# release the connection to the pool
session.close()
return count_delete
amount_of_result = 0
for record in records:
amount_of_result += 1
source_path = record.getSourcePath()
if source_path in files:
try:
#removing db entry from the os list
files.remove(source_path)
except:
print "Entry doesn't exist"
#print "New list len: " , len(files)
counter += 1
offset += limit
print "List len: ", len(files)
print len(files) , " Untracked files:"
for file in files:
new_source = SourceController(host, file)
if (new_source.get_owner() == 'na62cdr' and new_source.get_group() == 'vl'):
print file + " Was tracked"
new_source = SourceController(host, file)
print 'deleting'
new_source.delete()
#new_source.rename()
else:
print file + " Was not tracked"
continue
else:
print "Nothing untracked"
logger.info('Running on: ' + host)
records = (session.query(Transfer).filter_by(is_invalid=0).filter_by(
status='SUBMITTED').filter_by(resubmit_id=None).filter_by(
src_host=host).filter(
Transfer.updated_at < allow_resubmit_before).order_by(
Transfer.transfer_attemp.desc()).limit(10))
resubmit_count = 0
suitable_for_resubmission_count = 0
if records.count() > 0:
castor = Gfal2Controller()
for record in records:
source_path = record.src_path + record.file_name
source = SourceController(host, source_path)
local_exist = source.exists()
remote_exist = castor.file_exist(record.getDestinationPath())
if not remote_exist and local_exist:
suitable_for_resubmission_count += 1
#if remote_exist:
# print "Exist"
## if remote_size == 0:
## size_zero_count += 1
## if database_size == local_size:
## suitable_for_overwrite_count += 1
## else:
## size_non_zero_count += 1
## if database_size == local_size and database_size == local_size:
## fail_fetch_status_count += 1
#.order_by(Transfer.updated_at.desc())
#.filter(Transfer.updated_at < allow_resubmit_before)
.order_by(Transfer.transfer_attemp.desc()).limit(1000))
resubmit_count = 0
size_zero_count = 0
size_non_zero_count = 0
suitable_for_overwrite_count = 0
fail_checksum_count = 0
file_recreated_count = 0
remote_dont_exist = 0
if records.count() > 0:
castor = Gfal2Controller()
for record in records:
new_source = SourceController(host, record.getSourcePath())
database_size = record.file_size
local_size = new_source.get_size()
if castor.file_exist(record.getDestinationPath()):
remote_size = castor.get_size(record.getDestinationPath())
if remote_size == 0:
size_zero_count += 1
if database_size == local_size:
suitable_for_overwrite_count += 1
else:
size_non_zero_count += 1
if database_size == local_size and database_size == remote_size:
fail_checksum_count += 1
else:
file_recreated_count += 1
logger.info(str(record.id) + ' ' + record.job_id + " " + str(record.transfer_attemp) +\
def delete_over_threshold(self, session, file_dir):
"""
Deletes file transferred but not on tape
"""
number_of_cicle = 0
#if self.is_disk_full(file_dir):
while self.is_disk_full(file_dir) and number_of_cicle < 15:
number_of_cicle += 1
#Check first for files on tape non deleted because of the timeout
records = (session.query(Transfer).filter_by(
is_invalid=0).filter_by(src_host=self.host).filter_by(
status='FINISHED').filter_by(is_src_deleted=0).filter_by(
is_on_tape=1).order_by(
Transfer.created_at.asc()).limit(100))
count_deleted_on_tape = 0
count_deleted_not_on_tape = 0
if records.count() > 0:
for record in records:
self.logger.info("Deleting Source on tape: " +
record.getSourcePath())
new_source = SourceController(self.host,
record.getSourcePath())
new_source.delete()
try:
record.is_src_deleted = 1
record.deleted_at = datetime.now()
session.merge(record)
session.commit()
self.logger.info(
"Commit on the DB %s is_source_deleted",
record.job_id)
count_delete_on_tape += 1
except Exception as exception:
session.rollback()
self.logger.error(
"Failed Commit on the DB %s is_source_deleted exception\
message %s", record.job_id, exception.message)
break
if count_deleted_on_tape == 0:
# I need to delete data non stored on tape
records = (session.query(Transfer).filter_by(
is_invalid=0).filter_by(src_host=self.host).filter_by(
status='FINISHED').filter_by(
is_src_deleted=0).filter_by(is_on_tape=0).order_by(
Transfer.created_at.asc()).limit(100))
if records.count() > 0:
for record in records:
self.logger.info("Deleting Source not on tape: " +
record.getSourcePath())
new_source = SourceController(self.host,
record.getSourcePath())
new_source.delete()
try:
record.is_src_deleted = 1
record.deleted_at = datetime.now()
record.comment = 'File has been deleted when was not on tape'
session.merge(record)
session.commit()
self.logger.info(
"Commit on the DB %s is_source_deleted",
record.job_id)
count_deleted_not_on_tape += 1
except Exception as exception:
session.rollback()
self.logger.error(
"Failed Commit on the DB %s is_source_deleted exception\
message %s", record.job_id, exception.message)
break
if count_deleted_not_on_tape == 0:
self.logger.error("Cannot delete any file!")
self.logger.info("number of cycle " + str(number_of_cicle) +
" Sleeping..")
time.sleep(1)
def do_overwrite(self, session, amount=1):
"""
Resubmit with overwrite true in case the remote file size is 0
This function use a wrapper around Castor to understand if the remote
file exist and the size is 0
Check for transfer status
"""
records = (
session.query(Transfer).filter_by(is_invalid=0).filter_by(
src_host=self.host).filter_by(status='FAILED').filter_by(
resubmit_id=None).filter(
Transfer.transfer_attemp >= 3).order_by(
Transfer.transfer_attemp.desc())
#.order_by(Transfer.updated_at.desc())
#.filter(Transfer.updated_at < allow_resubmit_before)
.limit(amount))
overwrite_count = 0
if records.count() > 0:
for record in records:
#stackoverflow.com/questions/8253978/sqlalchemy-get-object-not-bound-to-a-session
#If you want a bunch of objects produced by querying a session to be usable outside
# the scope of the session, you need to expunge them for the session.
session.expunge_all()
source_path = record.getSourcePath()
source = SourceController(self.host, source_path)
#Protecting by uncontrolled deleted sources
if not source.exists():
#Invalidating
self.logger.error(
'Error File doesnt exist on the source host')
try:
record.is_invalid = 1
record.comment = 'File doesnt exist on the source host'
session.merge(record)
session.commit()
except Exception as exception:
session.rollback()
self.logger.error(
'Failed to commit on the database: ' +
'Exception message' + exception.message)
continue
dst = record.getDestinationPath()
if not self.castor.file_exist(dst):
self.logger.error(
'File doesnt exist on the remote host with why I am overwriting? '
+ dst)
continue
remote_size = self.castor.get_size(dst)
database_size = record.file_size
local_size = source.get_size()
if self.castor.is_size_zero(dst):
if database_size == local_size:
# Cases where the file has 0 size on castor
self.logger.info('File suitable for overwrite: ' + dst)
try:
self.resubmit(session,
source,
record,
overwrite=True) #overwrite
overwrite_count += 1
except Exception as exception:
self.logger.error(
'Failed resubmission with overwrite file: ' +
dst + 'exception message: ' +
exception.message)
continue
else:
self.logger.error('File not suitable for overwrite:' +
dst)
continue
else:
# Cases where the file has the same size on database mergers and castor
# it must be a checksum error
all_sizes_are_equal = local_size == database_size and \
database_size == remote_size
if all_sizes_are_equal:
self.logger.info('File suitable for overwrite: ' +
dst + ' All sizes match size: ' +
str(remote_size))
try:
self.resubmit(session,
source,
record,
overwrite=True) #overwrite
overwrite_count += 1
except Exception as exception:
self.logger.error(
'Failed resubmission with overwrite file: ' +
dst + 'exception message: ' +
exception.message)
continue
else:
## In those cases file has been recreated with the same name..
self.logger.error('Sizes mismatch ' + dst)
else:
self.logger.info('No record to overwrite!')
# release the connection to the pool
session.close()
return overwrite_count
#!/usr/bin/env python
'''
Author: Marco Boretto
Mail: [email protected]
Python Version: 2.7
'''
# check for submited file with size 0
from datetime import datetime, timedelta
from models import Transfer, configSectionMap, showDatabase, Session
from controllers import SourceController
session = Session()
host = configSectionMap("Default")['host']
print "Running on: " + host
records = (session.query(Transfer).filter_by(is_invalid=0).filter_by(
status='FAILED').filter_by(resubmit_id=None).filter_by(
src_host=host).filter_by(file_size=0).limit(10))
#print str(records)
if records.count() > 0:
for record in records:
new_source = SourceController(record.src_host, record.getSourcePath())
print record
print "Source File size: " + str(
record.file_size) + " " + record.getSourcePath()
else:
print "No records found"
print "Running on: " + host
records = (session.query(Transfer)
.filter_by(is_invalid=0)
.filter_by(status='FAILED')
.filter_by(resubmit_id=None)
.filter_by(src_host=host)
.filter_by(file_size=0)
.limit(10)
)
#print str(records)
if records.count() > 0:
for record in records:
print record
print "Deleting Source File size: " + str(record.file_size) + " " + record.getSourcePath()
new_source = SourceController(record.src_host, record.getSourcePath())
if new_source.get_size() == record.file_size:
new_source.delete()
#new_source.rename()
try:
record.is_src_deleted = 1
record.is_invalid = 1
record.comment = "Invalid transfer File size is 0"
record.deleted_at = datetime.now()
session.merge(record)
session.commit()
#log.info("Commit on the DB %s is_source_deleted" % (record.job_id))
#except:
except Exception as e:
print traceback.format_exception(*sys.exc_info())
session.rollback()