def transfer_changed_datafile(data_file, hit_directory, log_directory = DirectoriesConfig().log_directory):
"""Transfers a changed file to XCDB by calling `transfer_file(data_file)`
:param data_file: The soakdb that we want to check if it updated, :class:`TransferChangedDataFile` self.data_file
:type data_file: str
:param hit_directory: Directory to which the soakdb corresponds to. Usually :class:`TransferChangedDataFile` self.hit_directory
:type hit_directory: str
:return: Should return nothing but will update the
:rtype: None
"""
print(data_file)
maint_exists = db_functions.check_table_sqlite(data_file, 'mainTable')
if maint_exists == 1:
soakdb_query = SoakdbFiles.objects.get(filename=data_file)
print(soakdb_query)
split_path = data_file.split('database')
search_path = split_path[0]
# remove pandda data transfer done file
if os.path.isfile(os.path.join(search_path, 'transfer_pandda_data.done')):
os.remove(os.path.join(search_path, 'transfer_pandda_data.done'))
log_files = find_log_files(search_path).rsplit()
print(log_files)
for log in log_files:
print(f"{log}.run.done")
if os.path.isfile(f"{log}.run.done"):
os.remove(f"{log}.run.done")
if os.path.isfile(f"{log}.sites.done"):
os.remove(f"{log}.sites.done")
if os.path.isfile(f"{log}.events.done"):
os.remove(f"{log}.events.done")
# find_logs_out_files = glob.glob(str(search_path + '*.txt'))
find_logs_out_files = glob.glob(f"{search_path}*.txt")
for f in find_logs_out_files:
if is_date(f.replace(search_path, '').replace('.txt', '')):
os.remove(f)
out, err, proposal = db_functions.pop_soakdb(data_file)
db_functions.pop_proposals(proposal)
else:
print('Main Table does not exist!')
transfer_file(data_file)
def run_transfer(filename):
status_query = SoakdbFiles.objects.get(filename=filename).status
print('status query: ' + str(status_query) + '(0=new, 1=updated)')
# if it is a changed file - do the delete things
if status_query == 1:
# maint_exists = db_functions.check_table_sqlite(filename, 'mainTable')
#
# if maint_exists == 1:
soakdb_query = SoakdbFiles.objects.get(filename=filename)
print('Updating/adding a new entry for file... (not deleting)')
#soakdb_query.delete()
# the next step is always the same
out, err, proposal = db_functions.pop_soakdb(filename)
db_functions.pop_proposals(proposal)
transfer_file(filename)
def run(self):
# delete all fields from soakdb filename
maint_exists = db_functions.check_table_sqlite(self.data_file,
'mainTable')
if maint_exists == 1:
soakdb_query = SoakdbFiles.objects.get(filename=self.data_file)
crystals = Crystal.objects.filter(visit=soakdb_query)
for crystal in crystals:
refinement = Refinement.objects.get(crystal_name=crystal)
target_name = str(crystal.target.target_name).upper()
crystal_name = str(crystal.crystal_name)
proasis_crystal_directory = os.path.join(
self.hit_directory, target_name, crystal_name)
if ProasisHits.objects.filter(crystal_name=crystal).exists():
proasis_hit = ProasisHits.objects.filter(
crystal_name=crystal)
for hit in proasis_hit:
for path in glob.glob(
os.path.join(
os.getcwd(), 'logs/proasis/hits',
str(hit.crystal_name.crystal_name + '_' +
hit.modification_date + '*'))):
os.remove(path)
if os.path.isdir(proasis_crystal_directory):
shutil.rmtree(
os.path.join(proasis_crystal_directory))
if ProasisOut.objects.filter(proasis=hit).exists:
for obj in ProasisOut.objects.filter(proasis=hit):
obj.delete()
hit.delete()
soakdb_query.delete()
out, err, proposal = db_functions.pop_soakdb(self.data_file)
db_functions.pop_proposals(proposal)
transfer_file(self.data_file)
with self.output().open('w') as f:
f.write('')
def transfer_all_fed_ids_and_datafiles(soak_db_filelist):
"""Transfers fedids and datafiles from soakdb filepaths to XCDB, used to populate the proposals table
:param soak_db_filelist: Soakdb filepath as defined within :class:`transfer_soakdb.TransferAllFedIDsAndDatafiles` as self.input()
:type soak_db_filelist: str
:return: Should return nothing, but will populate the proposals table in XCDB.
:rtype: None
"""
print(soak_db_filelist)
with open(soak_db_filelist, 'rb') as database_list:
for database_file in database_list.readlines():
database_file = database_file.replace('\n', '')
out, err, proposal = db_functions.pop_soakdb(database_file)
print(out)
print(err)
print(proposal)
proposal_list = list(SoakdbFiles.objects.values_list('proposal', flat=True))
for proposal_number in set(proposal_list):
db_functions.pop_proposals(proposal_number)
def run(self):
# use list from previous step as input to write to postgres
with self.input().open('r') as database_list:
for database_file in database_list.readlines():
database_file = database_file.replace('\n', '')
# populate the soakdb table for each db file found by FindSoakDBFiles
out, err, proposal = db_functions.pop_soakdb(database_file)
print(out)
print(err)
print(proposal)
# return a list of all proposals from db
proposal_list = list(SoakdbFiles.objects.values_list('proposal', flat=True))
# add fedid permissions via proposals table
for proposal_number in set(proposal_list):
db_functions.pop_proposals(proposal_number)
# write output to show job done
with self.output().open('w') as f:
f.write('TransferFeDIDs DONE')
def run(self):
# connect to central postgres db
conn, c = db_functions.connectDB()
# use list from previous step as input to write to postgres
with self.input().open('r') as database_list:
for database_file in database_list.readlines():
database_file = database_file.replace('\n', '')
out, err, proposal = db_functions.pop_soakdb(database_file)
proposal_list = []
c.execute('SELECT proposal FROM soakdb_files')
rows = c.fetchall()
for row in rows:
proposal_list.append(str(row[0]))
for proposal_number in set(proposal_list):
db_functions.pop_proposals(proposal_number)
c.close()
with self.output().open('w') as f:
f.write('TransferFeDIDs DONE')
def run(self):
# a list to hold filenames that have been checked
checked = []
# Status codes:-
# 0 = new
# 1 = changed
# 2 = not changed
print('INPUT NAME:')
print(self.input()[1].path)
with open(self.input()[1].path, 'r') as f:
files = f.readlines()
print('FILES:')
print(files)
for filename in files:
# remove any newline characters
filename_clean = filename.rstrip('\n')
# find the relevant entry in the soakdbfiles table
soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))
print(len(soakdb_query))
# raise an exception if the file is not in the soakdb table
if len(soakdb_query) == 0:
print('LEN=0')
out, err, prop = db_functions.pop_soakdb(filename_clean)
db_functions.pop_proposals(prop)
# only one entry should exist per file
if len(soakdb_query) == 1:
print('LEN=1')
# get the filename back from the query
data_file = soakdb_query[0].filename
# add the file to the list of those that have been checked
checked.append(data_file)
# get the modification date as stored in the db
old_mod_date = soakdb_query[0].modification_date
# get the current modification date of the file
current_mod_date = misc_functions.get_mod_date(data_file)
# get the id of the entry to write to
id_number = soakdb_query[0].id
print(old_mod_date)
if not old_mod_date:
soakdb_query[0].modification_date = current_mod_date
soakdb_query[0].save()
old_mod_date = 0
print(current_mod_date)
# if the file has changed since the db was last updated for the entry, change status to indicate this
try:
if int(current_mod_date) > int(old_mod_date):
update_status = SoakdbFiles.objects.get(id=id_number)
update_status.status = 1
update_status.save()
except ValueError:
raise Exception(str('current_mod_date: ' + str(current_mod_date)
+ ', old_mod_date: ' + str(old_mod_date)))
# if there is more than one entry, raise an exception (should never happen - filename field is unique)
if len(soakdb_query) > 1:
raise Exception('More than one entry for file! Something has gone wrong!')
# if the file is not in the database at all
if filename_clean not in checked:
# add the file to soakdb
out, err, proposal = db_functions.pop_soakdb(filename_clean)
# add the proposal to proposal
db_functions.pop_proposals(proposal)
# retrieve the new db entry
soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))
# get the id to update
id_number = soakdb_query[0].id
# update the relevant status to 0, indicating it as a new file
update_status = SoakdbFiles.objects.get(id=id_number)
update_status.status = 0
update_status.save()
# if the lab table is empty, no data has been transferred from the datafiles, so set status of everything to 0
lab = list(Lab.objects.all())
if not lab:
# this is to set all file statuses to 0 (new file)
soakdb = SoakdbFiles.objects.all()
for filename in soakdb:
filename.status = 0
filename.save()
# write output to signify job done
with self.output().open('w') as f:
f.write('')
def run(self):
print(self.data_file)
# delete all fields from soakdb filename
maint_exists = db_functions.check_table_sqlite(self.data_file, 'mainTable')
if maint_exists == 1:
soakdb_query = SoakdbFiles.objects.get(filename=self.data_file)
print(soakdb_query)
# for pandda file finding
split_path = self.data_file.split('database')
search_path = split_path[0]
# remove pandda data transfer done file
if os.path.isfile(os.path.join(search_path, 'transfer_pandda_data.done')):
os.remove(os.path.join(search_path, 'transfer_pandda_data.done'))
log_files = find_log_files(search_path).rsplit()
print(log_files)
for log in log_files:
print(str(log + '.run.done'))
if os.path.isfile(str(log + '.run.done')):
os.remove(str(log + '.run.done'))
if os.path.isfile(str(log + '.sites.done')):
os.remove(str(log + '.sites.done'))
if os.path.isfile(str(log + '.events.done')):
os.remove(str(log + '.events.done'))
find_logs_out_files = glob.glob(str(search_path + '*.txt'))
for f in find_logs_out_files:
if is_date(f.replace(search_path, '').replace('.txt', '')):
os.remove(f)
crystals = Crystal.objects.filter(visit=soakdb_query)
for crystal in crystals:
target_name = str(crystal.target.target_name).upper()
crystal_name = str(crystal.crystal_name)
proasis_crystal_directory = os.path.join(self.hit_directory, target_name.upper(), crystal_name)
if ProasisHits.objects.filter(crystal_name=crystal).exists():
proasis_hit = ProasisHits.objects.filter(crystal_name=crystal)
for hit in proasis_hit:
for path in glob.glob(os.path.join(DirectoriesConfig().log_directory, 'proasis/hits',
str(hit.crystal_name.crystal_name +
'_' + hit.modification_date + '*'))):
os.remove(path)
if os.path.isdir(proasis_crystal_directory):
shutil.rmtree(os.path.join(proasis_crystal_directory), ignore_errors=True)
if ProasisOut.objects.filter(proasis=hit).exists:
for obj in ProasisOut.objects.filter(proasis=hit):
if obj.root:
delete_files = ['verne.transferred', 'PROPOSALS', 'VISITS', 'visits_proposals.done']
for f in delete_files:
if os.path.isfile(os.path.join(obj.root, '/'.join(obj.start.split('/')[:-2]),
f)):
os.remove(os.path.join(obj.root, '/'.join(obj.start.split('/')[:-2]), f))
shutil.rmtree(os.path.join(obj.root, obj.start))
obj.delete()
hit.delete()
soakdb_query.delete()
out, err, proposal = db_functions.pop_soakdb(self.data_file)
db_functions.pop_proposals(proposal)
else:
print('MAIN TABLE DOES NOT EXIST!')
transfer_file(self.data_file)
with self.output().open('w') as f:
f.write('')
def check_files(soak_db_filepath):
"""Check if soakdb file has been updated since last run
:param soak_db_filepath: Soakdb filepath as defined
within :class:`transfer_soakdb.CheckFiles` as self.input()[1].path
:type soak_db_filepath: str
:return: Will return nothing but should update the status of soakdb file if it needs to...
:rtype: None
"""
# Beginning of run(self)
checked = []
# Status codes:-
# 0 = new
# 1 = changed
# 2 = not changed
# self.input()[1].path = soak_db_filepath?
print(f'INPUT NAME: {soak_db_filepath}')
# Open file
with open(soak_db_filepath, 'r') as f:
files = f.readlines()
print(f'FILES: {files}')
for filename in files:
filename_clean = filename.rstrip('\n')
soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))
print(len(soakdb_query))
# Consider Switch instead of IFs?
if len(soakdb_query) == 0:
print('LEN=0')
out, err, prop = db_functions.pop_soakdb(filename_clean)
db_functions.pop_proposals(prop)
if len(soakdb_query) == 1:
print('LEN=1')
# Get filename from query
data_file = soakdb_query[0].filename
# add file to list which have been checked
checked.append(data_file)
# Get last modification date as stored in soakdb
old_mod_date = soakdb_query[0].modification_date
# Get current modification date of file
current_mod_date = misc_functions.get_mod_date(data_file)
# get the id of entry to write to
id_number = soakdb_query[0].id
print(old_mod_date)
if not old_mod_date:
soakdb_query[0].modification_date = current_mod_date
soakdb_query[0].save()
old_mod_date = 0
print(current_mod_date)
# if the file has changed since the db was last updated for the entry, change status to indicate this
try:
if int(current_mod_date) > int(old_mod_date):
update_status = SoakdbFiles.objects.get(id=id_number)
update_status.status = 1
update_status.save()
except ValueError:
raise Exception(f"current_mod_date: {current_mod_date}, old_mod_date: {old_mod_date}")
if len(soakdb_query) > 1:
raise Exception('More than one entry for file! Something has gone wrong!')
# If file isn't in XCDB
if filename_clean not in checked:
# Add to soakdb
out, err, proposal = db_functions.pop_soakdb(filename_clean)
db_functions.pop_proposals(proposal)
soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))
id_number = soakdb_query[0].id
update_status = SoakdbFiles.objects.get(id=id_number)
update_status.status = 0
update_status.save()
lab = list(Lab.objects.all())
if not lab:
# Set all file statuses to 0
soak_db = SoakdbFiles.objects.all()
for filename in soak_db:
filename.status = 0
filename.save()
def run(self):
conn, c = db_functions.connectDB()
exists = db_functions.table_exists(c, 'soakdb_files')
checked = []
# Status codes:-
# 0 = new
# 1 = changed
# 2 = not changed
if exists:
with self.input().open('r') as f:
files = f.readlines()
for filename in files:
filename_clean = filename.rstrip('\n')
c.execute(
'select filename, modification_date, status_code from soakdb_files where filename like %s;',
(filename_clean, ))
for row in c.fetchall():
if len(row) > 0:
data_file = str(row[0])
checked.append(data_file)
old_mod_date = str(row[1])
current_mod_date = misc_functions.get_mod_date(
data_file)
if current_mod_date > old_mod_date:
c.execute(
'UPDATE soakdb_files SET status_code = 1 where filename like %s;',
(filename_clean, ))
c.execute(
'UPDATE soakdb_files SET modification_date = %s where filename like %s;',
(current_mod_date, filename_clean))
conn.commit()
if filename_clean not in checked:
out, err, proposal = db_functions.pop_soakdb(
filename_clean)
db_functions.pop_proposals(proposal)
c.execute(
'UPDATE soakdb_files SET status_code = 0 where filename like %s;',
(filename_clean, ))
conn.commit()
c.execute('select filename from soakdb_files;')
# for row in c.fetchall():
# if str(row[0]) not in checked:
# data_file = str(row[0])
exists = db_functions.table_exists(c, 'lab')
if not exists:
c.execute('UPDATE soakdb_files SET status_code = 0;')
conn.commit()
with self.output().open('w') as f:
f.write('')
def check_file(filename):
status=2
# remove any newline characters
filename_clean = filename.rstrip('\n')
# find the relevant entry in the soakdbfiles table
soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))
# raise an exception if the file is not in the soakdb table - not necessary here, I think
# if len(soakdb_query) == 0:
# print('LEN=0')
# out, err, prop = db_functions.pop_soakdb(filename_clean)
# db_functions.pop_proposals(prop)
# only one entry should exist per file
if len(soakdb_query) == 1:
# get the filename back from the query
data_file = soakdb_query[0].filename
# add the file to the list of those that have been checked
# checked.append(data_file)
# get the modification date as stored in the db
old_mod_date = soakdb_query[0].modification_date
# get the current modification date of the file
current_mod_date = misc_functions.get_mod_date(data_file)
# get the id of the entry to write to
id_number = soakdb_query[0].id
if not old_mod_date:
soakdb_query[0].modification_date = current_mod_date
soakdb_query[0].save()
old_mod_date = 0
# if the file has changed since the db was last updated for the entry, change status to indicate this
try:
if int(current_mod_date) > int(old_mod_date):
update_status = SoakdbFiles.objects.get(id=id_number)
update_status.status = 1
status = 1
update_status.save()
except ValueError:
raise Exception(str('current_mod_date: ' + str(current_mod_date)
+ ', old_mod_date: ' + str(old_mod_date)))
print(current_mod_date)
print(old_mod_date)
# if there is more than one entry, raise an exception (should never happen - filename field is unique)
if len(soakdb_query) > 1:
raise Exception('More than one entry for file! Something has gone wrong!')
# if the file is not in the database at all
if len(soakdb_query) == 0:
print('This is a new soakDB file, just setting it up in the database!')
# add the file to soakdb
out, err, proposal = db_functions.pop_soakdb(filename_clean)
# add the proposal to proposal
db_functions.pop_proposals(proposal)
# retrieve the new db entry
soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))
# get the id to update
id_number = soakdb_query[0].id
# update the relevant status to 0, indicating it as a new file
update_status = SoakdbFiles.objects.get(id=id_number)
update_status.status = 0
update_status.save()
status = 0
# else:
# print('The file has not been updated, using existing XCDB data...')
# status = 2
# if the lab table is empty, no data has been transferred from the datafiles, so set status of everything to 0
lab = list(Lab.objects.all())
if not lab:
# this is to set all file statuses to 0 (new file)
soakdb = SoakdbFiles.objects.all()
for filename in soakdb:
filename.status = 0
filename.save()
return status