def transfer_file(data_file):
"""Transfer contents of soakdb.sqlite file to xcdb into separate
(crystal, lab, refinement, dimple and data_processing tables.
:param data_file: File-path(?) pointing towards a soakdb.sqlite file
:type data_file: str
:return: Function should not return anything but will write to a database.
:rtype: None
"""
maint_exists = db_functions.check_table_sqlite(data_file, 'mainTable')
if maint_exists == 1:
db_functions.transfer_table(translate_dict=db_functions.crystal_translations(), filename=data_file,
model=Crystal)
db_functions.transfer_table(translate_dict=db_functions.lab_translations(), filename=data_file,
model=Lab)
db_functions.transfer_table(translate_dict=db_functions.refinement_translations(), filename=data_file,
model=Refinement)
db_functions.transfer_table(translate_dict=db_functions.dimple_translations(), filename=data_file,
model=Dimple)
db_functions.transfer_table(translate_dict=db_functions.data_processing_translations(),
filename=data_file, model=DataProcessing)
soakdb_query = SoakdbFiles.objects.get(filename=data_file)
soakdb_query.status = 2
soakdb_query.save()
def transfer_file(data_file):
maint_exists = db_functions.check_table_sqlite(data_file, 'mainTable')
if maint_exists == 1:
db_functions.transfer_table(
translate_dict=db_functions.crystal_translations(),
filename=data_file,
model=Crystal)
db_functions.transfer_table(
translate_dict=db_functions.lab_translations(),
filename=data_file,
model=Lab)
db_functions.transfer_table(
translate_dict=db_functions.refinement_translations(),
filename=data_file,
model=Refinement)
db_functions.transfer_table(
translate_dict=db_functions.dimple_translations(),
filename=data_file,
model=Dimple)
db_functions.transfer_table(
translate_dict=db_functions.data_processing_translations(),
filename=data_file,
model=DataProcessing)
soakdb_query = SoakdbFiles.objects.get(filename=data_file)
soakdb_query.status = 2
soakdb_query.save()
def transfer_changed_datafile(data_file, hit_directory, log_directory = DirectoriesConfig().log_directory):
"""Transfers a changed file to XCDB by calling `transfer_file(data_file)`
:param data_file: The soakdb that we want to check if it updated, :class:`TransferChangedDataFile` self.data_file
:type data_file: str
:param hit_directory: Directory to which the soakdb corresponds to. Usually :class:`TransferChangedDataFile` self.hit_directory
:type hit_directory: str
:return: Should return nothing but will update the
:rtype: None
"""
print(data_file)
maint_exists = db_functions.check_table_sqlite(data_file, 'mainTable')
if maint_exists == 1:
soakdb_query = SoakdbFiles.objects.get(filename=data_file)
print(soakdb_query)
split_path = data_file.split('database')
search_path = split_path[0]
# remove pandda data transfer done file
if os.path.isfile(os.path.join(search_path, 'transfer_pandda_data.done')):
os.remove(os.path.join(search_path, 'transfer_pandda_data.done'))
log_files = find_log_files(search_path).rsplit()
print(log_files)
for log in log_files:
print(f"{log}.run.done")
if os.path.isfile(f"{log}.run.done"):
os.remove(f"{log}.run.done")
if os.path.isfile(f"{log}.sites.done"):
os.remove(f"{log}.sites.done")
if os.path.isfile(f"{log}.events.done"):
os.remove(f"{log}.events.done")
# find_logs_out_files = glob.glob(str(search_path + '*.txt'))
find_logs_out_files = glob.glob(f"{search_path}*.txt")
for f in find_logs_out_files:
if is_date(f.replace(search_path, '').replace('.txt', '')):
os.remove(f)
out, err, proposal = db_functions.pop_soakdb(data_file)
db_functions.pop_proposals(proposal)
else:
print('Main Table does not exist!')
transfer_file(data_file)
def run(self):
# delete all fields from soakdb filename
maint_exists = db_functions.check_table_sqlite(self.data_file,
'mainTable')
if maint_exists == 1:
soakdb_query = SoakdbFiles.objects.get(filename=self.data_file)
crystals = Crystal.objects.filter(visit=soakdb_query)
for crystal in crystals:
refinement = Refinement.objects.get(crystal_name=crystal)
target_name = str(crystal.target.target_name).upper()
crystal_name = str(crystal.crystal_name)
proasis_crystal_directory = os.path.join(
self.hit_directory, target_name, crystal_name)
if ProasisHits.objects.filter(crystal_name=crystal).exists():
proasis_hit = ProasisHits.objects.filter(
crystal_name=crystal)
for hit in proasis_hit:
for path in glob.glob(
os.path.join(
os.getcwd(), 'logs/proasis/hits',
str(hit.crystal_name.crystal_name + '_' +
hit.modification_date + '*'))):
os.remove(path)
if os.path.isdir(proasis_crystal_directory):
shutil.rmtree(
os.path.join(proasis_crystal_directory))
if ProasisOut.objects.filter(proasis=hit).exists:
for obj in ProasisOut.objects.filter(proasis=hit):
obj.delete()
hit.delete()
soakdb_query.delete()
out, err, proposal = db_functions.pop_soakdb(self.data_file)
db_functions.pop_proposals(proposal)
transfer_file(self.data_file)
with self.output().open('w') as f:
f.write('')
def requires(self):
if not os.path.isfile(StartTransfers(date=self.date, soak_db_filepath=self.soak_db_filepath).output().path):
return StartTransfers(date=self.date, soak_db_filepath=self.soak_db_filepath)
else:
soakdb_files = [obj.filename for obj in SoakdbFiles.objects.all()]
m = [Lab, Dimple, DataProcessing, Refinement]
zipped = []
for filename in soakdb_files:
for model in m:
try:
maint_exists = db_functions.check_table_sqlite(filename, 'mainTable')
except OperationalError:
if not os.path.isfile(filename):
f = SoakdbFiles.objects.get(filename=filename)
f.delete()
continue
else:
raise Exception(str(traceback.format_exc() + '; db_file=' + filename))
if maint_exists == 1:
zipped.append(tuple([filename, model]))
return [CheckFileUpload(filename=filename, model=model) for (filename, model) in zipped]
def requires(self):
if not os.path.isfile(
StartTransfers(
date=self.date,
soak_db_filepath=self.soak_db_filepath).output().path):
return StartTransfers(date=self.date,
soak_db_filepath=self.soak_db_filepath)
else:
soakdb_files = [obj.filename for obj in SoakdbFiles.objects.all()]
models = [Lab, Dimple, DataProcessing, Refinement]
zipped = []
for filename in soakdb_files:
for model in models:
maint_exists = db_functions.check_table_sqlite(
filename, 'mainTable')
if maint_exists == 1:
zipped.append(tuple([filename, model]))
return [
CheckFileUpload(filename=filename, model=model)
for (filename, model) in zipped
]
def run(self):
print(self.data_file)
# delete all fields from soakdb filename
maint_exists = db_functions.check_table_sqlite(self.data_file, 'mainTable')
if maint_exists == 1:
soakdb_query = SoakdbFiles.objects.get(filename=self.data_file)
print(soakdb_query)
# for pandda file finding
split_path = self.data_file.split('database')
search_path = split_path[0]
# remove pandda data transfer done file
if os.path.isfile(os.path.join(search_path, 'transfer_pandda_data.done')):
os.remove(os.path.join(search_path, 'transfer_pandda_data.done'))
log_files = find_log_files(search_path).rsplit()
print(log_files)
for log in log_files:
print(str(log + '.run.done'))
if os.path.isfile(str(log + '.run.done')):
os.remove(str(log + '.run.done'))
if os.path.isfile(str(log + '.sites.done')):
os.remove(str(log + '.sites.done'))
if os.path.isfile(str(log + '.events.done')):
os.remove(str(log + '.events.done'))
find_logs_out_files = glob.glob(str(search_path + '*.txt'))
for f in find_logs_out_files:
if is_date(f.replace(search_path, '').replace('.txt', '')):
os.remove(f)
crystals = Crystal.objects.filter(visit=soakdb_query)
for crystal in crystals:
target_name = str(crystal.target.target_name).upper()
crystal_name = str(crystal.crystal_name)
proasis_crystal_directory = os.path.join(self.hit_directory, target_name.upper(), crystal_name)
if ProasisHits.objects.filter(crystal_name=crystal).exists():
proasis_hit = ProasisHits.objects.filter(crystal_name=crystal)
for hit in proasis_hit:
for path in glob.glob(os.path.join(DirectoriesConfig().log_directory, 'proasis/hits',
str(hit.crystal_name.crystal_name +
'_' + hit.modification_date + '*'))):
os.remove(path)
if os.path.isdir(proasis_crystal_directory):
shutil.rmtree(os.path.join(proasis_crystal_directory), ignore_errors=True)
if ProasisOut.objects.filter(proasis=hit).exists:
for obj in ProasisOut.objects.filter(proasis=hit):
if obj.root:
delete_files = ['verne.transferred', 'PROPOSALS', 'VISITS', 'visits_proposals.done']
for f in delete_files:
if os.path.isfile(os.path.join(obj.root, '/'.join(obj.start.split('/')[:-2]),
f)):
os.remove(os.path.join(obj.root, '/'.join(obj.start.split('/')[:-2]), f))
shutil.rmtree(os.path.join(obj.root, obj.start))
obj.delete()
hit.delete()
soakdb_query.delete()
out, err, proposal = db_functions.pop_soakdb(self.data_file)
db_functions.pop_proposals(proposal)
else:
print('MAIN TABLE DOES NOT EXIST!')
transfer_file(self.data_file)
with self.output().open('w') as f:
f.write('')
