def transfer_file(data_file): """Transfer contents of soakdb.sqlite file to xcdb into separate (crystal, lab, refinement, dimple and data_processing tables. :param data_file: File-path(?) pointing towards a soakdb.sqlite file :type data_file: str :return: Function should not return anything but will write to a database. :rtype: None """ maint_exists = db_functions.check_table_sqlite(data_file, 'mainTable') if maint_exists == 1: db_functions.transfer_table(translate_dict=db_functions.crystal_translations(), filename=data_file, model=Crystal) db_functions.transfer_table(translate_dict=db_functions.lab_translations(), filename=data_file, model=Lab) db_functions.transfer_table(translate_dict=db_functions.refinement_translations(), filename=data_file, model=Refinement) db_functions.transfer_table(translate_dict=db_functions.dimple_translations(), filename=data_file, model=Dimple) db_functions.transfer_table(translate_dict=db_functions.data_processing_translations(), filename=data_file, model=DataProcessing) soakdb_query = SoakdbFiles.objects.get(filename=data_file) soakdb_query.status = 2 soakdb_query.save()
def transfer_file(data_file): maint_exists = db_functions.check_table_sqlite(data_file, 'mainTable') if maint_exists == 1: db_functions.transfer_table( translate_dict=db_functions.crystal_translations(), filename=data_file, model=Crystal) db_functions.transfer_table( translate_dict=db_functions.lab_translations(), filename=data_file, model=Lab) db_functions.transfer_table( translate_dict=db_functions.refinement_translations(), filename=data_file, model=Refinement) db_functions.transfer_table( translate_dict=db_functions.dimple_translations(), filename=data_file, model=Dimple) db_functions.transfer_table( translate_dict=db_functions.data_processing_translations(), filename=data_file, model=DataProcessing) soakdb_query = SoakdbFiles.objects.get(filename=data_file) soakdb_query.status = 2 soakdb_query.save()
def transfer_changed_datafile(data_file, hit_directory, log_directory = DirectoriesConfig().log_directory): """Transfers a changed file to XCDB by calling `transfer_file(data_file)` :param data_file: The soakdb that we want to check if it updated, :class:`TransferChangedDataFile` self.data_file :type data_file: str :param hit_directory: Directory to which the soakdb corresponds to. Usually :class:`TransferChangedDataFile` self.hit_directory :type hit_directory: str :return: Should return nothing but will update the :rtype: None """ print(data_file) maint_exists = db_functions.check_table_sqlite(data_file, 'mainTable') if maint_exists == 1: soakdb_query = SoakdbFiles.objects.get(filename=data_file) print(soakdb_query) split_path = data_file.split('database') search_path = split_path[0] # remove pandda data transfer done file if os.path.isfile(os.path.join(search_path, 'transfer_pandda_data.done')): os.remove(os.path.join(search_path, 'transfer_pandda_data.done')) log_files = find_log_files(search_path).rsplit() print(log_files) for log in log_files: print(f"{log}.run.done") if os.path.isfile(f"{log}.run.done"): os.remove(f"{log}.run.done") if os.path.isfile(f"{log}.sites.done"): os.remove(f"{log}.sites.done") if os.path.isfile(f"{log}.events.done"): os.remove(f"{log}.events.done") # find_logs_out_files = glob.glob(str(search_path + '*.txt')) find_logs_out_files = glob.glob(f"{search_path}*.txt") for f in find_logs_out_files: if is_date(f.replace(search_path, '').replace('.txt', '')): os.remove(f) out, err, proposal = db_functions.pop_soakdb(data_file) db_functions.pop_proposals(proposal) else: print('Main Table does not exist!') transfer_file(data_file)
def run(self): # delete all fields from soakdb filename maint_exists = db_functions.check_table_sqlite(self.data_file, 'mainTable') if maint_exists == 1: soakdb_query = SoakdbFiles.objects.get(filename=self.data_file) crystals = Crystal.objects.filter(visit=soakdb_query) for crystal in crystals: refinement = Refinement.objects.get(crystal_name=crystal) target_name = str(crystal.target.target_name).upper() crystal_name = str(crystal.crystal_name) proasis_crystal_directory = os.path.join( self.hit_directory, target_name, crystal_name) if ProasisHits.objects.filter(crystal_name=crystal).exists(): proasis_hit = ProasisHits.objects.filter( crystal_name=crystal) for hit in proasis_hit: for path in glob.glob( os.path.join( os.getcwd(), 'logs/proasis/hits', str(hit.crystal_name.crystal_name + '_' + hit.modification_date + '*'))): os.remove(path) if os.path.isdir(proasis_crystal_directory): shutil.rmtree( os.path.join(proasis_crystal_directory)) if ProasisOut.objects.filter(proasis=hit).exists: for obj in ProasisOut.objects.filter(proasis=hit): obj.delete() hit.delete() soakdb_query.delete() out, err, proposal = db_functions.pop_soakdb(self.data_file) db_functions.pop_proposals(proposal) transfer_file(self.data_file) with self.output().open('w') as f: f.write('')
def requires(self): if not os.path.isfile(StartTransfers(date=self.date, soak_db_filepath=self.soak_db_filepath).output().path): return StartTransfers(date=self.date, soak_db_filepath=self.soak_db_filepath) else: soakdb_files = [obj.filename for obj in SoakdbFiles.objects.all()] m = [Lab, Dimple, DataProcessing, Refinement] zipped = [] for filename in soakdb_files: for model in m: try: maint_exists = db_functions.check_table_sqlite(filename, 'mainTable') except OperationalError: if not os.path.isfile(filename): f = SoakdbFiles.objects.get(filename=filename) f.delete() continue else: raise Exception(str(traceback.format_exc() + '; db_file=' + filename)) if maint_exists == 1: zipped.append(tuple([filename, model])) return [CheckFileUpload(filename=filename, model=model) for (filename, model) in zipped]
def requires(self): if not os.path.isfile( StartTransfers( date=self.date, soak_db_filepath=self.soak_db_filepath).output().path): return StartTransfers(date=self.date, soak_db_filepath=self.soak_db_filepath) else: soakdb_files = [obj.filename for obj in SoakdbFiles.objects.all()] models = [Lab, Dimple, DataProcessing, Refinement] zipped = [] for filename in soakdb_files: for model in models: maint_exists = db_functions.check_table_sqlite( filename, 'mainTable') if maint_exists == 1: zipped.append(tuple([filename, model])) return [ CheckFileUpload(filename=filename, model=model) for (filename, model) in zipped ]
def run(self): print(self.data_file) # delete all fields from soakdb filename maint_exists = db_functions.check_table_sqlite(self.data_file, 'mainTable') if maint_exists == 1: soakdb_query = SoakdbFiles.objects.get(filename=self.data_file) print(soakdb_query) # for pandda file finding split_path = self.data_file.split('database') search_path = split_path[0] # remove pandda data transfer done file if os.path.isfile(os.path.join(search_path, 'transfer_pandda_data.done')): os.remove(os.path.join(search_path, 'transfer_pandda_data.done')) log_files = find_log_files(search_path).rsplit() print(log_files) for log in log_files: print(str(log + '.run.done')) if os.path.isfile(str(log + '.run.done')): os.remove(str(log + '.run.done')) if os.path.isfile(str(log + '.sites.done')): os.remove(str(log + '.sites.done')) if os.path.isfile(str(log + '.events.done')): os.remove(str(log + '.events.done')) find_logs_out_files = glob.glob(str(search_path + '*.txt')) for f in find_logs_out_files: if is_date(f.replace(search_path, '').replace('.txt', '')): os.remove(f) crystals = Crystal.objects.filter(visit=soakdb_query) for crystal in crystals: target_name = str(crystal.target.target_name).upper() crystal_name = str(crystal.crystal_name) proasis_crystal_directory = os.path.join(self.hit_directory, target_name.upper(), crystal_name) if ProasisHits.objects.filter(crystal_name=crystal).exists(): proasis_hit = ProasisHits.objects.filter(crystal_name=crystal) for hit in proasis_hit: for path in glob.glob(os.path.join(DirectoriesConfig().log_directory, 'proasis/hits', str(hit.crystal_name.crystal_name + '_' + hit.modification_date + '*'))): os.remove(path) if os.path.isdir(proasis_crystal_directory): shutil.rmtree(os.path.join(proasis_crystal_directory), ignore_errors=True) if ProasisOut.objects.filter(proasis=hit).exists: for obj in ProasisOut.objects.filter(proasis=hit): if obj.root: delete_files = ['verne.transferred', 'PROPOSALS', 'VISITS', 'visits_proposals.done'] for f in delete_files: if os.path.isfile(os.path.join(obj.root, '/'.join(obj.start.split('/')[:-2]), f)): os.remove(os.path.join(obj.root, '/'.join(obj.start.split('/')[:-2]), f)) shutil.rmtree(os.path.join(obj.root, obj.start)) obj.delete() hit.delete() soakdb_query.delete() out, err, proposal = db_functions.pop_soakdb(self.data_file) db_functions.pop_proposals(proposal) else: print('MAIN TABLE DOES NOT EXIST!') transfer_file(self.data_file) with self.output().open('w') as f: f.write('')