def main(args): """ Main entry point """ logger.debug('Starting file structure scan.') for nc_file in ilist_files(args.directory): nc_file_name = os.path.basename(nc_file) db_files = DataFile.objects.filter(name=nc_file_name) if db_files.count() == 0: logger.error('File not found in database: {}'.format(nc_file)) elif db_files.count() > 1: logger.error('{} entries found in database for file: {}'. format(db_files.count(), nc_file)) else: db_file = db_files.first() act_size = os.path.getsize(nc_file) if act_size != db_file.size: logger.info('File %s has size %d', db_file.name, act_size) db_file.online = False db_file.directory = None db_file.save() os.remove(nc_file) if not is_same_gws(nc_file, BASE_OUTPUT_DIR): sym_link_path = os.path.join(BASE_OUTPUT_DIR, construct_drs_path(db_file), db_file.name) try: if os.path.exists(sym_link_path): os.remove(sym_link_path) except OSError: logger.error('Unable to delete sym link %s', sym_link_path)
def scan_database(): """ Start the scan of the database. """ logger.debug('Starting database scan.') for data_file in DataFile.objects.filter(online=True).iterator(): full_path = os.path.join(data_file.directory, data_file.name) if not os.path.exists(full_path): logger.warning('File cannot be found on disk, status changed to ' 'offline: {}'.format(full_path)) data_file.online = False data_file.directory = None data_file.save() continue if not is_same_gws(data_file.directory, BASE_OUTPUT_DIR): sym_link_dir = os.path.join(BASE_OUTPUT_DIR, construct_drs_path(data_file)) sym_link_path = os.path.join(sym_link_dir, data_file.name) if not os.path.exists(sym_link_path): if not os.path.exists(sym_link_dir): os.makedirs(sym_link_dir) os.symlink(full_path, sym_link_path) logger.warning('Created symlink for file {} from {}'. format(data_file.name, sym_link_path)) logger.debug('Completed database scan.')
def main(args): """ Main entry point """ base_output_dir = Settings.get_solo().base_output_dir for data_file in DataFile.objects.filter(online=True): gws_pattern = r'^/group_workspaces/jasmin2/primavera(\d)/(\S*)' gws = re.match(gws_pattern, data_file.directory) if not gws: logger.error('No GWS match for {}'.format(data_file.name)) continue new_gws = '/gws/nopw/j04/primavera' + gws.group(1) new_dir = os.path.join(new_gws, gws.group(2)) new_path = os.path.join(new_dir, data_file.name) if not os.path.exists(new_path): logger.error('Cannot find {}'.format(new_path)) continue data_file.directory = new_dir data_file.save() if not is_same_gws(data_file.directory, base_output_dir): link_path = os.path.join(base_output_dir, construct_drs_path(data_file), data_file.name) # it's got to be a link but check anyway if os.path.islink(link_path): os.remove(link_path) os.symlink(os.path.join(data_file.directory, data_file.name), link_path) else: logger.error('Expected a link but found a file at {}'. format(link_path))
def main(args): """ Main entry point """ dfs = DataFile.objects.filter(climate_model__short_name='MPI-ESM1-2-XR', experiment__short_name='highres-future', version='v20190617') prim_gws = '/gws/nopw/j04/primavera5/stream1' old_dirs = [] for df in dfs: old_drs_path = construct_drs_path(df) df.version = 'v20190517' df.save() if df.online: # file itself gws = get_gws(df.directory) old_dir = df.directory new_dir = os.path.join(gws, construct_drs_path(df)) if not os.path.exists(new_dir): os.makedirs(new_dir) os.rename(os.path.join(df.directory, df.name), os.path.join(new_dir, df.name)) df.directory = new_dir df.save() if old_dir not in old_dirs: old_dirs.append(old_dir) # sym link if not is_same_gws(df.directory, prim_gws): old_sym_dir = os.path.join(prim_gws, old_drs_path) old_sym = os.path.join(old_sym_dir, df.name) # TODO next line doesn't work as this is now a broken symlink so returns false if os.path.exists(old_sym): if os.path.islink(old_sym): os.remove(old_sym) else: logger.warning(f'Not symlink as expected: {old_sym}') new_sym_dir = os.path.join(prim_gws, construct_drs_path(df)) if not os.path.exists(new_sym_dir): os.makedirs(new_sym_dir) os.symlink(os.path.join(new_dir, df.name), os.path.join(new_sym_dir, df.name)) if old_sym_dir not in old_dirs: old_dirs.append(old_sym_dir) logger.debug(f'Removing {len(old_dirs)} old dirs') for old_dir in old_dirs: delete_drs_dir(old_dir)
def main(args): """Main entry point""" base_dir = Settings.get_solo().base_output_dir for extracted_file in ilist_files(args.top_dir): found_name = os.path.basename(extracted_file) try: data_file = DataFile.objects.get(name=found_name) except django.core.exceptions.ObjectDoesNotExist: logger.warning('Cannot find DMT entry. Skipping {}'. format(extracted_file)) continue found_checksum = adler32(extracted_file) if not found_checksum == data_file.checksum_set.first().checksum_value: logger.warning("Checksum doesn't match. Skipping {}". format(found_name)) continue dest_dir = os.path.join(get_gws_any_dir(extracted_file), 'stream1', construct_drs_path(data_file)) dest_path = os.path.join(dest_dir, found_name) if os.path.exists(dest_path): logger.warning('Skipping {} as it already exists at {}'. format(found_name, dest_path)) continue # create the directory if it doesn't exist if not os.path.exists(dest_dir): os.makedirs(dest_dir) os.rename(extracted_file, dest_path) # create a link from the base dir if not is_same_gws(dest_path, base_dir): link_dir = os.path.join(base_dir, construct_drs_path(data_file)) link_path = os.path.join(link_dir, data_file.name) if not os.path.exists(link_dir): os.makedirs(link_dir) os.symlink(dest_path, link_path) data_file.online = True data_file.directory = dest_dir data_file.save()
def _rename_file(self): """ Rename the file on disk and move to its new directory. Update the link from the primary directory. """ if not os.path.exists(self.new_directory): os.makedirs(self.new_directory) os.rename(os.path.join(self.old_directory, self.old_filename), os.path.join(self.new_directory, self.new_filename)) # check for empty directory if not os.listdir(self.old_directory): delete_drs_dir(self.old_directory) # Update the symbolic link if required if not is_same_gws(self.old_directory, BASE_OUTPUT_DIR): old_link_path = os.path.join(self.old_sym_link_dir, self.old_filename) if os.path.lexists(old_link_path): if not os.path.islink(old_link_path): logger.error("{} exists and isn't a symbolic link.".format( old_link_path)) raise SymLinkIsFileError(old_link_path) else: # it is a link so remove it os.remove(old_link_path) # check for empty directory if not os.listdir(self.old_sym_link_dir): delete_drs_dir(self.old_sym_link_dir) new_link_dir = os.path.join(BASE_OUTPUT_DIR, construct_drs_path(self.datafile)) if not os.path.exists(new_link_dir): os.makedirs(new_link_dir) os.symlink(os.path.join(self.new_directory, self.new_filename), os.path.join(new_link_dir, self.new_filename))
def move_dirs(data_req, new_gws): """ Move the files :param pdata_app.models.DataRequest data_req: the data request to move :param int new_gws: the number of the gws to move to """ single_dir = '{}{}'.format(COMMON_GWS_NAME, new_gws) existing_dirs = data_req.directories() # ignore data that is offline if None in existing_dirs: existing_dirs.remove(None) use_single_dir = False for exist_dir in existing_dirs: if exist_dir.startswith(single_dir): use_single_dir = True break if not use_single_dir: # As a quick sanity check, generate an error if there is no # data already in the requested output directory logger.error('The new output directory is {} but no data from ' 'this variable is currently in this directory.'. format(single_dir)) sys.exit(1) for exist_dir in existing_dirs: if exist_dir.startswith(single_dir): continue files_to_move = data_req.datafile_set.filter(directory=exist_dir) logger.debug('Moving {} files from {}'.format( files_to_move.count(), exist_dir)) for file_to_move in files_to_move: # Move the file src = os.path.join(exist_dir, file_to_move.name) dest_path = os.path.join(single_dir, 'stream1', construct_drs_path(file_to_move)) if not os.path.exists(dest_path): os.makedirs(dest_path) dest = os.path.join(dest_path, file_to_move.name) # remove existing link if about to write over it if dest.startswith(BASE_OUTPUT_DIR): if os.path.exists(dest): if os.path.islink(dest): os.remove(dest) # Move the file shutil.move(src, dest) # Update the file's location in the DB file_to_move.directory = dest_path file_to_move.save() # Check that it was safely copied actual_checksum = adler32(dest) db_checksum = file_to_move.checksum_set.first().checksum_value if not actual_checksum == db_checksum: logger.error('For {}\ndatabase checksum: {}\n' 'actual checksum: {}'. format(dest, db_checksum, actual_checksum)) sys.exit(1) # Update the symlink if not is_same_gws(dest_path, BASE_OUTPUT_DIR): primary_path_dir = os.path.join( BASE_OUTPUT_DIR, construct_drs_path(file_to_move)) primary_path = os.path.join(primary_path_dir, file_to_move.name) if os.path.lexists(primary_path): if not os.path.islink(primary_path): logger.error("{} exists and isn't a symbolic " "link.".format(primary_path)) sys.exit(1) else: # it is a link so remove it os.remove(primary_path) if not os.path.exists(primary_path_dir): os.makedirs(primary_path_dir) os.symlink(dest, primary_path) delete_drs_dir(exist_dir)
def main(args): """ Main entry point """ logger.debug('Starting incoming_to_drs.py') data_sub = _get_submission_object(os.path.normpath(args.directory)) if not args.alternative: drs_base_dir = BASE_OUTPUT_DIR else: drs_base_dir = args.alternative errors_encountered = False for data_file in data_sub.datafile_set.order_by('name'): # make full path of existing file existing_path = os.path.join(data_file.directory, data_file.name) # make full path of where it will live drs_sub_path = construct_drs_path(data_file) drs_dir = os.path.join(drs_base_dir, drs_sub_path) drs_path = os.path.join(drs_dir, data_file.name) # check the destination directory exists if not os.path.exists(drs_dir): os.makedirs(drs_dir) # link if on same GWS, or else copy this_file_error = False try: os.rename(existing_path, drs_path) except OSError as exc: logger.error('Unable to link from {} to {}. {}'.format( existing_path, drs_path, str(exc))) errors_encountered = True this_file_error = True # update the file's location in the database if not this_file_error: data_file.directory = drs_dir if not data_file.online: data_file.online = True data_file.save() # if storing the files in an alternative location, create a sym link # from the primary DRS structure to the file if not is_same_gws(BASE_OUTPUT_DIR, drs_base_dir): primary_path = os.path.join(BASE_OUTPUT_DIR, drs_sub_path) try: if not os.path.exists(primary_path): os.makedirs(primary_path) os.symlink(drs_path, os.path.join(primary_path, data_file.name)) except OSError as exc: logger.error('Unable to link from {} to {}. {}'.format( drs_path, os.path.join(primary_path, data_file.name), str(exc))) errors_encountered = True # summarise what happened and keep the DB updated if not errors_encountered: logger.debug('All files copied with no errors. Data submission ' 'incoming directory can be deleted.') else: logger.error('Errors were encountered. Please fix these before ' 'deleting the incoming directory.') logger.debug('Completed incoming_to_drs.py')
def test_slightly_bad_path(self): path1 = '/group_workspaces/jasmin2/primavera2/some/dir' path2 = '/group_workspaces/jasmin1/primavera1/some/dir' self.assertFalse(is_same_gws(path1, path2))
def test_archive_second(self): path1 = '/group_workspaces/jasmin2/primavera2/some/dir' path2 = '/badc/cmip6' self.assertFalse(is_same_gws(path1, path2))
def test_new_diff(self): path1 = '/gws/nopw/j04/primavera1/some/dir' path2 = '/gws/nopw/j04/primavera5/another/dir' self.assertFalse(is_same_gws(path1, path2))
def test_same(self): path1 = '/group_workspaces/jasmin2/primavera1/some/dir' path2 = '/group_workspaces/jasmin2/primavera1/another/dir' self.assertTrue(is_same_gws(path1, path2))
def main(args): """ Main entry point """ dreqs_hr = DataRequest.objects.filter( climate_model__short_name='CMCC-CM2-HR4', experiment__short_name__in=['hist-1950', 'control-1950'], variable_request__table_name__startswith='SI', datafile__isnull=False).distinct() dreqs_vhr = DataRequest.objects.filter( climate_model__short_name='CMCC-CM2-VHR4', experiment__short_name='hist-1950', variable_request__table_name__startswith='SI', datafile__isnull=False).distinct() dreqs = dreqs_hr | dreqs_vhr logger.debug(f'Found {dreqs.count()} data requests') for dreq in dreqs: logger.debug(f'Processing {dreq}') old_directories = [] for df in dreq.datafile_set.order_by('name'): if not df.online: logger.error(f'Not online {df.name}') continue if df.version == NEW_VERSION: logger.warning(f'Already at {NEW_VERSION} {df.name}') continue # save the sym link directory before we make any changes if not is_same_gws(BASE_OUTPUT_DIR, df.directory): old_sym_link_dir = os.path.join(BASE_OUTPUT_DIR, construct_drs_path(df)) # now get back to updating the version df.version = NEW_VERSION gws = get_gws(df.directory) new_dir = os.path.join(gws, construct_drs_path(df)) old_directory = df.directory if not os.path.exists(new_dir): os.mkdir(new_dir) os.rename(os.path.join(df.directory, df.name), os.path.join(new_dir, df.name)) df.directory = new_dir df.save() if old_directory not in old_directories: old_directories.append(old_directory) # Update any sym links too if not is_same_gws(BASE_OUTPUT_DIR, df.directory): sym_link_path = os.path.join(old_sym_link_dir, df.name) if os.path.lexists(sym_link_path): if os.path.islink(sym_link_path): os.remove(sym_link_path) if old_sym_link_dir not in old_directories: old_directories.append(old_sym_link_dir) sym_link_dir = os.path.join(BASE_OUTPUT_DIR, construct_drs_path(df)) if not os.path.exists(sym_link_dir): os.makedirs(sym_link_dir) sym_link_path = os.path.join(sym_link_dir, df.name) os.symlink(os.path.join(df.directory, df.name), sym_link_path) for directory in old_directories: if not os.listdir(directory): delete_drs_dir(directory) else: logger.error(f'Not empty {directory}')
def copy_et_files_into_drs(data_files, retrieval_dir, args): """ Copy files from the restored data cache into the DRS structure. :param list data_files: The DataFile objects to copy. :param str retrieval_dir: The path that the files were retrieved to. :param argparse.Namespace args: The parsed command line arguments namespace. """ logger.debug('Copying elastic tape files') for data_file in data_files: file_submission_dir = data_file.incoming_directory filename = (data_file.name if not args.incoming else data_file.incoming_name) extracted_file_path = os.path.join(retrieval_dir, file_submission_dir.lstrip('/'), filename) if not os.path.exists(extracted_file_path): msg = ('Unable to find file {} in the extracted data at {}. The ' 'expected path was {}'.format(filename, retrieval_dir, extracted_file_path)) logger.error(msg) sys.exit(1) drs_path = construct_drs_path(data_file) if not args.alternative: drs_dir = os.path.join(BASE_OUTPUT_DIR, drs_path) else: drs_dir = os.path.join(args.alternative, drs_path) dest_file_path = os.path.join(drs_dir, filename) # create the path if it doesn't exist if not os.path.exists(drs_dir): os.makedirs(drs_dir) if os.path.exists(dest_file_path): msg = 'File already exists on disk: {}'.format(dest_file_path) logger.warning(msg) else: os.rename(extracted_file_path, dest_file_path) if not args.skip_checksums: try: _check_file_checksum(data_file, dest_file_path) except ChecksumError: # warning message has already been displayed and so move on # to next file continue # create symbolic link from main directory if storing data in an # alternative directory if args.alternative and not is_same_gws(dest_file_path, BASE_OUTPUT_DIR): primary_path = os.path.join(BASE_OUTPUT_DIR, drs_path) if not os.path.exists(primary_path): os.makedirs(primary_path) os.symlink(dest_file_path, os.path.join(primary_path, filename)) # set directory and set status as being online data_file.directory = drs_dir data_file.online = True data_file.save() logger.debug('Finished copying elastic tape files')