def test_broken_link(): """ CommandLine: python -m ubelt.tests.test_links test_broken_link """ dpath = ub.ensure_app_cache_dir('ubelt', 'test_broken_link') ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) util_links._dirstats(dpath) broken_fpath = join(dpath, 'broken_fpath.txt') broken_flink = join(dpath, 'broken_flink.txt') ub.touch(broken_fpath, verbose=2) util_links._dirstats(dpath) ub.symlink(broken_fpath, broken_flink, verbose=2) util_links._dirstats(dpath) ub.delete(broken_fpath, verbose=2) util_links._dirstats(dpath) # make sure I am sane that this is the correct check. can_symlink = util_links._can_symlink() print('can_symlink = {!r}'.format(can_symlink)) if can_symlink: # normal behavior assert islink(broken_flink) assert not exists(broken_flink) else: # on windows hard links are essentially the same file. # there is no trace that it was actually a link. assert exists(broken_flink)
def setup_dpath(self, short=True, hashed=True): train_info = self.train_info(short, hashed) train_dpath = ub.ensuredir(train_info['train_dpath']) train_info_fpath = join(train_dpath, 'train_info.json') util.write_json(train_info_fpath, train_info) # setup symlinks ub.ensuredir(dirname(train_info['link_dpath'])) ub.symlink(train_info['train_dpath'], train_info['link_dpath'], overwrite=True, verbose=3) if train_info['nice_dpath']: ub.ensuredir(dirname(train_info['nice_dpath'])) ub.symlink(train_info['train_dpath'], train_info['nice_dpath'], overwrite=True, verbose=3) print('+=========') # print('hyper_strid = {!r}'.format(params.hyper_id())) # print('train_init_id = {!r}'.format(train_info['input_id'])) # print('arch = {!r}'.format(train_info['arch_id'])) # print('train_hyper_hashid = {!r}'.format(train_info['train_hyper_hashid'])) print('hyper = {}'.format(ub.repr2(train_info['hyper'], nl=3))) print('train_hyper_id_brief = {!r}'.format(train_info['train_hyper_id_brief'])) print('train_id = {!r}'.format(train_info['train_id'])) print('+=========') return train_info
def test_cant_overwrite_file_with_symlink(): if ub.WIN32: # Can't distinguish this case on windows pytest.skip() dpath = ub.ensure_app_cache_dir('ubelt', 'test_cant_overwrite_file_with_symlink') ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) happy_fpath = join(dpath, 'happy_fpath.txt') happy_flink = join(dpath, 'happy_flink.txt') for verbose in [2, 1, 0]: print('=======') print('verbose = {!r}'.format(verbose)) ub.delete(dpath, verbose=verbose) ub.ensuredir(dpath, verbose=verbose) ub.touch(happy_fpath, verbose=verbose) ub.touch(happy_flink) # create a file where a link should be util_links._dirstats(dpath) with pytest.raises(FileExistsError): # file exists error ub.symlink(happy_fpath, happy_flink, overwrite=False, verbose=verbose) with pytest.raises(FileExistsError): # file exists error ub.symlink(happy_fpath, happy_flink, overwrite=True, verbose=verbose)
def setup_loggers(harn): if harn.train_dpath is None: harn.warn('harn.train_dpath is None, cannot setup loggers') return use_file_logger = True if use_file_logger and harn.flog is None: flog_fname = 'fitlog_{}.log'.format(ub.timestamp()) flog_fpath = os.path.join(harn.train_dpath, flog_fname) flog = logging.getLogger(harn.__class__.__name__) formatter = logging.Formatter('%(asctime)s : %(message)s') handler = logging.FileHandler(flog_fpath, mode='w') handler.setFormatter(formatter) flog.propagate = False flog.setLevel(logging.DEBUG) flog.addHandler(handler) harn.flog = flog harn.debug('initialized file logger') flog_link = os.path.join(harn.train_dpath, 'fit.log') ub.symlink(flog_fpath, flog_link, overwrite=True) if tensorboard_logger: train_base = os.path.dirname(harn.nice_dpath or harn.train_dpath) harn.log('dont forget to start: tensorboard --logdir ' + train_base) harn.log('Initializing tensorboard') harn.tlog = tensorboard_logger.Logger(harn.train_dpath, flush_secs=2) else: harn.log('Tensorboard is not available')
def test_modify_directory_symlinks(): dpath = ub.ensure_app_cache_dir('ubelt', 'test_modify_symlinks') ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) happy_dpath = join(dpath, 'happy_dpath') happy_dlink = join(dpath, 'happy_dlink') ub.ensuredir(happy_dpath, verbose=2) ub.symlink(happy_dpath, happy_dlink, verbose=2) # Test file inside directory symlink file_path1 = join(happy_dpath, 'file.txt') file_path2 = join(happy_dlink, 'file.txt') ub.touch(file_path1, verbose=2) assert exists(file_path1) assert exists(file_path2) ub.writeto(file_path1, 'foo') assert ub.readfrom(file_path1) == 'foo' assert ub.readfrom(file_path2) == 'foo' ub.writeto(file_path2, 'bar') assert ub.readfrom(file_path1) == 'bar' assert ub.readfrom(file_path2) == 'bar' ub.delete(file_path2, verbose=2) assert not exists(file_path1) assert not exists(file_path2) # Test directory inside directory symlink dir_path1 = join(happy_dpath, 'dir') dir_path2 = join(happy_dlink, 'dir') ub.ensuredir(dir_path1, verbose=2) assert exists(dir_path1) assert exists(dir_path2) subfile_path1 = join(dir_path1, 'subfile.txt') subfile_path2 = join(dir_path2, 'subfile.txt') ub.writeto(subfile_path2, 'foo') assert ub.readfrom(subfile_path1) == 'foo' assert ub.readfrom(subfile_path2) == 'foo' ub.writeto(subfile_path1, 'bar') assert ub.readfrom(subfile_path1) == 'bar' assert ub.readfrom(subfile_path2) == 'bar' ub.delete(dir_path1, verbose=2) assert not exists(dir_path1) assert not exists(dir_path2)
def setup_dpath(self, train_dpath, short=True, hashed=True): train_info = self.train_info(train_dpath, short, hashed) train_dpath = ub.ensuredir(train_info['train_dpath']) # backwards compatability code, # can eventually remove after a major version change if True: # backwards compatability code if os.path.exists( train_info['old_train_dpath']) and not os.path.islink( train_info['old_train_dpath']): ub.delete(train_info['train_dpath']) ub.symlink(train_info['old_train_dpath'], train_info['train_dpath'], overwrite=True, verbose=3) # setup symlinks # ub.ensuredir(dirname(train_info['link_dpath'])) # ub.symlink(train_info['train_dpath'], train_info['link_dpath'], # overwrite=True, verbose=3) if train_info['nice_dpath']: ub.ensuredir(dirname(train_info['nice_dpath'])) ub.symlink(train_info['train_dpath'], train_info['nice_dpath'], overwrite=True, verbose=3) verbose = 0 if verbose: print('+=========') # print('hyper_strid = {!r}'.format(params.hyper_id())) # print('train_init_id = {!r}'.format(train_info['input_id'])) # print('arch = {!r}'.format(train_info['arch_id'])) # print('train_hyper_hashid = {!r}'.format(train_info['train_hyper_hashid'])) print('hyper = {}'.format(ub.repr2(train_info['hyper'], nl=3))) print('train_hyper_id_brief = {!r}'.format( train_info['train_hyper_id_brief'])) print('train_id = {!r}'.format(train_info['train_id'])) print('+=========') return train_info
def test_modify_file_symlinks(): """ CommandLine: python -m ubelt.tests.test_links test_modify_symlinks """ # TODO: test that we handle broken links dpath = ub.ensure_app_cache_dir('ubelt', 'test_modify_symlinks') happy_fpath = join(dpath, 'happy_fpath.txt') happy_flink = join(dpath, 'happy_flink.txt') ub.touch(happy_fpath, verbose=2) ub.symlink(happy_fpath, happy_flink, verbose=2) # Test file symlink ub.writeto(happy_fpath, 'foo') assert ub.readfrom(happy_fpath) == 'foo' assert ub.readfrom(happy_flink) == 'foo' ub.writeto(happy_flink, 'bar') assert ub.readfrom(happy_fpath) == 'bar' assert ub.readfrom(happy_flink) == 'bar'
def test_rel_file_link(): dpath = ub.ensure_app_cache_dir('ubelt', 'test_rel_file_link') ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) real_fpath = join(ub.ensuredir((dpath, 'dir1')), 'real') link_fpath = join(ub.ensuredir((dpath, 'dir2')), 'link') ub.touch(real_fpath) orig = os.getcwd() try: os.chdir(dpath) real_path = relpath(real_fpath, dpath) link_path = relpath(link_fpath, dpath) link = ub.symlink(real_path, link_path) import sys if sys.platform.startswith('win32') and isfile(link): # Note: if windows hard links the file there is no way we can # tell that it was a symlink. Just verify it exists. from ubelt import _win32_links assert _win32_links._win32_is_hardlinked(real_fpath, link_fpath) else: pointed = ub.util_links._readlink(link) resolved = os.path.realpath( ub.expandpath(join(dirname(link), pointed))) assert os.path.realpath(ub.expandpath(real_fpath)) == resolved except Exception: util_links._dirstats(dpath) util_links._dirstats(join(dpath, 'dir1')) util_links._dirstats(join(dpath, 'dir2')) print('TEST FAILED: test_rel_link') print('real_fpath = {!r}'.format(real_fpath)) print('link_fpath = {!r}'.format(link_fpath)) print('real_path = {!r}'.format(real_path)) print('link_path = {!r}'.format(link_path)) try: if 'link' in vars(): print('link = {!r}'.format(link)) if 'pointed' in vars(): print('pointed = {!r}'.format(pointed)) if 'resolved' in vars(): print('resolved = {!r}'.format(resolved)) except Exception: print('...rest of the names are not available') raise finally: util_links._dirstats(dpath) util_links._dirstats(join(dpath, 'dir1')) util_links._dirstats(join(dpath, 'dir2')) os.chdir(orig)
def test_rel_dir_link(): dpath = ub.ensure_app_cache_dir('ubelt', 'test_rel_dir_link') ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) real_dpath = join(ub.ensuredir((dpath, 'dir1')), 'real') link_dpath = join(ub.ensuredir((dpath, 'dir2')), 'link') ub.ensuredir(real_dpath) orig = os.getcwd() try: os.chdir(dpath) real_path = relpath(real_dpath, dpath) link_path = relpath(link_dpath, dpath) link = ub.symlink(real_path, link_path) # Note: on windows this is hacked. pointed = ub.util_links._readlink(link) resolved = os.path.realpath(ub.expandpath(join(dirname(link), pointed))) assert os.path.realpath(ub.expandpath(real_dpath)) == resolved except Exception: util_links._dirstats(dpath) util_links._dirstats(join(dpath, 'dir1')) util_links._dirstats(join(dpath, 'dir2')) print('TEST FAILED: test_rel_link') print('real_dpath = {!r}'.format(real_dpath)) print('link_dpath = {!r}'.format(link_dpath)) print('real_path = {!r}'.format(real_path)) print('link_path = {!r}'.format(link_path)) try: if 'link' in vars(): print('link = {!r}'.format(link)) if 'pointed' in vars(): print('pointed = {!r}'.format(pointed)) if 'resolved' in vars(): print('resolved = {!r}'.format(resolved)) except Exception: print('...rest of the names are not available') raise finally: util_links._dirstats(dpath) util_links._dirstats(join(dpath, 'dir1')) util_links._dirstats(join(dpath, 'dir2')) os.chdir(orig)
def test_overwrite_symlink(): """ CommandLine: python -m ubelt.tests.test_links test_overwrite_symlink """ # TODO: test that we handle broken links dpath = ub.ensure_app_cache_dir('ubelt', 'test_overwrite_symlink') ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) happy_fpath = join(dpath, 'happy_fpath.txt') other_fpath = join(dpath, 'other_fpath.txt') happy_flink = join(dpath, 'happy_flink.txt') for verbose in [2, 1, 0]: print('=======') print('verbose = {!r}'.format(verbose)) ub.delete(dpath, verbose=verbose) ub.ensuredir(dpath, verbose=verbose) ub.touch(happy_fpath, verbose=verbose) ub.touch(other_fpath, verbose=verbose) util_links._dirstats(dpath) ub.symlink(happy_fpath, happy_flink, verbose=verbose) # Creating a duplicate link util_links._dirstats(dpath) ub.symlink(happy_fpath, happy_flink, verbose=verbose) util_links._dirstats(dpath) with pytest.raises(Exception): # file exists error ub.symlink(other_fpath, happy_flink, verbose=verbose) ub.symlink(other_fpath, happy_flink, verbose=verbose, overwrite=True) ub.delete(other_fpath, verbose=verbose) with pytest.raises(Exception): # file exists error ub.symlink(happy_fpath, happy_flink, verbose=verbose) ub.symlink(happy_fpath, happy_flink, verbose=verbose, overwrite=True)
def test_delete_symlinks(): """ CommandLine: python -m ubelt.tests.test_links test_delete_symlinks """ # TODO: test that we handle broken links dpath = ub.ensure_app_cache_dir('ubelt', 'test_delete_links') happy_dpath = join(dpath, 'happy_dpath') happy_dlink = join(dpath, 'happy_dlink') happy_fpath = join(dpath, 'happy_fpath.txt') happy_flink = join(dpath, 'happy_flink.txt') broken_dpath = join(dpath, 'broken_dpath') broken_dlink = join(dpath, 'broken_dlink') broken_fpath = join(dpath, 'broken_fpath.txt') broken_flink = join(dpath, 'broken_flink.txt') def check_path_condition(path, positive, want, msg): if not want: positive = not positive msg = 'not ' + msg if not positive: util_links._dirstats(dpath) print('About to raise error: {}'.format(msg)) print('path = {!r}'.format(path)) print('exists(path) = {!r}'.format(exists(path))) print('islink(path) = {!r}'.format(islink(path))) print('isdir(path) = {!r}'.format(isdir(path))) print('isfile(path) = {!r}'.format(isfile(path))) raise AssertionError('path={} {}'.format(path, msg)) def assert_sometrace(path, want=True): # Either exists or is a broken link positive = exists(path) or islink(path) check_path_condition(path, positive, want, 'has trace') def assert_broken_link(path, want=True): if util_links._can_symlink(): print('path={} should{} be a broken link'.format( path, ' ' if want else ' not')) positive = not exists(path) and islink(path) check_path_condition(path, positive, want, 'broken link') else: # TODO: we can test this # positive = util_links._win32_is_junction(path) print('path={} should{} be a broken link (junction)'.format( path, ' ' if want else ' not')) print('cannot check this yet') # We wont be able to differentiate links and nonlinks for junctions # positive = exists(path) # check_path_condition(path, positive, want, 'broken link') util_links._dirstats(dpath) ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) util_links._dirstats(dpath) ub.ensuredir(happy_dpath, verbose=2) ub.ensuredir(broken_dpath, verbose=2) ub.touch(happy_fpath, verbose=2) ub.touch(broken_fpath, verbose=2) util_links._dirstats(dpath) ub.symlink(broken_fpath, broken_flink, verbose=2) ub.symlink(broken_dpath, broken_dlink, verbose=2) ub.symlink(happy_fpath, happy_flink, verbose=2) ub.symlink(happy_dpath, happy_dlink, verbose=2) util_links._dirstats(dpath) # Deleting the files should not delete the symlinks (windows) ub.delete(broken_fpath, verbose=2) util_links._dirstats(dpath) ub.delete(broken_dpath, verbose=2) util_links._dirstats(dpath) assert_broken_link(broken_flink, 1) assert_broken_link(broken_dlink, 1) assert_sometrace(broken_fpath, 0) assert_sometrace(broken_dpath, 0) assert_broken_link(happy_flink, 0) assert_broken_link(happy_dlink, 0) assert_sometrace(happy_fpath, 1) assert_sometrace(happy_dpath, 1) # broken symlinks no longer exist after they are deleted ub.delete(broken_dlink, verbose=2) util_links._dirstats(dpath) assert_sometrace(broken_dlink, 0) ub.delete(broken_flink, verbose=2) util_links._dirstats(dpath) assert_sometrace(broken_flink, 0) # real symlinks no longer exist after they are deleted # but the original data is fine ub.delete(happy_dlink, verbose=2) util_links._dirstats(dpath) assert_sometrace(happy_dlink, 0) assert_sometrace(happy_dpath, 1) ub.delete(happy_flink, verbose=2) util_links._dirstats(dpath) assert_sometrace(happy_flink, 0) assert_sometrace(happy_fpath, 1)
def ensure_selenium_chromedriver(): """ os.environ['webdriver.chrome.driver'] = ensure_selenium_chromedriver() """ import requests import zipfile timeout = 5.0 def latest_version(): rsp = requests.get( 'http://chromedriver.storage.googleapis.com/LATEST_RELEASE', timeout=timeout) if rsp.status_code != 200: raise Exception version = rsp.text.strip() return version # version = latest_version() # version = '91.0.4472.19' # version = '90.0.4430.24' version = '92.0.4515.107' known_hashs = { '91.0.4472.19': '49622b740b1c7e66b87179a2642f6c57f21a97fc844c84b30a48', '90.0.4430.24': 'b85313de6abc1b44f26a0e12e20cb66657b840417f5ac6018946', '92.0.4515.107': '844c0e04bbbfd286617af2d7facd3d6cf7d3491b1e78120f8e0', } url = 'http://chromedriver.storage.googleapis.com/{}/chromedriver_linux64.zip'.format( version) bin_dpath = pathlib.Path(ub.expandpath('~/.local/bin')) download_dpath = bin_dpath / f'chromedriver_{version}' download_dpath.mkdir(exist_ok=True, parents=True) zip_fpath = ub.grabdata( url, hash_prefix=known_hashs.get(version, 'unknown-version'), dpath=download_dpath, ) zip_fpath = pathlib.Path(zip_fpath) # dpath = zip_fpath.parent # TODO: version the binary chromedriver_fpath_real = download_dpath / 'chromedriver' chromedriver_fpath_link = bin_dpath / 'chromedriver' if not chromedriver_fpath_real.exists( ) or not chromedriver_fpath_link.exists(): # Also check hash? zfile = zipfile.ZipFile(str(zip_fpath)) try: fpath = zfile.extract('chromedriver', path=chromedriver_fpath_real.parent) finally: zfile.close() chromedriver_fpath_real_ = pathlib.Path(fpath) assert chromedriver_fpath_real_.exists() ub.symlink(chromedriver_fpath_real_, chromedriver_fpath_link, overwrite=True) if not ub.WIN32: print('add permission chromedriver_fpath_real_ = {!r}'.format( chromedriver_fpath_real_)) st = os.stat(chromedriver_fpath_real_) os.chmod(chromedriver_fpath_real_, st.st_mode | stat.S_IEXEC) os.environ['PATH'] = os.pathsep.join( ub.oset(os.environ['PATH'].split(os.pathsep)) | ub.oset([str(chromedriver_fpath_link.parent)])) return chromedriver_fpath_link
def 创建连接(文件路径, 目标路径, 覆盖=False, 显示信息=0): return ub.symlink(文件路径, 目标路径, 覆盖, 显示信息)
def test_overwrite_symlink(): """ CommandLine: python ~/code/ubelt/tests/test_links.py test_overwrite_symlink """ # TODO: test that we handle broken links dpath = ub.ensure_app_cache_dir('ubelt', 'test_overwrite_symlink') ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) happy_fpath = join(dpath, 'happy_fpath.txt') other_fpath = join(dpath, 'other_fpath.txt') happy_flink = join(dpath, 'happy_flink.txt') for verbose in [2, 1, 0]: print('@==========@') print('verbose = {!r}'.format(verbose)) print('[test] Setup') ub.delete(dpath, verbose=verbose) ub.ensuredir(dpath, verbose=verbose) ub.touch(happy_fpath, verbose=verbose) ub.touch(other_fpath, verbose=verbose) print('[test] Dirstats dpath') util_links._dirstats(dpath) print('[test] Create initial link (to happy)') ub.symlink(happy_fpath, happy_flink, verbose=verbose) print('[test] Dirstats dpath') util_links._dirstats(dpath) # Creating a duplicate link print('[test] Create a duplicate link (to happy)') ub.symlink(happy_fpath, happy_flink, verbose=verbose) print('[test] Dirstats dpath') util_links._dirstats(dpath) print('[test] Create an unauthorized overwrite link (to other)') with pytest.raises(Exception) as exc_info: # file exists error ub.symlink(other_fpath, happy_flink, verbose=verbose) print(' * exc_info = {!r}'.format(exc_info)) print('[test] Create an authorized overwrite link (to other)') ub.symlink(other_fpath, happy_flink, verbose=verbose, overwrite=True) print('[test] Dirstats dpath') ub.delete(other_fpath, verbose=verbose) print('[test] Create an unauthorized overwrite link (back to happy)') with pytest.raises(Exception) as exc_info: # file exists error ub.symlink(happy_fpath, happy_flink, verbose=verbose) print(' * exc_info = {!r}'.format(exc_info)) print('[test] Create an authorized overwrite link (back to happy)') ub.symlink(happy_fpath, happy_flink, verbose=verbose, overwrite=True)
def _ensure_image_cog(gpath, cache_gpath, config, hack_use_cli=True): """ Returns a special array-like object with a COG GeoTIFF backend """ cog_gpath = cache_gpath if not exists(cog_gpath): if not exists(gpath): raise OSError(( 'Source image gpath={!r} ' 'does not exist!').format(gpath)) ub.ensuredir(dirname(cog_gpath)) # If the file already is a cog, just use it from ndsampler.utils.validate_cog import validate as _validate_cog warnings, errors, details = _validate_cog(gpath) if DEBUG_LOAD_COG: from multiprocessing import current_process from threading import current_thread is_main = ( current_thread().name == 'MainThread' and current_process().name == 'MainProcess' ) DEBUG = 0 if is_main: DEBUG = 2 else: DEBUG = 0 if DEBUG: print('\n<DEBUG INFO>') print('Missing cog_gpath={}'.format(cog_gpath)) print('gpath = {!r}'.format(gpath)) gpath_is_cog = not bool(errors) if ub.argflag('--debug-validate-cog') or DEBUG > 2: print('details = {}'.format(ub.repr2(details, nl=1))) print('errors (why we need to ensure) = {}'.format(ub.repr2(errors, nl=1))) print('warnings = {}'.format(ub.repr2(warnings, nl=1))) print('gpath_is_cog = {!r}'.format(gpath_is_cog)) if gpath_is_cog: # If we already are a cog, then just use it if DEBUG: print('Image is already a cog, symlinking to cache') ub.symlink(gpath, cog_gpath) else: config = config.copy() if DEBUG: print('BUILDING COG REPRESENTATION') print('gpath = {!r}'.format(gpath)) print('cog config = {}'.format(ub.repr2(config, nl=2))) if DEBUG > 1: try: import netharn as nh print(' * info(gpath) = ' + ub.repr2(nh.util.get_file_info(gpath))) except ImportError: pass try: _locked_cache_write(_cog_cache_write, gpath, cache_gpath=cog_gpath, config=config) except CorruptCOG: import warnings msg = ('!!! COG was corrupted, trying once more') warnings.warn(msg) print(msg) _locked_cache_write(_cog_cache_write, gpath, cache_gpath=cog_gpath, config=config) if DEBUG: print('cog_gpath = {!r}'.format(cog_gpath)) if DEBUG > 1: try: import netharn as nh print(' * info(cog_gpath) = ' + ub.repr2(nh.util.get_file_info(cog_gpath))) except ImportError: pass print('</DEBUG INFO>') file = util_gdal.LazyGDalFrameFile(cog_gpath) return file