def test_expand_vars(self):
self.fs.makedir("TYRIONLANISTER")
self.fs.makedir("$FOO")
path = self.fs.getsyspath("$FOO")
os.environ["FOO"] = "TYRIONLANISTER"
fs1 = osfs.OSFS(path)
fs2 = osfs.OSFS(path, expand_vars=False)
self.assertIn("TYRIONLANISTER", fs1.getsyspath("/"))
self.assertNotIn("TYRIONLANISTER", fs2.getsyspath("/"))
def my_main(dataset_name):
img_loc = osfs.OSFS('./runs', create=True)
img_loc.removetree('.')
samples = img_loc.makedir('samples')
weights = img_loc.makedir('weights')
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
assert hasattr(datasets, dataset_name)
ds = getattr(datasets, dataset_name)
data_loc = fs.appfs.UserCacheFS('torchvision').getsyspath(dataset_name)
train_data = ds(data_loc, download=True)
tr = trainer.RelGAN(networks.Generator(), networks.Classifier(),
train_data.train_data.float() / 256, device)
for i in tqdm(range(50000), miniters=50):
tr.step()
if i % 100 == 0:
with torch.no_grad():
g = tr.generator(10)
v = torch.cat([g[j] for j in range(10)], 1).detach()
if v.device != 'cpu':
v = v.cpu()
plt.imshow(v.numpy())
plt.savefig(samples.getsyspath('step_{:06d}.png'.format(i)))
plt.close('all')
with weights.open('weights-{:06d}.pkl'.format(i), 'wb') as f:
torch.save(tr.generator, f)
torch.save(tr.critic, f)
def main():
args = parse_args()
config = read_config()
logger, logger_stream = get_logger()
# Para crear las tablas de los modelos, se debería utilizar la receta
# 'migrate', que se asegura de ejecutar todas las migraciones de Alembic
# que existan. Sin embargo, para facilitar el uso del ETL durante el
# desarrollo ('interactive'), se crean las tablas automáticamente si no se
# encuentran. Notar que esto puede llevar a situaciones como que se cree
# una tabla que en realidad debería haber sido creada en una migración, lo
# cual resultaria en errores cuando se intente ejecutar la migración.
init_models = args.mode == 'interactive'
ctx = Context(config=config,
fs=osfs.OSFS(config.get('etl', 'files_dir'),
create=True,
create_mode=constants.DIR_PERMS),
engine=create_engine(config['db'],
echo=args.verbose,
init_models=init_models),
report=Report(logger, logger_stream),
mode=args.mode)
if args.command == 'etl':
etl(args.processes, args.start, args.end, args.no_mail, ctx)
elif args.command == 'console':
console(ctx)
elif args.command == 'info':
info(ctx)
elif args.command == 'stats':
stats(ctx)
else:
raise RuntimeError('Invalid command.')
def setUp(self):
self.temp_dir = tempfile.mkdtemp(u"fstest")
open(os.path.join(self.temp_dir, u".dotfile"), 'w').close()
open(os.path.join(self.temp_dir, u"regularfile"), 'w').close()
os.mkdir(os.path.join(self.temp_dir, u".dotdir"))
os.mkdir(os.path.join(self.temp_dir, u"regulardir"))
self.fs = HideDotFilesFS(osfs.OSFS(self.temp_dir))
def test_create(self):
"""Test create=True"""
dir_path = tempfile.mkdtemp()
try:
create_dir = os.path.join(dir_path, "test_create")
with osfs.OSFS(create_dir, create=True):
self.assertTrue(os.path.isdir(create_dir))
self.assertTrue(os.path.isdir(create_dir))
finally:
shutil.rmtree(dir_path)
# Test exception when unable to create dir
with tempfile.NamedTemporaryFile() as tmp_file:
with self.assertRaises(errors.CreateFailed):
# Trying to create a dir that exists as a file
osfs.OSFS(tmp_file.name, create=True)
def setUp(self):
self.fs = tempfs.TempFS()
watchfs = osfs.OSFS(self.fs.root_path)
self.watchfs = ensure_watchable(watchfs, poll_interval=0.1)
if watch_inotify is not None:
self.assertEqual(watchfs, self.watchfs)
if watch_win32 is not None:
self.assertEqual(watchfs, self.watchfs)
def test_unicode_paths(self):
dir_path = tempfile.mkdtemp()
try:
fs_dir = os.path.join(dir_path, "te\u0161t_\u00fanicod\u0113")
os.mkdir(fs_dir)
with osfs.OSFS(fs_dir):
self.assertTrue(os.path.isdir(fs_dir))
finally:
shutil.rmtree(dir_path)
def deidentify_file(deid_profile, file_path, output_directory):
"""
Args:
deid_profile(DeIdProfile): the de-identification profile to use to process the file
file_path: the path to the file to be de-identified
output_directory(str): the directory to which to output the de-identified file
Returns:
str: path to the de-identified file
"""
dirname, basename = os.path.split(file_path)
with osfs.OSFS(dirname) as src_fs:
with osfs.OSFS(output_directory) as dst_fs:
deid_profile.process_file(src_fs=src_fs,
src_file=basename,
dst_fs=dst_fs)
deid_files = [dst_fs.getsyspath(fp) for fp in dst_fs.walk.files()]
if deid_files:
deid_path = deid_files[0]
else:
deid_path = ''
return deid_path
def __init__(self, fsname, write):
# Writable Zip fs
self.datafs = zipfs.ZipFS(fsname, write)
self.osfs = None
self.write = write
if write:
self.osfs = osfs.OSFS(u'/')
self.cwd = os.getcwd()
with self.create_file(u'/cwd') as f:
f.write(self.cwd.encode('utf-8'))
else:
assert (os.path.exists(fsname))
with self.datafs.open(u'/cwd', encoding='utf-8') as f:
self.cwd = f.read()
if os.path.altsep is not None:
self.cwd += os.path.altsep
else:
self.cwd += os.path.sep
def __init__(self,
remote_url: str,
discriminator: str = "type",
read_only: bool = False):
self.read_only = read_only
if isinstance(remote_url, str):
if remote_url[-1] == "/":
remote_url = remote_url[:-1]
self.remote_fs = open_fs(remote_url, create=True)
else:
self.remote_fs = remote_url
self.remote_is_local = self.remote_fs.hassyspath(".")
if self.remote_is_local and not read_only:
self.local_fs = self.remote_fs
else:
self.local_fs = osfs.OSFS(tempfile.mkdtemp())
self.new = not self.remote_fs.exists(self.db_file)
if (not self.new and not self.remote_is_local) or self.read_only:
# store exists
copy.copy_file(self.remote_fs, self.db_file, self.local_fs,
self.db_file)
dbpath = path.join(self.local_fs.getsyspath("."), self.db_file)
self.connection = sqlite3.connect(dbpath)
self.connection.row_factory = sqlite3.Row
self._options = dict()
self._schemas = dict()
if self.new:
self._create_options_table()
self.set_strict(True)
self.set_discriminator(discriminator)
self._tables = self._get_tables()
def deidentify_files(profile_path,
input_directory,
profile_name='dicom',
file_list=None,
output_directory=None,
date_increment=None):
"""
Given profile_path to a valid flywheel de-id profile with a "dicom" namespace, this function
replaces original files with de-identified copies of DICOM files .
Returns a list of paths to the deidentified files. If no changes were imposed by the profile,
no files will be modified
Args:
profile_path (str): Path to the de-id profile to apply
input_directory (str): Directory containing the dicoms to be de-identified
profile_name (str): Name of the profile to pass .get_file_profile()
output_directory (str): Directory to which to save de-identified files. If not provided, originals will be
replaced
date_increment (str): Date offset to apply to the profile
file_list (list, optional): Optional list of relative paths of files to process, if not provided, will work
on all files in the input_directory
Returns:
list: list of paths to deidentified files or None if no files are de-identified
"""
with tempfile.TemporaryDirectory() as tmp_deid_dir:
# Load the de-id profile from a file
deid_profile = deidentify.load_profile(profile_path)
# Load the de-id profile as a dictionary
template_dict = parse_deid_template(profile_path)
if date_increment:
deid_profile.date_increment = date_increment
# OSFS setup
src_fs = osfs.OSFS(input_directory)
dst_fs = osfs.OSFS(tmp_deid_dir)
if not output_directory:
output_directory = input_directory
if not file_list:
# Get list of files (dicom files do not always have an extension in the wild)
file_list = [
match.path
for match in src_fs.glob('**/*', case_sensitive=False)
if not match.info.is_dir
]
# Monkey-patch get_dest_path to return the original path
# This necessitates creating any missing subdirectories
def default_path(state, record, path):
dst_fs.makedirs(fs.path.dirname(path), recreate=True)
return path
# Get the dicom profile from the de-id profile
file_profile = deid_profile.get_file_profile(profile_name)
if template_dict.get(profile_name):
file_dict = template_dict.get(profile_name)
if file_dict.get('remove_private_tags') == True:
file_profile.remove_private_tags = True
file_profile.get_dest_path = default_path
file_profile.process_files(src_fs, dst_fs, file_list)
# get list of modified files in tmp_deid_dir
deid_files = [
match.path for match in dst_fs.glob('**/*', case_sensitive=False)
if not match.info.is_dir
]
deid_paths = list()
for deid_file in deid_files:
deid_file = deid_file.lstrip(os.path.sep)
# Create list of de-identified files
deid_path = os.path.join(output_directory, deid_file)
deid_paths.append(deid_path)
tmp_filepath = os.path.join(tmp_deid_dir, deid_file)
replace_filepath = os.path.join(output_directory, deid_file)
shutil.move(tmp_filepath, replace_filepath)
if not deid_paths:
return None
return deid_paths
def setUp(self):
self.temp_dir = tempfile.mkdtemp("fstest")
self.fs = osfs.OSFS(self.temp_dir)
def __init__(self, name, fs=None):
self.sp = SelfPrint(leading="- ")
self.name = name
if fs is None:
fs = osfs.OSFS(".")
self.fs = fs
def make_fs(self):
temp_dir = tempfile.mkdtemp("fstestosfs")
return osfs.OSFS(temp_dir)
def test_not_exists(self):
with self.assertRaises(errors.CreateFailed):
osfs.OSFS("/does/not/exists/")
def setUp(self):
self.temp_dir = tempfile.mkdtemp("fstest")
self.parent_fs = osfs.OSFS(self.temp_dir)
self.parent_fs.makedir("foo/bar", recursive=True)
self.fs = self.parent_fs.opendir("foo/bar")
from fs import zipfs, osfs, path
import zipfile
import logging
from dataflow.utils import input_protection
import backend
from .common import File, Chapter, Page
import config
import workers.unzippers as unzippers
logger = logging.getLogger(__name__)
# noinspection PyUnresolvedReferences
root_filestore = osfs.OSFS(config.storage_dir)
data_filestore = fs.path.join('/data', 'raw_files')
@input_protection()
def unzip(input: File, output_chapter: Queue, output_page: Queue):
"""
First, open the file and try to guess what chapters it contains.
Emit a chapter object for each chapter.
Then emit a page for each page in that chapter.
--
"""
logger.debug('Entering unzip')
def make_fs_from_string(string, _cache={}):
"""
Create a FS object from a string. Uses a cache to avoid creating multiple
FS objects for any given string (except for tempfs which allows multple
instances).
"""
if string == 'tempfs':
# Use a temporary filesystem which is only available to the current
# process, and will be cleaned up automatically. Bypass the cache
# for this type of filesystem, as they are unique and self-contained.
return tempfs.TempFS()
if string.startswith('~/'):
string = os.path.expanduser(string)
if string in _cache:
return _cache[string]
if string.startswith('/'):
# Use a simple directory on the filesystem.
if not os.path.exists(string):
osfs.OSFS('/', dir_mode=0775).makedir(
path=string,
recursive=True,
allow_recreate=True,
)
fs = osfs.OSFS(string, dir_mode=0775)
elif string.startswith('s3:'):
# Use an S3 bucket.
s3_bucket = string[3:]
if '/' in s3_bucket:
s3_bucket, path = s3_bucket.split('/', 1)
else:
path = ''
# The S3FS class can poll S3 for a file's etag after writing
# to it, to ensure that the file upload has been written to
# all relevant nodes in the S3 cluster.
# S3 has read-after-write consistency for PUTS of new objects
# and eventual consistency for overwrite PUTS and DELETES.
# See http://aws.amazon.com/s3/faqs/
# Most of our operations are writing to new files, so disable
# this mostly wasteful check. This might need to be revisited
# if there is a special case where we're updating files.
key_sync_timeout = None
fs = s3fs.S3FS(s3_bucket, key_sync_timeout=key_sync_timeout)
if path:
fs = fs.makeopendir(path, recursive=True)
elif string.startswith('http://'):
fs = httpfs.HTTPFS(string)
else:
raise ValueError('Unsupported storage string %r' % string)
_cache[string] = fs
return fs
def setUp(self):
self.temp_dir = tempfile.mkdtemp("fstest")
self.parent_fs = osfs.OSFS(self.temp_dir)
self.parent_fs.makedir("__subdir__")
self.fs = self.parent_fs.opendir("__subdir__")
def setUp(self):
self.temp_dir = tempfile.mkdtemp(u"fstest")
self.fs = wrapfs.WrapFS(osfs.OSFS(self.temp_dir))