def merge_cat(args): if not os.path.exists(args.target): args.target = os.path.join(args.archive, cat._subdir, os.path.basename(args.target)) if not os.path.exists(args.target): tgt = cat.new() else: tgt = json.load(open(args.target, 'rU')) if not tgt['namespace'] == cat._ns: raise ValueError("'%s' is not a catalog" % args.target) log.info("appending to '%s', resources=%d", args.target, len(tgt['resources'])) for source in args.source: src = json.load(open(source, 'rU')) if not src['namespace'] == cat._ns: log.error("'%s' is not a catalog; skipping" % source) continue log.info("reading from '%s', resources=%d", source, len(src['resources'])) cat.merge(src, tgt, args.no_confirm) json.dump(tgt, open(args.target, 'wt'), indent=2, separators=(',', ': ')) log.info("wrote merged catalog '%s', resources=%d", args.target, len(tgt['resources']))
def store_files(args): log.info("version: %s", nbank.__version__) log.info("run time: %s", datetime.datetime.now()) cfg = nbank.get_config(args.archive) if cfg is None: raise ValueError("%s not a neurobank archive. Use '-A' or set NBANK_PATH in environment." % args.archive) if args.read_stdin: args.file.extend(l.strip() for l in sys.stdin) if len(args.file) == 0: raise ValueError("no files specified") if os.path.exists(args.catalog): raise ValueError("catalog '%s' already exists. Write to a new file, then merge" % args.catalog) files = [] for fname in args.file: if not os.path.isfile(fname) and not os.path.isdir(fname): log.warn("warning: '%s' is not a file or directory - skipping", fname) continue path, base, ext = util.fileparts(fname) try: id = args.func_id(fname) except IOError as e: if os.path.exists(fname): log.warn( "warning: '%s' is a directory, can't be used a a source file - skipping", fname ) else: log.warn("warning: '%s' does not exist - skipping", fname) continue if cfg['policy'][args.target]['keep_filename']: id += '_' + base if args.suffix: id += '_' + args.suffix if cfg['policy'][args.target]['keep_extension']: id += ext try: mode = int(cfg['policy'][args.target]['mode'], base=8) except (KeyError, ValueError) as e: log.warn("E: %s", e) mode = 0o440 tgt = nbank.store_file(fname, args.archive, id, mode) if args.target == 'data' and tgt is None: # id collisions are errors for data files. This should never happen # with uuids raise ValueError("id assigned to '%s' already exists in archive: %s" % (fname, id)) files.append({'id': id, 'name': base + ext}) if tgt is not None: # file was moved to database log.info("%s -> %s", fname, id) if args.link: try: os.symlink(os.path.abspath(tgt), os.path.join(path, id)) except OSError as e: log.warn("error creating link: %s", e) else: log.info("'%s' already in archive as '%s'", fname, id) json.dump(cat.new(files), open(args.catalog, 'wt'), indent=2, separators=(',', ': ')) log.info("wrote resource catalog to '%s'", args.catalog)