def test_mc_init(self): proj_list = mcapi.get_all_projects() proj_dict = {p.name:p for p in proj_list} self.assertFalse('CLITest' in proj_dict) testargs = ['mc', 'init'] with captured_output(wd=self.proj_path) as (sout, serr): init_subcommand(testargs) #print_stringIO(sout) out = sout.getvalue().splitlines() remote = mcapi.use_remote() url = remote.config.mcurl self.assertEqual(out[0], "Created new project at: " + url) testargs = ['mc', 'init'] with captured_output(wd=self.proj_path) as (sout, serr): init_subcommand(testargs) #print_stringIO(sout) out = sout.getvalue().splitlines() self.assertTrue(re.match("Already in project. name: CLITest", out[0])) proj_list = mcapi.get_all_projects() proj_dict = {p.name:p for p in proj_list} self.assertTrue('CLITest' in proj_dict) with working_dir(self.proj_path): proj = make_local_project() self.assertEqual(proj.name, 'CLITest')
def get_all_objects(self, args): if self.requires_project and _proj_path() is None: raise Exception("Not in any Materials Commons project directory") if not self.proj_member: data = self.get_all() else: proj = make_local_project() if self.expt_member and args.expt: expt = make_local_expt(proj) data = self.get_all_from_experiment(expt) else: data = self.get_all_from_project(proj) def _any_match(value): for n in args.expr: if re.match(n, value): return True return False if args.expr: if args.id: objects = [d for d in data if _any_match(d.id)] elif self.has_owner and args.owner: objects = [d for d in data if _any_match(d.owner)] else: objects = [d for d in data if _any_match(d.name)] else: objects = data return objects
def up_subcommand(argv=sys.argv): """ upload files to Materials Commons mc up [-r] [<pathspec> ...] """ parser = argparse.ArgumentParser( description='upload files', prog='mc up') parser.add_argument('paths', nargs='*', default=None, help='Files or directories') parser.add_argument('-r', '--recursive', action="store_true", default=False, help='Upload directory contents recursively') parser.add_argument('--limit', nargs=1, type=float, default=[50], help='File size upload limit (MB). Default=50MB.') # ignore 'mc up' args = parser.parse_args(argv[2:]) limit = args.limit[0] proj = make_local_project() local_abspaths = set([os.path.abspath(p) for p in args.paths]) if not args.recursive: dirs = [p for p in local_abspaths if os.path.isdir(p)] for d in dirs: local_abspaths.remove(d) files = [os.path.join(p, f) for f in os.listdir(d) if os.path.isfile(os.path.join(p,f))] local_abspaths.update(files) if args.recursive and proj.local_path in local_abspaths: # treat upload of everything specially local_abspaths = set([os.path.join(proj.local_path, f) for f in os.listdir(proj.local_path) if f != ".mc"]) for p in local_abspaths: if os.path.isfile(p): #print "uploading:", p try: result = proj.add_file_by_local_path(p, verbose=True, limit=limit) except Exception as e: print "Could not upload:", p print "Error:" print e elif os.path.isdir(p) and args.recursive: #print "uploading:", p result, error = proj.add_directory_tree_by_local_path(p, verbose=True, limit=limit) if len(error): for file in error: print "Could not upload:", file print "Error:" print error[file] return
def get_proj(cls): return make_local_project(cls.proj_path)
def expt_subcommand(argv=sys.argv): """ List, create, delete, and modify experiments mc expt mc expt [-c] <exptname> """ parser = argparse.ArgumentParser( description='List, create, delete, and modify experiments', prog='mc expt') parser.add_argument('expts', nargs='*', default=None, help='Experiment names (or id if --id given)') parser.add_argument('-l', '--list', action="store_true", default=True, help='List experiments (default)') parser.add_argument('-c', '--create', action="store_true", default=False, help='Create experiments') parser.add_argument('-d', '--delete', action="store_true", default=False, help='Delete experiments') parser.add_argument('-s', '--set', action="store_true", default=False, help='Set current experiment') parser.add_argument('--id', action="store_true", default=False, help='Input experiment id instead of name') parser.add_argument('--desc', type=str, default="", help='Experiment description') parser.add_argument('-n', '--dry-run', action="store_true", default=False, help='Dry run deletion') parser.add_argument('-a', '--all', action="store_true", default=False, help='Delete all samples and processes') #parser.add_argument('-m', '--rename', type=str, default='origin', help='Rename experiment') # ignore 'mc expt' args = parser.parse_args(argv[2:]) proj = make_local_project() expt_list = proj.get_all_experiments() expts = {e.name:e for e in expt_list} expt_ids = {e.id:e for e in expt_list} if args.create: if len(args.expts) != 1: print 'create one experiment at a time' print 'example: mc expt -c ExptName --desc "short description"' parser.print_help() exit(1) for name in args.expts: if name not in expts: expt = proj.create_experiment(name, args.desc) print 'Created experiment:', expt.name set_current_experiment(proj, expt) print "Set current experiment: '" + expt.name + "'" else: print 'experiment: \'' + name + '\' already exists' elif args.delete: for name in args.expts: if args.id: _expts = expt_ids else: _expts = expts _expts[name].delete(dry_run=args.dry_run, delete_processes_and_samples=args.all) print 'Deleted experiment:', name for key, val in _expts[name].delete_tally.__dict__.iteritems(): print key, val print "" elif args.set: if len(args.expts) != 1: print 'set one current experiment at a time' print 'example: mc expt -s ExptName' parser.print_help() exit(1) if args.id: set_current_experiment(proj, expt_ids[args.expts[0]]) print "Set current experiment: '" + expt_ids[args.expts[0]].name + "'" else: set_current_experiment(proj, expts[args.expts[0]]) print "Set current experiment: '" + expts[args.expts[0]].name + "'" elif args.list: _print_experiments(proj.get_all_experiments(), make_local_expt(proj)) return
def get_proj(self): return make_local_project(self.proj_path)
def ls_subcommand(argv=sys.argv): """ 'ls' a project directory to see local and remote files and directories. mc ls [<pathspec> ...] """ parser = argparse.ArgumentParser( description='List local & remote directory contents', prog='mc ls') parser.add_argument('paths', nargs='*', default=[os.getcwd()], help='Files or directories') parser.add_argument('--checksum', action="store_true", default=False, help='Calculate MD5 checksum for local files') parser.add_argument('--json', action="store_true", default=False, help='Print JSON exactly') # ignore 'mc ls' args = parser.parse_args(argv[2:]) local_abspaths = [os.path.abspath(p) for p in args.paths] proj = make_local_project() # act on local paths (df, files, dirs, remotes) = _ls_group(proj, local_abspaths, files_only=True, checksum=args.checksum, json=args.json) # print warnings for type mismatches for file in files: if file in dirs and os.path.isfile(file): print "warning!", file, "is a local file and remote directory!" if file in dirs and os.path.isdir(file): print "warning!", file, "is a local directory and remote file!" if args.json: for r in remotes: print r.input_data else: if df.shape[0]: print df.to_string(index=False) print "" for d in dirs: _locals = [] if os.path.exists(d): _locals = [os.path.join(d, f) for f in os.listdir(d)] if os.path.abspath(d) == proj.local_path: remote_dir = proj.get_top_directory() else: remote_dir = proj.get_by_local_path(d) _remotes = [] if remote_dir is not None: _remotes = [os.path.join(d, child.name) for child in remote_dir.get_children()] _local_abspaths = SortedSet(_locals + _remotes) (df, files, dirs, remotes) = _ls_group(proj, _local_abspaths, files_only=False, checksum=args.checksum, json=args.json) if args.json: for r in remotes: print r.input_data else: if df.shape[0]: print os.path.relpath(d, os.getcwd()) + ":" #print df.to_string(index=False) print tabulate(df, showindex=False, headers=df.columns, tablefmt="plain") print "" return