def test_globus_mkdir():
''' test globus mkdir '''
global test_ep, test_dir, mkdir_url
gsm = GlobusStorageManager()
gsm.activate_endpoint(test_ep)
log.debug('test_globus_mkdir: creating {}'.format(test_dir))
mkcode = gsm.mkdir(mkdir_url)
log.debug('mkdir {} returned {}'.format(mkdir_url, mkcode))
def test_globus_rename():
''' test globus rename '''
global mkdir_path, test_ep
srcp = '{}/{}'.format(mkdir_url, 'testfile.dst')
dstp = '{}/{}'.format(mkdir_url, 'testfile.end')
gsm = GlobusStorageManager()
gsm.activate_endpoint(test_ep)
if not gsm.rename(srcp, dstp):
raise Exception('rename from {} to {} failed.'.format(srcp, dstp))
def get_gsm(self):
log.debug('ArchivedVideoFile.get_gsm()')
if self.gsm is None:
self.gsm = GlobusStorageManager()
self.gsm.wait_timeout = PUBLICATION_TRANSFER_TIMEOUT
return self.gsm
def test_globus_cp():
''' test globus cp '''
global test_dir, mkdir_local_path, mkdir_url
srcfp = os.path.join(mkdir_local_path, 'testfile.src')
with open(srcfp, 'w') as f:
f.write('test file for {}'.format(test_dir))
gsm = GlobusStorageManager()
gsm.activate_endpoint(test_ep)
srcp = '{}/{}'.format(mkdir_url, 'testfile.src')
dstp = '{}/{}'.format(mkdir_url, 'testfile.dst')
if not gsm.cp(srcp, dstp):
raise Exception('cp from {} to {} failed.'.format(srcp, dstp))
def test_globus_ls():
''' test globus ls '''
global test_ep, test_ep_local_path, test_ep_url
test_fname = 'test.txt'
test_fpath = os.path.join(test_ep_local_path, test_fname)
if not os.path.exists(test_fpath):
with open(test_fpath, 'w') as f:
f.write('test file for ls')
gsm = GlobusStorageManager()
gsm.activate_endpoint(test_ep)
file_list = gsm.ls(test_ep_url)
log.debug('test_globus_ls(): file_list: {}'.format(file_list))
os.remove(test_fpath)
files = [e['name'] for e in file_list['DATA']]
if test_fname not in files:
raise Exception("{} not in entries".format(test_fname))
def publication_login(*args):
cfgname = args[0] if len(args) else 'local'
if 'custom' in dj.config and 'globus.token' in dj.config['custom']:
del dj.config['custom']['globus.token']
from pipeline.globus import GlobusStorageManager
GlobusStorageManager()
if cfgname == 'local':
dj.config.save_local()
elif cfgname == 'global':
dj.config.save_global()
else:
log.warning('unknown configuration {}. not saving'.format(cfgname))
def test_globus_activate_ep():
''' test adding an endpoint '''
global test_ep
gsm = GlobusStorageManager()
try:
gsm.activate_endpoint(None)
except Exception:
pass
gsm.activate_endpoint(test_ep)
def test_globus_rmdir():
''' test globus rmdir '''
global test_ep, mkdir_url
gsm = GlobusStorageManager()
log.debug('test_globus_rmdir: removing {}'.format(mkdir_url))
gsm.activate_endpoint(test_ep)
subscription_fixed_and_cp_tested = False
if subscription_fixed_and_cp_tested:
if gsm.rmdir(mkdir_url):
raise Exception('rmdir {} should have failed.'.format(mkdir_url))
if not gsm.rmdir(mkdir_url, recursive=True):
raise Exception('rmdir of {} failed.'.format(mkdir_url))
def test_globus_constructor():
''' test globus constructor '''
gsm = GlobusStorageManager()
return gsm
def get_gsm(self):
log.debug('ArchivedVideoFile.get_gsm()')
if self.gsm is None:
self.gsm = GlobusStorageManager()
return self.gsm
def get_gsm(self):
log.debug('ArchivedRawEphysTrial.get_gsm()')
if self.gsm is None:
self.gsm = GlobusStorageManager()
return self.gsm
def __init__(self, env={}):
self._env = env # Environment (grab bag atm)
self._gsm = GlobusStorageManager() # Globus Storage Manager
self._aep = set() # Active Endpoints
self._cep = None # Current Endpoint
self._cwd = '/' # Current Directory
class GlobusShell:
def __init__(self, env={}):
self._env = env # Environment (grab bag atm)
self._gsm = GlobusStorageManager() # Globus Storage Manager
self._aep = set() # Active Endpoints
self._cep = None # Current Endpoint
self._cwd = '/' # Current Directory
def _getpath(self, ep=None, wd=None):
ep = ep if ep else self._cep
wd = wd if wd else self._cwd
if ep == self._cep:
wd = wd if wd.startswith('/') else path.join(self._cwd, wd)
else:
wd = path.join('/', wd)
return ep, wd
def env(self):
print(self._env)
def cd(self, ep=None, wd='/'):
ep, wd = self._getpath(ep, wd)
log.debug('cd: {}:{}'.format(ep, wd))
if ep not in self._aep:
self._gsm.activate_endpoint(ep)
self._aep = self._aep.union({(ep)})
self._cep = ep
self._cwd = wd
def pwd(self):
print('{}:{}'.format(self._cep, self._cwd))
def mkdir(self, ep, wd):
ep, wd = self._getpath(ep, wd)
log.debug('mkdir: {}:{}'.format(ep, wd))
return self._gsm.mkdir('{}:{}'.format(ep, wd))
def ls(self, ep=None, wd=None):
ep, wd = self._getpath(ep, wd)
log.debug('ls: {}:{}'.format(ep, wd))
lsdat = self._gsm.ls('{}:{}'.format(ep, wd))
for i in iter(lsdat):
if i['type'] == 'file':
print('f: {}'.format(i['name']))
else:
print('d: {}'.format(i['name']))
return True
def rm(self, ep, wd):
ep, wd = self._getpath(ep, wd)
log.debug('rm: {}:{}'.format(ep, wd))
return self._gsm.rm('{}:{}'.format(ep, wd))
def rmdir(self, ep, wd):
ep, wd = self._getpath(ep, wd)
log.debug('rmdir: {}:{}'.format(ep, wd))
return self._gsm.rmdir('{}:{}'.format(ep, wd))
def rm_r(self, ep, wd):
# XXX: drop when shell gets 'args' support & add -r flag
ep, wd = self._getpath(ep, wd)
log.debug('rm_r: {}:{}'.format(ep, wd))
return self._gsm.rm('{}:{}'.format(ep, wd), recursive=True)
def mv(self, ep1, wd1, ep2, wd2):
ep1, wd1 = self._getpath(ep1, wd1)
ep2, wd2 = self._getpath(ep2, wd2)
if ep1 != ep2:
return False
log.debug('mv: {}:{} -> {}:{}'.format(ep1, wd1, ep2, wd2))
return self._gsm.rename('{}:{}'.format(ep1, wd1),
'{}:{}'.format(ep2, wd2))
def cp(self, ep1, wd1, ep2, wd2):
# XXX: delete_destination_xtra for mirroring?
ep1, wd1 = self._getpath(ep1, wd1)
ep2, wd2 = self._getpath(ep2, wd2)
log.debug('cp: {}:{} -> {}:{}'.format(ep1, wd1, ep2, wd2))
return self._gsm.cp('{}:{}'.format(ep1, wd1), '{}:{}'.format(ep2, wd2))
def cp_r(self, ep1, wd1, ep2, wd2):
ep1, wd1 = self._getpath(ep1, wd1)
ep2, wd2 = self._getpath(ep2, wd2)
log.debug('cp_r: {}:{} -> {}:{}'.format(ep1, wd1, ep2, wd2))
return self._gsm.cp('{}:{}'.format(ep1, wd1),
'{}:{}'.format(ep2, wd2),
recursive=True)
def find(self, ep=None, wd=None):
ep, wd = self._getpath(ep, wd)
log.debug('find: {}:{}'.format(ep, wd))
for ep, dirname, node in self._gsm.fts('{}:{}'.format(ep, wd)):
if node['DATA_TYPE'] == 'file':
t, basename = 'f', node['name']
else:
t, basename = 'd', None
print('{}: {}:{}/{}'.format(t, ep, dirname, basename) if t ==
'f' else '{}: {}:{}'.format(t, ep, dirname))
def sh(self, prompt='globus% '):
cmds = set(('env', 'pwd', 'cd', 'ls', 'find', 'cp', 'cp_r', 'mkdir',
'mv', 'rm', 'rmdir', 'rm_r'))
while True:
try:
data = input(prompt).split(' ')
except EOFError:
return
cmd, args = data[0], data[1:]
log.debug('cmd input: {} -> {}, {}'.format(data, cmd, args))
if not cmd or cmd.startswith('#'):
continue
elif cmd in cmds:
try:
# TODO: need 'flags' handling so e.g. 'cp -r' set recursive
args = (i.split(':') if ':' in i else (None, i)
for i in args)
ret = getattr(self, cmd)((*[j for i in args for j in i]))
if ret not in {True, None}:
print(ret)
except Exception as e:
print(repr(e))
else:
print('commands: \n - {}'.format('\n - '.join(cmds)))
class GlobusShell:
def __init__(self, env={}):
self._env = env # Environment (grab bag atm)
self._gsm = GlobusStorageManager() # Globus Storage Manager
self._aep = set() # Active Endpoints
self._cep = None # Current Endpoint
self._cwd = '/' # Current Directory
def env(self):
print(self._env)
def pwd(self):
print('{}:{}'.format(self._cep, self._cwd))
def cd(self, ep=None, path=None):
ep = ep if ep else self._cep
path = (path if
(ep == self._cep and path.startswith('/')) else '/'.join(
(self._cwd, path)))
if ep not in self._aep:
self._gsm.activate_endpoint(ep)
self._aep = self._aep.union({(ep)})
self._cep = ep
self._cwd = '/' + path.lstrip('/')
def ls(self, ep=None, path=None):
# TODO: should strip cwd from each pathname..
ep = ep if ep else self._cep
path = path if path else self._cwd
# todo: support ls with arg and also no path cwd
# path1 = (path1 if path1.startswith('/')
# else '{}/{}'.format(self._cwd, path1))
lsdat = self._gsm.ls('{}:{}'.format(ep, path))
for i in iter(lsdat):
if i['type'] == 'file':
print('f: {}'.format(i['name']))
else:
print('d: {}'.format(i['name']))
return True
def find(self, ep=None, path=None):
# FIXME: cwd interpoloate
ep = ep if ep else self._cep
path = path if path else self._cwd
path = '/' if path is None else path
for ep, dirname, node in self._gsm.fts('{}:/{}'.format(ep, path)):
if node['DATA_TYPE'] == 'file':
t, basename = 'f', node['name']
else:
t, basename = 'd', node['path']
print('{}: {}:{}/{}'.format(t, ep, dirname, basename))
def mv(self, ep1, path1, ep2, path2):
ep1 = ep1 if ep1 else self._cep
ep2 = ep2 if ep2 else self._cep
if ep1 != ep2:
return False
path1 = (path1 if path1.startswith('/') else '{}/{}'.format(
self._cwd, path1))
path2 = (path2 if path2.startswith('/') else '{}/{}'.format(
self._cwd, path2))
return self._gsm.rename('{}:/{}'.format(ep1, path1),
'{}:/{}'.format(ep2, path2))
def rm(self, ep, path):
ep = ep if ep else self._cep
path = (path if path.startswith('/') else '{}/{}'.format(
self._cwd, path))
return self._gsm.rmdir('{}:/{}'.format(ep, path))
def rm(self, ep, path):
ep = ep if ep else self._cep
path = (path if path.startswith('/') else '{}/{}'.format(
self._cwd, path))
return self._gsm.rm('{}:/{}'.format(ep, path))
def rm_r(self, ep, path):
# XXX: drop when shell gets 'args' support & add -r flag
ep = ep if ep else self._cep
path = (path if path.startswith('/') else '{}/{}'.format(
self._cwd, path))
return self._gsm.rm('{}:/{}'.format(ep, path), recursive=True)
def cp(self, ep1, path1, ep2, path2, recursive=False):
# XXX: delete_destination_xtra for mirroring?
# XXX: recursive -> make flag - false required for file,
# true reqd? (or at least accepted) for dirs
print('cp', ep1, path1, ep2, path2, recursive)
ep1 = ep1 if ep1 else self._cep
ep2 = ep2 if ep2 else self._cep
path1 = (path1 if path1.startswith('/') else '{}/{}'.format(
self._cwd, path1))
path2 = (path2 if path2.startswith('/') else '{}/{}'.format(
self._cwd, path2))
return self._gsm.cp('{}:/{}'.format(ep1, path1),
'{}:/{}'.format(ep2, path2), recursive)
def mkdir(self):
raise NotImplementedError('mkdef() plox')
def sh(self, prompt='globus% '):
cmds = set(
('env', 'pwd', 'cd', 'ls', 'find', 'cp', 'mv', 'rm', 'rm_r'))
while True:
try:
data = input(prompt).split(' ')
except EOFError:
return
cmd, args = data[0], data[1:]
print('cmd input: {} -> ({}, {}'.format(data, cmd, args))
if not cmd or cmd.startswith('#'):
continue
elif cmd in cmds:
try:
# TODO: need 'flags' handling so e.g. 'cp -r' set recursive
args = (i.split(':') if ':' in i else (None, i)
for i in args)
ret = getattr(self, cmd)((*[j for i in args for j in i]))
if ret not in {True, None}:
print(ret)
except Exception as e:
print(repr(e))
else:
print('commands: {}'.format(cmds))
