def testprepend_datapath(self):
env = InstanceManager()
local_dp = 'local_dp'
tmp_global_dp = 'tmp_global_dp'
global_dp = 'global_dp'
env['slimvars']['localtmpdir'] = local_dp
env['slimvars']['globaltmpdir'] = tmp_global_dp
env['slimvars']['datapath'] = global_dp
itr = [
[(None, None), None],
[(False, None), None],
[(True, None), None],
[(None, True), None],
[(True, True), local_dp],
[(False, True), tmp_global_dp],
[(None, False), global_dp],
[(True, False), global_dp],
[(False, False), global_dp],
]
for (is_loc, is_tmp), ans in itr:
res = Unix_Pipes.prepend_datapath(is_loc, is_tmp, 'com')
if ans is None:
self.assertFalse("DATAPATH" in res, "expected no datapath")
else:
self.assertTrue(ans in res, "expected %(ans) got %(res)s")
def testCreateComand(self):
# (cmdlist, node_name, source, target,err ), ans
items = [
(
((['cmdlist'], ), {
'node_name': 'localhost',
'source': 'source',
'target': 'target',
'err': 'err'
}),
'< source cmdlist > target 2> err',
),
(
((['c1 r="p"', 'c2'], ), {
'node_name': 'foo',
'source': 'source',
'target': 'target',
'err': None
}),
'ssh foo "< source c1 r=\\"p\\" | c2 > target"',
),
(
((['c1'], ), {
'source': 'source',
}),
'< source c1',
),
]
for (pars, kw), ans in items:
res = Unix_Pipes.CreateComand(*pars, **kw)
self.assertEquals(res, ans)
def node_copy(self, node_name):
"""
copy from one node to another
"""
if self.is_global or self.isempty():
return
if node_name in self.nodenames:
return
tmddir = self.env['slimvars']['localtmpdir']
from_cmd = "%s form=xdr" % self.SFDD
to_cmd = "%s form=native" % self.SFDD
from_node = list(self.nodenames)[0]
from_data = self.get_data(from_node)
to_data = self.get_data(node_name)
is_tmp = self.istmp()
to_cmd = Unix_Pipes.CreateComand([to_cmd],
node_name=node_name,
target=to_data,
is_local=[True],
is_tmp=[is_tmp],
ssh_localhost=True)
frm_node = list(self.nodenames)[0]
self.add_node_name(node_name)
cmd = Unix_Pipes.CreateComand([from_cmd, to_cmd],
is_local=[None, None],
is_tmp=[is_tmp, is_tmp],
node_name=frm_node,
source=from_data,
ssh_localhost=False)
print >> self.env['record'](1, 'cmd'), cmd
assert frm_node != node_name, "can not copy from same node to same node"
p0 = Popen(cmd, shell=True, stderr=__PIPE__)
p0.wait()
print >> self.env['record'](2, 'cmd'), "finished:: cp", node_name
if p0.returncode:
out = p0.stderr.read()
raise IOError("copying from node to node retuned error %s\n\n%s" %
(p0.returncode, out))
def testreplace_quotes(self):
init = '"foo"'
i2 = '\\"foo\\"'
i3 = '\\\\\\"foo\\\\\\"'
res = init
for ans in [init, i2, i3]:
self.assertEquals(res, ans)
res = Unix_Pipes.replace_quotes(res)
def sync(synccmd):
sync_command = Unix_Pipes.CreateComand([synccmd], node)
print >> cmd_log1, sync_command
p0 = Popen(sync_command, shell=True, stderr=__PIPE__)
ret = p0.wait()
if ret:
print >> self.env[
'record'], 'error running %(sync_command)s ' % vars()
print >> self.env[
'record'], 'try running SLIMpy with "sync_disk_om_rm=False" ' % vars(
)
def GetShapeFwd3d( command ,space ):
nbd = command['nbd']
nbs = command['nbs']
ac = command['ac']
n1 = space.shape[0]
n2 = space.shape[1]
n3 = space.shape[2]
key = (nbd,nbs,ac)
if curvelets.has_key(key):
ck = curvelets[key]
mshape = tuple(space.shape)
if ck.has_key( mshape ):
return ck[ mshape ]
if not isfile( SFFDCT3() ):
raise EnvironmentError( "could not find file 'sffdct2' " )
if not isfile( SFMATH() ):
raise EnvironmentError( "could not find file 'sfmath' " )
sizes = join ( __env__['slimvars']['globaltmpdir'] , 'sizes3d.rsf' )
MATH = MATHCMD_3D.substitute( math=SFMATH(), n1=n1, n2=n2, n3=n3 )
cmd = MAPONLY_3D.substitute( SFFDCT=SFFDCT3(), nbd=nbd, nbs=nbs, ac=ac, sizes=sizes )
sizes_command = Unix_Pipes.CreateComand([MATH,cmd], 'localhost',
is_local=[None,False],
is_tmp=[None,True] )
print >> __env__['record']( 5, 'cmd','sizes' ) , sizes_command
p0 = Popen( sizes_command, shell=True, stderr=PIPE )
retcode = p0.wait()
if retcode:
out = p0.stderr.read()
raise Exception( "Command: '%(cmd)s' failed\n%(out)s" %vars() )
sizes_cont = rsf_data_container( sizes )
shape = sizes_cont.params.shape
sizes_cont.rm()
ck = curvelets.setdefault( key, {} )
ck[ tuple(space.shape) ] = shape
ck[ tuple(shape) ] = tuple(space.shape)
return shape
def getHeader(self):
"""
return open header, data must exist
"""
node, data = self._get_any_node()
if node == 'localhost':
return open(data)
else:
cat = Unix_Pipes.CreateComand(['cat'], node_name=node, source=data)
p0 = Popen(cat, shell=True, stdout=__PIPE__)
return p0.stdout
def pipe_format(self):
#
# clist= []
clist = self.format()
# If any target is non temp then send it to global storage
is_local = self.make_locals_list()
is_tmp = self.make_tmp_list()
if self.source is None:
clist.insert(0, 'true')
is_local.insert(0, None)
is_tmp.insert(0, None)
if self.is_muti_proc():
nodename = 'localhost'
else:
nodename = self.node_name
source = self.source
if source:
source = source.get_data(self.node_name)
target = self.target
if target:
target = target.get_data(self.node_name)
pipecommand = Unix_Pipes.CreateComand(
clist,
node_name=nodename,
source=source,
target=target,
is_local=is_local,
is_tmp=is_tmp,
)
all_scalars = SCALAR(pipecommand)
scalars_map = self.env['table'].scalars_map
for scal in all_scalars:
scalar_value = str(scalars_map[scal])
pipecommand = pipecommand.replace(scal, scalar_value)
return pipecommand
def diagnostic(self, nodename=None):
"""
run a check if this data is valid
"""
log = self.env['record'](1, 'cmd', 'err')
log2 = self.env['record'](2, 'diagnostic')
print >> log
print >> log, "Runnind diagnostic on data %s" % self
print >> log
acmd = self.ATTRMD.substitute(attr=self.SFATTR,
file=str(self),
want='all',
lval=2)
if nodename is None:
nodename = list(self.nodenames)
elif isinstance(nodename, str):
nodename = [nodename]
did_run = False
for node in nodename:
did_run = True
attr_command = Unix_Pipes.CreateComand([acmd], node)
print >> log2, attr_command
p0 = Popen(attr_command,
shell=True,
stderr=_STDOUT,
stdout=__PIPE__)
ret = p0.wait()
lines = p0.stdout.read()
print >> log2, lines
if ret:
print >> log, 'Error running Diagnostic on data "%(self)s"' % vars(
)
return False
else:
print >> log, 'File "%(self)s" is OK.' % vars()
return did_run
def teststdout_join(self):
res = Unix_Pipes.stdout_join('cmd', 'foo')
ans = "cmd 1> foo"
self.assertEqual(res, ans)
def teststdin_join(self):
res = Unix_Pipes.stdin_join('foo', "cmd")
ans = "< foo cmd"
self.assertEqual(res, ans)
def rm(self):
"""
removes the file on disc
"""
print >> self.env['record'](
10, 'cleaner'), "call to remove %s:" % self.data
print >> self.env['record'](
11, 'cleaner'), "\tistmp=%s , isfull=%s" % (self.istmp(),
self.isfull())
if not self.istmp():
return False
if self.isempty():
return False
err = 0
cmd_log1 = self.env['record'](1, 'cmd')
# print "rm called"
synccmd = self.env['slimvars']['SYNC_CMD']
do_sync = self.env['slimvars']['sync_disk_om_rm']
def sync(synccmd):
sync_command = Unix_Pipes.CreateComand([synccmd], node)
print >> cmd_log1, sync_command
p0 = Popen(sync_command, shell=True, stderr=__PIPE__)
ret = p0.wait()
if ret:
print >> self.env[
'record'], 'error running %(sync_command)s ' % vars()
print >> self.env[
'record'], 'try running SLIMpy with "sync_disk_om_rm=False" ' % vars(
)
for node in self.nodenames.copy():
data = self.get_data(node)
sfrm = self.RMCMD.substitute(rm=self.SFRM, file=data)
rm = self.RMCMD.substitute(rm='rm', file=data)
if do_sync:
sync(synccmd)
command = Unix_Pipes.CreateComand([sfrm], node)
print >> cmd_log1, command
p0 = Popen(command, shell=True, stderr=__PIPE__)
ret = p0.wait()
print >> self.env['record'](2), "finished::", node, 'rm'
if ret:
err += 1
msg = p0.stderr.read()
p0.stderr.close()
print >> self.env[
'record'], 'error %(ret)s on %(command)s: removeing header file:\n%(msg)s' % vars(
)
command = Unix_Pipes.CreateComand([rm], node)
print >> cmd_log1, command
p0 = Popen(command, shell=True, stderr=__PIPE__)
ret = p0.wait()
if ret:
msg = p0.stderr.read()
print >> cmd_log1, "could not 'sfrm' or rm 'data'\n%(msg)s" % vars(
)
else:
self.nodenames.remove(node)
else:
self.nodenames.remove(node)
p0.stderr.close()
return not err