def link(self, lfn, dir):
lfn = ddm_getlocalabspath(lfn)
dir = ddm_getlocalabspath(dir)
_logger.debug('LOCAL: Try to link file from %s to %s' % (lfn, dir))
try:
proc = subprocess.Popen(['/bin/bash'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
out = proc.communicate("mkdir -p %s && ln -s %s %s" % (dir, lfn, dir))
except:
_logger.error('Unable to link:%s to %s' % (lfn, dir))
def link(self, lfn, dir):
lfn = ddm_getlocalabspath(lfn)
dir = ddm_getlocalabspath(dir)
_logger.debug('LOCAL: Try to link file from %s to %s' % (lfn, dir))
try:
proc = subprocess.Popen(['/bin/bash'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
out = proc.communicate("mkdir -p %s && ln -s %s %s" %
(dir, lfn, dir))
except:
_logger.error('Unable to link:%s to %s' % (lfn, dir))
def get(self, src, dest):
if not ddm_localisdir(dest):
ddm_localmakedirs(dest)
dest = ddm_getlocalabspath(dest)
src = ddm_getlocalabspath(src)
_logger.debug('LOCAL: Try to copy file from %s to %s' % (src, dest))
try:
if not os.path.isfile(src):
_logger.error("%s: File not found" % src)
shutil.copy2(src, dest)
except:
_logger.error('Unable to copy:%s %s' % (src, dest))
def get(self, src, dest):
if not ddm_localisdir(dest):
ddm_localmakedirs(dest)
dest = ddm_getlocalabspath(dest)
src = ddm_getlocalabspath(src)
_logger.debug('LOCAL: Try to copy file from %s to %s' % (src, dest))
try:
if not os.path.isfile(src):
_logger.error("%s: File not found" % src)
shutil.copy2(src, dest)
except:
_logger.error('Unable to copy:%s %s' % (src, dest))
def get(self, url, dest):
dest = ddm_getlocalabspath(dest)
_logger.debug('HTTP: Try to get file from %s to %s' % (url, dest))
try:
import urllib2
file_name = url.split('/')[-1]
u = urllib2.urlopen(url)
f = open(os.path.join(dest, file_name), 'wb')
meta = u.info()
file_size = int(meta.getheaders("Content-Length")[0])
print "Downloading: %s Bytes: %s" % (file_name, file_size)
file_size_dl = 0
block_sz = 8192
while True:
buffer = u.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
status = status + chr(8)*(len(status)+1)
print status,
f.close()
except:
_logger.error('Unable to download:%s to %s' % (url, dest))
def jobLog(id):
"""
Returns job stdout & stderr
:param id: Job id
:return: json data
"""
job = jobs_.get(id=id)
extractLog(id)
locdir = '/%s/.sys/%s' % (getScope(job.owner.username), job.container.guid)
absdir = ddm_getlocalabspath(locdir)
fout = find('payload_stdout.txt', absdir)
ferr = find('payload_stderr.txt', absdir)
out = ''
err = ''
if len(fout) > 0:
with open(fout[0]) as f:
out = f.read()
if len(ferr) > 0:
with open(ferr[0]) as f:
err = f.read()
data = {}
data['id'] = id
data['out'] = out
data['err'] = err
return make_response(jsonify({'data': data}), 200)
def put(self, src, dest):
src = ddm_getlocalabspath(src)
_logger.debug('DROPBOX: Try to put file from %s to %s' % (src, dest))
# put file from local se to dropbox
f = open(src, 'rb')
response = self.client.put_file(dest, f)
_logger.debug('uploaded: ', response)
def get(self, url, dest):
dest = ddm_getlocalabspath(dest)
_logger.debug('HTTP: Try to get file from %s to %s' % (url, dest))
try:
import urllib2
file_name = url.split('/')[-1]
u = urllib2.urlopen(url)
f = open(os.path.join(dest, file_name), 'wb')
meta = u.info()
file_size = int(meta.getheaders("Content-Length")[0])
print "Downloading: %s Bytes: %s" % (file_name, file_size)
file_size_dl = 0
block_sz = 8192
while True:
buffer = u.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
status = r"%10d [%3.2f%%]" % (file_size_dl,
file_size_dl * 100. / file_size)
status = status + chr(8) * (len(status) + 1)
print status,
f.close()
except:
_logger.error('Unable to download:%s to %s' % (url, dest))
def put(self, src, dest):
src = ddm_getlocalabspath(src)
dest = self.datadir + dest
_logger.debug('HPC: Try to put file from %s to %s' % (src, dest))
try:
proc = subprocess.Popen(['/bin/bash'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
out = proc.communicate("ssh -i %s %s@%s 'mkdir -p %s'" % (self.key, self.user, self.host, dest))
proc = subprocess.Popen(['/bin/bash'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
out = proc.communicate("rsync --bwlimit=2500 -av -e 'ssh -i %s' %s %s@%s:%s/" % (self.key, src, self.user, self.host, dest))
except:
_logger.error('Unable to upload:%s to %s' % (src, dest))
def get(self, src, dest):
src = self.datadir + src
dest = ddm_getlocalabspath(dest)
_logger.debug('HPC: Try to get file from %s to %s' % (src, dest))
try:
if not src.startswith('/'):
src = '/' + src
proc = subprocess.Popen(['/bin/bash'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
out = proc.communicate("rsync --bwlimit=2500 -av -e 'ssh -i %s' %s@%s:%s %s/" % (self.key, self.user, self.host, src, dest))
except:
_logger.error('Unable to download:%s to %s' % (src, dest))
def get(self, src, dest):
dest = ddm_getlocalabspath(dest)
_logger.debug('DROPBOX: Try to get file from %s to %s' % (src, dest))
fname = src.split('/')[-1]
# get file from dropbox to local se
out = open(os.path.join(dest, fname), 'wb')
f, metadata = self.client.get_file_and_metadata(src)
with f:
out.write(f.read())
_logger.debug('downloaded: ' + metadata)
def get(self, src, dest):
src = self.datadir + src
dest = ddm_getlocalabspath(dest)
_logger.debug('HPC: Try to get file from %s to %s' % (src, dest))
try:
if not src.startswith('/'):
src = '/' + src
proc = subprocess.Popen(['/bin/bash'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
out = proc.communicate(
"rsync --bwlimit=2500 -av -e 'ssh -i %s' %s@%s:%s %s/" %
(self.key, self.user, self.host, src, dest))
except:
_logger.error('Unable to download:%s to %s' % (src, dest))
def put(self, src, dest):
src = ddm_getlocalabspath(src)
_logger.debug('FTP: Try to put file from %s to %s' % (src, dest))
try:
dest = dest.split('ftp://')[-1]
host = dest.split('/')[0]
path = '/'.join(dest.split('/')[1:-1])
fname = src.split('/')[-1]
ftp = self.connect(host, self.login, self.password)
ftp.cwd(path)
f = open(src, "wb")
ftp.storbinary("STOR " + fname, f)
f.close()
except:
_logger.error('Unable to download:%s to %s' % (src, dest))
def put(self, src, dest):
src = ddm_getlocalabspath(src)
dest = self.datadir + dest
_logger.debug('HPC: Try to put file from %s to %s' % (src, dest))
try:
proc = subprocess.Popen(['/bin/bash'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
out = proc.communicate("ssh -i %s %s@%s 'mkdir -p %s'" %
(self.key, self.user, self.host, dest))
proc = subprocess.Popen(['/bin/bash'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
out = proc.communicate(
"rsync --bwlimit=2500 -av -e 'ssh -i %s' %s %s@%s:%s/" %
(self.key, src, self.user, self.host, dest))
except:
_logger.error('Unable to upload:%s to %s' % (src, dest))
def put(self, src, dest):
src = ddm_getlocalabspath(src)
_logger.debug('FTP: Try to put file from %s to %s' % (src, dest))
try:
dest = dest.split('ftp://')[-1]
host = dest.split('/')[0]
path = '/'.join(dest.split('/')[1:-1])
fname = src.split('/')[-1]
ftp = self.connect(host, self.login, self.password)
ftp.cwd(path)
f = open(src, "wb")
ftp.storbinary("STOR " + fname, f)
f.close()
except:
_logger.error('Unable to download:%s to %s' % (src, dest))
def get(self, url, dest):
dest = ddm_getlocalabspath(dest)
_logger.debug('FTP: Try to get file from %s to %s' % (url, dest))
try:
url = url.split('ftp://')[-1]
host = url.split('/')[0]
path = '/'.join(url.split('/')[1:-1])
fname = url.split('/')[-1]
destfile = os.path.join(dest, fname)
ftp = self.connect(host, self.login, self.password)
ftp.cwd(path)
f = open(destfile, "wb")
ftp.retrbinary("RETR " + fname, f.write, 8 * 1024)
f.close()
except:
_logger.error('Unable to download:%s to %s' % (url, dest))
def get(self, url, dest):
dest = ddm_getlocalabspath(dest)
_logger.debug('FTP: Try to get file from %s to %s' % (url, dest))
try:
url = url.split('ftp://')[-1]
host = url.split('/')[0]
path = '/'.join(url.split('/')[1:-1])
fname = url.split('/')[-1]
destfile = os.path.join(dest, fname)
ftp = self.connect(host, self.login, self.password)
ftp.cwd(path)
f = open(destfile, "wb")
ftp.retrbinary("RETR " + fname, f.write, 8*1024)
f.close()
except:
_logger.error('Unable to download:%s to %s' % (url, dest))
def jobLogAPI(id):
"""Returns job stdout & stderr"""
g.user = request.oauth.user
job = jobs_.get(id)
extractLog(id)
locdir = '/%s/.sys/%s' % (getScope(job.owner.username), job.container.guid)
absdir = ddm_getlocalabspath(locdir)
fout = find('payload.stdout', absdir)
ferr = find('payload.stderr', absdir)
out = ''
err = ''
if len(fout) > 0:
with open(fout[0]) as f:
out = f.read()
if len(ferr) > 0:
with open(ferr[0]) as f:
err = f.read()
data = dict()
data['id'] = id
data['out'] = out
data['err'] = err
return data
def jobLogAPI(id):
"""Returns job stdout & stderr"""
g.user = request.oauth.user
job = jobs_.get(id)
extractLog(id)
locdir = '/%s/.sys/%s' % (getScope(job.owner.username), job.container.guid)
absdir = ddm_getlocalabspath(locdir)
fout = find('payload.stdout', absdir)
ferr = find('payload.stderr', absdir)
out = ''
err = ''
if len(fout) > 0:
with open(fout[0]) as f:
out = f.read()
if len(ferr) > 0:
with open(ferr[0]) as f:
err = f.read()
data = dict()
data['id'] = id
data['out'] = out
data['err'] = err
return data