def retrieve_logs ( self ):
""" runs the RETRIEVE_LOGS operation"""
# get remote XML file and print it on stdout,
myRsync = Rsync (
server = self.__CONFIG['rsync']['rsync server'][1],
module = 'TRANSACTION_XML',
port = self.__CONFIG['rsync']['rsync port'][1],
timeout = self.__CONFIG['rsync']['rsync timeout'][1],
login = self.__CONFIG['my resif node']['my node name'][1].lower(),
password = self.__CONFIG['my resif node']['my node password'][1],
dryrun = self.myTestOnly,
command = self.__CONFIG['rsync']['rsync command full path'][1],
extraargs = self.__CONFIG['rsync']['rsync extra args'][1]
)
tempxml= os.path.join ( self.__CONFIG['system']['working directory'][1], self.myTransactionID + '.xml.tmp' )
logging.info ("Getting XML file from rsync server for transaction %s" % self.myTransactionID)
try:
myRsync.pull ( remote = '%s.xml' % self.myTransactionID, local = tempxml )
except:
msg='Could not retrieve XML file for this transaction.'
logging.error ( msg )
sys.stderr.write ( msg + '\n' )
return 1
# cat temp file on stdout
with open(tempxml,'r') as f:
data = f.read()
sys.stdout.write(data)
# remove temp file
os.remove(tempxml)
return 0
class Transfer(Rsync, SFtp, Copy):
def __init__(self, method):
self.methodDict = ["rsync", "copy", "sftp"]
self.setServerFlag = 0
self.method = method
if self.method not in self.methodDict:
raise TransferError(method + " type of transfer is not " + "implemented")
def setServerParams(self, server, uname, passwd):
if self.method == "rsync":
self.rsyncObj = Rsync(server, uname, passwd)
elif self.method == "sftp":
self.sftpObj = SFtp(server, uname, passwd)
def copy(self, src, dest, toServer=None):
self.copyObj = Copy()
if self.method == "rsync":
if not toServer == None:
self.rsyncObj.run(src, dest, toServer)
else:
raise TransferError("Did not specify transfer " + "direction")
return
if os.path.isdir(src):
if self.method == "sftp":
raise TransferError("Cannot transfer directories " + "using sftp")
else:
self.copyObj.copytree(src, dest)
else:
if self.method == "sftp":
if os.path.isdir(dest):
raise TransferError("Cannot transfer to dir " + "using sftp")
self.sftpObj.copy_file(src, dest)
else:
self.copyObj.copy(src, dest)
def run(argv):
parser = argparse.ArgumentParser(
description='Add a more elaborate description here, later.'
)
parser.add_argument('-v', '--verbose', action='store_true',
help='throw a racket about what we are doing')
parser.add_argument('-V', '--version', action='version',
version='%(prog)s 0.1-alpha')
args = parser.parse_args(argv)
conf = Config()
if args.verbose:
conf.verbose = True
utils = Utils()
if not utils.exists_rsync():
print "Could not find rsync -- is it installed?"
sys.exit(1)
if not utils.check_version((3,0,9)):
print "Installed rsync is too old. Please upgrade!"
sys.exit(1)
for server in conf.servers:
rsync = Rsync(server, conf)
rsync.run()
def pull_report(report_source):
l.debug("Pulling remote report '%s'" % report_source)
tmpdir = tempfile.mkdtemp(prefix="b1000-remote-report-")
try:
rsync = Rsync("remote_report", report_source, tmpdir, "")
rsync.run()
reports = os.listdir(tmpdir)
report = ConfigParser.RawConfigParser()
report.readfp(open(tmpdir + "/" + reports[0]))
except Exception, e:
raise NoReportException("Could not read report file '%s'. Exception: %s" % (report_source, str(e)))
def copy(self):
excludes = self.job.exclude + " " + self.exclude
l.debug("Copying '%s' to '%s' on destination '%s' excluding: '%s'" % (self.job.include, self.path, self.name, excludes))
self.rsync = Rsync(self.name, self.job.include, self.path, excludes)
self.rsync.set_verbosity(self.verbosity)
self.retries -= 1
self.rsync.run()
def test_get_rsync_dir():
rsync = Rsync(host='127.0.0.1', user='******', local_dir='kkk',
remote_dir='zzz')
rsync.add_ssh_port(222)
rsync.add_ssh_key('my_ssh_key.key')
rsync.add_exclude_file('*.pyc')
rsync.add_exclude_file('*.swp')
assert rsync.get_cmd() == """rsync -e 'ssh -p 222 -i my_ssh_key.key' kkk [email protected]:zzz --exclude="*.pyc" --exclude="*.swp" -rlvx --delete"""
def __notify_remote(self, status):
l.debug("Sending notification '%s' to remote '%s'" % (status, self.report_source))
tmpdir = tempfile.mkdtemp(prefix="b1000-remote-notify-")
status_file = tmpdir + "/" + self.remote_report_file + "." + platform.node() + "." + status
# prepare message file
open(status_file, "w").close()
# rsync messge file
try:
rsync = Rsync("remote_notify", status_file, self.report_source, "")
rsync.run()
except:
raise
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
class DestRsync(Dest):
params_required = ['type', 'path']
params_allowed = ['verbosity', 'exclude']
# --------------------------------------------------------------------
def __init__(self, job, cfg, name, bg):
l.debug("Adding rsync destination '%s' (background: %s) to job '%s'" % (name, str(bg), job.real_name))
Dest.__init__(self, job, cfg, name, bg)
self.exclude = cfg.get_def(self.sname, "exclude", "")
self.verbosity = int(cfg.get_def(self.sname, "verbosity", "1"))
self.path = self.__prepare_path()
self.retries = int(cfg.get_def("global", "copy_retries", '3'))
self.retry_sleep = int(cfg.get_def("global", "copy_retries", '60'))
# --------------------------------------------------------------------
def __prepare_path(self):
path = self.cfg.get(self.sname, "path")
if not path.endswith("/"):
path += "/"
path += self.job.get_job_path()
return path
# --------------------------------------------------------------------
def copy(self):
excludes = self.job.exclude + " " + self.exclude
l.debug("Copying '%s' to '%s' on destination '%s' excluding: '%s'" % (self.job.include, self.path, self.name, excludes))
self.rsync = Rsync(self.name, self.job.include, self.path, excludes)
self.rsync.set_verbosity(self.verbosity)
self.retries -= 1
self.rsync.run()
def setServerParams(self, server, uname, passwd):
if self.method == "rsync":
self.rsyncObj = Rsync(server, uname, passwd)
elif self.method == "sftp":
self.sftpObj = SFtp(server, uname, passwd)
def build_copy(self, fname, folder, key):
if self.ssh.mode != 0:
copy = Rsync()
elif self.ssh.mode != 1:
copy = ToSsh()
elif self.ssh.mode != 2:
copy = FromSsh()
#copy.set_logger(logging)
copy.set_filename(fname)
copy.set_ssh(self.ssh)
copy.set_source(folder)
copy.set_target(self.target)
copy.set_link("{}/{}".format(self.link, key))
copy.set_rsyncconf(self.rsyncconf)
return copy
def send_data ( self ):
""" runs the SEND_DATA operation """
# calculate dir size
mycommandline = '%s %s %s' % (
self.__CONFIG['system']['disk usage command full path'][1],
self.__CONFIG['system']['disk usage command arguments'][1],
self.myDirectoryName )
args = shlex.split ( mycommandline )
logging.info('Calculating size of %s (%s)' % (self.myDirectoryName, mycommandline) )
proc = subprocess.Popen ( args, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
(stdoutdata, stderrdata) = proc.communicate()
if proc.returncode != 0 :
raise Exception('External process returned non 0 value : %s' % mycommandline)
sizeGb = float ( stdoutdata.split()[0] ) / 1024**3
logging.info('Directory size is %.2fGb' % sizeGb)
# check limits (logbook)
if ( self.myLogbook and not self.ignoreLimits):
logging.info('Checking size limits : must be under %.2fGb per week.' % self.__CONFIG['limits']['weekly max size'][1])
totalsize = self.logbook_compute_size(offset=7)
logging.info('Size of data transferred since one week: %.2fGb' % totalsize )
if (totalsize + sizeGb > self.__CONFIG['limits']['weekly max size'][1]):
raise Exception('Transfer size exceeds limit (see log file for details)')
# build XML object, write XML file
logging.info ('Building XML object')
tree = Transaction()
self.myTransactionID = tree.get_transaction_id()
logging.info ( 'Transaction ID is %s' % self.myTransactionID )
tree.set_status('0')
tree.set_resif_node(self.__CONFIG['my resif node']['my node name'][1])
tree.set_data_type(self.myDataType)
tree.set_client_size( '%.4f' % sizeGb )
tree.set_comment('data sent to datacentre by client-side application, waiting for processing.')
xmlfile = os.path.join ( self.__CONFIG['system']['working directory'][1], self.myTransactionID + '.xml' )
logging.info ('Writing XML in %s' % xmlfile)
tree.write(xmlfile)
# send data + XML file
myRsync = Rsync (
server = self.__CONFIG['rsync']['rsync server'][1],
module = 'INCOMING_' + self.__CONFIG['my resif node']['my node name'][1],
port = self.__CONFIG['rsync']['rsync port'][1],
timeout = self.__CONFIG['rsync']['rsync timeout'][1],
login = self.__CONFIG['my resif node']['my node name'][1].lower(),
password = self.__CONFIG['my resif node']['my node password'][1],
compress = self.__CONFIG['rsync']['rsync compress'][1],
dryrun = self.myTestOnly,
command = self.__CONFIG['rsync']['rsync command full path'][1],
bwlimit = None if self.ignoreLimits else self.__CONFIG['limits']['bandwidth max'][1],
extraargs = self.__CONFIG['rsync']['rsync extra args'][1]
)
logging.info ('Calling rsync to transfer %s and %s' % ( self.myDirectoryName, xmlfile))
(stdoutdata,stderrdata) = myRsync.push ( source = self.myDirectoryName + ' ' + xmlfile, destination = self.myTransactionID )
# print transaction identifier on stdout
sys.stdout.write ( self.myTransactionID + '\n' )
logging.debug('rsync stderr follows: %s' % stderrdata)
os.remove(xmlfile)
# update logbook
self.myLogbook.append ( { 'date': time.strftime ( self.__DATE_FORMAT,time.gmtime() ),
'node': self.__CONFIG['my resif node']['my node name'][1],
'directory':self.myDirectoryName,
'transactionID':self.myTransactionID,
'size':sizeGb,
'datatype':self.myDataType })
if not self.myTestOnly:
logging.info ('Updating transfer loogbook')
with open ( self.__CONFIG['logging']['logbook'][1], 'w' ) as f: json.dump(self.myLogbook,f,indent=2)
# on success, returns 0
return 0