def send_data ( self ):
""" runs the SEND_DATA operation """
# calculate dir size
mycommandline = '%s %s %s' % (
self.__CONFIG['system']['disk usage command full path'][1],
self.__CONFIG['system']['disk usage command arguments'][1],
self.myDirectoryName )
args = shlex.split ( mycommandline )
logging.info('Calculating size of %s (%s)' % (self.myDirectoryName, mycommandline) )
proc = subprocess.Popen ( args, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
(stdoutdata, stderrdata) = proc.communicate()
if proc.returncode != 0 :
raise Exception('External process returned non 0 value : %s' % mycommandline)
sizeGb = float ( stdoutdata.split()[0] ) / 1024**3
logging.info('Directory size is %.2fGb' % sizeGb)
# check limits (logbook)
if ( self.myLogbook and not self.ignoreLimits):
logging.info('Checking size limits : must be under %.2fGb per week.' % self.__CONFIG['limits']['weekly max size'][1])
totalsize = self.logbook_compute_size(offset=7)
logging.info('Size of data transferred since one week: %.2fGb' % totalsize )
if (totalsize + sizeGb > self.__CONFIG['limits']['weekly max size'][1]):
raise Exception('Transfer size exceeds limit (see log file for details)')
# build XML object, write XML file
logging.info ('Building XML object')
tree = Transaction()
self.myTransactionID = tree.get_transaction_id()
logging.info ( 'Transaction ID is %s' % self.myTransactionID )
tree.set_status('0')
tree.set_resif_node(self.__CONFIG['my resif node']['my node name'][1])
tree.set_data_type(self.myDataType)
tree.set_client_size( '%.4f' % sizeGb )
tree.set_comment('data sent to datacentre by client-side application, waiting for processing.')
xmlfile = os.path.join ( self.__CONFIG['system']['working directory'][1], self.myTransactionID + '.xml' )
logging.info ('Writing XML in %s' % xmlfile)
tree.write(xmlfile)
# send data + XML file
myRsync = Rsync (
server = self.__CONFIG['rsync']['rsync server'][1],
module = 'INCOMING_' + self.__CONFIG['my resif node']['my node name'][1],
port = self.__CONFIG['rsync']['rsync port'][1],
timeout = self.__CONFIG['rsync']['rsync timeout'][1],
login = self.__CONFIG['my resif node']['my node name'][1].lower(),
password = self.__CONFIG['my resif node']['my node password'][1],
compress = self.__CONFIG['rsync']['rsync compress'][1],
dryrun = self.myTestOnly,
command = self.__CONFIG['rsync']['rsync command full path'][1],
bwlimit = None if self.ignoreLimits else self.__CONFIG['limits']['bandwidth max'][1],
extraargs = self.__CONFIG['rsync']['rsync extra args'][1]
)
logging.info ('Calling rsync to transfer %s and %s' % ( self.myDirectoryName, xmlfile))
(stdoutdata,stderrdata) = myRsync.push ( source = self.myDirectoryName + ' ' + xmlfile, destination = self.myTransactionID )
# print transaction identifier on stdout
sys.stdout.write ( self.myTransactionID + '\n' )
logging.debug('rsync stderr follows: %s' % stderrdata)
os.remove(xmlfile)
# update logbook
self.myLogbook.append ( { 'date': time.strftime ( self.__DATE_FORMAT,time.gmtime() ),
'node': self.__CONFIG['my resif node']['my node name'][1],
'directory':self.myDirectoryName,
'transactionID':self.myTransactionID,
'size':sizeGb,
'datatype':self.myDataType })
if not self.myTestOnly:
logging.info ('Updating transfer loogbook')
with open ( self.__CONFIG['logging']['logbook'][1], 'w' ) as f: json.dump(self.myLogbook,f,indent=2)
# on success, returns 0
return 0
