def retrieve_logs ( self ): """ runs the RETRIEVE_LOGS operation""" # get remote XML file and print it on stdout, myRsync = Rsync ( server = self.__CONFIG['rsync']['rsync server'][1], module = 'TRANSACTION_XML', port = self.__CONFIG['rsync']['rsync port'][1], timeout = self.__CONFIG['rsync']['rsync timeout'][1], login = self.__CONFIG['my resif node']['my node name'][1].lower(), password = self.__CONFIG['my resif node']['my node password'][1], dryrun = self.myTestOnly, command = self.__CONFIG['rsync']['rsync command full path'][1], extraargs = self.__CONFIG['rsync']['rsync extra args'][1] ) tempxml= os.path.join ( self.__CONFIG['system']['working directory'][1], self.myTransactionID + '.xml.tmp' ) logging.info ("Getting XML file from rsync server for transaction %s" % self.myTransactionID) try: myRsync.pull ( remote = '%s.xml' % self.myTransactionID, local = tempxml ) except: msg='Could not retrieve XML file for this transaction.' logging.error ( msg ) sys.stderr.write ( msg + '\n' ) return 1 # cat temp file on stdout with open(tempxml,'r') as f: data = f.read() sys.stdout.write(data) # remove temp file os.remove(tempxml) return 0
def build_copy(self, fname, folder, key): if self.ssh.mode != 0: copy = Rsync() elif self.ssh.mode != 1: copy = ToSsh() elif self.ssh.mode != 2: copy = FromSsh() #copy.set_logger(logging) copy.set_filename(fname) copy.set_ssh(self.ssh) copy.set_source(folder) copy.set_target(self.target) copy.set_link("{}/{}".format(self.link, key)) copy.set_rsyncconf(self.rsyncconf) return copy
def send_data ( self ): """ runs the SEND_DATA operation """ # calculate dir size mycommandline = '%s %s %s' % ( self.__CONFIG['system']['disk usage command full path'][1], self.__CONFIG['system']['disk usage command arguments'][1], self.myDirectoryName ) args = shlex.split ( mycommandline ) logging.info('Calculating size of %s (%s)' % (self.myDirectoryName, mycommandline) ) proc = subprocess.Popen ( args, stdout = subprocess.PIPE, stderr = subprocess.PIPE) (stdoutdata, stderrdata) = proc.communicate() if proc.returncode != 0 : raise Exception('External process returned non 0 value : %s' % mycommandline) sizeGb = float ( stdoutdata.split()[0] ) / 1024**3 logging.info('Directory size is %.2fGb' % sizeGb) # check limits (logbook) if ( self.myLogbook and not self.ignoreLimits): logging.info('Checking size limits : must be under %.2fGb per week.' % self.__CONFIG['limits']['weekly max size'][1]) totalsize = self.logbook_compute_size(offset=7) logging.info('Size of data transferred since one week: %.2fGb' % totalsize ) if (totalsize + sizeGb > self.__CONFIG['limits']['weekly max size'][1]): raise Exception('Transfer size exceeds limit (see log file for details)') # build XML object, write XML file logging.info ('Building XML object') tree = Transaction() self.myTransactionID = tree.get_transaction_id() logging.info ( 'Transaction ID is %s' % self.myTransactionID ) tree.set_status('0') tree.set_resif_node(self.__CONFIG['my resif node']['my node name'][1]) tree.set_data_type(self.myDataType) tree.set_client_size( '%.4f' % sizeGb ) tree.set_comment('data sent to datacentre by client-side application, waiting for processing.') xmlfile = os.path.join ( self.__CONFIG['system']['working directory'][1], self.myTransactionID + '.xml' ) logging.info ('Writing XML in %s' % xmlfile) tree.write(xmlfile) # send data + XML file myRsync = Rsync ( server = self.__CONFIG['rsync']['rsync server'][1], module = 'INCOMING_' + self.__CONFIG['my resif node']['my node name'][1], port = self.__CONFIG['rsync']['rsync port'][1], timeout = self.__CONFIG['rsync']['rsync timeout'][1], login = self.__CONFIG['my resif node']['my node name'][1].lower(), password = self.__CONFIG['my resif node']['my node password'][1], compress = self.__CONFIG['rsync']['rsync compress'][1], dryrun = self.myTestOnly, command = self.__CONFIG['rsync']['rsync command full path'][1], bwlimit = None if self.ignoreLimits else self.__CONFIG['limits']['bandwidth max'][1], extraargs = self.__CONFIG['rsync']['rsync extra args'][1] ) logging.info ('Calling rsync to transfer %s and %s' % ( self.myDirectoryName, xmlfile)) (stdoutdata,stderrdata) = myRsync.push ( source = self.myDirectoryName + ' ' + xmlfile, destination = self.myTransactionID ) # print transaction identifier on stdout sys.stdout.write ( self.myTransactionID + '\n' ) logging.debug('rsync stderr follows: %s' % stderrdata) os.remove(xmlfile) # update logbook self.myLogbook.append ( { 'date': time.strftime ( self.__DATE_FORMAT,time.gmtime() ), 'node': self.__CONFIG['my resif node']['my node name'][1], 'directory':self.myDirectoryName, 'transactionID':self.myTransactionID, 'size':sizeGb, 'datatype':self.myDataType }) if not self.myTestOnly: logging.info ('Updating transfer loogbook') with open ( self.__CONFIG['logging']['logbook'][1], 'w' ) as f: json.dump(self.myLogbook,f,indent=2) # on success, returns 0 return 0