def clearFile(self, srcDir, filename):
try:
os.remove(srcDir + "/" + filename)
return 1
except:
print MNIUtil.timeLog() + "ERROR: Error clearing " + filename
return 0
def pushFile(self, src, dst, host, user, key=""):
finalDst = src.split('/')[-1]
if[dst != ""]:
finalDst = dst+"/"+finalDst
try:
c = "scp "
if(not os.path.isfile(key)):
keyfile = os.path.expanduser('~') + "/.ssh/id_dsa"
c += "-i "+keyfile+" "
c += src+" "+user+"@"+host+":"+finalDst
ret = os.system(c)
if(not ret == 0):
raise Exception("scp returned "+str(ret))
return 1
except Exception, e:
print MNIUtil.timeLog() + "ERROR: Could not push file " + src + \
" to host " + host + " - ", e
print MNIUtil.timeLog() + "INFO: Failed command was " + c
sys.stdout.flush()
return 0
def pushFile(self, src, dst, host, user, key=""):
finalDst = src.split('/')[-1]
if[dst != ""]:
finalDst = dst+"/"+finalDst
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, 22))
trans = paramiko.Transport(sock)
trans.start_client()
if(isinstance(key, paramiko.PKey)):
key = key
elif((isinstance(key, str)) and key != ""):
# Password Auth
trans.auth_password(user, key)
else:
# Key Auth
p = os.path.expanduser('~') + "/.ssh/id_dsa"
key = paramiko.DSSKey.from_private_key_file(p)
trans.auth_publickey(user, key)
chan = trans.open_session()
chan.get_pty()
chan.setblocking(120)
f = file(src, 'rb')
chan.exec_command('scp -q -t '+finalDst)
# Send data
dl = int(os.stat(src)[6])
head = 'C' + oct(os.stat(src).st_mode)[-4:] +\
' ' + str(dl) + \
' ' + src.split('/')[-1] + '\n'
data = f.read()
chan.send(head)
chan.sendall(data)
dl += len(head) + 3 + data.count('\n')
rl = 0
# Seek to the end of stdin
while(rl < dl):
msg = chan.recv(1)
rl += len(msg)
# Check for success code
if(msg[-1] == '\x00'):
print MNIUtil.timeLog() + "WARN: No success response on "+\
src + " to host " + host
#raise Exception("Invalid response (non NULL over scp)")
f.close()
chan.close();
trans.close();
sock.close();
return 1
except Exception, e:
print MNIUtil.timeLog() + "ERROR: Could not push file " + \
src + " to host " + host + " - ", e
return 0
def getHosts(trans_home, ):
hf = cfg.cfgGet('hosts.file')
if hf[0] == '/':
hf = trans_home+hf
hosts = parseHosts(hf)
for h in hosts:
if(not archiver.testArchiving(h[0])):
print MNIUtil.timeLog()+"FATAL: Error archiving to " +\
archiver.dst+"/"+h[0]
sys.exit(2)
print MNIUtil.timeLog()+"HOST: "+h[0]+" - "+h[2]+"@"+\
h[1]+":"+h[4]
return hosts
def archiveFile(self, srcDir, filename, subdir=""):
d = self.dst
if(subdir != ""):
d += "/"+subdir
try:
shutil.copy2(srcDir+"/"+filename, d+"/"+filename)
return 1
except:
print MNIUtil.timeLog() + "ERROR: Error archiving " + filename
if(testArchiving(subdir)):
return archiveFile(srcDir, filename, subdir)
else:
print MNIUtil.timeLog() + "ERROR: Attempt to fix " + \
"archiving has failed. Please diagnose and restart."
return 0
def pushFile(xml_dir, file, host):
use_pass = (cfg.cfgGet('push.usepass').lower().strip() in
['true', '1', 't', 'y', 'yes'])
pwd = host[3]
if(not use_pass):
pwd = ""
# Push
if(pusher.pushFile(xml_dir+"/"+file, host[4], host[1], host[2], pwd)):
# Archive
archiver.archiveFile(xml_dir, file)
print MNIUtil.timeLog()+"PUSHED "+file+" to host " + host[0]
sys.stdout.flush()
return 1
else:
print MNIUtil.timeLog()+"FAILED PUSH " + file + \
" to host " + host[0] + ". Retrying in " + \
cfg.cfgGet("push.faildelay", 5) + " seconds"
sys.stdout.flush()
return 0
def pushFile(self, src, dst, host, user, key=""):
finalDst = src.split('/')[-1]
if[dst != ""]:
finalDst = dst+"/"+finalDst
try:
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect(host, username=user, password=key)
ftp = ssh.open_sftp()
ftp.put(src, finalDst)
ftp.close()
ssh.close()
return 1
except Exception, e:
print MNIUtil.timeLog()+"ERROR: Could not push file " + src + \
" to host " +host + " - ", e
#if ftp != None:
# ftp.close()
#if ssh != None:
# ssh.close()
return 0
def main(argv):
if(len(argv) < 2):
print "Usage: pushfiles.py <config file>"
sys.exit(1)
# Parse cfg files
global cfg, archiver, pusher
cfg = MNICfg(argv[1], "pushfiles.py")
# Grab and cleanup directories
trans_home = cfg.cfgGet('module.home')
xml_dir = cfg.cfgGet('data.dir')
arc_dir = cfg.cfgGet('archive.dir')
if (xml_dir.split('/')[-1] != ''):
xml_dir = xml_dir+'/'
if (arc_dir.split('/')[-1] != ''):
arc_dir = arc_dir+'/'
if (trans_home.split('/')[-1] != ''):
trans_home = trans_home+'/'
paramiko.util.log_to_file(trans_home+'/paramiko_debug.log')
r = re.compile("^n[0-9]+_[a-z]+_[a-z0-9]+.xml$", re.IGNORECASE)
archiver = MNIArchiver(arc_dir)
pusher = getPusher()
# Version and startup logging
print MNIUtil.timeLog()+"mnnewsmlfeed - " + \
"V: $Id: pushfiles.py 2712 2011-09-29 18:27:12Z jcerda $"
print MNIUtil.timeLog()+"HOME "+trans_home
print MNIUtil.timeLog()+"DATA "+xml_dir
print MNIUtil.timeLog()+"ARCHIVE "+arc_dir
print MNIUtil.timeLog()+"PUSHER "+cfg.cfgGet('push.method').strip()
sys.stdout.flush()
hosts = getHosts(trans_home)
# Main loop
while (True):
# Grab file list
l = os.listdir(xml_dir)
l.sort()
mylen = len(l)
if(mylen > 0):
if(mylen > 1500):
l = l[:1500]
# Filter out non-xml and template
l = [x.strip() for x in l]
l = filter(r.search,l)
# Push and move files to archive
# If push fails, break. Order matters.
for f in l:
print MNIUtil.timeLog()+"STARTING "+f
for h in hosts:
if (len(h) < 4):
print MNIUtil.timeLog()+"WARN: One invalid host " +\
"line. Skipping"
continue
while(not pushFile(xml_dir, f, h)):
# Sleep a little so we don't kill servers
time.sleep(float(cfg.cfgGet("push.faildelay", 5)))
# Move file
if(not archiver.clearFile(xml_dir, f)):
print MNIUtil.timeLog()+"FATAL: Fatal error. " +\
"Could not move " + f + ". Exiting to avoid " +\
"double publishing."
sys.exit(3)
print MNIUtil.timeLog()+"FINISHED "+f
sys.stdout.flush()
# Sleep a little so we don't kill server
time.sleep(float(cfg.cfgGet("push.delay", 0.5)))
def getPusher():
pm = cfg.cfgGet('push.method').strip()
if(not issubclass(eval(pm), MNIPush)):
print MNIUtil.timeLog()+"FATAL: Invalid push method - " + pm
sys.exit(4)
return eval(pm+"()")