Beispiel #1
0
 def clearFile(self, srcDir, filename):
     try:
         os.remove(srcDir + "/" + filename)
         return 1
     except:
         print MNIUtil.timeLog() + "ERROR: Error clearing " + filename
         return 0
Beispiel #2
0
    def pushFile(self, src, dst, host, user, key=""):
        finalDst = src.split('/')[-1]
        if[dst != ""]:
            finalDst = dst+"/"+finalDst

        try:
            c = "scp "

            if(not os.path.isfile(key)):
                keyfile = os.path.expanduser('~') + "/.ssh/id_dsa"

            c += "-i "+keyfile+" "
            c += src+" "+user+"@"+host+":"+finalDst

            ret = os.system(c)
            if(not ret == 0):
                raise Exception("scp returned "+str(ret))

            return 1
        except Exception, e:
            print MNIUtil.timeLog() + "ERROR: Could not push file " + src + \
                " to host " + host + " - ", e
            print MNIUtil.timeLog() + "INFO: Failed command was " + c
            sys.stdout.flush()
            return 0
Beispiel #3
0
    def pushFile(self, src, dst, host, user, key=""):
        finalDst = src.split('/')[-1]
        if[dst != ""]:
            finalDst = dst+"/"+finalDst

        try:
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect((host, 22))
            trans = paramiko.Transport(sock)
            trans.start_client()

            if(isinstance(key, paramiko.PKey)):
                key = key
            elif((isinstance(key, str)) and key != ""):
                # Password Auth
                trans.auth_password(user, key)
            else:
                # Key Auth
                p = os.path.expanduser('~') + "/.ssh/id_dsa"
                key = paramiko.DSSKey.from_private_key_file(p)
                trans.auth_publickey(user, key)

                chan = trans.open_session()
                chan.get_pty()
                chan.setblocking(120)
                f = file(src, 'rb')
                chan.exec_command('scp -q -t '+finalDst)

                # Send data
                dl = int(os.stat(src)[6])
                head = 'C' + oct(os.stat(src).st_mode)[-4:] +\
                        ' ' + str(dl) + \
                        ' ' + src.split('/')[-1] + '\n'
                data = f.read()
                chan.send(head)
                chan.sendall(data)

                dl += len(head) + 3 + data.count('\n')
                rl = 0

                # Seek to the end of stdin
                while(rl < dl):
                    msg = chan.recv(1)
                    rl += len(msg)

                # Check for success code
                if(msg[-1] == '\x00'):
                    print MNIUtil.timeLog() + "WARN: No success response on "+\
                        src + " to host " + host
                    #raise Exception("Invalid response (non NULL over scp)")

                f.close()
                chan.close();
                trans.close();
                sock.close();
                return 1
        except Exception, e:
                print MNIUtil.timeLog() + "ERROR: Could not push file " + \
                    src + " to host " + host + " - ", e
                return 0
Beispiel #4
0
def getHosts(trans_home, ):
    hf = cfg.cfgGet('hosts.file')
    if hf[0] == '/':
        hf = trans_home+hf

    hosts = parseHosts(hf)
    for h in hosts:
        if(not archiver.testArchiving(h[0])):
            print MNIUtil.timeLog()+"FATAL: Error archiving to " +\
                archiver.dst+"/"+h[0]
            sys.exit(2)
        print MNIUtil.timeLog()+"HOST:    "+h[0]+" - "+h[2]+"@"+\
            h[1]+":"+h[4]
    return hosts
Beispiel #5
0
    def archiveFile(self, srcDir, filename, subdir=""):
        d = self.dst
        if(subdir != ""):
            d += "/"+subdir

        try:
            shutil.copy2(srcDir+"/"+filename, d+"/"+filename)
            return 1
        except:
            print MNIUtil.timeLog() + "ERROR: Error archiving " + filename

            if(testArchiving(subdir)):
                return archiveFile(srcDir, filename, subdir)
            else:
                print MNIUtil.timeLog() + "ERROR: Attempt to fix " + \
                    "archiving has failed. Please diagnose and restart."
                return 0
Beispiel #6
0
def pushFile(xml_dir, file, host):
    use_pass = (cfg.cfgGet('push.usepass').lower().strip() in
                ['true', '1', 't', 'y', 'yes'])
    pwd = host[3]
    if(not use_pass):
        pwd = ""

    # Push
    if(pusher.pushFile(xml_dir+"/"+file, host[4], host[1], host[2], pwd)):
        # Archive
        archiver.archiveFile(xml_dir, file)
        print MNIUtil.timeLog()+"PUSHED     "+file+" to host " + host[0]
        sys.stdout.flush()
        return 1
    else:
        print MNIUtil.timeLog()+"FAILED PUSH " + file + \
            " to host " + host[0] + ". Retrying in " + \
            cfg.cfgGet("push.faildelay", 5) + " seconds"
        sys.stdout.flush()
        return 0
Beispiel #7
0
    def pushFile(self, src, dst, host, user, key=""):
        finalDst = src.split('/')[-1]
        if[dst != ""]:
            finalDst = dst+"/"+finalDst

        try:
            ssh = paramiko.SSHClient()
            ssh.load_system_host_keys()
            ssh.connect(host, username=user, password=key)
            ftp = ssh.open_sftp()
            ftp.put(src, finalDst)
            ftp.close()
            ssh.close()
            return 1
        except Exception, e:
            print MNIUtil.timeLog()+"ERROR: Could not push file " + src + \
                " to host " +host + " - ", e
            #if ftp != None:
            #        ftp.close()
            #if ssh != None:
            #        ssh.close()
            return 0
Beispiel #8
0
def main(argv):
    if(len(argv) < 2):
        print "Usage: pushfiles.py <config file>"
        sys.exit(1)

    # Parse cfg files
    global cfg, archiver, pusher
    cfg = MNICfg(argv[1], "pushfiles.py")

    # Grab and cleanup directories
    trans_home = cfg.cfgGet('module.home')
    xml_dir    = cfg.cfgGet('data.dir')
    arc_dir    = cfg.cfgGet('archive.dir')

    if (xml_dir.split('/')[-1] != ''):
        xml_dir = xml_dir+'/'
    if (arc_dir.split('/')[-1] != ''):
        arc_dir = arc_dir+'/'
    if (trans_home.split('/')[-1] != ''):
        trans_home = trans_home+'/'

    paramiko.util.log_to_file(trans_home+'/paramiko_debug.log')
    r = re.compile("^n[0-9]+_[a-z]+_[a-z0-9]+.xml$", re.IGNORECASE)
    archiver = MNIArchiver(arc_dir)
    pusher = getPusher()

    # Version and startup logging
    print MNIUtil.timeLog()+"mnnewsmlfeed - " + \
        "V: $Id: pushfiles.py 2712 2011-09-29 18:27:12Z jcerda $"
    print MNIUtil.timeLog()+"HOME       "+trans_home
    print MNIUtil.timeLog()+"DATA       "+xml_dir
    print MNIUtil.timeLog()+"ARCHIVE    "+arc_dir
    print MNIUtil.timeLog()+"PUSHER     "+cfg.cfgGet('push.method').strip()
    sys.stdout.flush()

    hosts = getHosts(trans_home)

    # Main loop
    while (True):
        # Grab file list
        l = os.listdir(xml_dir)
        l.sort()
        mylen = len(l)

        if(mylen > 0):
            if(mylen > 1500):
                l = l[:1500]

            # Filter out non-xml and template
            l = [x.strip() for x in l]
            l = filter(r.search,l)

            # Push and move files to archive
            # If push fails, break. Order matters.
            for f in l:
                print MNIUtil.timeLog()+"STARTING   "+f

                for h in hosts:
                    if (len(h) < 4):
                        print MNIUtil.timeLog()+"WARN: One invalid host " +\
                            "line. Skipping"
                        continue

                    while(not pushFile(xml_dir, f, h)):
                        # Sleep a little so we don't kill servers
                        time.sleep(float(cfg.cfgGet("push.faildelay", 5)))

                # Move file
                if(not archiver.clearFile(xml_dir, f)):
                    print MNIUtil.timeLog()+"FATAL: Fatal error. " +\
                        "Could not move " + f + ". Exiting to avoid " +\
                        "double publishing."
                    sys.exit(3)

                print MNIUtil.timeLog()+"FINISHED   "+f
                sys.stdout.flush()

                # Sleep a little so we don't kill server
                time.sleep(float(cfg.cfgGet("push.delay", 0.5)))
Beispiel #9
0
def getPusher():
    pm = cfg.cfgGet('push.method').strip()
    if(not issubclass(eval(pm), MNIPush)):
        print MNIUtil.timeLog()+"FATAL: Invalid push method - " + pm
        sys.exit(4)
    return eval(pm+"()")