示例#1
0
def simulate(fil, s3key_name, enable_delete, p_dst):
    pprint_kv("File to be uploaded", fil)
    pprint_kv(" Filename used on s3", s3key_name)

    if enable_delete:
        _c, f_is_writable = can_write(fil)
        pprint_kv(" File would be deleted", fil)
        if not f_is_writable:
            pprint_kv(" ! File can't be deleted", fil)
    else:
        if p_dst is not None:
            bname = os.path.basename(fil)
            fdst = os.path.join(p_dst, bname)
            _c, d_is_writable = can_write(p_dst)
            pprint_kv(" File would be moved to", fdst)
            if not d_is_writable:
                pprint_kv(" ! File can't be moved to", p_dst)
示例#2
0
def run(
    dst_path=None,
    topic_name=None,
    queue_name=None,
    mod_sub=None,
    mod_pub=None,
    polling_interval=None,
    force=False,
    node_id=None,
    force_delete=False,
    delete_queue=None,
    proto_n=None,
    proto_m=None,
    **_
):

    signal.signal(signal.SIGTERM, handlerSigTerm)

    ### STARTUP CHECKS
    ##################

    if proto_n > proto_m:
        raise Exception("Parameter 'n' must be smaller than 'm'")

    if os.path.isdir(dst_path):
        raise Exception("'dst_path' must be a filename, not a directory")

    if force_delete:
        logging.info("Attempting to delete '%s'" % dst_path)
        rm(dst_path)

    if os.path.isfile(dst_path):
        raise Exception("'dst_path' must not exists at startup: use -fd to delete")

    dir_path = os.path.dirname(dst_path)
    code, _msg = can_write(dir_path)
    if not code.startswith("ok"):
        raise Exception("directory '%s' is not writable" % dir_path)

    ### SETUP
    ###########################
    if node_id is None:
        node_id = str(uuid.uuid1())

    logging.info("Node id: %s" % node_id)

    proc = protocol_processor(node_id, proto_n, proto_m)

    ### START MAIN LOOP
    ######################################
    run_aws(node_id, proc, polling_interval, queue_name, topic_name, dst_path, delete_queue)
    """
示例#3
0
def run(source_path=None, move_path=None, check_path=None, 
        batch_size=5, 
        polling_interval=None, enable_delete=False
        ,**_):

    if check_path is not None:
        ct=check_transition()

    if enable_delete and move_path is not None:
        raise Exception("Options '-mp' and '-d' are mutually exclusive")
        
    code, rp=resolve_path(source_path)
    if not code.startswith("ok"):
        raise Exception("can't resolve source path '%s'" % source_path)
    source_path=rp
    
    if move_path is not None:
        code, rp=resolve_path(move_path)
        if not code.startswith("ok"):
            raise Exception("can't resolve 'move_path' '%s'" % move_path)
        move_path=rp

        logging.info("Creating (if necessary) 'move' path: %s" % move_path)
        code, msg=mkdir_p(move_path)
        if not code.startswith("ok"):
            raise Exception("Can't create move path '%s': %s" % (move_path, str(msg)))
            
        logging.info("Checking if 'move' directory is writable")
        code, msg=can_write(move_path)
        if not code.startswith("ok"):
            raise Exception("Can't write to 'move' directory")
            
    to_skip=[]
    logging.info("Process pid: %s" % os.getpid())
    ppid=os.getppid()
    logging.info("Parent pid : %s" % ppid)
    logging.info("Starting loop...")
    while True:
        if os.getppid()!=ppid:
            logging.warning("Parent terminated... exiting")
            break
        
        if check_path is not None:
            try:    exists=os.path.exists(check_path)
            except: exists=False
            
            maybe_tr, _=ct.send(exists)
            if maybe_tr=="tr" and exists:
                logging.info("Check path: passed")
            if maybe_tr=="tr" and not exists:
                logging.info("Check path: failed - skipping")
        else:
            ## fake 'exists'
            exists=True

        if exists:        
            code, files=get_root_files(source_path)
            if not code.startswith("ok"):
                logging.error("Can't get root files from %s" % source_path)
            else:                
                ###############################################################
                files=files[:batch_size]
                try:
                    for src_file in files:
                        
                        if src_file in to_skip:
                            continue
                        
                        code, _=can_write(src_file)
                        if not code.startswith("ok"):
                            to_skip.append(src_file)
                            logging.error("Would not be able to move/delete source file '%s'... skipping streaming" % src_file)
                            continue
        
                        dst_file=None                
                        if move_path is not None:
                            bn=os.path.basename(src_file)
                            dst_file=os.path.join(move_path, bn)
                        
                        code, maybe_error=process(src_file, dst_file, enable_delete)
                        if not code.startswith("ok"):
                            to_skip.append(src_file)
                            logging.warning("Problem processing file '%s': %s" % (src_file, maybe_error))
                except BrokenPipe:
                    raise
                except KeyboardInterrupt:
                    raise
                except Exception, e:
                    logging.error("processing file '%s': %s" % (src_file, str(e)))
                ###############################################################            
        
        
        logging.debug("...sleeping for %s seconds" % polling_interval)
        sleep(polling_interval)
示例#4
0
def run(source_path=None, dest_path=None, check_path=None, 
        batch_size=5, 
        polling_interval=None, delete_fetch_error=False
        ,**_):
    
    if check_path is not None:
        ct=check_transition()
    
    logging.info("Creating (if necessary) destination path: %s" % dest_path)
    code, msg=mkdir_p(dest_path)
    if not code.startswith("ok"):
        raise Exception("Can't create destination path '%s': %s" % (dest_path, str(msg)))
            
    to_skip=[]
    ppid=os.getppid()
    logging.info("Process pid: %s" % os.getpid())
    logging.info("Parent pid : %s" % ppid)
    logging.info("Starting loop...")
    while True:
        if os.getppid()!=ppid:
            logging.warning("Parent terminated... exiting")
            break
        
        if check_path is not None:
            try:    exists=os.path.exists(check_path)
            except: exists=False
            
            maybe_tr, _=ct.send(exists)
            if maybe_tr=="tr" and exists:
                logging.info("Check path: passed")
            if maybe_tr=="tr" and not exists:
                logging.info("Check path: failed - skipping")
        else:
            ## fake 'exists'
            exists=True

        if exists:        
            code, files=get_root_files(source_path)
            if not code.startswith("ok"):
                logging.error("Can't get root files from %s" % source_path)
                continue
            
            ###############################################################
            files=files[:batch_size]
            try:
                for src_file in files:
                    
                    if src_file in to_skip:
                        continue
                    
                    code, _=can_write(src_file)
                    if not code.startswith("ok"):
                        to_skip.append(src_file)
                        logging.error("Would not be able to delete source file '%s'... skipping download" % src_file)
                        continue
                    
                    process(src_file, dest_path, delete_fetch_error)
            except BrokenPipe:
                raise
            except Exception, e:
                logging.error("processing file '%s': %s" % (src_file, str(e)))
            ###############################################################            
        
        
        logging.debug("...waiting for %s seconds (max)" % polling_interval)
        
        ### Implement a "pass-through" for stdin --> stdout
        ###  whilst also handling a maximum time-out
        start_time=time.time()
        while True:
            ir, _w, _e=select.select([sys.stdin], [], [], polling_interval)
            if len(ir):
                iline=sys.stdin.readline()
                sys.stdout.write(iline)
                
            elapsed_time = time.time() - start_time
            if elapsed_time > polling_interval:
                break
示例#5
0
def run(dest_path=None 
        ,**_):
    
    if dest_path is not None:
        if not os.path.isdir(dest_path):
            raise Exception("Expecting a valid destination path '%s'" % dest_path)

    ppid=os.getppid()            
    logging.info("Process pid: %s" % os.getpid())
    logging.info("Parent pid : %s" % ppid)
    logging.info("Starting loop...")
    while True:
        if os.getppid()!=ppid:
            logging.warning("Parent terminated... exiting")
            break
        
        iline=sys.stdin.readline().strip()
        
        #################### VALIDATE
        ## if we received two strings on the same line:  url  dst_path
        bits=iline.split(" ")
        l=len(bits)
        if l > 2 or l==0:
            logging.error("Invalid input line: %s" % iline)
            continue
        
        url=bits[0]
        bn=os.path.basename(url)
        
        if len(bits)==2:
            path=bits[1]
        else:
            if dest_path is not None:
                path=os.path.join(dest_path, bn)
            else:
                logging.warning("Didn't receive 'dest_path' from stdin and none specified on command line...")
                continue
            
        ####### WRITE CAPABILITY VERIFICATION
        code, result=can_write(path)
        if not code.startswith("ok") or not result:
            logging.warning("Won't be able to write to path '%s'... skipping download" % path)
            continue
            
        ####### DOWNLOAD
            
        code, (http_code, headers, data)=fetch(url)
        if not code.startswith("ok"):
            logging.warning("Error attempting to download: %s" % url)
            continue
        
        try:     http_code=int(http_code)
        except:  pass
        
        if http_code!=200:
            logging.warning("Can't fetch url '%s', http response code: %s" % (url, http_code))
            continue
                        
        code, msg=atomic_write(path, data)
        if not code.startswith("ok"):
            raise Exception("Can't write to file '%s': %s" % (path, msg))
        
        ctx={
             "dest_filename": path
             ,"url": url
             ,"http_code": http_code
             ,"headers": headers
             }
        
        try:    sys.stdout.write(json.dumps(ctx)+"\n")
        except: 
            raise Exception("Exiting... probably broken pipe")