예제 #1
0
def process_dir(delete_source, 
                dir_src, 
                bn_dir_src, 
                dir_dst, 
                dir_tmp):
    logging.debug("Processing dir BEGIN: %s" % dir_src)

    zfile_path_tmp=tempfile.mktemp(dir=dir_tmp)
    
    logging.debug("Creating archive '%s'" % zfile_path_tmp)
    try:
        zfile=zipfile.ZipFile(zfile_path_tmp, "w")
        
        os.chdir(dir_src)
        
        code, files=get_root_files(dir_src, strip_dirname=True)
        if code!="ok":
            raise Exception("Can't get files from dir: %s" % dir_src)
        
        for fichier in files:
            zfile.write(fichier)
            
        zfile.close()
        
    except Exception, e:
        rm(zfile_path_tmp)
        raise Exception("Can't generate zip archive: %s (%s)" % (zfile_path_tmp, e))
예제 #2
0
def process_dir(ext_done, mode_dir, del_dir_dst, dir_src, bn_dir_src, dir_dst, dir_tmp):
    logging.debug("Processing dir: %s" % dir_src)
    
    if not mode_dir:
        code, maybe_files=get_root_files(dir_src, strip_dirname=True)
        if not code.startswith("ok"):
            raise Exception("Can't get the filenames of dir: %s" % dir_src)
        
        for f in maybe_files:
            if not f.endswith(ext_done):
                df="%s.%s" % (f, ext_done)
                if not df in maybe_files:
                    logging.debug("Missing 'done' file for: %s" % f)
                    return
    
    do_move(mode_dir, ext_done, del_dir_dst, dir_src, bn_dir_src, dir_dst, dir_tmp)
예제 #3
0
def do_move(mode_dir, ext_done, del_dir_dst, dir_src, bn_dir_src, dir_dst, dir_tmp):
    """
    - Move path directory to a temporary dir in 'dir_tmp'
    - Delete 'file.ext_done' files
    - Move path directory to 'dir_dst'
    """
    dpath, tdir, tpath=do_common(mode_dir, bn_dir_src, del_dir_dst, dir_src, dir_dst, dir_tmp)
    
    if not mode_dir: 
        
        ## next, delete all files with extension 'ext_done'
        code, maybe_files=get_root_files(tpath)
        if not code.startswith("ok"):
            rmdir(tdir)
            raise Exception("Can't get the filenames of temp dir: %s" % tpath)
        
        def _cmp(path):
            return path.endswith(ext_done)
        
        liste=filter(_cmp, maybe_files)
        
        logging.debug("Deleting '%s' files with extension '%s'" % (len(liste), ext_done))
        for f in liste:
            code, _=rm(f)
            if not code.startswith("ok"):
                logging.warning("Can't delete file '%s'... aborting" % f)
                rmdir(tdir)
                return
    
    ## last, move to final destination
    
    logging.debug("Moving '%s' to final dir '%s'" % (tpath, dpath))
    code, _=move(tpath, dpath)
    rmdir(tdir)
    logging.debug("Removed temp dir: %s" % tdir)
    if not code.startswith("ok"):
        raise Exception("Can't move '%s' to directory" % dpath)
    
    logging.info("Processed directory: %s" % dir_src)
예제 #4
0
def run(path_source=None
        ,polling_interval=None
        ,ext_include=None
        ,ext_exclude=None
        ,batch_size=None
        ,**_
        ):

    if ext_include is not None and ext_exclude is not None:
        raise Exception("'ee' and 'ei' options are mutually exclusive")
    
    criteria="include" if ext_include else "exclude"
    elist=ext_include or ext_exclude

    code, path_source=resolve_path(path_source)
    if not code.startswith("ok"):
        raise Exception("can't resolve path '%s'" % path_source)

    ppid=os.getppid()
    logging.info("Process pid: %s" % os.getpid())
    logging.info("Parent pid : %s" % ppid)
    logging.info("Starting loop...")    
    while True:
        if os.getppid()!=ppid:
            logging.warning("Parent terminated... exiting")
            break
        
        code, files=get_root_files(path_source)
        l=filter_files_by_ext(criteria, elist, (code, files))
        
        if l:
            for bunch in batch(l, batch_size):
                output(path_source, bunch)
        
        logging.debug("...sleeping for %s seconds" % polling_interval)
        sleep(polling_interval)
예제 #5
0
def run(source_path=None, move_path=None, check_path=None, 
        batch_size=5, 
        polling_interval=None, enable_delete=False
        ,**_):

    if check_path is not None:
        ct=check_transition()

    if enable_delete and move_path is not None:
        raise Exception("Options '-mp' and '-d' are mutually exclusive")
        
    code, rp=resolve_path(source_path)
    if not code.startswith("ok"):
        raise Exception("can't resolve source path '%s'" % source_path)
    source_path=rp
    
    if move_path is not None:
        code, rp=resolve_path(move_path)
        if not code.startswith("ok"):
            raise Exception("can't resolve 'move_path' '%s'" % move_path)
        move_path=rp

        logging.info("Creating (if necessary) 'move' path: %s" % move_path)
        code, msg=mkdir_p(move_path)
        if not code.startswith("ok"):
            raise Exception("Can't create move path '%s': %s" % (move_path, str(msg)))
            
        logging.info("Checking if 'move' directory is writable")
        code, msg=can_write(move_path)
        if not code.startswith("ok"):
            raise Exception("Can't write to 'move' directory")
            
    to_skip=[]
    logging.info("Process pid: %s" % os.getpid())
    ppid=os.getppid()
    logging.info("Parent pid : %s" % ppid)
    logging.info("Starting loop...")
    while True:
        if os.getppid()!=ppid:
            logging.warning("Parent terminated... exiting")
            break
        
        if check_path is not None:
            try:    exists=os.path.exists(check_path)
            except: exists=False
            
            maybe_tr, _=ct.send(exists)
            if maybe_tr=="tr" and exists:
                logging.info("Check path: passed")
            if maybe_tr=="tr" and not exists:
                logging.info("Check path: failed - skipping")
        else:
            ## fake 'exists'
            exists=True

        if exists:        
            code, files=get_root_files(source_path)
            if not code.startswith("ok"):
                logging.error("Can't get root files from %s" % source_path)
            else:                
                ###############################################################
                files=files[:batch_size]
                try:
                    for src_file in files:
                        
                        if src_file in to_skip:
                            continue
                        
                        code, _=can_write(src_file)
                        if not code.startswith("ok"):
                            to_skip.append(src_file)
                            logging.error("Would not be able to move/delete source file '%s'... skipping streaming" % src_file)
                            continue
        
                        dst_file=None                
                        if move_path is not None:
                            bn=os.path.basename(src_file)
                            dst_file=os.path.join(move_path, bn)
                        
                        code, maybe_error=process(src_file, dst_file, enable_delete)
                        if not code.startswith("ok"):
                            to_skip.append(src_file)
                            logging.warning("Problem processing file '%s': %s" % (src_file, maybe_error))
                except BrokenPipe:
                    raise
                except KeyboardInterrupt:
                    raise
                except Exception, e:
                    logging.error("processing file '%s': %s" % (src_file, str(e)))
                ###############################################################            
        
        
        logging.debug("...sleeping for %s seconds" % polling_interval)
        sleep(polling_interval)
예제 #6
0
def maybe_process_ok(ctx, _ok, _, primary_path, compare_path, just_basename, dest_path):
    
    if dest_path:
        logging.debug("Emptying dest path: %s" % dest_path)
        rmdir(dest_path)
        
        code, _msg=mkdir_p(dest_path)
        if code!="ok":
            raise Exception("Can't create path: %s" % dest_path)
        
    ### log rate limiter helper -- need to update status in context 'ctx'
    doOnTransition(ctx, "status_file.contents", "down", True, None)
    
    codep, primary_files=get_root_files(primary_path, strip_dirname=True)
    codec, compare_files=get_root_files(compare_path, strip_dirname=True)
    
    ### output some log info on transitions
    tm=ctx["tm"]
    tm.send(("pp_log", codep=="ok"))
    tm.send(("zp_log", codec=="ok"))
    
    ### not much to do if either path isn't accessible...
    if not codep.startswith("ok") or not codec.startswith("ok"):
        return
    
    exts=ctx["exts"]
    if exts is not None:
        primary_files=filter(filtre(exts), primary_files)
        compare_files=filter(filtre(exts), compare_files)
    
    def _mapper(path):
        bn=os.path.basename(path)
        return os.path.splitext(bn)[0]
    
    if just_basename:
        pfiles=map(_mapper, primary_files)
        cfiles=map(_mapper, compare_files)
    else:
        pfiles=primary_files
        cfiles=compare_files
    
    try:
        setpf=set(pfiles)
        setcf=set(cfiles)
        common=setpf.intersection(setcf)
        
        diff={
               "pp":     primary_path
              ,"zp":     compare_path
              ,"pp-zp":  list(setpf-setcf)
              ,"zp-pp":  list(setcf-setpf)
              ,"common": list(common)
              }
        
        topic_name=ctx["topic_name"]
        
        if topic_name is not None:
            diff["topic"]=topic_name
        
        if ctx["just_list"]:
            doout(ctx, diff, dest_path)
        else:
            stdoutj(diff)
            stdoutf()
            
    except Exception, e:
        logging.error("Can't compute diff between paths: %s" % str(e))
예제 #7
0
def run(source_path=None, dest_path=None, check_path=None, 
        batch_size=5, 
        polling_interval=None, delete_fetch_error=False
        ,**_):
    
    if check_path is not None:
        ct=check_transition()
    
    logging.info("Creating (if necessary) destination path: %s" % dest_path)
    code, msg=mkdir_p(dest_path)
    if not code.startswith("ok"):
        raise Exception("Can't create destination path '%s': %s" % (dest_path, str(msg)))
            
    to_skip=[]
    ppid=os.getppid()
    logging.info("Process pid: %s" % os.getpid())
    logging.info("Parent pid : %s" % ppid)
    logging.info("Starting loop...")
    while True:
        if os.getppid()!=ppid:
            logging.warning("Parent terminated... exiting")
            break
        
        if check_path is not None:
            try:    exists=os.path.exists(check_path)
            except: exists=False
            
            maybe_tr, _=ct.send(exists)
            if maybe_tr=="tr" and exists:
                logging.info("Check path: passed")
            if maybe_tr=="tr" and not exists:
                logging.info("Check path: failed - skipping")
        else:
            ## fake 'exists'
            exists=True

        if exists:        
            code, files=get_root_files(source_path)
            if not code.startswith("ok"):
                logging.error("Can't get root files from %s" % source_path)
                continue
            
            ###############################################################
            files=files[:batch_size]
            try:
                for src_file in files:
                    
                    if src_file in to_skip:
                        continue
                    
                    code, _=can_write(src_file)
                    if not code.startswith("ok"):
                        to_skip.append(src_file)
                        logging.error("Would not be able to delete source file '%s'... skipping download" % src_file)
                        continue
                    
                    process(src_file, dest_path, delete_fetch_error)
            except BrokenPipe:
                raise
            except Exception, e:
                logging.error("processing file '%s': %s" % (src_file, str(e)))
            ###############################################################            
        
        
        logging.debug("...waiting for %s seconds (max)" % polling_interval)
        
        ### Implement a "pass-through" for stdin --> stdout
        ###  whilst also handling a maximum time-out
        start_time=time.time()
        while True:
            ir, _w, _e=select.select([sys.stdin], [], [], polling_interval)
            if len(ir):
                iline=sys.stdin.readline()
                sys.stdout.write(iline)
                
            elapsed_time = time.time() - start_time
            if elapsed_time > polling_interval:
                break