def is_encode_ok(s,to_enc,buf_err):
    try:
        s.encode(to_enc)
        ret = True
    except UnicodeEncodeError as e:
        msg = "'"+s[e.start:e.end]+"'"
        eol = get_eol(s)
        #when string is multiline,get number of line that contains error chars        
        if eol != '':
            lno=len(re.findall(eol,s[0:e.start]))+1
            msg += " at line " + str(lno)
        
        buf_err.write(msg)
        ret = False
    return ret
def process(start_dir,pattern,to_enc,to_eol_type,preview):
    if not os.path.exists(start_dir):
        print(_("%s: does'nt exists") % start_dir )
        return
    
    files = find_all_files(start_dir)
    count =0
    files_processed=[]  #files to be processed
    files_skipped=[]    #files to be skipped
    files_dec_ng=[]     #files that can't be decoded
    files_enc_ng =[]    #files that can't be encoded
    #gather information of files.current encoding,end of line
    for path in files:
        if not is_match_patterns_fnmatch(path, pattern.split(',')):
            continue
        try:
            encoding,data = get_encoding(path)
        except DecodeException as e:
            files_dec_ng.append(path)
            continue
        
        info = {'path':path,'encoding':encoding,'eol_type':inv_eol[get_eol(data)]}
        todo = get_todo(info, to_enc, to_eol_type)
        if len(todo)==0:
            files_skipped.append(info)
            continue
        
        if 'encoding' in todo:
            #test encoding
            buf_err = io.StringIO()
            if  not is_encode_ok(data, to_enc,buf_err):
                info['err_str']=buf_err.getvalue()
                files_enc_ng.append(info)
                continue

        files_processed.append(info)
    
    #print files that can't be decoded
    if len(files_dec_ng)>0:
        print(_("Can't decode these files.They are not processed:"))
        for path in files_dec_ng:
            print(path)
        print("---")
    
    #print files that can't be encoded
    if len(files_enc_ng)>0:
        print(_("Can't encode these files.They are not processed:"))
        arr=[]
        for info in files_enc_ng:
            arr.append( [info['encoding'],info['eol_type'],info['path'],info['err_str']])
        print_arr(arr, "[%s,%s] %s:%s")
        print("---")
    
    #print files to be skipped
    if len(files_skipped)>0:
        print(_("files to skip:"))
        arr=[]
        for info in files_skipped:
            arr.append([info['encoding'],info['eol_type'],info['path']])
        print_arr(arr,"[%s,%s] %s")
        print("---")
    
    #print files to be converted
    if len(files_processed)>0:
        print(_("files to convert:"))
        arr=[]
        for info in files_processed:
            todo = get_todo(info, to_enc, to_eol_type)
            
            if 'encoding' in todo:
                msg_to_enc = to_enc
            else:
                msg_to_enc = info['encoding']
            if 'eol' in todo:
                msg_to_eol_type = to_eol_type
            else:
                msg_to_eol_type = info['eol_type']

            arr.append([info["encoding"],info['eol_type'],msg_to_enc,msg_to_eol_type,info["path"]])
        print_arr(arr,"[%s,%s]->[%s,%s] %s")
        print("---")
    else:
        print(_("nothing to do."))
        return
    
    #return here if preview mode
    if preview:
        print (_("***preview mode***"))
        return
    
    #convert
    for info in files_processed:
        todo = get_todo(info, to_enc, to_eol_type)
        if len(todo)>0:
            eol = None
            if 'eol' in todo:
                eol = tbl_eol[to_eol_type]

            if to_enc == 'skip':
                conv_encoding(info["path"], info['encoding'],eol)    #specify original encoding,change only end of line
            else:
                conv_encoding(info["path"], to_enc,eol)
            count+=1

    print (count,_("files changed"))