def get_list(self,tb_name) : name = os.path.join(Recover.Iname,tb_name+'.hpp') txt_name = tb_name+'.txt' s = read(name) s = create_txt(tb_name,s) write(txt_name,s, False) return s
def read_and_hash_async(path_to_file, block_size, method, chunks_count): global async_hashes_list # on default sets cpu_count pool = Pool() i = 0 async_hashes_list = [None] * chunks_count f_read = open(path_to_file, 'rb') start_time = time.time() while True: chunk = f_read.read(block_size) if not chunk: break pool.apply_async(hash_async, args=(chunk, i, method), callback=collect_results) i += 1 pool.close() pool.join() end_time = time.time() - start_time print('read and hash in async mode: ' + str(end_time)) print('hashing speed in async mode: ' + str(chunks_count * block_size / (1024 * 1024) / end_time) + ' Mb/sec') f_read.close() file_utils.write(async_hashes_list, path_to_file, method)
def make_new_bench_file(self,name,mode) : oldb = self.get_old_bench_text(name) newb = self.get_base(mode) newb = sub_list('\$name\$',name,newb) newb = sub_list('\$tb_name\$',self.get_tb_name(),newb) newb = newb[:14]+self.define_types(oldb,mode)+newb[14:] newb = newb[:-2]+sub_list(' ','',self.get_range_list(name,oldb,mode))+newb[-2:] # show(newb) write(os.path.join(self.get_new_path(mode),name+'.cpp'),newb,False)
def replace(self): for f in self.get_list_cpps(): full_f = os.path.join(self.get_unit_simd_path(), f) if self.ok(full_f): s = Rep_units_simd.Simd_unit s = sub_list("\$name\$", f[:-4], s) s = sub_list("\$tb_name\$", self.get_tb_name(), s) show(s) write(full_f, s, False)
def modify_tpl_defs(self) : for k in self.get_dict().keys() : if self.get_dict()[k][1] : p = os.path.join(self.get_new_def_path(),k+'.hpp') s = read(p) pattern = " *NT2_FUNCTION_IMPLEMENTATION.*" rep = " NT2_CRLIBM_FUNCTION_IMPLEMENTATION_TPL("+k+")" s = sub_list(pattern,rep,s) show(s) write(p,s,False)
def replace_scalar_units(self) : fctrs = self.get_functor_list(self.get_tb_name()) for name in fctrs : s = self.replace_scalar_unit(name) ## print "----------------------------" show(s) ## print "============================" p = os.path.join(self.get_tb_path(self.get_tb_name()),'unit/scalar',name+'.cpp') print p write(p,s,False)
def update_CMake(self,check) : lp = self.__root_abs_path.split('/') p = os.path.join('/'.join(lp[:-4]),'CMakeLists.txt') s= read(p) pattern = "ADD_SUBDIRECTORY\(\${PROJECT_BINARY_DIR}/include/nt2/toolbox/"+self.get_tb_name()+'\)' for l in s : m = re.match(pattern, l) if m : return s.append(string.replace(pattern,'\\','')) write(p,s,False)
def get_replacmt(self,target) : for d in self.get_list_dirs() : for h in self.get_list_hpps() : f = os.path.join(d,h) if exist(f) : s = read(f) st ='\n'.join(s) if re.search(target,st) : s=sub_list(target, Insert.Replacement[target],s) write(f,s,False) print "%s -> %s"%(target,Insert.Replacement[target])
def create_summary_from_domains_json(domains_as_json): logging.info('create_summary_from_domains_json()') all_domains = [] available_domains = [] for domain_key in domains_as_json: all_domains.append(domain_key) if domains_as_json[domain_key]['is_available']: available_domains.append(domain_key) file_utils.write('output/all_domains.txt', '\n'.join(all_domains)) file_utils.write('output/available_domains.txt', '\n'.join(available_domains)) print(f'len(all_domains)={len(all_domains)}') print(f'len(available_domains)={len(available_domains)}')
def write_header2(self,path,check=True,flag=None): if (flag is None) or (flag == 'full') : l = self.get_banner()+self.get_guard_begin()+self.__inner+self.get_guard_end() elif flag == 'inner' : l = self.__inner elif flag == 'banner+inner' : l = self.get_banner()+self.__inner else : print "unexpected flag : %s " %flag raise SystemExit if len(self.__fill)==1 : l=sub_if_match_list(" @"+self.__comment, "@", self.__fill, l) write(path,l,check)
def get_matches(self,target) : for d in self.get_list_dirs() : for h in self.get_list_hpps() : f = os.path.join(d,h) if exist(f) : s = read(f) st ='\n'.join(s) if re.search(target,st) : rep = Insert.Insertions[target] if not re.search(rep,st) : s.insert(10,rep) write(f,s,False) print "%s -> %s"%(target,h)
def callback( arg, dirname, fnames ): for file in fnames: if file[-4:] =='.hpp' : fname =os.path.join(dirname,file) s =read(fname) v = gen_proper_guard(dirname[46:],file[:-4]) if len(s)>7 and s[8] != "" and s[8] !=v[0] : print "file %s, dir %s " %(file,dirname[46:]) print "correct %s "%v[0] print "existing %s "%s[8] s1 = [] s1.extend(banner) s1.extend(v) s1.extend(s[10:]) write(fname,s1,False)
def ydm(self,fct_name,acts,subs_dict,action_data,check=True) : """modify a file (reverse of mdy): deleting a line""" fname = action_data["file"].replace('$root_name$',self.get_tb_name()) file2modify = os.path.join(self.get_tb_abs_path(),acts,fname) text = read(file2modify) subs_dict["\$self.tb_pathfnt2\$"]=self.get_tb_pathfnt2() subs_dict["\$fct_name\$"]=fct_name line2rmv = self.__treat(action_data["l2ad"], subs_dict) test, text = self.__rmv_line(text, line2rmv) if test : self.logger.info( "\nmodifying \n%s\nfor definition of functor %s\n" % (file2modify,fct_name) ) write(file2modify,text,False) else : self.logger.warning( "\nFile %s was not present in file\n%s\n" % (fct_name,file2modify) )
def write_header(self,path=None,check=True,flag=None): # print "self.total %s"%self.get_total_path() # print "path %s"%path path2headerfile = os.path.join(nt2_dir() if path is None else path,self.get_total_path()) # print "path2headerfile %s " % path2headerfile if (flag is None) or (flag == 'full') : l = self.get_banner()+self.get_guard_begin()+self.__inner+self.get_guard_end() elif flag == 'inner' : l = self.__inner elif flag == 'banner+inner' : l = self.get_banner()+self.__inner else : print "unexpected *** flag : %s " %flag raise SystemExit if len(self.__fill)==1 : l=sub_if_match_list(" @"+self.__comment, "@", self.__fill, l) self.logger.info( "header written to:\n %s\n" % path2headerfile) write(path2headerfile,l,check)
def mdy(self,fct_name,acts,subs_dict,action_data,check=True) : """modify a file, inserting a line""" fname = action_data["file"].replace('$root_name$',self.get_tb_name()) file2modify = os.path.join(self.get_tb_abs_path(),acts,fname) text = read(file2modify) subs_dict["\$self.tb_pathfnt2\$"]=self.get_tb_pathfnt2() subs_dict["\$fct_name\$"]=fct_name line2add = self.__treat(action_data["l2ad"], subs_dict) token = action_data["tokn"] test, text = self.__add_line(text, line2add, token) if test : self.logger.info( "\nmodifying \n%s\nfor definition of functor %s\n" % (file2modify,fct_name) ) write(file2modify,text,False) else : self.logger.warning( "\nFile %s was already included\n" % fct_name + "in file\n%s\n" % file2modify )
def modify_cpp(self,tb_name,cpps) : unit_dir = self.get_unit_dir(tb_name) d = {} if type(cpps) is str : d["#fct_name"]= cpps d["#tb_name"] = tb_name repls = self.update_replacmnt(d) ## sinserts = self.update_strait_insertions(d) ## finserts = self.update_final_insertions(d) p = os.path.join(unit_dir,cpps+'.cpp') s = read(p) s,doner = self.mk_replace(s,repls) ## s,dones = self.mk_inserts(s,sinserts) ## s,donef = self.mk_insertf(s,finserts) if True or doner :#or dones or donef: print "%s was modified" % cpps #show(s) self.update(s,d) # show(s) write(p,s,False) else : for cpp in cpps : self.modify_cpp(tb_name,cpp)
def write_def_impl(self,fct_name,s) : p = os.path.join(self.get_def_path(),fct_name+'.hpp') write(p,s,False) show(s) return s
for fct_name in s : # print "fct_name : %s"% fct_name p1=os.path.join(p,fct_name+'.hpp') # print "p1 : %s" %p1 arity = get_arity(p1) # print arity if arity !="" : r.append(fct_name+(30-len(fct_name))*' '+arity) return r if __name__ == "__main__" : Mylogging.set_level('CRITICAL') length = len(sys.argv) if length >= 2 : tb_name = sys.argv[1] if length == 2 : p = '/home/jt/DevC++/dev_lasmea/docnt4/nt2-cleanup/nt2/core/numeric/function/' else : p = os.path.join('/home/jt/DevC++/dev_lasmea/docnt4/nt2-cleanup/nt2/toolbox',tb_name,'function') print "p %s" % p s = create_txt(tb_name,p) show(s) write(tb_name+'.txt',s) else : print __doc__ sys.path.pop(0)
def write_new_scalar_impl(self,fct_name, new) : p = os.path.join(self.get_new_scal_path(),fct_name+'.hpp') s = write(p,new,False) # show(s) return s
def write_def_impl(self,fct_name,s) : print "writing %s def" % fct_name p = os.path.join(self.get_def_path(),fct_name+'.hpp') write(p,s,False) # show(s) return s
def write_Cmake_txt(self,s) : print "writing CMakeLists.txt" p = os.path.join(self.get_bench_path(),'CMakeLists.txt') write(p,s,False) #show(s) return s
return "" def create_txt(tb_name, s): r = [tb_name, " "] for l in s: m = re.search("\#include <nt2/core/numeric/function/(.*)>", l) if m: fct_name = m.groups()[0][:-4] p = os.path.join( "/home/jt/DevC++/dev_lasmea/docnt4/nt2-cleanup/nt2/core/numeric/function/", fct_name + ".hpp" ) arity = get_arity(p) if arity != "": r.append(fct_name + (30 - len(fct_name)) * " " + arity) return r if __name__ == "__main__": Mylogging.set_level("CRITICAL") length = len(sys.argv) if length >= 2: tb_name = sys.argv[1] pname = os.path.join("/home/jt/DevC++/dev_lasmea/docnt4/nt2-cleanup/nt2/include/functions", tb_name + ".hpp") s = read(pname) s = create_txt(tb_name, s) write(tb_name + ".txt", s) else: print __doc__ sys.path.pop(0)
def create_text(self) : write(self.get_tb_name()+'.txt',self.create_arity_list(),True)
def write_functor(self,fct_name,type,s,check) : fct_name_path = os.path.join(self.get_tb_abs_path(),'function',type,fct_name+'.hpp') r = write(fct_name_path,s,check)
def create_tb_texts(self) : for tb_name in self.get_tb_list() : s = get_list(tb_name) s = create_txt(tb_name,s) write(tb_name+'.txt',s, True)
def write_Cmake_txt(self,s,test_type,mode) : print "writing CMakeLists.txt" p = os.path.join(self.get_test_path(test_type,mode),'CMakeLists.txt') write(p,s,False) #show(s) return s
def save_page(self, filename, encoding="utf8"): fu.write(filename=filename, data=self.page(), encoding=encoding)
def write_bench(self,name,style,s): p = os.path.join(self.get_tb_path(self.get_tb_name()),'bench',style,name+'.cpp') print name # print "apr-Aès"-b # show(s) write(p,s,False)
import image_check_exceptions __author__ = 'A.P. Rajshekhar' import validate_images import argparse from image_check_exceptions import ImageCheckException import file_utils def parse_arg(): parser = argparse.ArgumentParser() parser.add_argument("--env", help="the environment against which to run search") args = parser.parse_args() if args.env: return args.env else: return 'ci' if __name__ == '__main__': check_image = validate_images.SearchAndValidate() try: check_image.start_check(parse_arg()) except ImageCheckException, e: raise except: file_utils.delete_file() file_utils.write("Build has thrown exception not related to pull of image. Please check.")