def git_repo_root(): try : cmd = "git rev-parse --show-toplevel" mout, merr = os_system(cmd) path = mout.split("\n")[0] if len(path) < 1: return None except : return None return path
def test1(): from utilmy import (os_makedirs, Session, global_verbosity, os_system) os_makedirs('ztmp/ztmp2/myfile.txt') os_makedirs('ztmp/ztmp3/ztmp4') os.system("ls ztmp") print('verbosity', global_verbosity( __file__, "config.json", 40, )) sess = Session("ztmp/session") sess.save('mysess', globals(), '01') os.system("ls ztmp/session") res = os_system(f" ls . ", doprint=True) print(res) print("success")
def test1(): from utilmy import (pd_show, git_current_hash, ) ############################################################################ from utilmy import pd_read_file import pandas as pd, random ncols = 7 nrows = 100 ll = [[ random.random() for i in range(0, ncols)] for j in range(0, nrows) ] # Required for it to be detected in Session's globals() global df df = pd.DataFrame(ll, columns = [str(i) for i in range(0,ncols)]) n0 = len(df) s0 = df.values.sum() os.makedirs("data/parquet/", exist_ok= True) ##### m_job , n_pool tests ############################## ncopy = 20 for i in range(0, ncopy) : df.to_csv( f"data/parquet/ppf_{i}.csv.gz", compression='gzip' , index=False) df1 = pd_read_file("data/parquet/ppf*.gz", verbose=1, n_pool= 7 ) assert len(df1) == ncopy * n0, f"df1 {len(df1) }, original {n0}" assert round(df1.values.sum(), 5) == round(ncopy * s0,5), f"df1 {df1.values.sum()}, original {ncopy*s0}" #################################################### df.to_csv( "data/parquet/fa0b2.csv.gz", compression='gzip' , index=False) df.to_csv( "data/parquet/fab03.csv.gz", compression='gzip' , index=False) df.to_csv( "data/parquet/fabc04.csv.gz", compression='gzip' , index=False) df.to_csv( "data/parquet/fa0bc05.csv.gz", compression='gzip' , index=False) df1 = pd_read_file("data/parquet/fab*.*", verbose=1) assert len(df1) == 2 * n0, f"df1 {len(df1) }, original {n0}" ##### Stresss n_pool df2 = pd_read_file("data/parquet/fab*.*", n_pool=1000 ) assert len(df2) == 2 * n0, f"df1 {len(df2) }, original {n0}" ################################################################################### ################################################################################### from utilmy import git_repo_root print(git_repo_root()) assert not git_repo_root() == None, "err git repo" ################################################################################### ################################################################################### from utilmy import os_makedirs, os_system, os_removedirs os_makedirs('ztmp/ztmp2/myfile.txt') os_makedirs('ztmp/ztmp3/ztmp4') os_makedirs('/tmp/one/two') os_makedirs('/tmp/myfile') os_makedirs('/tmp/one/../mydir/') os_makedirs('./tmp/test') os.system("ls ztmp") path = ["/tmp/", "ztmp/ztmp3/ztmp4", "/tmp/", "./tmp/test","/tmp/one/../mydir/"] for p in path: f = os.path.exists(os.path.abspath(p)) assert f == True, "path " + p rev_stat = os_removedirs("ztmp/ztmp2") assert not rev_stat == False, "cannot delete root folder" res = os_system( f" ls . ", doprint=True) print(res) res = os_system( f" ls . ", doprint=False) ################################################################################### ################################################################################### from utilmy import global_verbosity print('verbosity', global_verbosity(__file__, "config.json", 40,)) print('verbosity', global_verbosity('../', "config.json", 40,)) print('verbosity', global_verbosity(__file__)) verbosity = 40 gverbosity = global_verbosity(__file__) assert gverbosity == 5, "incorrect default verbosity" gverbosity =global_verbosity(__file__, "config.json", 40,) assert gverbosity == verbosity, "incorrect verbosity " ################################################################################### ################################################################################### from utilmy import Session sess = Session("ztmp/session") sess.save('mysess', globals(), '01') os.system("ls ztmp/session") sess.save('mysess', globals(), '02') sess.show() import glob flist = glob.glob("ztmp/session/" + "/*") for f in flist: t = os.path.exists(os.path.abspath(f)) assert t == True, "session path not created " pickle_created = os.path.exists(os.path.abspath(f + "/df.pkl")) assert pickle_created == True, "Pickle file not created" sess.load('mysess') sess.load('mysess', None, '02') ################################################################################### from utilmy.decorators import timer @timer def dummy_func(): time.sleep(2) class DummyClass: @timer def method(self): time.sleep(3) dummy_func() a = DummyClass() a.method() ################################################################################### from utilmy.decorators import profiler_deco, profiler_context @profiler_deco def profiled_sum(): return sum(range(100000)) profiled_sum() with profiler_context(): x = sum(range(1000000)) print(x) ################################################################################### from utilmy import profiler_start, profiler_stop profiler_start() print(sum(range(1000000))) profiler_stop() ################################################################################### from utilmy import os_platform_os assert os_platform_os() == sys.platform
def test1(): from utilmy import ( Session, global_verbosity, os_makedirs, os_system, os_removedirs, pd_read_file, pd_show, git_repo_root, git_current_hash, ) #################################################################################### import pandas as pd, random ncols = 7 nrows = 100 ll = [[random.random() for i in range(0, ncols)] for j in range(0, nrows)] df = pd.DataFrame(ll, columns=[str(i) for i in range(0, ncols)]) n0 = len(df) s0 = df.values.sum() os.makedirs("data/parquet/", exist_ok=True) ##### m_job , n_pool tests ############################## ncopy = 20 for i in range(0, ncopy): df.to_csv(f"data/parquet/ppf_{i}.csv.gz", compression='gzip', index=False) df1 = pd_read_file("data/parquet/ppf*.gz", verbose=1, n_pool=7) assert len(df1) == ncopy * n0, f"df1 {len(df1) }, original {n0}" assert round(df1.values.sum(), 5) == round(ncopy * s0, 5), f"df1 {df1.values.sum()}, original {ncopy*s0}" ########################################################### df.to_csv("data/parquet/fa0b2.csv.gz", compression='gzip', index=False) df.to_csv("data/parquet/fab03.csv.gz", compression='gzip', index=False) df.to_csv("data/parquet/fabc04.csv.gz", compression='gzip', index=False) df.to_csv("data/parquet/fa0bc05.csv.gz", compression='gzip', index=False) df1 = pd_read_file("data/parquet/fab*.*", verbose=1) assert len(df1) == 2 * n0, f"df1 {len(df1) }, original {n0}" ##### Stresss n_pool df2 = pd_read_file("data/parquet/fab*.*", n_pool=1000) assert len(df2) == 2 * n0, f"df1 {len(df2) }, original {n0}" ################################################################################### ################################################################################### print(git_repo_root()) assert not git_repo_root() == None, "err git repo" ################################################################################### ################################################################################### os_makedirs('ztmp/ztmp2/myfile.txt') os_makedirs('ztmp/ztmp3/ztmp4') os_makedirs('/tmp/') os_makedirs('/tmp/one/two') os_makedirs('/tmp/myfile') os_makedirs('/tmp/one/../mydir/') os_makedirs('./tmp/test') os.system("ls ztmp") path = [ "/tmp/", "ztmp/ztmp3/ztmp4", "/tmp/", "./tmp/test", "/tmp/one/../mydir/" ] for p in path: f = os.path.exists(os.path.abspath(p)) assert f == True, "path " rev_stat = os_removedirs("ztmp/ztmp2") assert not rev_stat == False, "cannot delete root folder" res = os_system(f" ls . ", doprint=True) print(res) res = os_system(f" ls . ", doprint=False) ################################################################################### ################################################################################### print('verbosity', global_verbosity( __file__, "config.json", 40, )) print('verbosity', global_verbosity( '../', "config.json", 40, )) print('verbosity', global_verbosity(__file__)) verbosity = 40 gverbosity = global_verbosity(__file__) assert gverbosity == 5, "incorrect default verbosity" gverbosity = global_verbosity( __file__, "config.json", 40, ) assert gverbosity == verbosity, "incorrect verbosity " ################################################################################### ################################################################################### sess = Session("ztmp/session") sess.save('mysess', globals(), '01') os.system("ls ztmp/session") sess.save('mysess', globals(), '02') sess.show() import glob flist = glob.glob("ztmp/session/" + "/*") for f in flist: t = os.path.exists(os.path.abspath(f)) assert t == True, "session path not created " sess.load('mysess') sess.load('mysess', None, '02')
def test1(): from utilmy import (Session, global_verbosity, os_makedirs, os_system , os_removedirs, pd_read_file, pd_show, git_repo_root, git_current_hash, ) import pandas as pd, random ll = [[ random.random() for i in range(0, 77)] for j in range(0,103) ] df =pd.DataFrame(ll, columns = [i for i in range(0,100)]) os.makedirs("data/parquet/", exist_ok= True) df.to_csv( "data/parquet/f01.csv.gz", compression='gzip' ) df.to_csv( "data/parquet/fa02.csv.gz", compression='gzip' ) df.to_csv( "data/parquet/fab03.csv.gz", compression='gzip' ) df.to_csv( "data/parquet/fabc04.csv.gz", compression='gzip' ) df.to_csv( "data/parquet/fabc05.csv", ) df1 = pd_read_file("data/parquet/f*.gz", verbose=1, n_pool=3) print('pd_read_file gzip ', df1) b = df1.mean() a = df.mean() assert round(a,5) == round(b,5), "Sum of loaded df <> Original df" df1 = pd_read_file("data/parquet/fab*.*", verbose=1) print('pd_read_file csv ', df) df1 = pd_read_file("data/parquet/fab*.*", n_pool=1 ) print('pd_read_file csv ', df) df1 = pd_read_file("data/parquet/f*.gz", verbose=1, n_pool=3) print('pd_read_file gzip ', df1) b = df1.mean() a = df.mean() print(a.equals(b)) # for index, val in a.iteritems(): # print(f'{index}: {round( val, 5)}') # for index, val in b.iteritems(): # print(f'{index}: {round( val, 5)}') # the 1st df1 = pd_read_file("data/parquet/fab*.*", n_pool=0 ) df1 = pd_read_file("data/parquet/fab*.*", n_pool=1000 ) df1 = pd_read_file("data/parquet/fac*.*") df1 = pd_read_file("data/parquet/") # the 2nd # pd_show() # the 3rd print(git_repo_root()) ############################################################# os_makedirs('ztmp/ztmp2/myfile.txt') os_makedirs('ztmp/ztmp3/ztmp4') os_makedirs('/tmp/') os_makedirs('/tmp/one/two') os_makedirs('/tmp/myfile') os_makedirs('/tmp/one/../mydir/') os_makedirs('./tmp/test') os.system("ls ztmp") os_removedirs("ztmp/ztmp2") print('verbosity', global_verbosity(__file__, "config.json", 40,)) print('verbosity', global_verbosity('../', "config.json", 40,)) print('verbosity', global_verbosity(__file__)) sess = Session("ztmp/session") sess.save('mysess', globals(), '01') os.system("ls ztmp/session") sess.save('mysess', globals(), '02') sess.show() sess.load('mysess') sess.load('mysess', None, '02') res = os_system( f" ls . ", doprint=True) print(res) res = os_system( f" ls . ", doprint=False) res = os_system( f" ls . ", doprint=True) print("success")