Esempio n. 1
0
def test1():
    from utilmy import (os_makedirs, Session, global_verbosity, os_system)

    os_makedirs('ztmp/ztmp2/myfile.txt')
    os_makedirs('ztmp/ztmp3/ztmp4')
    os.system("ls ztmp")

    print('verbosity', global_verbosity(
        __file__,
        "config.json",
        40,
    ))

    sess = Session("ztmp/session")
    sess.save('mysess', globals(), '01')
    os.system("ls ztmp/session")

    res = os_system(f" ls . ", doprint=True)
    print(res)

    print("success")
Esempio n. 2
0
def test1():
   from utilmy import (pd_show,
                       git_current_hash,
                      )

   ############################################################################
   from utilmy import pd_read_file
   import pandas as pd, random
   ncols = 7
   nrows = 100
   ll = [[ random.random() for i in range(0, ncols)] for j in range(0, nrows) ]
   # Required for it to be detected in Session's globals()
   global df
   df = pd.DataFrame(ll, columns = [str(i) for i in range(0,ncols)])
   n0 = len(df)
   s0 = df.values.sum()
   os.makedirs("data/parquet/", exist_ok= True)

   ##### m_job , n_pool tests  ##############################
   ncopy = 20
   for i in range(0, ncopy) :
      df.to_csv( f"data/parquet/ppf_{i}.csv.gz",  compression='gzip' , index=False)

   df1 = pd_read_file("data/parquet/ppf*.gz", verbose=1, n_pool= 7 )

   assert len(df1) == ncopy * n0,         f"df1 {len(df1) }, original {n0}"
   assert round(df1.values.sum(), 5) == round(ncopy * s0,5), f"df1 {df1.values.sum()}, original {ncopy*s0}"


   ####################################################
   df.to_csv( "data/parquet/fa0b2.csv.gz",   compression='gzip' , index=False)
   df.to_csv( "data/parquet/fab03.csv.gz",   compression='gzip' , index=False)
   df.to_csv( "data/parquet/fabc04.csv.gz",  compression='gzip' , index=False)
   df.to_csv( "data/parquet/fa0bc05.csv.gz", compression='gzip' , index=False)

   df1 = pd_read_file("data/parquet/fab*.*", verbose=1)
   assert len(df1) == 2 * n0, f"df1 {len(df1) }, original {n0}"


   ##### Stresss n_pool
   df2 = pd_read_file("data/parquet/fab*.*", n_pool=1000 )
   assert len(df2) == 2 * n0, f"df1 {len(df2) }, original {n0}"



   ###################################################################################
   ###################################################################################
   from utilmy import git_repo_root
   print(git_repo_root())
   assert not git_repo_root() == None, "err git repo"



   ###################################################################################
   ###################################################################################
   from utilmy import os_makedirs, os_system, os_removedirs
   os_makedirs('ztmp/ztmp2/myfile.txt')
   os_makedirs('ztmp/ztmp3/ztmp4')
   os_makedirs('/tmp/one/two')
   os_makedirs('/tmp/myfile')
   os_makedirs('/tmp/one/../mydir/')
   os_makedirs('./tmp/test')
   os.system("ls ztmp")

   path = ["/tmp/", "ztmp/ztmp3/ztmp4", "/tmp/", "./tmp/test","/tmp/one/../mydir/"]
   for p in path:
       f = os.path.exists(os.path.abspath(p))
       assert  f == True, "path " + p

   rev_stat = os_removedirs("ztmp/ztmp2")
   assert not rev_stat == False, "cannot delete root folder"

   res = os_system( f" ls . ",  doprint=True)
   print(res)
   res = os_system( f" ls . ",  doprint=False)



   ###################################################################################
   ###################################################################################
   from utilmy import global_verbosity
   print('verbosity', global_verbosity(__file__, "config.json", 40,))
   print('verbosity', global_verbosity('../', "config.json", 40,))
   print('verbosity', global_verbosity(__file__))

   verbosity = 40
   gverbosity = global_verbosity(__file__)
   assert gverbosity == 5, "incorrect default verbosity"
   gverbosity =global_verbosity(__file__, "config.json", 40,)
   assert gverbosity == verbosity, "incorrect verbosity "





   ###################################################################################
   ###################################################################################
   from utilmy import Session
   sess = Session("ztmp/session")
   sess.save('mysess', globals(), '01')
   os.system("ls ztmp/session")

   sess.save('mysess', globals(), '02')
   sess.show()

   import glob
   flist = glob.glob("ztmp/session/" + "/*")
   for f in flist:
       t = os.path.exists(os.path.abspath(f))
       assert  t == True, "session path not created "

       pickle_created = os.path.exists(os.path.abspath(f + "/df.pkl"))
       assert  pickle_created == True, "Pickle file not created"

   sess.load('mysess')
   sess.load('mysess', None, '02')

   
   ###################################################################################   
   from utilmy.decorators import timer
   @timer
   def dummy_func():
       time.sleep(2)

   class DummyClass:
       @timer
       def method(self):
           time.sleep(3)

   dummy_func()
   a = DummyClass()
   a.method()


   ###################################################################################
   from utilmy.decorators import profiler_deco, profiler_context
   
   @profiler_deco
   def profiled_sum():
       return sum(range(100000))

   profiled_sum()

   with profiler_context():
       x = sum(range(1000000))
       print(x)

      
   ###################################################################################
   from utilmy import profiler_start, profiler_stop
   
   profiler_start()
   print(sum(range(1000000)))
   profiler_stop()

   
   ###################################################################################
   from utilmy import os_platform_os
   assert os_platform_os() == sys.platform
Esempio n. 3
0
def test1():
    from utilmy import (
        Session,
        global_verbosity,
        os_makedirs,
        os_system,
        os_removedirs,
        pd_read_file,
        pd_show,
        git_repo_root,
        git_current_hash,
    )

    ####################################################################################
    import pandas as pd, random

    ncols = 7
    nrows = 100
    ll = [[random.random() for i in range(0, ncols)] for j in range(0, nrows)]
    df = pd.DataFrame(ll, columns=[str(i) for i in range(0, ncols)])
    n0 = len(df)
    s0 = df.values.sum()
    os.makedirs("data/parquet/", exist_ok=True)

    ##### m_job , n_pool tests  ##############################
    ncopy = 20
    for i in range(0, ncopy):
        df.to_csv(f"data/parquet/ppf_{i}.csv.gz",
                  compression='gzip',
                  index=False)

    df1 = pd_read_file("data/parquet/ppf*.gz", verbose=1, n_pool=7)

    assert len(df1) == ncopy * n0, f"df1 {len(df1) }, original {n0}"
    assert round(df1.values.sum(),
                 5) == round(ncopy * s0,
                             5), f"df1 {df1.values.sum()}, original {ncopy*s0}"

    ###########################################################
    df.to_csv("data/parquet/fa0b2.csv.gz", compression='gzip', index=False)
    df.to_csv("data/parquet/fab03.csv.gz", compression='gzip', index=False)
    df.to_csv("data/parquet/fabc04.csv.gz", compression='gzip', index=False)
    df.to_csv("data/parquet/fa0bc05.csv.gz", compression='gzip', index=False)

    df1 = pd_read_file("data/parquet/fab*.*", verbose=1)
    assert len(df1) == 2 * n0, f"df1 {len(df1) }, original {n0}"

    ##### Stresss n_pool
    df2 = pd_read_file("data/parquet/fab*.*", n_pool=1000)
    assert len(df2) == 2 * n0, f"df1 {len(df2) }, original {n0}"

    ###################################################################################
    ###################################################################################
    print(git_repo_root())
    assert not git_repo_root() == None, "err git repo"

    ###################################################################################
    ###################################################################################
    os_makedirs('ztmp/ztmp2/myfile.txt')
    os_makedirs('ztmp/ztmp3/ztmp4')
    os_makedirs('/tmp/')
    os_makedirs('/tmp/one/two')
    os_makedirs('/tmp/myfile')
    os_makedirs('/tmp/one/../mydir/')
    os_makedirs('./tmp/test')

    os.system("ls ztmp")

    path = [
        "/tmp/", "ztmp/ztmp3/ztmp4", "/tmp/", "./tmp/test",
        "/tmp/one/../mydir/"
    ]
    for p in path:
        f = os.path.exists(os.path.abspath(p))
        assert f == True, "path "

    rev_stat = os_removedirs("ztmp/ztmp2")
    assert not rev_stat == False, "cannot delete root folder"

    res = os_system(f" ls . ", doprint=True)
    print(res)
    res = os_system(f" ls . ", doprint=False)

    ###################################################################################
    ###################################################################################
    print('verbosity', global_verbosity(
        __file__,
        "config.json",
        40,
    ))
    print('verbosity', global_verbosity(
        '../',
        "config.json",
        40,
    ))
    print('verbosity', global_verbosity(__file__))

    verbosity = 40
    gverbosity = global_verbosity(__file__)
    assert gverbosity == 5, "incorrect default verbosity"
    gverbosity = global_verbosity(
        __file__,
        "config.json",
        40,
    )
    assert gverbosity == verbosity, "incorrect verbosity "

    ###################################################################################
    ###################################################################################
    sess = Session("ztmp/session")
    sess.save('mysess', globals(), '01')
    os.system("ls ztmp/session")

    sess.save('mysess', globals(), '02')
    sess.show()

    import glob
    flist = glob.glob("ztmp/session/" + "/*")
    for f in flist:
        t = os.path.exists(os.path.abspath(f))
        assert t == True, "session path not created "

    sess.load('mysess')
    sess.load('mysess', None, '02')
Esempio n. 4
0
def test1():

   from utilmy import (Session,
                       global_verbosity,




                       os_makedirs,
                       os_system ,
                       os_removedirs,





                       pd_read_file,
                       pd_show,



                       git_repo_root,
                       git_current_hash,


                      )




   import pandas as pd, random

   ll = [[ random.random() for i in range(0, 77)] for j in range(0,103) ]
   df =pd.DataFrame(ll, columns = [i for i in range(0,100)])
   os.makedirs("data/parquet/", exist_ok= True)
   df.to_csv( "data/parquet/f01.csv.gz", compression='gzip' )
   df.to_csv( "data/parquet/fa02.csv.gz", compression='gzip' )
   df.to_csv( "data/parquet/fab03.csv.gz", compression='gzip' )
   df.to_csv( "data/parquet/fabc04.csv.gz", compression='gzip' )
   df.to_csv( "data/parquet/fabc05.csv", )


   df1 = pd_read_file("data/parquet/f*.gz", verbose=1, n_pool=3)
   print('pd_read_file gzip ', df1)
   b = df1.mean()
   a = df.mean()
   assert round(a,5) == round(b,5), "Sum of loaded df <> Original df"



   df1 = pd_read_file("data/parquet/fab*.*", verbose=1)
   print('pd_read_file csv ', df)

   df1 = pd_read_file("data/parquet/fab*.*", n_pool=1 )
   print('pd_read_file csv ', df)

   df1 = pd_read_file("data/parquet/f*.gz", verbose=1, n_pool=3)
   print('pd_read_file gzip ', df1)
   b = df1.mean()
   a = df.mean()
   
   print(a.equals(b))
   # for index, val in a.iteritems():
   #  print(f'{index}: {round( val, 5)}')

   # for index, val in b.iteritems():
   #  print(f'{index}: {round( val, 5)}')

   # the 1st
   df1 = pd_read_file("data/parquet/fab*.*", n_pool=0 )

   df1 = pd_read_file("data/parquet/fab*.*", n_pool=1000 )

   df1 = pd_read_file("data/parquet/fac*.*")

   df1 = pd_read_file("data/parquet/")


   # the 2nd
   # pd_show()

   # the 3rd
   print(git_repo_root())


   #############################################################
   os_makedirs('ztmp/ztmp2/myfile.txt')
   os_makedirs('ztmp/ztmp3/ztmp4')
   os_makedirs('/tmp/')
   os_makedirs('/tmp/one/two')
   os_makedirs('/tmp/myfile')
   os_makedirs('/tmp/one/../mydir/')
   os_makedirs('./tmp/test')
    
   os.system("ls ztmp")


   os_removedirs("ztmp/ztmp2")



   print('verbosity', global_verbosity(__file__, "config.json", 40,))
   print('verbosity', global_verbosity('../', "config.json", 40,))
   print('verbosity', global_verbosity(__file__))

   sess = Session("ztmp/session")
   sess.save('mysess', globals(), '01')
   os.system("ls ztmp/session")

   sess.save('mysess', globals(), '02')
   sess.show()

   sess.load('mysess')
   sess.load('mysess', None, '02')


   res = os_system( f" ls . ",  doprint=True)
   print(res)

   res = os_system( f" ls . ",  doprint=False) 
   res = os_system( f" ls . ",  doprint=True) 

   print("success")