def int_to_size_str(size,b1024=True,zero='0 B',less_than_zero='%s', ):
'''Convert a file size to human-readable form.
Keyword arguments:
size -- file size in bytes
b1024 -- if True (default), use multiples of 1024
if False, use multiples of 1000
Returns: string
test git
'''
U,T,N,F=py.importUTNF()
if py.istr(size) or py.isbyte(size):
size=U.len(size)
size=py.int(size)
if size < 0:
if py.callable(less_than_zero):
return less_than_zero(size)
if py.istr(less_than_zero):
if '%' in less_than_zero:
return less_than_zero%size
if '{' in less_than_zero and '}' in less_than_zero:
return less_than_zero.format(size)
return less_than_zero
if size==0:return zero
# raise ValueError('number must be non-negative')
multiple = 1024.0 if b1024 else 1000.0 #another if statement style
if size<=multiple:return py.str(size)+' B'
for suffix in SUFFIXES[multiple]:
size /= multiple
if size < multiple:
return '{0:.3f} {1}'.format(size, suffix)
raise ValueError('number too large')
def dill_dump_bytes(obj,file=None,protocol=None,dill_ext='.dill'): ''' #TODO file=0 Not write '../0.dill' dill.dump(obj, file, protocol=None, byref=None, fmode=None, recurse=None) dill.dumps(obj, protocol=None, byref=None, fmode=None, recurse=None) ValueError: pickle protocol must be <= 4 r=request.get ... F.readableSize(len(F.dill_dump(protocol=None,obj=r) ) )#'14.192 KiB' F.readableSize(len(F.dill_dump(protocol=0,obj=r) ) ) #'15.773 KiB' F.readableSize(len(F.dill_dump(protocol=1,obj=r) ) ) #'19.177 KiB' F.readableSize(len(F.dill_dump(protocol=2,obj=r) ) ) #'18.972 KiB' F.readableSize(len(F.dill_dump(protocol=3,obj=r) ) ) #'14.192 KiB' F.readableSize(len(F.dill_dump(protocol=4,obj=r) ) ) #'13.694 KiB' ''' import dill if file: if py.istr(obj) and py.len(obj)<333 and '.dill' in obj: if not py.istr(file) or '.dill' not in file: file,obj=obj,file file=auto_path(file,ext=dill_ext) with py.open(file,'wb') as f: dill.dump(obj=obj,file=f,protocol=protocol) return file else: return dill.dumps(obj=obj,protocol=protocol)
def get(p=False, edit=False, edit_prompt='', only_edit_firstline=True, **ka):
'''win32con.
CF_DSPTEXT ', 129],
CF_OEMTEXT ', 7],
CF_RETEXTOBJ ', 'RichEdit Text and Objects'],
CF_TEXT ', 1],
CF_UNICODETEXT ', 13],
in py3 win32con.CF_TEXT return b' '
prompt [präm(p)t 不发音p] 提示 ;
promte 提升
'''
U = py.importU()
if U.istermux(): return U.cmd('termux-clipboard-get')
p = U.get_duplicated_kargs(ka,
'_print',
'print_',
'show',
'PRINT',
default=p)
edit = U.get_duplicated_kargs(ka,
'e',
'E',
'input_edit',
'input',
'edit_clipboard',
default=edit)
w.OpenClipboard()
d = w.GetClipboardData(win32con.CF_UNICODETEXT)
w.CloseClipboard()
if p: U.pln(d)
if edit:
type = U.get_duplicated_kargs(ka, 'type', 'edit_type', 't')
edit_prompt = U.get_duplicated_kargs(ka,
'title',
'msg',
'edit_msg',
'edit_prompt',
'prompt',
'promte',
'promot',
'promote',
default=edit_prompt)
only_edit_firstline = U.get_duplicated_kargs(
ka,
'edit_firstline',
'firstline',
'fl',
'line0',
'l0',
default=only_edit_firstline)
if not edit_prompt:
if py.istr(edit): edit_prompt = edit
else: edit_prompt = ''
if only_edit_firstline and py.len(d.splitlines()) > 0:
d = d.splitlines()[0]
d = U.input(edit_prompt, default=d, type=type)
return d
def write(file,data,mod='w',encoding='utf-8',mkdir=False,autoArgs=True,pretty=True,seek=None):
'''py3 open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None)
py2 open(name[, mode[, buffering]])
pretty=True Format a Python object into a pretty-printed representation.
'''
U=py.importU()
try:
if autoArgs:
if py.istr(data) and py.len(file)>py.len(data)>0:
if '.' in data and '.' not in file and isFileName(data):
file,data=data,file
U.warring('F.write fn,data but seems data,fn auto corrected（v 纠正')
except:pass
# try:
file=autoPath(file)
if not encoding:encoding=U.encoding
if mkdir:makeDirs(file,isFile=True)
# if 'b' not in mod and py.isbytes(data):mod+='b'# 自动检测 data与 mod 是否匹配
if 'b' not in mod: #强制以 byte 写入
mod+='b'
f=py.open(file,mod)
#f.write(强制unicode) 本来只适用 py.is3() ，但 py2 中 有 from io import open
if py.isint(seek):
f.seek(seek)
# with open(file,mod) as f:
if py.isbyte(data):#istr(data) or (py.is3() and py.isinstance(data,py.bytes) ) :
f.write(data)
elif (py.is2() and py.isinstance(data,py.unicode)) :
f.write(data.encode(encoding))
elif (py.is3() and py.istr(data)):
# if 'b' in mod.lower():
f.write(data.encode(encoding))
# else:f.write(data)#*** UnicodeEncodeError: 'gbk' codec can't encode character '\xa9' in
else:
# if py.is2():print >>f,data
# else:
if pretty:
data=U.pformat(data)
U.pln(data,file=f)
f.close()
return f.name
def trace_variable(code): if py.istr(code):a=U.parse_code(code ) # one line return <_ast.Expr,_ast.For...,multi return [Assign,Expr,For..] if getattr(code,'value',0):a=code if isinstance(a,_ast.Expr): a=a.value if isinstance(a,_ast.Call): a.args a.keywords
def copy(src,dst,src_base='',skip=''):
r''' src : sFilePath , list ,or \n strs
dst:sPath
'''
from shutil import copy as _copy
if not py.istr(dst):raise py.ArgumentError('dst must be str')
if py.istr(src):
if '\n' in src:
src=src.splitlines()
return copy(src,dst)
if src[-1] in ['/','\\']:
src=F.ls(src,r=1)
else:
return _copy(src,dst)
if not src_base:
U,T,N,F=py.importUTNF()
dl=U.unique(U.len(*src),ct=1)
min=py.min(*dl)
f=[i for i in src if py.len(i)==min][0]
if f[-1] in ['/','\\']:f=f[:-1]
# Path(f).absolute().parent.absolute().__str__()
src_base=f[:py.len(T.sub_last(f.replace('\\','/'),'','/') )+1]
src_base_len=py.len(src_base)
print('src_base: %r'%src_base,'len(src)==%s'%py.len(src))
while dst[-1] not in ['/','\\']:
dst=U.input('not dir! rewrite dst:',default=dst)
if py.iterable(src):
fns=[]
skips=[]
for i in src:
if skip and skip in i:
skips.append(i)
continue
# fn=getName(i)
# if fn in fns:
# fn=T.fileName(i)
fn=i[src_base_len:]
if fn[-1] in ['/','\\']:
mkdir(dst+fn)
else:
_copy(i,dst+fn)
fns.append(fn)
if skips:return skips,fns
return fns
raise py.ArgumentUnsupported(src)
def __new__(cls, *a, **ka): #int() argument must be a string, a bytes-like object or a number, not if py.istr(a[0]) or py.isbyte(a[0]) or py.isnumber(a[0]): self= py.int.__new__(cls, *a) else: self= a[0] self.ka=ka return self
def set_title(title, h=0):
'''在python内设置的，退出后 会还原
py3 : SetWindowTextW
'''
if not py.istr(title): title = py.str(title)
if not h: h = getCmdHandle()
if py.is3():
return user32.SetWindowTextW(h, title)
else:
return user32.SetWindowTextA(h, title)
def isDir(ast):
'''#TODO:
'''
if not py.istr(ast):ast=py.str(ast)
if not ast:return False
ast=ast.replace('\\','/')
if exist(ast):
return _p.isdir(ast)
else:
if ast.endswith('/'):return True
else: return False
def read_xlsx(file,sheet=0):
''' 加载一个8M多xlsx文件，很慢,'''
import openpyxl
if py.isinstance(file,openpyxl.workbook.workbook.Workbook):
wb=file
elif py.istr(file):
wb = openpyxl.load_workbook(file)# 这句很耗时
else:
raise py.ArgumentUnsupported(file)
if py.isint(sheet):
ws=wb[wb.sheetnames[sheet]]
elif py.istr(sheet):
ws=wb[sheet]
else:raise py.ArgumentError('sheet is index or sheet_names')
r=[]
for row in ws.values:#<Worksheet "1 lemmas">
ri=[]
for value in row:
ri.append(value)
r.append(ri)
return r
def jupyter_password(passphrase='',salt='0'*12,algorithm='sha1'):
import hashlib
from ipython_genutils.py3compat import cast_bytes,str_to_bytes
if py.isbytes(salt):
bsalt=salt
salt=bsalt.decode('ascii')
elif py.istr(salt):
bsalt=str_to_bytes(salt, 'ascii')
else:
raise py.ArgumentError(salt)
h=hashlib.new(algorithm)
h.update(cast_bytes(passphrase, 'utf-8') + bsalt)
return ':'.join((algorithm, salt, h.hexdigest()))
def detect_file_encoding(file,confidence=0.7,default=py.No('not have default encoding'),buffer_size=9999,p=True,**ka):
U,T,N,F=py.importUTNF()
p=U.get_duplicated_kargs(ka,'print_file_encoding','print_detect_encoding','print',default=p,no_pop=True)
if py.istr(file):
with py.open(file,'rb') as f:
b=f.read(buffer_size)
elif py.isfile(file):
if 'b' not in file.mode:raise py.ArgumentError("'b' not in file.mode",file)
i=file.tell()
b=file.read(buffer_size)
file.seek(i)
else:raise py.ArgumentError('need str or file')
c= T.detect(b,confidence=confidence,default=default)
if p:print(file,c) #TODO U.get_or_set('detect_file_encoding.p',True)
return c
def modPathInSys(mod=None):
if mod:
if not py.istr(mod):mod=mod.__file__
else:mod=__file__
mod=mod.replace('\\','/')
inPath=False
if os.path.isabs(mod):
for i in sys.path:
i=i.replace('\\','/')
if i and mod.startswith(i):
if not i.endswith('/'):i+='/'
i+='qgb'
if mod.startswith(i):inPath=True
else:
raise NotImplementedError('__file__ not abs')
return inPath
def deSerialize(obj=None,file=None):
'''The protocol version of the pickle is detected automatically, so no
protocol argument is needed. Bytes past the pickled object's
representation are ignored.
'''
if not py.isbyte(obj) and not file:raise py.ArgumentError('need bytes or file=str ')
import pickle
if py.istr(obj):
file=obj
obj=None
U.log('autoArgs file=%s'%file)
if py.isbyte(obj):
return pickle.loads(obj)
else:
file=autoPath(file)
with py.open(file,'rb') as f:
return pickle.load(f)
def set(skey,
name,
value,
root=HKEY_CURRENT_USER,
type='auto,or REG_TYPE int',
returnType=True):
r = OpenKey(root, skey, 0, KEY_SET_VALUE)
if not py.isint(type):
if py.isint(value): type = 4
if py.istr(value): type = 1
if py.isbyte(value): type = 3 #TODO test,and add more rule
SetValueEx(r, 'ProxyEnable', 0, type, value)
if get(skey, name, root=root, returnType=False) == value:
return 'reg.set [{}] {}={} sucess!'.format(skey[-55:], name, value)
else:
return 'reg.set [{}] {}={} Failed !'.format(skey, name, value)
def open(file,mode='r',**ka):
'''py.open(
file,
mode='r',
buffering=-1,
encoding=None,
errors=None,
newline=None,
closefd=True,
opener=None,
)
'''
U,T,N,F=py.importUTNF()
mode=U.get_duplicated_kargs(ka,'mode','mod','m',default=mode)
if py.isfile(file):
#if file.closed==False:
return file
elif py.istr(file):
return py.open(file,mode=mode,**ka)
else:
raise py.ArgumentUnsupported(file,ka)
def win32_shellcopy(src, dest):
"""
Copy files and directories using Windows shell.
:param src: Path or a list of paths to copy. Filename portion of a path
(but not directory portion) can contain wildcards ``*`` and
``?``.
:param dst: destination directory.
:returns: ``True`` if the operation completed successfully,
``False`` if it was aborted by user (completed partially).
:raises: ``WindowsError`` if anything went wrong. Typically, when source
file was not found.
.. seealso:
`SHFileperation on MSDN <http://msdn.microsoft.com/en-us/library/windows/desktop/bb762164(v=vs.85).aspx>`
"""
from win32com.shell import shell, shellcon
U, T, N, F = py.importUTNF()
if py.istr(src): # in Py3 replace basestring with str
src = F.abspath(src).replace('/', '\\')
else: # iterable
src = '\0'.join(F.abspath(path) for path in src)
result, aborted = shell.SHFileOperation((
0,
shellcon.FO_COPY,
src,
F.abspath(dest).replace('/', '\\'),
shellcon.FOF_NOCONFIRMMKDIR, # flags
None,
None))
if not aborted and result != 0:
# Note: raising a WindowsError with correct error code is quite
# difficult due to SHFileOperation historical idiosyncrasies.
# Therefore we simply pass a message.
raise WindowsError('SHFileOperation failed: 0x%08x' % result)
return not aborted
def outType(t=None,start=0,stop=U.IMAX,len=py.range(U.IMAX)):
'''t is type to flit
is3:range(start, stop[, step]) -> range object
'''
if t !=None:
# if type(t) is U.instance: # 这个什么意思来着
# t=t.__class__
if py.istr(t):
def m(a):return t in py.repr(a)
elif type(t) is U.classType:#没有考虑 取出Class 类型的情况
def m(a):return isinstance(a,t)
else:
if not isinstance(t,type):
t=type(t)
def m(a):return type(a) is t
r={}
for index,i in py.enumerate(gOut):
if index<start or index>stop:continue
if t !=None:
if not m(gOut[i]):continue
r[i]=type(gOut[i]),U.len(gOut[i]),U.sizeof(gOut[i])
return r
def list(ap='.',type='',t='',r=False,d=False,dir=False,f=False,
file=False,include='',exclude='',timeout=None,print_result=False,**ka):
'''Parms:bool r recursion
str (type,t) '(d,f,a,r)'
default return all'''
U=py.importU()
print_result=U.get_duplicated_kargs(ka,'print_r','print','p',default=print_result)
if dir:d=True
if file:f=True
if t and not type:type=t
if 'd' in type:d=True
if 'f' in type:f=True
if 'a' in type:d=True;f=True
if 'r' in type:r=True
if d or dir or f or file:pass
else:d=f=True #default return all
if not py.istr(ap) or py.len(ap)<1:
setErr('F.list arguments ap error')
ap='.'
# if len(ap)==2 and ap.endswith(':'):ap+='/'
ap=ap.replace('\\','/')
if not ap.endswith('/'):#U.inMuti(ap,'/','\\',f=str.endswith):
if isDir(ap):
ap+='/'
else:
return [exists(ap)]
# if not ap.endswith('/'):ap+='/'
# else:ap+='\\'
# U.repl()
########## below r is result
rls=[]
try:r3=py.list(walk(ap))
except Exception as ew:
# pln ap;raise ew
return py.No(ew)
if ap=='./':ap=''
# U.repl()
r3[1]=[ap+i+'/' for i in r3[1]] #dirs
r3[2]=[ap+i for i in r3[2]] #files
if d:rls.extend(r3[1])
#
if r:
for i in r3[1]:rls.extend(list(i,r=r,d=d,f=f))
if f:rls.extend(r3[2])
if include:rls=[i for i in rls if include in i]
if exclude:rls=[i for i in rls if exclude not in i]
if print_result:
U.pprint(rls)
return rls
def save(file=None,lines=-1,tryExcept=False,out=False,columns=70,overide=True,del_other=False,out_max=9999,**ka):
'''file is str or (mod a)
在没有ipython实例时，不能get_ipython()
当file被指定时，overide 参数无效
#BUG #TODO
出现输入记录不同步的现象，应该添加一个报警
In [302]: _i265
Out[302]: '[i for i in sku if i[0] in [0.03, 0.04, 0.05, 0.06, 0.07, 0.18] ]'
In [303]: In[265]
Out[303]: 'page=pa=await tb.get_or_new_page(U.cbg(e=1))'
'''
del_other=U.get_duplicated_kargs(ka,'delOther','delete','delattr',default=del_other)
if ka:raise py.ArgumentError('没有处理这个ka，是不是多传了参数',ka)
try:
if py.isint(lines) and lines>0:
# lsta,lend=0,lines
lsta,lend=lines,-1
elif len(lines)==2:
lsta,lend=lines
lsta=lsta if lsta>0 else 0
else:raise Exception
except:
lsta,lend=0,gIn.__len__()
if file:#当指定file 名时，总是 overide
if T.istr(file):
file=T.filename(file)[:255-3]# 如果用 pathname 不好处理 含有 /斜杠的问题
if U.is_linux():
while py.len(file.encode('utf-8'))> 255-3: # 256-3 too long !
file=file[:-1]
file=F.autoPath(file,ext='.py',default=gsavePath)
#255-3(.py) 防止文件名过长 OSError: [Errno 22] Invalid argument: "\\\\?\\C:\\test
F.new(file)
if py.is2():file=open(file,'a')
else:file=open(file,'a',encoding="utf8")
elif py.isfile(file):
if file.mode!='a':raise Exception('file mode should be "a" ')
else:
raise Exception('invalid argument file')
else:
if gdTimeName and overide:
file=gdTimeName[py.list(gdTimeName.keys() )[-1]]#is3:TypeError: 'dict_keys' object does not support indexing
file=F.autoPath(file,ext='.py',default=gsavePath)
if py.is2():file=open(file,'w')
else:file=open(file,'w',encoding="utf8")
# last=-1
# for t in gdTimeName:
# if t>last:file,last=name,d
# last=gdTimeName.values()
# last.sort()#从小到大排序,ACS正序, DESC倒序 Ascending and Descending
# file=[n for n,d in gdTimeName.items() if d==last[-1]][0]
else:
file='{0}{1}.py'.format(gsavePath,U.stime())
if py.is2():file=open(file,'a')
else:file=open(file,'a',encoding="utf8")
U.pln(gshead,file=file)
####### get exec_lines to gsqgb
gsqgb=U.main(display=False)
#join( <traitlets.config.loader.LazyConfigValue at 0x20066295400> ) ### TypeError: can only join an iterable
gsexec_lines=U.getNestedValue(gipy.config,'InteractiveShellApp','exec_lines') #_class%20'traitlets.config.loader.LazyConfigValue'_.html
if not py.iterable(gsexec_lines):
gsexec_lines=[]
gsexec_lines=' ;'.join(gsexec_lines)
if gsexec_lines:
if not 'from qgb import' in gsexec_lines:
gsqgb='{0}\n{1}'.format(gsqgb,gsexec_lines)
else:gsqgb=gsexec_lines
U.pln(gsqgb+'\n',file=file )
# print >>file,'import sys;sys.path.append('{0}');from qgb import *'.format(gspath)
#using single quote for cmd line
#-4 为了去除 /qgb
#ipython --InteractiveShellApp.exec_lines=['%qp%'] 不会改变In[0],始终为''
for i,v in enumerate(gIn[lsta:lend]):
skip=False
if i==0 and lsta==0:continue
i=lsta+i
v=v.strip()
# U.isSyntaxError(u'_{0}={1}'.format(i,v) ) :
# pass
if i in gOut:
if i==1 and py.istr(gOut[1]) and gOut[1].endswith(':/QGB/babun/cygwin/lib/python2.7/'):
pass
else:
v=u'_{0}={1}'.format(i,v)
if U.isSyntaxError(v) or U.multin(gIgnoreIn,v):
# or u'from qgb import *' in v or sum(map(lambda a:v.startswith(a),gIgnoreStart) ):
v=u'#'+v
skip=True
if tryExcept and (not skip):
v='#########################\n\t'+v
if py.is2():v=gsTryExcept.format(i,v).encode('utf-8')
else:v=gsTryExcept.format(i,v)
U.p(v,file=file )
else:
if py.is2():v=v.encode('utf-8')
else:pass
U.p(v,' '*(columns-len(v.splitlines()[-1])),file=file )
if out and (i in gOut) and (not skip):
pout=pformat(gOut[i])
if py.len(pout) > out_max:
pout='#Warning# len(pout)={} > out_max'.format(py.len(pout))
print( '#ipy.save In[{}]{}# len(pout)={} > out_max'.format(i,gIn[i],py.len(pout)) )
U.pln(';"""#{0}'.format(i),file=file )
# U.pln('"""',file=file )
U.pln(pout,file=file )
U.pln('"""',file=file )
else:
U.pln('#',i,file=file )
# if i in [14]:import pdb;pdb.set_trace()#U.repl()
# gipy.magic(u'save save.py 0-115')
# U.pln(gIn
# U.pln(gOut.keys()
file.close()
gdTimeName[U.time()]=file.name
if del_other:
for t,f in gdTimeName.items():
if f == file.name:continue
print('del:',U.stime(t),F.delete(f) or [f,False])
return '{0} {1} success!'.format(save.name,file.name)