def main():
setup()
if os.path.exists(args.database):
db = sqlite3.connect(args.database)
c = db.cursor()
c.execute("PRAGMA default_cache_size=900000")
c.execute("PRAGMA cache_size=900000")
c.execute("PRAGMA synchronous = OFF")
c.execute("PRAGMA journal_mode = OFF")
c.execute("PRAGMA locking_mode = EXCLUSIVE")
c.execute("PRAGMA temp_store = MEMORY")
c.execute("PRAGMA count_changes = OFF")
c.execute("PRAGMA PAGE_SIZE = 4096")
c.execute("PRAGMA compile_options")
c.execute('SELECT username,name,id FROM projects WHERE processed = 0 LIMIT %s' % (args.multi))
res = c.fetchall()
if len(res) == 0:
print 'nothing to analyize'
m_count = 0
for i in res:
my_path = os.path.join(args.folder,i[0],i[1])
for r,d,f in betterwalk.walk(my_path):
for file in f:
# POPULATE files table
#print file
c.execute("SELECT count(name) FROM files WHERE name = '%s'" % (file.replace("'", '')))
res = c.fetchall()
if res[0][0] == 0:
c.execute("INSERT INTO files (name) values ('%s')" % (file.replace("'", '')))
else:
c.execute("UPDATE files SET count = count + 1 WHERE name = '%s'" % (file.replace("'", '')))
for dir in d:
# POPULATE dirs table
#print dir
c.execute("SELECT count(name) FROM dirs WHERE name = '%s'" % (dir.replace("'", '')))
res = c.fetchall()
if res[0][0] == 0:
c.execute("INSERT INTO dirs (name) values ('%s')" % (dir.replace("'", '')))
else:
c.execute("UPDATE dirs SET count = count + 1 WHERE name = '%s'" % (dir.replace("'", '')))
m_count += 1
c.execute("UPDATE projects SET processed = '1' WHERE username = '******' AND name = '%s'" % (i[0], i[1]))
db.commit()
print '%s: Completed username: %s and project: %s' % (m_count, i[0],i[1])
try:
c.close()
except:
pass
db.close()
else:
print 'Database does not exist...'
sys.exit()
def tearDown(self):
# Tear everything down. This is a decent use for bottom-up on
# Windows, which doesn't have a recursive delete command. The
# (not so) subtlety is that rmdir will fail unless the dir's
# kids are removed first, so bottom up is essential.
for root, dirs, files in betterwalk.walk(self.testfn, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
dirname = os.path.join(root, name)
if not os.path.islink(dirname):
os.rmdir(dirname)
else:
os.remove(dirname)
os.rmdir(self.testfn)
def test_traversal(self):
# Build:
# TESTFN/
# TEST1/ a file kid and two directory kids
# tmp1
# SUB1/ a file kid and a directory kid
# tmp2
# SUB11/ no kids
# SUB2/ a file kid and a dirsymlink kid
# tmp3
# link/ a symlink to TESTFN.2
# TEST2/
# tmp4 a lone file
walk_path = os.path.join(self.testfn, "TEST1")
sub1_path = os.path.join(walk_path, "SUB1")
sub11_path = os.path.join(sub1_path, "SUB11")
sub2_path = os.path.join(walk_path, "SUB2")
tmp1_path = os.path.join(walk_path, "tmp1")
tmp2_path = os.path.join(sub1_path, "tmp2")
tmp3_path = os.path.join(sub2_path, "tmp3")
link_path = os.path.join(sub2_path, "link")
t2_path = os.path.join(self.testfn, "TEST2")
tmp4_path = os.path.join(self.testfn, "TEST2", "tmp4")
# Create stuff.
os.makedirs(sub11_path)
os.makedirs(sub2_path)
os.makedirs(t2_path)
for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path:
f = file(path, "w")
f.write("I'm " + path + " and proud of it. Blame test_os.\n")
f.close()
if hasattr(os, "symlink"):
os.symlink(os.path.abspath(t2_path), link_path)
sub2_tree = (sub2_path, ["link"], ["tmp3"])
else:
sub2_tree = (sub2_path, [], ["tmp3"])
# Walk top-down.
all = list(betterwalk.walk(walk_path))
self.assertEqual(len(all), 4)
# We can't know which order SUB1 and SUB2 will appear in.
# Not flipped: TESTFN, SUB1, SUB11, SUB2
# flipped: TESTFN, SUB2, SUB1, SUB11
flipped = all[0][1][0] != "SUB1"
all[0][1].sort()
self.assertEqual(all[0], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
self.assertEqual(all[1 + flipped], (sub1_path, ["SUB11"], ["tmp2"]))
self.assertEqual(all[2 + flipped], (sub11_path, [], []))
self.assertEqual(all[3 - 2 * flipped], sub2_tree)
# Prune the search.
all = []
for root, dirs, files in betterwalk.walk(walk_path):
all.append((root, dirs, files))
# Don't descend into SUB1.
if 'SUB1' in dirs:
# Note that this also mutates the dirs we appended to all!
dirs.remove('SUB1')
self.assertEqual(len(all), 2)
self.assertEqual(all[0], (walk_path, ["SUB2"], ["tmp1"]))
self.assertEqual(all[1], sub2_tree)
# Walk bottom-up.
all = list(betterwalk.walk(walk_path, topdown=False))
self.assertEqual(len(all), 4)
# We can't know which order SUB1 and SUB2 will appear in.
# Not flipped: SUB11, SUB1, SUB2, TESTFN
# flipped: SUB2, SUB11, SUB1, TESTFN
flipped = all[3][1][0] != "SUB1"
all[3][1].sort()
self.assertEqual(all[3], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
self.assertEqual(all[flipped], (sub11_path, [], []))
self.assertEqual(all[flipped + 1], (sub1_path, ["SUB11"], ["tmp2"]))
self.assertEqual(all[2 - 2 * flipped], sub2_tree)
if hasattr(os, "symlink"):
# Walk, following symlinks.
for root, dirs, files in betterwalk.walk(walk_path, followlinks=True):
if root == link_path:
self.assertEqual(dirs, [])
self.assertEqual(files, ["tmp4"])
break
else:
self.fail("Didn't follow symlink with followlinks=True")
def do_betterwalk():
for root, dirs, files in betterwalk.walk(path):
pass
def SearchFile(Path,Search_Condition,Search_Type):
#Create New DataFrame
columns1=['File Name', 'Path', 'Folder']
index1 = np.arange(30000)
df = pd.DataFrame(columns=columns1, index = index1)
#Search => None
if Search_Type == 'None':
i=(-1)
#Path=unicode(Path,'utf8')
for pathinfile, subdirs, files in betterwalk.walk(Path):
for name in files:
if Search_Condition in name:
i+=1
fullPath = os.path.join(pathinfile,name)
df.loc[i, 'Path']=fullPath
df.loc[i, 'File Name']=name
#drop N/A
df = df[(pd.notnull(df['File Name']))]
#Search => OR
if Search_Type == 'OR':
SearchORArr=Search_Condition.split(',')
i=(-1)
for pathinfile, subdirs, files in betterwalk.walk(Path):
for name in files:
ORresult = map(lambda x:re.findall(x,name),SearchORArr)
if not isListEmpty(ORresult):
i+=1
fullPath = os.path.join(pathinfile,name)
df.loc[i, 'Path']=fullPath
df.loc[i, 'File Name']=name
#drop N/A
df = df[(pd.notnull(df['File Name']))]
#Search => AND
if Search_Type == 'AND':
SearchANDArr=Search_Condition.split(',')
i=(-1)
for pathinfile, subdirs, files in betterwalk.walk(Path):
for name in files:
ANDresult = map(lambda x:re.findall(x,name),SearchANDArr)
if ANDCheckEmpty(ANDresult)== True:
i+=1
fullPath = os.path.join(pathinfile,name)
df.loc[i, 'Path']=fullPath
df.loc[i, 'File Name']=name
#drop N/A
df = df[(pd.notnull(df['File Name']))]
if df.empty:
return ('No Results')
os.chdir('//ecsbks01/swap/DDD00/virtualenv/WinPython-32bit-2.7.10.2/python-2.7.10/Project_Evaluate_Excel/Search_History')
#Search for files
#word1=Search_Condition.decode('utf-8')
#df['Search Result']=df['File Name'].str.contains(Search_Condition)
#result = df[(df['Search Result']==True)]
#search for files write into excel
write_df=df.loc[:,['File Name','Path']]
writer = ExcelWriter('Result-Output.xls')
write_df.to_excel(writer,'Result',index=False)
writer.save()
#turn search to files into hyperlink
CWPath = '\\\\ecsbks01\\swap\\DDD00\\virtualenv\\WinPython-32bit-2.7.10.2\\python-2.7.10\\Project_Evaluate_Excel\\Search_History'
Excel_Path = os.path.join(CWPath, 'Result-Output.xls')
wb = Workbook(Excel_Path)
wb = Workbook.caller()
checkArr = Range('B2').vertical.value
i = 2
for check in checkArr:
RangeName=('B%d' % (i))
displayRange=('A%d' % (i))
address=Range(RangeName).value
display_name = Range(displayRange).value
i+=1
try:
Range(RangeName).add_hyperlink(address, text_to_display=address)
except:
pass
return "FINISH"
try:
shutil.rmtree(MsgFolder)
except:
pass
os.remove(MailmsgPath)
newpath='C:\\temp_read_Excel\\'
directory_path='P:\\final_price'
if not os.path.exists(newpath):
os.makedirs(newpath)
fileNum=0
for pathinfile, subdirs, files in betterwalk.walk(directory_path):
for name in files:
fileEXT = os.path.splitext(name)[1] #filename extension
fileNum+=1
print fileNum , "exception"
#if the file is msg
if (fileEXT=='.msg') and fileNum>90:
MsgPath=os.path.join(pathinfile,name)
print pathinfile
try:
UnMsg(pathinfile,name)
except:
pass
#if the file is Excel
if (fileEXT == '.xls' or fileEXT== '.xlsx') and fileNum>90: #and fileNum<=355 and fileNum>=0:
#coding=utf-8
import os.path, time
import os, sys
import datetime
import shutil
import betterwalk
from Tkinter import *
Last_Time=0
CLast_Time=0
LocalPath ='//Ecs01//ddb00/final_price'
for pathinfile, subdirs, files in betterwalk.walk(LocalPath):
for TempFilePath in map(lambda x:os.path.join(pathinfile,x),files):
Create_Time = os.path.getctime(TempFilePath)
Modified_Time = os.path.getmtime(TempFilePath)
if Last_Time<Modified_Time and os.path.splitext(TempFilePath)[1]!='.db': #except thumb.db
Last_Time=Modified_Time
FileName = TempFilePath
if CLast_Time<Create_Time and os.path.splitext(TempFilePath)[1]!='.db': #except thumb.db
CLast_Time=Create_Time
CFileName = TempFilePath
Result = time.ctime(Last_Time)
CResult = time.ctime(CLast_Time)
if __name__ == "__main__":
root = Tk()
root.geometry('900x250+200+300')
root.configure(background ='#E2E2E2')
