def do(self, i_args):
data = i_args['data']
res = ''
for errorRE in ErrorsRE:
if errorRE.search(data) is not None:
self.error = True
break
lines = data.split('\n')
for line in lines:
m = IMAGE_RE.match(line)
if m:
self.appendFile(m.group(1))
m = PDG_IMG_RE.search(line)
if m:
img = m.group(1)
if cgruutils.isImageExt(img):
self.appendFile(self.expandEnvVars(img))
percent_pos = data.find(PERCENT)
if percent_pos > -1:
ppos = data.find('%')
if ppos > -1:
try:
percent = int(data[percent_pos + PERCENT_len:ppos])
except: # TODO: too broad exception clause
pass
if -1 <= percent <= 100:
self.percentframe = percent
self.calculate()
# Search for mantra pear memory usage
for line in lines:
match = PeakMem_RE.match(line)
if match:
mem = cgruutils.memStrToBytes(match.group(1))
if mem:
mem = 'gb:{:.2f}'.format(float(mem) / (2**30))
self.mantra_mem = 'mantra_' + mem.strip('.0')
# Resources and host peak memory
if self.res_mem_peak_mb:
res += ' mem_peak_mb:%d' % self.res_mem_peak_mb
# Resources and mantra peak memory
if self.mantra_mem:
res += ' %s' % self.mantra_mem
# Resources and averarge CPU
if self.res_cpu_agv:
res += ' cpu_avg:%d' % self.res_cpu_agv
# Add resources to parser object if any
res = res.strip()
if len(res):
self.resources = res
def walkdir(i_path, i_subwalk, i_curdepth=0):
global Progress
global PrevFiles
global CurFiles
global ThumbFolderCount
global ReportFolderCount
global TotalSpace
# Output current path:
if Options.verbose > i_curdepth and i_subwalk:
outInfo('cur_path', i_path)
# Output report:
if Options.report is not None:
if i_path.find(os.path.dirname(Options.output)) == -1:
if ReportFolderCount % Options.report == 0:
print('REPORT: %s - %.1f GB' %
(i_path, TotalSpace / 1024 / 1024 / 1024))
sys.stdout.flush()
ReportFolderCount += 1
out = jsonLoad(os.path.join(i_path, Options.output))
if out is None:
out = dict()
checkDict(out, True)
try:
entries = os.listdir(i_path)
except:
outInfo('error_listdir', str(sys.exc_info()[1]))
return None
for entry in entries:
path = os.path.join(i_path, entry)
st = None
try:
st = os.lstat(path)
except:
outInfo('error_listdir', str(sys.exc_info()[1]))
continue
# We are not walking in links:
if stat.S_ISLNK(st.st_mode):
continue
if stat.S_ISDIR(st.st_mode):
out['num_folders'] += 1
out['num_folders_total'] += 1
size, space = getSizeSpace(st)
out['space'] += space
TotalSpace += space
fout = None
if i_subwalk:
# Recursively walk in a subfolder:
fout = walkdir(path, True, i_curdepth + 1)
else:
# Load previous walk data:
fout = jsonLoad(os.path.join(path, Options.output))
if fout is not None:
checkDict(fout)
# We do not need info for each subfolder in a child folder:
del fout['files']
del fout['folders']
# Create an empty folder entry if not preset:
if not 'folders' in out:
out['folders'] = dict()
if not isinstance(out['folders'], dict):
out['folders'] = dict()
if not entry in out['folders']:
out['folders'][entry] = dict()
# Merge keys from subfolder:
for key in fout.keys():
out['folders'][entry][key] = fout[key]
out['num_folders_total'] += fout['num_folders_total']
out['num_files_total'] += fout['num_files_total']
out['size_total'] += fout['size_total']
out['space'] += fout['space']
if stat.S_ISREG(st.st_mode):
CurFiles += 1
size, space = getSizeSpace(st)
if entry[0] != '.':
out['num_files'] += 1
if cgruutils.isImageExt(path):
if Options.thumb is not None:
if out['num_images'] == 0:
if ThumbFolderCount % Options.thumb == 0 and size < 10000000:
print('@IMAGE!@' + path)
sys.stdout.flush()
ThumbFolderCount += 1
out['num_images'] += 1
elif cgruutils.isMovieExt(path) and Options.mediainfo:
obj = mediainfo.processMovie(path)
if obj and 'mediainfo' in obj:
out['files'][entry] = obj['mediainfo']
out['num_files_total'] += 1
out['size_total'] += size
out['size'] += size
out['space'] += space
TotalSpace += space
# Just output progress:
if PrevFiles:
cur_progress = int(100.0 * CurFiles / PrevFiles)
if cur_progress != Progress:
Progress = cur_progress
outInfo('progress', 'PROGRESS: %d%%' % Progress)
# Skip soting data in .rules folders, or we will create '.rules/.rules' folders
if os.path.basename(i_path) == os.path.dirname(Options.output):
return out
# Store current walk data:
filename = os.path.join(i_path, Options.output)
if not os.path.isdir(os.path.dirname(filename)):
try:
os.makedirs(os.path.dirname(filename))
except:
outInfo('error_make_dir', str(sys.exc_info()[1]))
if os.path.isdir(os.path.dirname(filename)):
try:
with open(filename, 'w') as f:
json.dump(out, f, indent=1)
except:
outInfo('error_file_write', str(sys.exc_info()[1]))
return out
ThumbTime = 0
while True:
data = Process.stdout.readline()
if data is None: break
if len(data) < 1: break
sys.stdout.write(cgruutils.toStr(data))
if data.find(Key) != -1 and FilesTotal:
if data not in Files:
print('PROGRESS: %d%%' % int(100.0 * len(Files) / FilesTotal))
Files.append(data)
curtime = time.time()
if Options.thumbsec is not None and curtime - ThumbTime > Options.thumbsec:
data = data[data.find(Key):]
data = data[len(Key):]
data = data.strip(' \n\r')
data = data.split(' ')[0]
#data = os.path.abspath(data)
if cgruutils.isImageExt(data) and os.path.isfile(data):
print('@IMAGE!@' + data)
ThumbTime = curtime
sys.stdout.flush()
returncode = Process.wait()
sys.exit(returncode)
if root_basename.lower().find(skip) != -1:
to_skip = True
break
if to_skip:
if Options.verbose:
print('Skipping: "%s"' % root)
continue
if Options.verbose:
print('Scanning folder "%s"...' % root)
images = []
for afile in files:
if afile[0] in '._':
continue
if cgruutils.isImageExt(afile):
images.append(afile)
elif cgruutils.isMovieExt(afile) and not Options.nomovie:
new_movie = os.path.join(root, afile)
new_mtime = int(os.path.getmtime(new_movie))
if new_mtime > cur_mtime:
Movie = new_movie
cur_mtime = new_mtime
if len(images) == 0:
continue
new_mtime = int(os.path.getmtime(os.path.join(root, images[0])))
if new_mtime > cur_mtime:
Images = []
Movie = None
images.sort()
ThumbTime = 0
while True:
data = Process.stdout.readline()
if data is None: break
if len(data) < 1: break
sys.stdout.write(data)
if data.find(Key) != -1 and FilesTotal:
if data not in Files:
print('PROGRESS: %d%%' % int( 100.0 * len(Files) / FilesTotal))
Files.append( data)
curtime = time.time()
if Options.thumbsec is not None and curtime - ThumbTime > Options.thumbsec:
data = data[data.find(Key):]
data = data[len(Key):]
data = data.strip(' \n\r')
data = data.split(' ')[0]
#data = os.path.abspath(data)
if cgruutils.isImageExt(data) and os.path.isfile( data):
print('@IMAGE!@' + data)
ThumbTime = curtime
sys.stdout.flush()
returncode = Process.wait()
sys.exit(returncode)
def walkdir(i_path, i_subwalk, i_curdepth=0):
global Progress
global PrevFiles
global CurFiles
global ThumbFolderCount
if Options.verbose > i_curdepth and i_subwalk:
outInfo('cur_path',i_path)
out = dict()
checkDict(out)
try:
entries = os.listdir(i_path)
except:
outInfo('error_listdir',str(sys.exc_info()[1]))
return None
for entry in entries:
# Skip result folder (.rules):
if entry == os.path.dirname(Options.output):
continue
path = os.path.join(i_path, entry)
# We are not walking in links:
if os.path.islink(path):
continue
if os.path.isdir(path):
out['num_folders'] += 1
out['num_folders_total'] += 1
fout = None
if i_subwalk:
# Recursively walk in a subfolder:
fout = walkdir(path, True, i_curdepth + 1)
else:
# Load previous walk data:
fout = jsonLoad(os.path.join(path, Options.output))
if fout is not None:
checkDict(fout)
# We do not need info for each subfolder in a child folder:
del fout['files']
del fout['folders']
out['folders'][entry] = fout
out['num_folders_total'] += fout['num_folders_total']
out['num_files_total'] += fout['num_files_total']
out['size_total'] += fout['size_total']
if os.path.isfile(path):
CurFiles += 1
size = os.path.getsize(path)
if entry[0] != '.':
out['num_files'] += 1
if cgruutils.isImageExt( path):
if Options.thumb is not None:
if out['num_images'] == 0:
if ThumbFolderCount % Options.thumb == 0 and size < 10000000:
print('@IMAGE!@'+path)
sys.stdout.flush()
ThumbFolderCount += 1
out['num_images'] += 1
out['num_files_total'] += 1
out['size_total'] += size
out['size'] += size
# Just output progress:
if PrevFiles:
cur_progress = int(100.0 * CurFiles / PrevFiles)
if cur_progress != Progress:
Progress = cur_progress
outInfo('progress','PROGRESS: %d%%' % Progress)
# Store current walk data:
filename = os.path.join(i_path, Options.output)
if not os.path.isdir(os.path.dirname(filename)):
try:
os.makedirs(os.path.dirname(filename))
except:
outInfo('error_make_dir',str(sys.exc_info()[1]))
if os.path.isdir(os.path.dirname(filename)):
try:
with open(filename, 'w') as f:
json.dump(out, f, indent=1)
except:
outInfo('error_file_write',str(sys.exc_info()[1]))
return out
def walkdir(i_path, i_subwalk, i_curdepth=0):
global Progress
global PrevFiles
global CurFiles
global ThumbFolderCount
global ReportFolderCount
global TotalSpace
# Output current path:
if Options.verbose > i_curdepth and i_subwalk:
outInfo('cur_path',i_path)
# Output report:
if Options.report is not None:
if i_path.find( os.path.dirname( Options.output)) == -1:
if ReportFolderCount % Options.report == 0:
print('REPORT: %s - %.1f GB' % ( i_path, TotalSpace / 1024 / 1024 / 1024 ))
sys.stdout.flush()
ReportFolderCount += 1
out = jsonLoad( os.path.join( i_path, Options.output))
if out is None:
out = dict()
checkDict(out, True)
try:
entries = os.listdir(i_path)
except:
outInfo('error_listdir',str(sys.exc_info()[1]))
return None
for entry in entries:
path = os.path.join(i_path, entry)
st = None
try:
st = os.lstat( path)
except:
outInfo('error_listdir',str(sys.exc_info()[1]))
continue
# We are not walking in links:
if stat.S_ISLNK( st.st_mode):
continue
if stat.S_ISDIR( st.st_mode):
out['num_folders'] += 1
out['num_folders_total'] += 1
size, space = getSizeSpace( st)
out['space'] += space
TotalSpace += space
fout = None
if i_subwalk:
# Recursively walk in a subfolder:
fout = walkdir(path, True, i_curdepth + 1)
else:
# Load previous walk data:
fout = jsonLoad(os.path.join(path, Options.output))
if fout is not None:
checkDict(fout)
# We do not need info for each subfolder in a child folder:
del fout['files']
del fout['folders']
# Create an empty folder entry if not preset:
if not 'folders' in out:
out['folders'] = dict()
if not isinstance(out['folders'], dict):
out['folders'] = dict()
if not entry in out['folders']:
out['folders'][entry] = dict()
# Merge keys from subfolder:
for key in fout.keys():
out['folders'][entry][key] = fout[key]
out['num_folders_total'] += fout['num_folders_total']
out['num_files_total'] += fout['num_files_total']
out['size_total'] += fout['size_total']
out['space'] += fout['space']
if stat.S_ISREG( st.st_mode):
CurFiles += 1
size, space = getSizeSpace( st)
if entry[0] != '.':
out['num_files'] += 1
if cgruutils.isImageExt( path):
if Options.thumb is not None:
if out['num_images'] == 0:
if ThumbFolderCount % Options.thumb == 0 and size < 10000000:
print('@IMAGE!@'+path)
sys.stdout.flush()
ThumbFolderCount += 1
out['num_images'] += 1
elif cgruutils.isMovieExt( path) and Options.mediainfo:
obj = mediainfo.processMovie( path)
if obj and 'mediainfo' in obj:
out['files'][entry] = obj['mediainfo']
out['num_files_total'] += 1
out['size_total'] += size
out['size'] += size
out['space'] += space
TotalSpace += space
# Just output progress:
if PrevFiles:
cur_progress = int(100.0 * CurFiles / PrevFiles)
if cur_progress != Progress:
Progress = cur_progress
outInfo('progress','PROGRESS: %d%%' % Progress)
# Skip soting data in .rules folders, or we will create '.rules/.rules' folders
if os.path.basename(i_path) == os.path.dirname(Options.output):
return out
# Store current walk data:
filename = os.path.join(i_path, Options.output)
if not os.path.isdir(os.path.dirname(filename)):
try:
os.makedirs(os.path.dirname(filename))
except:
outInfo('error_make_dir',str(sys.exc_info()[1]))
if os.path.isdir(os.path.dirname(filename)):
try:
with open(filename, 'w') as f:
json.dump(out, f, indent=1)
except:
outInfo('error_file_write',str(sys.exc_info()[1]))
return out
if root_basename.lower().find(skip) != -1:
to_skip = True
break
if to_skip:
if Options.verbose:
print('Skipping: "%s"' % root)
continue
if Options.verbose:
print('Scanning folder "%s"...' % root)
images = []
for afile in files:
if afile[0] in '._':
continue
if cgruutils.isImageExt(afile):
images.append(afile)
elif cgruutils.isMovieExt(afile) and not Options.nomovie:
new_movie = os.path.join(root, afile)
new_mtime = int(os.path.getmtime(new_movie))
if new_movie > cur_mtime:
Movie = new_movie
cur_mtime = new_mtime
if len(images) == 0:
continue
new_mtime = int(os.path.getmtime(os.path.join(root, images[0])))
if new_mtime > cur_mtime:
Images = []
Movie = None
images.sort()
