def get_matching_excluded_paths(context_path: [str], excluded_paths: [str],
included_paths: [str],
exclude_gitignored_files: bool) -> [str]:
def abs_path_glob_including_snapshot(p):
return os.path.abspath(os.path.join(context_path, p))
def expand_if_dir(path):
if os.path.isdir(path):
# Return the dir as well as the files inside
return itertools.chain(
iter([path]),
glob2.iglob(os.path.join(path, '**'),
recursive=True,
include_hidden=True))
else:
return iter([path])
included_paths = set(
ip for p in included_paths
for ip in glob2.iglob(abs_path_glob_including_snapshot(p),
recursive=True,
include_hidden=True))
# Get the files inside the directories
included_paths = set(p for ip in included_paths for p in expand_if_dir(ip))
all_included_prefixes = set(
os.sep.join(ip.split(os.sep)[:n + 1]) for ip in included_paths
for n in range(len(ip.split(os.sep))))
# Expand the globs
excluded_paths = [
ep for p in excluded_paths
for ep in glob2.iglob(abs_path_glob_including_snapshot(p),
recursive=True,
include_hidden=True)
]
# Add the git ignored files
git_ignored_files = []
# A value of None for exclude_gitignored_files means "exclude if git is
# available"
use_git = exclude_gitignored_files or (exclude_gitignored_files is None
and is_git_present(context_path))
if use_git:
git_ignored_files = [abs_path_glob_including_snapshot('.git')] + \
get_ignored_git_files(context_path)
excluded_paths += git_ignored_files
excluded_and_not_included_paths = []
for ep in excluded_paths:
if ep in all_included_prefixes:
excluded_and_not_included_paths += (
p for p in expand_if_dir(ep) if p not in all_included_prefixes)
else:
excluded_and_not_included_paths.append(ep)
return [
p[len(os.path.abspath(context_path)) + 1:]
for p in excluded_and_not_included_paths
]
def main():
logger.info('started')
notDone=True
while notDone:
#anyone done?
for process,file in readers:
if not process.is_alive():
logger.info('{0}/{1} marked as done'.format(process,file))
readers.remove((process,file))
for afile in glob2.iglob(options.filemask):
#logger.debug('noticed file {0}'.format(afile))
#if we aren't reading a file
#spin up a process to read the file
activefiles=[]
for process,filename in readers:
activefiles.append(filename)
if afile not in activefiles:
logger.info('starting a reader for {0}'.format(afile))
readingProcess=Process(target=readCEFFile,args=(afile,),name="cef2mozdefReadFile")
readers.append((readingProcess,afile))
readingProcess.start()
#if we are reading a file no longer in the list of active files (rotated, etc),
#tell it's reader it's ok to stop.
for process,filename in readers:
if filename not in glob2.iglob(options.filemask):
logger.info('{0} no longer valid, stopping file reader'.format(filename))
process.terminate()
process.join()
#change this if you want it to stop (cronjob, or debugging), else it will run forever (ala /etc/init service)
#notDone=False
time.sleep(2)
logger.info('finished')
def build_trigger_pd():
run_num = [
int(num[0]) for file, num in (
glob2.iglob(f"{data_folder}/raw_dat/RUN_*/", with_matches=True))
]
run_num = [num for num in run_num if num > 106]
gemroc = []
run = []
subrun = []
trigger = []
time_end = []
for run_n in run_num:
for filename, sub in glob2.iglob(
f"{data_folder}/raw_dat//RUN_{run_n}/ACQ_log_*",
with_matches=True):
with open(filename, 'r') as filelog:
for line in filelog.readlines():
if "total" in line:
run.append(int(run_n))
subrun.append(int(sub[0]))
gemroc.append(int(line.split()[11]))
trigger.append(int(line.split()[-1]))
time_end.append(line.split("--")[0])
trigger_pd = pd.DataFrame({
"gemroc": gemroc,
"run": run,
"subrun": subrun,
"triggers": trigger,
"time": time_end
})
return trigger_pd
def directoryU():
global counter
images = list(glob2.iglob("static/img/*.*"))
path = images[counter]
Invoice_No = request.form['Invoice_No']
Invoice_Date = request.form['Invoice_Date']
Supplier_Name = request.form['Supplier_Name']
Amount = request.form['Amount']
CGST = request.form['CGST']
SGST = request.form['SGST']
GST_No_TVS = request.form['GST_No_TVS']
GST_No_SUPPLIER = request.form['GST_No_SUPPLIER']
PAN_No_SUPPLIER = request.form['PAN_No_SUPPLIER']
PAN_NO_TVS = request.form['PAN_NO_TVS']
PO_No = request.form['PO_No']
PO_Date = request.form['PO_Date']
sample = {
"Invoice_No": Invoice_No,
"Invoice_Date": Invoice_Date,
"Supplier_Name": Supplier_Name,
"Amount": Amount,
"CGST": CGST,
"SGST": SGST,
"GST_No_TVS": GST_No_TVS,
"GST_No_SUPPLIER": GST_No_SUPPLIER,
"PAN_No_SUPPLIER": PAN_No_SUPPLIER,
"PAN_NO_TVS": PAN_NO_TVS,
"PO_No": PO_No,
"PO_Date": PO_Date
}
#{"Invoice_No","Invoice_Date","Supplier_Name",Amount","CGST","SGST","GST_No_TVS","GST_No_SUPPLIER","PAN_No_SUPPLIER","PAN_NO_TVS","PO_No","PO_Date"}
print("In custom : ", sample)
Iname = path.split('/')[2].split('.')[0]
path = 'uploads/user_response/' + Iname
# if not os.path.isfile(path):
# os.mkdir(path)
#uuid = randint(0, 99)
book = xlwt.Workbook(encoding="utf-8")
sheet1 = book.add_sheet("sheet1")
colunm_count = 0
data = sample
source_photo = os.path.join(os.getcwd(), images[counter])
dest = app.config['FINAL_DEST_OF_PHOTOS']
shutil.move(source_photo, dest)
for title, value in data.items():
sheet1.write(0, colunm_count, title)
sheet1.write(1, colunm_count, value)
colunm_count += 1
file_name = path + ".xls" % ()
book.save(file_name)
images = list(glob2.iglob("static/img/*.*"))
if len(images) == 0:
return render_template("index.html")
counter = 0
return render_template("extract.html", image_path=images[counter])
def find_praat_test_files():
for dir in glob.iglob(os.path.join(PRAAT_TEST_BASE_DIR, "*", "")):
if os.path.basename(os.path.dirname(dir)) in PRAAT_TEST_IGNORE_SUBDIRS:
continue
for fn in glob.iglob(os.path.join(dir, "**", "*.praat"),
recursive=True):
if "_GUI_" in fn:
continue
rel_fn = os.path.relpath(fn, PRAAT_TEST_BASE_DIR)
yield pytest.param(fn, id=rel_fn)
def archive_files(from_pattern, archive_dir, to_pattern, batch_id):
"""
Archives files matching from_pattern and renames to to_pattern based on uid in a batch folder
:param from_pattern:
:param archive_dir
:param to_pattern:
:param batch_id:
:return:
"""
# place archived files in a timestamped folder
archive_folder = os.path.join(archive_dir, batch_id)
if not os.path.isdir(archive_folder):
os.mkdir(archive_folder)
os.mkdir(os.path.join(archive_folder, 'unsigned_certs'))
os.mkdir(os.path.join(archive_folder, 'signed_certs'))
os.mkdir(os.path.join(archive_folder, 'sent_txs'))
os.mkdir(os.path.join(archive_folder, 'receipts'))
os.mkdir(os.path.join(archive_folder, 'blockchain_certificates'))
archived_file_pattern = os.path.join(archive_folder, to_pattern)
[
shutil.move(filename,
models.convert_file_name(archived_file_pattern, uid))
for filename, (uid, ) in glob2.iglob(from_pattern, with_matches=True)
]
def sort_allrepeat(dir):
sort_dir = os.path.join(dir, 'sort')
if os.path.exists(sort_dir):
shutil.rmtree(sort_dir)
os.mkdir(sort_dir)
for path in glob2.iglob(dir + '/*.jpg'):
image = Image.open(path)
def extract():
images = list(glob2.iglob("static/img/*.*"))
if len(images) == 0:
return render_template("index.html")
print(images)
return render_template("extract.html", image_path=images[counter])
def elab_on_run_dict(self, run):
time_dict_start = {}
time_dict_end = {}
for filename, (subrun, ) in glob2.iglob(
self.path + "/RUN_{}/ACQ_log_*".format(run),
with_matches=True):
lenght_list = []
with open(filename, "r") as flog:
lines = flog.readlines()
if len(lines) != 0:
if "--" in lines[0]:
utc_time = time.strptime(lines[0].split("--")[0],
"%a %b %d %H:%M:%S %Y ")
start_time = timegm(utc_time)
else:
start_time = -1
if "--" in lines[-1]:
utc_time = time.strptime(lines[-1].split("--")[0],
"%a %b %d %H:%M:%S %Y ")
end_time = timegm(utc_time)
else:
end_time = start_time
time_dict_start[subrun] = start_time
time_dict_end[subrun] = end_time
else:
time_dict_start[subrun] = -1
time_dict_end[subrun] = -1
return time_dict_start, time_dict_end
def main():
logger.info('started')
notDone = True
while notDone:
#anyone done?
for process, file in readers:
if not process.is_alive():
logger.info('{0}/{1} marked as done'.format(process, file))
readers.remove((process, file))
for afile in glob2.iglob(options.filemask):
#if we aren't reading a file
#or at the end of the file
#spin up a process to read the file
activefiles = []
for process, filename in readers:
activefiles.append(filename)
if afile not in activefiles:
pygtail = Pygtail(afile, pretend=False)
if getsize(afile) > pygtail._offset:
logger.info('starting a reader for {0}'.format(afile))
readingProcess = Process(target=readCEFFile,
args=(afile, pygtail),
name="cef2mozdefReadFile")
readers.append((readingProcess, afile))
readingProcess.start()
#change this if you want it to stop (cronjob, or debugging), else it will run forever (ala /etc/init service)
#notDone=False
time.sleep(2)
logger.info('finished')
def app_css(template_reference):
"""Generate app css link tags for Flask app."""
link_tags = ""
# Excluded css libs
for ex_lib in G.css_config['excluded_libs']:
if ex_lib.startswith('http'):
link_tags += link(ex_lib, absolute=True)
else:
link_tags += link(ex_lib)
# if False:
if G.debug:
# libs
link_tags += ''.join(map(link, G.css_config['libs']))
# layout
for layout_path in G.css_config['layout']:
# 支持通配符
if '*' in layout_path:
for path in glob2.iglob(
os.path.join(G.static_path, layout_path)):
relative_path = path.split(G.static_path)[1][1:]
link_tags += link(relative_path)
else:
link_tags += link(layout_path)
# page
template_name = _get_template_name(template_reference)
page_css_path = os.path.join(G.css_config['page'],
template_name.replace('html', 'css'))
link_tags += link(page_css_path)
else:
link_tags += link(APP_CSS)
return Markup(link_tags)
def MAE_Value(resDir, gtDir):
p = parameter()
gtThreshold = p['gtThreshold']
gtImgs = glob2.iglob(gtDir + '/*.png')
MAE = []
for gtName in gtImgs:
dir, name = os.path.split(gtName)
mapName = os.path.join(resDir, name)
if len(name) > 7:
continue
#if method!='CGL' and method!='MFSR' and method!='TSAA'and method!='PDNet' and method!='MFCF' and method!='Ours':
#mapName = mapName.replace('.png', '_' + 'RT_' + '.png')
curMap = im2double(cv2.imread(mapName, cv2.IMREAD_GRAYSCALE))
curGT = im2double(cv2.imread(gtName, cv2.IMREAD_GRAYSCALE))
curGT = (curGT >= gtThreshold).astype(np.float32)
if curMap.shape[0] != curGT.shape[0]:
curMap = cv2.resize(curMap, (curGT.shape[1], curGT.shape[0]))
diff = np.abs(curMap - curGT)
MAE.append(np.mean(diff))
return np.mean(MAE)
def scan_keys(keys_dir=KEYS_DIR):
keys_map = KeysMap()
patterns = ['Key-6.dat', '*.jks', '*.zs2', '*.ZS2']
if glob2:
files = sum(
(list(glob2.iglob(os.path.join(KEYS_DIR, '**/%s' % pattern)))
for pattern in patterns), [])
else:
files = sum((list(
glob.iglob(os.path.join(KEYS_DIR, '**/%s' % pattern),
recursive=True)) for pattern in patterns), [])
log.info('Keys (%s) in %s', len(files), keys_dir)
for filename in files:
filename = os.path.abspath(filename)
if filename not in keys_map.values():
log.info('Checking new key %s', filename)
cabinet = Cabinet()
try:
inn, fio, expires = cabinet.pre_login_cert(filename)
except Exception as e:
log.exception('Error occured on key processing %s %s',
filename, repr(e))
if DEBUG:
import pdb
pdb.set_trace() # noqa
continue
finally:
cabinet.quit()
log.info('Adding key inn=%s fio=%s expires=%s filename=%s', inn,
fio, expires, filename)
keys_map.add_key(inn, fio, filename, expires)
def main():
logger.info('started')
notDone=True
while notDone:
#anyone done?
for process,file in readers:
if not process.is_alive():
logger.info('{0}/{1} marked as done'.format(process,file))
readers.remove((process,file))
for afile in glob2.iglob(options.filemask):
#if we aren't reading a file
#or at the end of the file
#spin up a process to read the file
activefiles=[]
for process,filename in readers:
activefiles.append(filename)
if afile not in activefiles:
pygtail = Pygtail(afile,pretend=False)
if getsize(afile)> pygtail._offset:
logger.info('starting a reader for {0}'.format(afile))
readingProcess=Process(target=readCEFFile,args=(afile,pygtail),name="cef2mozdefReadFile")
readers.append((readingProcess,afile))
readingProcess.start()
#change this if you want it to stop (cronjob, or debugging), else it will run forever (ala /etc/init service)
#notDone=False
time.sleep(2)
logger.info('finished')
def elab_on_run(self, run):
rname = self.elabpath + "{}".format(run)
rname = rname + '/time_run_{}.root'.format(run)
rootFile = R.TFile(rname, 'recreate')
tree = R.TTree('tree', '')
tree_struct = R.TreeStruct()
tree.Branch('runNo', R.AddressOf(tree_struct, 'runNo'), 'runNo/I')
tree.Branch('subRunNo', R.AddressOf(tree_struct, 'subRunNo'),
'subRunNo/I')
tree.Branch('triggers', R.AddressOf(tree_struct, 'triggers'),
'triggers/I')
tree.Branch('start_time', R.AddressOf(tree_struct, 'start_time'),
'start_time/I')
tree.Branch('end_time', R.AddressOf(tree_struct, 'end_time'),
'end_time/I')
tree_struct.runNo = int(run)
for filename, (subrun, ) in glob2.iglob(
self.path + "/RUN_{}/ACQ_log_*".format(run),
with_matches=True):
tree_struct.subRunNo = int(subrun)
lenght_list = []
with open(filename, "r") as flog:
lines = flog.readlines()
for line in lines:
if "packets" in line:
lenght_list.append(
int(line.split("TM.dat, total packets= ")[1]))
if len(lines) != 0:
if "--" in lines[0]:
utc_time = time.strptime(lines[0].split("--")[0],
"%a %b %d %H:%M:%S %Y ")
start_time = timegm(utc_time)
else:
start_time = -1
if "--" in lines[-1]:
utc_time = time.strptime(lines[-1].split("--")[0],
"%a %b %d %H:%M:%S %Y ")
end_time = timegm(utc_time)
else:
end_time = start_time
tree_struct.start_time = start_time
tree_struct.end_time = end_time
else:
tree_struct.start_time = -1
tree_struct.end_time = -1
lenght_list = np.asarray(lenght_list)
if len(lenght_list) > 0 and (np.count_nonzero(lenght_list)
): #interrotto da timeout, con dati
tree_struct.triggers = int(
np.min(lenght_list[np.nonzero(lenght_list)]))
else:
tree_struct.triggers = -1
tree.Fill()
rootFile.Write()
rootFile.Close()
def _lookup_savegame_path(self, name, pathname):
path = self.configurator.settings.get(name, 'path', fallback=None)
if not path or not os.path.exists(path):
path = os.path.normpath(
os.path.expandvars(
os.path.expanduser(pathname)))
if not os.path.isabs(path):
for search_path in self.search_paths:
glob_pattern = os.path.join(
os.path.normpath(search_path), path)
glob_path = list(itertools.islice(
glob2.iglob(glob_pattern), 1))
if glob_path:
path = glob_path[0]
break
if os.path.exists(path):
if not self.configurator.settings.has_section(name):
self.configurator.settings.add_section(name)
self.configurator.settings.set(name, 'path', path)
self.configurator.save_settings_file()
else:
path = None
# TODO logging
# raise RuntimeError(
# f"Savegame path could not be found: {pathname}")
return path
def in_valid_paths(self, root, path, valid_paths):
for valid_path in [p for p in valid_paths if isinstance(p, str)]:
if path in list(map(normalize_path, glob(valid_path))):
return True
for valid_path in [p for p in valid_paths if not isinstance(p, str)]:
for nested_valid_path in valid_path:
for found_nested_path, matches in iglob(nested_valid_path,
with_matches=True):
found_nested_path = normalize_path(found_nested_path)
if found_nested_path == path:
# check matches
matches = map(normalize_path, matches)
for match in matches:
for related_path in valid_path:
if related_path != found_nested_path:
related_path = related_path.replace(
'*', match, 1)
if not os.path.isfile(related_path):
rel_path = normalize_path(
os.path.relpath(path, root))
rel_related_path = normalize_path(
os.path.relpath(
related_path, root))
raise ValidationError(
'{file} missing related file {related}'
.format(file=rel_path,
related=rel_related_path))
return True
raise ValidationError('{file} is not allowed'.format(file=path))
def MAE_Value(resDir, gtDir):
p = parameter()
gtThreshold = p['gtThreshold']
gtImgs = glob2.iglob(gtDir + '/*.jpg')
MAE = []
for gtName in gtImgs:
dir, name = os.path.split(gtName)
mapName = os.path.join(resDir, name[:-4] + '.png') ######
#print(mapName)
if os.path.exists(mapName) is False:
mapName = mapName.replace('.png', '.jpg')
if os.path.exists(mapName) is False:
mapName = mapName.replace('.jpg', '.bmp')
curMap = im2double(cv2.imread(mapName, cv2.IMREAD_GRAYSCALE))
curGT = im2double(cv2.imread(gtName, cv2.IMREAD_GRAYSCALE))
curGT = (curGT >= gtThreshold).astype(np.float32)
if curMap.shape[0] != curGT.shape[0]:
curMap = cv2.resize(curMap, (curGT.shape[1], curGT.shape[2]))
diff = np.abs(curMap - curGT)
MAE.append(np.mean(diff))
return np.mean(MAE)
def preview(self, request, pk=None):
job = self.get_object()
ips = job.rule.information_packages.all()
files = []
for ip in ips:
datadir = os.path.join(ip.policy.cache_storage.value,
ip.object_identifier_value)
for pattern, spec in six.iteritems(job.rule.specification):
for path in iglob(datadir + '/' + pattern):
if os.path.isdir(path):
for root, dirs, files in walk(path):
rel = os.path.relpath(root, datadir)
for f in files:
files.append({
'ip': ip.object_identifier_value,
'document': os.path.join(rel, f)
})
elif os.path.isfile(path):
rel = os.path.relpath(path, datadir)
files.append({
'ip': ip.object_identifier_value,
'document': rel
})
return Response(files)
def buildListOfFiles(searchGlob):
"""
Build a master list of all files that we should check.
"""
return [
fpath for fpath in glob2.iglob(searchGlob) if os.path.isfile(fpath)
]
def convertpdftoimage():
for i, pages in enumerate(glob2.iglob('uploads/*.pdf')):
print(pages)
pg = pdf2image.convert_from_path(pages, 300)
os.remove(pages)
for j in range(len(pg)):
pg[j].save('uploads/skp{}{}.jpg'.format(i, j), 'JPEG')
def files(self):
"""
a generator to return all files watched by this dog
"""
for path in self.paths:
for file in glob2.iglob(path):
yield file
def dec_run_header_trailer(self):
"""
Decodes one run in parallel
:return:
"""
input_list = []
return_list_hd = []
return_list_tr = []
return_list_udp = []
self.decoder = pl_lib.decoder(1, self.run_number, downsamplig=self.downsampling)
if not self.silent:
print ("Decoding")
for filename ,(subrun,gemroc) in glob2.iglob(self.data_folder+"/raw_dat/RUN_{}/SubRUN_*_GEMROC_*_TM.dat".format(self.run_number), with_matches=True):
input_list.append((filename, int(subrun), int(gemroc)))
if len(input_list)>0:
with Pool(processes=self.cpu_to_use) as pool:
with tqdm(total=len(input_list), disable=self.silent) as pbar:
for i, x in enumerate(pool.imap_unordered(self.decode_on_file_header_trailer, input_list)):
return_list_hd.append(x[0])
return_list_tr.append(x[1])
return_list_udp.append(x[2])
pbar.update()
header_pd = pd.concat(return_list_hd)
trailer_pd = pd.concat(return_list_tr)
UDP_pd = pd.concat(return_list_udp)
header_pd.to_pickle(os.path.join(self.data_folder,"raw_root",f"{self.run_number}","header_pd.gzip"), compression ="gzip")
trailer_pd.to_pickle(os.path.join(self.data_folder,"raw_root",f"{self.run_number}","trailer_pd.gzip"), compression ="gzip")
UDP_pd.to_pickle(os.path.join(self.data_folder,"raw_root",f"{self.run_number}","UDP_pd.gzip"), compression ="gzip")
else:
print (f"Can't find any .dat file in {self.data_folder}/raw_dat/RUN_{self.run_number}")
def get_features(paths, **kwargs):
"""Get features for given paths.
:param list paths: `list` of paths (file or dirs)
:return: `list` of `Feature` objects.
"""
seen_names = set()
features = []
for path in paths:
if path not in seen_names:
seen_names.add(path)
if op.isdir(path):
features.extend(
get_features(
glob2.iglob(op.join(path, "**", "*.feature")),
**kwargs
)
)
else:
base, name = op.split(path)
feature = Feature.get_feature(base, name, **kwargs)
features.append(feature)
features.sort(key=lambda feature: feature.name or feature.filename)
return features
def app_css(template_reference):
"""Generate app css link tags for Flask app."""
link_tags = ""
# Excluded css libs
for ex_lib in G.css_config['excluded_libs']:
if ex_lib.startswith('http'):
link_tags += link(ex_lib, absolute=True)
else:
link_tags += link(ex_lib)
# if False:
if G.debug:
# libs
link_tags += ''.join(map(link, G.css_config['libs']))
# layout
for layout_path in G.css_config['layout']:
# 支持通配符
if '*' in layout_path:
for path in glob2.iglob(os.path.join(G.static_path, layout_path)):
relative_path = path.split(G.static_path)[1][1:]
link_tags += link(relative_path)
else:
link_tags += link(layout_path)
# page
template_name = _get_template_name(template_reference)
page_css_path = os.path.join(G.css_config['page'],
template_name.replace('html', 'css'))
link_tags += link(page_css_path)
else:
link_tags += link(APP_CSS)
return Markup(link_tags)
def get_features(paths, **kwargs):
"""Get features for given paths.
:param list paths: `list` of paths (file or dirs)
:return: `list` of `Feature` objects.
"""
seen_names = set()
features = []
for path in paths:
if path not in seen_names:
seen_names.add(path)
if op.isdir(path):
features.extend(
get_features(
glob2.iglob(op.join(path, "**", "*.feature")),
**kwargs
)
)
else:
base, name = op.split(path)
feature = Feature.get_feature(base, name, **kwargs)
features.append(feature)
features.sort(key=lambda feature: feature.name or feature.filename)
return features
def process_data(partition):
""" Reads audio waveform and transcripts from a dataset partition
and generates mfcc featues.
Args:
parition - represents the dataset partition name.
Returns:
feats: dict containing mfcc feature per utterance
transcripts: dict of lists representing transcript.
utt_len: dict of ints holding sequence length of each
utterance in time frames.
"""
feats = {}
transcripts = {}
utt_len = {} # Required for sorting the utterances based on length
for filename in glob2.iglob(partition + '/**/*.txt'):
with open(filename, 'r') as f:
for line in f:
parts = line.split()
audio_file = parts[0]
file_path = os.path.join(os.path.dirname(filename),
audio_file + '.flac')
audio, sample_rate = sf.read(file_path)
feats[audio_file] = compute_mfcc(audio, sample_rate)
utt_len[audio_file] = feats[audio_file].shape[0]
target = ' '.join(parts[1:])
transcripts[audio_file] = [CHAR_TO_IX[i] for i in target]
return feats, transcripts, utt_len
def extract_num_triggers_run(data_raw_folder, run_number):
return_dict = {}
for filename, subrun in glob2.iglob("{}/RUN_{}/ACQ_log_*".format(
data_raw_folder, run_number),
with_matches=True):
return_dict[subrun[0]] = int(extract_num_triggers_subrun(filename))
return return_dict
def add_patterns(self, patterns):
for p in patterns:
for log_file in glob2.iglob(p):
try:
self.add_available(log_file)
except LogError, e:
logger.warning(e)
def _process(self):
for source_path in iglob(os.path.join(self.source, '**')):
path = os.path.relpath(source_path, self.source)
target_path = os.path.join(self.target, path)
try:
target_time = os.path.getmtime(target_path)
except OSError:
target_time = 0
try:
self.timestamp = os.path.getmtime(source_path)
if self.timestamp == target_time:
raise ValueError
item = next(x for x in self.handlers if x['regex'].match(path))
_in = open(source_path, 'rb')
except(OSError, StopIteration, ValueError):
continue
for _filter in item['filters']:
out = BytesIO()
_filter(_in, out, path)
_in = out
_in.seek(0)
for action in item['actions']:
out = BytesIO()
target_path = action(_in, out, target_path)
_in.seek(0)
out.seek(0)
if target_path:
self._write(out, target_path)
_in.close()
def merg_dec_func(self, subrun ):
pd_list = []
for filename, (gemroc) in glob2.iglob(self.data_folder + "/raw_root/{}/SubRUN_{}_GEMROC_*_TM.pickle.gzip".format(self.run_number, subrun), with_matches=True):
pd_list.append(pd.read_pickle(filename, compression="gzip"))
if len(pd_list)>0:
subrun_pd = pd.concat(pd_list, ignore_index=True)
subrun_pd.to_pickle(self.data_folder + "/raw_root/{}/Sub_RUN_dec_{}.pickle.gzip".format(self.run_number, subrun), compression="gzip")
def find_praat_test_files():
for dir in glob.iglob(os.path.join(PRAAT_TEST_BASE_DIR, "*", "")):
if os.path.basename(os.path.dirname(dir)) in PRAAT_TEST_IGNORE_SUBDIRS:
continue
for fn in glob.iglob(os.path.join(dir, "**", "*.praat"),
recursive=True):
rel_fn = os.path.relpath(fn, PRAAT_TEST_BASE_DIR)
marks = []
if rel_fn == os.path.join("num", "mean.praat"):
marks.append(
pytest.mark.xfail(
sys.platform == 'win32',
reason=
"`long double` is only double precision when compiling with MSVC, causing `assert mean ({ -1e18, 3, 1e18 }) = 1` to fail"
))
yield pytest.param(fn, id=rel_fn, marks=marks)
def import_raw_hands(hands_dir):
for hands_filepath in glob2.iglob(os.path.join(hands_dir, '*.txt')):
with open(hands_filepath, 'rt', encoding='latin-1') as hands_file:
print(hands_filepath)
file_text = replace_euro_symbol(hands_file.read())
for raw_hand in file_text.split('\n\n\n\n\n'):
if raw_hand and is_valid_hand(raw_hand):
yield raw_hand
def glob(self, basedir, expr):
"""Runs when a glob expression needs to be resolved.
"""
expr = path.join(basedir, expr)
for filename in glob.iglob(expr):
if path.isdir(filename):
continue
yield filename
def extract_num_triggers_run(self, run_number):
tot = 0
for filename, subrun in glob2.iglob(
"/{}/raw_dat/RUN_{}/ACQ_log_*".format(self.data_folder,
run_number),
with_matches=True):
tot += int(extract_num_triggers_subrun(filename))
return tot
def glob(self, basedir, expr):
"""Runs when a glob expression needs to be resolved.
"""
expr = path.join(basedir, expr)
for filename in glob.iglob(expr):
if path.isdir(filename):
continue
yield filename
def start_encrypt(key):
#Encrypts all files recursively starting from startPath
for filename in glob2.iglob(startPath):
if (os.path.isfile(filename)):
#print('Encrypting> ' + filename)
if (os.path.splitext(filename)[1] != extension):
encrypt_file(key, filename)
os.remove(filename)
def migrate_tests():
"""Migrate outdated tests to the most recent form."""
if len(sys.argv) != 2:
print 'Usage: pytestbdd_migrate_tests <path>'
sys.exit(1)
path = sys.argv[1]
for file_path in glob2.iglob(os.path.join(os.path.abspath(path), '**', '*.py')):
migrate_tests_in_file(file_path)
def add_patterns(self, patterns):
for p in patterns:
for log_file in glob2.iglob(p):
try:
self.add_available(log_file)
except LogError as e:
logger.warning(e)
logger.info("Added %d log file(s)", len(self.available))
def glob(self, basedir, expr):
"""Generator that runs when a glob expression needs to be
resolved. Yields a list of absolute filenames.
"""
expr = path.join(basedir, expr)
for filename in glob.iglob(expr):
if path.isdir(filename):
continue
yield filename
def _iglob_within(self, root, pathname):
try:
from glob2 import iglob
except ImportError:
from glob import iglob
import os
for path in iglob(os.path.join(root, pathname)):
yield path.replace(root, '').lstrip('/')
def prepare_issuance_batch(unsigned_certs_dir, signed_certs_dir, blockchain_certs_dir, work_dir,
file_extension=JSON_EXT):
"""
Prepares file system for issuing a batch of certificates. Copies inputs to work_dir, and ensures
that all output dirs required for processing the batch exist.
:param unsigned_certs_dir: input certificates
:param signed_certs_dir: output dir
:param blockchain_certs_dir: output dir
:param work_dir: work dir
:return:
"""
# create work dir if it doesn't already exist
os.makedirs(work_dir, exist_ok=True)
# create final output dirs if they don't already exist
os.makedirs(blockchain_certs_dir, exist_ok=True)
os.makedirs(signed_certs_dir, exist_ok=True)
# ensure previous processing state, if any, is cleaned up
for item in os.listdir(work_dir):
file_path = os.path.join(work_dir, item)
if os.path.isdir(file_path):
shutil.rmtree(file_path)
# define work subdirs
unsigned_certs_work_dir = os.path.join(work_dir, UNSIGNED_CERTIFICATES_DIR)
signed_certs_work_dir = os.path.join(work_dir, SIGNED_CERTIFICATES_DIR)
blockchain_certs_work_dir = os.path.join(work_dir, BLOCKCHAIN_CERTIFICATES_DIR)
# copy input certs to unsigned certs work subdir and create output subdirs
shutil.copytree(unsigned_certs_dir, unsigned_certs_work_dir)
os.makedirs(signed_certs_work_dir, exist_ok=True)
os.makedirs(blockchain_certs_work_dir, exist_ok=True)
cert_info = collections.OrderedDict()
input_file_pattern = str(os.path.join(unsigned_certs_work_dir, '*' + file_extension))
matches = glob2.iglob(input_file_pattern, with_matches=True)
if not matches:
logging.warning('No certificates to process')
raise NoCertificatesFoundError('No certificates to process')
# create certificate metadata for each certificates
for filename, (uid,) in sorted(matches):
certificate_metadata = CertificateMetadata(uid=uid,
unsigned_certs_dir=unsigned_certs_work_dir,
signed_certs_dir=signed_certs_work_dir,
blockcerts_dir=blockchain_certs_work_dir,
final_blockcerts_dir=blockchain_certs_dir,
file_extension=file_extension)
cert_info[uid] = certificate_metadata
logging.info('Processing %d certificates', len(cert_info))
return cert_info
def eglob(path, exclude=None):
"""Like glob.glob, but supports "/path/**/{a,b,c}.txt" lookup"""
fi = itertools.chain.from_iterable
paths = list(fi(glob2.iglob(d) for d in expand_paths(path)))
if exclude:
cached_regex = cached_regices.get(exclude, None)
if not cached_regex:
cached_regex = cached_regices[exclude] = re.compile(exclude)
paths = [x for x in paths if not cached_regex.search(x)]
return paths
def match(patterns):
for pattern in patterns:
if os.path.isfile(pattern):
yield pattern
elif os.path.isdir(pattern):
for path, names, filenames in os.walk(pattern):
for filename in filenames:
yield os.path.join(path, filename)
else:
for filename in glob2.iglob(pattern):
yield filename
def add_patterns(self, patterns):
i = 0
for p in patterns:
for log_file in glob2.iglob(p):
try:
self.add_available(log_file)
i += 1
except LogError as e:
logger.warning(e)
logger.info('Added %d log file(s)', i)
return i
def list(self, path):
for root in self.locations:
for absolute_path in glob2.iglob(os.path.join(root, path)):
if os.path.isfile(absolute_path):
logical_path = os.path.relpath(absolute_path, root)
# Filter out any files that don't end in the specified extensions
if not self.ext_pattern.search(absolute_path):
continue
# Filter out ignored paths
if self.ignore_pattern.search(absolute_path):
continue
yield logical_path, absolute_path
def find_unsigned_certificates(app_config):
cert_info = collections.OrderedDict()
for filename, (uid,) in sorted(glob2.iglob(
app_config.unsigned_certs_file_pattern, with_matches=True)):
with open(filename) as cert_file:
cert_raw = cert_file.read()
cert_json = json.loads(cert_raw)
certificate_metadata = make_certificate_metadata(app_config,
uid,
cert_json['recipient']['publicKey'])
cert_info[uid] = certificate_metadata
return cert_info
def cp(sources, dest):
"""Copy a file or a list of files.
Copy one or more files. Also is possible copy directories. There is
one different between this two sources, `some/folder` and
`/some/folder/*`. In the first case the folder is copied to the
destination, in the second case the files inside the directory are
copied, but not the directory itself.
"""
logging.info('-----')
for source in _as_list(sources):
for path in iglob(source):
copy_rec(path, dest)
def glob(self, expr):
"""Globs the given expression with the base path of the module.
This uses the glob2 module and supports recursive globs ('**/*').
Args:
expr: Glob expression.
Returns:
A list of all files that match the glob expression.
"""
if not expr or not len(expr):
return []
base_path = os.path.dirname(self.path)
glob_path = os.path.join(base_path, expr)
return list(glob2.iglob(glob_path))
def find_unsigned_certificates(app_config):
cert_info = {}
for filename, (uid,) in glob2.iglob(app_config.unsigned_certs_file_pattern, with_matches=True):
with open(filename) as cert_file:
cert_raw = cert_file.read()
cert_json = json.loads(cert_raw)
certificate_metadata = CertificateMetadata(app_config,
uid,
cert_json['recipient']['givenName'] + ' ' +
cert_json['recipient'][
'familyName'],
cert_json['recipient']['pubkey'])
cert_info[uid] = certificate_metadata
return cert_info
def make_archive(self, task, test_dir):
# Return early if no test_dir is specified
if test_dir is None:
return None
# Return early if there are no globs specified
if task.task.artifact_archive_globs is None or len(task.task.artifact_archive_globs) == 0:
return None
all_matched = set()
total_size = 0
for g in task.task.artifact_archive_globs:
try:
matched = glob2.iglob(test_dir + "/" + g)
for m in matched:
canonical = os.path.realpath(m)
if not canonical.startswith(test_dir):
LOG.warn("Glob %s matched file outside of test_dir, skipping: %s" % (g, canonical))
continue
total_size += os.stat(canonical).st_size
all_matched.add(canonical)
except Exception as e:
LOG.warn("Error while globbing %s: %s" % (g, e))
if len(all_matched) == 0:
return None
max_size = 200*1024*1024 # 200MB max uncompressed size
if total_size > max_size:
# If size exceeds the maximum size, upload a zip with an error message instead
LOG.info("Task %s generated too many bytes of matched artifacts (%d > %d)," \
+ "uploading archive with error message instead.",
task.task.get_id(), total_size, max_size)
archive_buffer = cStringIO.StringIO()
with zipfile.ZipFile(archive_buffer, "w") as myzip:
myzip.writestr("_ARCHIVE_TOO_BIG_",
"Size of matched uncompressed test artifacts exceeded maximum size" \
+ "(%d bytes > %d bytes)!" % (total_size, max_size))
return archive_buffer
# Write out the archive
archive_buffer = cStringIO.StringIO()
with zipfile.ZipFile(archive_buffer, "w", zipfile.ZIP_DEFLATED) as myzip:
for m in all_matched:
arcname = os.path.relpath(m, test_dir)
while arcname.startswith("/"):
arcname = arcname[1:]
myzip.write(m, arcname)
return archive_buffer
def _get_mtimes(paths):
files = {}
for path in paths:
if '*' not in path and not os.path.exists(path):
logging.error("WARNING, file not found: '{}'".format(path))
if os.path.isdir(path):
path = '{}/{}'.format(path.rstrip('/'), '**')
for file_path in iglob(path):
if os.path.isfile(file_path):
files[file_path] = os.path.getmtime(file_path)
return files
def main():
'''For each file matching our filemask
read in lines until we've got a complete slapd connection/termination
parse out the interesting transactions and move on to the next
sleeping in between to be nice to our cpu for really large files
'''
for afile in glob2.iglob(options.filemask):
#cache list of log lines to help deal with the multi-line slapd format
linecache=[]
#defaults
ipAddress='0.0.0.0'
bindName='Unknown'
errCode=None
errName='Unknown'
#dictionary to store bits of bind transactions as we find them in the multi-line logs
bindDetails={}
if exists(afile): #sometimes files can move/archive while we iterate the list
#have pygtail feed us lines without incrementing the offset until we've posted any results we've found.
pt = Pygtail(afile,pretend=True)
for line in pt:
if 'slapd' in line and ('ACCEPT' in line or 'RESULT' in line or 'BIND' in line ):
#store lines until we match a multi line slapd connection structure
#ACCEPT yields the source IP, BIND yields the dn [email protected], RESULT yields the success/failure
if len(linecache)>options.cachelength:
linecache.remove(linecache[0])
linecache.append(line)
searchforBind(line,bindDetails)
if 'RESULT' in line:
#maybe it's the termination of an earlier bind attempt
searchforBindResult(line,bindDetails,linecache)
postBindResults(bindDetails,pt,linecache)
trimBindDetails(bindDetails,linecache)
sleep(.00001) #be nice, but not too nice
#post any remaining bindDetails
postBindResults(bindDetails,pt,linecache,True)
def get_features(paths):
"""Get features for given paths.
:param paths: `list` of paths (file or dirs)
:return: `list` of `Feature` objects
"""
seen_names = set()
features = []
for path in paths:
if path in seen_names:
continue
seen_names.add(path)
if os.path.isdir(path):
features.extend(get_features(glob2.iglob(os.path.join(path, '**', '*.feature'))))
else:
base, name = os.path.split(path)
feature = Feature.get_feature(base, name)
features.append(feature)
features.sort(key=lambda feature: feature.name or feature.filename)
return features
def archive_files(from_pattern, archive_dir, to_pattern, batch_id):
"""
Archives files matching from_pattern and renames to to_pattern based on uid in a batch folder
:param from_pattern:
:param archive_dir
:param to_pattern:
:param batch_id:
:return:
"""
# place archived files in a timestamped folder
archive_folder = os.path.join(archive_dir, batch_id)
if not os.path.isdir(archive_folder):
os.mkdir(archive_folder)
os.mkdir(os.path.join(archive_folder, 'unsigned_certs'))
os.mkdir(os.path.join(archive_folder, 'signed_certs'))
os.mkdir(os.path.join(archive_folder, 'sent_txs'))
os.mkdir(os.path.join(archive_folder, 'receipts'))
os.mkdir(os.path.join(archive_folder, 'blockchain_certificates'))
archived_file_pattern = os.path.join(archive_folder, to_pattern)
[shutil.move(filename, models.convert_file_name(archived_file_pattern, uid))
for filename, (uid,) in glob2.iglob(from_pattern, with_matches=True)]
def app_css(template_reference):
"""Generate app css link tags for Flask app."""
link_tags = ""
# if False:
if G.debug:
# layout
for layout_path in G.css_config['layout']:
# 支持通配符
if '*' in layout_path:
for path in glob2.iglob(os.path.join(G.static_path, layout_path)):
relative_path = path.split(G.static_path)[1][1:]
link_tags += link(relative_path)
else:
link_tags += link(layout_path)
# page
template_name = _get_template_name(template_reference)
page_css_path = os.path.join(G.css_config['page'],
template_name.replace('html', 'css'))
link_tags += link(page_css_path)
else:
link_tags += link(APP_CSS)
return Markup(link_tags)
def archive_files(from_pattern, to_pattern, timestamp):
[shutil.copyfile(filename,
convert_file_name(to_pattern, uid) + '-' + timestamp)
for filename, (uid,) in glob2.iglob(from_pattern, with_matches=True)]
# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--input", required = True,
help = "input directory of images")
ap.add_argument("-o", "--output", required = True,
help = "output directory")
ap.add_argument("-c", "--csv", required = True,
help = "path to CSV file for image counts")
args = vars(ap.parse_args())
# open the output file for writing
output = open(args["csv"], "w")
# loop over the input images
for imagePath in glob2.iglob(args["input"] + "/*/*.jpg"):
# generate a random filename for the image and copy it to
# the output location
filename = str(uuid.uuid4()) + ".jpg"
shutil.copy(imagePath, args["output"] + "/" + filename)
# there is a 1 in 500 chance that multiple copies of this
# image will be used
if random.randint(0, 500) == 0:
# initialize the number of times the image is being
# duplicated and write it to the output CSV file
numTimes = random.randint(1, 8)
output.write("%s,%d\n" % (filename, numTimes))
# loop over a random number of times for this image to
# be duplicated
def get(self, pattern):
_folder = CleverString(self._folder).value
_pattern = CleverString(pattern).value
for item in glob2.iglob(str(f(_folder)/_pattern)):
yield item
def eglob(path):
"""Like glob.glob, but supports "/path/**/{a,b,c}.txt" lookup"""
fi = itertools.chain.from_iterable
paths = expand_paths(path)
return list(fi(glob2.iglob(d) for d in paths))
