def main():
SDL_Init(SDL_INIT_VIDEO)
window = SDL_CreateWindow(b"Bolidozor Snapshot Browser",
SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED,
430, 600, SDL_WINDOW_SHOWN)
windowsurface = SDL_GetWindowSurface(window)
main_thread_queue = queue.Queue()
def run_on_main_thread(func):
main_thread_queue.put(func)
event = SDL_Event()
event.type = SDL_USEREVENT
SDL_PushEvent(event)
thpool = mpdummy.Pool(processes=3)
connector = bzpost.HTTPConnector(
"http://space.astro.cz/bolidozor/OBSUPICE/OBSUPICE-R3/")
connector.connect()
drawable_snapshots = []
def put_up_snapshot(snapshot):
print("downloading %s..." % snapshot.url)
x = urllib.request.urlopen(snapshot.url)
fits = pyfits.open(io.BytesIO(x.read()))
print("downloading %s... done" % snapshot.url)
imunit = None
for unit in fits:
if unit.data is not None:
imunit = unit
img = imunit.data[:, :]
imin, imax = np.min(img), np.max(img)
rgbimg = np.repeat(
((img - imin) / (imax - imin) * 255).astype(np.uint8), 3)
def finish():
h, w = img.shape
surface = SDL_CreateRGBSurfaceFrom(rgbimg.ctypes.data, w, h, 24,
3 * w, 0, 0, 0, 0)
drawable_snapshots.append({
'time': snapshot.time,
'surface': surface,
'imgdata': rgbimg
})
run_on_main_thread(finish)
collection = SnapshotCollection(
connector, lambda a: thpool.apply_async(put_up_snapshot, (a, )))
time = 1421000000
running = True
event = SDL_Event()
run_on_main_thread(lambda: None)
while running:
while SDL_WaitEvent(ctypes.byref(event)) != 0:
if event.type == SDL_QUIT:
running = False
break
if event.type == SDL_MOUSEWHEEL:
time -= event.wheel.y * 10 * TIME_PER_PIX
break
if event.type == SDL_USEREVENT:
if not main_thread_queue.empty():
while not main_thread_queue.empty():
main_thread_queue.get()()
break
collection.cover(time - 800 * TIME_PER_PIX * 4,
time + 800 * TIME_PER_PIX * 5)
sdl2.ext.fill(windowsurface.contents, BLACK)
for snapshot in drawable_snapshots:
y = (snapshot['time'] - bzpost.normalize_time(
int(time))).total_seconds() / TIME_PER_PIX
SDL_BlitSurface(snapshot['surface'], None, windowsurface,
SDL_Rect(10, int(y), 0, 0))
SDL_UpdateWindowSurface(window)
SDL_DestroyWindow(window)
SDL_Quit()
return 0
def __init__(self,
policies: Sequence[py_policy.PyPolicy],
multithreading: bool = True):
"""Batch together multiple (non-batched) py policies.
The policies can be different but must use the same action and
observation specs.
Args:
policies: List python policies (must be non-batched).
multithreading: Python bool describing whether interactions with the
given policies should happen in their own threadpool. If `False`,
then all interaction is performed serially in the current thread.
This may be combined with `TFPyPolicy(..., py_policy_is_batched=True)`
to ensure that multiple policies are all run in the same thread.
Raises:
ValueError: If policies is not a list or tuple, or is zero length, or if
one of the policies is already batched.
ValueError: If the action or observation specs don't match.
"""
if not isinstance(policies, (list, tuple)):
raise ValueError("policies must be a list or tuple. Got: %s" %
policies)
self._parallel_execution = multithreading
self._policies = policies
self._num_policies = len(policies)
self._time_step_spec = self._policies[0].time_step_spec
self._action_spec = self._policies[0].action_spec
self._policy_state_spec = self._policies[0].policy_state_spec
self._info_spec = self._policies[0].info_spec
self._policy_step_spec = self._policies[0].policy_step_spec
self._trajectory_spec = self._policies[0].trajectory_spec
self._collect_data_spec = self._policies[0].collect_data_spec
self._observation_and_action_constraint_splitter = \
self._policies[0].observation_and_action_constraint_splitter
self._validate_spec(py_policy.PyPolicy.time_step_spec,
self._time_step_spec)
self._validate_spec(py_policy.PyPolicy.action_spec, self._action_spec)
self._validate_spec(py_policy.PyPolicy.policy_state_spec,
self._policy_state_spec)
self._validate_spec(py_policy.PyPolicy.info_spec, self._info_spec)
self._validate_spec(py_policy.PyPolicy.policy_step_spec,
self._policy_step_spec)
self._validate_spec(py_policy.PyPolicy.trajectory_spec,
self._trajectory_spec)
self._validate_spec(py_policy.PyPolicy.collect_data_spec,
self._collect_data_spec)
self._validate_spec(
py_policy.PyPolicy.observation_and_action_constraint_splitter,
self._observation_and_action_constraint_splitter)
# Create a multiprocessing threadpool for execution.
if multithreading:
self._pool = mp_threads.Pool(self._num_policies)
super(BatchedPyPolicy,
self).__init__(self._time_step_spec, self._action_spec,
self._policy_state_spec, self._info_spec,
self._observation_and_action_constraint_splitter)
with open(str(file), "r", encoding="utf8") as f:
content = f.readlines()
if not only_sleep1(
content) or len(content) <= 5: #FIXME: that's broken test
replay_file_list.append(file)
else:
print("skipping " + str(file))
print(replay_file_list)
if len(replay_file_list) == 0:
continue
pool_size = 8
with mp.Pool(processes=pool_size) as pool:
task_list = []
for image_name in image_name_list:
task = pool.apply_async(run_cases_on_image, (
apk_path,
replay_file_list,
image_name,
))
task_list.append(task)
for task in task_list:
try:
print("[*]", str(task.get(timeout=400)), "DONE!")
except Exception as e:
print(e)
logging.info("ALL DONE!")
def init():
global MAX_TIMEOUT, UPGRADE_ONLY
if sys.version_info < (3, 0):
sys.stdout.write("Sorry, requires Python 3.x, not Python 2.x\n")
sys.exit(1)
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="Detect and exploit insecure forwarding of h2c upgrades.",
epilog="Example Usage:\n"
+ sys.argv[0] + " --scan-list urls.txt --threads 5\n"
+ sys.argv[0] + " -x https://edgeserver http://localhost\n"
+ sys.argv[0] + " -x https://edgeserver -XPOST -d "
"'{\"data\":1}' -H \"Content-Type: application/json\""
"-H \"X-ADMIN: true\" http://backend/private/endpoint"
)
parser.add_argument("--scan-list",
help="list of URLs for scanning")
parser.add_argument("--threads",
type=int,
default=5,
help="# of threads (for use with --scan-list)")
parser.add_argument("--upgrade-only",
default=False,
action="store_true",
help="drop HTTP2-Settings from outgoing "
"Connection header")
parser.add_argument("-x", "--proxy",
help="proxy server to try to bypass")
parser.add_argument("-i", "--wordlist",
help="list of paths to bruteforce")
parser.add_argument("-X", "--request",
default="GET",
help="smuggled verb")
parser.add_argument("-d", "--data",
help="smuggled data")
parser.add_argument("-H", "--header",
action="append",
help="smuggled headers")
parser.add_argument("-m", "--max-time",
type=float,
default=10,
help="socket timeout in seconds "
"(type: float; default 10)")
parser.add_argument("-t", "--test",
help="test a single proxy server",
action="store_true")
parser.add_argument("-v", "--verbose",
action="store_true")
parser.add_argument("url", nargs="?")
args = parser.parse_args()
MAX_TIMEOUT = args.max_time
UPGRADE_ONLY = args.upgrade_only
if args.scan_list:
lines = []
with open(args.scan_list) as fd:
lines = [line.strip() for line in fd.readlines()]
p = mp.Pool(args.threads)
p.map(scan, lines)
p.close()
p.join()
sys.exit(1)
if not args.proxy:
print("Please provide a server for tunneling ('-x') flag ",
file=sys.stderr)
sys.exit(1)
if not args.test and not args.url:
print("Please specify the '-t' flag or provide smuggled URL")
sys.exit(1)
if args.url and not urlparse(args.url).scheme:
print("Please specify scheme (e.g., http[s]://) for: " + args.url)
sys.exit(1)
if not urlparse(args.proxy).scheme:
print("Please specify scheme (e.g., http[s]://) for: " + args.proxy)
sys.exit(1)
main(args)
def main(test, m):
p = multiprocessing.Pool()
results = p.map(lambda f: f(test), [read_RuFactEval, runtest])
p.close()
p.join()
factsRuFactEval = results[0]
# runtest(test) #если есть результат, то не нужно запускать повторно
# factsRuFactEval = read_RuFactEval(test)
factsPatterns = read_OurFact(test)
allFactsPatternsCount = 0
for book in factsPatterns:
allFactsPatternsCount += len(factsPatterns[book])
allFactsRuFactEvalCount = 0
for book in factsRuFactEval:
allFactsRuFactEvalCount += len(factsRuFactEval[book])
gaz = cleanBYgaz(factsRuFactEval)
hard = cleanBYhard(factsRuFactEval)
sent = cleanBYsent(test, factsRuFactEval)
all = cleanBYhard(cleanBYgaz((sent)))
print("Отфильтрованные базы для", test, "получены")
p = multiprocessing.Pool()
run = [[testPHexact, "EQ", factsRuFactEval.copy()],
[testPHabout, "Normalize",
factsRuFactEval.copy()],
[testPHexact, "EQ-gaz-filter",
gaz.copy()], [testPHabout, "Normalize-gaz-filter",
gaz.copy()],
[testPHexact, "EQ-hard-filter",
hard.copy()], [testPHabout, "Normalize-hard-filter",
hard.copy()],
[testPHexact, "EQ-sent-filter",
sent.copy()], [testPHabout, "Normalize-sent-filter",
sent.copy()],
[testPHexact, "EQ-all-filter",
all.copy()], [testPHabout, "Normalize-all-filter",
all.copy()]]
results = p.map(
lambda f: analyse_results(test, f[2], factsPatterns.copy(), f[0], f[1]
), run)
# print(results)
p.close()
p.join()
trueFactsPatternsCount = results[0][0]
trueFactsPatternsCount2 = results[1][0]
th1 = results[0][1]
th2 = results[1][1]
data = ""
data += m
data += ("\n\nВсего извлечено = \t" + str(allFactsPatternsCount))
data += ("\nВсего должно быть извлечено = \t" +
str(allFactsRuFactEvalCount))
data += ("\n\nИзвлечено правильно (точное совпадение) = \t" +
str(trueFactsPatternsCount))
data += ("\n\nИз них повышенной сложности = \t" + str(th1))
if trueFactsPatternsCount > 0:
data += PRF(trueFactsPatternsCount, allFactsPatternsCount,
allFactsRuFactEvalCount)
if trueFactsPatternsCount2 > 0:
# print("\nА ещё мы посчитали результаты с учётом форм слов")
data += ("\n\nИзвлечено правильно, с учётом форм слов = " +
str(trueFactsPatternsCount2))
data += ("\n\nИз них повышенной сложности = \t" + str(th2))
data += PRF(trueFactsPatternsCount2, allFactsPatternsCount,
allFactsRuFactEvalCount)
k = 1
k += 1
ro = results[k][0]
if ro > 0:
data += (
"\n\nРезульаты с очисткой по газеттиру (точное совпадение), правильно = \t"
+ str(ro))
data += ("\n\nРазмер словаря правильных ответов = \t" + str(len(gaz)))
data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)
k += 1
ro = results[k][0]
if ro > 0:
data += (
"\n\nРезульаты с очисткой по газеттиру (нормализация), правильно = "
+ str(ro))
data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)
k += 1
ro = results[k][0]
if ro > 0:
data += (
"\n\nРезульаты с очисткой по сложности (точное совпадение), правильно = "
+ str(ro))
data += ("\n\nРазмер словаря правильных ответов = \t" + str(len(hard)))
data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)
k += 1
ro = results[k][0]
if ro > 0:
data += (
"\n\nРезульаты с очисткой по сложности (нормализация), правильно = "
+ str(ro))
data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)
k += 1
ro = results[k][0]
if ro > 0:
data += (
"\n\nРезульаты с очисткой по предложениям (точное совпадение), правильно = "
+ str(ro))
data += ("\n\nРазмер словаря правильных ответов = \t" + str(len(sent)))
data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)
k += 1
ro = results[k][0]
if ro > 0:
data += (
"\n\nРезульаты с очисткой по предложениям (нормализация), правильно = "
+ str(ro))
data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)
k += 1
ro = results[k][0]
if ro > 0:
data += (
"\n\nРезульаты с полной очисткой (точное совпадение), правильно = "
+ str(ro))
data += ("\n\nРазмер словаря правильных ответов = \t" + str(len(all)))
data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)
k += 1
ro = results[k][0]
if ro > 0:
data += (
"\n\nРезульаты с полной очисткой (нормализация), правильно = " +
str(ro))
data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)
wfile = open("Results.txt", "a", encoding="cp1251")
wfile.write(data)
wfile.close()
print("результаты записаны:", test)
def rdsItem(self, current, aconnect, itemlist):
added = False
printColor(['_____LISTING RDS [] now....in .%s' % (aconnect._region)])
threaded = aconnect.getMultiThread()
nEnv = aconnect._env
lfound = []
objs = []
rows = []
pools = 2 # 171.35 seconds #168 seconds
totalItems = len(itemlist)
q = None
if threaded:
pools = auditMeth.poolThreadNumber(totalItems, mp.cpu_count())
p = mp.Pool(1 if pools == 0 else pools)
else:
p = mp.Pool()
m = mp.Manager()
q = m.Queue()
if totalItems > 0:
objs.append([
'Name[%s]' % (current.svc), 'Audit', 'Owner', 'Engine',
'Size (GB)', 'Instance', 'MutliAZ', 'VPC', 'last_Modified',
'connections', 'cost'
])
if not threaded:
for unit in itemlist:
name = unit['DBInstanceIdentifier']
#sg=unit['VpcSecurityGroups'][0]['VpcSecurityGroupId']
#ec2 = aconnect.__get_client__('ec2')
#sgUnit = ec2.describe_security_groups(GroupIds=[sg])['SecurityGroups'][0]
#account = sgUnit['OwnerId']
if nEnv in name or aconnect._useAccounts:
if pools == 0:
objs, row = rdsDefine(unit, name, q)
else:
getit = p.apply_async(rdsDefine, (unit, name, q))
#getit.get()
lfound.append(name)
#objs,row = self.rdsDefine(current,aconnect,objs,unit,name)
#rows.append(row)
else:
#print itemlist
lfound = [unit['DBInstanceIdentifier'] for unit in itemlist]
results = p.map(rdsDefine, (unit for unit in itemlist))
if pools > 0:
p.close()
p.join()
if threaded:
for que in results:
newobjs, row = que
#print ' DYNAMO --got[C] result', row
objs = objs + newobjs
rows.append(row)
else:
while not q.empty():
newobjs, row = q.get()
objs = objs + newobjs
# print newobjs
rows.append(row)
print objs
return (lfound, objs, rows)
def get_multiple_entities(entities):
import multiprocessing.dummy as multiprocessing
return dict(multiprocessing.Pool(THREAD_POOL_PROCESSES).map(_get_entity, entities))
from .bookkeeping import Job
from openmod.sh.api import (provide_element_api, json_to_db,
provide_elements_api, provide_sequence_api,
allowed_file, explicate_hubs,
delete_element_from_db, results_to_db,
get_hub_results, create_transmission,
get_flow_results, get_flow_result, get_co2_results)
from openmod.sh.forms import ComputeForm
from openmod.sh.visualization import make_graph_plot
from openmod.sh.web import app, csrf
from openmod.sh import mcbeth
from openmod.sh.config import get_config
# Set up a pool of workers to which jobs can be submitted and a dictionary
# which stores the asynchronous result objects.
app.workers = mpd.Pool(1) if app.debug else mpp.Pool(1)
app.results = OD()
babel = Babel(app)
@babel.localeselector
def get_locale():
# if a user is logged in, use the locale from the user settings
user = getattr(flask.g, 'user', None)
if user is not None:
return user.locale
# otherwise try to guess the language from the user accept
# header the browser transmits. The best match wins.
return flask.request.accept_languages.best_match(['de', 'en'])
if __name__ == "__main__":
PARSER_ = argparse.ArgumentParser(description="Parameters")
PARSER_.add_argument("--annotation",
nargs="?",
type=str,
default=None,
help="File annotation path")
PARSER_.add_argument("--path",
nargs="?",
type=str,
default=None,
help="Dataset path")
ARGS_ = PARSER_.parse_args()
annotation_file = ARGS_.annotation
dataset_path = ARGS_.path
cloud_path = dataset_path + '/cloud'
pathlib.Path(cloud_path).mkdir(parents=True, exist_ok=True)
with open(annotation_file, 'r') as f:
f.readline()
fx, fy, rx, ry = csv_camera_info(f.readline())
f.readline()
f.readline()
pf = mp.Pool(12)
for i, _ in enumerate(pf.imap_unordered(thread_cloud, f, 1)):
#sys.stderr.write('\rdone {0:%}\n'.format(i/scene_.m_total_frames))
sys.stderr.write('\rdone: {}'.format(i))
def __init__(self, num_dims: int = 16):
super().__init__()
self.embedding_net = EmbeddingNet(num_dims)
self.pool = mp.Pool(processes=3)
def divide_train_val():
all_masks = []
for dir_name in ['obj', 'scene', 'scene_only']:
image_dir = os.path.join(os.getcwd(), 'data', dir_name)
if tf.io.gfile.exists(image_dir):
tf.io.gfile.rmtree(image_dir)
tf.io.gfile.makedirs(image_dir)
train_dir = os.path.join(image_dir, 'train')
tf.io.gfile.makedirs(train_dir)
val_dir = os.path.join(image_dir, 'val')
tf.io.gfile.makedirs(val_dir)
loc_table = {}
lines = tf.io.gfile.GFile(os.path.join(image_dir + '_all',
'loc.txt')).readlines()
for line in lines:
loc_table[line.split(' ')[0]] = line.split(' ')[1]
classes = tf.io.gfile.listdir(image_dir + '_all')
classes.remove('loc.txt')
classes.sort()
pool = multiprocessing.Pool(100)
val_lines = []
loc_lines = []
for i in range(len(classes)):
a_class = classes[i]
if dir_name == 'obj':
all_masks.append(
np.load(os.path.join(os.getcwd(), 'data', 'coco',
'segments', a_class, 'val_mask'),
allow_pickle=True))
files = np.array(
tf.io.gfile.listdir(os.path.join(image_dir + '_all', a_class)))
train_files = []
val_files = []
for f in files:
if re.search('9[0-9]{2}\.jpg', f):
val_files.append(f)
else:
train_files.append(f)
val_files.sort()
tf.io.gfile.makedirs(os.path.join(train_dir, a_class))
pool.map(
lambda f: tf.io.gfile.copy(
os.path.join(image_dir + '_all', a_class, f),
os.path.join(train_dir, a_class, f)), train_files)
pool.map(
lambda f: tf.io.gfile.copy(
os.path.join(image_dir + '_all', a_class, f),
os.path.join(val_dir, f)), val_files)
val_lines += [f + ' ' + str(i) + '\n' for f in val_files]
loc_lines += [f + ' ' + loc_table[f] for f in val_files]
val_txt_path = os.path.join(image_dir, 'val.txt')
with tf.io.gfile.GFile(val_txt_path, 'w') as f:
f.write(''.join(sorted(val_lines)))
with tf.io.gfile.GFile(os.path.join(image_dir, 'val_loc.txt'),
'w') as f:
f.write(''.join(sorted(loc_lines)))
# Copy over the object masks for the validation set
np.save(tf.io.gfile.GFile(os.path.join(image_dir, 'val_mask'), 'w'),
np.concatenate(all_masks))
shutil.copy(os.path.join(image_dir + '_all', 'loc.txt'),
os.path.join(image_dir, 'loc.txt'))
tf.io.gfile.rmtree(image_dir + '_all')
# img = cv2.imread(wsi_path + '/' + name)
# cv2.imwrite(save_path + '/%f_%s' % (grade , name) , img)
reco_path = r'X:\GXB\SNS\test_result\unet_4down_wsi_reco'
path = r'X:\GXB\SNS\test_result\unet_4down_wsi'
wsi_names = os.listdir(path)
for wsi_name in wsi_names[1:]:
print(wsi_name)
img_names = os.listdir(path + '/' + wsi_name)
img_names = [x for x in img_names if x.find('.tif') != -1]
mean_value = []
names = []
pool = dummy.Pool(cpu_count())
for img_name in img_names:
t = pool.apply_async(post_handle,
args=(path + '/%s/%s' % (wsi_name, img_name), ))
mean_value.append(t.get())
names.append(img_name)
pool.close()
pool.join()
mean_value = np.array(mean_value)
print(np.shape(mean_value))
recommand(reco_path + '/' + wsi_name, path + '/' + wsi_name, names,
mean_value)
def map(self, func, args):
with thrd.Pool(processes=self.processes) as pool:
return pool.map(func, args)
def _prepare_pool(self, n_jobs=None):
if n_jobs == 1:
return dummy_processing.Pool(processes=n_jobs)
return actual_processing.Pool(processes=n_jobs)
def servers_get_stats(servers):
threads = __servers_pool_size(len(servers))
pool = multiprocessing.Pool(threads)
pool.map(Nginx.get_stats, servers)
def similarity(cd, num_random_exp=None, num_workers=25):
"""Returns cosine similarity of all discovered concepts.
Args:
cd: The ConceptDiscovery module for discovered conceps.
num_random_exp: If None, calculates average similarity using all the class's
random concepts. If a number, uses that many random counterparts.
num_workers: If greater than 0, runs the function in parallel.
Returns:
A similarity dict in the form of {(concept1, concept2):[list of cosine
similarities]}
"""
def concepts_similarity(cd, concepts, rnd, bn):
"""Calcualtes the cosine similarity of concept cavs.
This function calculates the pairwise cosine similarity of all concept cavs
versus an specific random concept
Args:
cd: The ConceptDiscovery instance
concepts: List of concepts to calculate similarity for
rnd: a random counterpart
bn: bottleneck layer the concepts belong to
Returns:
A dictionary of cosine similarities in the form of
{(concept1, concept2): [list of cosine similarities], ...}
"""
similarity_dic = {}
for c1 in concepts:
cav1 = cd.load_cav_direction(c1, rnd, bn)
for c2 in concepts:
if (c1, c2) in similarity_dic.keys():
continue
cav2 = cd.load_cav_direction(c2, rnd, bn)
similarity_dic[(c1, c2)] = cosine_similarity(cav1, cav2)
similarity_dic[(c2, c1)] = similarity_dic[(c1, c2)]
return similarity_dic
similarity_dic = {bn: {} for bn in cd.bottlenecks}
if num_random_exp is None:
num_random_exp = cd.num_random_exp
randoms = ['random500_{}'.format(i) for i in np.arange(num_random_exp)]
concepts = {}
for bn in cd.bottlenecks:
concepts[bn] = [cd.target_class, cd.random_concept] + cd.dic[bn]['concepts']
for bn in cd.bottlenecks:
concept_pairs = [(c1, c2) for c1 in concepts[bn] for c2 in concepts[bn]]
similarity_dic[bn] = {pair: [] for pair in concept_pairs}
def t_func(rnd):
return concepts_similarity(cd, concepts[bn], rnd, bn)
if num_workers:
pool = multiprocessing.Pool(num_workers)
sims = pool.map(lambda rnd: t_func(rnd), randoms)
else:
sims = [t_func(rnd) for rnd in randoms]
while sims:
sim = sims.pop()
for pair in concept_pairs:
similarity_dic[bn][pair].append(sim[pair])
return similarity_dic
def dynamoItem(self, current, aconnect, itemlist):
added = False
printColor(
['_____LISTING DynamoDB [] now....in .%s' % (aconnect._region)])
threaded = aconnect.getMultiThread()
nEnv = aconnect._env
lfound = []
objs = []
rows = []
pools = 2 # 171.35 seconds #168 seconds
totalItems = len(itemlist)
if threaded:
pools = auditMeth.poolThreadNumber(totalItems, mp.cpu_count())
if pools > 0:
p = mp.Pool(pools)
elif pools == 0:
threaded = False
else:
p = mp.Pool()
m = mp.Manager()
q = m.Queue()
if totalItems > 0:
objs.append([
'Name[%s]' % (current.svc), 'Audit', 'Owner', 'Status',
'PartitionKey', 'indexes', 'totalRead', 'totalWrite'
])
client = aconnect.__get_client__('dynamodb')
if not threaded:
for name in itemlist:
unit = client.describe_table(TableName=name)['Table']
#account = dunit['TableArn']
if nEnv in name or aconnect._useAccounts:
if pools == 0:
objs, row = dynamoDefine(unit, name, q)
else:
getit = p.apply_async(dynamoDefine, (unit, name, q))
#getit.get()
lfound.append(name)
#objs,row=self.dynamoDefine(current,aconnect,objs,unit,name)
#rows.append(row)
else:
#print itemlist
lfound = [name for name in itemlist]
#print lfound
#print 'client ready?'
#dd = client.describe_table(TableName='Tags')['Table']
#print dd
results = p.map(dynamoDefine,
(client.describe_table(TableName=name)['Table']
for name in itemlist))
if pools > 0:
p.close()
p.join()
if threaded:
for que in results:
newobjs, row = que
#print ' RDS --got[C] result', row
objs = objs + newobjs
rows.append(row)
else:
while not q.empty():
newobjs, row = q.get()
objs = objs + newobjs
# print newobjs
rows.append(row)
return (lfound, objs, rows)
def parmap(fun,seq,N=None,Nt=1,chunksize=1,ordered=True,\
daemon=False,progress=False):
"""
parmap -- Simple parallel mapper that can split amongst processes (N)
and threads (Nt) (within the processes).
Does *NOT* require functions to be pickleable (unlike
vanilla multiprocess.Pool.map)
Inputs:
-------
fun
Single input function. Use lambdas or functools.partial
to enable/exapnd multi-input. See example
sequence
Sequence of inputs to map in parallel
Options:
--------
N [None] (integer or None)
Number of processes to use. If `None`, will use the CPU_COUNT
Nt [1] (integer)
Number of threads to use. See notes below on multi-threaded vs
multi-processes.
chunksize [1] (int)
How to be break up the incoming sequence. Useful if also using threads.
Will be (re)set to max(chunksize,Nt)
ordered [True] (bool)
Whether or not to order the results. If False, will return in whatever
order they finished.
daemon [False] (bool)
Sets the multiprocessing `daemon` flag. If True, can not spawn child
processes (i.e. cannot nest parmap) but should allow for CTRL+C type
stopping. Supposedly, there may be issues with CTRL+C with it set to
False. Use at your own risk
progress [False] (bool)
Display a progress bar or counter.
Warning: Inconsistant in iPython/Jupyter notebooks and may clear
other printed content. Instead, specify as 'nb' to use a Jupyter
Widget progress
Notes:
------
Performs SEMI-lazy iteration based on chunksize. It will exhaust the input
iterator but will yield as results are computed (This is similar to the
`multiprocessing.Pool().imap` behavior)
Explicitly wrap the parmap call in a list(...) to force immediate
evaluation
Threads and/or processes:
-------------------------
This tool has the ability to split work amongst python processes
(via multiprocessing) and python threads (via the multiprocessing.dummy
module). Python is not very performant in multi-threaded situations
(due to the GIL) therefore, processes are the usually the best for CPU
bound tasks and threading is good for those that release the GIL (such
as IO-bound tasks). Note that many NumPy functions *do* release the GIL
and can be threaded, but many NumPy functions are, themselves, multi-
threaded.
Alternatives:
-------------
This tool allows more data types, can split with threads, has an optional
progress bar, and has fewer pickling issues, but these come at a small cost.
For simple needs, the following may be better:
>>> import multiprocessing as mp
>>> pool = mp.Pool(N) # Or mp.Pool() for N=None
>>> results = list( pool.imap(fun,seq) ) # or just pool.map
>>> pool.close()
Process Method:
---------------
This code uses iterators/generators to handle and distribute the workload.
By doing this, it is easy to have all results pass through a common
counting function for display of the progress without the use of
global (multiprocessing manager) variables and locks.
With the exception of when N == 1 (where it falls back to serial methods)
the code works as follows:
- A background thread is started the will iterate over the incoming sequence
and add items to the queue. If the incoming sequence is exhausted, the
worker sends kill signals into the queue.
- The items are also chunked and indexed (used later)
- After the background thread is started a function to pull from the OUTPUT
queue is created. This counts the number of closed processes but otherwise
yields the computed result items
- A pool of workers is created. Each worker will read from the input queue
and distribute the work amongst threads (if using). It will then
return the resuts into a queue
- Now the main work happens. It is done as chain of generators/iterators.
The background worker has already begin adding items to the queue so
now we work through the output queue. Note that this is in serial
since the work was already done in parallel
- Generator to pull from the result queue
- Generator to count and display progress (if progress=True).
- Generator to hold on to and return items in a sorted manner
if sorting is requested. This can cause itermediate results to be
stored until they can be returned in order
- The output generator chain is iterated pulling items through and then
are yielded.
- cleanup.
Last Updated:
-------------
2018-07-30
"""
if N is None:
N = CPU_COUNT
chunksize = max(chunksize, Nt)
try:
tot = len(seq)
except TypeError:
tot = None
if tqdm is None:
if isinstance(progress,(str,unicode))\
and progress.lower() in ['jupyter','notebook','nb']:
counter = partial(_counter_nb, tot=tot)
else:
counter = partial(_counter, tot=tot)
else:
if isinstance(progress,(str,unicode))\
and progress.lower() in ['jupyter','notebook','nb']\
and hasattr(tqdm,'tqdm_notebook'):
counter = partial(tqdm.tqdm_notebook, total=tot)
else:
counter = partial(
tqdm.tqdm,
total=tot) # Set the total since tqdm won't be able to get it.
if N == 1:
if Nt == 1:
out = imap(fun, seq)
else:
pool = mpd.Pool(Nt) # thread pools don't have the pickle issues
out = pool.imap(fun, seq)
if progress:
out = counter(out)
for item in out:
yield item
if Nt > 1:
pool.close()
return
q_in = mp.JoinableQueue()
q_out = mp.Queue()
# Start the workers
workers = [
mp.Process(target=_worker, args=(fun, q_in, q_out, Nt))
for _ in xrange(N)
]
for worker in workers:
worker.daemon = daemon
worker.start()
# Create a separate thread to add to the queue in the background
def add_to_queue():
for iixs in _iter_chunks(enumerate(seq), chunksize):
q_in.put(iixs)
# Once (if ever) it is exhausted, send None to close workers
for _ in xrange(N):
q_in.put(None)
add_to_queue_thread = Thread(target=add_to_queue)
add_to_queue_thread.start()
# Generator we use to return
def queue_getter():
finished = 0
count = 0
while finished < N:
out = q_out.get()
if out is None:
finished += 1
continue
yield out
# Chain generators on output
out = queue_getter()
if progress:
out = counter(out)
if ordered:
out = _sort_generator_unique_integers(out, key=lambda a: a[0])
# Return
for item in out:
yield item[1]
# Clean up
q_in.join()
for worker in workers:
worker.join()
if pbar:
pbar.update(1)
return a+b
# make pies
a_list = range(16)
b_list = range(16)
c_list = []
pbar = tqdm(total=len(a_list), position=0, leave=True,
desc = f"aa: ", )
p=mp.Pool(workers)
c_list = p.starmap(foo, zip(a_list,b_list,repeat(pbar)))
p.close()
pbar.close()
p.join()
print(c_list)
# PIL read
# https://pillow.readthedocs.io/en/stable/reference/Image.html
from PIL import Image
import multiprocessing.dummy as mp
items = [0] * 10
def do_print(i):
print(i)
items[i] = i * 2 + 1
p = mp.Pool(4)
p.map(do_print,
range(0, 10)) # range(0,1000) if you want to replicate your example
p.close()
p.join()
print(items)
import numpy as np
import multiprocessing.dummy as mp
def dummy_function(x, y):
return x * y
def dummy_iter(i):
for i in range(i):
yield (i, i)
if __name__ == '__main__':
p = mp.Pool(1)
itera = dummy_iter(10)
results = p.starmap(dummy_function, itera)
p.close()
p.join()
print(results)
new_xml_name = new_filename + '.xml'
cv2.imwrite(new_img_name, combined_im)
xml_doc.find('./filename').text = os.path.basename(new_img_name)
xml_doc.write(new_xml_name, encoding='utf8')
# cv2.imshow("coin", coin_im)
# cv2.imshow("bg", bg_im)
# cv2.imshow(str(uuid.uuid4()), combined_im)
if __name__ == '__main__':
# for i in range(1):
# generate_new_image('T:\emb_fin\images\\190103_005.xml', 2)
# while True:
# cv2.waitKey(500)
if not os.path.exists(NEW_DATA_DIR):
os.mkdir(NEW_DATA_DIR)
xml_files = glob.glob(os.path.join(sys.argv[1], "*.xml"))
p = mp.Pool()
p.map(generate_new_image_with_blur, xml_files)
p.close()
p.join()
# for xml_path in xml_files:
# try:
# generate_new_image(xml_path)
# except Exception as e:
# print("Error when processing image: " + xml_path)
# raise e
# pass
def create_patches(self, class_names, method='slic', discovery_images=None,
param_dict=None):
"""Creates a set of image patches using superpixel methods.
This method takes in the concept discovery images and transforms it to a
dataset made of the patches of those images.
Args:
method: The superpixel method used for creating image patches. One of
'slic', 'watershed', 'quickshift', 'felzenszwalb'.
discovery_images: Images used for creating patches. If None, the images in
the target class folder are used.
param_dict: Contains parameters of the superpixel method used in the form
of {'param1':[a,b,...], 'param2':[z,y,x,...], ...}. For instance
{'n_segments':[15,50,80], 'compactness':[10,10,10]} for slic
method.
"""
self.class_names = class_names
print("START MAKING PATCHES")
if param_dict is None:
param_dict = {}
dataset, image_numbers, patches, segment_class_labels = [], [], [], []
if discovery_images is None:
print("target class images")
discovery_images= self.load_concept_imgs(self.target_class, self.num_discovery_imgs)
np.save(os.path.join(self.np_dir,"discovery_images.npy"), discovery_images)
else:
discovery_images_list = [self.load_concept_imgs(clss, math.floor(self.num_discovery_imgs/len(self.class_names)), return_filenames=True) for clss in self.class_names]
discovery_images = [i for i,j in discovery_images_list]
filenames = [j for i,j in discovery_images_list]
with open(os.path.join(self.np_dir,'discovery_filenames.txt'), 'w') as f:
for i, lst in enumerate(filenames):
for filename in lst:
f.write("{} {} {}\n".format(i, self.class_names[i], filename))
class_labels = [[i]*len(d) for i, d in enumerate(discovery_images)]
self.discovery_class_labels = np.concatenate(class_labels) # index for class in self.class_names
discovery_images = np.concatenate(discovery_images)
filenames=np.concatenate(filenames)
np.save(os.path.join(self.np_dir,"discovery_images.npy"), discovery_images)
np.save(os.path.join(self.np_dir,"filenames.npy"),filenames)
if self.num_workers:
pool = multiprocessing.Pool(self.num_workers)
outputs = pool.map(
lambda img: self._return_superpixels(img, method, param_dict),
discovery_images)
for fn, sp_outputs in enumerate(outputs):
image_superpixels, image_patches = sp_outputs
for superpixel, patch in zip(image_superpixels, image_patches):
dataset.append(superpixel)
patches.append(patch)
image_numbers.append(fn)
segment_class_labels.append(self.discovery_class_labels[fn])
else:
print("num images: {}".format(len(discovery_images)))
for fn, img in enumerate(discovery_images):
print(fn)
p_dict = param_dict.copy()
print(p_dict)
image_superpixels, image_patches = self._return_superpixels(
img, method, p_dict)
for superpixel, patch in zip(image_superpixels, image_patches):
dataset.append(superpixel)
patches.append(patch)
image_numbers.append(fn)
segment_class_labels.append(self.discovery_class_labels[fn])
image_superpixels.clear()
image_patches.clear()
print("starting np loading")
np.save(os.path.join(self.np_dir, "dataset.npy"),np.array(dataset, dtype=np.float16))
dataset.clear()
np.save(os.path.join(self.np_dir, "image_numbers.npy"),np.array(image_numbers, dtype=np.int8))
image_numbers.clear()
np.save(os.path.join(self.np_dir, "patches.npy"),np.array(patches, dtype=np.float16))
patches.clear()
self.segment_class_labels = np.array(segment_class_labels)
np.save(os.path.join(self.np_dir, "segment_class_labels.npy"), np.array(segment_class_labels,dtype=np.int8))
self.discovery_size=len(discovery_images)
# np_dataset = np.array(dataset, dtype=np.float16)
# np_image_numbers = np.array(image_numbers, dtype=np.int16)
# np_patches = np.array(patches, dtype=np.float16)
#
# all_objects = muppy.get_objects()
# sum1 = summary.summarize(all_objects)
# summary.print_(sum1)
print("end of np loading")
print("discover concepts done")
def test():
reuse = False
if os.path.isfile('distribution_results/activity.pkl') and reuse:
with open('distribution_results/activity.pkl', 'rb') as f:
video_label = pickle.load(f)
else:
video_label = {}
test_video_path = 'dataset/Video/BreakfastII_15fps_qvga_sync'
with tf.Session(config=config) as sess:
Network = activity_network(number_of_classes=49)
IO_tool = IO_manager(Batch_size, frames_per_step, window_size, sess)
IO_tool.openpose.load_openpose_weights()
IO_tool.hidden_states_dim = Network.lstm_units
sess.run(Network.init)
dataset = IO_tool.dataset.Train_dataset
untrimmed = IO_tool.dataset.untrimmed_train_dataset
train_writer = tf.summary.FileWriter("logdir/test", sess.graph)
tot_second = 0
for root, dirs, files in os.walk(test_video_path):
for video_name in files:
path = root + '/' + video_name
if path not in untrimmed.keys():
continue
video = cv2.VideoCapture(path)
video.set(cv2.CAP_PROP_POS_AVI_RATIO, 1)
max_len = int(video.get(cv2.CAP_PROP_POS_MSEC) / 1000)
tot_second += max_len
# Loading initial C3d or presaved network
use_pretrained_model = True
if os.path.isfile('./checkpoint/checkpoint') and use_pretrained_model:
print('new model loaded')
Network.model_saver.restore(
sess, tf.train.latest_checkpoint('./checkpoint'))
elif load_c3d and use_pretrained_model:
print('original c3d loaded')
Network.c3d_loader.restore(sess, c3d_saved_weigths)
with tf.name_scope('whole_saver'):
whole_saver = tf.train.Saver()
whole_saver.save(sess, model_filename, global_step=Network.global_step)
pbar_whole = tqdm(total=(tot_second), desc='Videos')
for root, dirs, files in os.walk(test_video_path):
for video_name in files:
print(video_name)
path = root + '/' + video_name
if path not in untrimmed.keys():
continue
video = cv2.VideoCapture(path)
video.set(cv2.CAP_PROP_POS_AVI_RATIO, 1)
max_len = video.get(cv2.CAP_PROP_POS_MSEC) / 1000
fps = video.get(cv2.CAP_PROP_FPS)
step_times = [v for v in range(1, int(max_len / 1))]
pbar_video = tqdm(total=(len(step_times)),
desc='Video_completition')
for t_end in step_times:
if video_name in video_label.keys():
if t_end in video_label[video_name].keys():
pbar_video.update(1)
continue
segment = [(t_end - 1) * fps + 2, t_end * fps]
pbar = tqdm(total=(tasks * Batch_size * frames_per_step +
tasks),
leave=False,
desc='Batch Generation')
def multiprocess_batch(x):
X, c, h, label = IO_tool.test_generator(
pbar, path, segment)
return {'X': X, 'c': c, 'h': h, 'label': label}
pool = mp.Pool(processes=tasks)
ready_batch = pool.map(multiprocess_batch, range(0, 1))
pbar.close()
pool.close()
pool.join()
batch = IO_tool.add_pose(ready_batch, sess,
augment=False)[0]
conv3 = sess.run(
[Network.conv3],
feed_dict={
Network.input_batch: batch['X'],
Network.c_input: batch['c'],
Network.h_input: batch['h']
})
if video_name not in video_label.keys():
video_label[video_name] = {}
video_label[video_name][t_end] = {}
video_label[video_name][t_end]['tensor'] = conv3
video_label[video_name][t_end]['label'] = batch['label']
video_label[video_name][t_end]['limb'] = batch['X'][
0, :, :, :, 3]
video_label[video_name][t_end]['joint'] = batch['X'][
0, :, :, :, 4]
pbar_video.update(1)
pbar_whole.update(1)
with open(
'distribution_results/activity' + video_name + '.pkl',
'wb') as f:
pickle.dump(video_label, f, pickle.HIGHEST_PROTOCOL)
pbar_video.close()
def listFilePath():
query = urllib.urlencode({
b'path': g_netDirPath.encode('utf-8')
}).decode('utf-8')
url = 'http://%s/listdir?%s' % (g_netHost, query)
for line in urllib2.urlopen(url).readlines():
g_filePathList.append(line.decode('utf-8').strip())
def userInput():
global g_netHost, g_netDirPath, g_localDirPath
i = raw_input('input net host(default=localhost:8080):').decode(
sys.getfilesystemencoding())
if i: g_netHost = i
g_netDirPath = raw_input('input net dir:').decode(
sys.getfilesystemencoding())
assert g_netDirPath
g_localDirPath = raw_input('input local dir:').decode(
sys.getfilesystemencoding())
if not g_localDirPath:
g_localDirPath = os.path.split(g_netDirPath)[1] or 'temp'
if not g_netDirPath.endswith(os.path.sep): g_netDirPath += os.path.sep
if not g_localDirPath.endswith(os.path.sep): g_localDirPath += os.path.sep
userInput()
listFilePath()
dummy.Pool(16).map(downloadFile, g_filePathList)
import tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from torchvision.datasets import ImageFolder
import cv2
import os
import numpy as np
from .transforms import get_default_transforms
from PIL import Image, ImageOps
import matplotlib.pyplot as plt
import multiprocessing.dummy as mp
n_threads = 16
p = mp.Pool(n_threads)
from numba import njit, jit, prange
@jit
def edge_promoting(root, save):
#from numba.errors import NumbaDeprecationWarning, NumbaPendingDeprecationWarning, NumbaWarning
#import warnings
#warnings.simplefilter('ignore', category=NumbaDeprecationWarning)
#warnings.simplefilter('ignore', category=NumbaPendingDeprecationWarning)
#warnings.simplefilter('ignore', category=NumbaWarning)
img_size = (384, 384)
def main(opts):
global verbose
if opts.verbose is not None:
verbose = opts.verbose
if verbose>1:opt_zopfli[0].append('-v')
if not is_exec(opts.nasm):
eprint("nasm binary not found, doesn't exist or is not executable.")
return
if not is_exec(opts.gzip ): opts.gzip = None
if not is_exec(opts.xz ): opts.xz = None
if not is_exec(opts.lzma ): opts.lzma = None
if not is_exec(opts.zopfli): opts.zopfli = None
comprs = sum([ \
[(opts.gzip , opt_gzip , True )] if opts.gzip is not None else [],
[(opts.xz , opt_xz , False)] if opts.xz is not None else [],
[(opts.lzma , opt_lzma , False)] if opts.lzma is not None else [],
[(opts.zopfli, opt_zopfli, True )] if opts.zopfli is not None else [],
[(opts.zstd , opt_zstd , 2 )] if opts.zstd is not None else [] \
], [])
if len(comprs) == 0:
eprint("No compression methods specified.")
return
mpool = multi.Pool(processes=opts.jobs)
allofthem = list(sum(map(lambda x: run_compr(x, mpool, opts.input_file),
comprs),[]))
if len(allofthem) == 0:
eprint("No useable results available. See error log.")
return
eprint(len(allofthem))
allofthem_s = sorted(allofthem, key=lambda x:len(x[0]))
best = allofthem_s[0]
if verbose > 0:
lbl = "xz"
if best[1] == 2:
lbl = "zstd"
elif best[1]:
lbl = "gzip"
eprint(len(best[0]), lbl)
res, stubbed = best[0], None
if not opts.nostub:
stubbed = do_vndh(opts.nasm, opts.vndh, best,
opts.vndh_tag, opts.vndh_vfork, opts.vndh_unibin,
opts.vndh_cheat,opts.vndh_no_argv)
if verbose > 0: eprint("final: "+str(len(res)))
opts.output_file.write(stubbed or res)
if stubbed is not None and opts.rawout is not None:
opts.rawout.write(res)
def servers_stop(servers):
threads = __servers_pool_size(len(servers))
pool = multiprocessing.Pool(threads)
pool.map(Nginx.stop, servers)
def collate(self):
# Set the stacksize to be unlimited
res.setrlimit(res.RLIMIT_STACK, (res.RLIM_INFINITY, res.RLIM_INFINITY))
# Locate the FMS collation tool
# Check config for collate executable
mppnc_path = self.expt.config.get('collate_exe')
if mppnc_path is None:
for f in os.listdir(self.expt.lab.bin_path):
if f.startswith('mppnccombine'):
mppnc_path = os.path.join(self.expt.lab.bin_path, f)
break
else:
if not os.path.isabs(mppnc_path):
mppnc_path = os.path.join(self.expt.lab.bin_path, mppnc_path)
assert mppnc_path
# Check config for collate command line options
collate_flags = self.expt.config.get('collate_flags')
if collate_flags is None:
collate_flags = '-n4 -z -m -r'
# Import list of collated files to ignore
collate_ignore = self.expt.config.get('collate_ignore')
if collate_ignore is None:
collate_ignore = []
elif type(collate_ignore) != list:
collate_ignore = [collate_ignore]
# Generate collated file list and identify the first tile
tile_fnames = [
f for f in os.listdir(self.output_path)
if f[-4:].isdigit() and f[-8:-4] == '.nc.'
]
mnc_tiles = defaultdict(list)
for t_fname in tile_fnames:
t_base, t_ext = os.path.splitext(t_fname)
t_ext = t_ext.lstrip('.')
# Skip any files listed in the ignore list
if t_base in collate_ignore:
continue
mnc_tiles[t_base].append(t_fname)
# If this is run interactively NCPUS is set in collate_cmd, otherwise
# the cpu_count will return the number of CPUs assigned to the PBS job
count = int(os.environ.get('NCPUS', multiprocessing.cpu_count()))
pool = multiprocessing.Pool(processes=count)
# Collate each tileset into a single file
results = []
for nc_fname in mnc_tiles:
nc_path = os.path.join(self.output_path, nc_fname)
# Remove the collated file if it already exists, since it is
# probably from a failed collation attempt
# TODO: Validate this somehow
if os.path.isfile(nc_path):
os.remove(nc_path)
cmd = '{} {} {} {}'.format(mppnc_path, collate_flags, nc_fname,
' '.join(mnc_tiles[nc_fname]))
print(cmd)
result = pool.apply_async(cmdthread, args=(cmd, self.output_path))
results.append(result.get())
pool.close()
pool.join()
# TODO: Categorise the return codes
if any(rc is not None for rc in results):
for p, rc in enumerate(results):
if rc is not None:
print('payu: error: Thread {} crased with error code {}.'
''.format(p, rc),
file=sys.stderr)
sys.exit(-1)
# vim:fileencoding=utf-8
import re, urllib2
import multiprocessing.dummy as dummy
charset_re = re.compile(r'charset=([\w-]+)')
title_re = re.compile(r'<title>([^<]+)</title>')
def captitle(url):
req = urllib2.Request(url)
req.add_header('User-Agent',
'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')
f = urllib2.urlopen(req)
content = f.read()
title = (['unkown'] + title_re.findall(content))[-1]
enc = ''
enc = enc or ([''] +
charset_re.findall(f.headers.dict['content-type']))[-1]
enc = enc or ([''] + charset_re.findall(content))[-1]
enc = enc or 'utf-8'
return url, title.decode(enc)
pool = dummy.Pool(16)
urls = [url.strip() for url in file('1.txt').readlines()]
for url, title in pool.map(captitle, urls):
print '%s : %s' % (url, title)
