def test_api(client_key):
# 1. create test image data and both processing and result queues
urls = ['https://demo.restb.ai/images/demo/demo-1.jpg',
'https://demo.restb.ai/images/demo/demo-2.jpg',
'https://demo.restb.ai/images/demo/demo-3.jpg',
'https://demo.restb.ai/images/demo/demo-4.jpg',
'https://demo.restb.ai/images/demo/demo-5.jpg',
'https://demo.restb.ai/images/demo/demo-6.jpg']
queue = mp.Queue()
image_id = 1
for url in urls:
for model in __MODELS.keys():
queue.put(dict(id=image_id, url=url, model=model))
image_id += 1
results = mp.Queue()
# 2. Pick which API endpoint to use (US vs. EU)
url = __URL_US
# 3. Define concurrency specific objects
# stats objects
lock_stats = mp.Lock()
counter = mp.Value('i', 0)
avg_req_time = mp.Value('f', 0)
time_start = mp.Value('f', 999999999999999)
time_end = mp.Value('f', 0)
# 4. Spawn processes/threads to process the images in the queue
pool = []
for i in range(__requests_per_second):
# pass in necessary parameters to thread, including client key, etc.
p = mp.Process(target=image_process_thread,
args=(url, client_key, queue, results,
lock_stats, counter, avg_req_time, time_start, time_end))
pool.append(p)
p.start()
# 5. clean-up after queue has been processed with "poison pill"
while not queue.empty():
# wait for queue to be processed
time.sleep(1)
for i in pool:
# seed shutdown messages / poison pills
queue.put(dict(id=-1, url='shutdown', model='shutdown'))
for p in pool:
# enforce clean shutdown of threads
p.join()
# 6. finally, return accumulated results
total = time_end.value - time_start.value
print('[{requests}] requests processed in [{seconds}] seconds with average time [{time}] ms, total throughput: [{throughput}] rps'.format(
requests=counter.value,
seconds=str(round(total / 1000.0, 1)),
time=str(round(avg_req_time.value / counter.value, 0)),
throughput=str(round(counter.value / (total / 1000.0), 2))
))
return results
def main(argv):
parse_args(argv)
init_config()
get_password()
authenticate()
q = multiprocessing.Queue()
# if we're processing a directory which contains X number of json files
if os.path.isfile(config['inputpath']):
q.put(config['inputpath'])
config['num_threads'] = 1
else:
for f in os.listdir('.'):
q.put(f)
threads = []
for i in range(config['num_threads']):
t = multiprocessing.Process(target=upload_dispatcher, args=(q, ))
t.setDaemon(True)
threads.append(t)
t.start()
q.put(None)
for t in threads:
t.join()
def __init__(self):
self.cellDict = {}
self.queue = multiprocessing.Queue()
logger.info('start process for parse Msg')
self.process = multiprocessing.Process(target=parseMsg,
args=(self, self.queue, logger))
self.process.start()
def __init__(self, cases=None):
"""
:param cases: case 列表
"""
super(Runner, self).__init__()
self.__cases = cases
self.__tasks = multithreading.JoinableQueue()
self.__results = multithreading.Queue()
def __init__(self,
seeds=2,
token_file_name="tokens.csv",
seed_list=None,
following_pages_limit=0):
# Get seeds from seeds.csv
self.seed_pool = FileImport().read_seed_file()
# Create seed_list if none is given by sampling from the seed_pool
if seed_list is None:
self.number_of_seeds = seeds
try:
self.seeds = self.seed_pool.sample(n=self.number_of_seeds)
except ValueError: # seed pool too small
stderr.write(
"WARNING: Seed pool smaller than number of seeds.\n")
self.seeds = self.seed_pool.sample(n=self.number_of_seeds,
replace=True)
self.seeds = self.seeds[0].values
else:
self.number_of_seeds = len(seed_list)
self.seeds = seed_list
self.seed_queue = mp.Queue()
for seed in self.seeds:
self.seed_queue.put(seed)
# Get authorized user tokens for app from tokens.csv
self.tokens = FileImport().read_token_file(token_file_name)
# and put them in a queue
self.token_queue = mp.Queue()
for token, secret in self.tokens.values:
self.token_queue.put((token, secret, {}, {}))
# Initialize DataBaseHandler for DB communication
self.dbh = DataBaseHandler()
self.following_pages_limit = following_pages_limit
def main():
db = sys.argv[1]
conn = sqlite3.connect(db, detect_types=sqlite3.PARSE_DECLTYPES)
cursor = conn.cursor()
cursor.execute("SELECT MIN(date) FROM input")
for mindate, in cursor.fetchall():
mindate = datetime.strptime(mindate, "%Y-%m-%d %H:%M:%S")
cursor.execute("SELECT MAX(date) FROM input")
for maxdate, in cursor.fetchall():
maxdate = datetime.strptime(maxdate, "%Y-%m-%d %H:%M:%S")
days = (maxdate - mindate).days
t_queue = []
for i in range(days + 2):
t_queue.append((i, mindate + timedelta(i), mindate + timedelta(i + 1)))
cursor.execute("DELETE FROM temporary_label_clusters")
result_queue = mp.Queue()
pool = mp.Pool(1)
robj = pool.imap_unordered(lambda x: process(x, db, result_queue), t_queue)
output_file = open('output.tmp', 'w')
# TODO: write result processing
max_cluster_id = 0
for x in robj:
tmp_cluster_id = 0
for line in x.split("\n"):
line = line.strip()
if len(line) == 0:
continue
identifier, _, cluster = line.partition(' ')
identifier = int(identifier)
cluster = int(cluster)
if cluster > tmp_cluster_id:
tmp_cluster_id = cluster
if cluster == 0:
continue
cluster += max_cluster_id
print >> output_file, identifier, cluster
max_cluster_id += tmp_cluster_id
def get_dbs():
q = multiprocessing.Queue()
unique = set()
# if a checkpoint exists, run _db_updates with since parameter, otherwise just run _db_updates
if config['incremental'] and os.path.exists(config['checkpointpath']):
f = open(config['checkpointpath'], 'r')
checkpoint_seq = f.read()
seq_obj = requests.get('{0}_db_updates?since={1}'.format(
config['baseurl'], checkpoint_seq),
headers=config['authheader']).json()
# write the latest checkpoint
write_checkpoint(seq_obj)
for db_object in seq_obj['results']:
db = db_object['dbname']
if db not in ['_replicator', 'metrics', 'dbs'
] and db not in unique:
unique.add(db)
q.put(db)
else:
# we need to get the latest sequence number to prepare for future incrementals
r = requests.get('{0}_db_updates?limit=0&descending=true'.format(
config['baseurl']),
headers=config['authheader'])
if r.status_code != 200:
print 'Warning: Failed to retrieve a sequence number. Please ensure the global changes feed is enabled.'
else:
# write the latest checkpoint
write_checkpoint(r.json())
# now we grab all the databases in the account.
dbs = requests.get('{0}_all_dbs'.format(config['baseurl']),
headers=config['authheader']).json()
for db in dbs:
if db not in ['_replicator', 'metrics', 'dbs']:
q.put(db)
# return the queue of databses
return q
def __init__(self, token_file_name="tokens.csv", token_queue=None):
self.credentials = FileImport().read_app_key_file()
self.ctoken = self.credentials[0]
self.csecret = self.credentials[1]
if token_queue is None:
self.tokens = FileImport().read_token_file(token_file_name)
self.token_queue = mp.Queue()
for token, secret in self.tokens.values:
self.token_queue.put((token, secret, {}, {}))
else:
self.token_queue = token_queue
self.token, self.secret, self.reset_time_dict, self.calls_dict = self.token_queue.get(
)
self.auth = tweepy.OAuthHandler(self.ctoken, self.csecret)
self.auth.set_access_token(self.token, self.secret)
self.api = tweepy.API(self.auth,
wait_on_rate_limit=False,
wait_on_rate_limit_notify=False)
def main():
print 'Starting bulk loading program...'
print 'List of test runs: {0}'.format(runs)
print 'Output file: {0}'.format(output_file)
with open(output_file, 'r+') as title:
title.write('Time,Threads,Request Size,Requests/Second,Docs Written')
for run in runs:
body_size = run['body_size']
trials = run['trials']
for trial in range(trials):
# read the external json file
print 'Reading {0} for test document to post...'.format(filename)
with open(filename) as fh:
doc = json.load(fh)
time.sleep(1)
# build the request body we want to send
print 'Building a {0} document request body...'.format(body_size)
request_body = {'docs': []}
for i in range(body_size):
request_body['docs'].append(doc)
time.sleep(1)
# delete the database if it exists and then recreate it
print 'Re-initializing the testing database...'
r = requests.get(base_url + '_all_dbs',
auth=(user, pwd),
verify=False).json()
if db in r:
requests.delete(base_url + db, auth=(user, pwd), verify=False)
requests.put(base_url + db, auth=(user, pwd), verify=False)
# insert ddocs if needed
if ddoc_filename != '':
print 'Posting ddocs to the database...'
with open(ddoc_filename) as df:
ddocs = json.load(df)
requests.post(bulk_url,
data=json.dumps(ddocs),
headers={'content-type': 'application/json'},
auth=(user, pwd),
verify=False)
time.sleep(1)
# start to create threads
print 'Initializing {0} threads...'.format(num_threads)
threads = []
q = multiprocessing.Queue()
for j in range(num_threads):
t = multiprocessing.Process(target=execute,
args=(q, request_body))
threads.append(t)
print '{0} threads created...'.format(num_threads)
# start the threads
for thread in threads:
thread.start()
print 'All threads started...'
# wait until they are done
for k in threads:
k.join()
# sum total docs written from the queue
total_requests = 0
while not q.empty():
total_requests += q.get()
# compute metrics
requests_per_second = float(total_requests) / interval
docs_written = total_requests * body_size
with open(output_file, 'a') as oh:
oh.write('\n{0},{1},{2},{3},{4}'.format(
interval, num_threads, body_size, requests_per_second,
docs_written))
print 'Trial complete...'
time.sleep(120)
print 'TEST RUN COMPLETE'
for page in range(1, 50 + 1):
pool.apply_async(self._get_urls,
args=(page, city),
callback=self._gu_callback)
pool.close()
pool.join()
pool = mpd.Pool()
for url in self.urls:
pool.apply_async(self._get_detail, (url, ),
callback=self._gd_callback)
pool.close()
pool.join()
SPIPQ = mpd.Queue()
SPIQ = mpd.Queue()
def run() -> None:
"""Run a spider."""
carspi = CarSpi()
carspi.run()
datas = list(carspi.details)
SPIQ.put(datas)
def w_data() -> int:
"""Write datas."""
while True:
try:
def main(subreddits):
"""
Get comment stream for subreddits and process them. Will continue
until interrupted.
Parameters
----------
subreddits : list[str]
The subreddits to process comments from
"""
logger = resources.LOGGER
reddit = resources.load_reddit()
username = reddit.user.me().name
comments, submissions = get_streams(subreddits)
main_queue = mp.Queue(1024)
stop_event = mp.Event() # for stopping workers
# save the reply dict when the script exits
atexit.register(lambda: logger.info("Exited nbviewerbot"))
# create workers to add praw objects to the queue
workers = []
comments_worker = mp.DummyProcess(
name="CommentWorker",
target=utils.load_queue,
args=(main_queue, comments, stop_event),
)
workers.append(comments_worker)
submissions_worker = mp.DummyProcess(
name="SubmissionWorker",
target=utils.load_queue,
args=(main_queue, submissions, stop_event),
)
workers.append(submissions_worker)
# make sure workers end on main thread end
atexit.register(lambda e: e.set(), stop_event)
# let's get it started in here
[w.start() for w in workers]
logger.info("Started nbviewerbot, listening for new comments...")
while not stop_event.is_set():
try:
praw_obj = main_queue.get(timeout=1)
process_praw_object(praw_obj, username)
except queue.Empty:
pass # no problems, just nothing in the queue
except KeyboardInterrupt:
stop_event.set()
logger.warn("Stopping nbviewerbot...")
except:
stop_event.set()
logger.exception(
"Uncaught exception on object, skipping. Details:")
raise
if not all([w.is_alive() for w in workers]):
stop_event.set()
raise InterruptedError("Praw worker died unexpectedly")
def ParallelIterator(keras_sequence,
epochs,
shuffle,
use_on_epoch_end,
workers=4,
queue_size=10):
sourceQueue = mp.Queue() # queue for getting batch indices
batchQueue = mp.Queue(
maxsize=queue_size) # queue for getting actual batches
indices = np.arange(len(keras_sequence)) # array of indices to be shuffled
use_on_epoch_end = 'on_epoch_end' in dir(
keras_sequence) if use_on_epoch_end == True else False
batchesLeft = 0
# printQueue = mp.Queue() # queue for printing messages
# import threading
# screenLock = threading.Semaphore(value=1)
# totalWorkers= 0
# def printer():
# nonlocal printQueue, printing
# while printing:
# while not printQueue.empty():
# text = printQueue.get(block=True)
# screenLock.acquire()
# print(text)
# screenLock.release()
# fills the batch indices queue (called when sourceQueue is empty -> a few batches before an epoch ends)
def fillSource():
nonlocal batchesLeft
# printQueue.put("Iterator: fill source - source qsize = " + str(sourceQueue.qsize()))
if shuffle == True:
np.random.shuffle(indices)
# puts the indices in the indices queue
batchesLeft += len(indices)
# printQueue.put("Iterator: batches left:" + str(batchesLeft))
for i in indices:
sourceQueue.put(i)
# function that will load batches from the Keras Sequence
def worker():
nonlocal sourceQueue, batchQueue, keras_sequence, batchesLeft
# nonlocal printQueue, totalWorkers
# totalWorkers += 1
# thisWorker = totalWorkers
while True:
# printQueue.put('Worker: ' + str(thisWorker) + ' will try to get item')
index = sourceQueue.get(block=True) # get index from the queue
# printQueue.put('Worker: ' + str(thisWorker) + ' got item ' + str(index) + " - source q size = " + str(sourceQueue.qsize()))
if index is None:
break
item = keras_sequence[index] # get batch from the sequence
batchesLeft -= 1
# printQueue.put('Worker: ' + str(thisWorker) + ' batches left ' + str(batchesLeft))
batchQueue.put((index, item),
block=True) # puts batch in the batch queue
# printQueue.put('Worker: ' + str(thisWorker) + ' added item ' + str(index) + ' - queue: ' + str(batchQueue.qsize()))
# printQueue.put("hitting end of worker" + str(thisWorker))
# #printing pool that will print messages from the print queue
# printing = True
# printPool = mp.Pool(1, printer)
# creates the thread pool that will work automatically as we get from the batch queue
pool = mp.Pool(workers, worker)
fillSource(
) # at this point, data starts being taken and stored in the batchQueue
# generation loop
for epoch in range(epochs):
# if not waiting for epoch end synchronization, always keeps 1 epoch filled ahead
if (use_on_epoch_end == False):
if epoch + 1 < epochs: # only fill if not last epoch
fillSource()
for batch in range(len(keras_sequence)):
# if waiting for epoch end synchronization, wait for workers to have no batches left to get, then call epoch end and fill
if use_on_epoch_end == True:
if batchesLeft == 0:
keras_sequence.on_epoch_end()
if epoch + 1 < epochs: # only fill if not last epoch
fillSource()
else:
batchesLeft = -1 # in the last epoch, prevents from calling epoch end again and again
# yields batches for the outside loop that is using this generator
originalIndex, batchItems = batchQueue.get(block=True)
yield epoch, batch, originalIndex, batchItems
# print("iterator epoch end")
# printQueue.put("closing threads")
# terminating the pool - add None to the queue so any blocked worker gets released
for i in range(workers):
sourceQueue.put(None)
pool.terminate()
pool.close()
pool.join()
# printQueue.put("terminated")
# printing = False
# printPool.terminate()
# printPool.close()
# printPool.join()
del pool, sourceQueue, batchQueue
def main(args):
cap = cv2.VideoCapture(args.INPUT)
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
reported_fps = int(cap.get(cv2.CAP_PROP_FPS))
reported_bitrate = int(cap.get(cv2.CAP_PROP_BITRATE))
print("Total frames: {0}".format(total_frames))
print("Reported FPS: {0}".format(reported_fps))
print("Reported Bitrate: {0}kbps".format(reported_bitrate))
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
size = (width, height)
original_path = None
if args.OUTPUT:
original_path = Path(args.OUTPUT)
else:
original_path = Path(args.INPUT)
original_name = original_path.stem
original_parent = original_path.parent
# Video with duplicated frames removed
result_queue = None
result_proc = None
if args.SAVE in (1, 3):
result_name = Path(original_parent).joinpath(original_name +
"_result.avi")
result_name.unlink(missing_ok=True)
result_queue = mp.Queue()
result_proc = mp.Process(target=writer_create,
args=(
str(result_name),
reported_fps,
size,
result_queue,
))
result_proc.start()
# Video with difference blend mode between original and result video
diff_queue = None
diff_proc = None
if args.SAVE in (2, 3):
diff_name = Path(original_parent).joinpath(original_name + "_diff.avi")
diff_name.unlink(missing_ok=True)
diff_queue = mp.Queue()
diff_proc = mp.Process(target=writer_create,
args=(
str(diff_name),
reported_fps,
size,
diff_queue,
))
diff_proc.start()
frames = []
frame_number = -1
prev_frame = None
# with tqdm(total=total_frames, unit="frames") as prog_bar:
time_start = time.time()
prog_bar = tqdm(total=total_frames, unit="frames", leave=True)
while (cap.isOpened()):
frame_number += 1
prog_bar.set_description(
"Processing frame number {}".format(frame_number))
prog_bar.update(1)
ret, frame = cap.read()
if frame_number == 0:
prev_frame = frame.copy()
continue
try:
# frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# prev_frame_gray = cv2.cvtColor(prev_frame, cv2.COLOR_BGR2GRAY)
# frame_diff = cv2.absdiff(frame_gray, prev_frame_gray)
frame_diff = cv2.absdiff(frame, prev_frame)
if diff_queue is not None:
# diff_writer.write(frame_diff)
diff_queue.put(frame_diff)
mean = frame_diff.mean()
if mean > args.THRESHOLD:
frames.append(True)
if result_queue is not None:
# result_writer.write(frame)
result_queue.put(frame)
else:
frames.append(False)
prev_frame = frame.copy()
except KeyboardInterrupt:
exit(1)
except Exception as e:
# print("\r\n{0}".format(e))
if frame_number > total_frames:
break
else:
continue
if frame_number > total_frames:
break
time_stop = time.time()
n = prog_bar.n
prog_bar.close()
msg = "Calculations "
cap.release()
if result_queue is not None:
print("Finishing writing to result video file.")
result_queue.put("STOP")
result_proc.join()
msg += "and writing to the result file "
if diff_queue is not None:
print("Finishing writing to difference video file.")
diff_queue.put("STOP")
diff_proc.join()
msg += "and writing to the difference file "
cv2.destroyAllWindows()
time_total = time_stop - time_start
app_fps = n / time_total
msg += "took {0} seconds to complete.\n"
msg += "Average frames calculated per second is {1}."
print(msg.format(time_total, app_fps))
res = [i for i in frames if i]
times = dict()
base = float((1 / reported_fps) * 1000)
for i in range(len(res)):
if i == 0:
times[i] = base
else:
times[i] = base + (base * (res[i] - res[i - 1]))
data = dict()
data["frame number"] = []
data["frametime"] = []
data["framerate"] = []
for k, v in times.items():
data["frame number"].append(k)
data["frametime"].append(v)
if v == 0:
data["framerate"].append("INF")
else:
data["framerate"].append(1 / (v / 1000))
df = pd.DataFrame.from_dict(data)
frametime_stats = pd.DataFrame(df, columns=["frametime"])
framerate_stats = pd.DataFrame(df, columns=["framerate"])
stats_basic = dict()
stats_frametime_dict = dict()
stats_framerate_dict = dict()
stats_basic["Number of Unique Frames"] = [int(sum(frames))]
stats_basic["Number of Duplicated Frames"] = [
int(len(frames) - sum(frames))
]
if len(frames) == 0:
stats_basic["Percentage of Unique Frames"] = ["0 %"]
stats_basic["Percentage of Duplicated Frames"] = ["0 %"]
else:
stats_basic["Percentage of Unique Frames"] = [
"{} %".format(sum(frames) / len(frames) * 100)
]
stats_basic["Percentage of Duplicated Frames"] = [
"{} %".format(stats_basic["Number of Duplicated Frames"][0] /
len(frames) * 100)
]
stats_frametime_dict["Lowest"] = dict(frametime_stats.min(axis=0))
stats_frametime_dict["Highest"] = dict(frametime_stats.max(axis=0))
stats_frametime_dict["Mean"] = dict(frametime_stats.mean(axis=0))
stats_frametime_dict["Median"] = dict(frametime_stats.median(axis=0))
stats_frametime_dict["0.1 Percent Lows"] = dict(
frametime_stats.quantile(q=0.001, axis=0))
stats_frametime_dict["1 Percent Lows"] = dict(
frametime_stats.quantile(q=0.01, axis=0))
stats_frametime_dict["99 Percent Lows"] = dict(
frametime_stats.quantile(q=0.99, axis=0))
stats_frametime_dict["99.9 Percent Lows"] = dict(
frametime_stats.quantile(q=0.999, axis=0))
stats_framerate_dict["Lowest"] = dict(framerate_stats.min(axis=0))
stats_framerate_dict["Highest"] = dict(framerate_stats.max(axis=0))
stats_framerate_dict["Mean"] = dict(framerate_stats.mean(axis=0))
stats_framerate_dict["Median"] = dict(framerate_stats.median(axis=0))
stats_framerate_dict["0.1 Percent Lows"] = dict(
framerate_stats.quantile(q=0.001, axis=0))
stats_framerate_dict["1 Percent Lows"] = dict(
framerate_stats.quantile(q=0.01, axis=0))
stats_framerate_dict["99 Percent Lows"] = dict(
framerate_stats.quantile(q=0.99, axis=0))
stats_framerate_dict["99.9 Percent Lows"] = dict(
framerate_stats.quantile(q=0.999, axis=0))
stats_basic_df = pd.DataFrame.from_dict(stats_basic)
stats_frametime_df = pd.DataFrame.from_dict(stats_frametime_dict)
stats_framerate_df = pd.DataFrame.from_dict(stats_framerate_dict)
stats_joined = pd.concat([stats_frametime_df, stats_framerate_df], axis=0)
print("\nStatistics")
print(stats_basic_df.transpose().to_string(header=False))
print("\n", stats_joined.transpose().to_string())
csv_name = Path(original_parent).joinpath(original_name + "_report.csv")
csv_name.unlink(csv_name)
df.to_csv(csv_name, index=False)
def next(self, nsolutions=None):
Log('=' * 80)
Log('Simplex Random Walk')
Log('=' * 80)
Log(" %i equations" % len(self.eq_list))
Log("%6s %6s %6s\n%6i %6i %6i" %
(">=", "<=", "=", self.geq_count, self.leq_count, self.eq_count))
if nsolutions == 0: return
assert nsolutions is not None
dim = self.nVars
dof = dim - self.eq_count
burnin_len = max(10, int(self.burnin_factor * dof))
redo = max(100, int((dof**self.redo_exp) * self.redo_factor))
nmodels = nsolutions
nthreads = self.nthreads
self.stride = int(dim + 1)
n_stored = 0
self.dim = dim
self.dof = dof
self.redo = redo
self.burnin_len = burnin_len
accept_rate = self.accept_rate
accept_rate_tol = self.accept_rate_tol
store = np.zeros((dim, 1 + burnin_len),
order='Fortran',
dtype=np.float64)
newp = np.zeros(dim, order='C', dtype=np.float64)
eval = np.zeros(dim, order='C', dtype=np.float64)
evec = np.zeros((dim, dim), order='F', dtype=np.float64)
self.eqs = np.zeros((self.eqn_count + dim, dim + 1),
order='C',
dtype=np.float64)
for i, [c, e] in enumerate(self.eq_list):
self.eqs[i, :] = e
for i in xrange(dim):
self.eqs[self.eqn_count + i, 1 + i] = 1
self.dist_eqs = np.zeros((self.eqn_count - self.eq_count, dim + 1),
order='C',
dtype=np.float64)
i = 0
for c, e in self.eq_list:
if c == 'eq':
continue
elif c == 'leq':
p = e
elif c == 'geq':
p = -e
self.dist_eqs[i, :] = p
i += 1
Log('Using lpsolve %s' % lpsolve('lp_solve_version'))
Log("random seed = %s" % self.random_seed)
Log("threads = %s" % self.nthreads)
Log("acceptence rate = %s" % self.accept_rate)
Log("acceptence rate tolerance = %s" % self.accept_rate_tol)
Log("dof = %s" % self.dof)
Log("sample distance = max(100,%s * %s^%s) = %s" %
(self.redo_factor, self.dof, self.redo_exp, redo))
Log("starting twiddle = %s" % self.twiddle)
Log("burn-in length = %s" % burnin_len)
time_begin_next = time.clock()
#-----------------------------------------------------------------------
# Create pseudo inverse matrix to reproject samples back into the
# solution space.
#-----------------------------------------------------------------------
P = np.eye(dim)
if self.eq_count > 0:
self.A = np.zeros((self.eq_count, dim),
order='C',
dtype=np.float64)
self.b = np.zeros(self.eq_count, order='C', dtype=np.float64)
for i, [c, e] in enumerate(self.eq_list[:self.eq_count]):
self.A[i] = e[1:]
self.b[i] = e[0]
self.Apinv = pinv(self.A)
P -= np.dot(self.Apinv, self.A)
else:
self.A = None
self.B = None
self.Apinv = None
ev, evec = eigh(P)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# Find a point that is completely inside the simplex
#-----------------------------------------------------------------------
Log('Finding first inner point')
time_begin_inner_point = time.clock()
self.inner_point(newp)
time_end_inner_point = time.clock()
ok, fail_count = self.in_simplex(newp, eq_tol=1e-12, tol=0, verbose=1)
assert ok
self.avg0 = newp
# eqs = self.eqs.copy('A')
# eqs[:,1:] = np.dot(self.eqs[:,1:], evec)
# print newp
# S = zeros(self.eqs.shape[0])
# newp[:] = np.dot(evec.T, newp)
# newp0 = newp.copy()
# steps = newp.copy()
# for q in range(100):
# csamplex.refine_center(self, eqs, newp, ev, S, steps)
# d = newp - newp0
# #print d
# print norm(d)
# #print
# newp0 = newp.copy()
# #assert 0
# newp[:] = np.dot(evec, newp)
store[:, 0] = newp
n_stored = 1
#-----------------------------------------------------------------------
# Estimate the eigenvectors of the simplex
#-----------------------------------------------------------------------
Log('Estimating eigenvectors')
time_begin_est_eigenvectors = time.clock()
self.measured_ev(newp, ev, eval, evec)
time_end_est_eigenvectors = time.clock()
#-----------------------------------------------------------------------
# Now we can start the random walk
#-----------------------------------------------------------------------
Log("Getting solutions")
q = MP.Queue()
ran_set_seed(self.random_seed)
seeds = np.random.choice(1000000 * nthreads, nthreads, replace=False)
#-----------------------------------------------------------------------
# Launch the threads
#-----------------------------------------------------------------------
threads = []
models_per_thread = nmodels // nthreads
models_under = nmodels - nthreads * models_per_thread
id, N = 0, 0
while id < nthreads and N < nmodels:
n = models_per_thread
if id < models_under:
n += 1
assert n > 0
Log('Thread %i gets %i' % (id, n))
cmdq = MP.Queue()
ackq = MP.Queue()
thr = MP.Process(target=rwalk_burnin,
args=(id, n, int(np.ceil(burnin_len / nthreads)),
self, q, cmdq, ackq, newp, self.twiddle,
eval.copy('A'), evec.copy('A'), seeds[id]))
threads.append([thr, cmdq, ackq])
N += n
id += 1
assert N == nmodels
for thr, cmdq, _ in threads:
thr.daemon = True
thr.start()
cmdq.put(['CONT'])
def drainq(q):
try:
while True:
q.get(block=False)
except QueueEmpty:
pass
def pause_threads(threads):
for _, cmdq, ackq in threads:
cmdq.put(['WAIT'])
assert ackq.get() == 'OK'
def adjust_threads(i, cont_cmd):
pause_threads(threads)
drainq(q)
Log('Computing eigenvalues... [%i/%i]' % (i, burnin_len))
self.compute_eval_evec(store, eval, evec, n_stored)
# new twiddle <-- average twiddle
t = 0
for _, cmdq, ackq in threads:
cmdq.put(['REQ TWIDDLE'])
t += ackq.get()
t /= len(threads)
Log('New twiddle %f' % t)
for _, cmdq, _ in threads:
cmdq.put(['NEW DATA', [eval.copy('A'), evec.copy('A'), t]])
cmdq.put([cont_cmd])
#-----------------------------------------------------------------------
# Burn-in
#-----------------------------------------------------------------------
time_begin_burnin = time.clock()
compute_eval_window = 2 * self.dof
j = 0
for i in xrange(burnin_len):
j += 1
k, vec = q.get()
store[:, n_stored] = vec
n_stored += 1
if j == compute_eval_window:
j = 0
adjust_threads(i + 1, 'CONT')
compute_eval_window = int(0.1 * burnin_len + 1)
elif len(threads) < compute_eval_window:
threads[k][1].put(['CONT'])
time_end_burnin = time.clock()
#-----------------------------------------------------------------------
# Actual random walk
#-----------------------------------------------------------------------
time_begin_get_models = time.clock()
adjust_threads(burnin_len, 'RWALK')
i = 0
while i < nmodels:
k, vec = q.get()
t = np.zeros(dim + 1, order='Fortran', dtype=np.float64)
t[1:] = vec
i += 1
Log('%i models left to generate' % (nmodels - i),
overwritable=True)
yield t
time_end_get_models = time.clock()
#-----------------------------------------------------------------------
# Stop the threads and get their running times.
#-----------------------------------------------------------------------
time_threads = []
for thr, cmdq, ackq in threads:
cmdq.put(['STOP'])
m, t = ackq.get()
assert m == 'TIME'
time_threads.append(t)
#thr.terminate()
time_end_next = time.clock()
max_time_threads = np.amax(time_threads) if time_threads else 0
avg_time_threads = np.mean(time_threads) if time_threads else 0
Log('-' * 80)
Log('SAMPLEX TIMINGS')
Log('-' * 80)
Log('Initial inner point %.2fs' %
(time_end_inner_point - time_begin_inner_point))
Log('Estimate eigenvectors %.2fs' %
(time_end_est_eigenvectors - time_begin_est_eigenvectors))
Log('Burn-in %.2fs' %
(time_end_burnin - time_begin_burnin))
Log('Modeling %.2fs' %
(time_end_get_models - time_begin_get_models))
Log('Max/Avg thread time %.2fs %.2fs' %
(max_time_threads, avg_time_threads))
Log('Total wall-clock time %.2fs' % (time_end_next - time_begin_next))
Log('-' * 80)
