def test_training(sagemaker_session, ecr_image, instance_type, instance_count):
hyperparameters = {
'sagemaker_parameter_server_enabled': True
} if instance_count > 1 else {}
hyperparameters['epochs'] = 1
mx = MXNet(entry_point=SCRIPT_PATH,
role='SageMakerRole',
train_instance_count=instance_count,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
image_name=ecr_image,
hyperparameters=hyperparameters)
with timeout(minutes=15):
prefix = 'mxnet_mnist/{}'.format(utils.sagemaker_timestamp())
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(DATA_PATH, 'train'),
key_prefix=prefix + '/train')
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(DATA_PATH, 'test'), key_prefix=prefix + '/test')
job_name = utils.unique_name_from_base('test-mxnet-image')
mx.fit({'train': train_input, 'test': test_input}, job_name=job_name)
dgl = MXNet(entry_point=DGL_SCRIPT_PATH,
role='SageMakerRole',
train_instance_count=1,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
image_name=ecr_image)
with timeout(minutes=15):
job_name = utils.unique_name_from_base('test-mxnet-dgl-image')
dgl.fit(job_name=job_name)
def __check_go(self, chessboard):
self.agent = imp.load_source('AI', self.script_file_path).AI(self.chessboard_size, -1, self.time_out)
try:
timeout(self.time_out)(self.agent.go)(np.copy(chessboard))
except Exception:
self.errormsg = "Error:" + traceback.format_exc()
return False
return True
def getConfirmMail(mb):
try:
with timeout(seconds=20):
msg_confirm = mb.getCorreo()
return getConfirmUrl(msg_confirm)
except:
resend_mail()
try:
with timeout(seconds=20):
msg_confirm = mb.getCorreo()
return getConfirmUrl(msg_confirm)
except:
print "No ha llegado el correo"
exit(90)
def infer_type(filepath):
with timeout(seconds=15):
if get_extension(filepath) in img_extensions:
return "image"
try:
extract_netcdf(filepath)
return "netcdf"
except Exception as e:
if e.__class__ == TimeoutError:
return "unknown"
pass
try:
extract_json_metadata(filepath)
return "json/xml"
except Exception as e:
if e.__class__ == TimeoutError:
return "unknown"
pass
try:
extract_columnar_metadata(filepath, parallel=False)
return "tabular"
except Exception as e:
if e.__class__ == TimeoutError:
return "unknown"
pass
try:
if extract_keyword(filepath)["keywords"]:
return "freetext"
else:
pass
except Exception as e:
if e.__class__ == TimeoutError:
return "unknown"
pass
return "unknown"
def _test_mnist_train(sagemaker_session, ecr_image, instance_type,
instance_count, script):
source_dir = 'test/resources/mnist'
with timeout(minutes=15):
data_path = 'test/resources/mnist/data'
chainer = Chainer(entry_point=script,
source_dir=source_dir,
role='SageMakerRole',
train_instance_count=instance_count,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
image_name=ecr_image,
hyperparameters={
'batch-size': 10000,
'epochs': 1
})
prefix = 'chainer_mnist/{}'.format(sagemaker_timestamp())
train_data_path = os.path.join(data_path, 'train')
key_prefix = prefix + '/train'
train_input = sagemaker_session.upload_data(path=train_data_path,
key_prefix=key_prefix)
test_path = os.path.join(data_path, 'test')
test_input = sagemaker_session.upload_data(path=test_path,
key_prefix=prefix + '/test')
chainer.fit({'train': train_input, 'test': test_input})
def get_proxies(num_proxies = 5):
print("INTIALIZING {} PROXIES".format(num_proxies))
# Go to the proxy list url, and get their website text
# url = 'https://free-proxy-list.net/'
url = 'https://free-proxy-list.net/'
response = requests.get(url)
parser = fromstring(response.text)
proxies = set()
# For each potential proxy, add it if it works.
for i in parser.xpath('//tbody/tr'):
if i.xpath('.//td[7][contains(text(),"yes")]'):
#Grabbing IP and corresponding PORT
proxy = ":".join([i.xpath('.//td[1]/text()')[0], i.xpath('.//td[2]/text()')[0]])
try:
sleep(.5)
# Check the proxy
urls = ["http://www.neopets.com/", "https://www.theatlantic.com/", "https://www.reddit.com/r/funny/", "http://www.pythonforbeginners.com/"]
url = random.sample(urls, 1)[0]
print(url)
print(proxy)
with timeout(seconds=2):
response = requests.get(url,proxies={"http": proxy, "https": proxy})
# If it works, add the proxy
proxies.add(proxy)
print("="*50)
print("We have added {} proxies of a total of {} proxies".format(len(proxies),num_proxies))
print("{}% Complete with initializing proxies".format(round(len(proxies)/num_proxies*100,2)))
if len(proxies) == num_proxies:
return proxies
except:
print("Failed Connection")
continue
return proxies
def test_mxnet_distributed(sagemaker_session, ecr_image, instance_type,
framework_version):
data_path = os.path.join(RESOURCE_PATH, 'mnist')
script_path = os.path.join(data_path, 'mnist.py')
mx = MXNet(entry_point=script_path,
role='SageMakerRole',
train_instance_count=2,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
image_name=ecr_image,
framework_version=framework_version,
hyperparameters={'sagemaker_parameter_server_enabled': True})
prefix = 'mxnet_mnist/{}'.format(sagemaker_timestamp())
with timeout(minutes=15):
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, 'train'),
key_prefix=prefix + '/train')
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, 'test'), key_prefix=prefix + '/test')
mx.fit({'train': train_input, 'test': test_input})
with timeout_and_delete_endpoint(estimator=mx, minutes=30):
predictor = mx.deploy(initial_instance_count=1,
instance_type=instance_type)
data = np.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def search(b, w, mover, passes=0, getval=False, timelim=2, depthlim=None, heur=None, printdata=True, forever=False):
global start, ttable, h_current, totdepth, numsearch
start = time.time()
movesleft = 64 - bitcount(b|w)
depth = 0
ttable = {}
h_current = heur if heur else h_base
while forever or ((time.time() - start < timelim) if not depthlim else (depth < depthlim)):
depth += 1
bval = bmov = None
if not hastimeout or depthlim is not None:
bval, bmov = negascout(b, w, mover, depth, getmove=True)
else:
left = (timelim + start) - time.time()
try:
with timeout(seconds=left):
bval, bmov = negascout(b, w, mover, depth, getmove=True)
except TimeoutError:
continue
yield (bmov, bval) if getval else bmov
if depth > movesleft: break
totdepth += depth
numsearch += 1
if printdata:
print("Plies:", depth)
print('Avg Plies: {:.4g}'.format(totdepth/numsearch))
def run_test(sagemaker_session,
ecr_image,
instance_type,
framework_version,
test_data,
record_wrapper_type=None):
source_path = os.path.join(os.path.dirname(__file__), '..', '..',
'resources', 'pipemode')
script = os.path.join(source_path, 'pipemode.py')
estimator = TensorFlow(entry_point=script,
role='SageMakerRole',
train_instance_type=instance_type,
train_instance_count=1,
sagemaker_session=sagemaker_session,
image_name=ecr_image,
framework_version=framework_version,
script_mode=True,
input_mode='Pipe',
hyperparameters={'dimension': DIMENSION})
input = s3_input(s3_data=test_data,
distribution='FullyReplicated',
record_wrapping=record_wrapper_type,
input_mode='Pipe')
with timeout(minutes=20):
estimator.fit(
{'elizabeth': input},
job_name=unique_name_from_base('test-sagemaker-pipemode'))
def search_neg(b,
timelim=inf,
depthlim=inf,
heuristic=None,
retval=False,
printdata=False):
global totdepth, numsearch, hastimeout
start = time.time()
depthlim = min(depthlim, 65 - b.discs)
depth = 0
while (time.time() - start < timelim):
depth += 1
val = movelist = None
if not hastimeout or timelim == inf:
val, movelist = negamax(b, depth)
else:
left = (timelim + start) - time.time()
try:
with timeout(seconds=left):
val, movelist = negamax(b, depth)
except TimeoutError:
pass
if movelist is None: break
yield (val, movelist[-1]) if retval else movelist[-1]
if depth >= depthlim: break
totdepth += depth
numsearch += 1
if printdata:
print("Plies:", depth)
print('Avg Plies: {:.4g}'.format(totdepth / numsearch))
def generate_feed(self):
"""
Fill the rss feed with `numpost` posts, each with at most 5 comments.
Attempt to make a summary of the post using summarizer.py.
"""
post_ids = self._get_post_ids(self.api)
for pid in post_ids:
post_data = self._get_post_data(pid)
post_title = post_data.get('title', "")
post_score = post_data.get('score', "")
post_author = post_data.get('by', "")
post_kids = post_data.get('kids', "")
post_time = self._format_time(post_data.get('time'))
post_url = post_data.get('url', "")
post_text = post_data.get('text', "")
if not post_text:
post_text = ("<h2>Automated summary of {}.</h2>\n"
"[There may be errors].\n<p>").format(post_url)
try:
with timeout(seconds=15):
post_text += summarizer.summarize(post_url)
logger.debug("No problem occurred during summary")
except TimeoutError:
post_text += "Automated summary timed out. No summary available."
logger.info("Timeout occurred during automated summary.")
except Exception:
post_text += "Unknown error occurred during automated " + \
"summary. No Summary available."
logger.error("Automated summary failed for UNKNOWN reason")
post_text += "</p>"
post_text += ("<p>Current post score: {}. "
"Full comments are at "
"<a href='https://news.ycombinator.com/item?id={}'>"
"https://news.ycombinator.com/item?id={}</a></p>"
).format(post_score, pid, pid)
if post_kids:
post_text += ("<h3> Top Comments </h3><ol>\n\n")
for kid in post_kids[:4]:
kid_data = self._get_post_data(kid)
kid_text = ("<h3><li>{author} at {time}</h3>\n"
"<p>{text}</li>").format(
author=kid_data.get('by', 'Someone'),
time=self._format_time(kid_data.get('time')),
text=kid_data.get('text'))
post_text += kid_text
if post_kids:
post_text += "</ol>\n"
self.feed.append_item(title=post_title,
author=post_author,
link=post_url,
pubDate=post_time,
description=post_text)
def check_disk_space(self, paths, max_wait=5):
"""
Checks for a given list of (path, min_percent)-tuples,
if at least min_percent are still available at
path (e.g. at least 10% free disk space).
"""
satisfied = True
for path, min_percent in paths:
# Use timeout as a hard-mounted disk may still be mounted
# while not responding, which could cause a infinite wait
try:
# Calculate available disk space using statvfs
stat = timeout(max_wait)(os.statvfs)(path)
disk_capacity = stat.f_blocks * stat.f_frsize
free = stat.f_bavail * stat.f_frsize
free_percentage = 100 - (free / (disk_capacity / 100))
if free_percentage < min_percent:
satisfied = False
except (OSError, TimeoutException):
satisfied = False
return satisfied
def yield_move(self, max_time):
"""Yields a move to play.
Args:
max_time: Max time to come up with a move in seconds.
Returns:
Best available move so far.
Raises:
PlayerResigned: If the agent resigns.
"""
print("Thinking... ")
try:
with timeout(max_time):
self._searcher.search(self._game.to_game_state())
except TimeoutError:
pass
except KeyboardInterrupt:
if self._resigns:
raise PlayerResigned
raise KeyboardInterrupt
move = self._searcher.request_move()
print(move)
return move
def aux(algs, names, file):
datasets = ['Protein', 'Reuters']
pos_files = ['SRC1521', 'earn']
neg_files = ['PKA_group15', 'acq']
times = {}
for data in datasets:
times[data] = {}
for alg in names:
times[data][alg] = []
for j, data in enumerate(datasets):
for i in range(len(algs)):
print()
alg = algs[i]
k = 1
while True:
try:
print("data: %s, alg: %s, k: %g" % (data, names[i], k))
with timeout(seconds=200):
s = time.perf_counter()
alg("%s/%s.txt" % (data, pos_files[j]), "%s/%s.txt" % (data, neg_files[j]), k, False)
e = time.perf_counter()
times[data][names[i]].append(e-s)
k *= 2
except (TimeoutError, RecursionError) as e:
print(e)
break
pickle.dump(times, open("../report/data/times_%s.p" % (file), "wb"))
def main():
with timeout(seconds=3):
sleep(2)
# Creating airport pairs to iterate
airports = ["NYC", "LAX"] #, "CHI", "BOS", "SFO"]
airport_pairs = list(itertools.permutations(airports, 2))
# Creating dates to iterate
num_days = 1
base = dt.datetime.today() + dt.timedelta(days=1)
dates = [base + dt.timedelta(days=x) for x in range(0, num_days)]
dates = list(map(lambda x: x.strftime('%m/%d/%Y'), dates))
total_length = len(airport_pairs) * num_days
# Generate list of queries for pool
queries = [[date, pair] for date in dates for pair in airport_pairs]
# Going through all results
print("Fetching flight details")
# Starting multiprocessing
pool = mp.Pool(2)
list_flights = pool.map(parse, queries)
pool.terminate()
pool.join()
print("done with results")
with open(
'/Users/adodd202/Documents/Bootcamp_Spring2018/NYC-Data-Science-Capstone-Project/flight-results-all.json',
'w') as fp:
json.dump(list_flights, fp, indent=4)
def compute_vertex_cover_with_timeout(self, cutoff):
try:
with timeout(seconds=cutoff):
self.compute_vertex_cover()
except TimeoutError as e:
pass
return self.get_current_best_solution()
def test_tuning(sagemaker_session, ecr_image, instance_type):
mx = MXNet(entry_point=SCRIPT_PATH,
role='SageMakerRole',
train_instance_count=1,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
image_name=ecr_image,
hyperparameters={'epochs': 1})
hyperparameter_ranges = {'learning-rate': ContinuousParameter(0.01, 0.2)}
objective_metric_name = 'Validation-accuracy'
metric_definitions = [
{'Name': 'Validation-accuracy', 'Regex': 'Validation-accuracy=([0-9\\.]+)'}]
tuner = HyperparameterTuner(mx,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=2,
max_parallel_jobs=2)
with timeout(minutes=20):
prefix = 'mxnet_mnist/{}'.format(utils.sagemaker_timestamp())
train_input = mx.sagemaker_session.upload_data(path=os.path.join(DATA_PATH, 'train'),
key_prefix=prefix + '/train')
test_input = mx.sagemaker_session.upload_data(path=os.path.join(DATA_PATH, 'test'),
key_prefix=prefix + '/test')
job_name = utils.unique_name_from_base('test-mxnet-image', max_length=32)
tuner.fit({'train': train_input, 'test': test_input}, job_name=job_name)
tuner.wait()
def test_tuning(sagemaker_session, ecr_image, instance_type, framework_version):
resource_path = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
script = os.path.join(resource_path, 'mnist', 'mnist.py')
estimator = TensorFlow(entry_point=script,
role='SageMakerRole',
train_instance_type=instance_type,
train_instance_count=1,
sagemaker_session=sagemaker_session,
image_name=ecr_image,
framework_version=framework_version,
script_mode=True)
hyperparameter_ranges = {'epochs': IntegerParameter(1, 2)}
objective_metric_name = 'accuracy'
metric_definitions = [{'Name': objective_metric_name, 'Regex': 'accuracy = ([0-9\\.]+)'}]
tuner = HyperparameterTuner(estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=2,
max_parallel_jobs=2)
with timeout(minutes=20):
inputs = estimator.sagemaker_session.upload_data(
path=os.path.join(resource_path, 'mnist', 'data'),
key_prefix='scriptmode/mnist')
tuning_job_name = unique_name_from_base('test-tf-sm-tuning', max_length=32)
tuner.fit(inputs, job_name=tuning_job_name)
tuner.wait()
def send_new_homework(openIDs, homework):
pushdata = {
"coursename": {
"value": homework["course_name"],
"color": "#228B22"
},
"title": {
"value": homework["title"],
"color": "#228B22"
},
"endtime": {
"value": str(homework["end_time"]),
"color": "#228B22"
},
"text": {
"value": homework["detail"],
"color": "#228B22"
}
}
for user_id in openIDs:
try:
with timeout(3):
wechat.send_template_message(user_id=user_id, template_id=_TEMPLATE_HOMEWORK, data=pushdata, url="")
except:
logger.debug("send_template_message timeout")
def upload_image(self, timeout_s):
with timeout.timeout(timeout_s):
if not self.override_image:
filename = self.image['filename']
disk_format = self.combined_glance_section.get(
'disk_format', 'qcow2')
container_format = self.combined_glance_section.get(
'container_format', 'bare')
min_disk = self.combined_glance_section.get(
'min_disk', 0)
min_ram = self.combined_glance_section.get(
'min_ram', 0)
protected = self.combined_glance_section.get(
'protected', False
)
print("Uploading image from %s (time limit is %s s)" % (
filename, timeout_s
))
self.os_image = self.os.upload_image(
self.image_name,
filename,
disk_format=disk_format,
container_format=container_format,
min_disk=min_disk,
min_ram=min_ram,
protected=protected,
meta=self.image['glance'].get('properties', {})
)
print("Image %s uploaded." % self.os_image.id)
def process_dir(self, path, st):
""" i_dir should be absolute path
st is the stat object associated with the directory
"""
last_report = MPI.Wtime()
count = 0
try:
with timeout(seconds=30):
entries = scandir(path)
except OSError as e:
log.warn(e, extra=self.d)
self.skipped += 1
except TimeoutError as e:
log.error("%s when scandir() on %s" % (e, path), extra=self.d)
self.skipped += 1
else:
for entry in entries:
if entry.is_symlink():
self.sym_links += 1
elif entry.is_file():
self.circle.enq(entry.path)
else:
self.circle.preq(entry.path)
count += 1
if (MPI.Wtime() - last_report) > self.interval:
print("Rank %s : Scanning [%s] at %s" % (self.circle.rank, path, count))
last_report = MPI.Wtime()
log.info("Finish scan of [%s], count=%s" % (path, count), extra=self.d)
if count > self.maxfiles:
self.maxfiles = count
self.maxfiles_dir = path
def test_training(sagemaker_session, ecr_image, instance_type, instance_count):
hyperparameters = {
'random_seed': True,
'num_steps': 50,
'smdebug_path': '/opt/ml/output/tensors',
'epochs': 1
}
mx = MXNet(entry_point=SCRIPT_PATH,
role='SageMakerRole',
train_instance_count=instance_count,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
image_name=ecr_image,
hyperparameters=hyperparameters)
with timeout(minutes=15):
prefix = 'mxnet_mnist_gluon_basic_hook_demo/{}'.format(
utils.sagemaker_timestamp())
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(DATA_PATH, 'train'),
key_prefix=prefix + '/train')
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(DATA_PATH, 'test'), key_prefix=prefix + '/test')
job_name = utils.unique_name_from_base('test-mxnet-image')
mx.fit({'train': train_input, 'test': test_input}, job_name=job_name)
def process(self):
""" process a work unit, spath, dpath refers to
source and destination respectively """
spath = self.circle.deq()
if spath:
try:
with timeout(seconds=15):
st = os.lstat(spath)
except OSError as e:
log.warn(e, extra=self.d)
self.skipped += 1
return False
except TimeoutError as e:
log.error("%s when stat() on %s" % (e, spath), extra=self.d)
self.skipped += 1
return
self.reduce_items += 1
if os.path.islink(spath):
self.sym_links += 1
# NOT TO FOLLOW SYM LINKS SHOULD BE THE DEFAULT
return
self.handle_file_or_dir(spath, st)
del spath
def get_frequent_sets(airport):
# Filter out transactions that doesn't have the airport
trans = filter(lambda x: airport in ' '.join(x), transactions)
print("Process %s of transactions for %s" % (len(trans), airport))
# Obscure data science voodoo
min_sup = 20 + len(trans) / 10
if min_sup > 100:
min_sup = 100
# Retry to find frequent sets by increasing min_sup until there is
# no timeout nor too many results
while True:
res = timeout(mine_frequent_set,
args=(trans, min_sup),
timeout=1,
default='timeout')
min_sup += 5
if res == 'timeout':
print 'Timeout, increase minimal support'
continue
# only keep sets with at least 3 items
result = filter(lambda x: len(x[0]) > 2, res)
# final set must include airport
result = filter(lambda x: airport in ' '.join(x[0]), result)
if len(result) < 50:
break
result.sort(key=lambda x: x[1], reverse=True)
return result
def generate_feed(self):
"""
Fill the rss feed with `numpost` posts, each with at most 5 comments.
Attempt to make a summary of the post using summarizer.py.
"""
post_ids = self._get_post_ids(self.api)
for pid in post_ids:
post_data = self._get_post_data(pid)
post_title = post_data.get('title', "")
post_score = post_data.get('score', "")
post_author = post_data.get('by', "")
post_kids = post_data.get('kids', "")
post_time = self._format_time(post_data.get('time'))
post_url = post_data.get('url', "")
post_text = post_data.get('text', "")
if not post_text:
post_text = ("<h2>Automated summary of {}.</h2>\n"
"[There may be errors].\n<p>").format(post_url)
try:
with timeout(seconds=15):
post_text += summarizer.summarize(post_url)
logger.debug("No problem occurred during summary")
except TimeoutError:
post_text += "Automated summary timed out. No summary available."
logger.info("Timeout occurred during automated summary.")
except Exception:
post_text += "Unknown error occurred during automated " + \
"summary. No Summary available."
logger.error("Automated summary failed for UNKNOWN reason")
post_text += "</p>"
post_text += (
"<p>Current post score: {}. "
"Full comments are at "
"<a href='https://news.ycombinator.com/item?id={}'>"
"https://news.ycombinator.com/item?id={}</a></p>").format(
post_score, pid, pid)
if post_kids:
post_text += ("<h3> Top Comments </h3><ol>\n\n")
for kid in post_kids[:4]:
kid_data = self._get_post_data(kid)
kid_text = ("<h3><li>{author} at {time}</h3>\n"
"<p>{text}</li>").format(
author=kid_data.get('by', 'Someone'),
time=self._format_time(kid_data.get('time')),
text=kid_data.get('text'))
post_text += kid_text
if post_kids:
post_text += "</ol>\n"
self.feed.append_item(title=post_title,
author=post_author,
link=post_url,
pubDate=post_time,
description=post_text)
def __init__(self, htmlText, isInputFile=False, preprocesshtml="bs4", forcePeriod=False, timeoutsec=5):
"""
byeHTML(htmlText, isInputFile=False, preprocesshtml = "justext", forcePeriod = False).
Case you want to extract a content from a file, use isInputFile=True and htmlText is the
path to a document.
The current available options for HTML preprocessor are:
- justext ---- recommended.
- bs4 (beautifulsoup4) ---- might result in encoding problems
In case forcePeriod is active, a period mark will be added to every sentence
extracted by the preprocessing html method employed.
"""
try:
if isInputFile:
htmlText = self.__extract_content(htmlText)
with timeout(seconds=timeoutsec):
self.text = self.preprocess_html(htmlText, preprocesshtml, forcePeriod)
except Exception as e:
print(("Error %s -- %s" % (type(e), e)))
self.text = ""
def search(b,
timelim=inf,
depthlim=inf,
heuristic=None,
retval=False,
printdata=False):
global ttable, totdepth, numsearch, hastimeout, h_current
if heuristic is not None:
h_current = heuristic
start = time.time()
ttable = {}
depthlim = min(depthlim, 65 - b.discs)
depth = 0
while (time.time() - start < timelim):
depth += 1
val = move = None
if not hastimeout or timelim == inf:
val, move = pvs(b, depth, -inf, inf, retmove=True)
else:
left = (timelim + start) - time.time()
try:
with timeout(seconds=left):
val, move = pvs(b, depth, -inf, inf, retmove=True)
except TimeoutError:
pass
if move is None: break
yield (val, move) if retval else move
if depth >= depthlim: break
totdepth += depth
numsearch += 1
if printdata:
print("Plies:", depth)
print('Avg Plies: {:.4g}'.format(totdepth / numsearch))
def aux(algs, names):
datasets = ['Protein', 'Reuters']
pos_files = ['SRC1521', 'earn']
neg_files = ['PKA_group15', 'acq']
lengths = {}
for data in datasets:
lengths[data] = {}
for alg in names:
lengths[data][alg] = []
for j, data in enumerate(datasets):
for i in range(len(algs)):
print()
alg = algs[i]
k = 1
while True:
try:
print("data: %s, alg: %s, k: %g" % (data, names[i], k))
res = []
with timeout(seconds=100):
res = alg("%s/%s.txt" % (data, pos_files[j]),
"%s/%s.txt" % (data, neg_files[j]), k, False,
True)
lengths[data][names[i]].append(mean([len(x) for x in res]))
k *= 2
except (TimeoutError, RecursionError) as e:
print(e)
break
pickle.dump(lengths, open("../report/data/lengths.p", "wb"))
def get_server(link):
with timeout(seconds=20):
try:
r = requests.get(link)
server = r.headers['Server']
return server
except TimeoutError:
raise TimeoutError
def test_custom_exception_message():
value = "Overslept"
with pytest.raises(TimeoutError) as exc_info:
with timeout(0.01, value=value):
time.sleep(0.015)
assert exc_info.value.args == (value, )
def test_custom_exception():
exc = RuntimeError("Overslept")
with pytest.raises(RuntimeError) as exc_info:
with timeout(0.01, exc):
time.sleep(0.015)
assert exc_info.value is exc
def check_functions(file_path,
function_name,
args,
answers,
time_out,
no_unpack=False,
nested=False,
unbracket=False):
result = False
try:
# get the function out of the file submitted
filename = os.path.basename(file_path).split('.')[0]
function = getattr(
importlib.import_module('uploads.{0}'.format(filename)),
function_name)
func = timeout(timeout=time_out)(function)
# make a copy to submit to function incase argument is mutatable
args_in = copy.copy(args)
# check the number of args and call function accrodingly
if len(args_in) == 0:
ans = func()
else:
if no_unpack:
ans = func(args_in)
else:
ans = func(*args_in)
# if answer is within an array
if nested:
if ans == answers:
result = True
else:
if [ans] == answers:
result = True
# remove bracktes for display in markerbot
if unbracket:
answers = answers[0]
return {
'input': args,
'result': result,
'output': ans,
'expected': answers
}
except Exception as e:
return {
'input': args,
'result': result,
'output': e.message,
'expected': answers
}
def charge_time():
GPIO.setup(b_pin, GPIO.IN)
GPIO.setup(a_pin, GPIO.OUT)
GPIO.output(a_pin, True)
t1 = time.time()
with timeout(seconds=5):
while not GPIO.input(b_pin):
pass
t2 = time.time()
return (t2 - t1) * 1000000 # microseconds
def charge_time():
GPIO.setup(b_pin, GPIO.IN)
GPIO.setup(a_pin, GPIO.OUT)
GPIO.output(a_pin, True)
t1 = time.time()
with timeout(seconds=5):
while not GPIO.input(b_pin):
pass
t2 = time.time()
return (t2 - t1) * 1000000 # microseconds
def _run(self):
try:
logging.info('Building matrices')
self.build_matrices()
finally:
logging.info('Cleaning up state table')
with timeout(self.cleanup_timeout):
self.state_table_generator.clean_up()
self.catwalk()
def _run(self):
try:
logging.info('Generating matrices')
self.generate_matrices()
finally:
logging.info('Cleaning up state table')
with timeout(self.cleanup_timeout):
self.state_table_generator.clean_up()
self.db_engine.execute('drop table if exists {}'.format(
self.labels_table_name))
self.train_and_test_models()
def send_bind_success_message(openID, studentnumber):
pushdata = {
"studentnumber": {
"value": studentnumber,
"color": "#ff0000"
}
}
try:
with timeout(3):
wechat.send_template_message(user_id=openID, template_id=_TEMPLATE_BIND_SUCCESS, data=pushdata, url="")
except:
logger.debug("send_template_message timeout")
def run(self):
print utils.art
self.client()
self.stage_upstream()
prs = self.client().repository(self.owner, self.repo).pull_requests()
new_clones = [self.clone_dir(pr) for pr in prs]
if os.path.exists(self.base_dir()):
discard_dirs = set(os.listdir(self.base_dir())) - set(self.safe_dirs()) - set(new_clones)
discard_dirs = list(discard_dirs)
discard_dirs = [os.path.join(self.base_dir(), d) for d in discard_dirs]
for discard_dir in discard_dirs:
logger.log("===> Unstage {0} since PR is closed.".format(discard_dir))
shutil.rmtree(discard_dir)
with timeout(self.timeout_seconds()):
for pr in prs:
self.process_pull(pr)
def process(self):
""" process a work unit, spath, dpath refers to
source and destination respectively """
spath = self.circle.deq()
self.logger.debug("BEGIN process object: %s" % spath, extra=self.d)
if spath:
if spath in EXCLUDE:
self.logger.warn("Skip excluded path: %s" % spath, extra=self.d)
self.skipped += 1
return
try:
with timeout(seconds=5):
st = os.lstat(spath)
except OSError as e:
self.logger.warn(e, extra=self.d)
self.skipped += 1
return None
except TimeoutError as e:
self.logger.error("%s when stat() on %s" % (e, spath), extra=self.d)
self.skipped += 1
return None
except Exception as e:
self.logger.error("Unknown: %s on %s" % (e, spath), extra=self.d)
self.skipped += 1
return None
self.reduce_items += 1
self.logger.debug("FIN lstat object: %s" % spath, extra=self.d)
# islink() return True if it is symbolic link
if os.path.islink(spath):
self.sym_links += 1
# NOT TO FOLLOW SYM LINKS SHOULD BE THE DEFAULT
return None
self.handle_file_or_dir(spath, st)
self.logger.debug("END process object: %s" % spath, extra=self.d)
def test_timeout_with_content_manager(seconds):
import time
with timeout.timeout(timeout=5):
print "start"
time.sleep(seconds)
print "end"
return checksum
if __name__ == '__main__':
urllib3.disable_warnings()
mswindows = (sys.platform == "win32") # learning from 'subprocess' module
url = "https://raw.githubusercontent.com/racaljk/hosts/master/hosts"
filename = url.split('/')[-1]
save = os.path.join("/tmp", filename).replace("\\", "/")
if mswindows:
# global socket timeout
socket.setdefaulttimeout(10.0)
print "Downloading", url
urllib.urlretrieve(url, filename=save)
else:
assert sys.platform == "linux2", "please run this script on Windows or Linux"
with timeout(timeout=10.0):
urllib.urlretrieve(url, filename=save)
if os.path.isfile(save):
print "Saved: '%s'" % save
print "md5sum:", get_hash_sum(save, method="md5")
print "sha1sum:", get_hash_sum(save, method="sha1sum")
print "sha256sum:", get_hash_sum(save, method="sha256sum")
else:
print "can not download", url
sys.exit(1)
for row in rows:
ctr += 1
local_id = row['Local ID']
print '%s Processing %d/%d: %s' % (datetime.utcnow(), ctr, len(rows),
local_id)
counters['total'] += 1
last_fail_timeout = False
def f():
doi = update(row, commit=commit, show_diff=show_diff,
counters=counters, doi_update_rows=doi_update_rows,
force_update_file=force_update_file)
try:
if last_fail_timeout:
print 'Previous attempt timed out, waiting 30 seconds...'
time.sleep(30)
timeout.timeout(f, 180)
counters['success'] += 1
last_fail_timeout = False
except KeyboardInterrupt: raise KeyboardInterrupt
except SystemExit: raise SystemExit
except Exception, e:
print 'ERROR on row %d: %s' % (ctr, traceback.format_exc())
if type(e) is timeout.TimeoutError:
counters['timeout'] += 1
last_fail_timeout = True
else:
counters['error'] += 1
print str(dict(counters))
print 'Consider running `python merge_doi_maps.py %s %s`' % (
doi_tsv, doi_update_tsv)
