def test_training(sagemaker_session, ecr_image, instance_type, instance_count): hyperparameters = { 'sagemaker_parameter_server_enabled': True } if instance_count > 1 else {} hyperparameters['epochs'] = 1 mx = MXNet(entry_point=SCRIPT_PATH, role='SageMakerRole', train_instance_count=instance_count, train_instance_type=instance_type, sagemaker_session=sagemaker_session, image_name=ecr_image, hyperparameters=hyperparameters) with timeout(minutes=15): prefix = 'mxnet_mnist/{}'.format(utils.sagemaker_timestamp()) train_input = mx.sagemaker_session.upload_data( path=os.path.join(DATA_PATH, 'train'), key_prefix=prefix + '/train') test_input = mx.sagemaker_session.upload_data( path=os.path.join(DATA_PATH, 'test'), key_prefix=prefix + '/test') job_name = utils.unique_name_from_base('test-mxnet-image') mx.fit({'train': train_input, 'test': test_input}, job_name=job_name) dgl = MXNet(entry_point=DGL_SCRIPT_PATH, role='SageMakerRole', train_instance_count=1, train_instance_type=instance_type, sagemaker_session=sagemaker_session, image_name=ecr_image) with timeout(minutes=15): job_name = utils.unique_name_from_base('test-mxnet-dgl-image') dgl.fit(job_name=job_name)
def __check_go(self, chessboard): self.agent = imp.load_source('AI', self.script_file_path).AI(self.chessboard_size, -1, self.time_out) try: timeout(self.time_out)(self.agent.go)(np.copy(chessboard)) except Exception: self.errormsg = "Error:" + traceback.format_exc() return False return True
def getConfirmMail(mb): try: with timeout(seconds=20): msg_confirm = mb.getCorreo() return getConfirmUrl(msg_confirm) except: resend_mail() try: with timeout(seconds=20): msg_confirm = mb.getCorreo() return getConfirmUrl(msg_confirm) except: print "No ha llegado el correo" exit(90)
def infer_type(filepath): with timeout(seconds=15): if get_extension(filepath) in img_extensions: return "image" try: extract_netcdf(filepath) return "netcdf" except Exception as e: if e.__class__ == TimeoutError: return "unknown" pass try: extract_json_metadata(filepath) return "json/xml" except Exception as e: if e.__class__ == TimeoutError: return "unknown" pass try: extract_columnar_metadata(filepath, parallel=False) return "tabular" except Exception as e: if e.__class__ == TimeoutError: return "unknown" pass try: if extract_keyword(filepath)["keywords"]: return "freetext" else: pass except Exception as e: if e.__class__ == TimeoutError: return "unknown" pass return "unknown"
def _test_mnist_train(sagemaker_session, ecr_image, instance_type, instance_count, script): source_dir = 'test/resources/mnist' with timeout(minutes=15): data_path = 'test/resources/mnist/data' chainer = Chainer(entry_point=script, source_dir=source_dir, role='SageMakerRole', train_instance_count=instance_count, train_instance_type=instance_type, sagemaker_session=sagemaker_session, image_name=ecr_image, hyperparameters={ 'batch-size': 10000, 'epochs': 1 }) prefix = 'chainer_mnist/{}'.format(sagemaker_timestamp()) train_data_path = os.path.join(data_path, 'train') key_prefix = prefix + '/train' train_input = sagemaker_session.upload_data(path=train_data_path, key_prefix=key_prefix) test_path = os.path.join(data_path, 'test') test_input = sagemaker_session.upload_data(path=test_path, key_prefix=prefix + '/test') chainer.fit({'train': train_input, 'test': test_input})
def get_proxies(num_proxies = 5): print("INTIALIZING {} PROXIES".format(num_proxies)) # Go to the proxy list url, and get their website text # url = 'https://free-proxy-list.net/' url = 'https://free-proxy-list.net/' response = requests.get(url) parser = fromstring(response.text) proxies = set() # For each potential proxy, add it if it works. for i in parser.xpath('//tbody/tr'): if i.xpath('.//td[7][contains(text(),"yes")]'): #Grabbing IP and corresponding PORT proxy = ":".join([i.xpath('.//td[1]/text()')[0], i.xpath('.//td[2]/text()')[0]]) try: sleep(.5) # Check the proxy urls = ["http://www.neopets.com/", "https://www.theatlantic.com/", "https://www.reddit.com/r/funny/", "http://www.pythonforbeginners.com/"] url = random.sample(urls, 1)[0] print(url) print(proxy) with timeout(seconds=2): response = requests.get(url,proxies={"http": proxy, "https": proxy}) # If it works, add the proxy proxies.add(proxy) print("="*50) print("We have added {} proxies of a total of {} proxies".format(len(proxies),num_proxies)) print("{}% Complete with initializing proxies".format(round(len(proxies)/num_proxies*100,2))) if len(proxies) == num_proxies: return proxies except: print("Failed Connection") continue return proxies
def test_mxnet_distributed(sagemaker_session, ecr_image, instance_type, framework_version): data_path = os.path.join(RESOURCE_PATH, 'mnist') script_path = os.path.join(data_path, 'mnist.py') mx = MXNet(entry_point=script_path, role='SageMakerRole', train_instance_count=2, train_instance_type=instance_type, sagemaker_session=sagemaker_session, image_name=ecr_image, framework_version=framework_version, hyperparameters={'sagemaker_parameter_server_enabled': True}) prefix = 'mxnet_mnist/{}'.format(sagemaker_timestamp()) with timeout(minutes=15): train_input = mx.sagemaker_session.upload_data( path=os.path.join(data_path, 'train'), key_prefix=prefix + '/train') test_input = mx.sagemaker_session.upload_data( path=os.path.join(data_path, 'test'), key_prefix=prefix + '/test') mx.fit({'train': train_input, 'test': test_input}) with timeout_and_delete_endpoint(estimator=mx, minutes=30): predictor = mx.deploy(initial_instance_count=1, instance_type=instance_type) data = np.zeros(shape=(1, 1, 28, 28)) predictor.predict(data)
def search(b, w, mover, passes=0, getval=False, timelim=2, depthlim=None, heur=None, printdata=True, forever=False): global start, ttable, h_current, totdepth, numsearch start = time.time() movesleft = 64 - bitcount(b|w) depth = 0 ttable = {} h_current = heur if heur else h_base while forever or ((time.time() - start < timelim) if not depthlim else (depth < depthlim)): depth += 1 bval = bmov = None if not hastimeout or depthlim is not None: bval, bmov = negascout(b, w, mover, depth, getmove=True) else: left = (timelim + start) - time.time() try: with timeout(seconds=left): bval, bmov = negascout(b, w, mover, depth, getmove=True) except TimeoutError: continue yield (bmov, bval) if getval else bmov if depth > movesleft: break totdepth += depth numsearch += 1 if printdata: print("Plies:", depth) print('Avg Plies: {:.4g}'.format(totdepth/numsearch))
def run_test(sagemaker_session, ecr_image, instance_type, framework_version, test_data, record_wrapper_type=None): source_path = os.path.join(os.path.dirname(__file__), '..', '..', 'resources', 'pipemode') script = os.path.join(source_path, 'pipemode.py') estimator = TensorFlow(entry_point=script, role='SageMakerRole', train_instance_type=instance_type, train_instance_count=1, sagemaker_session=sagemaker_session, image_name=ecr_image, framework_version=framework_version, script_mode=True, input_mode='Pipe', hyperparameters={'dimension': DIMENSION}) input = s3_input(s3_data=test_data, distribution='FullyReplicated', record_wrapping=record_wrapper_type, input_mode='Pipe') with timeout(minutes=20): estimator.fit( {'elizabeth': input}, job_name=unique_name_from_base('test-sagemaker-pipemode'))
def search_neg(b, timelim=inf, depthlim=inf, heuristic=None, retval=False, printdata=False): global totdepth, numsearch, hastimeout start = time.time() depthlim = min(depthlim, 65 - b.discs) depth = 0 while (time.time() - start < timelim): depth += 1 val = movelist = None if not hastimeout or timelim == inf: val, movelist = negamax(b, depth) else: left = (timelim + start) - time.time() try: with timeout(seconds=left): val, movelist = negamax(b, depth) except TimeoutError: pass if movelist is None: break yield (val, movelist[-1]) if retval else movelist[-1] if depth >= depthlim: break totdepth += depth numsearch += 1 if printdata: print("Plies:", depth) print('Avg Plies: {:.4g}'.format(totdepth / numsearch))
def generate_feed(self): """ Fill the rss feed with `numpost` posts, each with at most 5 comments. Attempt to make a summary of the post using summarizer.py. """ post_ids = self._get_post_ids(self.api) for pid in post_ids: post_data = self._get_post_data(pid) post_title = post_data.get('title', "") post_score = post_data.get('score', "") post_author = post_data.get('by', "") post_kids = post_data.get('kids', "") post_time = self._format_time(post_data.get('time')) post_url = post_data.get('url', "") post_text = post_data.get('text', "") if not post_text: post_text = ("<h2>Automated summary of {}.</h2>\n" "[There may be errors].\n<p>").format(post_url) try: with timeout(seconds=15): post_text += summarizer.summarize(post_url) logger.debug("No problem occurred during summary") except TimeoutError: post_text += "Automated summary timed out. No summary available." logger.info("Timeout occurred during automated summary.") except Exception: post_text += "Unknown error occurred during automated " + \ "summary. No Summary available." logger.error("Automated summary failed for UNKNOWN reason") post_text += "</p>" post_text += ("<p>Current post score: {}. " "Full comments are at " "<a href='https://news.ycombinator.com/item?id={}'>" "https://news.ycombinator.com/item?id={}</a></p>" ).format(post_score, pid, pid) if post_kids: post_text += ("<h3> Top Comments </h3><ol>\n\n") for kid in post_kids[:4]: kid_data = self._get_post_data(kid) kid_text = ("<h3><li>{author} at {time}</h3>\n" "<p>{text}</li>").format( author=kid_data.get('by', 'Someone'), time=self._format_time(kid_data.get('time')), text=kid_data.get('text')) post_text += kid_text if post_kids: post_text += "</ol>\n" self.feed.append_item(title=post_title, author=post_author, link=post_url, pubDate=post_time, description=post_text)
def check_disk_space(self, paths, max_wait=5): """ Checks for a given list of (path, min_percent)-tuples, if at least min_percent are still available at path (e.g. at least 10% free disk space). """ satisfied = True for path, min_percent in paths: # Use timeout as a hard-mounted disk may still be mounted # while not responding, which could cause a infinite wait try: # Calculate available disk space using statvfs stat = timeout(max_wait)(os.statvfs)(path) disk_capacity = stat.f_blocks * stat.f_frsize free = stat.f_bavail * stat.f_frsize free_percentage = 100 - (free / (disk_capacity / 100)) if free_percentage < min_percent: satisfied = False except (OSError, TimeoutException): satisfied = False return satisfied
def yield_move(self, max_time): """Yields a move to play. Args: max_time: Max time to come up with a move in seconds. Returns: Best available move so far. Raises: PlayerResigned: If the agent resigns. """ print("Thinking... ") try: with timeout(max_time): self._searcher.search(self._game.to_game_state()) except TimeoutError: pass except KeyboardInterrupt: if self._resigns: raise PlayerResigned raise KeyboardInterrupt move = self._searcher.request_move() print(move) return move
def aux(algs, names, file): datasets = ['Protein', 'Reuters'] pos_files = ['SRC1521', 'earn'] neg_files = ['PKA_group15', 'acq'] times = {} for data in datasets: times[data] = {} for alg in names: times[data][alg] = [] for j, data in enumerate(datasets): for i in range(len(algs)): print() alg = algs[i] k = 1 while True: try: print("data: %s, alg: %s, k: %g" % (data, names[i], k)) with timeout(seconds=200): s = time.perf_counter() alg("%s/%s.txt" % (data, pos_files[j]), "%s/%s.txt" % (data, neg_files[j]), k, False) e = time.perf_counter() times[data][names[i]].append(e-s) k *= 2 except (TimeoutError, RecursionError) as e: print(e) break pickle.dump(times, open("../report/data/times_%s.p" % (file), "wb"))
def main(): with timeout(seconds=3): sleep(2) # Creating airport pairs to iterate airports = ["NYC", "LAX"] #, "CHI", "BOS", "SFO"] airport_pairs = list(itertools.permutations(airports, 2)) # Creating dates to iterate num_days = 1 base = dt.datetime.today() + dt.timedelta(days=1) dates = [base + dt.timedelta(days=x) for x in range(0, num_days)] dates = list(map(lambda x: x.strftime('%m/%d/%Y'), dates)) total_length = len(airport_pairs) * num_days # Generate list of queries for pool queries = [[date, pair] for date in dates for pair in airport_pairs] # Going through all results print("Fetching flight details") # Starting multiprocessing pool = mp.Pool(2) list_flights = pool.map(parse, queries) pool.terminate() pool.join() print("done with results") with open( '/Users/adodd202/Documents/Bootcamp_Spring2018/NYC-Data-Science-Capstone-Project/flight-results-all.json', 'w') as fp: json.dump(list_flights, fp, indent=4)
def compute_vertex_cover_with_timeout(self, cutoff): try: with timeout(seconds=cutoff): self.compute_vertex_cover() except TimeoutError as e: pass return self.get_current_best_solution()
def test_tuning(sagemaker_session, ecr_image, instance_type): mx = MXNet(entry_point=SCRIPT_PATH, role='SageMakerRole', train_instance_count=1, train_instance_type=instance_type, sagemaker_session=sagemaker_session, image_name=ecr_image, hyperparameters={'epochs': 1}) hyperparameter_ranges = {'learning-rate': ContinuousParameter(0.01, 0.2)} objective_metric_name = 'Validation-accuracy' metric_definitions = [ {'Name': 'Validation-accuracy', 'Regex': 'Validation-accuracy=([0-9\\.]+)'}] tuner = HyperparameterTuner(mx, objective_metric_name, hyperparameter_ranges, metric_definitions, max_jobs=2, max_parallel_jobs=2) with timeout(minutes=20): prefix = 'mxnet_mnist/{}'.format(utils.sagemaker_timestamp()) train_input = mx.sagemaker_session.upload_data(path=os.path.join(DATA_PATH, 'train'), key_prefix=prefix + '/train') test_input = mx.sagemaker_session.upload_data(path=os.path.join(DATA_PATH, 'test'), key_prefix=prefix + '/test') job_name = utils.unique_name_from_base('test-mxnet-image', max_length=32) tuner.fit({'train': train_input, 'test': test_input}, job_name=job_name) tuner.wait()
def test_tuning(sagemaker_session, ecr_image, instance_type, framework_version): resource_path = os.path.join(os.path.dirname(__file__), '..', '..', 'resources') script = os.path.join(resource_path, 'mnist', 'mnist.py') estimator = TensorFlow(entry_point=script, role='SageMakerRole', train_instance_type=instance_type, train_instance_count=1, sagemaker_session=sagemaker_session, image_name=ecr_image, framework_version=framework_version, script_mode=True) hyperparameter_ranges = {'epochs': IntegerParameter(1, 2)} objective_metric_name = 'accuracy' metric_definitions = [{'Name': objective_metric_name, 'Regex': 'accuracy = ([0-9\\.]+)'}] tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions, max_jobs=2, max_parallel_jobs=2) with timeout(minutes=20): inputs = estimator.sagemaker_session.upload_data( path=os.path.join(resource_path, 'mnist', 'data'), key_prefix='scriptmode/mnist') tuning_job_name = unique_name_from_base('test-tf-sm-tuning', max_length=32) tuner.fit(inputs, job_name=tuning_job_name) tuner.wait()
def send_new_homework(openIDs, homework): pushdata = { "coursename": { "value": homework["course_name"], "color": "#228B22" }, "title": { "value": homework["title"], "color": "#228B22" }, "endtime": { "value": str(homework["end_time"]), "color": "#228B22" }, "text": { "value": homework["detail"], "color": "#228B22" } } for user_id in openIDs: try: with timeout(3): wechat.send_template_message(user_id=user_id, template_id=_TEMPLATE_HOMEWORK, data=pushdata, url="") except: logger.debug("send_template_message timeout")
def upload_image(self, timeout_s): with timeout.timeout(timeout_s): if not self.override_image: filename = self.image['filename'] disk_format = self.combined_glance_section.get( 'disk_format', 'qcow2') container_format = self.combined_glance_section.get( 'container_format', 'bare') min_disk = self.combined_glance_section.get( 'min_disk', 0) min_ram = self.combined_glance_section.get( 'min_ram', 0) protected = self.combined_glance_section.get( 'protected', False ) print("Uploading image from %s (time limit is %s s)" % ( filename, timeout_s )) self.os_image = self.os.upload_image( self.image_name, filename, disk_format=disk_format, container_format=container_format, min_disk=min_disk, min_ram=min_ram, protected=protected, meta=self.image['glance'].get('properties', {}) ) print("Image %s uploaded." % self.os_image.id)
def process_dir(self, path, st): """ i_dir should be absolute path st is the stat object associated with the directory """ last_report = MPI.Wtime() count = 0 try: with timeout(seconds=30): entries = scandir(path) except OSError as e: log.warn(e, extra=self.d) self.skipped += 1 except TimeoutError as e: log.error("%s when scandir() on %s" % (e, path), extra=self.d) self.skipped += 1 else: for entry in entries: if entry.is_symlink(): self.sym_links += 1 elif entry.is_file(): self.circle.enq(entry.path) else: self.circle.preq(entry.path) count += 1 if (MPI.Wtime() - last_report) > self.interval: print("Rank %s : Scanning [%s] at %s" % (self.circle.rank, path, count)) last_report = MPI.Wtime() log.info("Finish scan of [%s], count=%s" % (path, count), extra=self.d) if count > self.maxfiles: self.maxfiles = count self.maxfiles_dir = path
def test_training(sagemaker_session, ecr_image, instance_type, instance_count): hyperparameters = { 'random_seed': True, 'num_steps': 50, 'smdebug_path': '/opt/ml/output/tensors', 'epochs': 1 } mx = MXNet(entry_point=SCRIPT_PATH, role='SageMakerRole', train_instance_count=instance_count, train_instance_type=instance_type, sagemaker_session=sagemaker_session, image_name=ecr_image, hyperparameters=hyperparameters) with timeout(minutes=15): prefix = 'mxnet_mnist_gluon_basic_hook_demo/{}'.format( utils.sagemaker_timestamp()) train_input = mx.sagemaker_session.upload_data( path=os.path.join(DATA_PATH, 'train'), key_prefix=prefix + '/train') test_input = mx.sagemaker_session.upload_data( path=os.path.join(DATA_PATH, 'test'), key_prefix=prefix + '/test') job_name = utils.unique_name_from_base('test-mxnet-image') mx.fit({'train': train_input, 'test': test_input}, job_name=job_name)
def process(self): """ process a work unit, spath, dpath refers to source and destination respectively """ spath = self.circle.deq() if spath: try: with timeout(seconds=15): st = os.lstat(spath) except OSError as e: log.warn(e, extra=self.d) self.skipped += 1 return False except TimeoutError as e: log.error("%s when stat() on %s" % (e, spath), extra=self.d) self.skipped += 1 return self.reduce_items += 1 if os.path.islink(spath): self.sym_links += 1 # NOT TO FOLLOW SYM LINKS SHOULD BE THE DEFAULT return self.handle_file_or_dir(spath, st) del spath
def get_frequent_sets(airport): # Filter out transactions that doesn't have the airport trans = filter(lambda x: airport in ' '.join(x), transactions) print("Process %s of transactions for %s" % (len(trans), airport)) # Obscure data science voodoo min_sup = 20 + len(trans) / 10 if min_sup > 100: min_sup = 100 # Retry to find frequent sets by increasing min_sup until there is # no timeout nor too many results while True: res = timeout(mine_frequent_set, args=(trans, min_sup), timeout=1, default='timeout') min_sup += 5 if res == 'timeout': print 'Timeout, increase minimal support' continue # only keep sets with at least 3 items result = filter(lambda x: len(x[0]) > 2, res) # final set must include airport result = filter(lambda x: airport in ' '.join(x[0]), result) if len(result) < 50: break result.sort(key=lambda x: x[1], reverse=True) return result
def generate_feed(self): """ Fill the rss feed with `numpost` posts, each with at most 5 comments. Attempt to make a summary of the post using summarizer.py. """ post_ids = self._get_post_ids(self.api) for pid in post_ids: post_data = self._get_post_data(pid) post_title = post_data.get('title', "") post_score = post_data.get('score', "") post_author = post_data.get('by', "") post_kids = post_data.get('kids', "") post_time = self._format_time(post_data.get('time')) post_url = post_data.get('url', "") post_text = post_data.get('text', "") if not post_text: post_text = ("<h2>Automated summary of {}.</h2>\n" "[There may be errors].\n<p>").format(post_url) try: with timeout(seconds=15): post_text += summarizer.summarize(post_url) logger.debug("No problem occurred during summary") except TimeoutError: post_text += "Automated summary timed out. No summary available." logger.info("Timeout occurred during automated summary.") except Exception: post_text += "Unknown error occurred during automated " + \ "summary. No Summary available." logger.error("Automated summary failed for UNKNOWN reason") post_text += "</p>" post_text += ( "<p>Current post score: {}. " "Full comments are at " "<a href='https://news.ycombinator.com/item?id={}'>" "https://news.ycombinator.com/item?id={}</a></p>").format( post_score, pid, pid) if post_kids: post_text += ("<h3> Top Comments </h3><ol>\n\n") for kid in post_kids[:4]: kid_data = self._get_post_data(kid) kid_text = ("<h3><li>{author} at {time}</h3>\n" "<p>{text}</li>").format( author=kid_data.get('by', 'Someone'), time=self._format_time(kid_data.get('time')), text=kid_data.get('text')) post_text += kid_text if post_kids: post_text += "</ol>\n" self.feed.append_item(title=post_title, author=post_author, link=post_url, pubDate=post_time, description=post_text)
def __init__(self, htmlText, isInputFile=False, preprocesshtml="bs4", forcePeriod=False, timeoutsec=5): """ byeHTML(htmlText, isInputFile=False, preprocesshtml = "justext", forcePeriod = False). Case you want to extract a content from a file, use isInputFile=True and htmlText is the path to a document. The current available options for HTML preprocessor are: - justext ---- recommended. - bs4 (beautifulsoup4) ---- might result in encoding problems In case forcePeriod is active, a period mark will be added to every sentence extracted by the preprocessing html method employed. """ try: if isInputFile: htmlText = self.__extract_content(htmlText) with timeout(seconds=timeoutsec): self.text = self.preprocess_html(htmlText, preprocesshtml, forcePeriod) except Exception as e: print(("Error %s -- %s" % (type(e), e))) self.text = ""
def search(b, timelim=inf, depthlim=inf, heuristic=None, retval=False, printdata=False): global ttable, totdepth, numsearch, hastimeout, h_current if heuristic is not None: h_current = heuristic start = time.time() ttable = {} depthlim = min(depthlim, 65 - b.discs) depth = 0 while (time.time() - start < timelim): depth += 1 val = move = None if not hastimeout or timelim == inf: val, move = pvs(b, depth, -inf, inf, retmove=True) else: left = (timelim + start) - time.time() try: with timeout(seconds=left): val, move = pvs(b, depth, -inf, inf, retmove=True) except TimeoutError: pass if move is None: break yield (val, move) if retval else move if depth >= depthlim: break totdepth += depth numsearch += 1 if printdata: print("Plies:", depth) print('Avg Plies: {:.4g}'.format(totdepth / numsearch))
def aux(algs, names): datasets = ['Protein', 'Reuters'] pos_files = ['SRC1521', 'earn'] neg_files = ['PKA_group15', 'acq'] lengths = {} for data in datasets: lengths[data] = {} for alg in names: lengths[data][alg] = [] for j, data in enumerate(datasets): for i in range(len(algs)): print() alg = algs[i] k = 1 while True: try: print("data: %s, alg: %s, k: %g" % (data, names[i], k)) res = [] with timeout(seconds=100): res = alg("%s/%s.txt" % (data, pos_files[j]), "%s/%s.txt" % (data, neg_files[j]), k, False, True) lengths[data][names[i]].append(mean([len(x) for x in res])) k *= 2 except (TimeoutError, RecursionError) as e: print(e) break pickle.dump(lengths, open("../report/data/lengths.p", "wb"))
def get_server(link): with timeout(seconds=20): try: r = requests.get(link) server = r.headers['Server'] return server except TimeoutError: raise TimeoutError
def test_custom_exception_message(): value = "Overslept" with pytest.raises(TimeoutError) as exc_info: with timeout(0.01, value=value): time.sleep(0.015) assert exc_info.value.args == (value, )
def test_custom_exception(): exc = RuntimeError("Overslept") with pytest.raises(RuntimeError) as exc_info: with timeout(0.01, exc): time.sleep(0.015) assert exc_info.value is exc
def check_functions(file_path, function_name, args, answers, time_out, no_unpack=False, nested=False, unbracket=False): result = False try: # get the function out of the file submitted filename = os.path.basename(file_path).split('.')[0] function = getattr( importlib.import_module('uploads.{0}'.format(filename)), function_name) func = timeout(timeout=time_out)(function) # make a copy to submit to function incase argument is mutatable args_in = copy.copy(args) # check the number of args and call function accrodingly if len(args_in) == 0: ans = func() else: if no_unpack: ans = func(args_in) else: ans = func(*args_in) # if answer is within an array if nested: if ans == answers: result = True else: if [ans] == answers: result = True # remove bracktes for display in markerbot if unbracket: answers = answers[0] return { 'input': args, 'result': result, 'output': ans, 'expected': answers } except Exception as e: return { 'input': args, 'result': result, 'output': e.message, 'expected': answers }
def charge_time(): GPIO.setup(b_pin, GPIO.IN) GPIO.setup(a_pin, GPIO.OUT) GPIO.output(a_pin, True) t1 = time.time() with timeout(seconds=5): while not GPIO.input(b_pin): pass t2 = time.time() return (t2 - t1) * 1000000 # microseconds
def _run(self): try: logging.info('Building matrices') self.build_matrices() finally: logging.info('Cleaning up state table') with timeout(self.cleanup_timeout): self.state_table_generator.clean_up() self.catwalk()
def _run(self): try: logging.info('Generating matrices') self.generate_matrices() finally: logging.info('Cleaning up state table') with timeout(self.cleanup_timeout): self.state_table_generator.clean_up() self.db_engine.execute('drop table if exists {}'.format( self.labels_table_name)) self.train_and_test_models()
def send_bind_success_message(openID, studentnumber): pushdata = { "studentnumber": { "value": studentnumber, "color": "#ff0000" } } try: with timeout(3): wechat.send_template_message(user_id=openID, template_id=_TEMPLATE_BIND_SUCCESS, data=pushdata, url="") except: logger.debug("send_template_message timeout")
def run(self): print utils.art self.client() self.stage_upstream() prs = self.client().repository(self.owner, self.repo).pull_requests() new_clones = [self.clone_dir(pr) for pr in prs] if os.path.exists(self.base_dir()): discard_dirs = set(os.listdir(self.base_dir())) - set(self.safe_dirs()) - set(new_clones) discard_dirs = list(discard_dirs) discard_dirs = [os.path.join(self.base_dir(), d) for d in discard_dirs] for discard_dir in discard_dirs: logger.log("===> Unstage {0} since PR is closed.".format(discard_dir)) shutil.rmtree(discard_dir) with timeout(self.timeout_seconds()): for pr in prs: self.process_pull(pr)
def process(self): """ process a work unit, spath, dpath refers to source and destination respectively """ spath = self.circle.deq() self.logger.debug("BEGIN process object: %s" % spath, extra=self.d) if spath: if spath in EXCLUDE: self.logger.warn("Skip excluded path: %s" % spath, extra=self.d) self.skipped += 1 return try: with timeout(seconds=5): st = os.lstat(spath) except OSError as e: self.logger.warn(e, extra=self.d) self.skipped += 1 return None except TimeoutError as e: self.logger.error("%s when stat() on %s" % (e, spath), extra=self.d) self.skipped += 1 return None except Exception as e: self.logger.error("Unknown: %s on %s" % (e, spath), extra=self.d) self.skipped += 1 return None self.reduce_items += 1 self.logger.debug("FIN lstat object: %s" % spath, extra=self.d) # islink() return True if it is symbolic link if os.path.islink(spath): self.sym_links += 1 # NOT TO FOLLOW SYM LINKS SHOULD BE THE DEFAULT return None self.handle_file_or_dir(spath, st) self.logger.debug("END process object: %s" % spath, extra=self.d)
def test_timeout_with_content_manager(seconds): import time with timeout.timeout(timeout=5): print "start" time.sleep(seconds) print "end"
return checksum if __name__ == '__main__': urllib3.disable_warnings() mswindows = (sys.platform == "win32") # learning from 'subprocess' module url = "https://raw.githubusercontent.com/racaljk/hosts/master/hosts" filename = url.split('/')[-1] save = os.path.join("/tmp", filename).replace("\\", "/") if mswindows: # global socket timeout socket.setdefaulttimeout(10.0) print "Downloading", url urllib.urlretrieve(url, filename=save) else: assert sys.platform == "linux2", "please run this script on Windows or Linux" with timeout(timeout=10.0): urllib.urlretrieve(url, filename=save) if os.path.isfile(save): print "Saved: '%s'" % save print "md5sum:", get_hash_sum(save, method="md5") print "sha1sum:", get_hash_sum(save, method="sha1sum") print "sha256sum:", get_hash_sum(save, method="sha256sum") else: print "can not download", url sys.exit(1)
for row in rows: ctr += 1 local_id = row['Local ID'] print '%s Processing %d/%d: %s' % (datetime.utcnow(), ctr, len(rows), local_id) counters['total'] += 1 last_fail_timeout = False def f(): doi = update(row, commit=commit, show_diff=show_diff, counters=counters, doi_update_rows=doi_update_rows, force_update_file=force_update_file) try: if last_fail_timeout: print 'Previous attempt timed out, waiting 30 seconds...' time.sleep(30) timeout.timeout(f, 180) counters['success'] += 1 last_fail_timeout = False except KeyboardInterrupt: raise KeyboardInterrupt except SystemExit: raise SystemExit except Exception, e: print 'ERROR on row %d: %s' % (ctr, traceback.format_exc()) if type(e) is timeout.TimeoutError: counters['timeout'] += 1 last_fail_timeout = True else: counters['error'] += 1 print str(dict(counters)) print 'Consider running `python merge_doi_maps.py %s %s`' % ( doi_tsv, doi_update_tsv)