def get(): transports = [_tor_getter.get()] for file in files: try: with open(file, 'r') as transport_file: transports.append(transport_file.read().strip()) except FileNotFoundError: pass else: break else: time.sleep(1) return list(transports)
def __retry_internal( f, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0, ): """ Executes a function and retries it if it failed. :param f: the function to execute. :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. :param tries: the maximum number of attempts. default: -1 (infinite). :param delay: initial delay between attempts. default: 0. :param max_delay: the maximum value of delay. default: None (no limit). :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). :param jitter: extra seconds added to delay between attempts. default: 0. fixed if a number, random if a range tuple (min, max) :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. default: retry.logging_logger. if None, logging is disabled. :returns: the result of the f function. """ _tries, _delay = tries, delay while _tries: try: return f() except exceptions as e: _tries -= 1 if not _tries: raise if logger is not None: logger.warning("%s, retrying in %s seconds...", e, _delay) time.sleep(_delay) _delay *= backoff if isinstance(jitter, tuple): _delay += random.uniform(*jitter) else: _delay += jitter if max_delay is not None: _delay = min(_delay, max_delay)
def iter_galleries(self, start=0, end=None, delay=None, **kwargs): i = start o = None while True: resp = self.get_gallery(page=i, **kwargs) if len(resp) == 0: break o2 = json.dumps(resp) if o == o2: warn("Page: %i and %i return the same results" % (i - 1, i)) break debug("Points is %i" % resp[0]['points']) debug("Score is %i" % resp[0]['score']) debug("Ups is %i" % resp[0]['ups']) #if min_points and resp[0]['points'] < min_points: # info("Stop at page %i because points is less than %i" % (i, min_points)) # break for d in resp: yield d i += 1 if end and i > end: break if delay: sleep(delay)
def update_stats_on_main(): errors = 0 sleep(60) # initial sleep while(is_server_running()): try: req = get_data("http://" + main_server_addr + "/update_server_stats?auth_key=" + get_internal_auth_key().decode() + "&server_addr=" + self_server_addr, post=json.dumps({"load": 0.0})) if(not req or json.loads(req.read())["type"] != TYPE_OK): errors += 1 sleep(5 * 60) # 5 minutes sleep except Exception as ex: LOG_WARN(LOG_TYPE_EXCEPTION, ex="could not update on main server") sleep(1 * 60) # 1 minutes sleep
def test_persist_timer(self): self.i = 0 def add_1(args): self.i += 1 gt = GTimer(only_once=False) gt.start(add_1, 0.1, 2) gt.start(add_1, 0.1, 2) gt.start(add_1, 0.1, 2) gt.start(add_1, 0.1, 2) time.sleep(0.3) self.assertTrue(self.i >= 3 and self.i <= 4) gt.stop() time.sleep(0.3) self.assertTrue(self.i >= 3 and self.i <= 4) gt.start(add_1, 0.1, 2) time.sleep(0.3) self.assertTrue(self.i >= 6 and self.i <= 8)
def _request(self, endpoint, retries=5, delay=30, ignore_credits=False): if not ignore_credits: api_credits = self.credits if api_credits['UserRemaining'] <= 10 and api_credits[ 'UserReset'] - time.time() > -60: # just to be sure s = api_credits['UserReset'] - time.time() + 60 warn("Waiting %i seconds till UserRemaining credits are reset." % (s)) sleep(s) return self._request(endpoint, retries, delay, ignore_credits) while api_credits['ClientRemaining'] <= 20: warn("Waiting an hour till ClientRemaining credits are reset.") sleep(60 * 60) api_credits = self.get_credits() for i in range(retries): debug("Requesting '%s'. Retry: %i" % (endpoint, i)) if self.credits: debug("Credits: %s" % self.credits) if i > 0: sleep(i * delay) # TODO: try catch all the possible request exceptions try: data = requests.request("GET", "https://api.imgur.com/3/%s" % endpoint, headers=self.headers) except Exception as e: error("Exception fetching '%s' : %s" % (endpoint, e)) continue if data.status_code >= 400: error("Bad status code (%s) fetching '%s'" % (data.status_code, endpoint)) continue for k, v in data.headers.items(): if k.startswith('X-RateLimit-'): #print("Update %s header: %s" % (k.replace("X-RateLimit-", ""), int(v))) self.credits[k.replace("X-RateLimit-", "")] = int(v) return data.json() raise FailedFetchingException(endpoint)
def lookup_blocks_from_communicator(comm_inst): logger.info('Looking up new blocks') tryAmount = 2 newBlocks = '' # List of existing saved blocks existingBlocks = get_block_list() triedPeers = [] # list of peers we've tried this time around # Max amount of *new* block hashes to have in queue maxBacklog = 1560 lastLookupTime = 0 # Last time we looked up a particular peer's list new_block_count = 0 for i in range(tryAmount): # Defined here to reset it each time, time offset is added later listLookupCommand = 'getblocklist' if len(comm_inst.blockQueue) >= maxBacklog: break if not comm_inst.isOnline: break # check if disk allocation is used if comm_inst.storage_counter.is_full(): logger.debug( 'Not looking up new blocks due to maximum amount of disk used') break try: # select random online peer peer = onlinepeers.pick_online_peer(comm_inst) except onionrexceptions.OnlinePeerNeeded: time.sleep(1) continue # if we've already tried all the online peers this time around, stop if peer in triedPeers: if len(comm_inst.onlinePeers) == len(triedPeers): break else: continue triedPeers.append(peer) # Get the last time we looked up a peer's stamp, # to only fetch blocks since then. # Saved in memory only for privacy reasons try: lastLookupTime = comm_inst.dbTimestamps[peer] except KeyError: lastLookupTime = epoch.get_epoch() - \ config.get("general.max_block_age", onionrvalues.DEFAULT_EXPIRE) listLookupCommand += '?date=%s' % (lastLookupTime,) try: newBlocks = peeraction.peer_action( comm_inst, peer, listLookupCommand) # get list of new block hashes except Exception as error: logger.warn( f'Could not get new blocks from {peer}.', error=error) newBlocks = False if newBlocks != False: # noqa # if request was a success for i in newBlocks.split('\n'): if stringvalidators.validate_hash(i): i = reconstructhash.reconstruct_hash(i) # if newline seperated string is valid hash # if block does not exist on disk + is not already in queue if i not in existingBlocks: if i not in comm_inst.blockQueue: if onionrproofs.hashMeetsDifficulty(i) and \ not blacklist.inBlacklist(i): if len(comm_inst.blockQueue) <= 1000000: # add blocks to download queue comm_inst.blockQueue[i] = [peer] new_block_count += 1 comm_inst.dbTimestamps[peer] = \ epoch.get_rounded_epoch(roundS=60) else: if peer not in comm_inst.blockQueue[i]: if len(comm_inst.blockQueue[i]) < 10: comm_inst.blockQueue[i].append(peer) if new_block_count > 0: block_string = "" if new_block_count > 1: block_string = "s" logger.info( f'Discovered {new_block_count} new block{block_string}', terminal=True) comm_inst.download_blocks_timer.count = \ int(comm_inst.download_blocks_timer.frequency * 0.99) comm_inst.decrementThreadCount('lookup_blocks_from_communicator')
def test_once_timer(self): meta = GrpcMeta('dev-pinpoint:9992', self.agent_meta) meta.start() time.sleep(1) meta.stop()
def func1(): print(123) time.sleep(3) print(456)
def new_connection(self): print("new connection to " + self.host + ":" + str(self.port)) time.sleep(1) return None
def main_polling(self): self.weibo_login() while 1: self.polling() time.sleep(weibo_config.polling_time)
def func2(i): print(456) time.sleep(2) return i, 'func2'
def _wait(self, delay): if delay > 0: time.sleep(delay)
def close_connection(self): print("connection closed") time.sleep(1) return None
def publish(self, set): print("uploading: " + set) time.sleep(2) return None
def func1(i): print(123) time.sleep(1) return i, 'func1'
def openUrl(self, urlName): time.sleep(2) self.driver.get(self.config[urlName])
def parse(self, response): item = ScholaridItem() item['scid'] = response.meta['scid'] item['scurl'] = response.meta['scurl'] item['name'] = response.css('.p_name ::text').extract_first() item['mechanism'] = response.css('.p_affiliate ::text').extract_first() p_ach = response.css('.p_ach_num ::text').extract() item['citedtimes'] = p_ach[0] item['resultsnumber'] = p_ach[1] item['Hindex'] = p_ach[2] item['Gindex'] = p_ach[3] field = response.css('.person_domain ::text').extract() item['field'] = list(filter(lambda x: x != '/', field)) pie = response.css('.pieText .number ::text').extract() if len(pie) == 4: item['journal'] = pie[0] item['meeting'] = pie[1] item['professionwork'] = pie[2] item['other'] = pie[3] else: item['journal'] = '' item['meeting'] = '' item['professionwork'] = '' item['other'] = '' item['total'] = response.css( '.pieMapTotal .number ::text').extract_first() #爬取关系网 chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--headless') browser = webdriver.Chrome(chrome_options=chrome_options) browser.get(response.request.url) item['copinfo'] = list() #如果有关系网图,就爬取图的,否则爬取侧栏的合作学者 try: browser.find_element_by_css_selector( '.co_author_wr h3 a').click() #模拟点击更多按钮 time.sleep(0.5) sreach_window = browser.current_window_handle #重定位网页 co_persons = browser.find_elements_by_css_selector( '.co_relmap_person') for co_person in co_persons: person = dict() person['url'] = self.source2real( co_person.get_attribute('href')) co_person = co_person.find_element_by_css_selector( '.co_person_name') person['name'] = co_person.text person['count'] = co_person.get_attribute('paper-count') #合作次数 person['mechanism'] = co_person.get_attribute('affiliate') item['copinfo'].append(person) except NoSuchElementException: co_persons = response.css('.au_info') for co_person in co_persons: person = dict() person['url'] = self.source2real( 'http://xueshu.baidu.com' + co_person.css('a::attr(href)').extract_first()) person['name'] = co_person.css('a ::text').extract_first() person['mechanism'] = co_person.css( '.au_label ::text').extract_first() person['count'] = 1 #暂定,网页并没有合作次数 item['copinfo'].append(person) finally: browser.close() yield item
def func2(): print('---------') time.sleep(1) print('=========')
def waiter(job): while job.result is None: time.sleep(0.1) return job.result
def CheckQueue(self, str): print(str) time.sleep(1) return True