class HackerNewsStore: @tenacity.retry(stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_random(min=1, max=2), before_sleep=tenacity.before_sleep_log( logger, logging.DEBUG)) # noqa: E501 def add_article(self, article): connection = DatabaseHelper.get_connection() with connection: cursor = connection.cursor() try: cursor.execute(self._get_query('add_article'), (article.id, article.link, article.title)) except psycopg2.IntegrityError as e: logger.info(e) def purge(self): connection = DatabaseHelper.get_connection() with connection: cursor = connection.cursor() lastCreateTime = datetime.date.today() - datetime.timedelta( days=10) # noqa: E501 cursor.execute(self._get_query('purge_removed_articles'), (lastCreateTime, )) logger.info(f"deleted rows:{cursor.rowcount}") def _get_query(self, query_id): query = None if DATABASE_CONFIG['type'] == 'postgre': query = PostgreQueries[query_id] else: raise RuntimeError( "The specified db type:{} is not supported".format( DATABASE_CONFIG['type'])) # noqa: E501 return query
def test_wait_double_sum(self): r = Retrying(wait=tenacity.wait_random(0, 3) + tenacity.wait_fixed(5)) # Test it a few time since it's random for i in six.moves.range(1000): w = r.wait(1, 5) self.assertLess(w, 8) self.assertGreaterEqual(w, 5)
def __init__(self, config: dict, logger, exchange): self.conf = config self.logger = logger self._exchange = exchange cfg = config.bot self._purchase_size_percentage = cfg["purchase_size_percentage"] self._update_interval = cfg["update_interval"] self._symbol = cfg["symbol"] self.util = Utils(self._exchange, self.logger) self._aretry = tenacity.AsyncRetrying( wait=tenacity.wait_random(0, 2), retry=(tenacity.retry_if_exception(ccxt.DDoSProtection) | tenacity.retry_if_exception(ccxt.RequestTimeout))) self._markets = {} if self._symbol: self._markets[self._symbol] = { "curr_sold": 0, "curr_bought": 0, "buys": 0, "sells": 0 }
def save_configuration(self, net_connect): retry_kwargs = { 'wait': tenacity.wait_random(min=2, max=6), 'reraise': False, 'stop': tenacity.stop_after_delay(30) } @tenacity.retry(**retry_kwargs) def _save(): try: net_connect.write(("copy running-config startup-config" + '\n').encode('utf-8')) error_msg = net_connect.read_until("Invalid", timeout=2) logger.debug("execute command :%s" % error_msg) if "Configuration update aborted" in error_msg: result = net_connect.read_very_eager() logger.debug("execute command failed.error msg:%s" % result) raise exceptions.ConfigSwitchError( command="copy running-config startup-config", error=result) except Exception: raise return error_msg return _save()
class GitUrlWatcher(SubTask): def __init__(self, app_config: Dict): super().__init__(name="git repo watcher") self.watched_repos = [] watched_compose_files_config = app_config["main"]["watched_git_repositories"] for config in watched_compose_files_config: repo = GitRepo( repo_id=config["id"], repo_url=config["url"], branch=config["branch"], tags=config["tags"], pull_only_files=config["pull_only_files"], username=config["username"], password=config["password"], paths=config["paths"], ) self.watched_repos.append(repo) async def init(self) -> Dict: description = await _init_repositories(self.watched_repos) return description @retry( reraise=True, stop=stop_after_attempt(NUMBER_OF_ATTEMPS), wait=wait_random(min=1, max=MAX_TIME_TO_WAIT_S), after=after_log(log, logging.DEBUG), ) async def check_for_changes(self) -> Dict: return await _check_repositories(self.watched_repos) async def cleanup(self): await _delete_repositories(self.watched_repos)
def __init__(self, logger=None, config=None, db=None): self._logger = logger if config: self._rsi_timeframe = config["rsi_timeframe"] self._interval = config["update_interval"] self._over_bought = config["over_bought"] self._rsi_period = config["rsi_period"] self._free_fall = config["free_fall"] self._over_sold = config["over_sold"] self._mooning = config["mooning"] self._db = db self._exchange_market_prices = {} self._significant_markets = set() self._aretry = tenacity.AsyncRetrying( wait=tenacity.wait_random(0, 3), retry=( tenacity.retry_if_exception(ccxt.DDoSProtection) | tenacity.retry_if_exception(ccxt.RequestTimeout) | tenacity.retry_if_exception(aiohttp.ServerDisconnectedError) ) )
def _blocking_poll(self, timeout=None): """Poll and wait for the Future to be resolved. Args: timeout (int): How long to wait for the operation to complete. If None, wait indefinitely. """ if self._result_set: return retry_on = tenacity.retry_if_result( functools.partial(operator.is_not, True)) # Use exponential backoff with jitter. wait_on = (tenacity.wait_exponential(multiplier=1, max=10) + tenacity.wait_random(0, 1)) if timeout is None: retry = tenacity.retry(retry=retry_on, wait=wait_on) else: retry = tenacity.retry(retry=retry_on, wait=wait_on, stop=tenacity.stop_after_delay(timeout)) try: retry(self.done)() except tenacity.RetryError as exc: six.raise_from( concurrent.futures.TimeoutError( 'Operation did not complete within the designated ' 'timeout.'), exc)
def retry_on_conflict(func): wrapper = tenacity.retry(stop=tenacity.stop_after_attempt(11), wait=tenacity.wait_random(max=0.002), retry=tenacity.retry_if_exception_type( exception.ConcurrentTransaction), reraise=True) return wrapper(func)
async def get_current_version(self) -> version.Version: async with httpx.AsyncClient(timeout=self.timeout) as client: try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), reraise=True, retry=retry_if_exception_type(httpx.ConnectTimeout), wait=wait_random(min=self.timeout, max=self.timeout * 10), ): with attempt: r = await client.get(self.checkurl) except (httpx.HTTPStatusError, httpx.ReadTimeout): return None webpage = r.text # FIXME: this is a hardcode tarball_regex = re.compile(r"\bemacs-[0-9.]+\.tar\.[a-z]*\b") version_regex = re.compile(r"\b[0-9.]+\b") currentversion = None for tarball in tarball_regex.findall(webpage): versionstring = version_regex.findall(tarball)[0] if versionstring.endswith("."): versionstring = versionstring[:-1] emacsversion = version.parse(versionstring) if currentversion is None or currentversion < emacsversion: currentversion = emacsversion return emacsversion
def _retry_send(self, function: Callable, attempt=10, *args, **kwargs): retry_configuration = tenacity.Retrying( stop=tenacity.stop_after_attempt(attempt), wait=tenacity.wait_fixed(3) + tenacity.wait_random(0, 2), after=tenacity.after_log(logger, logger.level) if logger else None, reraise=True, ) return retry_configuration(function, *args, **kwargs)
def retry_on_conflict(func): wrapper = tenacity.retry( stop=tenacity.stop_after_attempt(11), wait=tenacity.wait_random(max=0.002), retry=tenacity.retry_if_exception_type(exception.ConcurrentTransaction), reraise=True, ) return wrapper(func)
def wrapped_retry(*args, **kwargs): retryer = tenacity.Retrying( wait=tenacity.wait_random(1, 3), retry=tenacity.retry_if_exception_type(exception_type), stop=tenacity.stop_after_attempt(retries + 1), reraise=True, before_sleep=log_retry) return retryer(func, *args, **kwargs)
def __init__(self, min=0, multiplier=1, max=tenacity._utils.MAX_WAIT, exp_base=2): super(wait_random_exponential, self).__init__(multiplier=multiplier, max=(max - min), exp_base=exp_base) self._random = tenacity.wait_random(min=min, max=(min + multiplier))
def __init__(self, exchange: ccxt.Exchange, logger): self._exchange = exchange self._logger = logger self._aretry = tenacity.AsyncRetrying( wait=tenacity.wait_random(0, 2), retry=(tenacity.retry_if_exception(ccxt.DDoSProtection) | tenacity.retry_if_exception(ccxt.RequestTimeout))) self.purchase_strategies = {}
def retry(cls, timeout: float, callback, throw: Exception, *args, **kwargs) -> any: @tenacity.retry(wait=tenacity.wait_random(min=timeout/1000, max=timeout/100), stop=tenacity.stop_after_delay(timeout), reraise=True) def retry_impl(callback, *args, **kwargs): result = callback(*args, **kwargs) if result: return result raise throw('Operation timed out') return retry_impl(callback, *args, **kwargs)
def test_wait_arbitrary_sum(self): r = Retrying(wait=sum([tenacity.wait_fixed(1), tenacity.wait_random(0, 3), tenacity.wait_fixed(5), tenacity.wait_none()])) # Test it a few time since it's random for i in six.moves.range(1000): w = r.wait(1, 5) self.assertLess(w, 9) self.assertGreaterEqual(w, 6)
def __init__(self): # throttling self.throttling_wait = wait_chain( # always wait 20-40s first wait_fixed(20) + wait_random(0, 20), # wait 20-40s again wait_fixed(20) + wait_random(0, 20), # wait from 30 to 630s, with full jitter and exponentially # increasing max wait time wait_fixed(30) + wait_random_exponential(multiplier=1, max=600) ) # connection errors, other client and server failures self.network_wait = ( # wait from 3s to ~1m wait_random(3, 7) + wait_random_exponential(multiplier=1, max=55) ) self.server_wait = self.network_wait
def retry_blob_ingest_to_adx(container_name: str, blob_file_path: str, new_container_name: str, new_blob_file_path: str) -> None: """ Re-trigger the ingest pipeline by moving blob to retry folder """ # Add a random retry delay plus exponential backoff to mitigate the concurrent access to Azure retryer = Retrying(stop=stop_after_attempt(BLOB_REQ_MAX_ATTEMPT), wait=wait_random(0, 5) + wait_exponential(multiplier=1, min=2, max=BLOB_REQ_MAX_RETRY_DELAY_SEC), reraise=True) retryer(move_blob_file, AZURE_STORAGE_CONNECTION_STRING, container_name, new_container_name, blob_file_path, new_blob_file_path)
def test_random_sleep(self): r = Retrying(wait=tenacity.wait_random(min=1, max=20)) times = set() for x in six.moves.range(1000): times.add(r.wait(1, 6546)) # this is kind of non-deterministic... self.assertTrue(len(times) > 1) for t in times: self.assertTrue(t >= 1) self.assertTrue(t < 20)
class BasicParticipant(): """ Base class for an FFL general user """ def __init__(self, context: Context): """ Class initializer. Throws: An exception on failure :param context: connection details :type context: :class:`.Context` """ if not context: raise Exception('Credentials must be specified.') self.messenger = None self.context = context self.queue = None def __enter__(self): """ Context manager enters - call connect. Throws: An exception on failure :return: self :rtype: :class:`.BasicParticipant` """ return self.connect() def __exit__(self, *args): """ Context manager exits - call close Throws: An exception on failure """ self.close() @tenacity.retry(wait=tenacity.wait_random(min=1, max=2), stop=tenacity.stop_after_delay(5), reraise=True) def connect(self): """ Connect to the messaging system. Throws: An exception on failure :return: self :rtype: :class:`.BasicParticipant` """ self.messenger = Messenger(self.context, subscribe_queue=self.queue) return self def close(self) -> None: """ Close the connection to the messaging system. Throws: An exception on failure """ self.messenger.stop() self.messenger = None
def enviroplus_retry(fn: Coroutine) -> Coroutine: @wraps(fn) @ten.retry(retry=ten.retry_if_exception_type( (aiohttp.ClientError, aiohttp.http_exceptions.HttpProcessingError)), reraise=True, wait=ten.wait_random(1, 3), stop=ten.stop_after_attempt(3), before_sleep=ten.before_sleep_log(logger, logging.DEBUG)) async def wrapped_fn(*args, **kwargs): return await fn(*args, **kwargs) return wrapped_fn
def bind(self, req, body, instance_id, app_id): db_service = db_cf.get_service_for_instance(instance_id) if not db_service: return {} service_id = db_service.service_id environment_id = db_service.environment_id token = req.headers['X-Auth-Token'] m_cli = _get_muranoclient(token, req) session_id = create_session(m_cli, environment_id) env = m_cli.environments.get(environment_id, session_id) LOG.debug('Got environment {0}'.format(env)) service = self._get_service(env, service_id) LOG.debug('Got service {0}'.format(service)) # NOTE(starodubcevna): Here we need to find an action which will return # us needed credentials. By default we will look for getCredentials # action. result = {} try: actions = service['?']['_actions'] for action_id in list(actions): if 'getCredentials' in action_id: @tenacity.retry( retry=tenacity.retry_if_exception_type(TypeError), wait=tenacity.wait_random(min=1, max=10), stop=tenacity.stop_after_delay(30), reraise=True) def _get_creds(client, task_id, environment_id): result = m_cli.actions.get_result( environment_id, task_id)['result'] return result task_id = m_cli.actions.call(environment_id, action_id) result = _get_creds(m_cli, task_id, environment_id) if not result: LOG.warning( _LW("This application doesn't have action " "getCredentials")) return response.Response(status=500) except KeyError: # NOTE(starodubcevna): In CF service broker API spec return # code for failed bind is not present, so we will return 500. LOG.warning(_LW("This application doesn't have actions at all")) return response.Response(status=500) if 'credentials' in list(result): return result else: return {'credentials': result}
class PolicyInsights2: def __init__(self, credentials=None): load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES) self.credentials = AzureConnections().get_authenticated_client() @retry(wait=wait_random(min=1, max=3), stop=stop_after_attempt(4)) def policy_states_summarize_for_subscription(self, subscriptionId): """ This method calls the policy compliance using the policy insights and query policy state :rtype: response :param subscriptionId: :return: """ api_endpoint = self.config["AZURESDK"][ 'policy_states_summarize_for_subscription'] api_endpoint = api_endpoint.format(subscriptionId=subscriptionId) with request_authenticated_azure_session() as req: policy_states_summary = req.post(api_endpoint) return policy_states_summary @retry(wait=wait_random(min=1, max=3), stop=stop_after_attempt(4)) def policy_states_summarize_for_policy_definition(self, subscription_id, policy_definition_name): api_endpoint = self.config["AZURESDK"][ 'policy_states_summarize_for_policy_definition'] api_endpoint = api_endpoint.format( subscriptionId=subscription_id, policyDefinitionName=policy_definition_name) with request_authenticated_azure_session() as req: policy_states_summary = req.post(api_endpoint) return policy_states_summary
def test_random_sleep_without_min(self): r = Retrying(wait=tenacity.wait_random(max=2)) times = set() times.add(r.wait(1, 6546)) times.add(r.wait(1, 6546)) times.add(r.wait(1, 6546)) times.add(r.wait(1, 6546)) # this is kind of non-deterministic... self.assertTrue(len(times) > 1) for t in times: self.assertTrue(t >= 0) self.assertTrue(t <= 2)
class DockerRegistriesWatcher(SubTask): def __init__(self, app_config: Dict, stack_cfg: Dict): super().__init__(name="dockerhub repo watcher") # get all the private registries self.private_registries = app_config["main"]["docker_private_registries"] # get all the images to check for self.watched_repos = [] if "services" in stack_cfg: for service_name in stack_cfg["services"].keys(): if "image" in stack_cfg["services"][service_name]: image_url = stack_cfg["services"][service_name]["image"] self.watched_repos.append({ "image": image_url }) async def init(self): log.debug("initialising docker watcher..") with docker_client(self.private_registries) as client: for repo in self.watched_repos: try: registry_data = client.images.get_registry_data(repo["image"]) log.debug("accessed to image %s: %s", repo["image"], registry_data.attrs) repo["registry_data_attrs"] = registry_data.attrs except docker.errors.APIError: # in case a new service that is not yet in the registry was added log.warning("could not find image %s, maybe a new image was added to the stack??", repo["image"]) repo["registry_data_attrs"] = "" log.debug("docker watcher initialised") @retry(reraise=True, stop=stop_after_attempt(NUMBER_OF_ATTEMPS), wait=wait_random(min=1, max=MAX_TIME_TO_WAIT_S), after=after_log(log, logging.DEBUG)) async def check_for_changes(self) -> Dict: changes = {} with docker_client(self.private_registries) as client: for repo in self.watched_repos: try: registry_data = client.images.get_registry_data(repo["image"]) if repo["registry_data_attrs"]['Descriptor'] != registry_data.attrs['Descriptor']: log.info("docker image %s signature changed from %s to %s!", repo["image"], repo["registry_data_attrs"], registry_data.attrs) changes[repo['image']] = "image signature changed" except docker.errors.APIError: if repo["registry_data_attrs"]: # in that case something is wrong...either docker or config log.exception("Error while retrieving image %s in registry", repo["image"]) # raise else: # in that case the registry does not contain yet the new service log.warning("the image %s is still not available in the registry", repo["image"]) return changes async def cleanup(self): pass
async def fetch_url(url: str, timeout: int = 5, retry: int = 5): async with httpx.AsyncClient(timeout=timeout) as client: try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), reraise=True, retry=retry_if_exception_type(httpx.ConnectTimeout), wait=wait_random(min=timeout, max=timeout * 10), ): with attempt: r = await client.get(url) except (httpx.HTTPStatusError, httpx.ReadTimeout) as e: return str(e) return r.json()
class Comms_worker: """ """ def __init__(self, commsffl, worker_real_name='Anonymous'): """ """ self.id = worker_real_name # unused by now... #self.task_name = task_name #self.commsffl = ffl.Factory.participant(context_w, task_name=self.task_name) self.name = 'pycloudmessenger' self.commsffl = commsffl def send(self, message, address=None): try: # address is not used here, a worker can only send to the master with self.commsffl: self.commsffl.send(message) except Exception as err: print('\n') print('*' * 80) print('Pycloudmessenger ERROR at send: %s' % err) print('*' * 80) print('\n') raise def receive(self, timeout=1): try: with self.commsffl: packet = self.commsffl.receive(timeout) message = packet.content except Exception as err: if 'pycloudmessenger.ffl.fflapi.TimedOutException' not in str( type(err)): # we skip the normal timeouts print('\n') print('*' * 80) print('Pycloudmessenger ERROR at receive: %s' % err) print('*' * 80) print('\n') else: message = None raise return message @tenacity.retry(stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_random(min=1, max=3)) def receive_poms_123(self, timeout=10): with self.commsffl: packet = self.commsffl.receive(timeout) return packet
class PoloniexBaseAPI(metaclass=ABCMeta): @abstractmethod def __init__( self, *, requests_per_second: int = REQUESTS_PER_SECOND, ) -> None: self._rate_limiter = RateLimiter(requests_per_second, 1, 1) self._requests_per_second = requests_per_second self._session = Session() @tenacity.retry( retry=tenacity.retry_if_exception(_retry_poloniex_error), wait=tenacity.wait_exponential() + tenacity.wait_random(0, 1), stop=tenacity.stop_after_attempt(4) | tenacity.stop_after_delay(8), reraise=True, ) def request( self, *args, **kwargs, ) -> Optional[Union[Dict[str, Any], List[Dict[str, Any]]]]: cls_name = type(self).__name__ while self._rate_limiter.check(cls_name) is False: sleep(1 / self._requests_per_second) request = Request(*args, **kwargs) prepared = self._session.prepare_request(request) response = self._session.send(prepared) status = response.status_code if status >= 200 and status < 300: data = None try: data = response.json() except ValueError: logger.exception('Error decoding response JSON') return data elif status >= 400 and status < 500: raise PoloniexRequestError(response) elif status >= 500: raise PoloniexServerError(response) else: # We shouldn't ever get 1xx responses (Poloniex doesn't send them) # or 3xx responses (requests follows redirects); return None to # make mypy happy return None
def save_configuration(self, net_connect, cmd='copy running-config startup-config',): retry_kwargs = {'wait': tenacity.wait_random(min=2, max=6), 'reraise': False, 'stop': tenacity.stop_after_delay(30)} @tenacity.retry(**retry_kwargs) def _save(): try: output = super(CiscoNxosSSH, net_connect).save_config() self._check_output(output, cmd) except Exception: raise return output return _save()
class HttpFetcher: SSL_VERIFY = os.environ.get("SSL_VERIFY", True) not in FALSY def __init__(self, logger): self.logger = logger if not self.SSL_VERIFY: logger.warning( "You have set ssl certificates to not be verified. " "This may leave you vulnerable. " "http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification" ) def get_size(self, url): try: head = requests.head(url, allow_redirects=True, verify=self.SSL_VERIFY) return int(head.headers["Content-Length"]) except: head = requests.get( url, allow_redirects=True, verify=self.SSL_VERIFY, headers={"Range": "bytes=0-1"}, ) crange = head.headers["Content-Range"] match = re.search(r"/(\d+)$", crange) if match: return int(match.group(1)) self.logger.error(traceback.format_exc()) raise FuseOSError(ENOENT) @retry(wait=wait_fixed(1) + wait_random(0, 2), stop=stop_after_attempt(2)) def get_data(self, url, start, end): headers = { "Range": "bytes={}-{}".format(start, end), "Accept-Encoding": "" } self.logger.info("getting %s %s %s", url, start, end) r = requests.get(url, headers=headers) self.logger.info("got %s", r.status_code) r.raise_for_status() block_data = np.frombuffer(r.content, dtype=np.uint8) return block_data
def run_migrations_online(): """Run migrations in 'online' mode. In this scenario we need to create an Engine and associate a connection with the context. """ # this callback is used to prevent an auto-migration from being generated # when there are no changes to the schema # reference: http://alembic.readthedocs.org/en/latest/cookbook.html def process_revision_directives(context, revision, directives): if getattr(config.cmd_opts, 'autogenerate', False): script = directives[0] if script.upgrade_ops.is_empty(): directives[:] = [] logger.info('No changes in schema detected.') engine = engine_from_config(config.get_section(config.config_ini_section), prefix='sqlalchemy.', poolclass=pool.NullPool) @tenacity.retry( stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5), before=tenacity.before_log(logging.getLogger('tenacity.retry'), logging.DEBUG), before_sleep=tenacity.before_sleep_log( logging.getLogger('tenacity.retry'), logging.INFO), after=tenacity.after_log(logging.getLogger('tenacity.retry'), logging.DEBUG)) def try_connect(db): return db.connect() with try_connect(engine) as connection: context.configure( connection=connection, target_metadata=target_metadata, process_revision_directives=process_revision_directives, **current_app.extensions['migrate'].configure_args) with context.begin_transaction(): context.run_migrations() connection.close()
def run_migrations_online(): """Run migrations in 'online' mode. In this scenario we need to create an Engine and associate a connection with the context. """ # this callback is used to prevent an auto-migration from being generated # when there are no changes to the schema # reference: http://alembic.readthedocs.org/en/latest/cookbook.html def process_revision_directives(context, revision, directives): if getattr(config.cmd_opts, 'autogenerate', False): script = directives[0] if script.upgrade_ops.is_empty(): directives[:] = [] logger.info('No changes in schema detected.') engine = engine_from_config(config.get_section(config.config_ini_section), prefix='sqlalchemy.', poolclass=pool.NullPool) connection = tenacity.Retrying( stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5), before=tenacity.before_log(logging.getLogger("tenacity.retry"), logging.DEBUG), before_sleep=tenacity.before_sleep_log(logging.getLogger("tenacity.retry"), logging.INFO), after=tenacity.after_log(logging.getLogger("tenacity.retry"), logging.DEBUG) ).call(engine.connect) context.configure(connection=connection, target_metadata=target_metadata, process_revision_directives=process_revision_directives, **current_app.extensions['migrate'].configure_args) try: with context.begin_transaction(): context.run_migrations() finally: connection.close()
import random import tenacity def do_something(): if random.randint(0, 1) == 0: print('Failure') raise RuntimeError print('Success') @tenacity.retry(wait=tenacity.wait_fixed(10) + tenacity.wait_random(0, 3)) def do_something_and_retry(): do_something() do_something_and_retry()
def process_data(self): stations = {} try: self.log.info('Processing FFVL data...') result = requests.get( 'http://data.ffvl.fr/json/balises.json', timeout=(self.connect_timeout, self.read_timeout)) ffvl_stations = result.json() for ffvl_station in ffvl_stations: ffvl_id = None try: ffvl_id = ffvl_station['idBalise'] station = self.save_station( ffvl_id, ffvl_station['nom'], ffvl_station['nom'], ffvl_station['latitude'], ffvl_station['longitude'], Status.GREEN, altitude=ffvl_station['altitude'], url=ffvl_station['url']) stations[station['_id']] = station except ProviderException as e: self.log.warn(f"Error while processing station '{ffvl_id}': {e}") except Exception as e: self.log.exception(f"Error while processing station '{ffvl_id}': {e}") except ProviderException as e: self.log.warn(f'Error while processing stations: {e}') except Exception as e: self.log.exception(f'Error while processing stations: {e}') try: @retry(wait=wait_random(min=1, max=3), stop=(stop_after_delay(15))) def request_data(): # data.ffvl.fr randomly returns an empty file instead the json doc result = requests.get( 'http://data.ffvl.fr/json/relevesmeteo.json', timeout=(self.connect_timeout, self.read_timeout)) return result.json() ffvl_measures = request_data() ffvl_tz = tz.gettz('Europe/Paris') for ffvl_measure in ffvl_measures: station_id = None try: ffvl_id = ffvl_measure['idbalise'] station_id = self.get_station_id(ffvl_id) if station_id not in stations: raise ProviderException(f"Unknown station '{station_id}'") station = stations[station_id] measures_collection = self.measures_collection(station_id) new_measures = [] key = arrow.get(ffvl_measure['date'], 'YYYY-MM-DD HH:mm:ss').replace(tzinfo=ffvl_tz).timestamp if not self.has_measure(measures_collection, key): measure = self.create_measure( station, key, ffvl_measure['directVentMoy'], ffvl_measure['vitesseVentMoy'], ffvl_measure['vitesseVentMax'], temperature=ffvl_measure['temperature'], humidity=ffvl_measure['hydrometrie'], pressure=Pressure(qfe=ffvl_measure['pression'], qnh=None, qff=None) ) new_measures.append(measure) self.insert_new_measures(measures_collection, station, new_measures) except ProviderException as e: self.log.warn(f"Error while processing measures for station '{station_id}': {e}") except Exception as e: self.log.exception(f"Error while processing measures for station '{station_id}': {e}") except ProviderException as e: self.log.warn(f'Error while processing FFVL: {e}') except Exception as e: self.log.exception(f'Error while processing FFVL: {e}') self.log.info('...Done!')