def start(self, loop: asyncio.BaseEventLoop = None): self.lock_cooking_number() for instruction in self.instructions: if not isinstance(instruction, dict): print(f'pressing key {instruction}') self.key_press(instruction) time.sleep(self.sleep_tm) else: for _ in range(instruction['times']): if 'press_key_down_time' not in instruction: self.key_press(instruction['key']) time.sleep(self.sleep_tm) else: self.key_down(instruction['key'], instruction['press_key_down_time']) time.sleep(self.sleep_tm) if self.additional_cooking_instructions: recipe = Recipe(**self.additional_cooking_instructions) loop.call_later(self.cook_tm, cooking_timer, self.order_num, recipe) else: loop.call_later(self.cook_tm, cooking_timer, self.order_num)
def _master_watcher(pid: int, loop: asyncio.BaseEventLoop): loop.call_later(MASTER_WATCHER_PERIOD, _master_watcher, pid, loop) try: os.kill(pid, 0) # check master process except ProcessLookupError: os._exit( os.EX_OK) # noqa: W0212 hard break better than deattached pocesses
def shared(loop: asyncio.BaseEventLoop): fut = loop.create_future() def callback(): print("called") fut.set_result("xxx") loop.call_later(0.2, callback) return fut
def bootstrap_dht_nodes( loop: asyncio.BaseEventLoop, table_client: azure.storage.table.TableService, ipaddress: str, num_attempts: int) -> None: """Bootstrap DHT router nodes :param asyncio.BaseEventLoop loop: event loop :param azure.storage.table.TableService table_client: table client :param str ipaddress: ip address :param int num_attempts: number of attempts """ found_self = False dht_nodes = [] try: entities = table_client.query_entities( _STORAGE_CONTAINERS['table_dht'], filter='PartitionKey eq \'{}\''.format(_PARTITION_KEY)) except azure.common.AzureMissingResourceHttpError: pass else: for entity in entities: dht_nodes.append((entity['RowKey'], entity['Port'])) if entity['RowKey'] == ipaddress: found_self = True if not found_self: entity = { 'PartitionKey': _PARTITION_KEY, 'RowKey': ipaddress, 'Port': _DEFAULT_PORT_BEGIN, } table_client.insert_entity(_STORAGE_CONTAINERS['table_dht'], entity) dht_nodes.insert(0, (ipaddress, _DEFAULT_PORT_BEGIN)) # TODO handle vm/ips no longer in pool for node in dht_nodes: if len(_DHT_ROUTERS) >= 3: break add_dht_node(node[0], node[1]) # ensure at least 3 DHT router nodes if possible if len(dht_nodes) < 3: num_attempts += 1 if num_attempts < 600: delay = 1 elif num_attempts < 1200: delay = 10 else: delay = 30 loop.call_later( delay, bootstrap_dht_nodes, loop, table_client, ipaddress, num_attempts)
def _renew_blob_lease(loop: asyncio.BaseEventLoop, blob_client: azureblob.BlockBlobService, container_key: str, resource: str, blob_name: str): """Renew a storage blob lease :param asyncio.BaseEventLoop loop: event loop :param azureblob.BlockBlobService blob_client: blob client :param str container_key: blob container index into _STORAGE_CONTAINERS :param str resource: resource :param str blob_name: blob name """ try: lease_id = blob_client.renew_blob_lease( container_name=_STORAGE_CONTAINERS[container_key], blob_name=blob_name, lease_id=_BLOB_LEASES[resource], ) except azure.common.AzureException as e: logger.exception(e) _BLOB_LEASES.pop(resource) _CBHANDLES.pop(resource) else: _BLOB_LEASES[resource] = lease_id _CBHANDLES[resource] = loop.call_later(15, _renew_blob_lease, loop, blob_client, container_key, resource, blob_name)
def start(self, loop: asyncio.BaseEventLoop): # we don't call self.event directly, because call_later has a max waiting time of 1 day # (see https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.time) # So we check if the timer needs to be rescheduled. delta = self.expires - datetime.datetime.now(datetime.timezone.utc) if delta.total_seconds() <= 0: self.event(*self.args) else: # loop.call_later((self.expires - datetime.utcnow()).total_seconds(), self.event, *self.args) self._timer = loop.call_later(delta.total_seconds(), self.start, loop)
def _renew_queue_message_lease(loop: asyncio.BaseEventLoop, queue_client: azure.storage.queue.QueueService, queue_key: str, cb_key: str, msg_id: str): """Renew a storage queue message lease :param asyncio.BaseEventLoop loop: event loop :param azure.storage.queue.QueueService queue_client: queue client :param str queue_key: queue name key index into _STORAGE_CONTAINERS :param str cb_key: callback handle key :param str msg_id: message id """ msg = queue_client.update_message(_STORAGE_CONTAINERS[queue_key], message_id=msg_id, pop_receipt=_QUEUE_MESSAGES[msg_id], visibility_timeout=45) if msg.pop_receipt is None: raise RuntimeError('update message failed for id={} pr={}'.format( msg_id, _QUEUE_MESSAGES[msg_id])) _QUEUE_MESSAGES[msg_id] = msg.pop_receipt _CBHANDLES[cb_key] = loop.call_later(15, _renew_queue_message_lease, loop, queue_client, queue_key, cb_key, msg_id)
async def _direct_download_resources_async( loop: asyncio.BaseEventLoop, blob_client: azure.storage.blob.BlockBlobService, queue_client: azure.storage.queue.QueueService, table_client: azure.storage.table.TableService, ipaddress: str, nglobalresources: int) -> None: """Direct download resource logic :param asyncio.BaseEventLoop loop: event loop :param azure.storage.blob.BlockBlobService blob_client: blob client :param azure.storage.queue.QueueService queue_client: queue client :param azure.storage.table.TableService table_client: table client :param str ipaddress: ip address :param int nglobalresources: number of global resources """ # iterate through downloads to see if there are any torrents available with _DIRECTDL_LOCK: if len(_DIRECTDL) == 0: return # go through queue and find resources we can download msgs = queue_client.get_messages( _STORAGE_CONTAINERS['queue_globalresources'], num_messages=1, visibility_timeout=60) if len(msgs) == 0: return msg = None _rmdl = [] _release_list = [] _start_torrent_list = [] with _DIRECTDL_LOCK: for _msg in msgs: if (msg is None and _msg.content in _DIRECTDL and _msg.content not in _DIRECTDL_DOWNLOADING): if _ENABLE_P2P: nseeds = _get_torrent_num_seeds(table_client, _msg.content) if nseeds < _SEED_BIAS: msg = _msg else: _start_torrent_list.append(_msg) _rmdl.append(_msg.content) _release_list.append(_msg) else: msg = _msg else: _release_list.append(_msg) # renew lease and create renew callback if msg is not None: if _ENABLE_P2P or not _NON_P2P_CONCURRENT_DOWNLOADING: _QUEUE_MESSAGES[msg.id] = msg.pop_receipt _CBHANDLES[msg.content] = loop.call_later( 15, _renew_queue_message_lease, loop, queue_client, 'queue_globalresources', msg.content, msg.id) else: _release_list.append(msg) # release all messages in release list for _msg in _release_list: try: queue_client.update_message( _STORAGE_CONTAINERS['queue_globalresources'], message_id=_msg.id, pop_receipt=_msg.pop_receipt, visibility_timeout=0) except azure.common.AzureMissingResourceHttpError as ex: # message not exist can happen if there are large delays from # message lease till now if ex.status_code != 404: raise # start any torrents for _msg in _start_torrent_list: _start_torrent_via_storage(blob_client, table_client, _msg.content) # remove messages out of rmdl if len(_rmdl) > 0: with _DIRECTDL_LOCK: for dl in _rmdl: try: logger.info( 'removing resource {} from direct downloads'.format( dl)) _DIRECTDL.remove(dl) except ValueError: pass if msg is None: return del _start_torrent_list del _release_list del _rmdl # pull and save docker image in thread if msg.content.startswith(_DOCKER_TAG): thr = DockerSaveThread(blob_client, queue_client, table_client, msg.content, msg.id, nglobalresources) thr.start() else: # TODO download via blob, explode uri to get container/blob # use download to path into /tmp and move to _TORRENT_DIR raise NotImplementedError()
def start_callback(index: int, loop: asyncio.BaseEventLoop, interval: int): print(f"start {index}th callback at {time.monotonic()}") loop.call_later(interval, start_callback, index + 1, loop, interval)
async def _direct_download_resources_async( loop: asyncio.BaseEventLoop, blob_client: azureblob.BlockBlobService, table_client: azuretable.TableService, nglobalresources: int) -> None: """Direct download resource logic :param asyncio.BaseEventLoop loop: event loop :param azureblob.BlockBlobService blob_client: blob client :param azuretable.TableService table_client: table client :param int nglobalresources: number of global resources """ # ensure we are not downloading too many sources at once with _DIRECTDL_LOCK: if len(_DIRECTDL_DOWNLOADING) > _CONCURRENT_DOWNLOADS_ALLOWED: return # retrieve a resource from dl queue _seen = set() while True: try: resource = _DIRECTDL_QUEUE.get() except queue.Empty: break else: if resource in _seen: _DIRECTDL_QUEUE.put(resource) resource = None break _seen.add(resource) with _DIRECTDL_LOCK: if resource not in _DIRECTDL_DOWNLOADING: break else: _DIRECTDL_QUEUE.put(resource) resource = None del _seen # attempt to get a blob lease if resource is not None: lease_id = None blob_name = None for i in range(0, _CONCURRENT_DOWNLOADS_ALLOWED): blob_name = '{}.{}'.format(compute_resource_hash(resource), i) try: lease_id = blob_client.acquire_blob_lease( container_name=_STORAGE_CONTAINERS['blob_globalresources'], blob_name=blob_name, lease_duration=60, ) break except azure.common.AzureConflictHttpError: blob_name = None pass if lease_id is None: logger.debug('no available blobs to lease for resource: {}'.format( resource)) _DIRECTDL_QUEUE.put(resource) return # create lease renew callback logger.debug('blob lease {} acquired for resource {}'.format( lease_id, resource)) _BLOB_LEASES[resource] = lease_id _CBHANDLES[resource] = loop.call_later(15, _renew_blob_lease, loop, blob_client, 'blob_globalresources', resource, blob_name) if resource is None: return # pull and save container image in thread if is_container_resource(resource): thr = ContainerImageSaveThread(blob_client, table_client, resource, blob_name, nglobalresources) thr.start() else: # TODO download via blob, explode uri to get container/blob # use download to path into /tmp and move to directory raise NotImplementedError()