def make_node(node): public_address = _filter_ipv4(node.public_ips)[0] if isinstance(public_address, unicode): public_address = public_address.encode("ascii") if self._use_private_addresses: private_address = _filter_ipv4(node.private_ips)[0] else: private_address = None if isinstance(private_address, unicode): private_address = private_address.encode("ascii") Message.log( message_type="flocker:provision:libcloud:node_created", name=node.name, id=node.id, public_address=public_address, private_address=private_address, ) return LibcloudNode( provisioner=self, node=node, address=public_address, private_address=private_address, distribution=distribution)
def _pytest_collection_modifyitems(task_arguments, worker, items, invenio_rule, option, batch_recids_, all_recids_): """Report allowed recids and jsonpaths to master and await start. :type config: :py:class:_pytest.config.Config :type items: list """ _ensure_only_one_test_function_exists_in_check(items) item = items[0] # TODO: There could be a flag that declares whether there are side-effects # in the DB, instead of checking for use of record_fetching_fixturenames # Inform the worker about allowed paths and recids if _test_func_uses_record_related_fixtures(item): worker.allowed_paths, worker.allowed_recids = \ _get_restrictions_from_check_class(item, task_arguments, batch_recids_, all_recids_) else: worker.allowed_paths, worker.allowed_recids = None, None # Check if there are conflicts with other workers worker.status = StatusWorker.ready with worker.lock(): blockers = _worker_conflicts_with_currently_running(worker) worker.retry_after_ids = {bl.uuid for bl in blockers} if blockers: Message.log(message_type='detected conflicting workers', value=str(blockers)) del items[:] else: option.invenio_reporters = load_reporters(invenio_rule, option.invenio_execution) worker.status = StatusWorker.running
def handlePLAINTEXTServer(self, header): """ Parse a complete HTTP-like Foolscap negotiation request and begin proxying to a destination selected based on the extract TubID. """ # the client sends us a GET message lines = header.split("\r\n") if not lines[0].startswith("GET "): raise BananaError("not right") command, url, version = lines[0].split() if not url.startswith("/id/"): # probably a web browser raise BananaError("not right") targetTubID = url[4:] Message.log(event_type=u"handlePLAINTEXTServer", tub_id=targetTubID) if targetTubID == "": # they're asking for an old UnauthenticatedTub. Refuse. raise NegotiationError("secure Tubs require encryption") if isSubstring("Upgrade: TLS/1.0\r\n", header): wantEncrypted = True else: wantEncrypted = False Message.log(event_type=u"handlePLAINTEXTServer", want_encrypted=wantEncrypted) self._handleTubRequest(header, targetTubID)
def stop(self): """ Stop the scenario from being maintained by stopping all the loops that may be executing. :return Deferred[Optional[Dict[unicode, Any]]]: Scenario metrics. """ self.is_started = False if self.monitor_loop.running: self.monitor_loop.stop() if self.loop.running: self.loop.stop() outstanding_requests = self.rate_measurer.outstanding() if outstanding_requests > 0: msg = ( "There are {num_requests} outstanding requests. " "Waiting {num_seconds} seconds for them to complete." ).format(num_requests=outstanding_requests, num_seconds=self.timeout) Message.log(key="outstanding_requests", value=msg) with start_action(action_type=u"flocker:benchmark:scenario:stop", scenario="request_load"): def no_outstanding_requests(): return self.rate_measurer.outstanding() == 0 scenario_stopped = loop_until(self.reactor, no_outstanding_requests, repeat(1)) timeout(self.reactor, scenario_stopped, self.timeout) scenario = DeferredContext(scenario_stopped) def handle_timeout(failure): failure.trap(CancelledError) msg = ("Force stopping the scenario. " "There are {num_requests} outstanding requests").format( num_requests=outstanding_requests ) Message.log(key="force_stop_request", value=msg) scenario.addErrback(handle_timeout) def scenario_cleanup(ignored): """ Calls the scenario cleanup, and wraps it inside an eliot start action, so we can see the logs if something goes wrong within the cleanup :return Deferred: that will fire once the cleanup has been completed """ with start_action(action_type=u"flocker:benchmark:scenario:cleanup", scenario="request_load"): return self.request.run_cleanup() scenario.addBoth(scenario_cleanup) def return_metrics(_ignore): return self.rate_measurer.get_metrics() scenario.addCallback(return_metrics) return scenario.addActionFinish()
def g(recurse): with start_action(action_type="a-recurse={}".format(recurse)): Message.log(message_type="m-recurse={}".format(recurse)) if recurse: set(g(False)) else: yield
def log_progress(): Message.log( message_type="flocker.benchmark.container_setup:progress", container_count=self.container_count, error_count=self.error_count, total_containers=total, )
def instance_error(failure): Message.log( message_type="flocker:provision:aws:async_get_node:failed" ) instance.terminate() write_failure(failure) return failure
def log_totals(result): Message.log( message_type="flocker.benchmark.container_setup:finish", container_count=self.container_count, error_count=self.error_count, ) return result
def _build_config(self, ignored): """ Build a Flocker deployment configuration for the given cluster and parameters. The configuration consists of identically configured applications (containers) uniformly spread over all cluster nodes. :return dict: containing the json we need to send to compose to create the datasets and containers we want. """ Message.log(action="Building config") application_root = {} applications = {} application_root["version"] = 1 application_root["applications"] = applications for node in self.nodes: for i in range(self.per_node): name = "app_%s_%d" % (node.public_address, i) applications[name] = deepcopy(self.application_template) deployment_root = {} nodes = {} deployment_root["nodes"] = nodes deployment_root["version"] = 1 for node in self.nodes: addr = "%s" % node.public_address nodes[addr] = [] for i in range(self.per_node): name = "app_%s_%d" % (node.public_address, i) nodes[addr].append(name) return {"applications": application_root, "deployment": deployment_root}
def _converge_route53_customer(actual, config, subscriptions, k8s, aws): """ Converge on the desired Route53 state relating to individual S4 subscriptions. Specifically, make sure the per-subscription domain names that grants access to individual subscription's introducer and storage servers exist and point at the right thing. Also clean up the names for any cancelled subscriptions. """ changes = _compute_changes( actual.subscriptions, _ChangeableZone( zone=actual.zone.zone, rrsets=actual.zone.rrsets, domain=config.domain, ), ) # XXX Probably would be nice to group changes. Also, issue changes insert # of delete/create. Some structured objects would make that easier. route53 = aws.get_route53_client() def delete(sid): return delete_route53_rrsets(route53, actual.zone.zone, [sid]) def create(subscription): return create_route53_rrsets(route53, actual.zone.zone, [subscription]) deletes = list(partial(delete, sid) for sid in changes.delete) creates = list(partial(create, s) for s in changes.create) Message.log( event=u"convergence-service:route53-customer", create=list(subscription.subscription_id for subscription in changes.create), delete=list(changes.delete), ) return deletes + creates
def send_delay_to_fsm(sleep): Message.log( message_type=u'flocker:node:_loop:CONVERGE:delay', log_level=u'INFO', message=u'Delaying until next convergence loop.', delay=sleep.delay_seconds ) return self.fsm.receive(sleep)
def _run_task(rule_name, master_id): del Logger._destinations._destinations[:] to_file(open(os.path.join(eliot_log_path, master_id), "ab")) with start_task(action_type="invenio_checker:supervisor:_run_task", master_id=master_id) as eliot_task: from .models import CheckerRule # cleanup_failed_runs() redis_master = None def cleanup_session(): print 'Cleaning up' if redis_master is not None: redis_master.zap() def sigint_hook(rcv_signal, frame): cleanup_session() def except_hook(type_, value, tback): cleanup_session() reraise(type_, value, tback) signal.signal(signal.SIGINT, sigint_hook) signal.signal(signal.SIGTERM, sigint_hook) sys.excepthook = except_hook with start_action(action_type='create master'): eliot_task_id = eliot_task.serialize_task_id() redis_master = RedisMaster(master_id, eliot_task_id, rule_name) with start_action(action_type='create subtasks'): rules = CheckerRule.from_ids((rule_name,)) bundles = rules_to_bundles(rules, redis_master.all_recids) subtasks = [] errback = handle_error.s() for rule, rule_chunks in bundles.iteritems(): for chunk in rule_chunks: task_id = uuid() redis_master.workers_append(task_id) eliot_task_id = eliot_task.serialize_task_id() RedisWorker(task_id, eliot_task_id, chunk) subtasks.append(run_test.subtask(args=(rule.filepath, redis_master.master_id, task_id), task_id=task_id, link_error=[errback])) Message.log(message_type='registered subtasks', value=str(redis_master.workers)) with start_action(action_type='run chord'): redis_master.status = StatusMaster.running header = subtasks callback = handle_results.subtask(link_error=[handle_errors.s(redis_master.master_id)]) my_chord = chord(header) result = my_chord(callback) redis_master.status = StatusMaster.running
def handle_timeout(failure): failure.trap(CancelledError) msg = ( "Force stopping the scenario. " "There are {num_requests} outstanding requests" ).format( num_requests=outstanding_requests ) Message.log(key='force_stop_request', value=msg)
def test_returns_unfired_deferred(self): Message.log(hello="world") # @eliot_logged_test automatically gives us an action context but it's # still our responsibility to maintain it across stack-busting # operations. d = DeferredContext(deferLater(reactor, 0.0, lambda: None)) d.addCallback(lambda ignored: Message.log(goodbye="world")) # We didn't start an action. We're not finishing an action. return d.result
def handle_failure(failure): Message.log( message_type=( u"flocker:provision:libcloud:retry_exception:got_exception" ), ) write_failure(failure) saved_failure[0] = failure return False
def create_configuration(deploy_config, details, model): """ Create the Kubernetes configuration resource necessary to provide service to the given subscription. """ configmap_template = model.v1.ConfigMap( metadata=_s4_customer_metadata(model), ) name = configmap_name(details.subscription_id) metadata = subscription_metadata(details) public_host = configmap_public_host(details.subscription_id, deploy_config.domain) private_host = deploy_config.private_host Message.log( event=u"convergence-service:key-notification", key_id=deploy_config.s3_access_key_id, secret_key_hash=sha256(deploy_config.s3_secret_key).hexdigest().decode("ascii"), ) configuration = marshal_tahoe_configuration( introducer_pem=details.introducer_node_pem, storage_pem=details.server_node_pem, storage_privkey=details.oldsecrets["server_node_privkey"], introducer_port=details.introducer_port_number, storageserver_port=details.storage_port_number, bucket_name=details.bucketname, key_prefix=details.key_prefix, publichost=public_host, privatehost=private_host, introducer_furl=details.external_introducer_furl, # Note we get the keys from the deployment configuration passed to us. # oldsecrets has this junk in it, too, but it's unreliable and we want # to get rid of that storage location for the information. Our # deployment configuration is more reliable anyway and lets us change # values more easily (in one place instead of N). s3_access_key_id=deploy_config.s3_access_key_id, s3_secret_key=deploy_config.s3_secret_key, log_gatherer_furl=deploy_config.log_gatherer_furl, stats_gatherer_furl=deploy_config.stats_gatherer_furl, ) return configmap_template.transform( [u"metadata", u"namespace"], deploy_config.kubernetes_namespace, # Assign it a unique identifier the deployment can use to refer to it. [u"metadata", u"name"], name, # Some other metadata to make inspecting this stuff a little easier. *metadata ).transform( # Dump the actual Tahoe-LAFS configuration into it. [u"data", u"introducer.json"], dumps({"introducer": configuration["introducer"]}).decode("ascii"), [u"data", u"storage.json"], dumps({"storage": configuration["storage"]}).decode("ascii"), )
def g(): Message.log(message_type="a") with start_action(action_type="confounding-factor"): Message.log(message_type="b") yield None with start_action(action_type="double-confounding-factor"): yield None Message.log(message_type="c") Message.log(message_type="d") Message.log(message_type="e")
def stop(self): """ Stop the scenario from being maintained by stopping all the loops that may be executing. :return Deferred[Optional[Dict[unicode, Any]]]: Scenario metrics. """ self.is_started = False if self.monitor_loop.running: self.monitor_loop.stop() if self.loop.running: self.loop.stop() outstanding_requests = self.rate_measurer.outstanding() if outstanding_requests > 0: msg = ( "There are {num_requests} outstanding requests. " "Waiting {num_seconds} seconds for them to complete." ).format( num_requests=outstanding_requests, num_seconds=self.timeout ) Message.log(key='outstanding_requests', value=msg) with start_action( action_type=u'flocker:benchmark:scenario:stop', scenario='request_load' ): def no_outstanding_requests(): return self.rate_measurer.outstanding() == 0 scenario_stopped = loop_until(self.reactor, no_outstanding_requests, repeat(1)) timeout(self.reactor, scenario_stopped, self.timeout) scenario = DeferredContext(scenario_stopped) def handle_timeout(failure): failure.trap(CancelledError) msg = ( "Force stopping the scenario. " "There are {num_requests} outstanding requests" ).format( num_requests=outstanding_requests ) Message.log(key='force_stop_request', value=msg) scenario.addErrback(handle_timeout) def return_metrics(_ignore): return self.rate_measurer.get_metrics() scenario.addCallback(return_metrics) return scenario.addActionFinish()
def deploy(self): """ Deploy the new configuration: create the requested containers and dataset in the cluster nodes. :return Deferred: that will fire once the request to create all the containers and datasets has been sent. """ Message.log(action="Listing current nodes") d = self.create_datasets_and_containers() return d
def got_welcome(ignored): # we're connected to the wormhole server; send our introduction # message intro = {u"abilities": {u"server-v1": {}}} Message.log(server_intro=intro) wormhole.send_message(json.dumps(intro)) # await the client's introduction d = DeferredContext(wormhole.get_message()) d.addCallback(json.loads) return d.result
def log_info(**kwargs): """ Simple logging wrapper around Eliot messages. This fills in the message type and passes all other arguments on to ``Message.log``. """ Message.log( message_type=_compute_message_type(stack()[1]), **kwargs )
def got_intro(client_intro): Message.log(client_intro=client_intro) if u'abilities' not in client_intro: raise Exception("Expected 'abilities' in client introduction") if u'client-v1' not in client_intro['abilities']: raise Exception("Expected 'client-v1' in client abilities") # a correctly-versioned client has opened a wormhole to us; # give them the configuration JSON Message.log(event=u"send-configuration") wormhole.send_message(json.dumps(configuration))
def check_service(self, database, config, subscriptions, k8s_state, aws): expected = new_service(config.kubernetes_namespace, self.kube_model) actual = k8s_state.services.item_by_name(expected.metadata.name) # Don't actually care about the status. That belongs to the server # anyway. tweaked = actual.set(u"status", None) assert_that( tweaked, GoodEquals(expected), ) Message.log(check_service=thaw(expected))
def do_we_have_enough_containers(containers): msg = ( "Waiting for the containers to be ready..." "Created {current_containers} of {total_containers}" ).format( current_containers=len(containers), total_containers=number_of_containers, ) Message.log(action=msg) return (len(containers) >= number_of_containers)
def predicate(): stack_report = get_stack_report(stack_id, aws_config) current_status = stack_report['StackStatus'] Message.log( function='wait_for_stack_status', stack_id=stack_id, target_status=target_status, current_status=current_status ) if current_status == target_status: return stack_report
def do_we_have_enough_datasets(datasets): created_datasets = len(datasets) - self._initial_num_datasets msg = ( "Waiting for the datasets to be ready..." "Created {current_datasets} of {datasets_to_create} " "(Total = {total_datasets})" ).format( current_datasets=created_datasets, datasets_to_create=number_of_datasets, total_datasets=len(datasets) ) Message.log(action=msg) return created_datasets >= number_of_datasets
def handle_create_error(failure, name): # XXX This could be dangerous... What about a pre-existing # node with the same name (or even multiple nodes if the name # does not have to be unique)? Message.log( message_type="flocker:provision:libcloud:create_node:failed", node_name=name, ) write_failure(failure) d = self._async_cleanup_node_named(reactor, name) d.addCallback(lambda _: failure) return d
def do_we_have_enough_datasets(datasets): msg = ( "Waiting for the datasets to be ready..." "Created {current_datasets} of {total_datasets}" ).format( current_datasets=len(datasets), total_datasets=number_of_datasets, ) Message.log(action=msg) return (len(datasets) >= number_of_datasets)
def do_we_have_enough_containers(containers): created_containers = len(containers) - self._initial_num_containers msg = ( "Waiting for the containers to be ready..." "Created {current_containers} of {containers_to_create} " "(Total = {total_containers})" ).format( current_containers=created_containers, containers_to_create=number_of_containers, total_containers=len(containers), ) Message.log(action=msg) return created_containers >= number_of_containers
def check_state(node): if node is not None: if node.state in target_states: Message.log( message_type=( u"flocker:provision:libcloud:node_in_state" ), name=node.name, id=node.id, state=node.state, ) return True return False
def test_generator_and_non_generator(self, logger): @eliot_friendly_generator_function def g(): Message.log(message_type=u"a") yield with start_action(action_type=u"action-a"): Message.log(message_type=u"b") yield Message.log(message_type=u"c") Message.log(message_type=u"d") yield with start_action(action_type=u"the-action"): generator = g() next(generator) Message.log(message_type=u"0") next(generator) Message.log(message_type=u"1") next(generator) Message.log(message_type=u"2") self.assertRaises(StopIteration, lambda: next(generator)) assert_expected_action_tree( self, logger, u"the-action", [ u"a", u"yielded", u"0", { u"action-a": [ u"b", u"yielded", u"c", ], }, u"1", u"d", u"yielded", u"2", ], )
def _collect_dark_curr_settings(self, settings): with start_action(action_type="dark_current_settings"): quit = False use_dark_curr = self.ui.dark_curr_checkbox.isChecked() Message.log(checked=use_dark_curr) if not use_dark_curr: Message.log(quit=quit) return settings, quit try: dark_curr_par = float(self.ui.dark_curr_par.text()) dark_curr_perp = float(self.ui.dark_curr_perp.text()) dark_curr_ref = float(self.ui.dark_curr_ref.text()) settings.dark_curr_par = dark_curr_par settings.dark_curr_perp = dark_curr_perp settings.dark_curr_ref = dark_curr_ref except ValueError: quit = True Message.log(quit=quit) return settings, quit
def _perform_upload(self, snapshot): """ Do the actual work of performing an upload. Failures are propagated to the caller. """ upload_started_at = time.time() Message.log(message_type="uploading") self._status.upload_started(snapshot.relpath) remote_snapshot = yield write_snapshot_to_tahoe( snapshot, self._config.author, self._tahoe_client, ) Message.log(remote_snapshot=remote_snapshot.relpath) snapshot.remote_snapshot = remote_snapshot yield self._config.store_uploaded_snapshot( remote_snapshot.relpath, remote_snapshot, upload_started_at, ) Message.log(message_type="upload_completed") returnValue(remote_snapshot)
def store_local_snapshot(self, path): Message.log( message_type=u"memory-snapshot-creator:store-local-snapshot", path=path.path, ) self.processed.append(path)
def processEnded(self, reason): with self._action.context(): Message.log(message_type=u"process-ended") self.exited.callback(None)
def test_returns_fired_deferred(self): Message.log(hello="world") return succeed(None)
def stop(self): """ Stop the scenario from being maintained by stopping all the loops that may be executing. :return Deferred[Optional[Dict[unicode, Any]]]: Scenario metrics. """ self.is_started = False if self.monitor_loop.running: self.monitor_loop.stop() if self.loop.running: self.loop.stop() outstanding_requests = self.rate_measurer.outstanding() if outstanding_requests > 0: msg = ( "There are {num_requests} outstanding requests. " "Waiting {num_seconds} seconds for them to complete." ).format( num_requests=outstanding_requests, num_seconds=self.timeout ) Message.log(key='outstanding_requests', value=msg) with start_action( action_type=u'flocker:benchmark:scenario:stop', scenario='request_load' ): def no_outstanding_requests(): return self.rate_measurer.outstanding() == 0 scenario_stopped = loop_until(self.reactor, no_outstanding_requests, repeat(1)) timeout(self.reactor, scenario_stopped, self.timeout) scenario = DeferredContext(scenario_stopped) def handle_timeout(failure): failure.trap(CancelledError) msg = ( "Force stopping the scenario. " "There are {num_requests} outstanding requests" ).format( num_requests=outstanding_requests ) Message.log(key='force_stop_request', value=msg) scenario.addErrback(handle_timeout) def scenario_cleanup(ignored): """ Calls the scenario cleanup, and wraps it inside an eliot start action, so we can see the logs if something goes wrong within the cleanup :return Deferred: that will fire once the cleanup has been completed """ with start_action( action_type=u'flocker:benchmark:scenario:cleanup', scenario='request_load' ): return self.request.run_cleanup() scenario.addBoth(scenario_cleanup) def return_metrics(_ignore): return self.rate_measurer.get_metrics() scenario.addCallback(return_metrics) return scenario.addActionFinish()
def instance_error(failure): Message.log( message_type="flocker:provision:aws:async_get_node:failed") instance.terminate() write_failure(failure) return failure
# 크롤링 대상 URL 리스트 PAGE_URL_LIST = [ 'https://eliot.readthedocs.io/en/1.0.0/', 'https://eliot.readthedocs.io/en/1.0.0/generating/index.html', 'https://example.com/notfound.html', ] def fetch_pages(): """페이지의 내용을 추출합니다.""" # 어떤 처리의 로그인지는 action_type으로 지정 with start_action(action_type="fetch_pages"): page_contents = {} for page_url in PAGE_URL_LIST: # 어떤 처리의 로그인지 action_type으로 출력 with start_action(action_type="download", url=page_url): try: r = requests.get(page_url, timeout=30) r.raise_for_status() except requests.exceptions.RequestException as e: write_traceback() # 예외가 발생하면 트레이스백 출력 continue page_contents[page_url] = r.text return page_contents if __name__ == '__main__': page_contents = fetch_pages() with open('page_contents.json', 'w') as f_page_contents: json.dump(page_contents, f_page_contents, ensure_ascii=False) # 단순하게 로그 메시지만 출력할 수도 있음 Message.log(message_type="info", msg="데이터를 저장했습니다.")
async def connect(self): self.cl = cl = make_client() await login(cl) asyncio.create_task(cl.sync_forever(timeout=30000, full_state=True)) await after_first_sync(cl) Message.log(transport="matrix", state="ready")
def _finish(result, render, request): """ Try to finish rendering the response to a request. This implements extra convenience functionality not provided by Twisted Web. Various resources in Tahoe-LAFS made use of this functionality when it was provided by Nevow. Rather than making that application code do the more tedious thing itself, we duplicate the functionality here. :param result: Something returned by a render method which we can turn into a response. :param render: The original render method which produced the result. :param request: The request being responded to. :return: ``None`` """ if isinstance(result, Failure): if result.check(CancelledError): return Message.log( message_type=u"allmydata:web:common-render:failure", message=result.getErrorMessage(), ) _finish( _renderHTTP_exception(request, result), render, request, ) elif IResource.providedBy(result): # If result is also using @render_exception then we don't want to # double-apply the logic. This leads to an attempt to double-finish # the request. If it isn't using @render_exception then you should # fix it so it is. Message.log( message_type=u"allmydata:web:common-render:resource", resource=fullyQualifiedName(type(result)), ) result.render(request) elif isinstance(result, str): Message.log(message_type=u"allmydata:web:common-render:unicode", ) request.write(result.encode("utf-8")) request.finish() elif isinstance(result, bytes): Message.log(message_type=u"allmydata:web:common-render:bytes", ) request.write(result) request.finish() elif isinstance(result, DecodedURL): Message.log(message_type=u"allmydata:web:common-render:DecodedURL", ) _finish(redirectTo(result.to_text().encode("utf-8"), request), render, request) elif result is None: Message.log(message_type=u"allmydata:web:common-render:None", ) request.finish() elif result == NOT_DONE_YET: Message.log(message_type=u"allmydata:web:common-render:NOT_DONE_YET", ) pass else: Message.log(message_type=u"allmydata:web:common-render:unknown", ) log.err( "Request for {!r} handled by {!r} returned unusable {!r}".format( request.uri, fullyQualifiedName(render), result, )) request.setResponseCode(http.INTERNAL_SERVER_ERROR) _finish(b"Internal Server Error", render, request)
async def connect(self): Message.log(transport="logging dummy", state="ready")
def set_pending_delay(self, delay): Message.log(message_type=u"watchdog:inotify:set-pending-delay", delay=delay) assert self._state != STARTED self._pending_delay = delay
async def disconnect(self): Message.log(msg="logging dummy transport is going to sleep")
def mark_overwrite(self, relpath, mtime, staged_content, prior_pathstate): """ This snapshot is an overwrite. Move it from the staging area over top of the existing file (if any) in the magic-folder. :param FilePath staged_content: a local path to the downloaded content. :return PathState: The path state of the file that was written. """ # Could be items associated with the action but then we have to define # an action type to teach Eliot how to serialize the remote snapshot # and the FilePath. Probably *should* do that but just doing this for # now... Maybe it should be /easier/ to do that? Message.log( message_type=u"downloader:filesystem:mark-overwrite", content_relpath=relpath, staged_content_path=staged_content.path, ) local_path = self.magic_path.preauthChild(relpath) tmp = None if local_path.exists(): # As long as the poller is running in-process, in the reactor, # it won't see this temporary file. # https://github.com/LeastAuthority/magic-folder/pull/451#discussion_r660885345 tmp = local_path.temporarySibling(".snaptmp") local_path.moveTo(tmp) Message.log( message_type= u"downloader:filesystem:mark-overwrite:set-aside-existing", source_path=local_path.path, target_path=tmp.path, ) os.utime(staged_content.path, (mtime, mtime)) # We want to get the path state of the file we are about to write. # Ideally, we would get this from the staged file. However, depending # on the operating system, the ctime can change when we rename. # Instead, we get the path state before and after the move, and use # the latter if the mtime and size match. staged_path_state = get_pathinfo(staged_content).state with start_action( action_type=u"downloader:filesystem:mark-overwrite:emplace", content_final_path=local_path.path, ): if not local_path.parent().exists(): local_path.parent().makedirs() else: if not local_path.parent().isdir(): # XXX this probably needs a better answer but for # now at least we can signal a problem. raise RuntimeError( "Wanted to write a subdir of {} but it's not a directory" .format(local_path.parent().path, )) staged_content.moveTo(local_path) path_state = get_pathinfo(local_path).state if (staged_path_state.mtime_ns != path_state.mtime_ns or staged_path_state.size != path_state.size or staged_path_state.ctime_ns > path_state.ctime_ns): # The path got changed after we moved it, return the staged # path_state so it is detected as a change. path_state = staged_path_state if tmp is not None: with start_action( action_type= u"downloader:filesystem:mark-overwrite:dispose-existing", ): # prior_pathstate comes from inside the state-machine # _immediately_ before we conclude "there's no # download conflict" (see _check_local_update in # Magicfile) .. here, we double-check that the # modification time and size match what we checked # against in _check_local_update. # if prior_pathstate is None, there was no file at all # before this function ran .. so we must preserve the # tmpfile # # otherwise, the pathstate of the tmp file must match # what was there before tmp.restat() modtime = seconds_to_ns(tmp.getModificationTime()) size = tmp.getsize() if prior_pathstate is None or \ prior_pathstate.mtime_ns != modtime or \ prior_pathstate.size != size: # "something" has written to the file (or the # tmpfile, after it was moved) causing mtime or # size to be differt; OR prior_pathstate points at # a directory (or some other reason causing # get_fileinfo() to return a None pathstate # above). Thus, we keep the file as a backup # the MagicFile statemachine knows about this # exception and processes it separately -- turning # it into a conflict raise BackupRetainedError(tmp, ) else: # all good, the times + size match tmp.remove() return path_state
def visited(self, people): Message.log(message_type="visited", people=people, place=self.name) for thing in self.contained: thing.visited(people)
def errReceived(self, data): with self._action.context(): Message.log(message_type=u"err-received", data=data) sys.stdout.write(data)
async def disconnect(self): Message.log(transport="matrix", state="shutdown")
qlog.begin('Cascade, Version {}'.format(__version__)) #--------------------------------------------------------------------- # Modules which use quicklog cannot be imported until the logger # has been initialized, so they are imported here. #--------------------------------------------------------------------- from cascade.main import main from cascade.util_eliot import rotating_logfile #--------------------------------------------------------------------- # Start eliot logging eliot_handler = rotating_logfile(os.path.join('log', 'cascade.eliot.log'), auto=False) Message.log(message_type="startup", command_line=' '.join(sys.argv), arguments=arguments, version=__version__) if arguments['-w']: print('Privacy warning enabled.') try: main(arguments) except FatalUserError as e: # "Normal" user errors which cause application to halt qlog.error(str(e)) except Exception as e: # Unexpected exceptions (i.e. application crashes) qlog.fatal_exception(e) if arguments['-g']: #Launch postmortem debugger
def g(): Message.log(message_type=u"hello") yield expected Message.log(message_type=u"goodbye")
def _pytest_collection_modifyitems(session, config, items): """Report allowed recids and jsonpaths to master and await start. :type session: :py:class:_pytest.main.Session :type config: :py:class:_pytest.config.Config :type items: list """ redis_worker = config.option.redis_worker unique_functions_found = set((item.function for item in items)) if not unique_functions_found: raise AssertionError( "No check functions were found." " Scroll up for exceptions that may have prevented collection!") elif not len(unique_functions_found) == 1: raise AssertionError( "We support one check function per file. Found {0} instead.". format(len(unique_functions_found))) item = items[0] # Set allowed_paths and allowed_recids if hasattr(item, 'cls'): if hasattr(item.cls, 'allowed_paths'): allowed_paths = item.cls.allowed_paths( config.option.invenio_rule.arguments) else: allowed_paths = set() if hasattr(item.cls, 'allowed_recids'): allowed_recids = item.cls.allowed_recids( config.option.invenio_rule.arguments, batch_recids(session), all_recids(session), search(session)) else: allowed_recids = batch_recids(session) if allowed_recids - all_recids(session): raise AssertionError('Check requested recids that are not in the' ' database!') # TODO `allowed_paths` must return jsonpointers (IETF RFC 6901) redis_worker.allowed_paths = allowed_paths redis_worker.allowed_recids = allowed_recids def worker_conflicts_with_currently_running(worker): foreign_running_workers = get_workers_with_unprocessed_results() blockers = set() for foreign in foreign_running_workers: if not _are_compatible(worker, foreign): blockers.add(foreign) return blockers redis_worker.status = StatusWorker.ready # XXX unused? with start_action(action_type='checking for conflicting running workers'): redis_worker.lock.get() try: blockers = worker_conflicts_with_currently_running(redis_worker) if blockers: Message.log(message_type='found conflicting workers', value=str(blockers)) print 'CONFLICT {} {}'.format(redis_worker.task_id, blockers) redis_worker.retry_after_ids = {bl.task_id for bl in blockers} del items[:] else: print 'RESUMING ' + str(redis_worker.task_id) redis_worker.status = StatusWorker.running finally: redis_worker.lock.release()
def honeymoon(family, destination): Message.log(message_type="honeymoon", people=family) destination.visited(family)
def handle_timeout(failure): failure.trap(CancelledError) msg = ("Force stopping the scenario. " "There are {num_requests} outstanding requests").format( num_requests=outstanding_requests) Message.log(key='force_stop_request', value=msg)
def g(): Message.log(message_type=u"a") yield Message.log(message_type=u"b")
def test_returns_none(self): Message.log(hello="world")
def g(): Message.log(message_type=u"a") yield succeed(None) Message.log(message_type=u"b")
def got_code(code): # We don't currently expire these at all. expires = _NoExpiration() Message.log(event=u"code-allocated") return _WormholeClaim(code, expires)
def g(): Message.log(message_type=u"a") yield waiting Message.log(message_type=u"b")
def log_message(message: str, **kwargs): """Log the message in the current action context.""" Message.log(message_type=message, **kwargs)
def _thread(self): try: _assert(self._filter is not None, "no watch set") # To call Twisted or Tahoe APIs, use reactor.callFromThread as described in # <http://twistedmatrix.com/documents/current/core/howto/threading.html>. fni = FileNotifyInformation() while True: self._state = STARTED action = start_action( action_type=u"read-changes", directory=self._path.path, recursive=self._recursive, filter=self._filter, ) try: with action: fni.read_changes(self._hDirectory, self._recursive, self._filter) except WindowsError as e: self._state = STOPPING if self._check_stop(): return for info in fni: path = self._path.preauthChild(info.filename) # FilePath with Unicode path if info.action == FILE_ACTION_MODIFIED and path.isdir(): Message.log( message_type=u"filtering-out", info=repr(info), ) continue else: Message.log( message_type=u"processing", info=repr(info), ) #mask = _action_to_inotify_mask.get(info.action, IN_CHANGED) @log_call( action_type=MAYBE_NOTIFY.action_type, include_args=[], include_result=False, ) def _do_pending_calls(): event_mask = IN_CHANGED self._pending_call = None for path1 in self._pending: if self._callbacks: for cb in self._callbacks: try: with CALLBACK(inotify_events=event_mask): cb(None, path1, event_mask) except Exception as e2: log.err(e2) self._pending = set() def _maybe_notify(path2): if path2 not in self._pending: self._pending.add(path2) if self._state not in [STOPPING, STOPPED]: _do_pending_calls() # if self._pending_call is None and self._state not in [STOPPING, STOPPED]: # self._pending_call = reactor.callLater(self._pending_delay, _do_pending_calls) reactor.callFromThread(_maybe_notify, path) if self._check_stop(): return except Exception as e: log.err(e) self._state = STOPPED raise