def make_node(node):
public_address = _filter_ipv4(node.public_ips)[0]
if isinstance(public_address, unicode):
public_address = public_address.encode("ascii")
if self._use_private_addresses:
private_address = _filter_ipv4(node.private_ips)[0]
else:
private_address = None
if isinstance(private_address, unicode):
private_address = private_address.encode("ascii")
Message.log(
message_type="flocker:provision:libcloud:node_created",
name=node.name,
id=node.id,
public_address=public_address,
private_address=private_address,
)
return LibcloudNode(
provisioner=self,
node=node, address=public_address,
private_address=private_address,
distribution=distribution)
def _pytest_collection_modifyitems(task_arguments, worker, items, invenio_rule,
option, batch_recids_, all_recids_):
"""Report allowed recids and jsonpaths to master and await start.
:type config: :py:class:_pytest.config.Config
:type items: list
"""
_ensure_only_one_test_function_exists_in_check(items)
item = items[0]
# TODO: There could be a flag that declares whether there are side-effects
# in the DB, instead of checking for use of record_fetching_fixturenames
# Inform the worker about allowed paths and recids
if _test_func_uses_record_related_fixtures(item):
worker.allowed_paths, worker.allowed_recids = \
_get_restrictions_from_check_class(item, task_arguments,
batch_recids_, all_recids_)
else:
worker.allowed_paths, worker.allowed_recids = None, None
# Check if there are conflicts with other workers
worker.status = StatusWorker.ready
with worker.lock():
blockers = _worker_conflicts_with_currently_running(worker)
worker.retry_after_ids = {bl.uuid for bl in blockers}
if blockers:
Message.log(message_type='detected conflicting workers', value=str(blockers))
del items[:]
else:
option.invenio_reporters = load_reporters(invenio_rule, option.invenio_execution)
worker.status = StatusWorker.running
def handlePLAINTEXTServer(self, header):
"""
Parse a complete HTTP-like Foolscap negotiation request and begin proxying
to a destination selected based on the extract TubID.
"""
# the client sends us a GET message
lines = header.split("\r\n")
if not lines[0].startswith("GET "):
raise BananaError("not right")
command, url, version = lines[0].split()
if not url.startswith("/id/"):
# probably a web browser
raise BananaError("not right")
targetTubID = url[4:]
Message.log(event_type=u"handlePLAINTEXTServer", tub_id=targetTubID)
if targetTubID == "":
# they're asking for an old UnauthenticatedTub. Refuse.
raise NegotiationError("secure Tubs require encryption")
if isSubstring("Upgrade: TLS/1.0\r\n", header):
wantEncrypted = True
else:
wantEncrypted = False
Message.log(event_type=u"handlePLAINTEXTServer", want_encrypted=wantEncrypted)
self._handleTubRequest(header, targetTubID)
def stop(self):
"""
Stop the scenario from being maintained by stopping all the
loops that may be executing.
:return Deferred[Optional[Dict[unicode, Any]]]: Scenario metrics.
"""
self.is_started = False
if self.monitor_loop.running:
self.monitor_loop.stop()
if self.loop.running:
self.loop.stop()
outstanding_requests = self.rate_measurer.outstanding()
if outstanding_requests > 0:
msg = (
"There are {num_requests} outstanding requests. " "Waiting {num_seconds} seconds for them to complete."
).format(num_requests=outstanding_requests, num_seconds=self.timeout)
Message.log(key="outstanding_requests", value=msg)
with start_action(action_type=u"flocker:benchmark:scenario:stop", scenario="request_load"):
def no_outstanding_requests():
return self.rate_measurer.outstanding() == 0
scenario_stopped = loop_until(self.reactor, no_outstanding_requests, repeat(1))
timeout(self.reactor, scenario_stopped, self.timeout)
scenario = DeferredContext(scenario_stopped)
def handle_timeout(failure):
failure.trap(CancelledError)
msg = ("Force stopping the scenario. " "There are {num_requests} outstanding requests").format(
num_requests=outstanding_requests
)
Message.log(key="force_stop_request", value=msg)
scenario.addErrback(handle_timeout)
def scenario_cleanup(ignored):
"""
Calls the scenario cleanup, and wraps it inside an eliot
start action, so we can see the logs if something goes
wrong within the cleanup
:return Deferred: that will fire once the cleanup has been
completed
"""
with start_action(action_type=u"flocker:benchmark:scenario:cleanup", scenario="request_load"):
return self.request.run_cleanup()
scenario.addBoth(scenario_cleanup)
def return_metrics(_ignore):
return self.rate_measurer.get_metrics()
scenario.addCallback(return_metrics)
return scenario.addActionFinish()
def g(recurse):
with start_action(action_type="a-recurse={}".format(recurse)):
Message.log(message_type="m-recurse={}".format(recurse))
if recurse:
set(g(False))
else:
yield
def log_progress():
Message.log(
message_type="flocker.benchmark.container_setup:progress",
container_count=self.container_count,
error_count=self.error_count,
total_containers=total,
)
def instance_error(failure):
Message.log(
message_type="flocker:provision:aws:async_get_node:failed"
)
instance.terminate()
write_failure(failure)
return failure
def log_totals(result):
Message.log(
message_type="flocker.benchmark.container_setup:finish",
container_count=self.container_count,
error_count=self.error_count,
)
return result
def _build_config(self, ignored):
"""
Build a Flocker deployment configuration for the given cluster
and parameters.
The configuration consists of identically configured applications
(containers) uniformly spread over all cluster nodes.
:return dict: containing the json we need to send to compose to
create the datasets and containers we want.
"""
Message.log(action="Building config")
application_root = {}
applications = {}
application_root["version"] = 1
application_root["applications"] = applications
for node in self.nodes:
for i in range(self.per_node):
name = "app_%s_%d" % (node.public_address, i)
applications[name] = deepcopy(self.application_template)
deployment_root = {}
nodes = {}
deployment_root["nodes"] = nodes
deployment_root["version"] = 1
for node in self.nodes:
addr = "%s" % node.public_address
nodes[addr] = []
for i in range(self.per_node):
name = "app_%s_%d" % (node.public_address, i)
nodes[addr].append(name)
return {"applications": application_root,
"deployment": deployment_root}
def _converge_route53_customer(actual, config, subscriptions, k8s, aws):
"""
Converge on the desired Route53 state relating to individual S4
subscriptions.
Specifically, make sure the per-subscription domain names that grants
access to individual subscription's introducer and storage servers exist
and point at the right thing. Also clean up the names for any cancelled
subscriptions.
"""
changes = _compute_changes(
actual.subscriptions,
_ChangeableZone(
zone=actual.zone.zone,
rrsets=actual.zone.rrsets,
domain=config.domain,
),
)
# XXX Probably would be nice to group changes. Also, issue changes insert
# of delete/create. Some structured objects would make that easier.
route53 = aws.get_route53_client()
def delete(sid):
return delete_route53_rrsets(route53, actual.zone.zone, [sid])
def create(subscription):
return create_route53_rrsets(route53, actual.zone.zone, [subscription])
deletes = list(partial(delete, sid) for sid in changes.delete)
creates = list(partial(create, s) for s in changes.create)
Message.log(
event=u"convergence-service:route53-customer",
create=list(subscription.subscription_id for subscription in changes.create),
delete=list(changes.delete),
)
return deletes + creates
def send_delay_to_fsm(sleep):
Message.log(
message_type=u'flocker:node:_loop:CONVERGE:delay',
log_level=u'INFO',
message=u'Delaying until next convergence loop.',
delay=sleep.delay_seconds
)
return self.fsm.receive(sleep)
def _run_task(rule_name, master_id):
del Logger._destinations._destinations[:]
to_file(open(os.path.join(eliot_log_path, master_id), "ab"))
with start_task(action_type="invenio_checker:supervisor:_run_task",
master_id=master_id) as eliot_task:
from .models import CheckerRule
# cleanup_failed_runs()
redis_master = None
def cleanup_session():
print 'Cleaning up'
if redis_master is not None:
redis_master.zap()
def sigint_hook(rcv_signal, frame):
cleanup_session()
def except_hook(type_, value, tback):
cleanup_session()
reraise(type_, value, tback)
signal.signal(signal.SIGINT, sigint_hook)
signal.signal(signal.SIGTERM, sigint_hook)
sys.excepthook = except_hook
with start_action(action_type='create master'):
eliot_task_id = eliot_task.serialize_task_id()
redis_master = RedisMaster(master_id, eliot_task_id, rule_name)
with start_action(action_type='create subtasks'):
rules = CheckerRule.from_ids((rule_name,))
bundles = rules_to_bundles(rules, redis_master.all_recids)
subtasks = []
errback = handle_error.s()
for rule, rule_chunks in bundles.iteritems():
for chunk in rule_chunks:
task_id = uuid()
redis_master.workers_append(task_id)
eliot_task_id = eliot_task.serialize_task_id()
RedisWorker(task_id, eliot_task_id, chunk)
subtasks.append(run_test.subtask(args=(rule.filepath,
redis_master.master_id,
task_id),
task_id=task_id,
link_error=[errback]))
Message.log(message_type='registered subtasks', value=str(redis_master.workers))
with start_action(action_type='run chord'):
redis_master.status = StatusMaster.running
header = subtasks
callback = handle_results.subtask(link_error=[handle_errors.s(redis_master.master_id)])
my_chord = chord(header)
result = my_chord(callback)
redis_master.status = StatusMaster.running
def handle_timeout(failure):
failure.trap(CancelledError)
msg = (
"Force stopping the scenario. "
"There are {num_requests} outstanding requests"
).format(
num_requests=outstanding_requests
)
Message.log(key='force_stop_request', value=msg)
def test_returns_unfired_deferred(self):
Message.log(hello="world")
# @eliot_logged_test automatically gives us an action context but it's
# still our responsibility to maintain it across stack-busting
# operations.
d = DeferredContext(deferLater(reactor, 0.0, lambda: None))
d.addCallback(lambda ignored: Message.log(goodbye="world"))
# We didn't start an action. We're not finishing an action.
return d.result
def handle_failure(failure):
Message.log(
message_type=(
u"flocker:provision:libcloud:retry_exception:got_exception"
),
)
write_failure(failure)
saved_failure[0] = failure
return False
def create_configuration(deploy_config, details, model):
"""
Create the Kubernetes configuration resource necessary to provide
service to the given subscription.
"""
configmap_template = model.v1.ConfigMap(
metadata=_s4_customer_metadata(model),
)
name = configmap_name(details.subscription_id)
metadata = subscription_metadata(details)
public_host = configmap_public_host(details.subscription_id, deploy_config.domain)
private_host = deploy_config.private_host
Message.log(
event=u"convergence-service:key-notification",
key_id=deploy_config.s3_access_key_id,
secret_key_hash=sha256(deploy_config.s3_secret_key).hexdigest().decode("ascii"),
)
configuration = marshal_tahoe_configuration(
introducer_pem=details.introducer_node_pem,
storage_pem=details.server_node_pem,
storage_privkey=details.oldsecrets["server_node_privkey"],
introducer_port=details.introducer_port_number,
storageserver_port=details.storage_port_number,
bucket_name=details.bucketname,
key_prefix=details.key_prefix,
publichost=public_host,
privatehost=private_host,
introducer_furl=details.external_introducer_furl,
# Note we get the keys from the deployment configuration passed to us.
# oldsecrets has this junk in it, too, but it's unreliable and we want
# to get rid of that storage location for the information. Our
# deployment configuration is more reliable anyway and lets us change
# values more easily (in one place instead of N).
s3_access_key_id=deploy_config.s3_access_key_id,
s3_secret_key=deploy_config.s3_secret_key,
log_gatherer_furl=deploy_config.log_gatherer_furl,
stats_gatherer_furl=deploy_config.stats_gatherer_furl,
)
return configmap_template.transform(
[u"metadata", u"namespace"], deploy_config.kubernetes_namespace,
# Assign it a unique identifier the deployment can use to refer to it.
[u"metadata", u"name"], name,
# Some other metadata to make inspecting this stuff a little easier.
*metadata
).transform(
# Dump the actual Tahoe-LAFS configuration into it.
[u"data", u"introducer.json"],
dumps({"introducer": configuration["introducer"]}).decode("ascii"),
[u"data", u"storage.json"],
dumps({"storage": configuration["storage"]}).decode("ascii"),
)
def g():
Message.log(message_type="a")
with start_action(action_type="confounding-factor"):
Message.log(message_type="b")
yield None
with start_action(action_type="double-confounding-factor"):
yield None
Message.log(message_type="c")
Message.log(message_type="d")
Message.log(message_type="e")
def stop(self):
"""
Stop the scenario from being maintained by stopping all the
loops that may be executing.
:return Deferred[Optional[Dict[unicode, Any]]]: Scenario metrics.
"""
self.is_started = False
if self.monitor_loop.running:
self.monitor_loop.stop()
if self.loop.running:
self.loop.stop()
outstanding_requests = self.rate_measurer.outstanding()
if outstanding_requests > 0:
msg = (
"There are {num_requests} outstanding requests. "
"Waiting {num_seconds} seconds for them to complete."
).format(
num_requests=outstanding_requests,
num_seconds=self.timeout
)
Message.log(key='outstanding_requests', value=msg)
with start_action(
action_type=u'flocker:benchmark:scenario:stop',
scenario='request_load'
):
def no_outstanding_requests():
return self.rate_measurer.outstanding() == 0
scenario_stopped = loop_until(self.reactor,
no_outstanding_requests,
repeat(1))
timeout(self.reactor, scenario_stopped, self.timeout)
scenario = DeferredContext(scenario_stopped)
def handle_timeout(failure):
failure.trap(CancelledError)
msg = (
"Force stopping the scenario. "
"There are {num_requests} outstanding requests"
).format(
num_requests=outstanding_requests
)
Message.log(key='force_stop_request', value=msg)
scenario.addErrback(handle_timeout)
def return_metrics(_ignore):
return self.rate_measurer.get_metrics()
scenario.addCallback(return_metrics)
return scenario.addActionFinish()
def deploy(self):
"""
Deploy the new configuration: create the requested containers
and dataset in the cluster nodes.
:return Deferred: that will fire once the request to create all
the containers and datasets has been sent.
"""
Message.log(action="Listing current nodes")
d = self.create_datasets_and_containers()
return d
def got_welcome(ignored):
# we're connected to the wormhole server; send our introduction
# message
intro = {u"abilities": {u"server-v1": {}}}
Message.log(server_intro=intro)
wormhole.send_message(json.dumps(intro))
# await the client's introduction
d = DeferredContext(wormhole.get_message())
d.addCallback(json.loads)
return d.result
def log_info(**kwargs):
"""
Simple logging wrapper around Eliot messages.
This fills in the message type and passes all other arguments on to
``Message.log``.
"""
Message.log(
message_type=_compute_message_type(stack()[1]),
**kwargs
)
def got_intro(client_intro):
Message.log(client_intro=client_intro)
if u'abilities' not in client_intro:
raise Exception("Expected 'abilities' in client introduction")
if u'client-v1' not in client_intro['abilities']:
raise Exception("Expected 'client-v1' in client abilities")
# a correctly-versioned client has opened a wormhole to us;
# give them the configuration JSON
Message.log(event=u"send-configuration")
wormhole.send_message(json.dumps(configuration))
def check_service(self, database, config, subscriptions, k8s_state, aws):
expected = new_service(config.kubernetes_namespace, self.kube_model)
actual = k8s_state.services.item_by_name(expected.metadata.name)
# Don't actually care about the status. That belongs to the server
# anyway.
tweaked = actual.set(u"status", None)
assert_that(
tweaked,
GoodEquals(expected),
)
Message.log(check_service=thaw(expected))
def do_we_have_enough_containers(containers):
msg = (
"Waiting for the containers to be ready..."
"Created {current_containers} of {total_containers}"
).format(
current_containers=len(containers),
total_containers=number_of_containers,
)
Message.log(action=msg)
return (len(containers) >= number_of_containers)
def predicate():
stack_report = get_stack_report(stack_id, aws_config)
current_status = stack_report['StackStatus']
Message.log(
function='wait_for_stack_status',
stack_id=stack_id,
target_status=target_status,
current_status=current_status
)
if current_status == target_status:
return stack_report
def do_we_have_enough_datasets(datasets):
created_datasets = len(datasets) - self._initial_num_datasets
msg = (
"Waiting for the datasets to be ready..."
"Created {current_datasets} of {datasets_to_create} "
"(Total = {total_datasets})"
).format(
current_datasets=created_datasets, datasets_to_create=number_of_datasets, total_datasets=len(datasets)
)
Message.log(action=msg)
return created_datasets >= number_of_datasets
def handle_create_error(failure, name):
# XXX This could be dangerous... What about a pre-existing
# node with the same name (or even multiple nodes if the name
# does not have to be unique)?
Message.log(
message_type="flocker:provision:libcloud:create_node:failed",
node_name=name,
)
write_failure(failure)
d = self._async_cleanup_node_named(reactor, name)
d.addCallback(lambda _: failure)
return d
def do_we_have_enough_datasets(datasets):
msg = (
"Waiting for the datasets to be ready..."
"Created {current_datasets} of {total_datasets}"
).format(
current_datasets=len(datasets),
total_datasets=number_of_datasets,
)
Message.log(action=msg)
return (len(datasets) >= number_of_datasets)
def do_we_have_enough_containers(containers):
created_containers = len(containers) - self._initial_num_containers
msg = (
"Waiting for the containers to be ready..."
"Created {current_containers} of {containers_to_create} "
"(Total = {total_containers})"
).format(
current_containers=created_containers,
containers_to_create=number_of_containers,
total_containers=len(containers),
)
Message.log(action=msg)
return created_containers >= number_of_containers
def check_state(node):
if node is not None:
if node.state in target_states:
Message.log(
message_type=(
u"flocker:provision:libcloud:node_in_state"
),
name=node.name,
id=node.id,
state=node.state,
)
return True
return False
def test_generator_and_non_generator(self, logger):
@eliot_friendly_generator_function
def g():
Message.log(message_type=u"a")
yield
with start_action(action_type=u"action-a"):
Message.log(message_type=u"b")
yield
Message.log(message_type=u"c")
Message.log(message_type=u"d")
yield
with start_action(action_type=u"the-action"):
generator = g()
next(generator)
Message.log(message_type=u"0")
next(generator)
Message.log(message_type=u"1")
next(generator)
Message.log(message_type=u"2")
self.assertRaises(StopIteration, lambda: next(generator))
assert_expected_action_tree(
self,
logger,
u"the-action",
[
u"a",
u"yielded",
u"0",
{
u"action-a": [
u"b",
u"yielded",
u"c",
],
},
u"1",
u"d",
u"yielded",
u"2",
],
)
def _collect_dark_curr_settings(self, settings):
with start_action(action_type="dark_current_settings"):
quit = False
use_dark_curr = self.ui.dark_curr_checkbox.isChecked()
Message.log(checked=use_dark_curr)
if not use_dark_curr:
Message.log(quit=quit)
return settings, quit
try:
dark_curr_par = float(self.ui.dark_curr_par.text())
dark_curr_perp = float(self.ui.dark_curr_perp.text())
dark_curr_ref = float(self.ui.dark_curr_ref.text())
settings.dark_curr_par = dark_curr_par
settings.dark_curr_perp = dark_curr_perp
settings.dark_curr_ref = dark_curr_ref
except ValueError:
quit = True
Message.log(quit=quit)
return settings, quit
def _perform_upload(self, snapshot):
"""
Do the actual work of performing an upload. Failures are
propagated to the caller.
"""
upload_started_at = time.time()
Message.log(message_type="uploading")
self._status.upload_started(snapshot.relpath)
remote_snapshot = yield write_snapshot_to_tahoe(
snapshot,
self._config.author,
self._tahoe_client,
)
Message.log(remote_snapshot=remote_snapshot.relpath)
snapshot.remote_snapshot = remote_snapshot
yield self._config.store_uploaded_snapshot(
remote_snapshot.relpath,
remote_snapshot,
upload_started_at,
)
Message.log(message_type="upload_completed")
returnValue(remote_snapshot)
def store_local_snapshot(self, path):
Message.log(
message_type=u"memory-snapshot-creator:store-local-snapshot",
path=path.path,
)
self.processed.append(path)
def processEnded(self, reason):
with self._action.context():
Message.log(message_type=u"process-ended")
self.exited.callback(None)
def test_returns_fired_deferred(self):
Message.log(hello="world")
return succeed(None)
def stop(self):
"""
Stop the scenario from being maintained by stopping all the
loops that may be executing.
:return Deferred[Optional[Dict[unicode, Any]]]: Scenario metrics.
"""
self.is_started = False
if self.monitor_loop.running:
self.monitor_loop.stop()
if self.loop.running:
self.loop.stop()
outstanding_requests = self.rate_measurer.outstanding()
if outstanding_requests > 0:
msg = (
"There are {num_requests} outstanding requests. "
"Waiting {num_seconds} seconds for them to complete."
).format(
num_requests=outstanding_requests,
num_seconds=self.timeout
)
Message.log(key='outstanding_requests', value=msg)
with start_action(
action_type=u'flocker:benchmark:scenario:stop',
scenario='request_load'
):
def no_outstanding_requests():
return self.rate_measurer.outstanding() == 0
scenario_stopped = loop_until(self.reactor,
no_outstanding_requests,
repeat(1))
timeout(self.reactor, scenario_stopped, self.timeout)
scenario = DeferredContext(scenario_stopped)
def handle_timeout(failure):
failure.trap(CancelledError)
msg = (
"Force stopping the scenario. "
"There are {num_requests} outstanding requests"
).format(
num_requests=outstanding_requests
)
Message.log(key='force_stop_request', value=msg)
scenario.addErrback(handle_timeout)
def scenario_cleanup(ignored):
"""
Calls the scenario cleanup, and wraps it inside an eliot
start action, so we can see the logs if something goes
wrong within the cleanup
:return Deferred: that will fire once the cleanup has been
completed
"""
with start_action(
action_type=u'flocker:benchmark:scenario:cleanup',
scenario='request_load'
):
return self.request.run_cleanup()
scenario.addBoth(scenario_cleanup)
def return_metrics(_ignore):
return self.rate_measurer.get_metrics()
scenario.addCallback(return_metrics)
return scenario.addActionFinish()
def instance_error(failure):
Message.log(
message_type="flocker:provision:aws:async_get_node:failed")
instance.terminate()
write_failure(failure)
return failure
# 크롤링 대상 URL 리스트
PAGE_URL_LIST = [
'https://eliot.readthedocs.io/en/1.0.0/',
'https://eliot.readthedocs.io/en/1.0.0/generating/index.html',
'https://example.com/notfound.html',
]
def fetch_pages():
"""페이지의 내용을 추출합니다."""
# 어떤 처리의 로그인지는 action_type으로 지정
with start_action(action_type="fetch_pages"):
page_contents = {}
for page_url in PAGE_URL_LIST:
# 어떤 처리의 로그인지 action_type으로 출력
with start_action(action_type="download", url=page_url):
try:
r = requests.get(page_url, timeout=30)
r.raise_for_status()
except requests.exceptions.RequestException as e:
write_traceback() # 예외가 발생하면 트레이스백 출력
continue
page_contents[page_url] = r.text
return page_contents
if __name__ == '__main__':
page_contents = fetch_pages()
with open('page_contents.json', 'w') as f_page_contents:
json.dump(page_contents, f_page_contents, ensure_ascii=False)
# 단순하게 로그 메시지만 출력할 수도 있음
Message.log(message_type="info", msg="데이터를 저장했습니다.")
async def connect(self):
self.cl = cl = make_client()
await login(cl)
asyncio.create_task(cl.sync_forever(timeout=30000, full_state=True))
await after_first_sync(cl)
Message.log(transport="matrix", state="ready")
def _finish(result, render, request):
"""
Try to finish rendering the response to a request.
This implements extra convenience functionality not provided by Twisted
Web. Various resources in Tahoe-LAFS made use of this functionality when
it was provided by Nevow. Rather than making that application code do the
more tedious thing itself, we duplicate the functionality here.
:param result: Something returned by a render method which we can turn
into a response.
:param render: The original render method which produced the result.
:param request: The request being responded to.
:return: ``None``
"""
if isinstance(result, Failure):
if result.check(CancelledError):
return
Message.log(
message_type=u"allmydata:web:common-render:failure",
message=result.getErrorMessage(),
)
_finish(
_renderHTTP_exception(request, result),
render,
request,
)
elif IResource.providedBy(result):
# If result is also using @render_exception then we don't want to
# double-apply the logic. This leads to an attempt to double-finish
# the request. If it isn't using @render_exception then you should
# fix it so it is.
Message.log(
message_type=u"allmydata:web:common-render:resource",
resource=fullyQualifiedName(type(result)),
)
result.render(request)
elif isinstance(result, str):
Message.log(message_type=u"allmydata:web:common-render:unicode", )
request.write(result.encode("utf-8"))
request.finish()
elif isinstance(result, bytes):
Message.log(message_type=u"allmydata:web:common-render:bytes", )
request.write(result)
request.finish()
elif isinstance(result, DecodedURL):
Message.log(message_type=u"allmydata:web:common-render:DecodedURL", )
_finish(redirectTo(result.to_text().encode("utf-8"), request), render,
request)
elif result is None:
Message.log(message_type=u"allmydata:web:common-render:None", )
request.finish()
elif result == NOT_DONE_YET:
Message.log(message_type=u"allmydata:web:common-render:NOT_DONE_YET", )
pass
else:
Message.log(message_type=u"allmydata:web:common-render:unknown", )
log.err(
"Request for {!r} handled by {!r} returned unusable {!r}".format(
request.uri,
fullyQualifiedName(render),
result,
))
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
_finish(b"Internal Server Error", render, request)
async def connect(self):
Message.log(transport="logging dummy", state="ready")
def set_pending_delay(self, delay):
Message.log(message_type=u"watchdog:inotify:set-pending-delay",
delay=delay)
assert self._state != STARTED
self._pending_delay = delay
async def disconnect(self):
Message.log(msg="logging dummy transport is going to sleep")
def mark_overwrite(self, relpath, mtime, staged_content, prior_pathstate):
"""
This snapshot is an overwrite. Move it from the staging area over
top of the existing file (if any) in the magic-folder.
:param FilePath staged_content: a local path to the downloaded
content.
:return PathState: The path state of the file that was written.
"""
# Could be items associated with the action but then we have to define
# an action type to teach Eliot how to serialize the remote snapshot
# and the FilePath. Probably *should* do that but just doing this for
# now... Maybe it should be /easier/ to do that?
Message.log(
message_type=u"downloader:filesystem:mark-overwrite",
content_relpath=relpath,
staged_content_path=staged_content.path,
)
local_path = self.magic_path.preauthChild(relpath)
tmp = None
if local_path.exists():
# As long as the poller is running in-process, in the reactor,
# it won't see this temporary file.
# https://github.com/LeastAuthority/magic-folder/pull/451#discussion_r660885345
tmp = local_path.temporarySibling(".snaptmp")
local_path.moveTo(tmp)
Message.log(
message_type=
u"downloader:filesystem:mark-overwrite:set-aside-existing",
source_path=local_path.path,
target_path=tmp.path,
)
os.utime(staged_content.path, (mtime, mtime))
# We want to get the path state of the file we are about to write.
# Ideally, we would get this from the staged file. However, depending
# on the operating system, the ctime can change when we rename.
# Instead, we get the path state before and after the move, and use
# the latter if the mtime and size match.
staged_path_state = get_pathinfo(staged_content).state
with start_action(
action_type=u"downloader:filesystem:mark-overwrite:emplace",
content_final_path=local_path.path,
):
if not local_path.parent().exists():
local_path.parent().makedirs()
else:
if not local_path.parent().isdir():
# XXX this probably needs a better answer but for
# now at least we can signal a problem.
raise RuntimeError(
"Wanted to write a subdir of {} but it's not a directory"
.format(local_path.parent().path, ))
staged_content.moveTo(local_path)
path_state = get_pathinfo(local_path).state
if (staged_path_state.mtime_ns != path_state.mtime_ns
or staged_path_state.size != path_state.size
or staged_path_state.ctime_ns > path_state.ctime_ns):
# The path got changed after we moved it, return the staged
# path_state so it is detected as a change.
path_state = staged_path_state
if tmp is not None:
with start_action(
action_type=
u"downloader:filesystem:mark-overwrite:dispose-existing",
):
# prior_pathstate comes from inside the state-machine
# _immediately_ before we conclude "there's no
# download conflict" (see _check_local_update in
# Magicfile) .. here, we double-check that the
# modification time and size match what we checked
# against in _check_local_update.
# if prior_pathstate is None, there was no file at all
# before this function ran .. so we must preserve the
# tmpfile
#
# otherwise, the pathstate of the tmp file must match
# what was there before
tmp.restat()
modtime = seconds_to_ns(tmp.getModificationTime())
size = tmp.getsize()
if prior_pathstate is None or \
prior_pathstate.mtime_ns != modtime or \
prior_pathstate.size != size:
# "something" has written to the file (or the
# tmpfile, after it was moved) causing mtime or
# size to be differt; OR prior_pathstate points at
# a directory (or some other reason causing
# get_fileinfo() to return a None pathstate
# above). Thus, we keep the file as a backup
# the MagicFile statemachine knows about this
# exception and processes it separately -- turning
# it into a conflict
raise BackupRetainedError(tmp, )
else:
# all good, the times + size match
tmp.remove()
return path_state
def visited(self, people):
Message.log(message_type="visited", people=people, place=self.name)
for thing in self.contained:
thing.visited(people)
def errReceived(self, data):
with self._action.context():
Message.log(message_type=u"err-received", data=data)
sys.stdout.write(data)
async def disconnect(self):
Message.log(transport="matrix", state="shutdown")
qlog.begin('Cascade, Version {}'.format(__version__))
#---------------------------------------------------------------------
# Modules which use quicklog cannot be imported until the logger
# has been initialized, so they are imported here.
#---------------------------------------------------------------------
from cascade.main import main
from cascade.util_eliot import rotating_logfile
#---------------------------------------------------------------------
# Start eliot logging
eliot_handler = rotating_logfile(os.path.join('log', 'cascade.eliot.log'),
auto=False)
Message.log(message_type="startup",
command_line=' '.join(sys.argv),
arguments=arguments,
version=__version__)
if arguments['-w']:
print('Privacy warning enabled.')
try:
main(arguments)
except FatalUserError as e:
# "Normal" user errors which cause application to halt
qlog.error(str(e))
except Exception as e:
# Unexpected exceptions (i.e. application crashes)
qlog.fatal_exception(e)
if arguments['-g']:
#Launch postmortem debugger
def g():
Message.log(message_type=u"hello")
yield expected
Message.log(message_type=u"goodbye")
def _pytest_collection_modifyitems(session, config, items):
"""Report allowed recids and jsonpaths to master and await start.
:type session: :py:class:_pytest.main.Session
:type config: :py:class:_pytest.config.Config
:type items: list
"""
redis_worker = config.option.redis_worker
unique_functions_found = set((item.function for item in items))
if not unique_functions_found:
raise AssertionError(
"No check functions were found."
" Scroll up for exceptions that may have prevented collection!")
elif not len(unique_functions_found) == 1:
raise AssertionError(
"We support one check function per file. Found {0} instead.".
format(len(unique_functions_found)))
item = items[0]
# Set allowed_paths and allowed_recids
if hasattr(item, 'cls'):
if hasattr(item.cls, 'allowed_paths'):
allowed_paths = item.cls.allowed_paths(
config.option.invenio_rule.arguments)
else:
allowed_paths = set()
if hasattr(item.cls, 'allowed_recids'):
allowed_recids = item.cls.allowed_recids(
config.option.invenio_rule.arguments, batch_recids(session),
all_recids(session), search(session))
else:
allowed_recids = batch_recids(session)
if allowed_recids - all_recids(session):
raise AssertionError('Check requested recids that are not in the'
' database!')
# TODO `allowed_paths` must return jsonpointers (IETF RFC 6901)
redis_worker.allowed_paths = allowed_paths
redis_worker.allowed_recids = allowed_recids
def worker_conflicts_with_currently_running(worker):
foreign_running_workers = get_workers_with_unprocessed_results()
blockers = set()
for foreign in foreign_running_workers:
if not _are_compatible(worker, foreign):
blockers.add(foreign)
return blockers
redis_worker.status = StatusWorker.ready # XXX unused?
with start_action(action_type='checking for conflicting running workers'):
redis_worker.lock.get()
try:
blockers = worker_conflicts_with_currently_running(redis_worker)
if blockers:
Message.log(message_type='found conflicting workers',
value=str(blockers))
print 'CONFLICT {} {}'.format(redis_worker.task_id, blockers)
redis_worker.retry_after_ids = {bl.task_id for bl in blockers}
del items[:]
else:
print 'RESUMING ' + str(redis_worker.task_id)
redis_worker.status = StatusWorker.running
finally:
redis_worker.lock.release()
def honeymoon(family, destination):
Message.log(message_type="honeymoon", people=family)
destination.visited(family)
def handle_timeout(failure):
failure.trap(CancelledError)
msg = ("Force stopping the scenario. "
"There are {num_requests} outstanding requests").format(
num_requests=outstanding_requests)
Message.log(key='force_stop_request', value=msg)
def g():
Message.log(message_type=u"a")
yield
Message.log(message_type=u"b")
def test_returns_none(self):
Message.log(hello="world")
def g():
Message.log(message_type=u"a")
yield succeed(None)
Message.log(message_type=u"b")
def got_code(code):
# We don't currently expire these at all.
expires = _NoExpiration()
Message.log(event=u"code-allocated")
return _WormholeClaim(code, expires)
def g():
Message.log(message_type=u"a")
yield waiting
Message.log(message_type=u"b")
def log_message(message: str, **kwargs):
"""Log the message in the current action context."""
Message.log(message_type=message, **kwargs)
def _thread(self):
try:
_assert(self._filter is not None, "no watch set")
# To call Twisted or Tahoe APIs, use reactor.callFromThread as described in
# <http://twistedmatrix.com/documents/current/core/howto/threading.html>.
fni = FileNotifyInformation()
while True:
self._state = STARTED
action = start_action(
action_type=u"read-changes",
directory=self._path.path,
recursive=self._recursive,
filter=self._filter,
)
try:
with action:
fni.read_changes(self._hDirectory, self._recursive, self._filter)
except WindowsError as e:
self._state = STOPPING
if self._check_stop():
return
for info in fni:
path = self._path.preauthChild(info.filename) # FilePath with Unicode path
if info.action == FILE_ACTION_MODIFIED and path.isdir():
Message.log(
message_type=u"filtering-out",
info=repr(info),
)
continue
else:
Message.log(
message_type=u"processing",
info=repr(info),
)
#mask = _action_to_inotify_mask.get(info.action, IN_CHANGED)
@log_call(
action_type=MAYBE_NOTIFY.action_type,
include_args=[],
include_result=False,
)
def _do_pending_calls():
event_mask = IN_CHANGED
self._pending_call = None
for path1 in self._pending:
if self._callbacks:
for cb in self._callbacks:
try:
with CALLBACK(inotify_events=event_mask):
cb(None, path1, event_mask)
except Exception as e2:
log.err(e2)
self._pending = set()
def _maybe_notify(path2):
if path2 not in self._pending:
self._pending.add(path2)
if self._state not in [STOPPING, STOPPED]:
_do_pending_calls()
# if self._pending_call is None and self._state not in [STOPPING, STOPPED]:
# self._pending_call = reactor.callLater(self._pending_delay, _do_pending_calls)
reactor.callFromThread(_maybe_notify, path)
if self._check_stop():
return
except Exception as e:
log.err(e)
self._state = STOPPED
raise
