def __init__(cls, name, bases, env_dict): # pylint: disable=no-self-argument
registry.MetaclassRegistry.__init__(cls, name, bases, env_dict)
has_mixin = False
for mixin in itervalues(cls.connection_mixins):
if issubclass(cls, mixin):
has_mixin = True
break
# ApiRegressionTest is a base class, so it doesn't make sense to generate
# _http_v1/_http_v2 classes for it.
# Generating classes from already generated classes would lead to infinite
# recursion. Skipping the generated ones.
if name == "ApiRegressionTest" or has_mixin:
return
for mixin in itervalues(ApiRegressionTestMetaclass.connection_mixins):
if (mixin.skip_legacy_dynamic_proto_tests and
getattr(cls, "uses_legacy_dynamic_protos", False)):
continue
# Do not generate combinations where the mixin demands relational db reads
# but the test is aff4 only.
if (getattr(cls, "aff4_only_test", False) and
getattr(mixin, "read_from_relational_db", False)):
continue
cls_name = "%s_%s" % (name, mixin.connection_type)
test_cls = compatibility.MakeType(
cls_name,
(mixin, cls, test_lib.GRRBaseTest),
# pylint: disable=protected-access
{"testForRegression": lambda x: x._testForRegression()})
module = sys.modules[cls.__module__]
setattr(module, cls_name, test_cls)
def prepareSourcestampListForTrigger(self):
if self.sourceStamps:
ss_for_trigger = {}
for ss in self.sourceStamps:
codebase = ss.get('codebase', '')
assert codebase not in ss_for_trigger, "codebase specified multiple times"
ss_for_trigger[codebase] = ss
return list(itervalues(ss_for_trigger))
if self.alwaysUseLatest:
return []
# start with the sourcestamps from current build
ss_for_trigger = {}
objs_from_build = self.build.getAllSourceStamps()
for ss in objs_from_build:
ss_for_trigger[ss.codebase] = ss.asDict()
# overrule revision in sourcestamps with got revision
if self.updateSourceStamp:
got = self.getAllGotRevisions()
for codebase in ss_for_trigger:
if codebase in got:
ss_for_trigger[codebase]['revision'] = got[codebase]
return list(itervalues(ss_for_trigger))
def ParseFileset(self, fileset=None):
"""Process linux system login files.
Orchestrates collection of account entries from /etc/passwd and
/etc/shadow. The passwd and shadow entries are reconciled and group
memberships are mapped to the account.
Args:
fileset: A dict of files mapped from path to an open file.
Yields:
- A series of User entries, each of which is populated with
group memberships and indications of the shadow state of the account.
- A series of anomalies in cases where there are mismatches between passwd
and shadow state.
"""
self.AddPassword(fileset)
self.AddShadow(fileset)
self.ReconcileShadow(self.shadow_store)
# Get group memberships using the files that were already collected.
# Separate out groups and anomalies.
for rdf in LinuxSystemGroupParser().ParseFileset(fileset):
if isinstance(rdf, rdf_client.Group):
self.groups[rdf.name] = rdf
else:
yield rdf
self.AddGroupMemberships()
for user in itervalues(self.entry):
yield user
for grp in itervalues(self.groups):
yield grp
for anom in self.FindAnomalies():
yield anom
def CreateNewZipWithSignedLibs(z_in,
z_out,
ignore_files=None,
signer=None,
skip_signing_files=None):
"""Copies files from one zip to another, signing all qualifying files."""
ignore_files = ignore_files or []
skip_signing_files = skip_signing_files or []
extensions_to_sign = [".sys", ".exe", ".dll", ".pyd"]
to_sign = []
for template_file in z_in.namelist():
if template_file not in ignore_files:
extension = os.path.splitext(template_file)[1].lower()
if (signer and template_file not in skip_signing_files and
extension in extensions_to_sign):
to_sign.append(template_file)
else:
CopyFileInZip(z_in, template_file, z_out)
temp_files = {}
for filename in to_sign:
fd, path = tempfile.mkstemp()
with os.fdopen(fd, "wb") as temp_fd:
temp_fd.write(z_in.read(filename))
temp_files[filename] = path
try:
signer.SignFiles(itervalues(temp_files))
except AttributeError:
for f in itervalues(temp_files):
signer.SignFile(f)
for filename, tempfile_path in iteritems(temp_files):
z_out.writestr(filename, open(tempfile_path, "rb").read())
def RunXinetdCheck(self,
chk_id,
svc,
disabled,
sym,
found,
xinetd=False,
should_detect=True):
host_data = self.SetKnowledgeBase()
cfgs = parsers_test_lib.GenXinetd(svc, disabled)
stats, files = parsers_test_lib.GenTestData(cfgs, itervalues(cfgs))
data = list(self.parser(stats, files, None))
# create entries on whether xinetd itself is setup to start or not
if xinetd:
cfgs = parsers_test_lib.GenInit("xinetd",
"the extended Internet services daemon")
stats, files = parsers_test_lib.GenTestData(cfgs, itervalues(cfgs))
lsb_parser = linux_service_parser.LinuxLSBInitParser()
data.extend(list(lsb_parser.ParseMultiple(stats, files, None)))
host_data["LinuxServices"] = self.SetArtifactData(parsed=data)
results = self.RunChecks(host_data)
if should_detect:
self.assertCheckDetectedAnom(chk_id, results, sym, found)
else:
self.assertCheckUndetected(chk_id, results)
def _process_changes(self, newRev, branch):
"""
Read changes since last change.
- Read list of commit hashes.
- Extract details from each commit.
- Add changes to database.
"""
# initial run, don't parse all history
if not self.lastRev:
return
if newRev in itervalues(self.lastRev):
# TODO: no new changes on this branch
# should we just use the lastRev again, but with a different branch?
pass
# get the change list
revListArgs = ([r'--format=%H', r'%s' % newRev] +
[r'^%s' % rev for rev in itervalues(self.lastRev)] +
[r'--'])
self.changeCount = 0
results = yield self._dovccmd('log', revListArgs, path=self.workdir)
# process oldest change first
revList = results.split()
revList.reverse()
self.changeCount = len(revList)
self.lastRev[branch] = newRev
if self.changeCount:
log.msg('gitpoller: processing %d changes: %s from "%s" branch "%s"'
% (self.changeCount, revList, self.repourl, branch))
for rev in revList:
dl = defer.DeferredList([
self._get_commit_timestamp(rev),
self._get_commit_author(rev),
self._get_commit_files(rev),
self._get_commit_comments(rev),
], consumeErrors=True)
results = yield dl
# check for failures
failures = [r[1] for r in results if not r[0]]
if failures:
# just fail on the first error; they're probably all related!
raise failures[0]
timestamp, author, files, comments = [r[1] for r in results]
yield self.master.data.updates.addChange(
author=author, revision=ascii2unicode(rev), files=files,
comments=comments, when_timestamp=timestamp,
branch=ascii2unicode(self._removeHeads(branch)),
project=self.project, repository=ascii2unicode(self.repourl),
category=self.category, src=u'git')
def build_training_data(dataset, language, data_augmentation_config,
random_state):
# Create class mapping
intents = dataset[INTENTS]
intent_index = 0
classes_mapping = dict()
for intent in sorted(intents):
classes_mapping[intent] = intent_index
intent_index += 1
noise_class = intent_index
# Computing dataset statistics
nb_utterances = [len(intent[UTTERANCES]) for intent in itervalues(intents)]
augmented_utterances = []
utterance_classes = []
for nb_utterance, intent_name in zip(nb_utterances, intents):
min_utterances_to_generate = max(
data_augmentation_config.min_utterances, nb_utterance)
utterances = augment_utterances(
dataset, intent_name, language=language,
min_utterances=min_utterances_to_generate,
capitalization_ratio=0.0, random_state=random_state)
augmented_utterances += utterances
utterance_classes += [classes_mapping[intent_name] for _ in
range(len(utterances))]
augmented_utterances = add_unknown_word_to_utterances(
augmented_utterances,
data_augmentation_config.unknown_words_replacement_string,
data_augmentation_config.unknown_word_prob,
random_state
)
# Adding noise
noisy_utterances = generate_noise_utterances(
augmented_utterances, len(intents), data_augmentation_config, language,
random_state)
augmented_utterances = [get_text_from_chunks(u[DATA])
for u in augmented_utterances]
augmented_utterances += noisy_utterances
utterance_classes += [noise_class for _ in noisy_utterances]
if noisy_utterances:
classes_mapping[NOISE_NAME] = noise_class
nb_classes = len(set(itervalues(classes_mapping)))
intent_mapping = [None for _ in range(nb_classes)]
for intent, intent_class in iteritems(classes_mapping):
if intent == NOISE_NAME:
intent_mapping[intent_class] = None
else:
intent_mapping[intent_class] = intent
return augmented_utterances, np.array(utterance_classes), intent_mapping
def addBuildset(self, waited_for, scheduler=None, sourcestamps=None, reason=u'',
properties=None, builderids=None, external_idstring=None,
parent_buildid=None, parent_relationship=None,
_reactor=reactor):
if sourcestamps is None:
sourcestamps = []
if properties is None:
properties = {}
if builderids is None:
builderids = []
submitted_at = int(_reactor.seconds())
bsid, brids = yield self.master.db.buildsets.addBuildset(
sourcestamps=sourcestamps, reason=reason,
properties=properties, builderids=builderids,
waited_for=waited_for, external_idstring=external_idstring,
submitted_at=epoch2datetime(submitted_at),
parent_buildid=parent_buildid, parent_relationship=parent_relationship)
yield BuildRequestCollapser(self.master, list(itervalues(brids))).collapse()
# get each of the sourcestamps for this buildset (sequentially)
bsdict = yield self.master.db.buildsets.getBuildset(bsid)
sourcestamps = []
for ssid in bsdict['sourcestamps']:
sourcestamps.append(
(yield self.master.data.get(('sourcestamps', str(ssid)))).copy()
)
# notify about the component build requests
brResource = self.master.data.getResourceType("buildrequest")
brResource.generateEvent(list(itervalues(brids)), 'new')
# and the buildset itself
msg = dict(
bsid=bsid,
external_idstring=external_idstring,
reason=reason,
submitted_at=submitted_at,
complete=False,
complete_at=None,
results=None,
scheduler=scheduler,
sourcestamps=sourcestamps)
# TODO: properties=properties)
self.produceEvent(msg, "new")
log.msg("added buildset %d to database" % bsid)
# if there are no builders, then this is done already, so send the
# appropriate messages for that
if not builderids:
yield self.maybeBuildsetComplete(bsid, _reactor=_reactor)
defer.returnValue((bsid, brids))
def AddGroupMemberships(self):
"""Adds aggregate group membership from group, gshadow and passwd."""
self.groups = {g.name: self._Members(g) for g in itervalues(self.groups)}
# Map the groups a user is a member of, irrespective of primary/extra gid.
for g in itervalues(self.groups):
for user in g.members:
membership = self.memberships.setdefault(user, set())
membership.add(g.gid)
# Now add the completed membership to the user account.
for user in itervalues(self.entry):
user.gids = self.memberships.get(user.username)
def ReadMessageHandlerRequests(self):
"""Reads all message handler requests from the database."""
res = []
leases = self.message_handler_leases
for requests in itervalues(self.message_handler_requests):
for r in itervalues(requests):
res.append(r.Copy())
existing_lease = leases.get(r.handler_name, {}).get(r.request_id, None)
res[-1].leased_until = existing_lease
return sorted(res, key=lambda r: -1 * r.timestamp)
def percentIndexOverlap(attr, results):
"""
Returns what fraction of index values are common to all NDFrames in results,
compared to the NDFrame with the most rows/columns.
Pass "index" or "columns" as ``attr`` to specify rows or columns.
"""
getter = operator.attrgetter(attr)
longest = max(len(getter(df)) for df in itervalues(results))
columnsIter = (getter(df) for df in itervalues(results))
mutualColumns = functools.reduce(lambda colA, colB: colA.intersection(colB), columnsIter)
return len(mutualColumns)/longest
def ReadClientMessages(self, client_id):
"""Reads all client messages available for a given client_id."""
res = []
for msgs_by_id in itervalues(self.client_messages):
for orig_msg in itervalues(msgs_by_id):
if db_utils.ClientIdFromGrrMessage(orig_msg) != client_id:
continue
msg = orig_msg.Copy()
current_lease = self.client_message_leases.get(msg.task_id)
if current_lease:
msg.leased_until, msg.leased_by = current_lease
res.append(msg)
return res
def canStartBuild(self):
"""
I am called when a build is requested to see if this worker
can start a build. This function can be used to limit overall
concurrency on the worker.
Note for subclassers: if a worker can become willing to start a build
without any action on that worker (for example, by a resource in use on
another worker becoming available), then you must arrange for
L{maybeStartBuildsForWorker} to be called at that time, or builds on
this worker will not start.
"""
if self.worker_status.isPaused():
return False
# If we're waiting to shutdown gracefully, then we shouldn't
# accept any new jobs.
if self.worker_status.getGraceful():
return False
if self.max_builds:
active_builders = [sb for sb in itervalues(self.workerforbuilders)
if sb.isBusy()]
if len(active_builders) >= self.max_builds:
return False
if not self.locksAvailable():
return False
return True
def mergeSourceStampsWith(self, others):
""" Returns one merged sourcestamp for every codebase """
# get all codebases from all requests
all_codebases = set(self.sources)
for other in others:
all_codebases |= set(other.sources)
all_merged_sources = {}
# walk along the codebases
for codebase in all_codebases:
all_sources = []
if codebase in self.sources:
all_sources.append(self.sources[codebase])
for other in others:
if codebase in other.sources:
all_sources.append(other.sources[codebase])
assert len(
all_sources) > 0, "each codebase should have atleast one sourcestamp"
# TODO: select the sourcestamp that best represents the merge,
# preferably the latest one. This used to be accomplished by
# looking at changeids and picking the highest-numbered.
all_merged_sources[codebase] = all_sources[-1]
return list(itervalues(all_merged_sources))
def checkSummary(self, got, step, build=None):
self.failUnless(all(isinstance(k, unicode) for k in got))
self.failUnless(all(isinstance(k, unicode) for k in itervalues(got)))
exp = {u'step': step}
if build:
exp[u'build'] = build
self.assertEqual(got, exp)
def old_way():
d = None
for b in itervalues(self.buildslave.slavebuilders):
if b.remote:
d = b.mind.callRemote("shutdown")
break
if d:
name = self.buildslave.slavename
log.msg("Shutting down (old) slave: %s" % name)
# The remote shutdown call will not complete successfully since
# the buildbot process exits almost immediately after getting
# the shutdown request.
# Here we look at the reason why the remote call failed, and if
# it's because the connection was lost, that means the slave
# shutdown as expected.
@d.addErrback
def _errback(why):
if why.check(pb.PBConnectionLost):
log.msg("Lost connection to %s" % name)
else:
log.err("Unexpected error when trying to shutdown %s"
% name)
return d
log.err("Couldn't find remote builder to shut down slave")
return defer.succeed(None)
def stopConsuming(self, key=None):
if key is not None:
self.qrefs[key].stopConsuming()
else:
for qref in itervalues(self.qrefs):
qref.stopConsuming()
self.qrefs = {}
def checkSummary(self, got, step, build=None):
self.assertTrue(all(isinstance(k, text_type) for k in got))
self.assertTrue(all(isinstance(k, text_type) for k in itervalues(got)))
exp = {u'step': step}
if build:
exp[u'build'] = build
self.assertEqual(got, exp)
def findChangeSourceId(self, name):
validation.verifyType(self.testcase, 'changesource name', name,
validation.StringValidator())
if name not in self.changesourceIds:
self.changesourceIds[name] = max(
[0] + list(itervalues(self.changesourceIds))) + 1
return defer.succeed(self.changesourceIds[name])
def activate(self):
yield NightlyBase.activate(self)
lastTrigger = yield self.getState("lastTrigger", None)
self._lastTrigger = None
if lastTrigger:
try:
if isinstance(lastTrigger[0], list):
self._lastTrigger = (
lastTrigger[0],
properties.Properties.fromDict(lastTrigger[1]),
lastTrigger[2],
lastTrigger[3],
)
# handle state from before Buildbot-0.9.0
elif isinstance(lastTrigger[0], dict):
self._lastTrigger = (
list(itervalues(lastTrigger[0])),
properties.Properties.fromDict(lastTrigger[1]),
None,
None,
)
except Exception:
pass
# If the lastTrigger isn't of the right format, ignore it
if not self._lastTrigger:
log.msg(
format="NightlyTriggerable Scheduler <%(scheduler)s>: "
"could not load previous state; starting fresh",
scheduler=self.name,
)
def check_single_master(self):
# check additional problems that are only valid in a single-master
# installation
if self.multiMaster:
return
if not self.workers:
error("no workers are configured")
if not self.builders:
error("no builders are configured")
# check that all builders are implemented on this master
unscheduled_buildernames = set([b.name for b in self.builders])
for s in itervalues(self.schedulers):
builderNames = s.listBuilderNames()
if interfaces.IRenderable.providedBy(builderNames):
unscheduled_buildernames.clear()
else:
for n in builderNames:
if interfaces.IRenderable.providedBy(n):
unscheduled_buildernames.clear()
elif n in unscheduled_buildernames:
unscheduled_buildernames.remove(n)
if unscheduled_buildernames:
error("builder(s) %s have no schedulers to drive them"
% (', '.join(unscheduled_buildernames),))
def forget_about_it(self, name):
validation.verifyType(self.testcase, 'scheduler name', name,
validation.StringValidator())
if name not in self.schedulerIds:
self.schedulerIds[name] = max(
[0] + list(itervalues(self.schedulerIds))) + 1
return defer.succeed(self.schedulerIds[name])
def test_api_collection_filter_and_order_desc(self):
yield self.render_resource(self.rsrc, '/test?field=info&order=-info')
self.assertRestCollection(typeName='tests',
items=sorted(list([{'info': v['info']}
for v in itervalues(endpoint.testData)]),
key=lambda v: v['info'], reverse=True),
total=8, orderSignificant=True)
def test_api_collection_filter_pagination(self):
yield self.render_resource(self.rsrc, '/test?success=false&limit=2')
# note that the limit/offset and total are *after* the filter
self.assertRestCollection(typeName='tests',
items=sorted([v for v in itervalues(endpoint.testData)
if not v['success']], key=lambda v: v['id'])[:2],
total=3)
def Lookup(self,
keywords,
start_time=FIRST_TIMESTAMP,
end_time=LAST_TIMESTAMP,
last_seen_map=None):
"""Finds objects associated with keywords.
Find the names related to all keywords.
Args:
keywords: A collection of keywords that we are interested in.
start_time: Only considers keywords added at or after this point in time.
end_time: Only considers keywords at or before this point in time.
last_seen_map: If present, is treated as a dict and populated to map pairs
(keyword, name) to the timestamp of the latest connection found.
Returns:
A set of potentially relevant names.
"""
posting_lists = self.ReadPostingLists(
keywords,
start_time=start_time,
end_time=end_time,
last_seen_map=last_seen_map)
results = list(itervalues(posting_lists))
relevant_set = results[0]
for hits in results:
relevant_set &= hits
if not relevant_set:
return relevant_set
return relevant_set
def ReadChildFlowObjects(self, client_id, flow_id):
"""Reads flows that were started by a given flow from the database."""
res = []
for flow in itervalues(self.flows):
if flow.client_id == client_id and flow.parent_flow_id == flow_id:
res.append(flow)
return res
def collection_view_listing_db(context, request):
result = {}
frame = request.params.get('frame', 'columns')
limit = request.params.get('limit', 25)
if limit in ('', 'all'):
limit = None
if limit is not None:
try:
limit = int(limit)
except ValueError:
limit = 25
items = (
item for item in itervalues(context)
if request.has_permission('view', item)
)
if limit is not None:
items = islice(items, limit)
result['@graph'] = [
request.embed(request.resource_path(item, '@@' + frame))
for item in items
]
if limit is not None and len(result['@graph']) == limit:
params = [(k, v) for k, v in request.params.items() if k != 'limit']
params.append(('limit', 'all'))
result['all'] = '%s?%s' % (request.resource_path(context), urlencode(params))
return result
def Handle(self, unused_args, token=None):
router_methods = self.router.__class__.GetAnnotatedMethods()
result = ApiListApiMethodsResult()
for router_method in itervalues(router_methods):
api_method = ApiMethod(
name=router_method.name,
category=router_method.category,
doc=router_method.doc,
http_route=router_method.http_methods[-1][1],
http_methods=[router_method.http_methods[-1][0]])
if router_method.args_type:
api_method.args_type_descriptor = (
api_value_renderers.BuildTypeDescriptor(router_method.args_type))
if router_method.result_type:
if router_method.result_type == router_method.BINARY_STREAM_RESULT_TYPE:
api_method.result_kind = api_method.ResultKind.BINARY_STREAM
else:
api_method.result_kind = api_method.ResultKind.VALUE
api_method.result_type_descriptor = (
api_value_renderers.BuildTypeDescriptor(router_method.result_type)
)
else:
api_method.result_kind = api_method.ResultKind.NONE
result.items.append(api_method)
return result
def check(result):
# finish up the debounced updateSummary before checking
self.debounceClock.advance(1)
self.assertEqual(self.expected_remote_commands, [],
"assert all expected commands were run")
# in case of unexpected result, display logs in stdout for debugging failing tests
if result != self.exp_result:
for loog in itervalues(self.step.logs):
print(loog.stdout)
print(loog.stderr)
self.assertEqual(result, self.exp_result, "expected result")
if self.exp_state_string:
stepStateString = self.master.data.updates.stepStateString
stepids = list(stepStateString)
assert stepids, "no step state strings were set"
self.assertEqual(stepStateString[stepids[0]],
self.exp_state_string,
"expected step state strings")
for pn, (pv, ps) in iteritems(self.exp_properties):
self.assertTrue(self.properties.hasProperty(pn),
"missing property '%s'" % pn)
self.assertEqual(self.properties.getProperty(pn),
pv, "property '%s'" % pn)
if ps is not None:
self.assertEqual(
self.properties.getPropertySource(pn), ps, "property '%s' source" % pn)
for pn in self.exp_missing_properties:
self.assertFalse(self.properties.hasProperty(pn),
"unexpected property '%s'" % pn)
for l, contents in iteritems(self.exp_logfiles):
self.assertEqual(
self.step.logs[l].stdout, contents, "log '%s' contents" % l)
def ParseMultiple(self, stats, knowledge_base):
"""Parse each returned registry value."""
user_dict = {}
for stat in stats:
sid_str = stat.pathspec.path.split("/", 3)[2]
if SID_RE.match(sid_str):
if sid_str not in user_dict:
user_dict[sid_str] = rdf_client.User(sid=sid_str)
if stat.registry_data.GetValue():
# Look up in the mapping if we can use this entry to populate a user
# attribute, and if so, set it.
reg_key_name = stat.pathspec.Dirname().Basename()
if reg_key_name in self.key_var_mapping:
map_dict = self.key_var_mapping[reg_key_name]
reg_key = stat.pathspec.Basename()
kb_attr = map_dict.get(reg_key)
if kb_attr:
value = artifact_utils.ExpandWindowsEnvironmentVariables(
stat.registry_data.GetValue(), knowledge_base)
value = artifact_utils.ExpandWindowsUserEnvironmentVariables(
value, knowledge_base, sid=sid_str)
user_dict[sid_str].Set(kb_attr, value)
# Now yield each user we found.
return itervalues(user_dict)
def _LegacyCheckHashesWithFileStore(self):
"""Check all queued up hashes for existence in file store (legacy).
Hashes which do not exist in the file store will be downloaded. This
function flushes the entire queue (self.state.pending_hashes) in order to
minimize the round trips to the file store.
If a file was found in the file store it is copied from there into the
client's VFS namespace. Otherwise, we request the client to hash every block
in the file, and add it to the file tracking queue
(self.state.pending_files).
"""
if not self.state.pending_hashes:
return
# This map represents all the hashes in the pending urns.
file_hashes = {}
# Store a mapping of hash to tracker. Keys are hashdigest objects,
# values are arrays of tracker dicts.
hash_to_tracker = {}
for index, tracker in iteritems(self.state.pending_hashes):
# We might not have gotten this hash yet
if tracker.get("hash_obj") is None:
continue
hash_obj = tracker["hash_obj"]
digest = hash_obj.sha256
file_hashes[index] = hash_obj
hash_to_tracker.setdefault(digest, []).append(tracker)
# First we get all the files which are present in the file store.
files_in_filestore = {}
# TODO(amoser): This object never changes, could this be a class attribute?
filestore_obj = aff4.FACTORY.Open(
legacy_filestore.FileStore.PATH,
legacy_filestore.FileStore,
mode="r",
token=self.token)
for file_store_urn, hash_obj in filestore_obj.CheckHashes(
itervalues(file_hashes), external=self.state.use_external_stores):
# Since checkhashes only returns one digest per unique hash we need to
# find any other files pending download with the same hash.
for tracker in hash_to_tracker[hash_obj.sha256]:
self.state.files_skipped += 1
file_hashes.pop(tracker["index"])
files_in_filestore[file_store_urn] = hash_obj
# Remove this tracker from the pending_hashes store since we no longer
# need to process it.
self.state.pending_hashes.pop(tracker["index"])
# Now that the check is done, reset our counter
self.state.files_hashed_since_check = 0
# Now copy all existing files to the client aff4 space.
for filestore_file_urn, hash_obj in iteritems(files_in_filestore):
for file_tracker in hash_to_tracker.get(hash_obj.sha256, []):
stat_entry = file_tracker["stat_entry"]
# Copy the existing file from the filestore to the client namespace.
target_urn = stat_entry.pathspec.AFF4Path(self.client_urn)
aff4.FACTORY.Copy(
filestore_file_urn, target_urn, update_timestamps=True)
with aff4.FACTORY.Open(
target_urn, mode="rw", token=self.token) as new_fd:
new_fd.Set(new_fd.Schema.STAT, stat_entry)
# Due to potential filestore corruption, the existing files
# can have 0 size.
if new_fd.size == 0:
new_fd.size = (file_tracker["bytes_read"] or stat_entry.st_size)
if data_store.RelationalDBWriteEnabled():
path_info = rdf_objects.PathInfo.FromStatEntry(stat_entry)
path_info.hash_entry = hash_obj
data_store.REL_DB.WritePathInfos(self.client_id, [path_info])
# Add this file to the filestore index.
filestore_obj.AddURNToIndex(str(hash_obj.sha256), target_urn)
# Report this hit to the flow's caller.
self._ReceiveFetchedFile(file_tracker)
# Now we iterate over all the files which are not in the store and arrange
# for them to be copied.
for index in file_hashes:
# Move the tracker from the pending hashes store to the pending files
# store - it will now be downloaded.
file_tracker = self.state.pending_hashes.pop(index)
self.state.pending_files[index] = file_tracker
# If we already know how big the file is we use that, otherwise fall back
# to the size reported by stat.
if file_tracker["bytes_read"] > 0:
file_tracker["size_to_download"] = file_tracker["bytes_read"]
else:
file_tracker["size_to_download"] = file_tracker["stat_entry"].st_size
# We do not have the file here yet - we need to retrieve it.
expected_number_of_hashes = (
file_tracker["size_to_download"] // self.CHUNK_SIZE + 1)
# We just hash ALL the chunks in the file now. NOTE: This maximizes client
# VFS cache hit rate and is far more efficient than launching multiple
# GetFile flows.
self.state.files_to_fetch += 1
for i in range(expected_number_of_hashes):
if i == expected_number_of_hashes - 1:
# The last chunk is short.
length = file_tracker["size_to_download"] % self.CHUNK_SIZE
else:
length = self.CHUNK_SIZE
self.CallClient(
server_stubs.HashBuffer,
pathspec=file_tracker["stat_entry"].pathspec,
offset=i * self.CHUNK_SIZE,
length=length,
next_state="CheckHash",
request_data=dict(index=index))
if self.state.files_hashed % 100 == 0:
self.Log("Hashed %d files, skipped %s already stored.",
self.state.files_hashed, self.state.files_skipped)
def test_api_collection_list_filter(self):
yield self.render_resource(self.rsrc, b'/test?tags__contains=a')
self.assertRestCollection(typeName='tests',
items=[v for v in itervalues(endpoint.testData)
if 'a' in v['tags']],
total=2)
def test_api_collection_operator_filter(self):
yield self.render_resource(self.rsrc, b'/test?info__lt=skipped')
self.assertRestCollection(typeName='tests',
items=[v for v in itervalues(endpoint.testData)
if v['info'] < 'skipped'],
total=4)
def test_api_collection_simple_filter(self):
yield self.render_resource(self.rsrc, b'/test?success=yes')
self.assertRestCollection(
typeName='tests',
items=[v for v in itervalues(endpoint.testData) if v['success']],
total=5)
def run(self):
schedulers_and_props = yield self.getSchedulersAndProperties()
schedulers_and_props_list = []
# To be back compatible we need to differ between old and new style
# schedulers_and_props can either consist of 2 elements tuple or
# dictionary
for element in schedulers_and_props:
if isinstance(element, dict):
schedulers_and_props_list = schedulers_and_props
break
else:
# Old-style back compatibility: Convert tuple to dict and make
# it important
d = {
'sched_name': element[0],
'props_to_set': element[1],
'unimportant': False
}
schedulers_and_props_list.append(d)
# post process the schedulernames, and raw properties
# we do this out of the loop, as this can result in errors
schedulers_and_props = [
(self.getSchedulerByName(entry_dict['sched_name']),
self.createTriggerProperties(entry_dict['props_to_set']),
entry_dict['unimportant'])
for entry_dict in schedulers_and_props_list
]
ss_for_trigger = self.prepareSourcestampListForTrigger()
dl = []
triggeredNames = []
results = SUCCESS
self.running = True
unimportant_brids = []
for sch, props_to_set, unimportant in schedulers_and_props:
idsDeferred, resultsDeferred = sch.trigger(
waited_for=self.waitForFinish,
sourcestamps=ss_for_trigger,
set_props=props_to_set,
parent_buildid=self.build.buildid,
parent_relationship=self.parent_relationship)
# we are not in a hurry of starting all in parallel and managing
# the deferred lists, just let the db writes be serial.
brids = {}
try:
bsid, brids = yield idsDeferred
except Exception as e:
yield self.addLogWithException(e)
results = EXCEPTION
if unimportant:
unimportant_brids.extend(itervalues(brids))
self.brids.extend(itervalues(brids))
for brid in brids.values():
# put the url to the brids, so that we can have the status from
# the beginning
url = self.master.status.getURLForBuildrequest(brid)
yield self.addURL("%s #%d" % (sch.name, brid), url)
dl.append(resultsDeferred)
triggeredNames.append(sch.name)
if self.ended:
defer.returnValue(CANCELLED)
self.triggeredNames = triggeredNames
if self.waitForFinish:
self.waitForFinishDeferred = defer.DeferredList(dl,
consumeErrors=1)
try:
rclist = yield self.waitForFinishDeferred
except defer.CancelledError:
pass
# we were interrupted, don't bother update status
if self.ended:
defer.returnValue(CANCELLED)
yield self.addBuildUrls(rclist)
results = yield self.worstStatus(results, rclist,
unimportant_brids)
else:
# do something to handle errors
for d in dl:
d.addErrback(
log.err,
'(ignored) while invoking Triggerable schedulers:')
defer.returnValue(results)
def _CheckHashesWithFileStore(self):
"""Check all queued up hashes for existence in file store.
Hashes which do not exist in the file store will be downloaded. This
function flushes the entire queue (self.state.pending_hashes) in order to
minimize the round trips to the file store.
If a file was found in the file store it is not scheduled for collection
and its PathInfo is written to the datastore pointing to the file store's
hash. Otherwise, we request the client to hash every block in the file,
and add it to the file tracking queue (self.state.pending_files).
"""
if not data_store.RelationalDBReadEnabled(category="filestore"):
return self._LegacyCheckHashesWithFileStore()
if not self.state.pending_hashes:
return
# This map represents all the hashes in the pending urns.
file_hashes = {}
# Store a mapping of hash to tracker. Keys are hashdigest objects,
# values are arrays of tracker dicts.
hash_to_tracker = {}
for index, tracker in iteritems(self.state.pending_hashes):
# We might not have gotten this hash yet
if tracker.get("hash_obj") is None:
continue
hash_obj = tracker["hash_obj"]
digest = hash_obj.sha256
file_hashes[index] = hash_obj
hash_to_tracker.setdefault(rdf_objects.SHA256HashID(digest),
[]).append(tracker)
# First we get all the files which are present in the file store.
files_in_filestore = set()
statuses = file_store.CheckHashes([
rdf_objects.SHA256HashID.FromBytes(ho.sha256.AsBytes())
for ho in itervalues(file_hashes)
])
for hash_id, status in iteritems(statuses):
self.HeartBeat()
if not status:
continue
# Since checkhashes only returns one digest per unique hash we need to
# find any other files pending download with the same hash.
for tracker in hash_to_tracker[hash_id]:
self.state.files_skipped += 1
file_hashes.pop(tracker["index"])
files_in_filestore.add(hash_id)
# Remove this tracker from the pending_hashes store since we no longer
# need to process it.
self.state.pending_hashes.pop(tracker["index"])
# Now that the check is done, reset our counter
self.state.files_hashed_since_check = 0
# Now copy all existing files to the client aff4 space.
for hash_id in files_in_filestore:
for file_tracker in hash_to_tracker.get(hash_id, []):
stat_entry = file_tracker["stat_entry"]
path_info = rdf_objects.PathInfo.FromStatEntry(stat_entry)
path_info.hash_entry = hash_obj
data_store.REL_DB.WritePathInfos(self.client_id, [path_info])
# Report this hit to the flow's caller.
self._ReceiveFetchedFile(file_tracker)
# Now we iterate over all the files which are not in the store and arrange
# for them to be copied.
for index in file_hashes:
# Move the tracker from the pending hashes store to the pending files
# store - it will now be downloaded.
file_tracker = self.state.pending_hashes.pop(index)
self.state.pending_files[index] = file_tracker
# If we already know how big the file is we use that, otherwise fall back
# to the size reported by stat.
if file_tracker["bytes_read"] > 0:
file_tracker["size_to_download"] = file_tracker["bytes_read"]
else:
file_tracker["size_to_download"] = file_tracker["stat_entry"].st_size
# We do not have the file here yet - we need to retrieve it.
expected_number_of_hashes = (
file_tracker["size_to_download"] // self.CHUNK_SIZE + 1)
# We just hash ALL the chunks in the file now. NOTE: This maximizes client
# VFS cache hit rate and is far more efficient than launching multiple
# GetFile flows.
self.state.files_to_fetch += 1
for i in range(expected_number_of_hashes):
if i == expected_number_of_hashes - 1:
# The last chunk is short.
length = file_tracker["size_to_download"] % self.CHUNK_SIZE
else:
length = self.CHUNK_SIZE
self.CallClient(
server_stubs.HashBuffer,
pathspec=file_tracker["stat_entry"].pathspec,
offset=i * self.CHUNK_SIZE,
length=length,
next_state="CheckHash",
request_data=dict(index=index))
if self.state.files_hashed % 100 == 0:
self.Log("Hashed %d files, skipped %s already stored.",
self.state.files_hashed, self.state.files_skipped)
def test_api_collection_fields(self):
yield self.render_resource(self.rsrc, b'/test?field=success&field=info')
self.assertRestCollection(typeName='tests',
items=[{'success': v['success'], 'info': v['info']}
for v in itervalues(endpoint.testData)],
total=8)
def added(self):
return itervalues(self._added)
def ReadFlowProcessingRequests(self):
"""Reads all flow processing requests from the database."""
return list(itervalues(self.flow_processing_requests))
def _get_videoids(videos):
"""Return a list of VideoId s for the videos"""
return [
common.VideoId.from_videolist_item(video)
for video in itervalues(videos)
]
def _get_titles(videos):
"""Return a list of videos' titles"""
return [
_get_title(video) for video in itervalues(videos) if _get_title(video)
]
def _process_changes(self, newRev, branch):
"""
Read changes since last change.
- Read list of commit hashes.
- Extract details from each commit.
- Add changes to database.
"""
# initial run, don't parse all history
if not self.lastRev:
return
if newRev in itervalues(self.lastRev):
# TODO: no new changes on this branch
# should we just use the lastRev again, but with a different branch?
pass
# get the change list
revListArgs = ([r'--format=%H', r'%s' % newRev] + [
r'^%s' % rev.encode('ascii', 'ignore')
for rev in itervalues(self.lastRev)
] + [r'--'])
self.changeCount = 0
results = yield self._dovccmd('log', revListArgs, path=self.workdir)
# process oldest change first
revList = results.split()
revList.reverse()
self.changeCount = len(revList)
self.lastRev[branch] = newRev
if self.changeCount:
log.msg(
'gitpoller: processing %d changes: %s from "%s" branch "%s"' %
(self.changeCount, revList, self.repourl, branch))
for rev in revList:
dl = defer.DeferredList([
self._get_commit_timestamp(rev),
self._get_commit_author(rev),
self._get_commit_files(rev),
self._get_commit_comments(rev),
],
consumeErrors=True)
results = yield dl
# check for failures
failures = [r[1] for r in results if not r[0]]
if failures:
# just fail on the first error; they're probably all related!
raise failures[0]
timestamp, author, files, comments = [r[1] for r in results]
yield self.master.data.updates.addChange(
author=author,
revision=ascii2unicode(rev),
files=files,
comments=comments,
when_timestamp=timestamp,
branch=ascii2unicode(self._removeHeads(branch)),
project=self.project,
repository=ascii2unicode(self.repourl),
category=self.category,
src=u'git')
def infer_spikes(self,
channel=0,
label=None,
gamma=None,
share_gamma=True,
mode='correct',
verbose=False):
"""Infer the most likely discretized spike train underlying a
fluorescence trace.
Parameters
----------
channel : string or int, optional
The channel to be used for spike inference.
label : string or None, optional
Text string indicating the signals from which spikes should be
inferred. Defaults to the most recently extracted signals.
gamma : float, optional
Gamma is 1 - timestep/tau, where tau is the time constant of the
AR(1) process. If no value is given, then gamma is estimated from
the data.
share_gamma : bool, optional
Whether to apply the same gamma estimate to all ROIs. Defaults to
True.
mode : {'correct', 'robust', 'psd'}, optional
The method for estimating sigma. The 'robust' method overestimates
the noise by assuming that gamma = 1. The 'psd' method estimates
sigma from the PSD of the fluorescence data. Default: 'correct'.
verbose : bool, optional
Whether to print status updates. Default: False.
Returns
-------
spikes : ndarray of float
The inferred normalized spike count at each time-bin. Values are
normalized to the maximum value over all time-bins.
Shape: (num_rois, num_timebins).
fits : ndarray of float
The inferred denoised fluorescence signal at each time-bin.
Shape: (num_rois, num_timebins).
parameters : dict of (str, ndarray of float)
Dictionary with values for 'sigma', 'gamma', and 'baseline'.
Notes
-----
We strongly recommend installing MOSEK (www.mosek.com; free for
academic use) which greatly speeds up the inference.
References
----------
* Pnevmatikakis et al. 2015. Submitted (arXiv:1409.2903).
* Machado et al. 2015. Submitted.
* Vogelstein et al. 2010. Journal of Neurophysiology. 104(6):
3691-3704.
"""
channel = self._resolve_channel(channel)
import sima.spikes
all_signals = self.signals(channel)
if label is None:
label = most_recent_key(all_signals)
signals = all_signals[label]
# estimate gamma for all cells
if mode == "psd":
if share_gamma:
mega_trace = np.concatenate(
[sigs for sigs in signals['raw'][0]])
sigma = sima.spikes.estimate_sigma(mega_trace)
gamma = sima.spikes.estimate_gamma(mega_trace, sigma)
sigma = [sigma for _ in signals['raw'][0]]
gamma = [gamma for _ in signals['raw'][0]]
else:
sigma = [
sima.spikes.estimate_sigma(sigs[0])
for sigs in signals['raw'][0]
]
gamma = [
sima.spikes.estimate_gamma(sigs[0], sigm)
for sigm, sigs in zip(sigma, signals['raw'][0])
]
else:
gamma = [
sima.spikes.estimate_parameters(sigs, gamma, sigma=0)[0]
for sigs in zip(*signals['raw'])
]
if share_gamma:
gamma = np.median(gamma)
# ensure that gamma is a list, one value per ROI
if isinstance(gamma, float):
gamma = [gamma for _ in signals['raw'][0]]
# estimate sigma values
sigma = [
sima.spikes.estimate_parameters(sigs, g)[1]
for g, sigs in zip(gamma, zip(*signals['raw']))
]
# perform spike inference
spikes, fits, parameters = [], [], []
for seq_idx, seq_signals in enumerate(signals['raw']):
spikes.append(np.zeros_like(seq_signals))
fits.append(np.zeros_like(seq_signals))
parameters.append(collections.defaultdict(list))
for i, trace in enumerate(seq_signals):
spikes[-1][i], fits[-1][i], p = sima.spikes.spike_inference(
trace, sigma[i], gamma[i], mode, verbose)
for k, v in iteritems(p):
parameters[-1][k].append(v)
for v in itervalues(parameters[-1]):
assert len(v) == len(spikes[-1])
parameters[-1] = dict(parameters[-1])
if self.savedir:
signals['spikes'] = spikes
signals['spikes_fits'] = fits
signals['spikes_params'] = parameters
all_signals[label] = signals
signals_filename = os.path.join(
self.savedir,
'signals_{}.pkl'.format(signals['signal_channel']))
pickle.dump(all_signals, open(signals_filename, 'wb'),
pickle.HIGHEST_PROTOCOL)
return spikes, fits, parameters
def itervalues(self):
return itervalues(self.dict)
def _validate_and_format_custom_entity(entity, utterance_entities, language,
builtin_entity_parser):
validate_type(entity, dict, object_label="entity")
# TODO: this is here temporarily, only to allow backward compatibility
if MATCHING_STRICTNESS not in entity:
strictness = entity.get("parser_threshold", 1.0)
entity[MATCHING_STRICTNESS] = strictness
mandatory_keys = [
USE_SYNONYMS, AUTOMATICALLY_EXTENSIBLE, DATA, MATCHING_STRICTNESS
]
validate_keys(entity, mandatory_keys, object_label="custom entity")
validate_type(entity[USE_SYNONYMS], bool, object_label="use_synonyms")
validate_type(entity[AUTOMATICALLY_EXTENSIBLE],
bool,
object_label="automatically_extensible")
validate_type(entity[DATA], list, object_label="entity data")
validate_type(entity[MATCHING_STRICTNESS], (float, int),
object_label="matching_strictness")
formatted_entity = dict()
formatted_entity[AUTOMATICALLY_EXTENSIBLE] = entity[
AUTOMATICALLY_EXTENSIBLE]
formatted_entity[MATCHING_STRICTNESS] = entity[MATCHING_STRICTNESS]
use_synonyms = entity[USE_SYNONYMS]
# Validate format and filter out unused data
valid_entity_data = []
for entry in entity[DATA]:
validate_type(entry, dict, object_label="entity entry")
validate_keys(entry, [VALUE, SYNONYMS], object_label="entity entry")
entry[VALUE] = entry[VALUE].strip()
if not entry[VALUE]:
continue
validate_type(entry[SYNONYMS], list, object_label="entity synonyms")
entry[SYNONYMS] = [s.strip() for s in entry[SYNONYMS] if s.strip()]
valid_entity_data.append(entry)
entity[DATA] = valid_entity_data
# Compute capitalization before normalizing
# Normalization lowercase and hence lead to bad capitalization calculation
formatted_entity[CAPITALIZE] = _has_any_capitalization(
utterance_entities, language)
validated_utterances = dict()
# Map original values an synonyms
for data in entity[DATA]:
ent_value = data[VALUE]
validated_utterances[ent_value] = ent_value
if use_synonyms:
for s in data[SYNONYMS]:
if s not in validated_utterances:
validated_utterances[s] = ent_value
# Number variations in entities values are expensive since each entity
# value is parsed with the builtin entity parser before creating the
# variations. We avoid generating these variations if there's enough entity
# values
# Add variations if not colliding
all_original_values = _extract_entity_values(entity)
if len(entity[DATA]) < VARIATIONS_GENERATION_THRESHOLD:
variations_args = {"case": True, "and_": True, "punctuation": True}
else:
variations_args = {"case": False, "and_": False, "punctuation": False}
variations_args["numbers"] = len(
entity[DATA]) < NUMBER_VARIATIONS_THRESHOLD
variations = dict()
for data in entity[DATA]:
ent_value = data[VALUE]
values_to_variate = {ent_value}
if use_synonyms:
values_to_variate.update(set(data[SYNONYMS]))
variations[ent_value] = set(
v for value in values_to_variate for v in get_string_variations(
value, language, builtin_entity_parser, **variations_args))
variation_counter = Counter(
[v for variations_ in itervalues(variations) for v in variations_])
non_colliding_variations = {
value: [
v for v in variations
if v not in all_original_values and variation_counter[v] == 1
]
for value, variations in iteritems(variations)
}
for entry in entity[DATA]:
entry_value = entry[VALUE]
validated_utterances = _add_entity_variations(
validated_utterances, non_colliding_variations, entry_value)
# Merge utterances entities
utterance_entities_variations = {
ent: get_string_variations(ent, language, builtin_entity_parser,
**variations_args)
for ent in utterance_entities
}
for original_ent, variations in iteritems(utterance_entities_variations):
if not original_ent or original_ent in validated_utterances:
continue
validated_utterances[original_ent] = original_ent
for variation in variations:
if variation and variation not in validated_utterances \
and variation not in utterance_entities:
validated_utterances[variation] = original_ent
formatted_entity[UTTERANCES] = validated_utterances
return formatted_entity
def building(self):
# A LatentWorkerForBuilder will only be busy if it is building.
return {
wfb
for wfb in itervalues(self.workerforbuilders) if wfb.isBusy()
}
def Flush(self):
super(OutputPluginWithOutputStreams, self).Flush()
for stream in itervalues(self.stream_objects):
stream.Flush()
def get(self, run, subsystem, filename=None):
# TODO: Validate
responseHeaders = {}
responseHeaders["run"] = run
responseHeaders["subsystem"] = subsystem
responseHeaders["filenames"] = []
# Return the filename for the particular run
subsystemContainer = db["runs"]["Run{0}".format(
run)].subsystems[subsystem]
# Handle special cases
if not filename:
# Return the available files
responseHeaders["filenames"] = [
tempFile.filename.split("/")[-1]
for tempFile in subsystemContainer.files.values()
]
return responseForSendingFile(additionalHeaders=responseHeaders)
elif filename == "combined":
# Return the combined file
responseHeaders["filenames"].append(
os.path.join(apiParameters["dirPrefix"],
subsystemContainer.combinedFile.filename))
response = responseForSendingFile(
filename=subsystemContainer.combinedFile.filename,
additionalHeaders=responseHeaders)
print("response: {}".format(response))
return response
#filename = secure_filename(filename)
# Look for the file
print(next(itervalues(subsystemContainer.files)).filename)
try:
requestedFile = next(
fileContainer
for fileContainer in subsystemContainer.files.values()
if fileContainer.filename.split("/")[-1] == filename)
except StopIteration as e:
print("Stop iteration error!")
response = responseForSendingFile(
additionalHeaders=responseHeaders)
response.headers[
"error"] = "Could not find requested file {0}".format(filename)
response.status_code = 404
return response
print("filename for requested file: {}".format(
os.path.join(apiParameters["dirPrefix"], requestedFile.filename)))
responseHeaders["filenames"].append(
os.path.join(apiParameters["dirPrefix"], requestedFile.filename))
#f = open(os.path.join(apiParameters["dirPrefix"], requestedFile.filename))
#return make_response(send_file(f, as_attachment = True, attachment_filename = filename))
#with open(os.path.join(apiParameters["dirPrefix"], requestedFile.filename)) as f:
with openFile(requestedFile.filename, "rb") as f:
# If StringIO is not used here then the file will go out of scope and be closed before the
# response is completed, which leads to "ValueError: I/O operation on closed file".
# I cannot seem to find a way around this. However, the file cannot just be opened, as otherwise
# it will leak memory.
# For more, see: https://stackoverflow.com/q/13344538 (no definitive solution)
# https://stackoverflow.com/a/25150805 (gave me the idea to just create a new StringIO)
# StringIO was selected based on https://stackoverflow.com/a/37463095
return make_response(
send_file(StringIO.StringIO(f.read()),
as_attachment=True,
attachment_filename=filename))
def thd(conn):
workers_tbl = self.db.model.workers
conn_tbl = self.db.model.connected_workers
cfg_tbl = self.db.model.configured_workers
bm_tbl = self.db.model.builder_masters
def selectWorker(q):
return q
# first, get the worker itself and the configured_on info
j = workers_tbl
j = j.outerjoin(cfg_tbl)
j = j.outerjoin(bm_tbl)
q = sa.select([
workers_tbl.c.id, workers_tbl.c.name, workers_tbl.c.info,
bm_tbl.c.builderid, bm_tbl.c.masterid
],
from_obj=[j],
order_by=[workers_tbl.c.id])
if _workerid is not None:
q = q.where(workers_tbl.c.id == _workerid)
if _name is not None:
q = q.where(workers_tbl.c.name == _name)
if masterid is not None:
q = q.where(bm_tbl.c.masterid == masterid)
if builderid is not None:
q = q.where(bm_tbl.c.builderid == builderid)
rv = {}
res = None
lastId = None
cfgs = None
for row in conn.execute(q):
if row.id != lastId:
lastId = row.id
cfgs = []
res = {
'id': lastId,
'name': row.name,
'configured_on': cfgs,
'connected_to': [],
'workerinfo': row.info
}
rv[lastId] = res
if row.builderid and row.masterid:
cfgs.append({
'builderid': row.builderid,
'masterid': row.masterid
})
# now go back and get the connection info for the same set of
# workers
j = conn_tbl
if _name is not None:
# note this is not an outer join; if there are unconnected
# workers, they were captured in rv above
j = j.join(workers_tbl)
q = sa.select([conn_tbl.c.workerid, conn_tbl.c.masterid],
from_obj=[j],
order_by=[conn_tbl.c.workerid])
if _workerid is not None:
q = q.where(conn_tbl.c.workerid == _workerid)
if _name is not None:
q = q.where(workers_tbl.c.name == _name)
if masterid is not None:
q = q.where(conn_tbl.c.masterid == masterid)
for row in conn.execute(q):
id = row.workerid
if id not in rv:
continue
rv[row.workerid]['connected_to'].append(row.masterid)
return list(itervalues(rv))
def getConfigDict(self):
return {
'name': self.name,
'childs':
[v.getConfigDict() for v in itervalues(self.namedServices)]
}
def _CheckAllRunning(self, action):
solutioncompleters = itervalues(self._completer_per_solution)
return all(
action(completer) for completer in solutioncompleters
if completer._ServerIsRunning())
def VerifierClassesForPlugin(cls, plugin_name):
if not plugin_name:
return []
return [c for c in itervalues(cls.classes) if c.plugin_name == plugin_name]
def Flush(self):
"""Writes the changes in this object to the datastore."""
if data_store.RelationalDBReadEnabled(category="message_handlers"):
message_handler_requests = []
leftover_responses = []
for r, timestamp in self.response_queue:
if r.request_id == 0 and r.session_id in session_id_map:
message_handler_requests.append(
rdf_objects.MessageHandlerRequest(
client_id=r.source and r.source.Basename(),
handler_name=session_id_map[r.session_id],
request_id=r.response_id,
request=r.payload))
else:
leftover_responses.append((r, timestamp))
if message_handler_requests:
data_store.REL_DB.WriteMessageHandlerRequests(
message_handler_requests)
self.response_queue = leftover_responses
self.data_store.StoreRequestsAndResponses(
new_requests=self.request_queue,
new_responses=self.response_queue,
requests_to_delete=self.requests_to_delete)
# We need to make sure that notifications are written after the requests so
# we flush after writing all requests and only notify afterwards.
mutation_pool = self.data_store.GetMutationPool()
with mutation_pool:
if data_store.RelationalDBReadEnabled(category="client_messages"):
if self.client_messages_to_delete:
data_store.REL_DB.DeleteClientMessages(
list(itervalues(self.client_messages_to_delete)))
else:
messages_by_queue = utils.GroupBy(
list(itervalues(self.client_messages_to_delete)),
lambda request: request.queue)
for queue, messages in iteritems(messages_by_queue):
self.Delete(queue, messages, mutation_pool=mutation_pool)
if self.new_client_messages:
for timestamp, messages in iteritems(
utils.GroupBy(self.new_client_messages,
lambda x: x[1])):
self.Schedule([x[0] for x in messages],
timestamp=timestamp,
mutation_pool=mutation_pool)
if self.notifications:
for notification in itervalues(self.notifications):
self.NotifyQueue(notification, mutation_pool=mutation_pool)
mutation_pool.Flush()
self.request_queue = []
self.response_queue = []
self.requests_to_delete = []
self.client_messages_to_delete = {}
self.notifications = {}
self.new_client_messages = []
def Shutdown(self):
if self.user_options['auto_stop_csharp_server']:
for solutioncompleter in itervalues(self._completer_per_solution):
solutioncompleter._StopServer()
def _process_changes(self, newRev, branch):
"""
Read changes since last change.
- Read list of commit hashes.
- Extract details from each commit.
- Add changes to database.
"""
# initial run, don't parse all history
if not self.lastRev:
return
rebuild = False
if newRev in itervalues(self.lastRev):
if self.buildPushesWithNoCommits and \
branch not in iterkeys(self.lastRev):
# we know the newRev but not for this branch
log.msg('gitpoller: rebuilding %s for new branch "%s"' %
(newRev, branch))
rebuild = True
# get the change list
revListArgs = ([r'--format=%H', r'%s' % newRev] + [
r'^%s' % rev.encode('ascii', 'ignore')
for rev in itervalues(self.lastRev)
] + [r'--'])
self.changeCount = 0
results = yield self._dovccmd('log', revListArgs, path=self.workdir)
# process oldest change first
revList = results.split()
revList.reverse()
if rebuild and len(revList) == 0:
revList = [newRev]
self.changeCount = len(revList)
self.lastRev[branch] = newRev
if self.changeCount:
log.msg(
'gitpoller: processing %d changes: %s from "%s" branch "%s"' %
(self.changeCount, revList, self.repourl, branch))
for rev in revList:
dl = defer.DeferredList([
self._get_commit_timestamp(rev),
self._get_commit_author(rev),
self._get_commit_files(rev),
self._get_commit_comments(rev),
],
consumeErrors=True)
results = yield dl
# check for failures
failures = [r[1] for r in results if not r[0]]
if failures:
for failure in failures:
log.error(
failure, "while processing changes for {} {}".format(
newRev, branch))
# just fail on the first error; they're probably all related!
failures[0].raiseException()
timestamp, author, files, comments = [r[1] for r in results]
yield self.master.data.updates.addChange(
author=author,
revision=ascii2unicode(rev),
files=files,
comments=comments,
when_timestamp=timestamp,
branch=ascii2unicode(self._removeHeads(branch)),
project=self.project,
repository=ascii2unicode(self.repourl),
category=self.category,
src=u'git')
def check(rc):
self.assertEqual(rc, 0)
self.assertEqual(calls, functions)
for repl in itervalues(repls):
repl.assert_called_with(config)
def expectedBuildTime(self):
if None in list(itervalues(self.times)):
return None
return sum(list(itervalues(self.times)))
def test_api_collection_order_desc(self):
yield self.render_resource(self.rsrc, b'/test?order=-info')
self.assertRestCollection(typeName='tests',
items=sorted(list(itervalues(endpoint.testData)),
key=lambda v: v['info'], reverse=True),
total=8, orderSignificant=True)
def __iter__(self):
for value in itervalues(self._backing):
yield value
def test_api_collection(self):
yield self.render_resource(self.rsrc, b'/test')
self.assertRestCollection(typeName='tests',
items=list(itervalues(endpoint.testData)),
total=8)
