def assign_score(cls, claim: ScorableClaim) -> float:
"""Calculate score for a given claim."""
if claim.report_type in ReportType.MUST_RUN_EVERY_SWEEP:
return MUST_RUN_SCORE
timer = Measure.timer(f'{__name__}.assign_score',
tags={
'entity_type': claim.entity_type,
'ad_account_id': claim.ad_account_id
},
sample_rate=0.01)
with timer:
hist_ratio = cls.historical_ratio(claim)
score_skew_ratio = cls.skew_ratio(claim)
account_skew = cls.account_skew(claim)
combined_ratio = hist_ratio * score_skew_ratio * account_skew
return int(MUST_RUN_SCORE * combined_ratio)
def iter_prioritized(
claims: Iterable[ScorableClaim]
) -> Generator[PrioritizationClaim, None, None]:
"""Assign score for each claim."""
_measurement_name_base = f'{__name__}.{iter_prioritized.__name__}'
_before_next_expectation = time.time()
for claim in claims:
_measurement_tags = {
'entity_type': claim.entity_type,
'ad_account_id': claim.ad_account_id
}
Measure.timing(f'{_measurement_name_base}.next_expected',
tags=_measurement_tags,
sample_rate=0.01)(
(time.time() - _before_next_expectation) * 1000)
try:
score = ScoreCalculator.assign_score(claim)
with Measure.timer(f'{_measurement_name_base}.yield_result',
tags=_measurement_tags):
yield PrioritizationClaim(
claim.entity_id,
claim.entity_type,
claim.report_type,
claim.job_signature,
score,
ad_account_id=claim.ad_account_id,
timezone=claim.timezone,
range_start=claim.range_start,
)
except ScoringException as e:
ErrorInspector.inspect(e, claim.ad_account_id,
{'job_id': claim.job_id})
_before_next_expectation = time.time()
def iter_persist_prioritized(
sweep_id: str, prioritized_iter: Iterable[PrioritizationClaim]
) -> Generator[PrioritizationClaim, None, None]:
"""Persist prioritized jobs and pass-through context objects for inspection."""
AccountCache.reset()
with SortedJobsQueue(sweep_id).JobsWriter() as add_to_queue:
_measurement_name_base = f'{__name__}.{iter_persist_prioritized.__name__}'
_before_next_prioritized = time.time()
for prioritization_claim in prioritized_iter:
job_type = detect_job_type(prioritization_claim.report_type,
prioritization_claim.entity_type)
_measurement_tags = {
'entity_type': prioritization_claim.entity_type,
'report_type': prioritization_claim.report_type,
'ad_account_id': prioritization_claim.ad_account_id,
'job_type': job_type,
'sweep_id': sweep_id,
}
Measure.timing(f'{_measurement_name_base}.next_prioritized',
tags=_measurement_tags,
sample_rate=0.01)(
(time.time() - _before_next_prioritized) * 1000)
score = prioritization_claim.score
if not should_persist(score):
logger.debug(
f'Not persisting job {prioritization_claim.job_id} due to low score: {score}'
)
continue
# Following are JobScope attributes we don't store on JobID
# so we need to store them separately.
# See JobScope object for exact attr names.
# At this point persister forms the
# auxiliary data blob for saving on Data Flower.
# We don't have to do that here.
# It can be pre-computed and placed on the JobSignature
# TODO: contemplate moving auxiliary data formation to
# place where JobSignatures are generated and use that
# data for Data Flower (as it was originally intended
# but not implemented because saving each job's data
# individually to Data Flower was too slow)
# So, here you would unpack
# **job_kwargs
# that you get from prioritization_claim.score_job_pairs
# ... Until then:
extra_data = {}
if prioritization_claim.timezone:
extra_data[
'ad_account_timezone_name'] = prioritization_claim.timezone
with Measure.timer(f'{_measurement_name_base}.add_to_queue',
tags=_measurement_tags):
if prioritization_claim.report_age_in_days is not None:
Measure.histogram(
f'{_measurement_name_base}.report_age',
tags=_measurement_tags)(
prioritization_claim.report_age_in_days)
add_to_queue(prioritization_claim.job_id, score, **extra_data)
# This time includes the time consumer of this generator wastes
# between reads from us. Good way to measure how quickly we are
# consumed (what pauses we have between each consumption)
with Measure.timer(f'{_measurement_name_base}.yield_result',
tags=_measurement_tags):
yield prioritization_claim
_before_next_prioritized = time.time()