def __init__(self,
aws_access_key=None,
aws_secret_key=None,
s3_bucket=None,
s3_filename=None,
s3_region_endpoint=None,
make_public=False,
content_type=None,
*args,
**kwargs):
super(CreateS3Report, self).__init__(*args, **kwargs)
self.aws_access_key = aws_access_key or self.aws_access_key
self.aws_secret_key = aws_secret_key or self.aws_secret_key
self.s3_bucket = s3_bucket or self.s3_bucket
self.s3_region_endpoint = s3_region_endpoint or self.s3_region_endpoint
self.s3_filename = s3_filename or self.s3_filename
self.make_public = make_public or self.make_public
self.content_type = content_type or self.content_type
if not self.aws_access_key:
raise NotConfigured("You must provide the AWS Access Key.")
if not self.aws_secret_key:
raise NotConfigured("You must provide the AWS Secret Key.")
if not self.s3_bucket:
raise NotConfigured("You must define the s3 bucket.")
if not self.s3_filename:
raise NotConfigured("You must define the s3 filename.")
def __init__(self, sender_token=None, sender_name=None):
sender_token = sender_token or self.sender_token
if not sender_token:
raise NotConfigured("You must provide a slack token.")
self.sender_name = sender_name or self.sender_name
if not self.sender_name:
raise NotConfigured("You must provide a slack sender name.")
self._client = SlackClient(sender_token)
self._users = None
def __init__(self,
aws_access_key=None,
aws_secret_key=None,
*args,
**kwargs):
super(SendSESEmail, self).__init__(*args, **kwargs)
self.aws_access_key = aws_access_key or self.aws_access_key
self.aws_secret_key = aws_secret_key or self.aws_secret_key
if not self.fake and not self.aws_access_key:
raise NotConfigured("You must provide the AWS Access Key.")
if not self.fake and not self.aws_secret_key:
raise NotConfigured("You must provide the AWS Secret Key.")
def __init__(
self,
sender_token=None,
sender_name=None,
recipients=None,
message=None,
message_template=None,
include_message=None,
attachments=None,
attachments_template=None,
include_attachments=None,
fake=None,
):
super(SendSlackMessage, self).__init__()
self.fake = fake or self.fake
self.manager = SlackMessageManager(
sender_token=sender_token or self.sender_token,
sender_name=sender_name or self.sender_name,
fake=self.fake,
)
self.recipients = recipients or self.recipients
self.message = message or self.message
self.message_template = message_template or self.message_template
self.include_message = include_message or self.include_message
self.attachments = attachments or self.attachments
self.attachments_template = attachments_template or self.attachments_template
self.include_attachments = include_attachments or self.include_attachments
if not self.fake and not self.recipients:
raise NotConfigured(
"You must provide at least one recipient for the message.")
def __init__(
self,
aws_access_key=None,
aws_secret_key=None,
aws_region_name=None,
*args,
**kwargs
):
super().__init__(*args, **kwargs)
self.aws_access_key = aws_access_key or self.aws_access_key
self.aws_secret_key = aws_secret_key or self.aws_secret_key
self.aws_region_name = aws_region_name or self.aws_region_name
if not self.fake and not self.aws_access_key:
raise NotConfigured("You must provide the AWS Access Key.")
if not self.fake and not self.aws_secret_key:
raise NotConfigured("You must provide the AWS Secret Key.")
def run_action(self):
if self.tags:
if not self.data.job:
raise NotConfigured("Job not available!")
job_metadata = self.data.job.metadata
self.process_tags(job_metadata)
job_metadata.save()
def run(self, result):
self.minimum_threshold = self.crawler.settings.getint(
SPIDERMON_MIN_ITEMS, 0)
if not self.minimum_threshold:
raise NotConfigured("You should specify a minimum number of items "
"to check against.")
return super(ItemCountMonitor, self).run(result)
def __init__(self, sender_token=None, fake=False):
sender_token = sender_token or self.sender_token
if not sender_token:
raise NotConfigured("You must provide a telegram bot token.")
self.fake = fake
self._client = SimplyTelegramClient(sender_token)
def run(self, result):
add_field_coverage_set = self.crawler.settings.getbool(
"SPIDERMON_ADD_FIELD_COVERAGE", False)
if not add_field_coverage_set:
raise NotConfigured(
"To enable field coverage monitor, set SPIDERMON_ADD_FIELD_COVERAGE=True in your project settings"
)
return super(FieldCoverageMonitor, self).run(result)
def run(self, result):
has_threshold_config = any([
hasattr(self, "threshold_setting"),
hasattr(self, "get_threshold")
])
if not has_threshold_config:
raise NotConfigured(
f"{self.__class__.__name__} should include a a `threshold_setting` attribute "
"to be configured in your project settings with the desired threshold "
"or a `get_threshold` method that returns the desired threshold."
)
if (hasattr(self, "threshold_setting") and self.threshold_setting
not in self.crawler.settings.attributes):
raise NotConfigured(
f"Configure {self.threshold_setting} to your project"
f"settings to use {self.monitor_name}.")
return super().run(result)
def __init__(
self,
sentry_dsn=None,
fake=None,
sentry_log_level=None,
project_name="",
environment="",
):
super(SendSentryMessage, self).__init__()
self.fake = fake or self.fake
self.sentry_log_level = sentry_log_level or self.sentry_log_level
self.sentry_dsn = sentry_dsn or self.sentry_dsn
self.project_name = project_name or self.project_name
self.environment = environment or self.environment
if not self.fake and not self.sentry_dsn:
raise NotConfigured("Missing SPIDERMON_SENTRY_DSN setting")
if not self.project_name:
raise NotConfigured("Missing SPIDERMON_SENTRY_PROJECT_NAME setting")
def __init__(
self,
sender,
to,
cc=None,
bcc=None,
reply_to=None,
subject=None,
subject_template=None,
body_text=None,
body_text_template=None,
body_html=None,
body_html_template=None,
fake=None,
*args,
**kwargs
):
super(SendEmail, self).__init__(*args, **kwargs)
self.sender = sender or self.sender
self.subject = subject or self.subject
self.subject_template = subject_template or self.subject_template
self.to = to or self.to
self.cc = cc or self.cc
self.bcc = bcc or self.bcc
self.reply_to = reply_to or self.reply_to
self.body_text = body_text or self.body_text
self.body_text_template = body_text_template or self.body_text_template
self.body_html = body_html or self.body_html
self.body_html_template = body_html_template or self.body_html_template
self.fake = fake or self.fake
if not self.fake and not self.to:
raise NotConfigured(
"You must provide at least one recipient for the message."
)
if not self.subject:
raise NotConfigured("You must provide a subject for the message.")
if not (
self.body_text or body_text_template or body_html or self.body_html_template
):
raise NotConfigured("You must provide a body for the message.")
def __init__(
self,
sender_token=None,
recipients=None,
message=None,
message_template=None,
fake=None,
):
super(SendTelegramMessage, self).__init__()
self.fake = fake or self.fake
self.manager = TelegramMessageManager(sender_token=sender_token
or self.sender_token,
fake=self.fake)
self.recipients = recipients or self.recipients
self.message = message or self.message
self.message_template = message_template or self.message_template
if not self.recipients:
raise NotConfigured(
"You must provide at least one recipient for the message.")
def __init__(self, template=None, *args, **kwargs):
super(CreateReport, self).__init__(*args, **kwargs)
self.template = template or self.template
self.report = ''
if not self.template:
raise NotConfigured("You must define one template file.")
def spider(self):
if not self.data.spider:
raise NotConfigured('Spider not available!')
return self.data.spider
def stats(self):
if not self.data.stats:
raise NotConfigured('Stats not available!')
return self.data.stats
def job(self):
if not self.data.job:
raise NotConfigured('Job not available!')
return self.data.job
def __getitem__(self, item):
if item in self._notconfigured:
raise NotConfigured("{} not available!".format(item))
return super(Context, self).__getitem__(item)
def __getitem__(self, item):
if item in self._notconfigured:
raise NotConfigured(f"{item} not available!")
return super().__getitem__(item)
def __init__(self, filename, *args, **kwargs):
super().__init__(*args, **kwargs)
self.filename = filename or self.filename
if not self.filename:
raise NotConfigured("You must define a template output file.")
def crawler(self):
if not self.data.crawler:
raise NotConfigured('Crawler not available!')
return self.data.crawler
def get_context_data(self):
raise NotConfigured('Context data needs to be set up')
