def __init__(self, aws_access_key=None, aws_secret_key=None, s3_bucket=None, s3_filename=None, s3_region_endpoint=None, make_public=False, content_type=None, *args, **kwargs): super(CreateS3Report, self).__init__(*args, **kwargs) self.aws_access_key = aws_access_key or self.aws_access_key self.aws_secret_key = aws_secret_key or self.aws_secret_key self.s3_bucket = s3_bucket or self.s3_bucket self.s3_region_endpoint = s3_region_endpoint or self.s3_region_endpoint self.s3_filename = s3_filename or self.s3_filename self.make_public = make_public or self.make_public self.content_type = content_type or self.content_type if not self.aws_access_key: raise NotConfigured("You must provide the AWS Access Key.") if not self.aws_secret_key: raise NotConfigured("You must provide the AWS Secret Key.") if not self.s3_bucket: raise NotConfigured("You must define the s3 bucket.") if not self.s3_filename: raise NotConfigured("You must define the s3 filename.")
def __init__(self, sender_token=None, sender_name=None): sender_token = sender_token or self.sender_token if not sender_token: raise NotConfigured("You must provide a slack token.") self.sender_name = sender_name or self.sender_name if not self.sender_name: raise NotConfigured("You must provide a slack sender name.") self._client = SlackClient(sender_token) self._users = None
def __init__(self, aws_access_key=None, aws_secret_key=None, *args, **kwargs): super(SendSESEmail, self).__init__(*args, **kwargs) self.aws_access_key = aws_access_key or self.aws_access_key self.aws_secret_key = aws_secret_key or self.aws_secret_key if not self.fake and not self.aws_access_key: raise NotConfigured("You must provide the AWS Access Key.") if not self.fake and not self.aws_secret_key: raise NotConfigured("You must provide the AWS Secret Key.")
def __init__( self, sender_token=None, sender_name=None, recipients=None, message=None, message_template=None, include_message=None, attachments=None, attachments_template=None, include_attachments=None, fake=None, ): super(SendSlackMessage, self).__init__() self.fake = fake or self.fake self.manager = SlackMessageManager( sender_token=sender_token or self.sender_token, sender_name=sender_name or self.sender_name, fake=self.fake, ) self.recipients = recipients or self.recipients self.message = message or self.message self.message_template = message_template or self.message_template self.include_message = include_message or self.include_message self.attachments = attachments or self.attachments self.attachments_template = attachments_template or self.attachments_template self.include_attachments = include_attachments or self.include_attachments if not self.fake and not self.recipients: raise NotConfigured( "You must provide at least one recipient for the message.")
def __init__( self, aws_access_key=None, aws_secret_key=None, aws_region_name=None, *args, **kwargs ): super().__init__(*args, **kwargs) self.aws_access_key = aws_access_key or self.aws_access_key self.aws_secret_key = aws_secret_key or self.aws_secret_key self.aws_region_name = aws_region_name or self.aws_region_name if not self.fake and not self.aws_access_key: raise NotConfigured("You must provide the AWS Access Key.") if not self.fake and not self.aws_secret_key: raise NotConfigured("You must provide the AWS Secret Key.")
def run_action(self): if self.tags: if not self.data.job: raise NotConfigured("Job not available!") job_metadata = self.data.job.metadata self.process_tags(job_metadata) job_metadata.save()
def run(self, result): self.minimum_threshold = self.crawler.settings.getint( SPIDERMON_MIN_ITEMS, 0) if not self.minimum_threshold: raise NotConfigured("You should specify a minimum number of items " "to check against.") return super(ItemCountMonitor, self).run(result)
def __init__(self, sender_token=None, fake=False): sender_token = sender_token or self.sender_token if not sender_token: raise NotConfigured("You must provide a telegram bot token.") self.fake = fake self._client = SimplyTelegramClient(sender_token)
def run(self, result): add_field_coverage_set = self.crawler.settings.getbool( "SPIDERMON_ADD_FIELD_COVERAGE", False) if not add_field_coverage_set: raise NotConfigured( "To enable field coverage monitor, set SPIDERMON_ADD_FIELD_COVERAGE=True in your project settings" ) return super(FieldCoverageMonitor, self).run(result)
def run(self, result): has_threshold_config = any([ hasattr(self, "threshold_setting"), hasattr(self, "get_threshold") ]) if not has_threshold_config: raise NotConfigured( f"{self.__class__.__name__} should include a a `threshold_setting` attribute " "to be configured in your project settings with the desired threshold " "or a `get_threshold` method that returns the desired threshold." ) if (hasattr(self, "threshold_setting") and self.threshold_setting not in self.crawler.settings.attributes): raise NotConfigured( f"Configure {self.threshold_setting} to your project" f"settings to use {self.monitor_name}.") return super().run(result)
def __init__( self, sentry_dsn=None, fake=None, sentry_log_level=None, project_name="", environment="", ): super(SendSentryMessage, self).__init__() self.fake = fake or self.fake self.sentry_log_level = sentry_log_level or self.sentry_log_level self.sentry_dsn = sentry_dsn or self.sentry_dsn self.project_name = project_name or self.project_name self.environment = environment or self.environment if not self.fake and not self.sentry_dsn: raise NotConfigured("Missing SPIDERMON_SENTRY_DSN setting") if not self.project_name: raise NotConfigured("Missing SPIDERMON_SENTRY_PROJECT_NAME setting")
def __init__( self, sender, to, cc=None, bcc=None, reply_to=None, subject=None, subject_template=None, body_text=None, body_text_template=None, body_html=None, body_html_template=None, fake=None, *args, **kwargs ): super(SendEmail, self).__init__(*args, **kwargs) self.sender = sender or self.sender self.subject = subject or self.subject self.subject_template = subject_template or self.subject_template self.to = to or self.to self.cc = cc or self.cc self.bcc = bcc or self.bcc self.reply_to = reply_to or self.reply_to self.body_text = body_text or self.body_text self.body_text_template = body_text_template or self.body_text_template self.body_html = body_html or self.body_html self.body_html_template = body_html_template or self.body_html_template self.fake = fake or self.fake if not self.fake and not self.to: raise NotConfigured( "You must provide at least one recipient for the message." ) if not self.subject: raise NotConfigured("You must provide a subject for the message.") if not ( self.body_text or body_text_template or body_html or self.body_html_template ): raise NotConfigured("You must provide a body for the message.")
def __init__( self, sender_token=None, recipients=None, message=None, message_template=None, fake=None, ): super(SendTelegramMessage, self).__init__() self.fake = fake or self.fake self.manager = TelegramMessageManager(sender_token=sender_token or self.sender_token, fake=self.fake) self.recipients = recipients or self.recipients self.message = message or self.message self.message_template = message_template or self.message_template if not self.recipients: raise NotConfigured( "You must provide at least one recipient for the message.")
def __init__(self, template=None, *args, **kwargs): super(CreateReport, self).__init__(*args, **kwargs) self.template = template or self.template self.report = '' if not self.template: raise NotConfigured("You must define one template file.")
def spider(self): if not self.data.spider: raise NotConfigured('Spider not available!') return self.data.spider
def stats(self): if not self.data.stats: raise NotConfigured('Stats not available!') return self.data.stats
def job(self): if not self.data.job: raise NotConfigured('Job not available!') return self.data.job
def __getitem__(self, item): if item in self._notconfigured: raise NotConfigured("{} not available!".format(item)) return super(Context, self).__getitem__(item)
def __getitem__(self, item): if item in self._notconfigured: raise NotConfigured(f"{item} not available!") return super().__getitem__(item)
def __init__(self, filename, *args, **kwargs): super().__init__(*args, **kwargs) self.filename = filename or self.filename if not self.filename: raise NotConfigured("You must define a template output file.")
def crawler(self): if not self.data.crawler: raise NotConfigured('Crawler not available!') return self.data.crawler
def get_context_data(self): raise NotConfigured('Context data needs to be set up')