def _get_recent(self, limit=100):
"""Downloads a list of the most recent pastes - the amount is limited by the <limit> parameter"""
r = Request()
endpoint = "api_scraping.php"
api_url = "{0}/{1}?limit={2}".format(self.api_base_url, endpoint, limit)
try:
response_data = r.get(api_url)
self._check_error(response_data)
pastes_dict = json.loads(response_data)
pastes = []
# Loop through the response and create objects by the data
for paste in pastes_dict:
paste_obj = Paste(key=paste.get("key"),
title=paste.get("title"),
user=paste.get("user"),
size=paste.get("size"),
date=paste.get("date"),
expire=paste.get("expire"),
syntax=paste.get("syntax"),
scrape_url=paste.get("scrape_url"),
full_url=paste.get("full_url"))
pastes.append(paste_obj)
return pastes
except Exception as e:
self.logger.error(e)
return None
def perform(self, paste, analyzer_name=None, matches=None):
"""
Sends the event to the MISP instance.
:param paste: The paste passed by the ActionHandler
:param analyzer_name: The name of the analyzer which matched the paste
"""
# Call transformer to construct payload
event = self.transformer(paste, analyzer_name)
if self.attributes:
# Add extra attributes
event['Attributes'].extend(self.attributes)
data = json.dumps({"Event": event})
# Send event to MISP instance
r = Request()
r.headers = {
'Authorization': self.access_key,
'Accept': 'application/json',
'Content-Type': 'application/json'
}
res = r.post(self.url + "/events", data=data)
# Error handling
if not res:
self.logger.warning("Empty response when adding event")
else:
res = json.loads(res)
if 'Event' in res:
self.logger.info('Event #%s successfully added to MISP',
res['Event']['id'])
else:
# An error has happened, but the 'errors' field is not always present
if 'errors' in res:
self.logger.error('Error when adding event: %s',
res['errors'])
self.logger.warning('Failed to add event: %s',
res.get('message'))
def perform(self, paste, analyzer_name=None):
"""Send a message via Discord to a specified channel, without checking for errors"""
r = Request()
if self.template is None:
text = "New paste matched by analyzer '{0}' - Link: {1}".format(
analyzer_name, paste.full_url)
else:
paste_dict = paste.to_dict()
paste_dict["analyzer_name"] = analyzer_name
text = self.template.safe_substitute(DictWrapper(paste_dict))
if self.webhook is not None:
# Send to a webhook (no authentication)
url = self.webhook
else:
# Send through Discord bot API (header-based authentication)
url = 'https://discordapp.com/api/channels/{0}/messages'.format(
self.channel_id)
r.headers = {'Authorization': 'Bot {}'.format(self.token)}
res = r.post(url, {"content": text})
if res == "":
# If the response is empty, skip further execution
return
res = json.loads(res)
if res.get(
'code'
) == 40001 and self.bot_available and self.webhook is None and not self.identified:
# Unauthorized access, bot token hasn't been identified to Discord Gateway
self.logger.info('Accessing Discord Gateway to initialize token')
self.initialize_gateway()
# Retry action
self.perform(paste, analyzer_name=analyzer_name)
def perform(self, paste, analyzer_name=None, matches=None):
"""Send a message via Discord to a specified channel, without checking for errors"""
r = Request()
text = TemplatingEngine.fill_template(paste,
analyzer_name,
template_string=self.template,
matches=matches)
if self.webhook_url is not None:
# Send to a webhook (no authentication)
url = self.webhook_url
else:
# Send through Discord bot API (header-based authentication)
url = 'https://discordapp.com/api/channels/{0}/messages'.format(
self.channel_id)
r.headers = {'Authorization': 'Bot {}'.format(self.token)}
res = r.post(url, {"content": text})
if res == "":
# If the response is empty, skip further execution
return
res = json.loads(res)
if res.get(
'code'
) == 40001 and self.bot_available and self.webhook_url is None and not self.identified:
# Unauthorized access, bot token hasn't been identified to Discord Gateway
self.logger.info('Accessing Discord Gateway to initialize token')
self.initialize_gateway()
# Retry action
self.perform(paste, analyzer_name=analyzer_name)
def __init__(self,
database=None,
proxies=None,
store_all_pastes=True,
ip_version=None):
"""
Basic PastePwn object handling the connection to pastebin and all the analyzers and actions
:param database: Database object extending AbstractDB
:param proxies: Dict of proxies as defined in the requests documentation
:param store_all_pastes: Bool to decide if all pastes should be stored into the db
:param ip_version: The IP version pastepwn should use (4|6)
"""
self.logger = logging.getLogger(__name__)
self.is_idle = False
self.database = database
self.paste_queue = Queue()
self.action_queue = Queue()
self.error_handlers = []
self.onstart_handlers = []
self.__exception_event = Event()
self.__request = Request(proxies) # initialize singleton
# We are trying to enforce a certain version of the Internet Protocol
enforce_ip_version(ip_version)
# Usage of ipify to get the IP - Uses the X-Forwarded-For Header which might
# lead to issues with proxies
try:
ip = self.__request.get("https://api.ipify.org")
except Exception as e:
ip = None
self.logger.warning(
"Could not fetch public IP via ipify: {0}".format(e))
if ip:
self.logger.info("Your public IP is {0}".format(ip))
self.scraping_handler = ScrapingHandler(
paste_queue=self.paste_queue,
exception_event=self.__exception_event)
self.paste_dispatcher = PasteDispatcher(
paste_queue=self.paste_queue,
action_queue=self.action_queue,
exception_event=self.__exception_event)
self.action_handler = ActionHandler(
action_queue=self.action_queue,
exception_event=self.__exception_event)
if self.database is not None and store_all_pastes:
# Save every paste to the database with the AlwaysTrueAnalyzer
self.logger.info("Database provided! Storing pastes in there.")
database_action = DatabaseAction(self.database)
always_true = AlwaysTrueAnalyzer(database_action)
self.add_analyzer(always_true)
elif store_all_pastes:
self.logger.info("No database provided!")
else:
self.logger.info("Not storing all pastes!")
def perform(self, paste, analyzer_name=None):
"""Send a message via a Telegram bot to a specified user, without checking for errors"""
r = Request()
text = TemplatingEngine.fill_template(paste,
analyzer_name,
template_string=self.template)
api_url = "https://api.telegram.org/bot{0}/sendMessage?chat_id={1}&text={2}".format(
self.token, self.receiver, text)
r.get(api_url)
def perform(self, paste, analyzer_name=None):
"""
Trigger the webhook
:param paste: The paste passed by the ActionHandler
:param analyzer_name: The name of the analyzer which matched the paste
:return: None
"""
if self.post_data is None:
paste_dict = None
else:
paste_dict = paste.to_dict()
r = Request()
r.post(url=self.url, data=paste_dict)
def perform(self, paste, analyzer_name=None):
"""Send a message via a Telegram bot to a specified user, without checking for errors"""
r = Request()
if self.template is None:
text = "New paste matched by analyzer '{0}' - Link: {1}".format(
analyzer_name, paste.full_url)
else:
paste_dict = paste.to_dict()
paste_dict["analyzer_name"] = analyzer_name
text = self.template.safe_substitute(DictWrapper(paste_dict))
api_url = "https://api.telegram.org/bot{0}/sendMessage?chat_id={1}&text={2}".format(
self.token, self.receiver, text)
r.get(api_url)
def _get_paste_content(self, key):
"""Downloads the content of a certain paste"""
r = Request()
endpoint = "api_scrape_item.php"
api_url = "{0}/{1}?i={2}".format(self.api_base_url, endpoint, key)
self.logger.debug("Downloading paste {0}".format(key))
try:
response_data = r.get(api_url)
except Exception as e:
self.logger.error(e)
raise e
self._check_error(response_data, key)
return response_data
def initialize_gateway(self):
"""Initialize the bot token so Discord identifies it properly."""
if self.webhook is not None:
raise NotImplementedError(
'Gateway initialization is only necessary for bot accounts.')
# Call Get Gateway Bot to get the websocket URL
r = Request()
r.headers = {'Authorization': 'Bot {}'.format(self.token)}
res = json.loads(r.get('https://discordapp.com/api/gateway/bot'))
ws_url = res.get('url')
# Start websocket client
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(self._identify(ws_url))
self.identified = True
def perform(self, paste, analyzer_name=None, matches=None):
"""
Sends the event to the MISP instance.
:param paste: The paste passed by the ActionHandler
:param analyzer_name: The name of the analyzer which matched the paste
"""
# Call transformer to construct payload
event = self.transformer(paste, analyzer_name)
if self.attributes:
# Add extra attributes
event["Attributes"].extend(self.attributes)
data = json.dumps({"Event": event})
# Send event to MISP instance
r = Request()
r.headers = {
"Authorization": self.access_key,
"Accept": "application/json",
"Content-Type": "application/json"
}
events_url = "{0}/events".format(self.url)
res = r.post(events_url, data=data)
# Error handling
if not res:
self.logger.warning("Empty response when adding event")
return
res = json.loads(res)
if "Event" in res:
event = res.get("Event")
self.logger.info("Event #%s successfully added to MISP",
event.get("id"))
return
# An error has happened, but the 'errors' field is not always present
if "errors" in res:
self.logger.error("Error when adding event: %s", res.get("errors"))
self.logger.warning("Failed to add event: %s", res.get("message"))
class PastePwn(object):
"""Represents an instance of the pastepwn core module"""
def __init__(self, database=None, proxies=None, store_all_pastes=True):
"""
Basic PastePwn object handling the connection to pastebin and all the analyzers and actions
:param database: Database object extending AbstractDB
:param proxies: Dict of proxies as defined in the requests documentation
:param store_all_pastes: Bool to decide if all pastes should be stored into the db
"""
self.logger = logging.getLogger(__name__)
self.is_idle = False
self.database = database
self.paste_queue = Queue()
self.action_queue = Queue()
self.error_handlers = list()
self.onstart_handlers = list()
self.__exception_event = Event()
self.__request = Request(proxies) # initialize singleton
# Usage of ipify to get the IP - Uses the X-Forwarded-For Header which might
# lead to issues with proxies
try:
ip = self.__request.get("https://api.ipify.org")
self.logger.info("Your public IP is {0}".format(ip))
except Exception as e:
self.logger.warning(
"Could not fetch public IP via ipify: {0}".format(e))
self.scraping_handler = ScrapingHandler(
paste_queue=self.paste_queue,
exception_event=self.__exception_event)
self.paste_dispatcher = PasteDispatcher(
paste_queue=self.paste_queue,
action_queue=self.action_queue,
exception_event=self.__exception_event)
self.action_handler = ActionHandler(
action_queue=self.action_queue,
exception_event=self.__exception_event)
if self.database is not None and store_all_pastes:
# Save every paste to the database with the AlwaysTrueAnalyzer
self.logger.info("Database provided! Storing pastes in there.")
database_action = DatabaseAction(self.database)
always_true = AlwaysTrueAnalyzer(database_action)
self.add_analyzer(always_true)
elif store_all_pastes:
self.logger.info("No database provided!")
else:
self.logger.info("Not storing all pastes!")
def add_scraper(self, scraper, restart_scraping=False):
"""
Adds a scraper to the list of scrapers. Scraping handler must be restarted for this to take effect.
:param scraper: Instance of a BasicScraper
:param restart_scraping: Indicates if the scraping handler should be restarted. Not setting this option results in your scraper not being started.
:return: None
"""
scraper.init_exception_event(self.__exception_event)
self.scraping_handler.add_scraper(scraper, restart_scraping)
def add_analyzer(self, analyzer):
"""
Adds a new analyzer to the list of analyzers
:param analyzer: Instance of a BasicAnalyzer
:return: None
"""
self.paste_dispatcher.add_analyzer(analyzer)
def add_error_handler(self, error_handler):
"""
Adds an error handler which will be called when an error happens
:param error_handler: Callable to be called when an error happens
:return: None
"""
if not callable(error_handler):
self.logger.error(
"The error handler you passed is not a function!")
return
self.error_handlers.append(error_handler)
def add_onstart_handler(self, onstart_handler):
"""Add a function as onstart_handler"""
if not callable(onstart_handler):
self.logger.error(
"The onstart handler you passed is not a function!")
return
self.onstart_handlers.append(onstart_handler)
def start(self):
"""Starts the pastepwn instance"""
if self.__exception_event.is_set():
self.logger.error(
"An exception occured. Aborting the start of PastePwn!")
exit(1)
if len(self.scraping_handler.scrapers) == 0:
from pastepwn.scraping.pastebin import PastebinScraper
pastebinscraper = PastebinScraper()
self.add_scraper(pastebinscraper, True)
self.scraping_handler.start()
self.paste_dispatcher.start()
self.action_handler.start()
for onstart_handler in self.onstart_handlers:
try:
onstart_handler()
except Exception as e:
self.logger.error(
"Onstart handler %s failed with error: %s. Pastepwn is still running."
% (onstart_handler.__name__, e))
def stop(self):
"""Stops the pastepwn instance"""
self.scraping_handler.stop()
self.paste_dispatcher.stop()
self.action_handler.stop()
self.is_idle = False
def signal_handler(self, signum, frame):
"""Handler method to handle signals"""
self.is_idle = False
self.logger.info("Received signal {}, stopping...".format(signum))
self.stop()
def idle(self, stop_signals=(SIGINT, SIGTERM, SIGABRT)):
"""
Blocks until one of the signals are received and stops the updater.
Thanks to the python-telegram-bot developers - https://github.com/python-telegram-bot/python-telegram-bot/blob/2cde878d1e5e0bb552aaf41d5ab5df695ec4addb/telegram/ext/updater.py#L514-L529
:param stop_signals: The signals to which the code reacts to
"""
self.is_idle = True
self.logger.info("In Idle!")
for sig in stop_signals:
signal(sig, self.signal_handler)
while self.is_idle:
if self.__exception_event.is_set():
self.logger.warning(
"An exception occurred. Calling exception handlers and going down!"
)
for handler in self.error_handlers:
# call the error handlers in case of an exception
handler()
self.is_idle = False
self.stop()
return
sleep(1)
def perform(self, paste, analyzer_name=None):
"""Trigger the webhook"""
# TODO - More post options ([custom] body, template, choose between GET/POST etc.)
r = Request()
r.post(url=self.url)
