def _transform_mobile_image( self, original_image_blob: Blob, new_image_blob: Blob ) -> Optional[Blob]: """ Create smaller image size to be served on mobile devices. :param Blob original_image_blob: Original image blob. :param Blob new_image_blob: New newly created Blob for mobile image. :returns: Optional[Blob] """ img_meta = self._get_image_meta(original_image_blob) img_bytes = original_image_blob.download_as_bytes() if img_bytes: stream = BytesIO(img_bytes) im = Image.open(stream) try: with BytesIO() as output: new_image = im.reduce(2) new_image.save(output, format=img_meta["format"]) new_image_blob.upload_from_string( output.getvalue(), content_type=img_meta["content-type"] ) LOGGER.success(f"Created mobile image `{new_image_blob.name}`") return new_image_blob except GoogleCloudError as e: LOGGER.error( f"GoogleCloudError while saving mobile image `{new_image_blob.name}`: {e}" ) except Exception as e: LOGGER.error( f"Unexpected exception while saving mobile image `{new_image_blob.name}`: {e}" )
def update_post(self, post_id: str, body: dict, slug: str) -> Optional[dict]: """ Update post by ID. :param str post_id: Ghost post ID :param dict body: Payload containing post updates. :param str slug: Human-readable unique identifier. :returns: Optional[dict] """ try: resp = requests.put( f"{self.admin_api_url}/posts/{post_id}/", json=body, headers={ "Authorization": self.session_token, "Content-Type": "application/json", }, ) if resp.status_code != 200: LOGGER.success(f"Successfully updated post `{slug}`") return resp.json() except HTTPError as e: LOGGER.error(f"HTTPError while updating Ghost post: {e}") except Exception as e: LOGGER.error(f"Unexpected error while updating Ghost post: {e}")
def get_author(self, author_id: int) -> Optional[List[str]]: """ Fetch single Ghost author. :param int author_id: ID of Ghost author to fetch. :returns: Optional[List[str]] """ try: params = {"key": self.content_api_key} headers = { "Content-Type": "application/json", } resp = requests.get( f"{self.content_api_url}/authors/{author_id}/", params=params, headers=headers, ) if resp.status_code == 200: return resp.json()["authors"] except HTTPError as e: LOGGER.error( f"Failed to fetch Ghost authorID={author_id}: {e.response.content}" ) except KeyError as e: LOGGER.error( f"KeyError while fetching Ghost authorID={author_id}: {e}")
def get_json_backup(self) -> Optional[dict]: """ Download JSON snapshot of Ghost database. Optional[dict] """ self._https_session() headers = { "Authorization": self.session_token, "accept": "text/html,application/xhtml+xml,application/xml;\ q=0.9,image/webp,image/apng,*/*;\ q=0.8,application/signed-exchange;\ v=b3;q=0.9", "accept-encoding": "gzip, deflate, br", "Origin": "hackersandslackers.com", "Authority": "hackersandslackers.com", } endpoint = f"{self.admin_api_url}/db/" try: resp = requests.get(endpoint, headers=headers) return resp.json() except HTTPError as e: LOGGER.error(f"HTTPError occurred while fetching JSON backup: {e}") except Exception as e: LOGGER.error( f"Unexpected error occurred while fetching JSON backup: {e}")
def welcome_newsletter_subscriber( subscriber: Member, ) -> Optional[SubscriptionWelcomeEmail]: """ Send welcome email to newsletter subscriber. :param Member subscriber: New Ghost member with newsletter subscription. :returns: Optional[SubscriptionWelcomeEmail] """ body = { "from": settings.MAILGUN_FROM_SENDER, "to": [subscriber.email], "subject": settings.MAILGUN_SUBJECT_LINE, "template": settings.MAILGUN_NEWSLETTER_TEMPLATE, "h:X-Mailgun-Variables": { "name": subscriber.name }, "o:tracking": True, } response = mailgun.send_email(body) if response.status_code != 200: LOGGER.error(f"Mailgun failed to send welcome email: {body}") return None return SubscriptionWelcomeEmail( from_email=settings.MAILGUN_PERSONAL_EMAIL, to_email=subscriber.email, subject=settings.MAILGUN_SUBJECT_LINE, template=settings.MAILGUN_NEWSLETTER_TEMPLATE, )
def _get_ssh_key(self): """ Fetch locally stored SSH key.""" try: self.ssh_key = RSAKey.from_private_key_file(self.ssh_key_filepath) LOGGER.info(f"Found SSH key at self {self.ssh_key_filepath}") return self.ssh_key except SSHException as e: LOGGER.error(e)
def _upload_ssh_key(self): try: system( f"ssh-copy-id -i {self.ssh_key_filepath}.pub {self.user}@{self.host}>/dev/null 2>&1" ) LOGGER.info(f"{self.ssh_key_filepath} uploaded to {self.host}") except FileNotFoundError as error: LOGGER.error(error)
def _upload_ssh_key(self): try: system( f"ssh-copy-id -i {self.ssh_key_filepath}.pub {self.user}@{self.host}>/dev/null 2>&1" ) LOGGER.info(f"{self.ssh_key_filepath} uploaded to {self.host}") except FileNotFoundError as error: LOGGER.error(error) except Exception as e: LOGGER.error(f"Unexpected error occurred: {e}") raise e
def get_pages(self) -> Optional[dict]: """ Fetch Ghost pages. :returns: Optional[dict] """ try: headers = { "Authorization": f"Ghost {self.session_token}", "Content-Type": "application/json", } endpoint = f"{self.admin_api_url}/pages" resp = requests.get(endpoint, headers=headers) if resp.json().get("errors") is not None: LOGGER.error( f"Failed to fetch Ghost pages: {resp.json().get('errors')[0]['message']}" ) return None post = resp.json()["pages"] LOGGER.info(f"Fetched Ghost pages` ({endpoint})") return post except HTTPError as e: LOGGER.error(f"Ghost HTTPError while fetching pages: {e}") except KeyError as e: LOGGER.error(f"KeyError for `{e}` occurred while fetching pages") except Exception as e: LOGGER.error( f"Unexpected error occurred while fetching pages: {e}")
def run_experiment(predicted_results, settings, limit, predicted_tag_count): """ Run the experiment with configuration """ tags_info = settings["tags_info"] sample_count = config.CLASSIFIER["sample_count"] # predicted_tag_count = settings["predicted_tag_count"] LOGGER.debug("Sample count: %d" % sample_count) LOGGER.debug("Max predicted tag count: %d" % predicted_tag_count) get_similarity = settings["get_similarity"] # run the test for index, predict_result in enumerate(predicted_results): if index > limit: break try: LOGGER.debug("%d/%d sample" % (index, sample_count)) orignal, scored_predicted = predict_result # TODO: HARD CODED Code again. if settings["should_rerank"]: scored_predicted = rerank_tags(scored_predicted[:30], get_similarity) scored_predicted = scored_predicted[:predicted_tag_count] predicted = [t for t, s in scored_predicted] # TODO: SOME PROBLEM may raise here predicted = predicted[:predicted_tag_count] for name, evaluator in settings["evaluators"].items(): evaluation = evaluator.update(orignal, predicted) log_message = "\nOriginal Result: %s\n"\ "Predicted Result: %s\n"\ "Evaluator Type: %s\n"\ "\tPrecision: %f\n"\ "\tRecall: %f\n" % ( str(to_named_tags(orignal, tags_info)), str(to_named_tags(predicted, tags_info)), name, evaluation[0], evaluation[1]) LOGGER.debug(log_message) except Exception as e: LOGGER.error("Error occurs %s" % (str(e))) evaluations = [] for name, evaluator in settings["evaluators"].items(): evaluation = evaluator.get_evaluation() LOGGER.info("%s Precision: %f\t Recall: %f" % (name, evaluation[0], evaluation[1])) evaluations.append(evaluation) return evaluations
def fetch_top_visited_urls(time_period, limit=20) -> List[Optional[dict]]: """ Fetch top visited URLs from Plausible. :param str time_period: Period of 12mo, 6mo, month, 30d, 7d, or day. :param int limit: Maximum number of results to be returned. :returns: Optional[List[dict]] """ try: headers = { "Content-Type": "application/json", "Authorization": f"Bearer {settings.PLAUSIBLE_API_TOKEN}", } params = { "site_id": "hackersandslackers.com", "period": time_period, "property": "event:page", "limit": limit, } resp = requests.get( settings.PLAUSIBLE_BREAKDOWN_ENDPOINT, params=params, headers=headers, ) if resp.status_code == 200: results_list = resp.json().get("results") ghost_pages = [f"/{page['slug']}/" for page in ghost.get_pages()] results_list = [ result for result in results_list if "/tag" not in result["page"] and "/page" not in result["page"] and "/author" not in result["page"] and result["page"] not in ghost_pages ] return [ enrich_url_with_post_data(result) for result in results_list if result is not None ] return [] except RequestException as e: LOGGER.error(f"RequestException when fetching Plausible top URLs: {e}") except Exception as e: LOGGER.error( f"Unexpected Exception when fetching Plausible top URLs: {e}")
def connection(self): """Open connection to remote host. """ try: client = SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(AutoAddPolicy()) client.connect( self.host, username=self.user, password=self.password, key_filename=self.ssh_key_filepath, timeout=5000, ) return client except AuthenticationException as e: LOGGER.error( f"Authentication failed: did you remember to create an SSH key? {e}" ) raise e
def rebuild_netlify_site(self) -> Tuple[str, int]: """ Trigger Netlify site rebuild. :returns: Tuple[str, int] """ try: resp = requests.post(self.netlify_build_url, ) LOGGER.info( f"Triggered Netlify build with status code {resp.status_code}." ) return ( f"Triggered Netlify build with status code {resp.status_code}.", resp.status_code, ) except HTTPError as e: LOGGER.error( f"Failed to rebuild Netlify site: {e.response.content}") return e.response.content, e.response.status_code
def persist_algolia_searches(table_name: str, timeframe: int) -> List[Optional[dict]]: """ Fetch single week of searches from Algolia API. :param timeframe: Number of days for which to fetch recent search analytics. :param str table_name: DB table name :returns: List[Optional[dict]] """ try: headers = { "x-algolia-application-id": settings.ALGOLIA_APP_ID, "x-algolia-api-key": settings.ALGOLIA_API_KEY, } params = { "index": "hackers_posts", "limit": 999999, "orderBy": "searchCount", "direction": "desc", "startDate": get_start_date_range(timeframe), } resp = requests.get(settings.ALGOLIA_SEARCHES_ENDPOINT, headers=headers, params=params) if resp.status_code == 200 and resp.json().get("searches") is not None: search_queries = resp.json().get("searches") search_queries = filter_search_queries(search_queries) if search_queries is not None: import_algolia_search_queries(search_queries, table_name) return search_queries return [] return [] except HTTPError as e: LOGGER.error( f"HTTPError while fetching Algolia searches for `{timeframe}`: {e}" ) except Exception as e: LOGGER.error( f"Unexpected error while fetching Algolia searches for `{timeframe}`: {e}" )
def _retry(self, f): """ The query is executed multiple times to overcome any operational error. Database error is logged immediately.""" count = 0 while True: try: return f() # http://initd.org/psycopg/docs/module.html#psycopg2.DatabaseError # handle operational error - memory allocation, unexpected disconnect except psycopg2.OperationalError, oe: count += 1 if count < self._max_retries: LOGGER.warn("Transient Error Received %s ", oe) time.sleep(self._retry_period) else: LOGGER.error("Unrecoverable Error %s", oe) raise oe # other database errors - integrity, internal, programming error etc except psycopg2.DatabaseError, de: LOGGER.error("Database Error %s", de) raise de
def create_member(self, body: dict) -> Tuple[str, int]: """ Create new Ghost member account used to receive newsletters. :param dict body: Payload containing member information. :returns: Optional[List[str]] """ try: resp = requests.post( f"{self.admin_api_url}/members/", json=body, headers={"Authorization": self.session_token}, ) response = f'Successfully created new Ghost member `{body.get("email")}: {resp.json()}.' LOGGER.success(response) return response, resp.status_code except HTTPError as e: LOGGER.error( f"Failed to create Ghost member: {e.response.content}") return e.response.content, e.response.status_code
def get_all_posts(self) -> Optional[List[str]]: """ Fetch all Ghost post URLs. :returns: Optional[List[str]] """ try: headers = { "Authorization": f"Ghost {self.session_token}", "Content-Type": "application/json", } params = { "limit": "200", } endpoint = f"{self.admin_api_url}/posts" resp = requests.get(endpoint, headers=headers, params=params) if resp.status_code == 200: posts = resp.json()["posts"] return [ post["url"].replace(".app", ".com") for post in posts if post["status"] == "published" ] except HTTPError as e: LOGGER.error(f"Ghost HTTPError while fetching posts: {e}") except KeyError as e: LOGGER.error(f"KeyError for `{e}` occurred while fetching posts") except Exception as e: LOGGER.error( f"Unexpected error occurred while fetching posts: {e}")
def remove_comment_upvote(db: Session, user_id: str, comment_id: int): """ Delete a record of a user's upvote for a given comment. :param Session db: ORM database session. :param str user_id: Primary key for account record. :param int comment_id: Unique ID of comment user attempted to upvote. :returns: CommentUpvote """ try: upvote = CommentUpvote(user_id=user_id, comment_id=comment_id) db.delete(upvote) db.commit() LOGGER.success( f"Removed upvote for comment `{comment_id}` from user `{user_id}`." ) except SQLAlchemyError as e: LOGGER.error( f"SQLAlchemyError while attempting to remove comment upvote: {e}") except IntegrityError as e: LOGGER.error( f"IntegrityError while attempting to remove comment upvote: {e}") except Exception as e: LOGGER.error( f"Unexpected error while attempting to remove comment upvote: {e}")
def create_donation(db: Session, donation: NewDonation) -> Donation: """ Create new BuyMeACoffee donation record. :param Session db: ORM database session. :param NewDonation donation: Donation schema object. :returns: Donation """ try: db_item = Donation( coffee_id=donation.coffee_id, email=donation.email, name=donation.name, count=donation.count, message=donation.message, link=donation.link, created_at=datetime.now(), ) db.add(db_item) db.commit() LOGGER.success( f"Successfully received donation: `{donation.count}` coffees from `{donation.name}`." ) return db_item except SQLAlchemyError as e: LOGGER.error(f"SQLAlchemyError while creating donation record: {e}") except IntegrityError as e: LOGGER.error(f"IntegrityError while creating donation record: {e}") except Exception as e: LOGGER.error(f"Unexpected error while creating donation record: {e}")
def get_all_authors(self) -> Optional[List[dict]]: """ Fetch all Ghost authors. :returns: Optional[List[dict]] """ try: params = {"key": self.content_api_key} headers = { "Authorization": f"Ghost {self.session_token}", "Content-Type": "application/json", } resp = requests.get(f"{self.admin_api_url}/users", params=params, headers=headers) if resp.status_code == 200: return resp.json().get("users") except HTTPError as e: LOGGER.error( f"Failed to fetch Ghost authors: {e.response.content}") except KeyError as e: LOGGER.error(f"KeyError while fetching Ghost authors: {e}")
def get_post(self, post_id: str) -> Optional[dict]: """ Fetch Ghost post by ID. :param str post_id: ID of post to fetch. :returns: Optional[dict] """ try: headers = { "Authorization": f"Ghost {self.session_token}", "Content-Type": "application/json", } params = { "include": "authors", "formats": "mobiledoc,html", } endpoint = f"{self.admin_api_url}/posts/{post_id}/" resp = requests.get(endpoint, headers=headers, params=params) if resp.json().get("errors") is not None: LOGGER.error( f"Failed to fetch post `{post_id}`: {resp.json().get('errors')[0]['message']}" ) return None elif resp.json().get("posts"): post = resp.json()["posts"][0] LOGGER.info( f"Fetched Ghost post `{post['slug']}` ({endpoint})") return post return None except HTTPError as e: LOGGER.error( f"Ghost HTTPError while fetching post `{post_id}`: {e}") except KeyError as e: LOGGER.error( f"KeyError for `{e}` occurred while fetching post `{post_id}`") except Exception as e: LOGGER.error( f"Unexpected error occurred while fetching post `{post_id}`: {e}" )
async def fetch_decoded_url(session, media_source: Dict[str, str], count: int, total_count: int, media_type: str) -> Dict[str, str]: """ Fetch URL and create a dictionary representing the resource. :param session: Async HTTP requests session. :type session: ClientSession :param media_source: Single resource to fetch. :type media_source: Dict[str, str] :param count: Current URL count. :type count: int :param total_count: Total URL count. :type total_count: int :param media_type: Type of media urls to generate. :type media_type: str :returns: Dict[str, str] """ try: async with session.post(media_source["url"]) as response: decoded_url = await response.text() if decoded_url == "": raise Exception( f"Decoded URL returned empty string; export may have expired." ) resource = {"url": decoded_url, "date": media_source["date"]} LOGGER.info( f"Decoded {count} of {total_count} {media_type}: {decoded_url}" ) return resource except InvalidURL as e: LOGGER.error( f"Unable to decode invalid URL `{media_source['date']}`: {e}") raise e except ClientError as e: LOGGER.error(f"Error while decoding URL `{media_source['date']}`: {e}") raise e except Exception as e: LOGGER.error(f"Unexpected error: {e}") raise e
async def fetch_snapchat_memory( session: ClientSession, memory: Dict[str, str], count: int, total_count: int, media_type: str, ): """ Download single media file and write to local directory. :param session: Async HTTP requests session. :type session: ClientSession :param memory: Resource URL to fetch with date as filename. :type memory: Dict[str, str] :param count: Current URL count. :type count: int :param total_count: Total URL count. :type total_count: int :param media_type: Type of media to fetch (photo or video). :type media_type: str """ filepath = f"{MEDIA_EXPORT_FILEPATH}/{media_type}/{memory['date']}{'.jpg' if media_type == 'photos' else '.mp4'}" try: async with session.get(memory["url"]) as response: if response.status == 200: data = await response.read() async with aiofiles.open(filepath, mode="wb+") as f: await f.write(data) LOGGER.info( f"Fetched {media_type} {count} of {total_count}: {memory['date']}" ) await f.close() except InvalidURL as e: LOGGER.error(f"Unable to decode invalid URL `{memory['url']}`: {e}") except ClientError as e: LOGGER.error(f"Error while decoding URL `{memory['url']}`: {e}") except Exception as e: LOGGER.error( f"Unexpected error while decoding URL `{memory['url']}`: {e}")
def create_comment(db: Session, comment: NewComment, user_role: Optional[str]) -> Comment: """ Create new user-submitted comment. :param Session db: ORM database session. :param NewComment comment: User comment object. :param Optional[str] user_role: Permissions of the comment author, if any. :returns: Comment """ try: LOGGER.info( f"Creating comment from {comment.user_email} on {comment.post_slug}..." ) new_comment = Comment( user_id=comment.user_id, user_name=comment.user_name, user_avatar=comment.user_avatar, user_email=comment.user_email, user_role=user_role, body=comment.body, post_slug=comment.post_slug, post_id=comment.post_id, created_at=datetime.now(), ) db.add(new_comment) db.commit() LOGGER.success( f"New comment created by user `{new_comment.user_name}` on post `{new_comment.post_slug}`" ) return new_comment except SQLAlchemyError as e: LOGGER.error(f"SQLAlchemyError while creating comment: {e}") except IntegrityError as e: LOGGER.error(f"IntegrityError while creating comment: {e}") except Exception as e: LOGGER.error(f"Unexpected error while creating comment: {e}")
def create_account(db: Session, account: NetlifyAccount) -> NetlifyAccount: """ Create new account record sourced from Netlify. :param Session db: ORM database session. :param account: User comment schema object. :param NetlifyAccount account: User account registered via Netlify. :returns: NetlifyAccount """ try: new_account = Account( id=account.id, full_name=account.user_metadata.full_name, avatar_url=account.user_metadata.avatar_url, email=account.email, role=account.role, provider=account.app_metadata.provider, created_at=datetime.now(), updated_at=datetime.now(), ) db.add(new_account) db.commit() LOGGER.success( f"New Netlify account created: `{account.user_metadata.full_name}`" ) return account except SQLAlchemyError as e: LOGGER.error( f"SQLAlchemyError while creating Netlify user account: {e}") except IntegrityError as e: LOGGER.error( f"IntegrityError while creating Netlify user account: {e}") except Exception as e: LOGGER.error( f"Unexpected error while creating Netlify user account: {e}")
if request and response: if request.method == 'GET' and '?' in request.url: # filter static URL if not static_reg.search(url): burp_traffic.append((request, response)) traffic_queue.put((request, response)) elif request.method == 'POST' and request.body: content_type = request.get_header('Content-Type') # save multipart if content_type and 'multipart/form-data; boundary=' in content_type: MULTIPART.append((request, response)) else: burp_traffic.append((request, response)) traffic_queue.put((request, response)) else: LOGGER.error('%s not exists!' % self.burp) @staticmethod def get_traffic_path(id): traffic_path = os.path.join(TRAFFIC_DIR, id + '.traffic') return traffic_path def get_render_task(self, url_list): render_task = [] i = len(url_list) k = 0 if i > self.process: j = i / self.process for i in range(self.process): if i == self.process - 1: urls = url_list[k:]
if file: with open(file) as f: scope_url = f.readline().strip() elif url: scope_url = url domain = get_domain_from_url(scope_url) if is_ip(scope_url): save_cookie_ip(args.cookie, domain) else: from cookie import save_cookie save_cookie(args.cookie, domain) if url or file or burp or args.id or args.filter: if args.id: id = args.id if not Engine.is_scanned(id): LOGGER.error('Task %s not found,exit.' % id) exit(0) else: id = gen_id() engine = Engine(id=id, url=url, file=file, burp=burp, process=num, browser=browser, coroutine=coroutine, filter=filter) try: result = engine.start() except KeyboardInterrupt, e: LOGGER.info(e)
def start(self): # check if traffic_path exists. if self.is_scanned(self.id): choice = raw_input( 'Task %s has been scanned, do you want to rescan?(Y/N)' % self.id) if choice == 'Y' or choice == 'y' or re.search( 'yes', choice, re.I): self.put_queue() self.send_end_sig() elif choice is 'N' or choice is 'n' or re.search( 'no', choice, re.I): exit(0) else: LOGGER.error('Incorrect choice.') exit(0) elif self.burp: self.put_burp_to_trafficqueue() self.send_end_sig() # save burp traffic if burp_traffic: self.save_traffic(burp_traffic, self.id) else: if self.url != '': url_list = [self.url] elif self.file: if os.path.exists(self.file): with open(self.file) as f: url_list = [] temp = [url.strip() for url in f.read().split('\n')] for i in temp: if i: url_list.append(i) if not self.file.endswith('.slice'): url_list = self.deduplicate(url_list) if self.filter: exit(0) # test 10000 urls # url_list = url_list[:100] else: LOGGER.error('%s not exists!' % self.file) exit(0) # decode url_list = self.urldecode(url_list) if self.browser: # render LOGGER.info('Start to request url with %s.' % self.browser) render_task = self.get_render_task(url_list) for i in render_task: i.start() for i in render_task: i.join() self.save_traffic(traffic_list, self.id) # put traffic tp queue for i in range(len(traffic_list)): request = traffic_list[i][0] response = traffic_list[i][1] traffic_queue.put((request, response)) self.send_end_sig() else: # traffic genetator LOGGER.info('Start to request url with urllib2.') traffic_maker = Traffic_generator(self.id, url_list, self.coroutine) traffic_maker.start() traffic_maker.join() self.put_queue() self.send_end_sig() # scan task = [Scan() for i in range(self.process)] for i in task: i.start() for i in task: i.join() # save reflect for analyzing self.save_reflect() if case_list: if self.browser: # verify Verify.verify_with_browser(self.browser, case_list, self.process) self.save_analysis() return openner_result else: # verify,async verify_result = Verify.verify_async(case_list, self.coroutine) self.save_analysis() return verify_result
def put_burp_to_trafficqueue(self): """ parse xxx.xml from burpsuite proxy. :return: """ if os.path.exists(self.burp): import base64 from xml.etree import cElementTree as ET from model import HttpRequest, HttpResponse with open(self.burp) as f: xmlstr = f.read() try: root = ET.fromstring(xmlstr) except cElementTree.ParseError, e: LOGGER.error('Parse burpsuite data error: ' + str(e)) exit(0) for child in root: if child.tag == 'item': req_headers = {} resp_headers = {} code = '' request, response = '', '' for child2 in child: if child2.tag == 'method': method = child2.text if child2.tag == 'url': url = child2.text # static url in burp if static_reg.search(url): break if child2.tag == 'status': code = child2.text if child2.tag == 'request': req_text = child2.text # base64 decode req_text = base64.b64decode(req_text) headers_list = req_text.split( '\r\n\r\n', 1)[0].split('\r\n')[1:] for header in headers_list: try: header_key, header_value = header.split( ': ')[0], header.split(': ')[1] if header_key not in req_headers.keys(): req_headers[header_key] = header_value # split header error except IndexError, e: LOGGER.warn(e) body = req_text.split('\r\n\r\n', 1)[1] request = HttpRequest(method, url, req_headers, body) if child2.tag == 'response': resp_text = child2.text # if response is not None if resp_text: # base64 decode resp_text = base64.b64decode(resp_text) reason = resp_text.split('\r\n')[0] headers_list = resp_text.split( '\r\n\r\n', 1)[0].split('\r\n')[1:] for header in headers_list: header_key, header_value = header.split( ': ')[0], header.split(': ')[1] if header_key not in resp_headers.keys(): resp_headers[header_key] = header_value data = resp_text.split('\r\n\r\n', 1)[1] response = HttpResponse( code, reason, resp_headers, data) if request and response: if request.method == 'GET' and '?' in request.url: # filter static URL if not static_reg.search(url): burp_traffic.append((request, response)) traffic_queue.put((request, response)) elif request.method == 'POST' and request.body: content_type = request.get_header('Content-Type') # save multipart if content_type and 'multipart/form-data; boundary=' in content_type: MULTIPART.append((request, response)) else: burp_traffic.append((request, response)) traffic_queue.put((request, response))
import simplejson as json from log import LOGGER # Set project base path BASE_DIR = path.abspath(path.dirname(__file__)) # Location of Snapchat data dump from https://accounts.snapchat.com/accounts/welcome SNAPCHAT_DATA_EXPORT = f"{BASE_DIR}/export" SNAPCHAT_MEMORIES_EXPORT = f"{SNAPCHAT_DATA_EXPORT}/json/memories_history.json" if path.exists(SNAPCHAT_MEMORIES_EXPORT): SNAPCHAT_MEMORIES_JSON = json.loads(open(SNAPCHAT_MEMORIES_EXPORT).read())[ "Saved Media" ] else: LOGGER.error(f"Snapchat data not found in `/export` folder.") raise Exception(f"Snapchat data not found in `/export` folder.") # Check if media URLs have been decoded and saved from a previous run SNAPCHAT_MEDIA_URLS = { "videos": None, "photos": None, } for k, v in SNAPCHAT_MEDIA_URLS.items(): if path.exists(f"{BASE_DIR}/urls/{k}.json"): decoded_urls = json.loads(open(f"{BASE_DIR}/urls/{k}.json").read()) if decoded_urls is not None and len(decoded_urls) > 0: SNAPCHAT_MEDIA_URLS[k] = decoded_urls