def test_clear_attachment(): with open('test_messages/text_three_attachments.txt', 'r') as f: html_parts = [] text_parts = [] msg = message_from_file(f) generator = mime.get_attachment_parts(msg, text_parts, html_parts) truncated_filename = '1' * 124 + '.png' for attachment_part, file_name, c_id in generator: assert file_name in [truncated_filename, 'green-_ball.png', 'redball.png'] assert c_id is None mime.clear_attachment(attachment_part) assert len(html_parts) == 0 assert len(text_parts) == 1 with open('test_messages/message_write.txt', 'w+') as f: g = Generator(f) g.flatten(msg) with open('test_messages/message_write.txt', 'r') as f: html_parts = [] text_parts = [] msg = message_from_file(f) generator = mime.get_attachment_parts(msg, text_parts, html_parts) for attachment_part, file_name, c_id in generator: assert False assert len(html_parts) == 3 assert len(text_parts) == 1
def test_date(self): filename = os.path.join(self.tempdir, 'set_date.html') # First test a message without any date defined def file_without_date(): open(filename, 'w').write('title: Some title\n\nSome content') file_without_date() command_date([filename, '2008-1-1'], None) message = email.message_from_file(open(filename)) self.assert_('date' in message) self.assertEqual(message['date'], str(datetime.date(2008, 1, 1))) # Then test a file which has already a date def file_with_date(): open(filename, 'w').write('title: Some title\ndate: 2008-12-31\n' '\nSome content') file_with_date() command_date([filename, '2008-1-1'], None) message = email.message_from_file(open(filename)) self.assert_('date' in message) self.assertEqual(message['date'], str(datetime.date(2008, 1, 1))) # Test aliases for alias in ('now', 'today', 'tomorrow', 'next_day'): file_without_date() command_date([filename, alias], None) message = email.message_from_file(open(filename)) self.assert_('date' in message)
def get_direct_emails(emails, addresses): """Return emails directly sent to addresses in given emails""" direct = [] cced = [] for msg_path in emails: try: msg_f = open(msg_path, 'r', encoding='utf8') msg = email.message_from_file(msg_f) except UnicodeDecodeError: msg_f = open(msg_path, 'r', encoding='latin1') msg = email.message_from_file(msg_f) try: msg_to = get_header_field(msg, 'To') msg_cc = get_header_field(msg, 'Cc') recipients = msg_to + '\n' + msg_cc if any(e in recipients for e in addresses): if any(e in msg_to for e in addresses) and len(re.split(r'[\n,]', msg_to)) == 1: direct.append(msg_path) else: cced.append(msg_path) except: continue return (direct, cced)
def handle(self, *args, **options): infile = args[0] if args else options['infile'] if infile: logger.info('Parsing mail loaded by filename') if six.PY3: with open(infile, 'rb') as file_: mail = email.message_from_binary_file(file_) else: with open(infile) as file_: mail = email.message_from_file(file_) else: logger.info('Parsing mail loaded from stdin') if six.PY3: mail = email.message_from_binary_file(sys.stdin.buffer) else: mail = email.message_from_file(sys.stdin) try: result = parse_mail(mail, options['list_id']) if result: sys.exit(0) logger.warning('Failed to parse mail') sys.exit(1) except Exception: logger.exception('Error when parsing incoming email', extra={'mail': mail.as_string()})
def testEmailParsing(self): # create an initial "from" email message = email.message_from_file( open("./testdata/mail3.txt", "r")) name, mto = parseaddr(message['to']) name, mfrom = parseaddr(message['from']) for part in message.walk(): if part.get_content_maintype() == "multipart": continue # it's just a container if part.get_content_type() == "text/plain": body = part.get_payload(decode=True) break try: user = CustomUser.objects.get( email=mfrom) except CustomUser.DoesNotExist: user = CustomUser.objects.create(email=mfrom, username=mfrom) user.set_unusable_password() user.save() recipient, _ = CustomUser.objects.get_or_create( email=mto, username=mto) mail = Mail.objects.create( subject=message['subject'], mfrom=user, mto=recipient, message=body) # now we simulate a reply, and override a couple of relevant # headers raw_email = email.message_from_file( open("./testdata/mail4.txt", "r")) new_mto = user.proxy_email raw_email.replace_header('in-reply-to', "<%s>" % mail.message_id) raw_email.replace_header('references', "<%s>" % mail.message_id) raw_email.replace_header('to', "%s <%s>" % (mail.message_id, new_mto)) fp = StringIO(raw_email.as_string()) response = make_response_from_file(fp) self.assertTrue(response) # now parse in an email that isn't a response to anything fp = open("./testdata/mail2.txt", "r") response = make_response_from_file(fp) self.assertEqual(response, EX_NOUSER) # and now try threading based on subject line as a fallback... raw_email = email.message_from_file( open("./testdata/mail2.txt", "r")) new_mto = user.proxy_email del(raw_email['in-reply-to']) del(raw_email['references']) raw_email.replace_header('subject', "Re:RE: re:%s" % mail.subject) raw_email.replace_header('to', "%s <%s>" % (mail.message_id, new_mto)) fp = StringIO(raw_email.as_string()) response = make_response_from_file(fp) self.assertTrue(response)
def __init__(self, path): if hasattr(path, 'readlines'): self.msg = email.message_from_file(path) else: with open(path, 'r') as fip: self.msg = email.message_from_file(fip) self.parts = [] self.headers = {} self.attachments = []
def setUp(self): if sys.version_info >= (3, 2): import io msg1 = email.message_from_file(io.open(join(DATA, 'test1.msg'), encoding='utf8')) msg2 = email.message_from_file(io.open(join(DATA, 'test2.msg'), encoding='utf8')) else: msg1 = email.message_from_file(open(join(DATA, 'test1.msg'))) msg2 = email.message_from_file(open(join(DATA, 'test2.msg'))) self.umessage1 = UMessage(msg1) self.umessage2 = UMessage(msg2)
def test_date_empty(self): filename = os.path.join(self.tempdir, 'empty_date.html') open(filename, 'w').write('title: title\n\ncontent') command_date([filename]) message = email.message_from_file(open(filename)) self.assert_('date' in message) self.assert_(message['date'] >= str(datetime.date.today())) open(filename, 'w').write('title: title\ndate: 2010-2-4\n\ncontent') command_date([filename]) message = email.message_from_file(open(filename)) self.assert_('date' in message) self.assert_(message['date'] >= str(datetime.date.today()))
def main(args): django.setup() logger = setup_error_handler() parser = argparse.ArgumentParser() parse_lock = None def list_logging_levels(): """Give a summary of all available logging levels.""" return sorted(list(VERBOSITY_LEVELS.keys()), key=lambda x: VERBOSITY_LEVELS[x]) parser.add_argument('--verbosity', choices=list_logging_levels(), help='logging level', default='info') args = vars(parser.parse_args()) logging.basicConfig(level=VERBOSITY_LEVELS[args['verbosity']]) mail = message_from_file(sys.stdin) try: parse_lock = lock() return parse_mail(mail) except: if logger: logger.exception('Error when parsing incoming email', extra={ 'mail': mail.as_string(), }) raise finally: release(parse_lock)
def build_attachments(self): """Build email's attachment messages""" attachments = [] for attachment in self.newsletter.attachment_set.all(): ctype, encoding = mimetypes.guess_type(attachment.file_attachment.path) if ctype is None or encoding is not None: ctype = 'application/octet-stream' maintype, subtype = ctype.split('/', 1) fd = open(attachment.file_attachment.path, 'rb') if maintype == 'text': message_attachment = MIMEText(fd.read(), _subtype=subtype) elif maintype == 'message': message_attachment = message_from_file(fd) elif maintype == 'image': message_attachment = MIMEImage(fd.read(), _subtype=subtype) elif maintype == 'audio': message_attachment = MIMEAudio(fd.read(), _subtype=subtype) else: message_attachment = MIMEBase(maintype, subtype) message_attachment.set_payload(fd.read()) encode_base64(message_attachment) fd.close() message_attachment.add_header('Content-Disposition', 'attachment', filename=attachment.title) attachments.append(message_attachment) return attachments
def test_file_sessions(self): """Make sure opening a connection creates a new file""" msg = EmailMessage('Subject', 'Content', '*****@*****.**', ['*****@*****.**'], headers={'From': '*****@*****.**'}) connection = mail.get_connection() connection.send_messages([msg]) self.assertEqual(len(os.listdir(self.tmp_dir)), 1) with open(os.path.join(self.tmp_dir, os.listdir(self.tmp_dir)[0])) as fp: message = email.message_from_file(fp) self.assertEqual(message.get_content_type(), 'text/plain') self.assertEqual(message.get('subject'), 'Subject') self.assertEqual(message.get('from'), '*****@*****.**') self.assertEqual(message.get('to'), '*****@*****.**') connection2 = mail.get_connection() connection2.send_messages([msg]) self.assertEqual(len(os.listdir(self.tmp_dir)), 2) connection.send_messages([msg]) self.assertEqual(len(os.listdir(self.tmp_dir)), 2) msg.connection = mail.get_connection() self.assertTrue(connection.open()) msg.send() self.assertEqual(len(os.listdir(self.tmp_dir)), 3) msg.send() self.assertEqual(len(os.listdir(self.tmp_dir)), 3)
def page_for_message(mlist, year, month, day, message): with open(message, "r") as fd: mail = email.message_from_file(fd) ## Headers # ['MIME-Version', 'Date', 'From', 'In-Reply-To', 'List-Id', # 'Message-Id', 'Precedence', 'References', 'Reply-To', # 'Return-Path', 'Sender', 'Subject', 'To', 'Content-Type', # 'Content-Transfer-Encoding'] data = dict(mail) body = None if mail.is_multipart(): body = mail.get_payload(0).get_payload() else: body = mail.get_payload(decode=True) data.update( { "Body": body, "List": mlist, "Year": year, "Month": month, "Day": day, ## Some older messages lack a `Date` "Date": data.get("Date", "Unknown"), } ) tmpl = Message.Message(searchList=[data]) return (unicode(tmpl), data)
def test_remove_next_part_from_content(self): with open(get_test_file("pipermail_nextpart.txt")) as email_file: msg = message_from_file(email_file) scrubber = Scrubber("*****@*****.**", msg) contents = scrubber.scrub()[0] self.failIf("-------------- next part --------------" in contents)
def read(self, length=None): try: return File.read(self,length) except IOError: pass if self.readptr > 0: return '' self.fd.seek(0) a=email.message_from_file(self.fd) my_part = self.inode.split('|')[-1] attachment_number = int(my_part[1:]) #print "attchement number %s" % attachment_number count = 0 for part in a.walk(): if part.get_content_maintype() == 'multipart': continue if count==attachment_number: self.message = part.get_payload(decode=1) self.readptr += len(self.message) #print "Returning %s" % part.get_payload() return self.message count+=1 return ''
def parseEmail(emailFile): escon = es.ESload() start = time.time() f = open(emailFile, 'r') msg = email.message_from_file(f) f.close() env = open(emailFile + '.envelope', 'r') envelope = env.readline() env.close() metaData = getMetaData(msg) try: if msg.is_multipart(): data = mimeEmail(msg, envelope, metaData, ) else: data = rawEmail(msg, envelope, metaData) except NoBoundaryInMultipartDefect: data = rawEmail(msg, envelope, metaData, ) key = createKey(metaData[0], envelope) envData = elasticEnvelope(envelope) #write data into ES escon.indexEmailData(data, key) escon.indexEnvelopeData(envData, key) duration = time.time() - start return duration
def get_bulk_email(site, filename): """ Show a specific bulk email saved on the disk """ save_path = get_log_dir(site) join = os.path.join if save_path: save_path = join(save_path, 'sent-bulk') if os.path.isdir(save_path): message_path = join(save_path, filename) try: message_file = open(message_path, 'r+') except IOError: return None mail = message_from_file(message_file) message_file.close() # Prepare the date to be formatted with utShowFullDateTime date = email_utils.parsedate_tz(mail.get('Date', '')) date = email_utils.mktime_tz(date) date = datetime.fromtimestamp(date) return { 'subject': mail.get('Subject', '(no-subject)'), 'content': mail.get_payload(decode=True).replace( '\n\n', '</p><p>').replace('\n', '<br/>'), 'recipients': mail.get_all('To'), 'sender': mail.get('From'), 'date': date, }
def test_write_mime_extra_headers(self): """write_mime() with extra headers.""" pr = ProblemReport(date="now!") pr["Simple"] = "bar" pr["TwoLine"] = "first\nsecond\n" io = StringIO() pr.write_mime(io, extra_headers={"Greeting": "hello world", "Foo": "Bar"}) io.seek(0) msg = email.message_from_file(io) self.assertEqual(msg["Greeting"], "hello world") self.assertEqual(msg["Foo"], "Bar") msg_iter = msg.walk() # first part is the multipart container part = msg_iter.next() self.assert_(part.is_multipart()) # second part should be an inline text/plain attachments with all short # fields part = msg_iter.next() self.assert_(not part.is_multipart()) self.assertEqual(part.get_content_type(), "text/plain") self.assert_("Simple: bar" in part.get_payload(decode=True)) # no more parts self.assertRaises(StopIteration, msg_iter.next)
def metadata(self): if self._metadata is None: metadata_file = self.data['metadata'] import email with open(metadata_file) as fp: self._metadata = email.message_from_file(fp) return self._metadata
def __init__(self, contents=None, filename="debian/control"): """ Parse an existing control file. @param contents: content of a control file @type contents: C{str} @param filename: name of the control file @type filename: C{str} @return: Control object @rtype: C{gbp.deb.conrol.Control} object """ if contents: control = email.message_from_string(contents) else: if not os.access(filename, os.F_OK): raise NoControlError("Control file %s does not exist" % filename) with open(filename) as f: control = email.message_from_file(f) if not control.items(): raise ParseControlError("Empty or invalid control file or contents") self._control = control self.filename = filename
def debug_hash_algorithm(opts): #mail_text = ''.join(sys.stdin.readlines()) #message = email.message_from_string(mail_text) message = email.message_from_file(sys.stdin) mail_hash, header_text = compute_hash_key(None, message, opts.message_id) print header_text print 'Hash:', mail_hash
def import_mail_list(corpus_path, file_list_path, db_config): db = build_db(db_config) if db is None: print 'ERROR: ImportEnronCorpus::import_corpus could not build db object' sys.exit(-1) error_list = [] mail_list = open(file_list_path, 'r').readlines() total_mail_files = len(mail_list) mail_count = 0 for mail_path in mail_list: mail_count += 1 if mail_count % 1000 == 0: print 'Stored {} / {}'.format(mail_count, total_mail_files) mail_full_path = os.path.join(corpus_path, mail_path.strip()) cur_email = email.message_from_file(open(mail_full_path)) if not exists_email_db(db, cur_email['Message-ID']): success, err_msg = insert_email_db(db, 'enron_corpus', cur_email) if not success: error_list.append(( mail_path, err_msg)) with open('insert_error.log', 'wb') as err_file: for email_path, error_msg in error_list: err_file.write('{}\t{}'.format(email_path, error_msg)) err_file.close()
def parse_from_file(email_file): ''' return_type: message.Message ''' with open(email_file) as f: e = email.message_from_file(f) # print(e["Message-ID"]) date = datetime.datetime.fromtimestamp(time.mktime(parsedate(e["Date"]))) print(type(e["Date"])) print(e["Date"]) print(type(date)) print(date) # print(e["From"]) # print(e["To"]) # print(e["Subject"]) # print(e["Mime-Version"]) # print(e["Content-Type"]) # print(e["Content-Transfer-Encoding"]) # print(e["X-From"]) # print(e["X-To"]) # print(e["X-cc"]) # print(e["X-bcc"]) # print(e["X-Folder"]) # print(e["X-Origin"]) # print(e["X-FileName"]) # print(e.get_payload()) if e.is_multipart(): for payload in e.get_payload(): # if payload.is_multipart(): ... print payload.get_payload() else: print e.get_payload() print len(e.get_payload())
def __init__(self, name, dir_name): self.name = name self.dir = dir_name metadata_name = os.path.join(dir_name, 'METADATA.txt') if os.path.exists(metadata_name): with open(metadata_name, 'rb') as stream: metadata = email.message_from_file(stream) self.description = metadata.get_payload() for key, value in metadata.items(): setattr(self, tagify(key).replace('-', '_'), value) else: self.title = name.title() self.description = '' # Scan the directory for data files. self.all_links = [] for file_name in os.listdir(dir_name): if file_name.endswith('.csv'): with open(os.path.join(dir_name, file_name), 'rb') as csv_file: reader = csv.DictReader(csv_file) for d in reader: if any(k and v for (k, v) in d.items()): self.all_links.append(Link(self, d)) # Prepare for indexing by keyword. self.facet_keywords = {'main': set()} for link in self.all_links: for facet_name, keywords in link.facet_keywords.items(): self.facet_keywords.setdefault(facet_name, set()).update(keywords) self.main_keywords = self.facet_keywords[MAIN] self.facet_keywords_by_tag = dict((tagify(f, k), (f, k)) for (f, keywords) in self.facet_keywords.items() for k in keywords)
def _msgobj(self, filename): fp = openfile(filename) try: msg = email.message_from_file(fp) finally: fp.close() return msg
def main(): parser = OptionParser(usage="""Hilfetext""") parser.add_option('-d', '--directory', type='string', action='store', help="""Hilfe""") opts, args = parser.parse_args() if not opts.directory: parser.print_help() sys.exit(1) try: msgfile = args[0] except IndexError: parser.print_help() sys.exit(1) fp = open(msgfile) msg = email.message_from_file(fp) fp.close() counter = 1 for part in msg.walk(): if part.get_content_maintype() == 'multipart' or part.get_content_maintype() == 'text': continue filename = part.get_filename() if not filename: continue fp = open(os.path.join(opts.directory, filename), 'wb') fp.write(part.get_payload(decode=True)) fp.close()
def test_custom_message_gets_policy_if_possible_from_file(self): source_file = io.StringIO("Subject: bogus\n\nmsg\n") msg = email.message_from_file(source_file, self.MyMessage, policy=self.MyPolicy) self.assertIsInstance(msg, self.MyMessage) self.assertIs(msg.check_policy, self.MyPolicy)
def decodeheader(file): fp = open(file,"r") msg = email.message_from_file(fp) subject = msg.get("subject") #取主题 #下面是调用python的email模块解码,实质是对MIME数据块解码,信体类似的 try: head = email.Header.Header(subject) #加上gbk或者utf-8参数可以让系统转码的时候适应中文 except: head = email.Header.Header(subject,"gbk") dh = email.Header.decode_header(head) subject = dh[0][0] #下面的打印内容可按需求调整 print "subject:" , subject print "from:", email.utils.parseaddr(msg.get("from"))[1] try: to = msg.get("to").split(',') print "to:", for i in to: print email.utils.parseaddr(i)[1], print "" except: print "to:", email.utils.parseaddr(msg.get("to"))[1] print "date:",msg.get("date") fp.close()
def print_message_score(msg_name, msg_fp): msg = email.message_from_file(msg_fp) bayes = CdbClassifier(open(DB_FILE, 'rb')) prob, evidence = bayes.spamprob(tokenize(msg), evidence=True) print msg_name, prob for word, prob in evidence: print ' ', repr(word), prob
def __init__(self, msg): if isinstance(msg, Message): self.message = msg elif hasattr(msg, 'read'): self.message = email.message_from_file(msg) else: self.message = email.message_from_string(str(msg))
def decodebody(file): fp = open(file,"r") msg = email.message_from_file(fp) for par in msg.walk(): #循环信件中的每一个mime数据块 if not par.is_multipart(): #判断是否属于multipart,不是的话其实是邮件的注释部分,没作用 name = par.get_param("name") #如果有附件,将会取出附件名 if name: #有附件 try: h = email.Header.Header(name) except: h = email.Header.Header(name,"gbk") dh = email.Header.decode_header(h) fname = dh[0][0] #下面这行是QP或base64解码 data = par.get_payload(decode=True) #解码出附件数据,存到文件中 try : f = open(fname, 'wb') except: f = open('aaa', 'wb') f.write(data) f.close() else: print par.get_payload(decode=True) #不是附件,直接解码输出文本内容
def email_recv(request): if not hasattr(settings, 'REPLY_EMAIL') \ or not hasattr(settings, 'EMAIL_KEY'): return HttpResponse(status=501) # Not Implemented key = request.POST.get('key') if key != settings.EMAIL_KEY: raise PermissionDenied if 'email' not in request.FILES: raise HttpResponse(status=400) # Bad Request msg = request.FILES['email'] msg = email.message_from_file(msg) mfrom = msg.get('From') mto = msg.get('To') subject = msg.get('Subject') if msg.is_multipart(): msgs = msg.get_payload() for m in msgs: if m.get_content_type == 'text/plain': content = m.get_payload(decode=True) break else: content = msgs[0].get_payload(decode=True) else: content = msg.get_payload(decode=True) if python_version < (3, ): content = content.decode('utf-8') addr = settings.REPLY_EMAIL pos = addr.find('@') name = addr[:pos] domain = addr[pos:] p = re.compile( '^%s\+(?P<project>[-\w]+)\.issue-(?P<issue>[0-9]+)(\.(?P<event>[0-9]+))?\.(?P<user>[0-9]+)\.(?P<token>[a-z0-9]+)%s$' % (name, domain)) m = None for _mto in map(lambda x: x.strip(), mto.split(',')): m = p.match(_mto) if m: break if not m: # no one matches raise Http404 project = get_object_or_404(Project, name=m.group('project')) issue = get_object_or_404(Issue, project=project, id=m.group('issue')) event = m.group('event') if event: event = get_object_or_404(Event, issue=issue, pk=event) user = get_object_or_404(User, pk=m.group('user')) token = m.group('token') mid = '%s.issue-%d' % (project.name, issue.id) if event: mid += '.%s' % event.pk reference_token = hexdigest_sha256(settings.SECRET_KEY, mid, user.pk) if token != reference_token: raise PermissionDenied event = Event(issue=issue, author=user, code=Event.COMMENT, additionnal_section=content) event.save() issue.subscribers.add(user) notify_new_comment(event) issue.save() return HttpResponse()
"ingv": "INGV", "ipgp": "IPGP", "knmi": "ODC", "resif": "RESIF", "uni-muenchen": "LMU" } for myfile in filelist: print 70 * '-' print "Processing", myfile, if not os.path.exists(myfile): scores[2] += 1 continue fid = open(myfile, "r") msg = email.message_from_file(fid) frm = msg['From'] # Need to decide if a file is one of: # (i) the HTML report from reqlogstats, as e-mail: use From to set 'who'. # (ii) the text report from fdsnws operators, as e-mail: use payload to set 'who' # (iii) the text report from fdsnws operators: payload still determines 'who'. # (iv) the HTML report from reqlogstats; 'who' must be given separately. contentType = "html" if os.path.basename(myfile).startswith("fdsnws_"): contentType = "text" try: subj = msg['Subject'].lower() except AttributeError:
def main(): for a in sys.argv[1:]: print "**********************************" parseOne(email.message_from_file(open(a)), a, "")
# прочитати файл, в унікоді погратися, зберегти в коремому файлі зміст. from email.parser import BytesParser from email.policy import default, HTTP, SMTP from email import message_from_file import locale import sys # using convenience functions with open("mail1", "r", encoding="utf-8") as f: cont = message_from_file(f, policy=default) print('start:----') print(cont) print('stop:----') with open("targ.txt", 'w', encoding="utf-8") as fp: fp.write(cont.as_string(maxheaderlen=90)) """ # reading in binary mode with open("mail1", "rb") as f: cont = BytesParser(policy=default).parse(f) for i in cont.walk(): if i.get_content_type() == "text/plain": with open("targ.txt", 'wb') as fp: fp.write(i.get_payload(decode=True)) """ ''' # reading message's body withou headers body = cont.get_body(preferencelist=('plain')) mes = body.get_content() with open("targ.txt", 'w', encoding="utf-8") as fp:
if not os.path.isdir(kde_dir): os.mkdir(kde_dir) arfile = os.path.join(kde_dir, 'duplicate_mails') duplicates = {} try: archive = open(arfile) except IOError: pass else: for line in archive: when, msgid = line[:-1].split(' ', 1) duplicates[msgid] = float(when) archive.close() redo_archive = False # suck message in from stdin and study it msg = email.message_from_file(sys.stdin) msgid = msg['Message-ID'] if msgid: if msgid in duplicates: # duplicate message: alter its subject subject = msg['Subject'] if subject is None: msg['Subject'] = '**** DUP **** ' + msgid else: del msg['Subject'] msg['Subject'] = '**** DUP **** ' + subject else: # non-duplicate message: redo the archive file redo_archive = True duplicates[msgid] = now else:
def randomly_pickup(dataset): email_path = random.choice(dataset) with open(email_path) as file: original_email = email.message_from_file(file) return original_email
def send_mail(message, filename="", whitelists=None): """sendmail wrapper, takes _either_ a message string or a file as arguments @type whitelists: list of (str or None) @param whitelists: path to whitelists. C{None} or an empty list whitelists everything, otherwise an address is whitelisted if it is included in any of the lists. In addition a global whitelist can be specified in Dinstall::MailWhiteList. """ maildir = Cnf.get('Dir::Mail') if maildir: path = os.path.join(maildir, datetime.datetime.now().isoformat()) path = find_next_free(path) with open(path, 'w') as fh: print(message, end=' ', file=fh) # Check whether we're supposed to be sending mail if "Dinstall::Options::No-Mail" in Cnf and Cnf[ "Dinstall::Options::No-Mail"]: return # If we've been passed a string dump it into a temporary file if message: (fd, filename) = tempfile.mkstemp() os.write(fd, message) os.close(fd) if whitelists is None or None in whitelists: whitelists = [] if Cnf.get('Dinstall::MailWhiteList', ''): whitelists.append(Cnf['Dinstall::MailWhiteList']) if len(whitelists) != 0: with open_file(filename) as message_in: message_raw = modemail.message_from_file(message_in) whitelist = [] for path in whitelists: with open_file(path, 'r') as whitelist_in: for line in whitelist_in: if not re_whitespace_comment.match(line): if re_re_mark.match(line): whitelist.append( re.compile(re_re_mark.sub("", line.strip(), 1))) else: whitelist.append( re.compile(re.escape(line.strip()))) # Fields to check. fields = ["To", "Bcc", "Cc"] for field in fields: # Check each field value = message_raw.get(field, None) if value is not None: match = [] for item in value.split(","): (rfc822_maint, rfc2047_maint, name, email) = fix_maintainer(item.strip()) mail_whitelisted = 0 for wr in whitelist: if wr.match(email): mail_whitelisted = 1 break if not mail_whitelisted: print("Skipping {0} since it's not whitelisted".format( item)) continue match.append(item) # Doesn't have any mail in whitelist so remove the header if len(match) == 0: del message_raw[field] else: message_raw.replace_header(field, ', '.join(match)) # Change message fields in order if we don't have a To header if "To" not in message_raw: fields.reverse() for field in fields: if field in message_raw: message_raw[fields[-1]] = message_raw[field] del message_raw[field] break else: # Clean up any temporary files # and return, as we removed all recipients. if message: os.unlink(filename) return fd = os.open(filename, os.O_RDWR | os.O_EXCL, 0o700) os.write(fd, message_raw.as_string(True)) os.close(fd) # Invoke sendmail (result, output) = commands.getstatusoutput( "%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename)) if (result != 0): raise SendmailFailedError(output) # Clean up any temporary files if message: os.unlink(filename)
def install(self, paths, maker, **kwargs): """ Install a wheel to the specified paths. If kwarg ``warner`` is specified, it should be a callable, which will be called with two tuples indicating the wheel version of this software and the wheel version in the file, if there is a discrepancy in the versions. This can be used to issue any warnings to raise any exceptions. If kwarg ``lib_only`` is True, only the purelib/platlib files are installed, and the headers, scripts, data and dist-info metadata are not written. The return value is a :class:`InstalledDistribution` instance unless ``options.lib_only`` is True, in which case the return value is ``None``. """ dry_run = maker.dry_run warner = kwargs.get('warner') lib_only = kwargs.get('lib_only', False) pathname = os.path.join(self.dirname, self.filename) name_ver = '%s-%s' % (self.name, self.version) data_dir = '%s.data' % name_ver info_dir = '%s.dist-info' % name_ver metadata_name = posixpath.join(info_dir, METADATA_FILENAME) wheel_metadata_name = posixpath.join(info_dir, 'WHEEL') record_name = posixpath.join(info_dir, 'RECORD') wrapper = codecs.getreader('utf-8') with ZipFile(pathname, 'r') as zf: with zf.open(wheel_metadata_name) as bwf: wf = wrapper(bwf) message = message_from_file(wf) wv = message['Wheel-Version'].split('.', 1) file_version = tuple([int(i) for i in wv]) if (file_version != self.wheel_version) and warner: warner(self.wheel_version, file_version) if message['Root-Is-Purelib'] == 'true': libdir = paths['purelib'] else: libdir = paths['platlib'] records = {} with zf.open(record_name) as bf: with CSVReader(stream=bf) as reader: for row in reader: p = row[0] records[p] = row data_pfx = posixpath.join(data_dir, '') info_pfx = posixpath.join(info_dir, '') script_pfx = posixpath.join(data_dir, 'scripts', '') # make a new instance rather than a copy of maker's, # as we mutate it fileop = FileOperator(dry_run=dry_run) fileop.record = True # so we can rollback if needed bc = not sys.dont_write_bytecode # Double negatives. Lovely! outfiles = [] # for RECORD writing # for script copying/shebang processing workdir = tempfile.mkdtemp() # set target dir later # we default add_launchers to False, as the # Python Launcher should be used instead maker.source_dir = workdir maker.target_dir = None try: for zinfo in zf.infolist(): arcname = zinfo.filename if isinstance(arcname, text_type): u_arcname = arcname else: u_arcname = arcname.decode('utf-8') # The signature file won't be in RECORD, # and we don't currently don't do anything with it if u_arcname.endswith('/RECORD.jws'): continue row = records[u_arcname] if row[2] and str(zinfo.file_size) != row[2]: raise DistlibException('size mismatch for ' '%s' % u_arcname) if row[1]: kind, value = row[1].split('=', 1) with zf.open(arcname) as bf: data = bf.read() _, digest = self.get_hash(data, kind) if digest != value: raise DistlibException('digest mismatch for ' '%s' % arcname) if lib_only and u_arcname.startswith((info_pfx, data_pfx)): logger.debug('lib_only: skipping %s', u_arcname) continue is_script = (u_arcname.startswith(script_pfx) and not u_arcname.endswith('.exe')) if u_arcname.startswith(data_pfx): _, where, rp = u_arcname.split('/', 2) outfile = os.path.join(paths[where], convert_path(rp)) else: # meant for site-packages. if u_arcname in (wheel_metadata_name, record_name): continue outfile = os.path.join(libdir, convert_path(u_arcname)) if not is_script: with zf.open(arcname) as bf: fileop.copy_stream(bf, outfile) outfiles.append(outfile) # Double check the digest of the written file if not dry_run and row[1]: with open(outfile, 'rb') as bf: data = bf.read() _, newdigest = self.get_hash(data, kind) if newdigest != digest: raise DistlibException('digest mismatch ' 'on write for ' '%s' % outfile) if bc and outfile.endswith('.py'): try: pyc = fileop.byte_compile(outfile) outfiles.append(pyc) except Exception: # Don't give up if byte-compilation fails, # but log it and perhaps warn the user logger.warning('Byte-compilation failed', exc_info=True) else: fn = os.path.basename(convert_path(arcname)) workname = os.path.join(workdir, fn) with zf.open(arcname) as bf: fileop.copy_stream(bf, workname) dn, fn = os.path.split(outfile) maker.target_dir = dn filenames = maker.make(fn) fileop.set_executable_mode(filenames) outfiles.extend(filenames) if lib_only: logger.debug('lib_only: returning None') dist = None else: # Generate scripts # Try to get pydist.json so we can see if there are # any commands to generate. If this fails (e.g. because # of a legacy wheel), log a warning but don't give up. commands = None file_version = self.info['Wheel-Version'] if file_version == '1.0': # Use legacy info ep = posixpath.join(info_dir, 'entry_points.txt') try: with zf.open(ep) as bwf: epdata = read_exports(bwf) commands = {} for key in ('console', 'gui'): k = '%s_scripts' % key if k in epdata: commands['wrap_%s' % key] = d = {} for v in epdata[k].values(): s = '%s:%s' % (v.prefix, v.suffix) if v.flags: s += ' %s' % v.flags d[v.name] = s except Exception: logger.warning('Unable to read legacy script ' 'metadata, so cannot generate ' 'scripts') else: try: with zf.open(metadata_name) as bwf: wf = wrapper(bwf) commands = json.load(wf).get('extensions') if commands: commands = commands.get('python.commands') except Exception: logger.warning('Unable to read JSON metadata, so ' 'cannot generate scripts') if commands: console_scripts = commands.get('wrap_console', {}) gui_scripts = commands.get('wrap_gui', {}) if console_scripts or gui_scripts: script_dir = paths.get('scripts', '') if not os.path.isdir(script_dir): raise ValueError('Valid script path not ' 'specified') maker.target_dir = script_dir for k, v in console_scripts.items(): script = '%s = %s' % (k, v) filenames = maker.make(script) fileop.set_executable_mode(filenames) if gui_scripts: options = {'gui': True } for k, v in gui_scripts.items(): script = '%s = %s' % (k, v) filenames = maker.make(script, options) fileop.set_executable_mode(filenames) p = os.path.join(libdir, info_dir) dist = InstalledDistribution(p) # Write SHARED paths = dict(paths) # don't change passed in dict del paths['purelib'] del paths['platlib'] paths['lib'] = libdir p = dist.write_shared_locations(paths, dry_run) if p: outfiles.append(p) # Write RECORD dist.write_installed_files(outfiles, paths['prefix'], dry_run) return dist except Exception: # pragma: no cover logger.exception('installation failed.') fileop.rollback() raise finally: shutil.rmtree(workdir)
def process_mailfile(filename, stats): with open(filename, 'r') as fp: mail = email.message_from_file(fp) process_mail(mail, filename, stats)
def _msgobj(self, filename): with openfile(filename) as fp: return email.message_from_file(fp, policy=self.policy)
parts['content'] += payload if msg["Date"]: e_date = email.utils.parsedate_tz(msg['Date']) if e_date: date = datetime.datetime.fromtimestamp( email.utils.mktime_tz(e_date)) parts['content'] += ' weekday_' + str( date.weekday()) + ' hour_of_day_' + str(date.hour) if msg['Subject']: # parts['subject'] = msg['Subject'] parts['content'] += ' ' + msg['Subject'] return parts ham_folder = os.path.join(os.getcwd(), 'Dataset', 'Ham') ham_list = [] for subdir, dirs, files in os.walk(ham_folder): for file_n in files: with open(os.path.join(subdir, file_n), encoding='latin-1') as f: ham_list.append(get_text_from_email(email.message_from_file(f))) print(len(ham_list)) df = pd.DataFrame(ham_list) ham_file = os.path.join(os.getcwd(), 'Dataset', 'csv_files', 'ham.csv') df.to_csv(ham_file) ham_file = os.path.join(os.getcwd(), 'Dataset', 'csv_files', 'ham_latin.csv') df.to_csv(ham_file, encoding='latin-1')
if e_date: date = datetime.datetime.fromtimestamp( email.utils.mktime_tz(e_date)) parts['content'] += ' weekday_' + str( date.weekday()) + ' hour_of_day_' + str(date.hour) except Exception: pass if msg['Subject']: parts['content'] += ' ' + msg['Subject'] return parts # input spam folder spam_folder = os.path.join(os.getcwd(), 'Dataset', sys.argv[1]) spam_list = [] for subdir, dirs, files in os.walk(spam_folder): for filename in files: try: with open(os.path.join(subdir, filename), encoding='latin-1') as f: spam_list.append( get_text_from_email(email.message_from_file(f))) except OSError: pass df = pd.DataFrame(spam_list) # Output file path. Saves as .csv file spam_file = os.path.join(os.getcwd(), 'Dataset', 'csv_files', sys.argv[2]) df.to_csv(spam_file)
def message_from_file(handle): '''Reads a mail from the given file-like object and returns an email object, very much like email.message_from_file. In addition to that OpenPGP encrypted data is detected and decrypted. If this succeeds, any mime messages found in the recovered plaintext message are added to the returned message object. :param handle: a file-like object :returns: :class:`email.message.Message` possibly augmented with decrypted data ''' m = email.message_from_file(handle) # make sure noone smuggles a token in (data from m is untrusted) del m[X_SIGNATURE_VALID_HEADER] del m[X_SIGNATURE_MESSAGE_HEADER] p = get_params(m) app_pgp_sig = 'application/pgp-signature' app_pgp_enc = 'application/pgp-encrypted' # handle OpenPGP signed data if (m.is_multipart() and m.get_content_subtype() == 'signed' and p.get('protocol') == app_pgp_sig): # RFC 3156 is quite strict: # * exactly two messages # * the second is of type 'application/pgp-signature' # * the second contains the detached signature malformed = False if len(m.get_payload()) != 2: malformed = u'expected exactly two messages, got {0}'.format( len(m.get_payload())) else: ct = m.get_payload(1).get_content_type() if ct != app_pgp_sig: malformed = u'expected Content-Type: {0}, got: {1}'.format( app_pgp_sig, ct) # TODO: RFC 3156 says the alg has to be lower case, but I've # seen a message with 'PGP-'. maybe we should be more # permissive here, or maybe not, this is crypto stuff... if not p.get('micalg', 'nothing').startswith('pgp-'): malformed = u'expected micalg=pgp-..., got: {0}'.format( p.get('micalg', 'nothing')) sigs = [] if not malformed: try: sigs = crypto.verify_detached( helper.email_as_string(m.get_payload(0)), m.get_payload(1).get_payload()) except GPGProblem as e: malformed = unicode(e) add_signature_headers(m, sigs, malformed) # handle OpenPGP encrypted data elif (m.is_multipart() and m.get_content_subtype() == 'encrypted' and p.get('protocol') == app_pgp_enc and 'Version: 1' in m.get_payload(0).get_payload()): # RFC 3156 is quite strict: # * exactly two messages # * the first is of type 'application/pgp-encrypted' # * the first contains 'Version: 1' # * the second is of type 'application/octet-stream' # * the second contains the encrypted and possibly signed data malformed = False ct = m.get_payload(0).get_content_type() if ct != app_pgp_enc: malformed = u'expected Content-Type: {0}, got: {1}'.format( app_pgp_enc, ct) want = 'application/octet-stream' ct = m.get_payload(1).get_content_type() if ct != want: malformed = u'expected Content-Type: {0}, got: {1}'.format(want, ct) if not malformed: try: sigs, d = crypto.decrypt_verify(m.get_payload(1).get_payload()) except GPGProblem as e: # signature verification failures end up here too if # the combined method is used, currently this prevents # the interpretation of the recovered plain text # mail. maybe that's a feature. malformed = unicode(e) else: # parse decrypted message n = message_from_string(d) # add the decrypted message to m. note that n contains # all the attachments, no need to walk over n here. m.attach(n) # add any defects found m.defects.extend(n.defects) # there are two methods for both signed and encrypted # data, one is called 'RFC 1847 Encapsulation' by # RFC 3156, and one is the 'Combined method'. if len(sigs) == 0: # 'RFC 1847 Encapsulation', the signature is a # detached signature found in the recovered mime # message of type multipart/signed. if X_SIGNATURE_VALID_HEADER in n: for k in (X_SIGNATURE_VALID_HEADER, X_SIGNATURE_MESSAGE_HEADER): m[k] = n[k] else: # an encrypted message without signatures # should arouse some suspicion, better warn # the user add_signature_headers(m, [], 'no signature found') else: # 'Combined method', the signatures are returned # by the decrypt_verify function. # note that if we reached this point, we know the # signatures are valid. if they were not valid, # the else block of the current try would not have # been executed add_signature_headers(m, sigs, '') if malformed: msg = u'Malformed OpenPGP message: {0}'.format(malformed) content = email.message_from_string(msg.encode('utf-8')) content.set_charset('utf-8') m.attach(content) return m
def msgfactory(fp): try: return email.message_from_file(fp) except email.Errors.MessageParseError: # Don't return None since that will stop the mailbox iterator return ''
def __init__(self, script_path): self.script_path = script_path self.parseConfig() options = self.parseArguments() self.quiet = options.quiet self.full_url = options.full_url self.urlList = None self.raw_file = None if not options.url: if options.urlList: with File(options.urlList) as urlList: if not urlList.exists(): print("The file with URLs does not exist") exit(1) if not urlList.is_valid(): print("The file with URLs is invalid") exit(1) if not urlList.can_read(): print("The file with URLs cannot be read") exit(1) self.urlList = list(urlList.get_lines()) elif options.cidr: self.urlList = [str(ip) for ip in IPv4Network(options.cidr)] elif options.stdin_urls: self.urlList = sys.stdin.read().splitlines() elif options.raw_file: with File(options.raw_file) as raw_content: if not raw_content.exists(): print("The file with the raw request does not exist") exit(1) if not raw_content.is_valid(): print("The file with the raw request is invalid") exit(1) if not raw_content.can_read(): print("The file with the raw request cannot be read") exit(1) self.raw_file = options.raw_file else: print("URL target is missing, try using -u <url>") exit(1) else: self.urlList = [options.url] if not options.extensions and not options.noExtension: print("WARNING: No extension was specified!") if options.noExtension: options.extensions = str() # Enable to use multiple dictionaries at once for dictFile in options.wordlist.split(","): with File(dictFile) as wordlist: if not wordlist.exists(): print("The wordlist file does not exist") exit(1) if not wordlist.is_valid(): print("The wordlist is invalid") exit(1) if not wordlist.can_read(): print("The wordlist cannot be read") exit(1) if options.proxyList: with File(options.proxyList) as plist: if not plist.exists(): print("The proxylist file does not exist") exit(1) if not plist.is_valid(): print("The proxylist is invalid") exit(1) if not plist.can_read(): print("The proxylist cannot be read") exit(1) self.proxylist = open(options.proxyList).read().splitlines() options.requestByHostname = True elif options.proxy: self.proxy = options.proxy options.requestByHostname = True else: self.proxy = None if options.replay_proxy: self.replay_proxy = options.replay_proxy options.requestByHostname = True else: self.replay_proxy = None if options.headers: try: self.headers = dict( email.message_from_file( StringIO("\r\n".join(options.headers)))) except Exception: print("Invalid headers") exit(1) else: self.headers = {} if options.headerList: try: with File(options.headerList) as hlist: if not hlist.exists(): print("The header list file does not exist") exit(1) if not hlist.is_valid(): print("The header list file is invalid") exit(1) if not hlist.can_read(): print("The header list cannot be read") exit(1) headers = dict( email.message_from_file(StringIO(hlist.read()))) for key, value in headers.items(): self.headers[key] = value except Exception as e: print("Error in headers file: " + str(e)) exit(1) if options.extensions == "*": self.extensions = [ "php", "inc.php", "jsp", "jsf", "asp", "aspx", "do", "action", "cgi", "pl", "html", "htm", "js", "css", "json", "txt", "tar.gz", "tgz" ] elif options.extensions == "CHANGELOG.md": print( "A weird extension was provided: CHANGELOG.md. Please do not use * as the extension or enclose it in double quotes" ) exit(0) else: self.extensions = list( oset([ extension.lstrip(' .') for extension in options.extensions.split(",") ])) if options.excludeExtensions: self.excludeExtensions = list( oset([ excludeExtension.lstrip(' .') for excludeExtension in options.excludeExtensions.split(",") ])) else: self.excludeExtensions = [] self.useragent = options.useragent self.useRandomAgents = options.useRandomAgents self.cookie = options.cookie if options.threadsCount < 1: print("Threads number must be greater than zero") exit(1) self.threadsCount = options.threadsCount self.includeStatusCodes = [] if options.includeStatusCodes: for statusCode in options.includeStatusCodes.split(","): try: if "-" in statusCode: statusCodes = [ i for i in range( int(statusCode.split("-")[0].strip()), int(statusCode.split("-")[1].strip()) + 1) ] self.includeStatusCodes.extend(statusCodes) else: self.includeStatusCodes.append(int(statusCode.strip())) except ValueError: print( "Invalid status code or status code range: {0}".format( statusCode)) exit(1) self.excludeStatusCodes = [] if options.excludeStatusCodes: for statusCode in options.excludeStatusCodes.split(","): try: if "-" in statusCode: statusCodes = [ i for i in range( int(statusCode.split("-")[0].strip()), int(statusCode.split("-")[1].strip()) + 1) ] self.excludeStatusCodes.extend(statusCodes) else: self.excludeStatusCodes.append(int(statusCode.strip())) except ValueError: print( "Invalid status code or status code range: {0}".format( statusCode)) exit(1) if options.excludeSizes: try: self.excludeSizes = list( oset([ excludeSize.strip().upper() if excludeSize else None for excludeSize in options.excludeSizes.split(",") ])) except ValueError: self.excludeSizes = [] else: self.excludeSizes = [] if options.excludeTexts: try: self.excludeTexts = list( oset([ excludeText.strip() if excludeText else None for excludeText in options.excludeTexts.split(",") ])) except ValueError: self.excludeTexts = [] else: self.excludeTexts = [] if options.excludeRegexps: try: self.excludeRegexps = list( oset([ excludeRegexp.strip() if excludeRegexp else None for excludeRegexp in options.excludeRegexps.split(",") ])) except ValueError: self.excludeRegexps = [] else: self.excludeRegexps = [] if options.excludeRedirects: try: self.excludeRedirects = list( oset([ excludeRedirect.strip() if excludeRedirect else None for excludeRedirect in options.excludeRedirects.split( ",") ])) except ValueError: self.excludeRedirects = [] else: self.excludeRedirects = [] self.prefixes = [] if not options.prefixes else list( oset([prefix.strip() for prefix in options.prefixes.split(",")])) self.suffixes = [] if not options.suffixes else list( oset([suffix.strip() for suffix in options.suffixes.split(",")])) if options.wordlist: self.wordlist = list( oset([ wordlist.strip() for wordlist in options.wordlist.split(",") ])) else: print("No wordlist was provided, try using -w <wordlist>") exit(1) self.lowercase = options.lowercase self.uppercase = options.uppercase self.capitalization = options.capitalization self.forceExtensions = options.forceExtensions self.data = options.data self.testFailPath = options.testFailPath self.color = options.color self.delay = options.delay self.timeout = options.timeout self.ip = options.ip self.maxRetries = options.maxRetries self.recursive = options.recursive self.minimumResponseSize = options.minimumResponseSize self.maximumResponseSize = options.maximumResponseSize self.noExtension = options.noExtension self.onlySelected = options.onlySelected self.simpleOutputFile = options.simpleOutputFile self.plainTextOutputFile = options.plainTextOutputFile self.jsonOutputFile = options.jsonOutputFile self.xmlOutputFile = options.xmlOutputFile self.markdownOutputFile = options.markdownOutputFile self.csvOutputFile = options.csvOutputFile if options.scanSubdirs: self.scanSubdirs = list( oset([ subdir.strip(" /") + "/" for subdir in options.scanSubdirs.split(",") ])) else: self.scanSubdirs = [] if options.excludeSubdirs: self.excludeSubdirs = list( oset([ subdir.strip(" /") + "/" for subdir in options.excludeSubdirs.split(",") ])) else: self.excludeSubdirs = None if len(set(self.extensions).intersection(self.excludeExtensions)): print( "Exclude extension list can not contain any extension that has already in the extension list" ) exit(1) self.redirect = options.followRedirects self.httpmethod = options.httpmethod self.scheme = options.scheme self.requestByHostname = options.requestByHostname self.exit_on_error = options.exit_on_error self.skip_on_429 = options.skip_on_429 self.debug = options.debug self.recursion_depth = options.recursion_depth if self.scheme not in ["http", "https"]: print("Invalid URI scheme: {0}".format(self.scheme)) exit(1)
def load_message(filepath): """ Load a message from the filesystem """ with open(filepath, 'r', encoding='utf-8') as file: return email.message_from_file(file)
def extractPDM(filename): with open(filename, "r") as f: oi = message_from_string(f.read()) msg = email.message_from_file(f, policy=policy.default) print(oi)
def parse_file(self, fd, prefix=None): m = message_from_file(fd) return self.parse(m, prefix)
def __init__(self, file, category=None): self.mail = email.message_from_file(file) self.category = category
def eom(self): """ End of Message """ self.fp.seek(0) msg = email.message_from_file(self.fp) msg_id = msg.get("In-Reply-To", None) if msg_id is None: self.log("No In-Reply-To, keep going") return Milter.CONTINUE # Ensure we don't get extra lines in the message-id msg_id = msg_id.split("\n")[0].strip() self.log("msg-ig %s" % msg_id) self.log("To %s" % msg["to"]) self.log("Cc %s" % msg.get("cc")) self.log("From %s" % msg["From"]) # Check the email was sent to the right address email_address = msg["to"] if "reply+" in msg.get("cc", ""): email_address = msg["cc"] if "reply+" not in email_address: self.log("No valid recipient email found in To/Cc: %s" % email_address) return Milter.CONTINUE # Ensure the user replied to his/her own notification, not that # they are trying to forge their ID into someone else's salt = _config.get("SALT_EMAIL") from_email = clean_item(msg["From"]) session = pagure.lib.model_base.create_session(_config["DB_URL"]) try: user = pagure.lib.query.get_user(session, from_email) except: self.log("Could not find an user in the DB associated with %s" % from_email) session.remove() return Milter.CONTINUE hashes = [] for email_obj in user.emails: m = hashlib.sha512("%s%s%s" % (msg_id, salt, email_obj.email)) hashes.append(m.hexdigest()) tohash = email_address.split("@")[0].split("+")[-1] if tohash not in hashes: self.log("hash list: %s" % hashes) self.log("tohash: %s" % tohash) self.log("Hash does not correspond to the destination") session.remove() return Milter.CONTINUE if msg["From"] and msg["From"] == _config.get("FROM_EMAIL"): self.log("Let's not process the email we send") session.remove() return Milter.CONTINUE msg_id = clean_item(msg_id) if msg_id and "-ticket-" in msg_id: self.log("Processing issue") session.remove() return self.handle_ticket_email(msg, msg_id) elif msg_id and "-pull-request-" in msg_id: self.log("Processing pull-request") session.remove() return self.handle_request_email(msg, msg_id) else: self.log("Not a pagure ticket or pull-request email, let it go") session.remove() return Milter.CONTINUE
def install(self, paths, maker, **kwargs): """ Install a wheel to the specified paths. If kwarg ``warner`` is specified, it should be a callable, which will be called with two tuples indicating the wheel version of this software and the wheel version in the file, if there is a discrepancy in the versions. This can be used to issue any warnings to raise any exceptions. If kwarg ``lib_only`` is True, only the purelib/platlib files are installed, and the headers, scripts, data and dist-info metadata are not written. If kwarg ``bytecode_hashed_invalidation`` is True, written bytecode will try to use file-hash based invalidation (PEP-552) on supported interpreter versions (CPython 2.7+). The return value is a :class:`InstalledDistribution` instance unless ``options.lib_only`` is True, in which case the return value is ``None``. """ dry_run = maker.dry_run warner = kwargs.get("warner") lib_only = kwargs.get("lib_only", False) bc_hashed_invalidation = kwargs.get("bytecode_hashed_invalidation", False) pathname = os.path.join(self.dirname, self.filename) name_ver = "%s-%s" % (self.name, self.version) data_dir = "%s.data" % name_ver info_dir = "%s.dist-info" % name_ver metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME) wheel_metadata_name = posixpath.join(info_dir, "WHEEL") record_name = posixpath.join(info_dir, "RECORD") wrapper = codecs.getreader("utf-8") with ZipFile(pathname, "r") as zf: with zf.open(wheel_metadata_name) as bwf: wf = wrapper(bwf) message = message_from_file(wf) wv = message["Wheel-Version"].split(".", 1) file_version = tuple([int(i) for i in wv]) if (file_version != self.wheel_version) and warner: warner(self.wheel_version, file_version) if message["Root-Is-Purelib"] == "true": libdir = paths["purelib"] else: libdir = paths["platlib"] records = {} with zf.open(record_name) as bf: with CSVReader(stream=bf) as reader: for row in reader: p = row[0] records[p] = row data_pfx = posixpath.join(data_dir, "") info_pfx = posixpath.join(info_dir, "") script_pfx = posixpath.join(data_dir, "scripts", "") # make a new instance rather than a copy of maker's, # as we mutate it fileop = FileOperator(dry_run=dry_run) fileop.record = True # so we can rollback if needed bc = not sys.dont_write_bytecode # Double negatives. Lovely! outfiles = [] # for RECORD writing # for script copying/shebang processing workdir = tempfile.mkdtemp() # set target dir later # we default add_launchers to False, as the # Python Launcher should be used instead maker.source_dir = workdir maker.target_dir = None try: for zinfo in zf.infolist(): arcname = zinfo.filename if isinstance(arcname, text_type): u_arcname = arcname else: u_arcname = arcname.decode("utf-8") if self.skip_entry(u_arcname): continue row = records[u_arcname] if row[2] and str(zinfo.file_size) != row[2]: raise DistlibException("size mismatch for " "%s" % u_arcname) if row[1]: kind, value = row[1].split("=", 1) with zf.open(arcname) as bf: data = bf.read() _, digest = self.get_hash(data, kind) if digest != value: raise DistlibException( "digest mismatch for " "%s" % arcname ) if lib_only and u_arcname.startswith((info_pfx, data_pfx)): logger.debug("lib_only: skipping %s", u_arcname) continue is_script = u_arcname.startswith( script_pfx ) and not u_arcname.endswith(".exe") if u_arcname.startswith(data_pfx): _, where, rp = u_arcname.split("/", 2) outfile = os.path.join(paths[where], convert_path(rp)) else: # meant for site-packages. if u_arcname in (wheel_metadata_name, record_name): continue outfile = os.path.join(libdir, convert_path(u_arcname)) if not is_script: with zf.open(arcname) as bf: fileop.copy_stream(bf, outfile) outfiles.append(outfile) # Double check the digest of the written file if not dry_run and row[1]: with open(outfile, "rb") as bf: data = bf.read() _, newdigest = self.get_hash(data, kind) if newdigest != digest: raise DistlibException( "digest mismatch " "on write for " "%s" % outfile ) if bc and outfile.endswith(".py"): try: pyc = fileop.byte_compile( outfile, hashed_invalidation=bc_hashed_invalidation ) outfiles.append(pyc) except Exception: # Don't give up if byte-compilation fails, # but log it and perhaps warn the user logger.warning("Byte-compilation failed", exc_info=True) else: fn = os.path.basename(convert_path(arcname)) workname = os.path.join(workdir, fn) with zf.open(arcname) as bf: fileop.copy_stream(bf, workname) dn, fn = os.path.split(outfile) maker.target_dir = dn filenames = maker.make(fn) fileop.set_executable_mode(filenames) outfiles.extend(filenames) if lib_only: logger.debug("lib_only: returning None") dist = None else: # Generate scripts # Try to get pydist.json so we can see if there are # any commands to generate. If this fails (e.g. because # of a legacy wheel), log a warning but don't give up. commands = None file_version = self.info["Wheel-Version"] if file_version == "1.0": # Use legacy info ep = posixpath.join(info_dir, "entry_points.txt") try: with zf.open(ep) as bwf: epdata = read_exports(bwf) commands = {} for key in ("console", "gui"): k = "%s_scripts" % key if k in epdata: commands["wrap_%s" % key] = d = {} for v in epdata[k].values(): s = "%s:%s" % (v.prefix, v.suffix) if v.flags: s += " [%s]" % ",".join(v.flags) d[v.name] = s except Exception: logger.warning( "Unable to read legacy script " "metadata, so cannot generate " "scripts" ) else: try: with zf.open(metadata_name) as bwf: wf = wrapper(bwf) commands = json.load(wf).get("extensions") if commands: commands = commands.get("python.commands") except Exception: logger.warning( "Unable to read JSON metadata, so " "cannot generate scripts" ) if commands: console_scripts = commands.get("wrap_console", {}) gui_scripts = commands.get("wrap_gui", {}) if console_scripts or gui_scripts: script_dir = paths.get("scripts", "") if not os.path.isdir(script_dir): raise ValueError("Valid script path not " "specified") maker.target_dir = script_dir for k, v in console_scripts.items(): script = "%s = %s" % (k, v) filenames = maker.make(script) fileop.set_executable_mode(filenames) if gui_scripts: options = {"gui": True} for k, v in gui_scripts.items(): script = "%s = %s" % (k, v) filenames = maker.make(script, options) fileop.set_executable_mode(filenames) p = os.path.join(libdir, info_dir) dist = InstalledDistribution(p) # Write SHARED paths = dict(paths) # don't change passed in dict del paths["purelib"] del paths["platlib"] paths["lib"] = libdir p = dist.write_shared_locations(paths, dry_run) if p: outfiles.append(p) # Write RECORD dist.write_installed_files(outfiles, paths["prefix"], dry_run) return dist except Exception: # pragma: no cover logger.exception("installation failed.") fileop.rollback() raise finally: shutil.rmtree(workdir)
def lambda_handler(event, context): s3 = boto3.client('s3') ses = boto3.client('ses') try: o2 = s3.get_object(Bucket=bucketConfig, Key=bucketKeyConfig) defaultConfig = json.loads(o2['Body'].read()) except: print "There was an error accessing the default Config in s3" defaultConfig = defaultConfigBackUp record = event['Records'][0] assert record['eventSource'] == 'aws:ses' o = s3.get_object(Bucket=defaultConfig['Ses_Incoming_Bucket'], Key=defaultConfig['prefix'] + record['ses']['mail']['messageId']) raw_mail = o['Body'] recipients = [] msg = message_from_file(raw_mail) del msg['DKIM-Signature'] del msg['Sender'] sender = msg['Return-Path'][1:-1] del msg['Return-Path'] defaultConfig['original_from'] = msg['From'] defaultConfig['previousSubj'] = msg['Subject'] del msg['Subject'] del msg['From'] msg['From'] = ( re.sub(r'\<.+?\>', '', defaultConfig['original_from']).strip() + ' <{}>'.format(defaultConfig['Verified_From_Email'])) msg['Return-Path'] = defaultConfig['Verified_From_Email'] recipient = record['ses']['receipt']['recipients'][0] defaultConfig['recipientName'] = recipient.split('@')[0] defaultConfig['subjectPrefix'] = ('[AWS ' + defaultConfig['recipientName'] + '] ' + defaultConfig['previousSubj']) msg['Subject'] = defaultConfig['subjectPrefix'] msg_string = msg.as_string() for key, value in defaultConfig['Forward_bySender_Mapping'].iteritems(): print sender if key == sender: recipients = value sendEmail(recipients, msg_string, ses) return for key, value in defaultConfig['Forward_byRecipient_Mapping'].iteritems(): if key == recipient: recipients = value sendEmail(recipients, msg_string, ses) return if recipient.endswith(key): recipients = defaultConfig['Forward_byRecipient_Mapping'].get( key, []) sendEmail(recipients, msg_string, ses) return return
elif part.get_content_type() == 'message/rfc822': continue if not os.path.isdir(newdir): os.mkdir(newdir) counter += 1 fd = file(os.path.join(newdir, filename), 'w') fd.write(part.get_payload(decode=1)) fd.close() else: description = msg.get_payload(decode=1).decode(msg.get_charsets()[0]) project = self.project_re.search(subject) if project: project = project.groups()[0] else: project = '' for partner in partners: self.rpc(('res.partner.event', 'create', {'name' : subject, 'partner_id' : partner, 'description' : description, 'project' : project})) if __name__ == '__main__': rpc_dispatcher = CommandDispatcher(RPCProxy(4, 'admin')) dispatcher = CommandDispatcher(ReceiverEmail2Event(rpc_dispatcher)) parser = EmailParser(['To', 'Cc', 'From'], dispatcher) parser.parse(email.message_from_file(sys.stdin)) # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
def from_file(fileobj): """Reads an email and cleans it up to make a MailBase.""" return from_message(email.message_from_file(fileobj))
def eom(self): ''' End of Message ''' self.fp.seek(0) msg = email.message_from_file(self.fp) msg_id = msg.get('In-Reply-To', None) if msg_id is None: self.log('No In-Reply-To, keep going') return Milter.CONTINUE # Ensure we don't get extra lines in the message-id msg_id = msg_id.split('\n')[0].strip() self.log('msg-ig %s' % msg_id) self.log('To %s' % msg['to']) self.log('Cc %s' % msg.get('cc')) self.log('From %s' % msg['From']) # Check the email was sent to the right address email_address = msg['to'] if 'reply+' in msg.get('cc', ''): email_address = msg['cc'] if 'reply+' not in email_address: self.log( 'No valid recipient email found in To/Cc: %s' % email_address) return Milter.CONTINUE # Ensure the user replied to his/her own notification, not that # they are trying to forge their ID into someone else's salt = _config.get('SALT_EMAIL') from_email = clean_item(msg['From']) session = pagure.lib.query.create_session(_config['DB_URL']) try: user = pagure.lib.query.get_user(session, from_email) except: self.log( "Could not find an user in the DB associated with %s" % from_email) session.remove() return Milter.CONTINUE hashes = [] for email_obj in user.emails: m = hashlib.sha512('%s%s%s' % (msg_id, salt, email_obj.email)) hashes.append(m.hexdigest()) tohash = email_address.split('@')[0].split('+')[-1] if tohash not in hashes: self.log('hash list: %s' % hashes) self.log('tohash: %s' % tohash) self.log('Hash does not correspond to the destination') session.remove() return Milter.CONTINUE if msg['From'] and msg['From'] == _config.get('FROM_EMAIL'): self.log("Let's not process the email we send") session.remove() return Milter.CONTINUE msg_id = clean_item(msg_id) if msg_id and '-ticket-' in msg_id: self.log('Processing issue') session.remove() return self.handle_ticket_email(msg, msg_id) elif msg_id and '-pull-request-' in msg_id: self.log('Processing pull-request') session.remove() return self.handle_request_email(msg, msg_id) else: self.log('Not a pagure ticket or pull-request email, let it go') session.remove() return Milter.CONTINUE
return urlopen(req) except URLError, e: raise ShortyError(e) def githubshrink(bigurl): gitio_pattern = 'http(s)?://((gist|raw|develop(er)?)\.)?github\.com' gitio_re = re.compile(gitio_pattern) if not gitio_re.search(bigurl): raise ShortyError('URL must match %s' % gitio_pattern) resp = request('http://git.io', post_data="url=%s" % bigurl) for header in resp.info().headers: if header.startswith("Location:"): return header[10:].strip('\n\r') raise ShortyError('Failed to shrink url') incoming = email.message_from_file(sys.stdin) if incoming.is_multipart(): payload = incoming.get_payload(0).get_payload(decode=True) else: payload = incoming.get_payload(decode=True) bodylines = payload.split("\n") subj = incoming.get("Subject") userparts = incoming.get("From").split("@", 1) theuser = userparts[0].split("<", 1) msgs = [] foundcommit = 0 foundmsg = 0
def extract(msgfile): message = message_from_file(msgfile) Text, Html = get_text(message) ret = {"text": Text.strip(), "html": Html.strip()} return ret
def message_from_filepath(filepath): import email fileobj = decompress(filepath) return email.message_from_file(fileobj)
M.user("18829210056") M.pass_("wyh123") #打印有多少封信 numMessages = len(M.list()[1]) print 'num of messages', numMessages for i in range(numMessages): m = M.retr(i + 1) buf = cStringIO.StringIO() for j in m[1]: print >> buf, j buf.seek(0) #解析信件内容 msg = email.message_from_file(buf) for part in msg.walk(): contenttype = part.get_content_type() filename = part.get_filename() if filename and contenttype == 'application/octet-stream': # 保存附件 f = open("mail%d.%s.attach" % (i + 1, filename), 'wb') f.write(base64.decodestring(part.get_payload())) f.close() elif contenttype == 'text/plain': # 保存正文 f = open("mail%d.txt" % (i + 1), 'wb') f.write(base64.decodestring(part.get_payload())) f.close()
def _load_msg_from_file(self, name): f = self._open_file(name) msg = message_from_file(f) f.close() return msg