def get_metadata(dist):
# type: (Distribution) -> Message
"""
:raises NoneMetadataError: if the distribution reports `has_metadata()`
True but `get_metadata()` returns None.
"""
metadata_name = 'METADATA'
if (isinstance(dist, pkg_resources.DistInfoDistribution)
and dist.has_metadata(metadata_name)):
metadata = dist.get_metadata(metadata_name)
elif dist.has_metadata('PKG-INFO'):
metadata_name = 'PKG-INFO'
metadata = dist.get_metadata(metadata_name)
else:
logger.warning("No metadata found in %s", display_path(dist.location))
metadata = ''
if metadata is None:
raise NoneMetadataError(dist, metadata_name)
feed_parser = FeedParser()
# The following line errors out if with a "NoneType" TypeError if
# passed metadata=None.
feed_parser.feed(metadata)
return feed_parser.close()
def _execute(self, http, order, requests):
"""Serialize batch request, send to server, process response.
Args:
http: httplib2.Http, an http object to be used to make the request with.
order: list, list of request ids in the order they were added to the
batch.
request: list, list of request objects to send.
Raises:
httplib2.Error if a transport error has occured.
apiclient.errors.BatchError if the response is the wrong format.
"""
message = MIMEMultipart('mixed')
# Message should not write out it's own headers.
setattr(message, '_write_headers', lambda self: None)
# Add all the individual requests.
for request_id in order:
request = requests[request_id]
msg = MIMENonMultipart('application', 'http')
msg['Content-Transfer-Encoding'] = 'binary'
msg['Content-ID'] = self._id_to_header(request_id)
body = self._serialize_request(request)
msg.set_payload(body)
message.attach(msg)
body = message.as_string()
headers = {}
headers['content-type'] = ('multipart/mixed; '
'boundary="%s"') % message.get_boundary()
resp, content = http.request(self._batch_uri, 'POST', body=body,
headers=headers)
if resp.status >= 300:
raise HttpError(resp, content, self._batch_uri)
# Now break out the individual responses and store each one.
boundary, _ = content.split(None, 1)
# Prepend with a content-type header so FeedParser can handle it.
header = 'content-type: %s\r\n\r\n' % resp['content-type']
for_parser = header + content
parser = FeedParser()
parser.feed(for_parser)
mime_response = parser.close()
if not mime_response.is_multipart():
raise BatchError("Response not in multipart/mixed format.", resp,
content)
for part in mime_response.get_payload():
request_id = self._header_to_id(part['Content-ID'])
headers, content = self._deserialize_response(part.get_payload())
self._responses[request_id] = (headers, content)
def get_pkg_info(pkg):
pkg_info = {
'name': pkg.project_name,
'version': pkg.version,
'namever': str(pkg),
}
metadata = None
if pkg.has_metadata('METADATA'):
metadata = pkg.get_metadata('METADATA')
if pkg.has_metadata('PKG-INFO') and metadata is None:
metadata = pkg.get_metadata('PKG-INFO')
if metadata is None:
for key in METADATA_KEYS:
pkg_info[key] = LICENSE_UNKNOWN
return pkg_info
feed_parser = FeedParser()
feed_parser.feed(metadata)
parsed_metadata = feed_parser.close()
for key in METADATA_KEYS:
pkg_info[key] = parsed_metadata.get(key, LICENSE_UNKNOWN)
if args.from_classifier and metadata is not None:
message = message_from_string(metadata)
pkg_info['license'] = find_license_from_classifier(message)
return pkg_info
def check_dist_requires_python(dist, absorb=True):
metadata = get_metadata(dist)
feed_parser = FeedParser()
feed_parser.feed(metadata)
pkg_info_dict = feed_parser.close()
requires_python = pkg_info_dict.get('Requires-Python')
if not absorb:
return requires_python
try:
if not check_requires_python(requires_python):
# raise exceptions.UnsupportedPythonVersion(
# "%s requires Python '%s' but the running Python is %s" % (
# dist.project_name,
# requires_python,
# '.'.join(map(str, sys.version_info[:3])),)
# )
return
except specifiers.InvalidSpecifier as e:
logger.warning(
"Package %s has an invalid Requires-Python entry %s - %s",
dist.project_name,
requires_python,
e,
)
return
def _deserialize_response(self, payload):
"""Convert string into httplib2 response and content.
Args:
payload: string, headers and body as a string.
Returns:
A pair (resp, content) like would be returned from httplib2.request.
"""
# Strip off the status line
status_line, payload = payload.split('\n', 1)
protocol, status, reason = status_line.split(' ', 2)
# Parse the rest of the response
parser = FeedParser()
parser.feed(payload)
msg = parser.close()
msg['status'] = status
# Create httplib2.Response from the parsed headers.
resp = httplib2.Response(msg)
resp.reason = reason
resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
content = payload.split('\r\n\r\n', 1)[1]
return resp, content
def upload_mime(request):
"""
This callback receives raw MIME messages.
Why? Because the URL ends with 'mime'
POST parameters are:
'sender'
'recipient'
'body-mime'
MIME is a raw message which can be saved into an .msg or .eml file, parsed
with Python's MIME parser, etc
Use callbacks like this when you want to have full control over messages,
for example when you need access to all original MIME headers, etc
"""
if request.method == 'POST':
sender = request.POST.get('sender', None)
recipient = request.POST.get('recipient', None)
body_mime = request.POST.get('body-mime', None)
# Simplistic MIME parsing:
parser = FeedParser()
parser.feed(body_mime)
message = parser.close()
# Returned text is ignored but HTTP status code matters:
# Mailgun wants to see 200, otherwise it will make another attempt
return HttpResponse('OK')
def request(self, query, headers):
# httplib would really help but there isn't any trivial way
# that I know of to use your own socket with it.
request = Message()
for k, v in headers.iteritems():
request[k] = v
self.sock.send(query + "\r\n" + request.as_string())
buffer = self.sock.recv(4096)
try:
[result, data] = buffer.split('\r\n', 1)
except ValueError:
traceback.print_exc()
print >> sys.stderr, 'Buffer:', buffer
print >> sys.stderr, 'Query:', query
print >> sys.stderr, 'Headers:', headers
return False
result = result.split(' ')
if int(result[1]) != 200:
self.log.info('Request failed: %s', ' '.join(result))
raise httplib.BadStatusLine(' '.join(result))
response = FeedParser()
response.feed(data)
return response.close()
def _deserialize_response(self, payload):
"""Convert string into httplib2 response and content.
Args:
payload: string, headers and body as a string.
Returns:
A pair (resp, content) like would be returned from httplib2.request.
"""
# Strip off the status line
status_line, payload = payload.split('\n', 1)
protocol, status, reason = status_line.split(' ')
# Parse the rest of the response
parser = FeedParser()
parser.feed(payload)
msg = parser.close()
msg['status'] = status
# Create httplib2.Response from the parsed headers.
resp = httplib2.Response(msg)
resp.reason = reason
resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
content = payload.split('\r\n\r\n', 1)[1]
return resp, content
def render_POST(self, request):
# XXX request.requestHeaders
headers = request.getAllHeaders()
form = FieldStorage(
fp=request.content,
headers=headers,
environ={
b'REQUEST_METHOD': request.method,
b'CONTENT_TYPE': headers[b'content-type'],
}
)
image = form[b"image"]
share = b"share" in form
p = FeedParser()
p.feed(
"Content-Disposition: " +
form['image'].headers.getheader('content-disposition'))
m = p.close()
filename = m.get_filename()
value = image.value
self.process_image(filename, value, share)
return redirectTo(form[b"return-url"].value, request)
def upload_mime(request):
"""
This callback receives raw MIME messages.
Why? Because the URL ends with 'mime'
POST parameters are:
'sender'
'recipient'
'body-mime'
MIME is a raw message which can be saved into an .msg or .eml file, parsed
with Python's MIME parser, etc
Use callbacks like this when you want to have full control over messages,
for example when you need access to all original MIME headers, etc
"""
if request.method == 'POST':
sender = request.POST.get('sender', None)
recipient = request.POST.get('recipient', None)
body_mime = request.POST.get('body-mime', None)
# Simplistic MIME parsing:
parser = FeedParser()
parser.feed(body_mime)
message = parser.close()
# Returned text is ignored but HTTP status code matters:
# Mailgun wants to see 200, otherwise it will make another attempt
return HttpResponse('OK')
def test_i18n_filenames(self):
parser = FeedParser(_factory=Message)
parser.feed("""\
Message-ID: <*****@*****.**>
Content-Type: multipart/mixed; boundary="------------050607040206050605060208"
This is a multi-part message in MIME format.
--------------050607040206050605060208
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: quoted-printable
Test message containing an attachment with an accented filename
--------------050607040206050605060208
Content-Disposition: attachment;
filename*=UTF-8''d%C3%A9jeuner.txt
Test content
--------------050607040206050605060208--
""")
msg = parser.close()
attachment = msg.get_payload(1)
try:
filename = attachment.get_filename()
except TypeError as error:
self.fail(error)
self.assertEqual(filename, u'd\xe9jeuner.txt')
def getLastMail():
ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv3)
#passwd = getpass.getpass()
mail = imaplib.IMAP4_SSL(config.imap['host'], config.imap['port'])
mail.login(config.imap['user'], config.imap['passwd'])
mail.select(config.imap['mailBox'], 1)
result, data = mail.search(None, "ALL")
ids = data[0] # data is a list.
id_list = ids.split() # ids is a space separated string
if len(id_list) == 0:
return (0, "")
latest_email_id = id_list[-1] # get the latest
result, data = mail.fetch(
latest_email_id,
"(RFC822)") # fetch the email body (RFC822) for the given ID
f = FeedParser()
f.feed(data[0][1])
rootMessage = f.close()
mailBody = rootMessage.get_payload(1).get_payload(decode=True)
# print (mailBody)
mail.close()
mail.logout()
del mail
return (latest_email_id, mailBody)
def email_as_list(self, email_id):
raw_email=self.raw_email(email_id)
f = FeedParser()
f.feed(raw_email)
rootMessage = f.close()
if(rootMessage.is_multipart()):
corps=rootMessage.get_payload(0).get_payload(decode=True).decode('utf-8')
# Récupérer le corps du mail en plain/text bien décodé
else:
corps=rootMessage.get_payload(decode=True).decode('utf-8')
subject=rootMessage.get('Subject')
#méthode Alex
# suppression des entêtes inutiles avec une regexp
subject=rootMessage.get('Subject')
for i in range(len(subject)):
if subject[i] == "=":
subject = subject[:i] + "%" + subject[i+1:]
elif subject[i] == "_":
subject = subject[:i] + " " + subject[i+1:]
subject = re.sub('(\n)*\%\?(UTF|utf)\-8\?(Q|B|q|b)\? *', '', subject)
subject = re.sub('\?\%(\r\n)*', '', subject)
subject=urllib.parse.unquote(subject)
#fin méthode Alex
date =rootMessage.get('Date')
exp=rootMessage.get('From')
email_liste=[]
email_liste.extend((email_id, exp, subject, corps, date))
return(email_liste)
def read_multi(self, environ, keep_blank_values, strict_parsing):
ib = self.innerboundary
if not valid_boundary(ib):
raise ValueError('Invalid boundary in multipart form: %r' % (ib,))
self.list = []
if self.qs_on_post:
query = urllib.parse.parse_qsl(self.qs_on_post, self.keep_blank_values, self.strict_parsing, encoding=self.encoding, errors=self.errors)
for (key, value) in query:
self.list.append(MiniFieldStorage(key, value))
FieldStorageClass = None
klass = self.FieldStorageClass or self.__class__
first_line = self.fp.readline()
if not isinstance(first_line, bytes):
raise ValueError('%s should return bytes, got %s' % (self.fp, type(first_line).__name__))
while True:
parser = FeedParser()
hdr_text = b''
while True:
data = self.fp.readline()
hdr_text += data
if not data.strip():
break
if not hdr_text:
break
parser.feed(hdr_text.decode(self.encoding, self.errors))
headers = parser.close()
part = klass(self.fp, headers, ib, environ, keep_blank_values, strict_parsing, self.limit - self.bytes_read, self.encoding, self.errors)
self.list.append(part)
if (part.done or self.bytes_read >= self.length) and self.length > 0:
break
self.skip_lines()
def pkg_info(self):
p = FeedParser()
data = self.egg_info_data("PKG-INFO")
if not data:
logger.warning("No PKG-INFO file found in %s", display_path(self.egg_info_path("PKG-INFO")))
p.feed(data or "")
return p.close()
def sendMail(msg, fromAddress):
toAddress = '*****@*****.**' # The 'To' address that Redmine emails should be sent to
try:
message = email.message_from_string(msg)
except:
print ("Cant set message variable with message_from_string")
try:
print raw_message
f = FeedParser()
f.feed(data[0][1])
message = f.close()
print "set via parser"
except:
print ("Can not set message variable")
try:
message.replace_header("From", fromAddress)
except:
print ("couldn't replace From, setting instead")
message.header.append("From", fromAddress)
print ("setting new To with " + toAddress)
try:
message.replace_header("To", toAddress)
except:
print ("couldn't replace To, setting instead")
message.header.append("To", toAddress)
server = smtplib.SMTP_SSL('myMailServer.myIntranet.intranet', 465) # Internal Email Server settings
server.set_debuglevel(1)
server.ehlo()
server.sendmail(fromAddress, toAddress, message.as_string())
server.quit()
print ("Sent an email")
def handle(self, *args, **options):
username = options.get('username')
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
raise CommandError("The specified user does not exist")
parser = FeedParser()
message_chunk = sys.stdin.read(10000)
while message_chunk:
parser.feed(message_chunk)
message_chunk = sys.stdin.read(10000)
message = parser.close()
try:
message_part = get_best_part(message)
subject = decode_header(message['Subject'])
body = make_text(message_part)
if int(options['verbosity']) > 1:
print(body)
action = Action(
user=user,
title=subject,
notes=body
)
action.insert_in_inbox()
action.save()
except RuntimeError:
# Recursion depth exceeded parsing message; throw the message away
pass
def test_i18n_filenames(self):
parser = FeedParser(_factory=Message)
parser.feed("""\
Message-ID: <*****@*****.**>
Content-Type: multipart/mixed; boundary="------------050607040206050605060208"
This is a multi-part message in MIME format.
--------------050607040206050605060208
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: quoted-printable
Test message containing an attachment with an accented filename
--------------050607040206050605060208
Content-Disposition: attachment;
filename*=UTF-8''d%C3%A9jeuner.txt
Test content
--------------050607040206050605060208--
""")
msg = parser.close()
attachment = msg.get_payload(1)
try:
filename = attachment.get_filename()
except TypeError as error:
self.fail(error)
self.assertEqual(filename, u'd\xe9jeuner.txt')
def get_pkg_license(pkg):
pkg_license = None
metadata = None
if pkg.has_metadata('METADATA'):
metadata = pkg.get_metadata('METADATA')
if pkg.has_metadata('PKG-INFO') and metadata is None:
metadata = pkg.get_metadata('PKG-INFO')
if metadata is None:
return None
feed_parser = FeedParser()
feed_parser.feed(metadata)
parsed_metadata = feed_parser.close()
pkg_info = {}
for key in METADATA_KEYS:
pkg_info[key] = parsed_metadata.get(key, LICENSE_UNKNOWN)
pkg_license = pkg_info.get('license')
if metadata is not None \
and pkg_license is not None \
and pkg_license is not LICENSE_UNKNOWN:
message = message_from_string(metadata)
pkg_license = find_license_from_classifier(message)
return pkg_license
def _execute(self, http, order, requests):
"""Serialize batch request, send to server, process response.
Args:
http: httplib2.Http, an http object to be used to make the request with.
order: list, list of request ids in the order they were added to the
batch.
request: list, list of request objects to send.
Raises:
httplib2.HttpLib2Error if a transport error has occured.
apiclient.errors.BatchError if the response is the wrong format.
"""
message = MIMEMultipart('mixed')
# Message should not write out it's own headers.
setattr(message, '_write_headers', lambda self: None)
# Add all the individual requests.
for request_id in order:
request = requests[request_id]
msg = MIMENonMultipart('application', 'http')
msg['Content-Transfer-Encoding'] = 'binary'
msg['Content-ID'] = self._id_to_header(request_id)
body = self._serialize_request(request)
msg.set_payload(body)
message.attach(msg)
body = message.as_string()
headers = {}
headers['content-type'] = ('multipart/mixed; '
'boundary="%s"') % message.get_boundary()
resp, content = http.request(self._batch_uri, 'POST', body=body,
headers=headers)
if resp.status >= 300:
raise HttpError(resp, content, uri=self._batch_uri)
# Now break out the individual responses and store each one.
boundary, _ = content.split(None, 1)
# Prepend with a content-type header so FeedParser can handle it.
header = 'content-type: %s\r\n\r\n' % resp['content-type']
for_parser = header + content
parser = FeedParser()
parser.feed(for_parser)
mime_response = parser.close()
if not mime_response.is_multipart():
raise BatchError("Response not in multipart/mixed format.", resp=resp,
content=content)
for part in mime_response.get_payload():
request_id = self._header_to_id(part['Content-ID'])
response, content = self._deserialize_response(part.get_payload())
self._responses[request_id] = (response, content)
def _collect(self):
pkg_infos = list(self.path.glob("**/PKG-INFO"))
if not pkg_infos:
log.warning("%r has no PKG-INFO", self)
return
pkg_info = pkg_infos[0]
# parse metadata
parser = FeedParser()
parser.feed(pkg_info.open().read())
message = parser.close()
for key, value in message.items():
value = value.strip()
if not value or value == "UNKNOWN":
continue
key = key.lower().replace("-", "_")
if key in self.MULTI_KEYS:
self.metadata.setdefault(key, set()).add(value)
else:
self.metadata[key] = value
# parse requires
requires_path = pkg_info.parent / "requires.txt"
if requires_path.exists():
requires = requires_path.open().read()
for extra, reqs in sorted(pkg_resources.split_sections(requires),
key=lambda x: x[0] or ""):
if extra is None:
extra = "run"
for req in reqs:
self.add_requirement(req, extra)
def messagesFromMBox(mbox):
''' Generator that reads a UNIX mailbox and yields Message objects.
TODO: do we still want readMBox()?
'''
needClose = False
if type(mbox) in StringTypes:
mbox = open(mbox)
needClose = True
P = None
for line in mbox:
assert line[-1] == '\n', "short line in %s" % (mbox,)
if line.startswith('From '):
if P is not None:
yield P.close()
P = FeedParser()
elif P is None:
raise ValueError("line in UNIX mailbox before first From_ line")
else:
P.feed(line)
if P is not None:
yield P.close()
if needClose:
mbox.close()
def request(self, query, headers):
# httplib would really help but there isn't any trivial way
# that I know of to use your own socket with it.
request = Message()
for k,v in headers.iteritems():
request[k] = v
self.sock.send(query + "\r\n" + request.as_string())
buffer = self.sock.recv(4096)
try:
[result, data] = buffer.split('\r\n', 1)
except ValueError:
traceback.print_exc()
print >> sys.stderr, 'Buffer:', buffer
print >> sys.stderr, 'Query:', query
print >> sys.stderr, 'Headers:', headers
return False
result = result.split(' ')
if int(result[1]) != 200:
self.log.info('Request failed: %s', ' '.join(result))
raise httplib.BadStatusLine(' '.join(result))
response = FeedParser()
response.feed(data)
return response.close()
def pkg_info(self):
p = FeedParser()
data = self.egg_info_data('PKG-INFO')
if not data:
logger.warn('No PKG-INFO file found in %s' %
display_path(self.egg_info_path('PKG-INFO')))
p.feed(data or '')
return p.close()
def _parse_pkg_info_file(self, filepath):
# The PKG-INFO generated by the egg-info command is in an email feed
# format, so we use an email feedparser here to extract the metadata
# from the PKG-INFO file.
data = self._osutils.get_file_contents(filepath, binary=False)
parser = FeedParser()
parser.feed(data)
return parser.close()
def get_emailmessage_from_file(filename='notification.email'):
"""Load a mail parse it into a email.message.
"""
email_string = get_string_from_file(filename)
parser = FeedParser()
parser.feed(email_string)
email_message = parser.close()
return email_message
def get_content_disposition(content_disposition):
""" Get content disposition filename from given header. Do not include
"Content-Disposition:". Returns a unicode string! """
parser = FeedParser()
parser.feed(b'Content-Disposition: ' + content_disposition)
name = parser.close().get_filename()
if not isinstance(name, six.text_type):
name = name.decode('latin1', 'ignore')
return name
def get_content_disposition(content_disposition):
""" Get content disposition filename from given header. Do not include
"Content-Disposition:". Returns a unicode string! """
parser = FeedParser()
parser.feed(b'Content-Disposition: ' + content_disposition)
name = parser.close().get_filename()
if not isinstance(name, six.text_type):
name = name.decode('latin1', 'ignore')
return name
def get_content_disposition(content_disposition):
""" Get content disposition filename from given header. Do not include
"Content-Disposition:". Returns a unicode string! """
parser = FeedParser()
parser.feed(b"Content-Disposition: " + content_disposition)
name = parser.close().get_filename()
if not isinstance(name, unicode):
name = name.decode("latin1", "ignore")
return name
def parse_message(file_input, sender, recipient):
"""Parse message from file input and create MailRequest"""
from .request import MailRequest # circular imports
parser = FeedParser(partial(MailRequest, sender, recipient))
for line in file_input:
parser.feed(line)
return parser.close()
def parse_email(self, email):
fp = FeedParser()
fp.feed(open(email).read())
message = fp.close()
if message.is_multipart():
texts, html, images, videos, applications = self.extract_multipart(message)
else:
texts, html, images, videos, applications = (message.get_payload(), 0, 0, 0, 0)
return (message.items(), html, texts, images, videos, applications)
def _parse_pkg_info_file(self, filepath):
# type: (str) -> Message
# The PKG-INFO generated by the egg-info command is in an email feed
# format, so we use an email feedparser here to extract the metadata
# from the PKG-INFO file.
data = self._osutils.get_file_contents(filepath, binary=False)
parser = FeedParser()
parser.feed(data)
return parser.close()
def get_pkg_info(pkg):
(license_file, license_text) = get_pkg_included_file(
pkg,
('LICENSE*', 'LICENCE*', 'COPYING*')
)
(notice_file, notice_text) = get_pkg_included_file(
pkg,
('NOTICE*',)
)
pkg_info = {
'name': pkg.project_name,
'version': pkg.version,
'namever': str(pkg),
'licensefile': license_file,
'licensetext': license_text,
'noticefile': notice_file,
'noticetext': notice_text,
}
metadata = None
if pkg.has_metadata('METADATA'):
metadata = pkg.get_metadata('METADATA')
if pkg.has_metadata('PKG-INFO') and metadata is None:
metadata = pkg.get_metadata('PKG-INFO')
if metadata is None:
for key in METADATA_KEYS:
pkg_info[key] = LICENSE_UNKNOWN
return pkg_info
feed_parser = FeedParser()
feed_parser.feed(metadata)
parsed_metadata = feed_parser.close()
for key in METADATA_KEYS:
pkg_info[key] = parsed_metadata.get(key, LICENSE_UNKNOWN)
if metadata is not None:
message = message_from_string(metadata)
pkg_info['license_classifier'] = \
find_license_from_classifier(message)
if args.filter_strings:
for k in pkg_info:
if isinstance(pkg_info[k], list):
for i, item in enumerate(pkg_info[k]):
pkg_info[k][i] = item. \
encode(args.filter_code_page, errors="ignore"). \
decode(args.filter_code_page)
else:
pkg_info[k] = pkg_info[k]. \
encode(args.filter_code_page, errors="ignore"). \
decode(args.filter_code_page)
return pkg_info
def read_multi(self, environ, keep_blank_values, strict_parsing):
"""Internal: read a part that is itself multipart."""
ib = self.innerboundary
if not valid_boundary(ib):
raise ValueError("Invalid boundary in multipart form: %r" % (ib, ))
self.list = []
if self.qs_on_post:
query = urllib.parse.parse_qsl(
self.qs_on_post,
self.keep_blank_values,
self.strict_parsing,
encoding=self.encoding,
errors=self.errors,
)
for key, value in query:
self.list.append(MiniFieldStorage(key, value))
FieldStorageClass = None
klass = self.FieldStorageClass or self.__class__
first_line = self.fp.readline() # bytes
if not isinstance(first_line, bytes):
raise ValueError("%s should return bytes, got %s" %
(self.fp, type(first_line).__name__))
self.bytes_read += len(first_line)
# first line holds boundary ; ignore it, or check that
# b"--" + ib == first_line.strip() ?
while True:
parser = FeedParser()
hdr_text = b""
while True:
data = self.fp.readline()
hdr_text += data
if not data.strip():
break
if not hdr_text:
break
# parser takes strings, not bytes
self.bytes_read += len(hdr_text)
parser.feed(hdr_text.decode(self.encoding, self.errors))
headers = parser.close()
part = klass(
self.fp,
headers,
ib,
environ,
keep_blank_values,
strict_parsing,
self.limit - self.bytes_read,
self.encoding,
self.errors,
)
self.bytes_read += part.bytes_read
self.list.append(part)
if part.done or self.bytes_read >= self.length > 0:
break
self.skip_lines()
def pkg_info(self):
p = FeedParser()
data = self.egg_info_data('PKG-INFO')
if not data:
logger.warning(
'No PKG-INFO app.ini found in %s',
display_path(self.egg_info_path('PKG-INFO')),
)
p.feed(data or '')
return p.close()
def pkg_info(self):
p = FeedParser()
data = self.egg_info_data("PKG-INFO")
if not data:
logger.warning(
"No PKG-INFO file found in %s",
display_path(self.egg_info_path("PKG-INFO")),
)
p.feed(data or "")
return p.close()
def pkg_info(self):
p = FeedParser()
data = self.egg_info_data('PKG-INFO')
if not data:
logger.warning(
'No PKG-INFO file found in %s',
display_path(self.egg_info_path('PKG-INFO')),
)
p.feed(data or '')
return p.close()
def read_multi(self, environ, keep_blank_values, strict_parsing):
"""Internal: read a part that is itself multipart."""
ib = self.innerboundary
if not valid_boundary(ib):
raise ValueError('Invalid boundary in multipart form: %r' % (ib, ))
self.list = []
if self.qs_on_post:
query = urllib.parse.parse_qsl(self.qs_on_post,
self.keep_blank_values,
self.strict_parsing,
encoding=self.encoding,
errors=self.errors)
for key, value in query:
self.list.append(MiniFieldStorage(key, value))
klass = self.FieldStorageClass or self.__class__
first_line = self.fp.readline() # bytes
if not isinstance(first_line, bytes):
raise ValueError("%s should return bytes, got %s" \
% (self.fp, type(first_line).__name__))
self.bytes_read += len(first_line)
# Ensure that we consume the file until we've hit our inner boundary
while (first_line.strip() != (b"--" + self.innerboundary)
and first_line):
first_line = self.fp.readline()
self.bytes_read += len(first_line)
while True:
parser = FeedParser()
hdr_text = b""
while True:
data = self.fp.readline()
hdr_text += data
if not data.strip():
break
if not hdr_text:
break
# parser takes strings, not bytes
self.bytes_read += len(hdr_text)
parser.feed(hdr_text.decode(self.encoding, self.errors))
headers = parser.close()
# Some clients add Content-Length for part headers, ignore them
if 'content-length' in headers:
del headers['content-length']
part = klass(self.fp, headers, ib, environ, keep_blank_values,
strict_parsing, self.limit - self.bytes_read,
self.encoding, self.errors)
self.bytes_read += part.bytes_read
self.list.append(part)
if part.done or self.bytes_read >= self.length > 0:
break
self.skip_lines()
def search_packages_info(query):
"""
Gather details from installed distributions. Print distribution name,
version, location, and installed files. Installed files requires a
pip generated 'installed-files.txt' in the distributions '.egg-info'
directory.
"""
installed = dict([(p.project_name.lower(), p)
for p in pkg_resources.working_set])
query_names = [name.lower() for name in query]
for dist in [installed[pkg] for pkg in query_names if pkg in installed]:
package = {
'name': dist.project_name,
'version': dist.version,
'location': dist.location,
'requires': [dep.project_name for dep in dist.requires()],
}
file_list = None
metadata = None
if isinstance(dist, pkg_resources.DistInfoDistribution):
# RECORDs should be part of .dist-info metadatas
if dist.has_metadata('RECORD'):
lines = dist.get_metadata_lines('RECORD')
paths = [l.split(',')[0] for l in lines]
paths = [os.path.join(dist.location, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata('METADATA'):
metadata = dist.get_metadata('METADATA')
else:
# Otherwise use pip's log for .egg-info's
if dist.has_metadata('installed-files.txt'):
paths = dist.get_metadata_lines('installed-files.txt')
paths = [os.path.join(dist.egg_info, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata('PKG-INFO'):
metadata = dist.get_metadata('PKG-INFO')
if dist.has_metadata('entry_points.txt'):
entry_points = dist.get_metadata_lines('entry_points.txt')
package['entry_points'] = entry_points
# @todo: Should pkg_resources.Distribution have a
# `get_pkg_info` method?
feed_parser = FeedParser()
feed_parser.feed(metadata)
pkg_info_dict = feed_parser.close()
for key in ('metadata-version', 'summary', 'home-page', 'author',
'author-email', 'license'):
package[key] = pkg_info_dict.get(key)
if file_list:
package['files'] = sorted(file_list)
yield package
def search_packages_info(query):
"""
Gather details from installed distributions. Print distribution name,
version, location, and installed files. Installed files requires a
pip generated 'installed-files.txt' in the distributions '.egg-info'
directory.
"""
installed = dict(
[(p.project_name.lower(), p) for p in pkg_resources.working_set])
query_names = [name.lower() for name in query]
for dist in [installed[pkg] for pkg in query_names if pkg in installed]:
package = {
'name': dist.project_name,
'version': dist.version,
'location': dist.location,
'requires': [dep.project_name for dep in dist.requires()],
}
file_list = None
metadata = None
if isinstance(dist, pkg_resources.DistInfoDistribution):
# RECORDs should be part of .dist-info metadatas
if dist.has_metadata('RECORD'):
lines = dist.get_metadata_lines('RECORD')
paths = [l.split(',')[0] for l in lines]
paths = [os.path.join(dist.location, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata('METADATA'):
metadata = dist.get_metadata('METADATA')
else:
# Otherwise use pip's log for .egg-info's
if dist.has_metadata('installed-files.txt'):
paths = dist.get_metadata_lines('installed-files.txt')
paths = [os.path.join(dist.egg_info, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata('PKG-INFO'):
metadata = dist.get_metadata('PKG-INFO')
if dist.has_metadata('entry_points.txt'):
entry_points = dist.get_metadata_lines('entry_points.txt')
package['entry_points'] = entry_points
# @todo: Should pkg_resources.Distribution have a
# `get_pkg_info` method?
feed_parser = FeedParser()
feed_parser.feed(metadata)
pkg_info_dict = feed_parser.close()
for key in ('metadata-version', 'summary',
'home-page', 'author', 'author-email', 'license'):
package[key] = pkg_info_dict.get(key)
if file_list:
package['files'] = sorted(file_list)
yield package
def _get_metadata_from_entrypoint(cls, entrypoint, extension_id):
"""Return metadata information from an entrypoint.
This is used internally to parse and validate package information from
an entrypoint for use in ExtensionInfo.
Args:
entrypoint (pkg_resources.EntryPoint):
The EntryPoint pointing to the extension class.
extension_id (unicode):
The extension's ID.
Returns:
dict:
The resulting metadata dictionary.
"""
dist = entrypoint.dist
try:
# Wheel, or other modern package.
lines = dist.get_metadata_lines('METADATA')
except IOError:
try:
# Egg, or other legacy package.
lines = dist.get_metadata_lines('PKG-INFO')
except IOError:
lines = []
logging.error('No METADATA or PKG-INFO found for the package '
'containing the %s extension. Information on '
'the extension may be missing.',
extension_id)
data = '\n'.join(lines)
# Try to decode the PKG-INFO content. If no decoding method is
# successful then the PKG-INFO content will remain unchanged and
# processing will continue with the parsing.
for enc in cls.encodings:
try:
data = data.decode(enc)
break
except UnicodeDecodeError:
continue
else:
logging.warning(
'Failed decoding PKG-INFO content for extension %s',
entrypoint.name)
p = FeedParser()
p.feed(data)
pkg_info = p.close()
return dict(pkg_info.items())
def _get_metadata_from_entrypoint(cls, entrypoint, extension_id):
"""Return metadata information from an entrypoint.
This is used internally to parse and validate package information from
an entrypoint for use in ExtensionInfo.
Args:
entrypoint (pkg_resources.EntryPoint):
The EntryPoint pointing to the extension class.
extension_id (unicode):
The extension's ID.
Returns:
dict:
The resulting metadata dictionary.
"""
dist = entrypoint.dist
try:
# Wheel, or other modern package.
lines = dist.get_metadata_lines('METADATA')
except IOError:
try:
# Egg, or other legacy package.
lines = dist.get_metadata_lines('PKG-INFO')
except IOError:
lines = []
logging.error(
'No METADATA or PKG-INFO found for the package '
'containing the %s extension. Information on '
'the extension may be missing.', extension_id)
data = '\n'.join(lines)
# Try to decode the PKG-INFO content. If no decoding method is
# successful then the PKG-INFO content will remain unchanged and
# processing will continue with the parsing.
for enc in cls.encodings:
try:
data = data.decode(enc)
break
except UnicodeDecodeError:
continue
else:
logging.warning(
'Failed decoding PKG-INFO content for extension %s',
entrypoint.name)
p = FeedParser()
p.feed(data)
pkg_info = p.close()
return dict(pkg_info.items())
def parse_metadata_file(contents: str) -> Message:
"""Parse :pep:`376` ``PKG-INFO``-style metadata files.
``METADATA`` and ``WHEEL`` files (as per :pep:`427`) use the same syntax
and can also be parsed using this function.
:param contents: The entire contents of the file
"""
feed_parser = FeedParser()
feed_parser.feed(contents)
return feed_parser.close()
def scriptTask(user, data, callback):
d = defer.Deferred()
parser = FeedParser()
parser.feed(data)
message = parser.close()
callback(user, message)
d.callback(None)
return d
def read_multi(self, environ, keep_blank_values, strict_parsing):
"""Internal: read a part that is itself multipart."""
ib = self.innerboundary
if not valid_boundary(ib):
raise ValueError('Invalid boundary in multipart form: %r' % (ib,))
self.list = []
if self.qs_on_post:
query = urllib.parse.parse_qsl(
self.qs_on_post, self.keep_blank_values, self.strict_parsing,
encoding=self.encoding, errors=self.errors)
for key, value in query:
self.list.append(MiniFieldStorage(key, value))
klass = self.FieldStorageClass or self.__class__
first_line = self.fp.readline() # bytes
if not isinstance(first_line, bytes):
raise ValueError("%s should return bytes, got %s" \
% (self.fp, type(first_line).__name__))
self.bytes_read += len(first_line)
# Ensure that we consume the file until we've hit our inner boundary
while (first_line.strip() != (b"--" + self.innerboundary) and
first_line):
first_line = self.fp.readline()
self.bytes_read += len(first_line)
while True:
parser = FeedParser()
hdr_text = b""
while True:
data = self.fp.readline()
hdr_text += data
if not data.strip():
break
if not hdr_text:
break
# parser takes strings, not bytes
self.bytes_read += len(hdr_text)
parser.feed(hdr_text.decode(self.encoding, self.errors))
headers = parser.close()
# Some clients add Content-Length for part headers, ignore them
if 'content-length' in headers:
del headers['content-length']
part = klass(self.fp, headers, ib, environ, keep_blank_values,
strict_parsing,self.limit-self.bytes_read,
self.encoding, self.errors)
self.bytes_read += part.bytes_read
self.list.append(part)
if part.done or self.bytes_read >= self.length > 0:
break
self.skip_lines()
def parse(self, fp, headersonly=True):
"""Create a message structure from the data in a file."""
feedparser = FeedParser(self._class)
feedparser._set_headersonly()
mp = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ)
data = ""
while True:
line = mp.readline()
data = data + line
if line == "\n":
break
feedparser.feed(data) # mp[0:5000])
return feedparser.close()
def get_metadata(dist):
if (isinstance(dist, pkg_resources.DistInfoDistribution)
and dist.has_metadata('METADATA')):
metadata = dist.get_metadata('METADATA')
elif dist.has_metadata('PKG-INFO'):
metadata = dist.get_metadata('PKG-INFO')
else:
logger.warning("No metadata found in %s", display_path(dist.location))
metadata = ''
feed_parser = FeedParser()
feed_parser.feed(metadata)
return feed_parser.close()
def get_metadata(dist):
# type: (Distribution) -> Message
if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata("METADATA"):
metadata = dist.get_metadata("METADATA")
elif dist.has_metadata("PKG-INFO"):
metadata = dist.get_metadata("PKG-INFO")
else:
logger.warning("No metadata found in %s", display_path(dist.location))
metadata = ""
feed_parser = FeedParser()
feed_parser.feed(metadata)
return feed_parser.close()
def get_message(mailbox, message):
s = siteconfig.storage(mailbox)
rawmsg = s.get_message(str(message))
if request.method == "HEAD":
parser = FeedParser()
parser.feed(rawmsg)
try:
msg = parser.close()
except:
return ("Unable to parse message", 500)
return "".join(["%{0}: {1}\n".format(k, v) for (k, v) in msg.items()])
else:
return (rawmsg, 200, {"Content-Type": "application/json"})
def get_metadata(dist):
if (isinstance(dist, pkg_resources.DistInfoDistribution) and
dist.has_metadata('METADATA')):
metadata = dist.get_metadata('METADATA')
elif dist.has_metadata('PKG-INFO'):
metadata = dist.get_metadata('PKG-INFO')
else:
logger.warning("No metadata found in %s", display_path(dist.location))
metadata = ''
feed_parser = FeedParser()
feed_parser.feed(metadata)
return feed_parser.close()
def read_multi(self, environ, keep_blank_values, strict_parsing):
"""Internal: read a part that is itself multipart."""
ib = self.innerboundary
if not valid_boundary(ib):
raise ValueError("Invalid boundary in multipart form: %r" % (ib,))
self.list = []
if self.qs_on_post:
query = urllib.parse.parse_qsl(
self.qs_on_post, self.keep_blank_values, self.strict_parsing, encoding=self.encoding, errors=self.errors
)
for key, value in query:
self.list.append(MiniFieldStorage(key, value))
FieldStorageClass = None
klass = self.FieldStorageClass or self.__class__
first_line = self.fp.readline() # bytes
if not isinstance(first_line, bytes):
raise ValueError("%s should return bytes, got %s" % (self.fp, type(first_line).__name__))
self.bytes_read += len(first_line)
# first line holds boundary ; ignore it, or check that
# b"--" + ib == first_line.strip() ?
while True:
parser = FeedParser()
hdr_text = b""
while True:
data = self.fp.readline()
hdr_text += data
if not data.strip():
break
if not hdr_text:
break
# parser takes strings, not bytes
self.bytes_read += len(hdr_text)
parser.feed(hdr_text.decode(self.encoding, self.errors))
headers = parser.close()
part = klass(
self.fp,
headers,
ib,
environ,
keep_blank_values,
strict_parsing,
self.limit - self.bytes_read,
self.encoding,
self.errors,
)
self.bytes_read += part.bytes_read
self.list.append(part)
if part.done or self.bytes_read >= self.length > 0:
break
self.skip_lines()
def __init__(self, entrypoint, ext_class):
data = '\n'.join(entrypoint.dist.get_metadata_lines('PKG-INFO'))
# Try to decode the PKG-INFO content. If no decoding method is
# successful then the PKG-INFO content will remain unchanged and
# processing will continue with the parsing.
for enc in self.encodings:
try:
data = data.decode(enc)
break
except UnicodeDecodeError:
continue
else:
logging.warning(
'Failed decoding PKG-INFO content for extension %s',
entrypoint.name)
p = FeedParser()
p.feed(data)
pkg_info = p.close()
# Extensions will often override "Name" to be something
# user-presentable, but we sometimes need the package name
self.package_name = pkg_info.get('Name')
metadata = dict(pkg_info.items())
if ext_class.metadata is not None:
metadata.update(ext_class.metadata)
self.metadata = metadata
self.name = metadata.get('Name')
self.version = metadata.get('Version')
self.summary = metadata.get('Summary')
self.description = metadata.get('Description')
self.author = metadata.get('Author')
self.author_email = metadata.get('Author-email')
self.license = metadata.get('License')
self.url = metadata.get('Home-page')
self.author_url = metadata.get('Author-home-page', self.url)
self.app_name = '.'.join(ext_class.__module__.split('.')[:-1])
self.enabled = False
self.installed = False
self.is_configurable = ext_class.is_configurable
self.has_admin_site = ext_class.has_admin_site
self.installed_htdocs_path = \
os.path.join(settings.MEDIA_ROOT, 'ext', self.package_name)
self.installed_static_path = \
os.path.join(settings.STATIC_ROOT, 'ext', ext_class.id)
def get_pkginfo(self):
"""Gets package info by reading PKG-INFO file"""
egg_info_dir = self._get_package_egg_info_path()
pkg_info_path = os.path.join(egg_info_dir or self.dist_dir, "PKG-INFO")
if not os.path.exists(pkg_info_path):
raise Exception("PKG-INFO not found %s" % self.name)
with open(pkg_info_path, 'r') as pkg_info:
data = pkg_info.read()
p = FeedParser()
p.feed(data.strip())
return p.close()
class PhotoMessage(object):
implements(smtp.IMessage)
def __init__(self):
self.email = FeedParser()
def lineReceived(self, line):
self.email.feed(line + '\n')
@defer.inlineCallbacks
def eomReceived(self):
email = self.email.close()
photo_id = yield photoIDFromKey(email['Subject'])
fname = extract_first_image(email, photo_id)
yield processImage(fname)
def search_packages_info(query):
from email.parser import FeedParser
import pkg_resources
installed = dict([(p.project_name.lower(), p) for p in pkg_resources.working_set])
query_names = [name.lower() for name in query]
for dist in [installed[pkg] for pkg in query_names if pkg in installed]:
package = {'name' : dist.project_name,
'version' : dist.version,
'location': dist.location,
'requires': [dep.project_name for dep in dist.requires()]}
return package
file_list = None
metadata=None
if isinstance(dist, pkg_resources.DistInfoDistribution):
if dist.has_metadata('RECORD'):
lines = dist.get_metadata_lines('RECORD')
paths = [l.split(',')[0] for l in lines]
paths = [os.path.join(dist.location, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata('METADATA'):
metadata = dist.get_metadata('METADATA')
else:
if dist.has_metadata('installed-files.txt'):
paths = dist.get_metadata_lines('installed-files.txt')
paths = [os.path.join(dist.egg_info, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata('PKG-INFO'):
metadata = dist.get_metadata('PKG-INFO')
if dist.has_metadata('entry_points.txt'):
entry_points = dist.get_metadata_lines('entry_points.txt')
package['entry_points'] = entry_points
feed_parser = FeedParser()
feed_parser.feed(metadata)
pkg_info_dict = feed_parser.close()
for key in ('metadata-version', 'summary',
'home-page', 'author', 'author-email', 'license', 'keywords'):
package[key] = pkg_info_dict.get(key)
if file_list:
package['files'] = sorted(file_list)
return package
def msg_to_dict(rawmsg):
parser = FeedParser()
parser.feed(rawmsg)
try:
msg = parser.close()
except:
return {}
data = {
"to": msg["to"] if "to" in msg else "",
"from": msg["from"] if "from" in msg else "",
"bcc": msg["bcc"] if "bcc" in msg else "",
"subject": msg["subject"] if "subject" in msg else "",
"cc": msg["cc"] if "cc" in msg else "",
"size": len(msg.__str__()),
"date": msg["date"] if "date" in msg else "0",
}
return data
def check_dist_requires_python(dist):
metadata = get_metadata(dist)
feed_parser = FeedParser()
feed_parser.feed(metadata)
pkg_info_dict = feed_parser.close()
requires_python = pkg_info_dict.get("Requires-Python")
try:
if not check_requires_python(requires_python):
raise exceptions.UnsupportedPythonVersion(
"%s requires Python '%s' but the running Python is %s"
% (dist.project_name, requires_python, ".".join(map(str, sys.version_info[:3])))
)
except specifiers.InvalidSpecifier as e:
logger.warning(
"Package %s has an invalid Requires-Python entry %s - %s" % (dist.project_name, requires_python, e)
)
return
def read_file(path):
if path == '-':
file = sys.stdin
else:
file = open(path, mode='rb')
parser = FeedParser()
logger.debug('loading %r', file.name)
with file as f:
data = f.read()
try:
parser.feed(data.decode('utf-8'))
except UnicodeDecodeError:
return data
message = parser.close()
if len(message._headers) == 0:
return data
return message
