def log(self, parent=None, limit=100):
# TODO(dcramer): we should make this streaming
cmd = ['log', '--template=%s' % (LOG_FORMAT,)]
if parent:
cmd.append('-r %s' % (parent,))
if limit:
cmd.append('--limit=%d' % (limit,))
result = self.run(cmd)
for chunk in BufferParser(result, '\x02'):
(sha, author, author_date, parents, branches, message) = chunk.split('\x01')
branches = filter(bool, branches.split(' ')) or ['default']
parents = filter(lambda x: x and x != '0' * 40, parents.split(' '))
author_date = datetime.utcfromtimestamp(
mktime_tz(parsedate_tz(author_date)))
yield RevisionResult(
id=sha,
author=author,
author_date=author_date,
message=message,
parents=parents,
branches=branches,
)
def get_delivery_time (msg):
# Figure out the delivery time.
dtime = None
if msg.has_key("Delivery-date"):
# eg. "Thu, 12 Jul 2001 08:47:20 -0400" to 994942040 (seconds
# since epoch in UTC)
dtime = mktime_tz(parsedate_tz(msg["Delivery-date"]))
elif msg.unixfrom:
# Parse eg.
# "From [email protected] Thu Jul 12 08:47:20 2001"
# -- this is the "From " line format used by Exim; hopefully other
# MTAs do the same!
m = re.match(r'^From (\S+) +(\w{3} \w{3}\s+\d\d? \d\d:\d\d:\d\d \d{4})$',
msg.unixfrom)
if not m:
warn("warning: could not parse \"From \" line: %s" % msg.unixfrom)
else:
(return_path, dtime_str) = m.groups()
# Eg. "Thu Jul 12 08:47:20 2001" -> 994945640 -- note that
# this might be different from what we get parsing the same
# date string above, because this one doesn't include the
# timezone. Sigh.
dtime = mktime(strptime(dtime_str, "%c"))
# Attempt to detect and correct for DST differences.
# (This works if we parsed a summer time during the winter;
# what about the inverse?)
dtime_str_curtz = ctime(dtime)
if dtime_str_curtz != dtime_str:
dtime_curtz = mktime(strptime(dtime_str_curtz, "%c"))
diff = dtime_curtz - dtime
dtime -= diff
return dtime
def loadfrommessage(self, msg):
self.tofield = msg.getaddrlist("To")
f = msg.getaddr("From")
self.fromfield = f[1]
self.realfromfield = f[0]
if not self.realfromfield:
self.realfromfield = self.fromfield
self.ccfield = msg.getaddrlist("Cc")
if not self.ccfield:
self.ccfield = ()
self.subjectfield = msg.getheader("Subject")
if not self.subjectfield:
self.subjectfield = ""
self.annotation = msg.getheader("X-SQmaiL-Annotation")
if not self.annotation:
self.annotation = ""
self.readstatus = "Unread"
# Work out the date the message arrived.
r = ""
for i in msg.getallmatchingheaders("Received"):
r = r + i
p = string.find(r, ";")
if (p == -1):
self.date = 0
else:
r = r[p+1:]
r = rfc822.parsedate_tz(r)
r = rfc822.mktime_tz(r)
self.date = r
self.headers = string.join(msg.headers, "")
self.body = msg.fp.read()
def get_pubdate(entry):
"""Try to determine the real pubDate of a feedparser entry
This basically takes the updated_parsed value, but also uses some more
advanced techniques to work around various issues with ugly feeds.
"published" now also takes precedence over "updated" (with updated used as
a fallback if published is not set/available). RSS' "pubDate" element is
"updated", and will only be used if published_parsed is not available.
"""
pubdate = entry.get('published_parsed', None)
if pubdate is None:
pubdate = entry.get('updated_parsed', None)
if pubdate is None:
# See http://code.google.com/p/feedparser/issues/detail?id=327
updated = entry.get('published', entry.get('updated', None))
if updated is not None:
# FIXME: This is kludgy. We should write our own date handler
# and register it with feedparser.registerDateHandler() and/or
# wait for feedparser to add support for this bogus date format.
pubdate = feedparser._parse_date(updated.replace(',', ''))
if pubdate is None:
# Cannot determine pubdate - party like it's 1970!
return 0
return mktime_tz(pubdate + (0,))
def _parse_sibling(self, sibling, headers, data):
"""
Parses a single sibling out of a response.
"""
sibling.exists = True
# Parse the headers...
for header, value in headers:
header = header.lower()
if header == "content-type":
sibling.content_type, sibling.charset = self._parse_content_type(value)
elif header == "etag":
sibling.etag = value
elif header == "link":
sibling.links = self._parse_links(value)
elif header == "last-modified":
sibling.last_modified = mktime_tz(parsedate_tz(value))
elif header.startswith("x-riak-meta-"):
metakey = header.replace("x-riak-meta-", "")
sibling.usermeta[metakey] = value
elif header.startswith("x-riak-index-"):
field = header.replace("x-riak-index-", "")
reader = csv.reader([value], skipinitialspace=True)
for line in reader:
for token in line:
token = decode_index_value(field, token)
sibling.add_index(field, token)
elif header == "x-riak-deleted":
sibling.exists = False
sibling.encoded_data = data
return sibling
def get_contents_to_filename(self, filename, headers=None,
cb=None, num_cb=10,
torrent=False,
version_id=None,
res_download_handler=None,
response_headers=None):
"""
Retrieve an object from S3 using the name of the Key object as the
key in S3. Store contents of the object to a file named by 'filename'.
See get_contents_to_file method for details about the
parameters.
:type filename: string
:param filename: The filename of where to put the file contents
:type headers: dict
:param headers: Any additional headers to send in the request
:type cb: function
:param cb: a callback function that will be called to report
progress on the upload. The callback should accept
two integer parameters, the first representing the
number of bytes that have been successfully
transmitted to S3 and the second representing the
size of the to be transmitted object.
:type cb: int
:param num_cb: (optional) If a callback is specified with
the cb parameter this parameter determines the
granularity of the callback by defining
the maximum number of times the callback will
be called during the file transfer.
:type torrent: bool
:param torrent: If True, returns the contents of a torrent file
as a string.
:type res_upload_handler: ResumableDownloadHandler
:param res_download_handler: If provided, this handler will
perform the download.
:type response_headers: dict
:param response_headers: A dictionary containing HTTP headers/values
that will override any headers associated with
the stored object in the response.
See http://goo.gl/EWOPb for details.
"""
fp = open(filename, 'wb')
self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent,
version_id=version_id,
res_download_handler=res_download_handler,
response_headers=response_headers)
fp.close()
# if last_modified date was sent from s3, try to set file's timestamp
if self.last_modified != None:
try:
modified_tuple = rfc822.parsedate_tz(self.last_modified)
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
os.utime(fp.name, (modified_stamp, modified_stamp))
except Exception: pass
def createEvent(evType, tstamp, name = None, contextList = [],
entityList = []):
"""
Create an XML element representing an event. Returns the XML object
It expects:
evType: Enum
tstamp: datetime object
name : string
contextList: List of context elements
entityList: List of entity elements
"""
result = etree.Element('event')
result.attrib['type'] = eventName(evType)
if tstamp == None:
tstamp = datetime.datetime.now()
result.attrib['datetime'] = rfc822.formatdate(rfc822.mktime_tz(rfc822.parsedate_tz(tstamp.strftime("%a, %d %b %Y %H:%M:%S"))))
if name != None:
result.attrib['name'] = name
for el in entityList + contextList:
result.append(el)
# Create the ID
m = hashlib.sha1()
m.update(etree.tostring(result))
result.attrib['id'] = m.hexdigest()
return result
def was_modified_since(header=None, mtime=0, size=0):
"""
Was something modified since the user last downloaded it?
header
This is the value of the If-Modified-Since header. If this is None,
I'll just return True.
mtime
This is the modification time of the item we're talking about.
size
This is the size of the item we're talking about.
"""
try:
if header is None:
raise ValueError
matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header,
re.IGNORECASE)
header_mtime = rfc822.mktime_tz(rfc822.parsedate_tz(
matches.group(1)))
header_len = matches.group(3)
if header_len and int(header_len) != size:
raise ValueError
if mtime > header_mtime:
raise ValueError
except (AttributeError, ValueError):
return True
return False
def matches_value( self, v ): t0 = time.time() then = rfc822.parsedate_tz( v ) t1 = rfc822.mktime_tz(then) return (t0 - t1) > self.age
def populate(self, sub):
file = open(os.path.join(self.archdir, str(sub), 'index'))
linepair = file.readline() + file.readline()
prev_timestamp = 0
while linepair:
match = _rx_index.match(linepair.rstrip())
if match:
g = match.groups()
msgnum = int(g[0])
try:
timestamp = rfc822.mktime_tz(rfc822.parsedate_tz(g[3]))
except:
timestamp = prev_timestamp + 1
prev_timestamp = timestamp
localtime = time.localtime(timestamp)
self.msgs[msgnum] = {
MSGNUM: msgnum,
THREADID: g[1],
SUBJECT: g[2],
DATE: g[3],
TIMESTAMP: timestamp,
AUTHORID: g[4],
AUTHOR: g[5],
MONTH: localtime[0] * 100 + localtime[1],
}
linepair = file.readline() + file.readline()
file.close()
def parse_pubdate(text):
"""Parse a date string into a Unix timestamp
>>> parse_pubdate('Fri, 21 Nov 1997 09:55:06 -0600')
880127706
>>> parse_pubdate('')
0
>>> parse_pubdate('unknown')
0
"""
if not text:
return 0
parsed = parsedate_tz(text)
if parsed is not None:
return int(mktime_tz(parsed))
# TODO: Fully RFC 3339-compliant parsing (w/ timezone)
try:
parsed = time.strptime(text[:19], '%Y-%m-%dT%H:%M:%S')
if parsed is not None:
return int(time.mktime(parsed))
except Exception:
pass
logger.error('Cannot parse date: %s', repr(text))
return 0
def execute(self, observation):
station_id = observation['station_id']
raw_time = observation['observation_time_rfc822']
parsed_time = datetime.datetime.fromtimestamp(rfc822.mktime_tz(rfc822.parsedate_tz(raw_time)))
epoch = datetime.datetime.utcfromtimestamp(0)
delta = int((parsed_time - epoch).total_seconds())
observation['ObservationTime'] = delta
observation['StationId'] = station_id
composite_key = "%s_%d" % (station_id, delta)
observation['CompositeKey'] = composite_key
region = os.environ['AWS_DEFAULT_REGION']
accessKey = os.environ['AWS_ACCESS_KEY']
secretKey = os.environ['AWS_SECRET_KEY']
try:
connx = boto.dynamodb2.connect_to_region(region, aws_access_key_id=accessKey, aws_secret_access_key=secretKey)
obs_table = Table('VocalPelicanObservation', connection = connx)
test_row = obs_table.get_item(CompositeKey=composite_key)
except JSONResponseError as responseError:
# authentication problem
print responseError
except boto.dynamodb2.exceptions.ItemNotFound as responseError:
# not found implies safe to add
return obs_table.put_item(observation)
return False
def open(self):
# XXX in future add support for compression
headers = {'Accept-Encoding': ''}
if _requests_version == '0':
self._data_response = self._session.get(self._url('data'),
prefetch=False,
headers=headers)
else:
self._data_response = self._session.get(self._url('data'),
stream=True,
headers=headers)
self._validate_response(self._data_response)
size = self._data_response.headers.get('Content-Length', None)
if size is not None:
size = int(size)
self._size = size
modified = self._data_response.headers.get('Last-Modified', None)
if modified is not None:
modified = rfc822.mktime_tz(rfc822.parsedate_tz(modified))
self._modified = modified
mimetype = self._data_response.headers.get('Content-Type',
'application/octet-stream')
self._mimetype = mimetype
return self._data_response.raw
def _parse_midmo_date(datestring):
"""
returns a local datetime corresponding to
the datestring given.
"""
# these appear to be rfc822/2822, not documented.
return datetime.fromtimestamp(rfc822.mktime_tz(rfc822.parsedate_tz(datestring)))
def copy_uri(self, uri, filename):
"""
Copies the contents of the resource given by 'uri' to 'filename'.
"""
source = Uri.UrlOpen(uri)
try:
source_mtime = source.headers.getdate_tz('last-modified')
source_mtime = rfc822.mktime_tz(source_mtime)
try:
target_mtime = os.path.getmtime(filename)
except OSError:
target_mtime = -1
if not (self.force or source_mtime > target_mtime):
self.announce("not copying %s (output up-to-date)" % uri, 1)
return filename, False
self.announce("copying %s -> %s" % (uri, filename), 2)
if not self.dry_run:
f = open(filename, 'wb')
try:
f.write(source.read())
finally:
f.close()
finally:
source.close()
return filename, True
def _onsuccess(response):
if response.status == 200:
checksum = response.headers['Etag'].strip('"')
last_modified = response.headers['Last-Modified']
modified_tuple = rfc822.parsedate_tz(last_modified)
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
return {'checksum': checksum, 'last_modified': modified_stamp}
def _parse_date_rfc822(dateString):
'''Parse an RFC822, RFC1123, RFC2822, or asctime-style date'''
data = dateString.split()
if not data:
return None
if data[0][-1] in (',', '.') or data[0].lower() in rfc822._daynames:
del data[0]
if len(data) == 4:
s = data[3]
i = s.find('+')
if i > 0:
data[3:] = [s[:i], s[i+1:]]
else:
data.append('')
dateString = " ".join(data)
# Account for the Etc/GMT timezone by stripping 'Etc/'
elif len(data) == 5 and data[4].lower().startswith('etc/'):
data[4] = data[4][4:]
dateString = " ".join(data)
if len(data) < 5:
dateString += ' 00:00:00 GMT'
tm = rfc822.parsedate_tz(dateString)
if tm:
# Jython doesn't adjust for 2-digit years like CPython does,
# so account for it by shifting the year so that it's in the
# range 1970-2069 (1970 being the year of the Unix epoch).
if tm[0] < 100:
tm = (tm[0] + (1900, 2000)[tm[0] < 70],) + tm[1:]
return time.gmtime(rfc822.mktime_tz(tm))
def get_pubdate(entry):
"""Try to determine the real pubDate of a feedparser entry
This basically takes the updated_parsed value, but also uses some more
advanced techniques to work around various issues with ugly feeds.
"published" now also takes precedence over "updated" (with updated used as
a fallback if published is not set/available). RSS' "pubDate" element is
"updated", and will only be used if published_parsed is not available.
If parsing the date into seconds since epoch returns an error (date is
before epoch or after the end of time), epoch is used as fallback.
This fixes https://bugs.gpodder.org/show_bug.cgi?id=2023
"""
pubdate = entry.get('published_parsed', None)
if pubdate is None:
pubdate = entry.get('updated_parsed', None)
if pubdate is None:
# Cannot determine pubdate - party like it's 1970!
return 0
try:
pubtimeseconds = mktime_tz(pubdate + (0,))
return pubtimeseconds
except(OverflowError,ValueError):
logger.warn('bad pubdate %s is before epoch or after end of time (2038)',pubdate)
return 0
def _readdate(txt):
"""Interpret the string as a date value."""
import rfc822
date = rfc822.parsedate_tz(txt.strip())
if date is not None:
return rfc822.mktime_tz(date)
return None
def _normalize_rfc822_date( self, date_string ):
return datetime.fromtimestamp(
rfc822.mktime_tz(
rfc822.parsedate_tz(
date_string
)
)
)
def parse(self, *args, **kwargs):
""" return the time value (in seconds since 1970) """
value = self.__call__(*args, **kwargs)
if value:
try:
return mktime_tz(parsedate_tz(value))
except TypeError:
raise HTTPBadRequest(("Received an ill-formed timestamp for %s: %s\r\n") % (self.name, value))
def log(self, parent=None, branch=None, author=None, offset=0, limit=100, paths=None):
""" Gets the commit log for the repository.
Each revision returned has exactly one branch name associated with it.
This is the branch name encoded into the revision changeset description.
See documentation for the base for general information on this function.
"""
start_time = time()
# TODO(dcramer): we should make this streaming
cmd = ['log', '--template=%s' % (LOG_FORMAT,)]
if parent and branch:
raise ValueError('Both parent and branch cannot be set')
# Build the -r parameter value into r_str with branch, parent and author
r_str = None
if branch:
cmd.append('-b{0}'.format(branch))
if parent:
r_str = ('ancestors(%s)' % parent)
if author:
r_str = ('({r}) and author("{0}")' if r_str else 'author("{0}")')\
.format(author, r=r_str)
if r_str:
cmd.append('-r reverse({0})'.format(r_str))
if limit:
cmd.append('--limit=%d' % (offset + limit,))
if paths:
cmd.extend(["glob:" + p.strip() for p in paths])
result = self.run(cmd)
self.log_timing('log', start_time)
for idx, chunk in enumerate(BufferParser(result, '\x02')):
if idx < offset:
continue
(sha, author, author_date, parents, branches, message) = chunk.split('\x01')
branches = filter(bool, branches.split(' ')) or ['default']
parents = filter(lambda x: x and x != '0' * 40, parents.split(' '))
author_date = datetime.utcfromtimestamp(
mktime_tz(parsedate_tz(author_date)))
yield RevisionResult(
id=sha,
author=author,
author_date=author_date,
message=message,
parents=parents,
branches=branches,
)
def addMail(self, mailString):
""" Store mail as news item
Returns created item
"""
archive = self.context
pw = self.context.portal_workflow
(header, body) = splitMail(mailString)
# if 'keepdate' is set, get date from mail,
if self.getValueFor('keepdate'):
timetuple = rfc822.parsedate_tz(header.get('date'))
time = DateTime(rfc822.mktime_tz(timetuple))
# ... take our own date, clients are always lying!
else:
time = DateTime()
(TextBody, ContentType, HtmlBody, Attachments) = unpackMail(mailString)
# Test Zeitangabe hinter Subject
from datetime import date
today = date.today()
mydate = today.strftime("%d.%m.%Y")
# let's create the news item
subject = mime_decode_header(header.get('subject', 'No Subject'))
sender = mime_decode_header(header.get('from','No From'))
#title = "%s / %s" % (subject, sender)
title = "%s" % (subject)
new_id = IUserPreferredURLNormalizer(self.request).normalize(title)
id = self._findUniqueId(new_id)
# ContentType is only set for the TextBody
if ContentType:
body = TextBody
else:
body = self.HtmlToText(HtmlBody)
# als vorlaeufige Loesung
desc = "%s..." % (body[:60])
uni_aktuell_body = "<p><strong>%s: %s</strong></p> <p> </p><pre>%s</pre>" % (mydate, sender, body)
# uni_aktuell_body = '<p> </p>' + body
objid = self.context.invokeFactory(NewsItem.meta_type, id=id, title=title, text=uni_aktuell_body, description=desc)
mailObject = getattr(self.context, objid)
try:
#original pw.doActionFor(mailObject, 'hide')
pw.doActionFor(mailObject, 'publish')
except:
pass
return mailObject
def _item_publish_date(self, item):
"""Returns the UTC date that the comic strip was published
Converts a RFC822 string to a UTC datetime.
"""
parts = rfc822.parsedate_tz(item['pubDate'])
timestamp = rfc822.mktime_tz(parts)
return datetime.fromtimestamp(timestamp, pytz.utc)
def _onsuccess(boto_key):
checksum = boto_key.etag.strip('"')
last_modified = boto_key.last_modified
modified_tuple = rfc822.parsedate_tz(last_modified)
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
return {'checksum': checksum,
'last_modified': modified_stamp,
'width': boto_key.metadata.width,
'height': boto_key.metadata.height}
def _item_publish_date_tz(self, item):
"""Returns the date that the comic strip was published.
The original timezone is preserved.
"""
parts = rfc822.parsedate_tz(item['pubDate'])
timestamp = rfc822.mktime_tz(parts)
return datetime.fromtimestamp(timestamp)
def parse_date_timestamp(value):
if not value:
return None
t = parsedate_tz(value)
if t is None:
# Could not parse
return None
t = mktime_tz(t)
return t
def createSession(name, tstart, tstop):
"""
Create an XML element representing a session. Returns the XML object. It
expects:
- name: string
- tstart: datetime.datetime
- tstop: datetime.datetime
"""
result = etree.Element('session')
if name != None:
result.attrib['name'] = str(name)
if tstart != None:
result.attrib['begin'] = rfc822.formatdate(rfc822.mktime_tz(rfc822.parsedate_tz(tstart.strftime("%a, %d %b %Y %H:%M:%S"))))
if tstop != None:
result.attrib['end'] = rfc822.formatdate(rfc822.mktime_tz(rfc822.parsedate_tz(tstop.strftime("%a, %d %b %Y %H:%M:%S"))))
return lookupElement(result)
def was_modified_since(request, last_modified):
if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
if not if_modified_since:
return True
try:
req_last_modified = rfc822.mktime_tz(rfc822.parsedate_tz(if_modified_since))
except (TypeError, OverflowError):
return True
return req_last_modified != int(time.mktime(last_modified.timetuple()))
def got_contents_to_filename(response):
fp.close()
# if last_modified date was sent from s3, try to set file's timestamp
if self.last_modified != None:
try:
modified_tuple = rfc822.parsedate_tz(self.last_modified)
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
os.utime(fp.name, (modified_stamp, modified_stamp))
except Exception: pass
if callable(callback):
callback(response)
def _TimeAsRfc822(self, dt):
return formatdate(
mktime_tz(parsedate_tz(dt.strftime('%a, %d %b %Y %H:%M:%S'))))
def log(self,
parent=None,
branch=None,
author=None,
offset=0,
limit=100,
paths=None):
""" Gets the commit log for the repository.
Each revision returned has exactly one branch name associated with it.
This is the branch name encoded into the revision changeset description.
See documentation for the base for general information on this function.
"""
start_time = time()
# TODO(dcramer): we should make this streaming
cmd = ['log', '--template=%s' % (LOG_FORMAT, )]
if parent and branch:
raise ValueError('Both parent and branch cannot be set')
# Build the -r parameter value into r_str with branch, parent and author
r_str = None
if branch:
cmd.append('-b{0}'.format(branch))
if parent:
r_str = ('ancestors(%s)' % parent)
if author:
r_str = ('({r}) and author("{0}")' if r_str else 'author("{0}")')\
.format(author, r=r_str)
if r_str:
cmd.append('-r reverse({0})'.format(r_str))
if limit:
cmd.append('--limit=%d' % (offset + limit, ))
if paths:
cmd.extend(["glob:" + p.strip() for p in paths])
result = self.run(cmd)
self.log_timing('log', start_time)
for idx, chunk in enumerate(BufferParser(result, '\x02')):
if idx < offset:
continue
(sha, author, author_date, parents, branches,
message) = chunk.split('\x01')
branches = filter(bool, branches.split(' ')) or ['default']
parents = filter(lambda x: x and x != '0' * 40, parents.split(' '))
author_date = datetime.utcfromtimestamp(
mktime_tz(parsedate_tz(author_date)))
yield RevisionResult(
id=sha,
author=author,
author_date=author_date,
message=message,
parents=parents,
branches=branches,
)
def timeAsrfc822(theTime):
import rfc822
return rfc822.formatdate(
rfc822.mktime_tz(
rfc822.parsedate_tz(theTime.strftime("%a, %d %b %Y %H:%M:%S"))))
def _onsuccess(boto_key):
checksum = boto_key.etag.strip('"')
last_modified = boto_key.last_modified
modified_tuple = rfc822.parsedate_tz(last_modified)
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
return {'checksum': checksum, 'last_modified': modified_stamp}
def format_rfc2822_date(date):
"""Formats a date according to RFC2822's desires."""
return formatdate(rfc822.mktime_tz(date.utctimetuple() + (0, )))
def clockTime(s):
# local not supported under windows
# since we are using english anyway we skip that
# locale.setlocale(locale.LC_ALL, 'en_GB')
# return time.mktime(time.strptime(s[:20], '%d %b %Y %H:%M:%S'))
return rfc822.mktime_tz(rfc822.parsedate_tz(s))
def parse_rfc2822(rfc2822):
time_tuple = rfc822.parsedate_tz(rfc2822)
timestamp = rfc822.mktime_tz(time_tuple)
return datetime.datetime.utcfromtimestamp(timestamp)
def _parse_date(date):
tm = rfc822.parsedate_tz(date)
if tm:
return rfc822.mktime_tz(tm)
return 0
class MailIndex(object):
"""This is a lazily parsing object representing a mailpile index."""
MSG_IDX = 0
MSG_PTRS = 1
MSG_UNUSED = 2 # Was size, now reserved for other fun things
MSG_ID = 3
MSG_DATE = 4
MSG_FROM = 5
MSG_SUBJECT = 6
MSG_TAGS = 7
MSG_REPLIES = 8
MSG_CONV_ID = 9
def __init__(self, config):
self.config = config
self.STATS = {}
self.INDEX = []
self.PTRS = {}
self.MSGIDS = {}
self.CACHE = {}
self.MODIFIED = set()
def l2m(self, line):
return line.decode('utf-8').split(u'\t')
def m2l(self, message):
return (u'\t'.join([unicode(p) for p in message])).encode('utf-8')
def load(self, session=None):
self.INDEX = []
self.PTRS = {}
self.MSGIDS = {}
def process_line(line):
try:
line = line.strip()
if line and not line.startswith('#'):
pos, ptrs, junk, msgid, rest = line.split('\t', 4)
pos = int(pos, 36)
while len(self.INDEX) < pos + 1:
self.INDEX.append('')
self.INDEX[pos] = line
self.MSGIDS[msgid] = pos
for msg_ptr in ptrs:
self.PTRS[msg_ptr] = pos
except ValueError:
pass
if session: session.ui.mark('Loading metadata index...')
try:
fd = open(self.config.mailindex_file(), 'r')
for line in fd:
if line.startswith(GPG_BEGIN_MESSAGE):
for line in decrypt_gpg([line], fd):
process_line(line)
else:
process_line(line)
fd.close()
except IOError:
if session:
session.ui.warning(('Metadata index not found: %s') %
self.config.mailindex_file())
if session:
session.ui.mark('Loaded metadata for %d messages' %
len(self.INDEX))
def save_changes(self, session=None):
mods, self.MODIFIED = self.MODIFIED, set()
if mods:
if session: session.ui.mark("Saving metadata index changes...")
fd = gpg_open(self.config.mailindex_file(),
self.config.get('gpg_recipient'), 'a')
for pos in mods:
fd.write(self.INDEX[pos] + '\n')
fd.close()
flush_append_cache()
if session: session.ui.mark("Saved metadata index changes")
def save(self, session=None):
self.MODIFIED = set()
if session: session.ui.mark("Saving metadata index...")
fd = gpg_open(self.config.mailindex_file(),
self.config.get('gpg_recipient'), 'w')
fd.write('# This is the mailpile.py index file.\n')
fd.write('# We have %d messages!\n' % len(self.INDEX))
for item in self.INDEX:
fd.write(item + '\n')
fd.close()
flush_append_cache()
if session: session.ui.mark("Saved metadata index")
def update_ptrs_and_msgids(self, session):
session.ui.mark('Updating high level indexes')
for offset in range(0, len(self.INDEX)):
message = self.l2m(self.INDEX[offset])
if len(message) > self.MSG_CONV_ID:
self.MSGIDS[message[self.MSG_ID]] = offset
for msg_ptr in message[self.MSG_PTRS].split(','):
self.PTRS[msg_ptr] = offset
else:
session.ui.warning('Bogus line: %s' % line)
def try_decode(self, text, charset):
for cs in (charset, 'iso-8859-1', 'utf-8'):
if cs:
try:
return text.decode(cs)
except (UnicodeEncodeError, UnicodeDecodeError, LookupError):
pass
return "".join(i for i in text if ord(i) < 128)
def hdr(self, msg, name, value=None):
try:
if value is None and msg:
value = msg[name]
decoded = email.header.decode_header(value or '')
return (' '.join([self.try_decode(t[0], t[1]) for t in decoded
])).replace('\r',
' ').replace('\t',
' ').replace('\n', ' ')
except email.errors.HeaderParseError:
return ''
def update_location(self, session, msg_idx, msg_ptr):
msg_info = self.get_msg_by_idx(msg_idx)
msg_ptrs = msg_info[self.MSG_PTRS].split(',')
self.PTRS[msg_ptr] = msg_idx
# If message was seen in this mailbox before, update the location
for i in range(0, len(msg_ptrs)):
if (msg_ptrs[i][:3] == msg_ptr[:3]):
msg_ptrs[i] = msg_ptr
msg_ptr = None
break
# Otherwise, this is a new mailbox, record this sighting as well!
if msg_ptr: msg_ptrs.append(msg_ptr)
msg_info[self.MSG_PTRS] = ','.join(msg_ptrs)
self.set_msg_by_idx(msg_idx, msg_info)
def scan_mailbox(self, session, idx, mailbox_fn, mailbox_opener):
try:
mbox = mailbox_opener(session, idx)
if mbox.editable:
session.ui.mark('%s: Skipped: %s' % (idx, mailbox_fn))
return 0
else:
session.ui.mark('%s: Checking: %s' % (idx, mailbox_fn))
except (IOError, OSError, NoSuchMailboxError), e:
session.ui.mark('%s: Error opening: %s (%s)' %
(idx, mailbox_fn, e))
return 0
unparsed = mbox.unparsed()
if not unparsed:
return 0
if len(self.PTRS.keys()) == 0:
self.update_ptrs_and_msgids(session)
added = 0
msg_date = int(time.time())
for ui in range(0, len(unparsed)):
if mailpile.util.QUITTING: break
i = unparsed[ui]
parse_status = ('%s: Reading your mail: %d%% (%d/%d messages)') % (
idx, 100 * ui / len(unparsed), ui, len(unparsed))
msg_ptr = mbox.get_msg_ptr(idx, i)
if msg_ptr in self.PTRS:
if (ui % 317) == 0: session.ui.mark(parse_status)
continue
else:
session.ui.mark(parse_status)
# Message new or modified, let's parse it.
msg = ParseMessage(mbox.get_file(i), pgpmime=False)
msg_id = b64c(
sha1b64((self.hdr(msg, 'message-id') or msg_ptr).strip()))
if msg_id in self.MSGIDS:
self.update_location(session, self.MSGIDS[msg_id], msg_ptr)
added += 1
else:
# Add new message!
msg_mid = b36(len(self.INDEX))
try:
last_date = msg_date
msg_date = int(
rfc822.mktime_tz(
rfc822.parsedate_tz(self.hdr(msg, 'date'))))
if msg_date > (time.time() + 24 * 3600):
session.ui.warning('=%s/%s is from the FUTURE!' %
(msg_mid, msg_id))
# Messages from the future are treated as today's
msg_date = last_date + 1
elif msg_date < 1:
session.ui.warning('=%s/%s is PREHISTORIC!' %
(msg_mid, msg_id))
msg_date = last_date + 1
except (ValueError, TypeError, OverflowError):
session.ui.warning('=%s/%s has a bogus date.' %
(msg_mid, msg_id))
#if session.config.get('debug'):
# session.ui.say(traceback.format_exc())
# This is a hack: We assume the messages in the mailbox are in
# chronological order and just add 1 second to the date of the last
# message. This should be a better-than-nothing guess.
msg_date += 1
keywords = self.index_message(
session,
msg_mid,
msg_id,
msg,
msg_date,
mailbox=idx,
compact=False,
filter_hooks=[self.filter_keywords])
tags = [
k.split(':')[0] for k in keywords if k.endswith(':tag')
]
msg_idx = len(self.INDEX)
self.set_msg_by_idx(
msg_idx,
[
msg_mid, # Our index ID
msg_ptr, # Location on disk
'', # UNUSED
msg_id, # Message-ID
b36(msg_date), # Date as a UTC timestamp
self.hdr(msg, 'from'), # From:
self.hdr(msg, 'subject'), # Subject
','.join(tags), # Initial tags
'', # No replies for now
''
]) # Conversation ID
self.set_conversation_ids(msg_mid, msg)
added += 1
if (added % 1000) == 0:
GlobalPostingList.Optimize(session, self, quick=True)
if added:
mbox.mark_parsed(i)
mbox.save(session)
session.ui.mark('%s: Indexed mailbox: %s' % (idx, mailbox_fn))
return added
def accept(game, locale, stream, extend=None):
global rootdir, orderbase, gamedir, gamename, sender
if extend is not None:
orderbase = orderbase + ".pre-" + extend
savedir = os.path.join(gamedir, orderbase)
# check if it's one of the pre-sent orders.
# create the save-directories if they don't exist
if not os.path.exists(gamedir): os.mkdir(gamedir)
if not os.path.exists(savedir): os.mkdir(savedir)
# parse message
message = Parser().parse(stream)
email = get_sender(message)
logger = logging.getLogger(email)
# write syslog
if email is None or valid_email(email) == 0:
logger.warning("invalid email address: " + str(email))
return -1
logger.info("received orders from " + email)
# get an available filename
maxdate, filename = available_file(savedir, prefix + email)
if filename is None:
logger.warning("more than " + str(maxfiles) + " orders from " + email)
return -1
# copy the orders to the file
turndate = None
maildate = message.get("Date")
if maildate is None:
turndate = time.time()
else:
turndate = rfc822.mktime_tz(rfc822.parsedate_tz(maildate))
text_ok = copy_orders(message, filename, email, turndate)
warning, msg, fail = None, "", False
if not maildate is None:
os.utime(filename, (turndate, turndate))
logger.debug("mail date is '%s' (%d)" % (maildate, turndate))
if False and turndate < maxdate:
logger.warning("inconsistent message date " + email)
warning = " (" + messages["warning-" + locale] + ")"
msg = msg + formatpar(
messages["maildate-" + locale] %
(time.ctime(maxdate), time.ctime(turndate)), 76, 2) + "\n"
else:
logger.warning("missing message date " + email)
warning = " (" + messages["warning-" + locale] + ")"
msg = msg + formatpar(messages["nodate-" + locale], 76, 2) + "\n"
print('ACCEPT_MAIL=' + email)
print('ACCEPT_FILE="' + filename + '"')
if not text_ok:
warning = " (" + messages["error-" + locale] + ")"
msg = msg + formatpar(messages["multipart-" + locale], 76, 2) + "\n"
logger.warning("rejected - no text/plain in orders from " + email)
os.unlink(filename)
savedir = savedir + "/rejected"
if not os.path.exists(savedir): os.mkdir(savedir)
maxdate, filename = available_file(savedir, prefix + email)
store_message(message, filename)
fail = True
if sendmail and warning is not None:
subject = gamename + " " + messages["subject-" + locale] + warning
print("mail " + subject)
ps = subprocess.Popen(['mutt', '-s', subject, email],
stdin=subprocess.PIPE)
ps.communicate(msg)
if not sendmail:
print text_ok, fail, email
print filename
if not fail:
queue = open(gamedir + "/orders.queue", "a")
queue.write("email=%s file=%s locale=%s game=%s\n" %
(email, filename, locale, game))
queue.close()
logger.info("done - accepted orders from " + email)
return 0
def addMail(self, mailString):
""" Store mail as news item
Returns created item
"""
pw = self.context.portal_workflow
(header, body) = splitMail(mailString)
# if 'keepdate' is set, get date from mail,
# XXX 'time' is unused
if self.getValueFor('keepdate'):
timetuple = rfc822.parsedate_tz(header.get('date'))
time = DateTime(rfc822.mktime_tz(timetuple))
# ... take our own date, clients are always lying!
else:
time = DateTime()
(TextBody, ContentType, HtmlBody, Attachments) = unpackMail(mailString)
# Test Zeitangabe hinter Subject
today = date.today()
mydate = today.strftime("%d.%m.%Y")
# let's create the news item
subject = mime_decode_header(header.get('subject', 'No Subject'))
sender = mime_decode_header(header.get('from', 'No From'))
title = "%s" % (subject)
new_id = IUserPreferredURLNormalizer(self.request).normalize(title)
id = self._findUniqueId(new_id)
# ContentType is only set for the TextBody
if ContentType:
body = TextBody
else:
body = self.HtmlToText(HtmlBody)
# XXX als vorlaeufige Loesung
from zope.component import getMultiAdapter
plone_view = getMultiAdapter((self.context, self.request),
name='plone')
desc = plone_view.cropText(body, 60)
body = '\n'.join([wrap_line(line) for line in body.splitlines()])
uni_aktuell_body = ("<p><strong>%s: %s</strong></p> "
"<p> </p><pre>%s</pre>" %
(mydate, sender, body))
objid = self.context.invokeFactory(
'News Item',
id=id,
title=title,
text=RichTextValue(uni_aktuell_body),
description=desc,
)
mailObject = getattr(self.context, objid)
images = [
att for att in Attachments
if att['maintype'] == 'image' and att['filename']
]
if images and hasattr(mailObject, 'image'):
image = Attachments[0]
mailObject.image = NamedBlobImage(
filename=safe_unicode(image['filename']),
data=image['filebody'],
)
try:
pw.doActionFor(mailObject, 'publish')
except Exception as e:
log.exception(e)
return mailObject
def get_seconds(headers, modified='Last-Modified'):
date_tuple = headers.getdate_tz(modified)
epoch_seconds = rfc822.mktime_tz(date_tuple)
return epoch_seconds
def convertFormat(s, timeformat = '%d.%m.%Y %H:%M:%S', timezone = None):
return convertClockTime(rfc822.mktime_tz(rfc822.parsedate_tz(s)), timeformat, timezone)
def build_date(theTime):
data = rfc822.parsedate_tz(theTime.strftime("%a, %d %b %Y %H:%M:%S"))
return rfc822.formatdate(rfc822.mktime_tz(data))
def get_contents_to_filename(self,
filename,
headers=None,
cb=None,
num_cb=10,
torrent=False,
version_id=None,
res_download_handler=None,
response_headers=None):
"""
Retrieve an object from S3 using the name of the Key object as the
key in S3. Store contents of the object to a file named by 'filename'.
See get_contents_to_file method for details about the
parameters.
:type filename: string
:param filename: The filename of where to put the file contents
:type headers: dict
:param headers: Any additional headers to send in the request
:type cb: function
:param cb: (optional) a callback function that will be called to
report progress on the download. The callback should
accept two integer parameters, the first representing
the number of bytes that have been successfully
transmitted from S3 and the second representing the
total number of bytes that need to be transmitted.
:type num_cb: int
:param num_cb: (optional) If a callback is specified with the
cb parameter this parameter determines the
granularity of the callback by defining the
maximum number of times the callback will be
called during the file transfer.
:type torrent: bool
:param torrent: If True, returns the contents of a torrent file
as a string.
:type res_upload_handler: ResumableDownloadHandler
:param res_download_handler: If provided, this handler will
perform the download.
:type response_headers: dict
:param response_headers: A dictionary containing HTTP headers/values
that will override any headers associated with
the stored object in the response.
See http://goo.gl/EWOPb for details.
"""
fp = open(filename, 'wb')
self.get_contents_to_file(fp,
headers,
cb,
num_cb,
torrent=torrent,
version_id=version_id,
res_download_handler=res_download_handler,
response_headers=response_headers)
fp.close()
# if last_modified date was sent from s3, try to set file's timestamp
if self.last_modified != None:
try:
modified_tuple = rfc822.parsedate_tz(self.last_modified)
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
os.utime(fp.name, (modified_stamp, modified_stamp))
except Exception:
pass
else:
language = settings.LANGUAGE_CODE
print language_model.language
activate(language)
nbitem = 0
xml = parse(urlopen(user.rss_feed))
feed_date = datetime.now()
for channel in xml.getElementsByTagName("channel"):
channelname = channel.getElementsByTagName("title")[0].firstChild.data
for item in channel.getElementsByTagName("item"):
msg={}
for node in item.childNodes:
if node.firstChild:
if node.nodeName == "pubDate":
pubDate = datetime.fromtimestamp(mktime_tz(parsedate_tz(node.firstChild.data)))
if feed_date > pubDate > (user.last_feed or datetime.min):
msg['publish'] = True
else:
msg[node.nodeName] = node.firstChild.data
if msg.get('publish'):
text = oiescape(u'%s : <p>%s</p><a href="%s">%s %s</a>'%(msg['title'], msg['description'], msg['link'], _("Published on"), channelname))
Message.objects.create(title=msg['title'], text=text, public=True, author=user.user, relevance=OI_SCORE_DEFAULT_RELEVANCE, parent=user.blog)
nbitem += 1
except Exception as e:
print "erreur du flux de %s"%user.user.username
print type(e)
print e
print msg['title']
finally:
def _onsuccess(h):
checksum = h.get('etag')[0].strip('"')
last_modified = h.get('last-modified')[0]
modified_tuple = rfc822.parsedate_tz(last_modified)
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
return {'checksum': checksum, 'last_modified': modified_stamp}
def scan_mailbox(self, session, idx, mailbox_fn, mailbox_opener):
try:
mbox = mailbox_opener(session, idx)
if mbox.editable:
session.ui.mark('%s: Skipped: %s' % (idx, mailbox_fn))
return 0
else:
session.ui.mark('%s: Checking: %s' % (idx, mailbox_fn))
except (IOError, OSError, NoSuchMailboxError):
session.ui.mark('%s: Error opening: %s' % (idx, mailbox_fn))
return 0
unparsed = mbox.unparsed()
if not unparsed:
return 0
if len(self.PTRS.keys()) == 0:
self.update_ptrs_and_msgids(session)
added = 0
msg_date = int(time.time())
for ui in range(0, len(unparsed)):
if mailpile.util.QUITTING: break
i = unparsed[ui]
parse_status = ('%s: Reading your mail: %d%% (%d/%d messages)') % (
idx, 100 * ui / len(unparsed), ui, len(unparsed))
msg_ptr = mbox.get_msg_ptr(idx, i)
if msg_ptr in self.PTRS:
if (ui % 317) == 0: session.ui.mark(parse_status)
continue
else:
session.ui.mark(parse_status)
# Message new or modified, let's parse it.
msg = ParseMessage(mbox.get_file(i), pgpmime=False)
msg_id = b64c(
sha1b64((self.hdr(msg, 'message-id') or msg_ptr).strip()))
if msg_id in self.MSGIDS:
self.update_location(session, self.MSGIDS[msg_id], msg_ptr)
added += 1
else:
# Add new message!
msg_mid = b36(len(self.INDEX))
try:
last_date = msg_date
msg_date = int(
rfc822.mktime_tz(
rfc822.parsedate_tz(self.hdr(msg, 'date'))))
if msg_date > (time.time() + 24 * 3600):
session.ui.warning('=%s/%s is from the FUTURE!' %
(msg_mid, msg_id))
# Messages from the future are treated as today's
msg_date = last_date + 1
elif msg_date < 1:
session.ui.warning('=%s/%s is PREHISTORIC!' %
(msg_mid, msg_id))
msg_date = last_date + 1
except (ValueError, TypeError, OverflowError):
session.ui.warning('=%s/%s has a bogus date.' %
(msg_mid, msg_id))
if session.config.get('debug'):
session.ui.say(traceback.format_exc())
# This is a hack: We assume the messages in the mailbox are in
# chronological order and just add 1 second to the date of the last
# message. This should be a better-than-nothing guess.
msg_date += 1
keywords = self.index_message(
session,
msg_mid,
msg_id,
msg,
msg_date,
mailbox=idx,
compact=False,
filter_hooks=[self.filter_keywords])
tags = [
k.split(':')[0] for k in keywords if k.endswith(':tag')
]
msg_idx = len(self.INDEX)
self.set_msg_by_idx(
msg_idx,
[
msg_mid, # Our index ID
msg_ptr, # Location on disk
'', # UNUSED
msg_id, # Message-ID
b36(msg_date), # Date as a UTC timestamp
self.hdr(msg, 'from'), # From:
self.hdr(msg, 'subject'), # Subject
','.join(tags), # Initial tags
'', # No replies for now
''
]) # Conversation ID
self.set_conversation_ids(msg_mid, msg)
added += 1
if (added % 1000) == 0:
GlobalPostingList.Optimize(session, self, quick=True)
if added:
mbox.mark_parsed(i)
mbox.save(session)
session.ui.mark('%s: Indexed mailbox: %s' % (idx, mailbox_fn))
return added
def datetime_to_epoch(dt):
if PYT3:
return email.utils.mktime_tz(dt.utctimetuple() + (0, ))
return rfc822.mktime_tz(dt.utctimetuple() + (0, ))