def handle(self, **options):
count = TLE.objects.count()
if count:
last_TLE = TLE.objects.first()
last_mktime = mktime_tz(parsedate_tz(last_TLE.datetime_in_lines.ctime()))
else:
last_mktime = 0
url = 'http://www.celestrak.com/NORAD/elements/stations.txt'
resp = get(url)
url_mktime = mktime_tz(parsedate_tz(resp.headers['last-modified']))
url_datetime = datetime.utcfromtimestamp(url_mktime)
if url_mktime > last_mktime:
self.stdout.write('Date and time of creation TLE: {}'.format(url_datetime.isoformat()))
self.stdout.write('New TLE found, downloading...')
result = urlretrieve(url, 'TLE.txt')
fh = open(result[0], 'r', encoding='utf8')
lines = fh.readlines()[:3]
fh.close()
os.remove(result[0])
title = lines[0].strip()
norad_id = int(lines[1][2:7])
sat, status = Satellite.objects.get_or_create(title=title, satellite_number=norad_id)
try:
self.stdout.write('Start create and save object - ' + sat.title + '\n')
TLE.objects.bulk_create(unique_tle(lines, sat))
self.stdout.write('Finished create and save object - ' + sat.title + '\n')
except:
self.stdout.write('Fail create and save object - ' + sat.title + '\n')
else:
self.stdout.write('No new TLE. A new attempt after 5 minutes...')
def tweet_processing(tweetFile):
tweet_file = open(tweetFile) # Open the file for reading
tweet_hash = {} # Define a dictionary for keeping the hashtags as values and their id as keys
tweet_id={} # Define a dictionary for keeping the created_at as values and their id as keys
first_tweet=True
latest_date=None
print("Started reading tweets")
for tweet_line in tweet_file: # Loop for every tweet in the tweets file
tweet = json.loads(tweet_line) # convert the json string to dictionary object
if "entities" in tweet.keys(): # Check whether entities tags present
hashtags = tweet["entities"]["hashtags"] # - if present then extract the hashtags
if hashtags:
if first_tweet:
latest_date=datetime.datetime.fromtimestamp(time.mktime(datetime.datetime.now().timetuple()),pytz.utc) #set latest date to current datetime if its first tweet
#latest_date=datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz("Fri Oct 30 15:29:44 +0000 2015")),pytz.utc)
first_tweet=False
tweet_hash[tweet["id"]]=remove_dups(['#'+str((remove_unicode(ht['text']))).lower() for ht in hashtags if ht != None and len((remove_unicode(ht['text'])))>1 ]) #extracts the hastags cleans them and checks if length of hashtag is more than 1 character
tweet_id[tweet["id"]]=[datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(tweet['created_at'])),pytz.utc),0]
create_graph(tweet_hash[tweet["id"]]) #calls the
for key,value in tweet_id.items(): #checks for old tweets if found evict them
if (latest_date-value[0]).total_seconds()>60:
if len(tweet_hash[key])>=2: evict_graph(tweet_hash[key],key)
#for i in older_tweet: #removes old tweets from if
# if i in tweet_id.keys():del tweet_id[i]
tweet_date=datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(tweet['created_at'])),pytz.utc)
if tweet_date>=latest_date:latest_date=tweet_date
feature2=open(outfile,'w')
for degree in rolling_degree:feature2.write(str(degree)+'\n') #write into output file
print("Processing is completed!!!")
def main():
ts = 'Fri Dec 07 16:12:48 +0100 2012'
dt = int(mktime_tz(parsedate_tz(ts.strip())))
print dt
ts = 'Fri Dec 07 16:12:48 +0000 2012'
dt = int(mktime_tz(parsedate_tz(ts.strip())))
print dt
def test_expires(self):
"""Check there are these headers and that expires is 3600 later."""
# We aren't bother going to test the actual time in expires, that
# way lies pain with broken tests later.
up = self.get(self.good_data)
hdrs = dict(up.get_headers(1))
lm = datetime(*utils.parsedate_tz(hdrs['Last-Modified'])[:7])
exp = datetime(*utils.parsedate_tz(hdrs['Expires'])[:7])
eq_((exp - lm).seconds, 3600)
def to_datetime(self,datestring): try: time_tuple = parsedate_tz(datestring.strip()) dt = datetime(*time_tuple[:6]) dt - timedelta(seconds=time_tuple[-1]) except Exception, e: datestring = "18 Feb 2007 16:30:00" time_tuple = parsedate_tz(datestring.strip()) dt = datetime(*time_tuple[:6]) dt - timedelta(seconds=time_tuple[-1])
def test_expires(self):
"""Check there are these headers and that expires is 3600 later."""
# We aren't bother going to test the actual time in expires, that
# way lies pain with broken tests later.
instance = self.get_update_instance(self.data)
headers = dict(instance.get_headers(1))
last_modified = datetime(
*utils.parsedate_tz(headers['Last-Modified'])[:7])
expires = datetime(*utils.parsedate_tz(headers['Expires'])[:7])
assert (expires - last_modified).seconds == 3600
def urllib_header_test():
import urllib2, time
import email.utils as eut
from datetime import datetime
f = urllib2.urlopen("http://127.0.0.1:8000/envsocial/client/v1/resources/environment/1/?format=xml")
print f.code
header_info = f.info()
h_date = header_info.get('Date')
print h_date
print eut.parsedate_tz(h_date)
t = time.mktime(eut.parsedate_tz(h_date)[:9])
now = time.mktime(time.gmtime())
def retry_after(cls, response, default):
"""Compute next `poll` time based on response ``Retry-After`` header.
Handles integers and various datestring formats per
https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.37
:param requests.Response response: Response from `poll`.
:param int default: Default value (in seconds), used when
``Retry-After`` header is not present or invalid.
:returns: Time point when next `poll` should be performed.
:rtype: `datetime.datetime`
"""
retry_after = response.headers.get('Retry-After', str(default))
try:
seconds = int(retry_after)
except ValueError:
# The RFC 2822 parser handles all of RFC 2616's cases in modern
# environments (primarily HTTP 1.1+ but also py27+)
when = parsedate_tz(retry_after)
if when is not None:
try:
tz_secs = datetime.timedelta(when[-1] if when[-1] else 0)
return datetime.datetime(*when[:7]) - tz_secs
except (ValueError, OverflowError):
pass
seconds = default
return datetime.datetime.now() + datetime.timedelta(seconds=seconds)
def _retrieve_mails(uri):
LOG.debug('Retrieving mail archive from uri: %s', uri)
content = utils.read_uri(uri)
if not content:
LOG.error('Error reading mail archive from uri: %s', uri)
return
content = utils.gzip_decompress(content)
LOG.debug('Mail archive is loaded, start processing')
content += TRAILING_RECORD
for rec in re.finditer(MAIL_BOX_PATTERN, content):
email = rec.groupdict()
email['author_email'] = email['author_email'].replace(' at ', '@', 1)
if not utils.check_email_validity(email['author_email']):
continue
email['date'] = int(email_utils.mktime_tz(
email_utils.parsedate_tz(email['date'])))
for pattern_name, pattern in six.iteritems(MESSAGE_PATTERNS):
collection = set()
for item in re.finditer(pattern, email['body']):
groups = item.groupdict()
item_id = groups['id']
if 'module' in groups:
item_id = groups['module'] + ':' + item_id
email['module'] = groups['module']
collection.add(item_id)
email[pattern_name] = list(collection)
yield email
def datetime_from_rfc822(value):
'''
Turns an RFC822 formatted date into a datetime object.
Example::
inputs.datetime_from_rfc822('Wed, 02 Oct 2002 08:00:00 EST')
:param str value: The RFC822-complying string to transform
:return: The parsed datetime
:rtype: datetime
:raises ValueError: if value is an invalid date literal
'''
raw = value
if not time_regex.search(value):
value = ' '.join((value, '00:00:00'))
try:
timetuple = parsedate_tz(value)
timestamp = mktime_tz(timetuple)
if timetuple[-1] is None:
return datetime.fromtimestamp(timestamp).replace(tzinfo=pytz.utc)
else:
return datetime.fromtimestamp(timestamp, pytz.utc)
except Exception:
raise ValueError('Invalid date literal "{0}"'.format(raw))
def setUp(self):
"""
Reads an arbitrary number of mail messages and
stores them in a brand new messages table.
"""
self.conn = msc.Connect(**loginInfo)
self.curs = self.conn.cursor()
self.curs.execute("DROP TABLE IF EXISTS {0}".format(TBLNM))
self.conn.commit()
curs.execute(TBLDEF)
conn.commit()
files = glob(FILESPEC)
self.msgIds = {} # Keyed by messageId
self.messageIds = {} # Keyed by id
self.msgdates = []
self.rowcount = 0
for f in files:
ff = open(f)
text = ff.read()
msg = message_from_string(text)
iD = self.msgIds[msg['message-id']] = maildb.store(msg, self.conn, self.curs, TBLNM)
self.messageIds[iD] = msg['message-id']
date = msg['date']
self.msgdates.append(datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(date))))
self.rowcount += 1 # Assuming no duplicated Message-IDs
ff.close()
def _refresh_cookie(self, c, delta):
"""
Takes a cookie string c and a time delta in seconds, and returns
a refreshed cookie string.
"""
try:
c = Cookie.SimpleCookie(str(c))
except Cookie.CookieError:
raise ValueError("Invalid Cookie")
for i in c.values():
if "expires" in i:
d = parsedate_tz(i["expires"])
if d:
d = mktime_tz(d) + delta
i["expires"] = formatdate(d)
else:
# This can happen when the expires tag is invalid.
# reddit.com sends a an expires tag like this: "Thu, 31 Dec
# 2037 23:59:59 GMT", which is valid RFC 1123, but not
# strictly correct according to the cookie spec. Browsers
# appear to parse this tolerantly - maybe we should too.
# For now, we just ignore this.
del i["expires"]
ret = c.output(header="").strip()
if not ret:
raise ValueError("Invalid Cookie")
return ret
def handler(doc):
# a 'rfc822' stores 'headers' as a dict, with each entry being a list.
# We only care about headers which rfc5322 must appear 0 or 1 times, so
# flatten the header values here...
headers = dict((k, v[0]) for (k, v) in doc["headers"].iteritems())
# for now, 'from' etc are all tuples of [identity_type, identity_id]
callbacks = []
ret = {}
if "from" in headers:
name, addr = parseaddr(headers["from"])
ret["from"] = ["email", addr.lower()]
ret["from_display"] = name
if "to" in headers:
id_list = ret["to"] = []
disp_list = ret["to_display"] = []
fill_identity_info(headers["to"], id_list, disp_list)
if "cc" in headers:
id_list = ret["cc"] = []
disp_list = ret["cc_display"] = []
fill_identity_info(headers["cc"], id_list, disp_list)
if "subject" in headers:
ret["subject"] = headers["subject"]
if "date" in headers:
dval = headers["date"]
if dval:
try:
ret["timestamp"] = mktime_tz(parsedate_tz(dval))
except (ValueError, TypeError), exc:
logger.debug("Failed to parse date %r in doc %r: %s", dval, doc["_id"], exc)
# later extensions will get upset if no attr exists
# XXX - is this still true? We should fix those extensions!
ret["timestamp"] = 0
def parse_date(value):
"""Parse one of the following date formats into a datetime object:
.. sourcecode:: text
Sun, 06 Nov 1994 08:49:37 GMT ; RFC 822, updated by RFC 1123
Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036
Sun Nov 6 08:49:37 1994 ; ANSI C's asctime() format
If parsing fails the return value is `None`.
:param value: a string with a supported date format.
:return: a :class:`datetime.datetime` object.
"""
if value:
t = parsedate_tz(value.strip())
if t is not None:
try:
year = t[0]
# unfortunately that function does not tell us if two digit
# years were part of the string, or if they were prefixed
# with two zeroes. So what we do is to assume that 69-99
# refer to 1900, and everything below to 2000
if year >= 0 and year <= 68:
year += 2000
elif year >= 69 and year <= 99:
year += 1900
return datetime(*((year,) + t[1:7])) - \
timedelta(seconds=t[-1] or 0)
except (ValueError, OverflowError):
return None
def refresh(self, now=None):
"""
This fairly complex and heuristic function refreshes a server
response for replay.
- It adjusts date, expires and last-modified headers.
- It adjusts cookie expiration.
"""
if not now:
now = time.time()
delta = now - self.timestamp_start
refresh_headers = [
"date",
"expires",
"last-modified",
]
for i in refresh_headers:
if i in self.headers:
d = parsedate_tz(self.headers[i])
if d:
new = mktime_tz(d) + delta
self.headers[i] = formatdate(new)
c = []
for set_cookie_header in self.headers.get_all("set-cookie"):
try:
refreshed = self._refresh_cookie(set_cookie_header, delta)
except ValueError:
refreshed = set_cookie_header
c.append(refreshed)
if c:
self.headers.set_all("set-cookie", c)
def _created_at_to_datetime(created_at):
'''
Convert a date represented in the Twitter format to a datetime
object.
'''
time_tuple = parsedate_tz(created_at)
return datetime(*time_tuple[:6])
def _parse_sibling(self, sibling, headers, data):
"""
Parses a single sibling out of a response.
"""
sibling.exists = True
# Parse the headers...
for header, value in headers:
header = header.lower()
if header == 'content-type':
sibling.content_type, sibling.charset = \
self._parse_content_type(value)
elif header == 'etag':
sibling.etag = value
elif header == 'link':
sibling.links = self._parse_links(value)
elif header == 'last-modified':
sibling.last_modified = mktime_tz(parsedate_tz(value))
elif header.startswith('x-riak-meta-'):
metakey = header.replace('x-riak-meta-', '')
sibling.usermeta[metakey] = value
elif header.startswith('x-riak-index-'):
field = header.replace('x-riak-index-', '')
reader = csv.reader([value], skipinitialspace=True)
for line in reader:
for token in line:
token = decode_index_value(field, token)
sibling.add_index(field, token)
elif header == 'x-riak-deleted':
sibling.exists = False
sibling.encoded_data = data
return sibling
def update_headers(self, resp):
headers = resp.headers
if 'expires' in headers:
return {}
if 'cache-control' in headers and headers['cache-control'] != 'public':
return {}
if resp.status not in self.cacheable_by_default_statuses:
return {}
if 'date' not in headers or 'last-modified' not in headers:
return {}
date = calendar.timegm(parsedate_tz(headers['date']))
last_modified = parsedate(headers['last-modified'])
if date is None or last_modified is None:
return {}
now = time.time()
current_age = max(0, now - date)
delta = date - calendar.timegm(last_modified)
freshness_lifetime = max(0, min(delta / 10, 24 * 3600))
if freshness_lifetime <= current_age:
return {}
expires = date + freshness_lifetime
return {'expires': time.strftime(TIME_FMT, time.gmtime(expires))}
def extract_file(self):
extracted_file = {}
i = 0 # Used to count actual JSON messages (with time and hashtags information)
# j=0 # Used to count API messages about connection and rate limits (without time and hashtags information)
for line in self.my_file:
temp = json.loads(line) # read data line by line
if 'created_at' in temp.keys(): # select the actual JSON messages
temp_time = temp['created_at'] # extract time
temp_tag = temp['entities']['hashtags'] # extract hashtags
# transfer time string to datatime frame
# temp_time = datetime.strptime(temp_time, '%a %b %d %H:%M:%S %z %Y')
# strptime is Python platform only, do not use the above query if run by shell
temp_time = datetime(1970, 1, 1) + timedelta(seconds=mktime_tz(parsedate_tz(temp_time)))
# store time, hashtags information,and later extracted hashtag words to new file
extracted_file[i] = [temp_time, temp_tag, []]
# extract the hashtag words
if temp_tag:
for tag in temp_tag:
extracted_file[i][2].append(tag['text'])
else: # no hashtags
pass
i += 1
else:
# these messages are Twitter API resulting from the rate-limit
# can be stored in apifile for future uses
# here we remove these messages from dataset
pass
# apifile[j] = temp
# j += 1
return extracted_file
def moveByLabel (self):
from email.utils import parsedate_tz,mktime_tz
from mailbox import NoSuchMailboxError
for folder in self.oldBox.list_folders():
_c_moved=0
_c_rej=0
_c_total=self.oldBox.get_folder(folder).__len__()
print("\n[I] Folder " + folder + "", end="")
for key, msg in self.oldBox.get_folder(folder).iteritems():
_date=msg['Date']
if _date:
if (mktime_tz(parsedate_tz(_date)) - self.deltaT) < 0:
if _c_moved == 0:
#To detect if no thing is moved, so this can be a new folder
try:
self.newBox.get_folder(folder)
except NoSuchMailboxError:
print("[I]\tCreating in new: %s" % folder)
self.newBox.add_folder(folder)
# Mooooooooooooo'ving!
self.newBox.get_folder(folder).add(msg)
self.oldBox.get_folder(folder).remove(key)
_c_moved += 1
print("\r[I]\tStats: Not moved (Bad Mail): %d/%d // Moved: %d/%d" % (_c_rej,_c_total,_c_moved,_c_total), end="")
else:
_c_rej += 1
if _c_moved >= _c_total:
print("\n[W]\tRemoving folder %s" % folder, end="")
print("")
def __init__(self, mbFile, listAddrProg, startDate=None, endDate=None):
self.mbFile = mbFile
mb = mailbox.UnixMailbox(file(mbFile, "r"), email.message_from_file)
checked = set()
count = 0
config = yaml.load(open("game.yaml"))
tz = pytz.timezone(config.get("timezone", "UTC"))
for msg in mb:
try:
if not listAddrProg.search(msg["to"]):
continue
timetuple = parsedate_tz(msg["date"])
timestamp = mktime_tz(timetuple)
date = datetime.fromtimestamp(timestamp)
date = date.replace(tzinfo=tz)
if startDate > date or date > endDate:
continue
mail = Mail(msg, date)
self.msgs.append(mail)
# count = count +1
# print "count...", count
# if count == 50:
# break
except Exception as e:
print "failed analyzing a message!" , e
def was_modified_since(self, header=None, mtime=0, size=0):
"""
Was something modified since the user last downloaded it?
header
This is the value of the If-Modified-Since header. If this is None,
I'll just return True.
mtime
This is the modification time of the item we're talking about.
size
This is the size of the item we're talking about.
"""
try:
if header is None:
raise ValueError
matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header,
re.IGNORECASE)
header_mtime = mktime_tz(parsedate_tz(matches.group(1)))
header_len = matches.group(3)
if header_len and int(header_len) != size:
raise ValueError()
if mtime > header_mtime:
raise ValueError()
except (AttributeError, ValueError, OverflowError):
return True
return False
def fixDir(dname):
for fname in os.listdir(dname):
fname = os.path.join(dname, fname)
fp = open(fname)
msg = Parser().parse(fp, True)
if 'Date' in msg:
date = parsedate_tz(msg['Date'])
if date:
# Ok I had some old emails with messed up Date headers as so:
# Date: Sun, 22 Aug US/E 13:01:00 -0400
# I knew these were briefly from '99-'00 so I manually fix that here.
'''
if date[0] < 1900:
if date[1] < 3:
year = 2000
else:
year = 1999
date = (year,) + date[1:]
print >> sys.stderr, "Fixing up year '%s' => '%s' for %s" % (msg['Date'], date, fname)
'''
try:
timestamp = mktime_tz(date)
os.utime(fname, (timestamp, timestamp))
except ValueError:
print >> sys.stderr, "Invalid date '%s' for %s: %s" % (msg['Date'], fname, date)
else:
print >> sys.stderr, "Could not parse date '%s' for %s" % (msg['Date'], fname)
else:
print >> sys.stderr, 'No Date header in %s' % (fname)
def parse_item(self, item):
ep = Storage()
ep.filename = item.findtext('{http://xmlns.ezrss.it/0.1/}torrent/{http://xmlns.ezrss.it/0.1/}fileName')
info = self.parse_title(ep.filename)
ep.update(info)
ep.title = item.findtext('title')
ep.link = item.findtext('link')
ep.description = item.findtext('description')
ep.guid = item.findtext('guid')
ep.pubdate = item.findtext('pubDate')
ep.magnet = item.findtext('{http://xmlns.ezrss.it/0.1/}torrent/{http://xmlns.ezrss.it/0.1/}magnetURI')
ep.filename = item.findtext('{http://xmlns.ezrss.it/0.1/}torrent/{http://xmlns.ezrss.it/0.1/}fileName')
ep.pubdate = datetime.datetime.utcfromtimestamp(mktime_tz(parsedate_tz(ep.pubdate)))
ep.size = item.find('enclosure').get('length')
try:
ep.size = int(ep.size)
except:
ep.size = 0
if ep.size < 100*1024*1024:
ep.size = 300*1024*1024
if ep.magnet == 'magnet:?xt=urn:btih:&dn=':
#check at least for a bt-chat info_hash
btchat = self.magnetr.search(ep.link)
if btchat:
hash = btchat.group(1)
ep.magnet = 'magnet:?xt=urn:btih:%s' % hash
ep.link = None
if not ep.guid:
ep.guid = ep.description
ep.filterwith = ep.title
return ep
def dec_date(data):
print("dec_date", data)
res = parsedate_tz(data or "")
if not res:
return time.strftime("%Y-%m-%d %H:%M:%S")
d = datetime.fromtimestamp(email.utils.mktime_tz(res))
return d.strftime("%Y-%m-%d %H:%M:%S")
def parse_item(self, item):
ep = Storage()
ep.title = item.findtext('title')
match = self.title_strip_quality.search(ep.title)
if match:
title_ = match.group(1)
else:
title_ = ep.title
info = self.parse_title(title_)
ep.update(info)
ep.description = item.findtext('description')
ep.link = item.find('enclosure').get('url')
ep.pubdate = item.findtext('pubDate')
ep.filename = item.findtext('title')
ep.pubdate = datetime.datetime.utcfromtimestamp(mktime_tz(parsedate_tz(ep.pubdate)))
ep.filterwith = ep.title
ep.size = item.find('enclosure').get('length')
try:
ep.size = int(ep.size)
except:
ep.size = 0
if ep.size < 100*1024*1024:
ep.size = 300*1024*1024
ep.guid = item.findtext('guid')
return ep
def parse_redis_message(payload):
try:
headers, body = payload.split('\r\n'*2, 1)
headers = dict(map(lambda h: h.split(': ', 1), headers.split('\r\n')))
return Message(mktime_tz(parsedate_tz(headers['Last-Modified'])), int(headers['Etag']), headers['Content-Type'], body)
except:
raise Message.Invalid()
def get_bulk_email(site, filename):
""" Show a specific bulk email saved on the disk """
save_path = get_log_dir(site)
join = os.path.join
if save_path:
save_path = join(save_path, 'sent-bulk')
if os.path.isdir(save_path):
message_path = join(save_path, filename)
try:
message_file = open(message_path, 'r+')
except IOError:
return None
mail = message_from_file(message_file)
message_file.close()
# Prepare the date to be formatted with utShowFullDateTime
date = email_utils.parsedate_tz(mail.get('Date', ''))
date = email_utils.mktime_tz(date)
date = datetime.fromtimestamp(date)
return {
'subject': mail.get('Subject', '(no-subject)'),
'content': mail.get_payload(decode=True).replace(
'\n\n', '</p><p>').replace('\n', '<br/>'),
'recipients': mail.get_all('To'),
'sender': mail.get('From'),
'date': date,
}
def refresh(self, now=None):
"""
This fairly complex and heuristic function refreshes a server
response for replay.
- It adjusts date, expires and last-modified headers.
- It adjusts cookie expiration.
"""
if not now:
now = time.time()
delta = now - self.timestamp_start
refresh_headers = [
"date",
"expires",
"last-modified",
]
for i in refresh_headers:
if i in self.headers:
d = parsedate_tz(self.headers[i][0])
if d:
new = mktime_tz(d) + delta
self.headers[i] = [formatdate(new)]
c = []
for i in self.headers["set-cookie"]:
c.append(self._refresh_cookie(i, delta))
if c:
self.headers["set-cookie"] = c
def parse_prod(logdate):
global stats
global users
maildate = ''.join([x[-2:] for x in logdate.split('-')])
mailarchive = join(utils.get_conf()['Dir::Base'], 'mail/archive',
'mail-%s.xz' % maildate)
if not isfile(mailarchive):
return
(fd, tmpfile) = utils.temp_filename(utils.get_conf()['Dir::TempPath'])
system('xzcat %s > %s' % (mailarchive, tmpfile))
for message in mbox(tmpfile):
if (message['subject']
and message['subject'].startswith('Comments regarding')):
try:
member = users[' '.join(message['From'].split()[:-1])]
except KeyError:
continue
ts = mktime_tz(parsedate_tz(message['date']))
timestamp = datetime.fromtimestamp(ts).strftime("%Y%m%d%H%M%S")
date = parse_timestamp(timestamp)
if date not in stats:
stats[date] = {'stats': {'NEW': 0, 'ACCEPT': 0,
'REJECT': 0, 'PROD': 0}, 'members': {}}
if member not in stats[date]['members']:
stats[date]['members'][member] = {'ACCEPT': 0, 'REJECT': 0,
'PROD': 0}
if member not in stats['history']['members']:
stats['history']['members'][member] = {'ACCEPT': 0,
'REJECT': 0, 'PROD': 0}
stats[date]['stats']['PROD'] += 1
stats[date]['members'][member]['PROD'] += 1
stats['history']['stats']['PROD'] += 1
stats['history']['members'][member]['PROD'] += 1
unlink(tmpfile)
def email_list(self, prnt = True):
'''
Returns a list containing list representations of emails.
The representations are comprised of the sender, the timestamp, the subject, and the body; in that order.
'''
list_of_email = []
if self.date_criterion is None:
self.set_start_time()
self.conn.select()
typ, data = self.conn.search(None, self.date_criterion, 'UNSEEN')#Filter for emails today that have not been seen
#typ, data = self.conn.search(None, 'SINCE "12-Jan-2016"', 'SEEN')
for num in data[0].split():
typ, data = self.conn.fetch(num, '(RFC822)')
e_list = []
e_data = parser.BytesParser().parsebytes(data[0][1])
e_sender = e_data['From']
e_sender = utils.parseaddr(e_sender)[1]
e_time = utils.mktime_tz(utils.parsedate_tz(e_data['Date'])) #utc timestamp
e_subject = e_data['Subject'] #subject
if e_data.is_multipart():
body = ""
for payload in e_data.get_payload():
if(payload.get_content_type() == "text/plain"):
body += payload.get_payload()
else:
body = e_data.get_payload()
e_body = body
#e_body = body.split("<html>")[0] #payload
e_body = e_body.replace("=C2=A0", "")
e_body = e_body.replace("=\r\n", "")
e_body = e_body.replace("\r\n", "\n")
#e_body = quopri.decodestring(e_body)
#e_body = e_body.decode('utf-8')
e_list.append(e_sender)
e_list.append(e_time)
e_list.append(e_subject)
e_list.append(e_body)
if prnt:
print("Email Found:\nFrom: %s\nTime: %s\nSubject: %s\nBody:\n%s" % (e_list[0], e_list[1], e_list[2], e_list[3]))
list_of_email.append(e_list)
return list_of_email
def parseTwitterTime(datetimeStr):
"""
Convert a datetime as string from Twitter API to a datetime object.
:param datetimeStr: datetime value as a string from the Twitter API.
e.g. 'Thu Mar 19 12:37:15 +0000 2015'
:return: datetime object in UTC time. Printing this out will reflect
in the system's timezone.
e.g. entering time 12:00 for +0000 timezone will show as 14:00 if
printing in a system set to +0200 timezone, whether doing str(obj) or
str(obj.hour).
"""
timeTuple = parsedate_tz(datetimeStr)
timestamp = mktime_tz(timeTuple)
return datetime.datetime.fromtimestamp(timestamp)
def get_parsed_msg(self, msg_id):
'''
Recieves a message in the Gmail api format
Returns a simplyfied version of the msg as a dictionary
'''
msg = self.get_msg(msg_id)
parsed_msg = {}
for header in msg['payload']['headers']:
if header['name'] == 'Subject':
parsed_msg['subject'] = header['value']
if header['name'] == 'Date':
parsed_msg['date'] = dt(
*parsedate_tz(header['value'])[:6]).strftime('%Y-%m-%d')
if header['name'] == 'From':
parsed_msg['from'] = header['value']
parsed_msg['body'] = msg['snippet']
return parsed_msg
def mkcsv(
self,
bulk=False,
j=None,
):
if bulk == False:
self.csvfile = StringIO()
writer = csv.DictWriter(
self.csvfile,
fieldnames=fieldnames,
extrasaction='ignore',
quoting=csv.QUOTE_ALL,
)
self.writer = writer
self.writer.writeheader()
# todo: error handling
if not j:
j = self.result.json()
if "date" in self.result.headers:
date = self.result.headers['date']
timestamp = mktime_tz(parsedate_tz(date))
utctime = datetime.datetime.utcfromtimestamp(timestamp)
j["UTCDate"] = utctime
for k in (
"OrgIndustrialCategoryL",
"OrgIndustrialCategoryM",
"OrgIndustrialCategoryS",
"OrgIndustrialCategoryT",
"OrgIpoType",
"OrgCapitalCode",
"OrgEmployeesCode",
"OrgGrossCode",
):
if j[k]:
codes = j[k].split(",")
cnames = []
for c in codes:
cname = self.docodata_lookup(k, c)
if cname not in cnames:
cnames.append(cname)
j[k] = "／".join(cnames)
if self.input:
j["Input"] = self.input
self.writer.writerow(j)
return
def parsetime(self, strdatetime):
'''尝试处理feedparser未能识别的时间格式'''
try:
# 针对Mon,13 May 2013 06:48:25 GMT+8这样的奇葩格式
if strdatetime[-5:-2] == 'GMT':
t = datetime.strptime(strdatetime[:-6], '%a,%d %b %Y %H:%M:%S')
return (t - timedelta(hours=int(strdatetime[-2:-1])) +
krconfig.timezone)
# feedparser对非utc时间的支持有问题（Wes, 22 May 2013 13:54:00 +0800这样的）
elif (strdatetime[-5:-3] == '+0'
or strdatetime[-5:-3] == '-0') and strdatetime[-2:] == '00':
a = parsedate_tz(strdatetime)
t = datetime(*a[:6]) - timedelta(seconds=a[-1])
return (t + krconfig.timezone)
else:
return (datetime.utcnow() + krconfig.timezone)
except Exception, e:
return (datetime.utcnow() + krconfig.timezone)
def parse_date(value):
if not value:
return None
try:
if not isinstance(value, str):
value = str(value, "latin-1")
except Exception:
return None
t = parsedate_tz(value)
if t is None:
# Could not parse
return None
t = mktime_tz(t)
return datetime.fromtimestamp(t, UTC)
def format_article(art, **response_args):
art["X-FromName"], art["X-FromEmail"] = parseaddr(art["From"] if "From" in
art else "")
if art["X-FromName"] == '': art["X-FromName"] = art["X-FromEmail"]
date = mktime_tz(parsedate_tz(art["Date"]))
if date < time.time() - 120:
title = "\x0314In \x0F\x03{0:02d}{Newsgroups}\x0F\x0314: on \x0F{Date}\x0314 by \x0F\x03{0:02d}{X-FromName}\x0F \x02{Subject}\x0F"
else:
title = "\x0314In \x0F\x03{0:02d}{Newsgroups}\x0F\x0314: by \x0F\x03{0:02d}{X-FromName}\x0F \x02{Subject}\x0F"
return Response(art.get_payload().replace('\n', ' '),
title=title.format(
adler32(art["Newsgroups"].encode()) & 0xf,
adler32(art["X-FromEmail"].encode()) & 0xf,
**{h: decode_header(i)
for h, i in art.items()}),
**response_args)
def __get_date(self, message):
parsed_date = parsedate_tz(message.get('date'))
if not parsed_date:
msgdate = datetime.datetime(*(1979, 2, 4, 0, 0))
tz_secs = 0
return msgdate, tz_secs
try:
msgdate = datetime.datetime(*parsed_date[:6])
if msgdate.year < 100:
msgdate = msgdate.replace(year=msgdate.year + 1900)
except ValueError:
msgdate = datetime.datetime(*(1979, 2, 4, 0, 0))
tz_secs = parsed_date[-1] or 0
return msgdate, tz_secs
def test_generate_telesign_headers_default_date_and_nonce(self):
method_name = 'GET'
resource = '/v1/resource'
actual_headers = RestClient.generate_telesign_headers(
self.customer_id,
self.api_key,
method_name,
resource,
'',
user_agent='unit_test')
self.assertFalse(parsedate_tz(actual_headers.get('Date')) is None)
try:
UUID(actual_headers.get('x-ts-nonce'))
except (TypeError, ValueError):
self.fail("x-ts-nonce is not a UUID")
def get_commit_info(info, logger):
import re
from datetime import datetime
from email.utils import parsedate_tz, mktime_tz
author_regex = re.compile("^Author: (?P<name>.*) <(?P<email>.*)>$")
date_regex = re.compile("^Date: (?P<date>.*)$")
title_regex = re.compile("^ (?P<title>.*)$")
lines = info.split('\n')
author_name = author_regex.search(lines[1]).group('name')
author_email = author_regex.search(lines[1]).group('email')
date_str = date_regex.search(lines[2]).group('date')
date = datetime.utcfromtimestamp(mktime_tz(parsedate_tz(date_str)))
title = title_regex.search(lines[4]).group('title')
return (author_name, author_email, date, title)
def getStartDate(self, default=_MARKER):
"""
Returns the date from the mail date
"""
if not self.hasFile():
# Return the standard start date if no file was provided
if default is _MARKER:
return self._baseGetStartDate()
else:
return self._baseGetStartDate(default)
date_string = self.getContentInformation().get('Date', None)
if date_string:
parsed_date_string = parsedate_tz(date_string)
if parsed_date_string is not None:
time = mktime_tz(parsed_date_string)
if time:
return DateTime(time)
return self.getCreationDate()
def parse_date(dtstr):
"""
Attempts to parse a date with given formats first, then default formats
"""
# Handle empty string or None
if not dtstr: return None
try:
# Attempt to use the email utils parser first
dt = parsedate_tz(dtstr)
if dt is not None:
return datetime.utcfromtimestamp(
mktime_tz(dt)).replace(tzinfo=tzutc())
# Otherwise use the dateutil parser
return parser.parse(dtstr)
except:
return None
def is_cookie_expired(cookie):
value = None
if cookie['max-age']:
value = cookie['max-age']
elif cookie['expires']:
value = cookie['expires']
if not value:
return False
if value.isdigit():
time_received = getattr(cookie, 'time_received', time.time())
expires = time_received + int(value)
else:
expires = parsedate_tz(value)
if expires:
expires = mktime_tz(expires)
if expires and expires <= time.time():
return True
return False
def make_call(query, org_id, page):
"""
Make a single UMAPI call with error handling and server-controlled throttling.
(Adapted from sample code at https://www.adobe.io/products/usermanagement/docs/samples#retry)
:param query: a query method from a UMAPI instance (callable as a function)
:param org_id: the organization being queried
:param page: the page number of the desired result set
:return: the json (dictionary) received from the server (if any)
"""
wait_time = 0
num_attempts = 0
while num_attempts < retry_max_attempts:
if wait_time > 0:
sleep(wait_time)
wait_time = 0
try:
num_attempts += 1
return query(org_id, page)
except UMAPIRetryError as e:
logger.warning("UMAPI service temporarily unavailable (attempt %d) -- %s", num_attempts, e.res.status_code)
if "Retry-After" in e.res.headers:
advice = e.res.headers["Retry-After"]
advised_time = parsedate_tz(advice)
if advised_time is not None:
# header contains date
wait_time = int(mktime_tz(advised_time) - time())
else:
# header contains delta seconds
wait_time = int(advice)
if wait_time <= 0:
# use exponential back-off with random delay
delay = randint(0, retry_random_delay_max)
wait_time = (int(pow(2, num_attempts)) * retry_exponential_backoff_factor) + delay
logger.warning("Next retry in %d seconds...", wait_time)
continue
except UMAPIRequestError as e:
logger.warning("UMAPI error processing request -- %s", e.code)
return {}
except UMAPIError as e:
logger.warning("HTTP error processing request -- %s: %s", e.res.status_code, e.res.text)
return {}
logger.error("UMAPI timeout...giving up on results page %d after %d attempts.", page, retry_max_attempts)
return {}
def insertarTwitter(self, codigo, texto, codUsuario, usuario, seguidores,
rtweet, favoritos, fechaC, contador, link):
mongo = MongoClient()
db = mongo.callaut
collection = db.twitterBusqueda
idText = codigo
texto = texto
idUsuario = codUsuario
arrobaUsuario = usuario
followers = seguidores
retweet = rtweet
favorite = favoritos
cuentaInsert = contador
link = link
fechaCreacion = fechaC
timestamp = mktime_tz(parsedate_tz(fechaCreacion))
colombia = datetime.fromtimestamp(timestamp).strftime(
'%d-%m-%Y %H:%M:%S')
#enviado = 'N'
ahora = datetime.now()
fecha2 = "%Y-%m-%d %H:%M:%S"
fecha = ahora.strftime(fecha2)
mensaje5a = texto.find("RT", 0, 2)
if mensaje5a == -1 and arrobaUsuario != 'Avianca':
enviado = 'N'
else:
enviado = 'S'
tw = {
"idText": idText,
"texto": texto,
"idUsuario": idUsuario,
"arrobaUsuario": arrobaUsuario,
"followers": followers,
"retweet": retweet,
"favorite": favorite,
"cuentaInsert": cuentaInsert,
"enviado": enviado,
"fechaCreacion": fechaCreacion,
"fechaCreacionColombia": colombia,
"fecha": fecha,
"link": link
}
collection.insert(tw)
print("Se imprimio la info correctamente")
def get_emails():
msgs = []
try:
inbox = mailbox.Maildir(INBOX, factory=None, create=False)
except mailbox.NoSuchMailboxError:
logger.critical("You must run 'offlineimap.py' to sync email.")
return msgs
pybossa = mailbox.Maildir(PROCESSED_MESSAGES_DIR,
factory=None,
create=True)
problems = mailbox.Maildir(PROBLEMS_MESSAGES_DIR,
factory=None,
create=True)
# If ever mbox is desired...
# pybossa_mbox = mailbox.mbox(PROCESSED_MESSAGES_DIR + '_mbox',
# factory=None, create=True)
# problems_mbox = mailbox.mbox(PROBLEMS_MESSAGES_DIR + '_mbox',
# factory=None, create=True)
#
# Uncomment to use existing messages.
#
## inbox = pybossa
# Sort by date, but must parse date header to actual time object.
date_keys = []
for key, msg in inbox.iteritems():
# http://docs.python.org/2/library/email.util.html
date_keys.append((mktime_tz(parsedate_tz(msg['date'])), key))
for date_key in sorted(date_keys, key=lambda x: x[0]):
ret = process_msg(date_key[1], inbox, pybossa, problems)
if ret:
msgs.append(ret)
inbox.close()
pybossa.close()
problems.close()
return msgs
def store(msg):
"""
Stores an email message, if necessary, returning its primary key.
"""
message_id = msg['message-id']
# text = msg.as_string()
curs.execute("SELECT msgID FROM message WHERE msgMessageID=%s", (message_id,))
result = curs.fetchone()
if result:
return result[0]
date = msg['date']
name, email = parseaddr(msg['from'])
dt = datetime.fromtimestamp(mktime_tz(parsedate_tz(date)))
text = msg.as_string()
curs.execute("INSERT INTO message (msgMessageID, msgDate,msgSenderName,msgSenderAddress,msgText) VALUES (%s, %s, %s,%s,%s)",
(message_id, dt, name, email, text))
conn.commit()
curs.execute("SELECT msgID FROM message WHERE msgMessageID=%s", (message_id,))
return curs.fetchone()[0]
def _parse_retry_after(header):
'''Parse headers for Retry-After value'''
hit = re.match(r'^\s*([0-9]+)\s*$', header)
if hit:
val = int(header)
else:
date = parsedate_tz(header)
if date is None:
log.warning('Unable to parse retry-after value: %s', header)
return None
val = mktime_tz(*date) - time.time()
if val > 300 or val < 0:
log.warning('Ignoring retry-after value of %.3f s, using 1 s instead',
val)
val = 1
return val
def _parse_retry_after(header):
'''Parse headers for Retry-After value'''
hit = re.match(r'^\s*([0-9]+)\s*$', header)
if hit:
val = int(header)
else:
date = parsedate_tz(header)
if date is None:
log.warning('Unable to parse retry-after value: %s', header)
return None
val = mktime_tz(*date) - time.time()
val_clamp = min(300, max(1, val))
if val != val_clamp:
log.warning('Ignoring retry-after value of %.3f s, using %.3f s instead', val, val_clamp)
val = val_clamp
return val
def parseMbox(locationName, beginTimeFrame=0, endTimeFrame=int(time.time())):
mbox = mailbox.mbox(locationName)
listOfMails = []
for message in mbox:
try:
timestamp = mktime_tz(parsedate_tz(message['date']))
if int(timestamp) < endTimeFrame and int(
timestamp) > beginTimeFrame:
#http://stackoverflow.com/questions/12160010/email-datetime-parsing-with-python
mailInfo = timestamp, 'Mbox', \
"From: " + str(message['from']) + ", " \
"To: " + str(message['to']) + ", " \
"Subject: " + str(message['subject'])
listOfMails.append(mailInfo)
except TypeError:
pass
return listOfMails
def update_headers(self, resp):
headers = resp.headers
if 'expires' in headers:
return {}
if 'cache-control' in headers and headers['cache-control'] != 'public':
return {}
if resp.status not in self.cacheable_by_default_statuses:
return {}
if 'last-modified' not in headers:
return {}
parsed_date = parsedate_tz(headers.get('date'))
if self.require_date and parsed_date is None:
return {}
if parsed_date is None:
date = time.time()
faked_date = True
else:
date = calendar.timegm(parsed_date)
faked_date = False
last_modified = parsedate(headers['last-modified'])
if last_modified is None:
return {}
now = time.time()
current_age = max(0, now - date)
delta = date - calendar.timegm(last_modified)
freshness_lifetime = max(0, min(delta * self.error_margin, 24 * 3600))
if freshness_lifetime <= current_age:
return {}
expires = date + freshness_lifetime
new_headers = {
'expires': time.strftime(TIME_FMT, time.gmtime(expires))
}
if faked_date:
new_headers['date'] = time.strftime(TIME_FMT, time.gmtime(date))
return new_headers
def __init__(self, raw):
"""Initialize structure from a raw mail."""
try:
self.mail = Message(raw)
except Exception as exc:
log.error('Parse message failed %s' % exc)
raise
if self.mail.defects:
# XXX what to do ?
log.warn('Defects on parsed mail %r' % self.mail.defects)
self.recipients = self._extract_recipients()
self.parts = self._extract_parts()
self.headers = self._extract_headers()
self.subject = self.mail.get('Subject')
tmp_date = parsedate_tz(self.mail['Date'])
self.date = datetime.fromtimestamp(mktime_tz(tmp_date))
self.external_message_id = self.mail.get('Message-Id')
self.external_parent_id = self.mail.get('In-Reply-To')
self.size = len(raw)
def get_header_value(msg, header):
value = msg.get(header, '')
if value:
if header == 'Subject':
value = decode_str(value)
elif header == 'Date':
print(msg.get('Date'))
try:
date_time = datetime.strptime(msg.get('Date'),
"%a, %d %b %Y %H:%M:%S %z (%Z)")
value = date_time.strftime('%Y-%m-%d %H:%M')
except ValueError as e:
date_time = parsedate_tz(msg.get('Date'))
value = "%s-%s-%s %s:%s" % date_time[0:5]
else:
hdr, addr = parseaddr(value)
name = decode_str(hdr)
value = u'%s %s' % (name, addr)
return value
def process_email_message(raw_message,
email_obj,
update_raw=True,
update_attachments=True):
msg = email.message_from_string(raw_message)
date_tuple = parsedate_tz(msg['date'])
if date_tuple:
local_date = datetime.fromtimestamp(mktime_tz(date_tuple))
email_obj.date = local_date
email_obj.sender = _decode_msg_part(" ".join(
msg['from'].split()[:-1]).strip('"'))
address = msg['from'].split()[-1]
email_obj.from_address = re.sub(r'[<>]', '', address)
message_id = msg['message-id'].strip()
email_obj.message_id = re.sub(r'[<>]', '', message_id)
email_obj.subject = _decode_msg_part(msg['subject'])
parsed_email = _parse_email(msg)
email_obj.plain_text = parsed_email['body']
email_obj.html = parsed_email['html']
if update_attachments:
for attachment in parsed_email['attachments']:
attm = EmailAttachments()
attm.name = attachment.name.strip('"')
attm.size = attachment.size
attm.content_type = attachment.content_type
attm.create_date = attachment.create_date
attm.mod_date = attachment.mod_date.strip('"')
attm.read_date = attachment.read_date
attm.checksum = attachment.checksum
attm.binary = attachment.getvalue()
email_obj.attachments.append(attm)
if update_raw:
email_obj.raw = raw_message
def verify_headers(btctxstore, headers, timeout_sec, sender_address,
recipient_address):
""" Verify authentication headers from http query.
Arguments:
btctxstore: BtcTxStore instance used to verify signature.
headers: Authentication headers to be verified.
timeout_sec: Timeout in seconds, where the date is valid.
sender_address: Sender bitcoin address used for signing.
recipient_address: The bitcoin address of the recipient.
Returns:
True if authentication headers are valid
Raises:
storjcore.auth.AuthError: if date or signature is invalid
storjcore.sanitize.ValidationError: if input was invalid
"""
# sanitize input
btctxstore = sanitize.is_btctxstore(btctxstore)
headers = sanitize.is_dict(headers)
signature = sanitize.is_signature(headers.get("Authorization"))
date = sanitize.is_header_date(headers.get("Date"))
timeout_sec = sanitize.is_unsigned_int(timeout_sec)
sender_address = sanitize.is_btcaddress(btctxstore, sender_address)
recipient_address = sanitize.is_btcaddress(btctxstore, recipient_address)
# verify date
clientdate = datetime.fromtimestamp(mktime_tz(parsedate_tz(date)))
timeout = timedelta(seconds=timeout_sec)
delta = abs(datetime.now() - clientdate)
if delta >= timeout:
msg = "Invalid header date {0} >= {1}!".format(delta, timeout)
raise AuthError(msg)
# verify signature
msg = recipient_address + " " + date
if not btctxstore.verify_signature_unicode(sender_address, signature, msg):
msg = "Invalid signature for auth addr {0}!".format(sender_address)
raise AuthError(msg)
return True
def __init__(self, msg=None):
self.message_id = None
self.references = []
self.subject = None
self.sender_address = None
self.sender_name = None
self.created = None
# read the sender data
m = sender_addr_regexp.search(msg.get('From'))
if m:
self.sender_address = m.group(1).strip().lower()
else:
self.sender_address = msg.get('From').lower()
m = sender_name_regexp.search(msg.get('From'))
if m:
self.sender_name = m.group(1).strip()
# read the message timestamp
self.created = int(mktime_tz(parsedate_tz(msg.get('Date'))))
# Set the message id
m = msgid_regexp.search(msg.get("Message-ID", ""))
if m is None:
raise ValueError("Message does not contain a Message-ID: header")
self.message_id = m.group(1)
# Get list of unique message IDs from the References: header
refs = msg.get("References", "")
self.references = msgid_regexp.findall(refs)
self.references = uniq(self.references)
self.subject = msg.get('Subject', "No subject")
# Get In-Reply-To: header and add it to references
in_reply_to = msg.get("In-Reply-To", "")
m = msgid_regexp.search(in_reply_to)
if m:
msg_id = m.group(1)
if msg_id not in self.references:
self.references.append(msg_id)
def date(*args, **kwargs):
""" Returns a Date from the given parameters:
- date(format=Date.format) => now
- date(int)
- date(string)
- date(string, format=Date.format)
- date(string, inputformat, format=Date.format)
- date(year, month, day, hours, minutes, seconds, format=Date.format)
If a string is given without an explicit input format, all known formats will be tried.
"""
d = None
if len(args) == 0 or args[0] == NOW:
# No parameters, return NOW.
d = Date.now()
elif len(args) == 1 \
and isinstance(args[0], int) \
or isinstance(args[0], basestring) and args[0].isdigit():
# One parameter, an int or string timestamp.
d = Date.fromtimestamp(int(args[0]))
elif len(args) == 1 and isinstance(args[0], basestring):
# One parameter, a date string for which we guess the input format.
# First, try to parse as a RFC2822 date.
# Otherwise, try to parse with a list of known formats.
try:
d = Date.fromtimestamp(mktime_tz(parsedate_tz(args[0])))
except:
for f in (kwargs.get("format") or []) + formats:
try:
d = Date.strptime(args[0], f)
break
except:
pass
if d is None:
raise DateError, "unknown date format for %s" % repr(args[0])
elif len(args) == 2 and isinstance(args[0], basestring):
# Two parameters, a date string and an explicit input format.
d = Date.strptime(args[0], args[1])
else:
# Many parameters: year, month, day, hours, minutes, seconds.
d = Date(*args[:7], **kwargs)
d.format = kwargs.get("format") or len(args) > 7 and args[7] or Date.format
return d
def getDetailInbox(service, msgId, userId='me'):
res = service.users().messages().get(userId=userId, id=msgId).execute()
msg = {}
res = res['payload']
for m in res['headers']:
msg[m['name']] = m['value']
# process timestamp
et = mktime_tz(parsedate_tz(msg[u'Date']))
et = datetime.datetime.fromtimestamp(et)
msg[u'Date'] = et.strftime('%a, %d %b %Y %I:%M %p')
# process body
msg['Attachments'] = []
if 'parts' in res:
for m in res['parts']:
if m['filename']:
getAttachments(service, 'me', msgId, m['body']['attachmentId'],
m['filename'])
msg['Attachments'].append(m['filename'])
else:
if m['mimeType'] == 'text/plain' or m[
'mimeType'] == 'text/html':
msg[u'body'] = (bd(str(m['body']['data']))).decode('utf-8')
if m['mimeType'] == 'multipart/alternative':
for n in m['parts']:
if n['mimeType'] == 'text/plain' or n[
'mimeType'] == 'text/html':
msg[u'body'] = (bd(str(
n['body']['data']))).decode('utf-8')
else:
msg[u'body'] = (bd(str(res['body']['data']))).decode('utf-8')
# process from
msg[u'From'] = msg[u'From'].split(' ')
if len(msg[u'From']) > 1:
msg[u'FromName'] = ' '.join(msg[u'From'][:-1])
msg[u'FromEmail'] = msg[u'From'][-1]
else:
msg[u'FromName'] = msg[u'From'][0]
msg[u'FromEmail'] = msg[u'From'][0]
return msg
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self):
m = message_from_string(cvs1_11_msg)
src = CVSMaildirSource('/dev/null')
src, chdict = src.parse(m)
self.assertNotEqual(chdict, None)
self.assertEqual(chdict['author'], 'andy')
self.assertEqual(len(chdict['files']), 1)
self.assertEqual(
chdict['files'][0], 'base/module/src/make/GNUmakefile')
self.assertEqual(chdict['comments'], 'Commented out some stuff.\n')
self.assertFalse(chdict['isdir'])
self.assertEqual(chdict['revision'], '2010-08-07 11:11:49')
dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000')
self.assertEqual(chdict['when'], mktime_tz(dateTuple))
self.assertEqual(chdict['branch'], None)
self.assertEqual(
chdict['repository'], ':ext:cvshost.example.com:/cvsroot')
self.assertEqual(chdict['project'], 'MyModuleName')
self.assertEqual(len(chdict['properties']), 0)
self.assertEqual(src, 'cvs')
def test_CVSMaildirSource_create_change_from_cvs1_12msg(self):
m = message_from_string(cvs1_12_msg)
src = CVSMaildirSource('/dev/null')
src, chdict = src.parse(m)
self.assertNotEqual(chdict, None)
self.assertEqual(chdict['author'], 'andy')
self.assertEqual(len(chdict['files']), 2)
self.assertEqual(chdict['files'][0], 'base/module/src/file1.cpp')
self.assertEqual(chdict['files'][1], 'base/module/src/file2.cpp')
self.assertEqual(chdict['comments'], 'Changes for changes sake\n')
self.assertFalse(chdict['isdir'])
self.assertEqual(chdict['revision'], '2010-08-11 04:56:44')
dateTuple = parsedate_tz('Wed, 11 Aug 2010 04:56:44 +0000')
self.assertEqual(chdict['when'], mktime_tz(dateTuple))
self.assertEqual(chdict['branch'], None)
self.assertEqual(
chdict['repository'], ':ext:cvshost.example.com:/cvsroot')
self.assertEqual(chdict['project'], 'MyModuleName')
self.assertEqual(len(chdict['properties']), 0)
self.assertEqual(src, 'cvs')