def test_newsgroup_link(self):
self.assertEqual(OrgFormat.newsgroup_link('foo'), '[[news:foo][foo]]')
self.assertEqual(OrgFormat.newsgroup_link('comp.emacs'),
'[[news:comp.emacs][comp.emacs]]')
with self.assertRaises(AssertionError):
OrgFormat.newsgroup_link('')
def __handle_file(self, photo_file, filename):
"""
checks if file is an image, try to get exif data and
write to org file
"""
logging.debug("handling file %s", filename)
# check if file is an image:
if imghdr.what(filename) != None:
datetime = get_exif_datetime(filename)
if datetime == None:
logging.debug("skipping: %s has no EXIF information", filename)
else:
try:
datetime = time.strptime(datetime, "%Y:%m:%d %H:%M:%S")
timestamp = OrgFormat.date(datetime, show_time=True)
output = OrgFormat.link(filename, photo_file)
properties = OrgProperties(photo_file + timestamp)
self._writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties)
except ValueError as e:
logging.warning("skipping: Could not parse " + \
"timestamp for %s : %s", filename, e)
def __get_datetime(self, mydate):
"""
@return string: Datetime - in Org Format
"""
mydate_tuple = OrgFormat.parse_basic_iso_datetime(mydate)
return OrgFormat.date(mydate_tuple)
def _handle_url(self, params):
timestamp = datetime.datetime.fromtimestamp(
int(params['timestamp'] / 1000000))
if not self._args.omit_drawer:
properties = OrgProperties()
if (params['title'] == ""):
params['title'] = params['url']
properties.add('URL', params['url'])
properties.add('VISIT_COUNT', params['visit_count'])
output = OrgFormat.link(params['url'], params['title'])
try:
output = self._args.output_format.decode('utf-8').format(**params)
except Exception:
pass
if self._args.omit_drawer:
self._writer.write_org_subitem(timestamp=OrgFormat.date(
timestamp, show_time=True),
output=output,
properties=None)
else:
self._writer.write_org_subitem(timestamp=OrgFormat.date(
timestamp, show_time=True),
output=output,
properties=properties)
def test_daterange_autodetect_time(self):
# NOTE: time.strptime() returns a time.struct_time
self.assertEqual(
OrgFormat.daterange_autodetect_time(
time.strptime('2011-11-29T20:38:42', '%Y-%m-%dT%H:%M:%S'),
time.strptime('2011-11-30T23:59:58', '%Y-%m-%dT%H:%M:%S'), inactive=True),
'[2011-11-29 Tue 20:38]--[2011-11-30 Wed 23:59]')
self.assertEqual(
OrgFormat.daterange_autodetect_time(
time.strptime('2011-11-29T20:38', '%Y-%m-%dT%H:%M'),
time.strptime('2011-11-30T23:59', '%Y-%m-%dT%H:%M'), inactive=True),
'[2011-11-29 Tue 20:38]--[2011-11-30 Wed 23:59]')
self.assertEqual(
OrgFormat.daterange_autodetect_time(
time.strptime('2011-11-29', '%Y-%m-%d'),
time.strptime('2011-11-30T23:59', '%Y-%m-%dT%H:%M'), inactive=True),
'[2011-11-29 Tue 00:00]--[2011-11-30 Wed 23:59]')
self.assertEqual(
OrgFormat.daterange_autodetect_time(
time.strptime('2011-11-29T20:38', '%Y-%m-%dT%H:%M'),
time.strptime('2011-11-30', '%Y-%m-%d'), inactive=True),
'[2011-11-29 Tue 20:38]--[2011-11-30 Wed 00:00]')
self.assertEqual(
OrgFormat.daterange_autodetect_time(
time.strptime('2011-11-29', '%Y-%m-%d'),
time.strptime('2011-11-30', '%Y-%m-%d'), inactive=True),
'[2011-11-29 Tue]--[2011-11-30 Wed]')
def test_mailto_link(self):
self.assertEqual(OrgFormat.mailto_link('Bob Bobby <*****@*****.**>'),
'[[mailto:[email protected]][Bob Bobby]]')
self.assertEqual(OrgFormat.mailto_link('<*****@*****.**>'),
'[[mailto:[email protected]][[email protected]]]')
self.assertEqual(OrgFormat.mailto_link('*****@*****.**'),
'[[mailto:[email protected]][[email protected]]]')
self.assertEqual(OrgFormat.mailto_link('foo bar'),
'[[mailto:foo bar][foo bar]]')
def test_parse_extended_iso_datetime(self):
# NOTE: time.strptime() returns a time.struct_time
self.assertEqual(OrgFormat.parse_extended_iso_datetime("2011-1-2"),
time.strptime('2011-01-02', '%Y-%m-%d'))
self.assertEqual(OrgFormat.parse_extended_iso_datetime("2011-1-2T3.4"),
time.strptime('2011-01-02 03.04', '%Y-%m-%d %H.%M'))
self.assertEqual(OrgFormat.parse_extended_iso_datetime("2011-1-2T3.4.5"),
time.strptime('2011-01-02 03.04.05', '%Y-%m-%d %H.%M.%S'))
self.assertEqual(OrgFormat.parse_extended_iso_datetime("2011-1-2T3:4"),
time.strptime('2011-01-02 03.04', '%Y-%m-%d %H.%M'))
self.assertEqual(OrgFormat.parse_extended_iso_datetime("2011-1-2T3:4:5"),
time.strptime('2011-01-02 03.04.05', '%Y-%m-%d %H.%M.%S'))
self.assertEqual(OrgFormat.parse_extended_iso_datetime("2011-1-2 3:4"),
time.strptime('2011-01-02 03.04', '%Y-%m-%d %H.%M'))
self.assertEqual(OrgFormat.parse_extended_iso_datetime("2011-1-2 3:4:5"),
time.strptime('2011-01-02 03.04.05', '%Y-%m-%d %H.%M.%S'))
self.assertEqual(OrgFormat.parse_extended_iso_datetime("2011-1-2 3.4"),
time.strptime('2011-01-02 03.04', '%Y-%m-%d %H.%M'))
self.assertEqual(OrgFormat.parse_extended_iso_datetime("2011-1-2 3.4.5"),
time.strptime('2011-01-02 03.04.05', '%Y-%m-%d %H.%M.%S'))
def test_link(self):
self.assertEqual(OrgFormat.link('foo/bar'),
'[[foo/bar]]')
self.assertEqual(OrgFormat.link('foo/bar', 'my description'),
'[[foo/bar][my description]]')
self.assertEqual(OrgFormat.link('file:foo/bar/some file.pdf'),
'[[file:foo/bar/some%20file.pdf]]')
self.assertEqual(OrgFormat.link('file:foo/bar/some file.pdf', 'my description'),
'[[file:foo/bar/some%20file.pdf][my description]]')
self.assertEqual(OrgFormat.link('file:foo/bar/some file.pdf', 'my description', replacespaces=False),
'[[file:foo/bar/some file.pdf][my description]]')
def __getTimestamp(self, time, onlyDate=False):
"""
converts xml timestamp into org readable timestamp
Do 6 Nov 21:22:17 2014
"""
time = time.strip()
mail_date = datetime.strptime(time,"%c")
if onlyDate is False:
return OrgFormat.date(mail_date, show_time=True)
return OrgFormat.date(mail_date)
def test_fix_struct_time_wday(self):
self.assertEqual(OrgFormat.fix_struct_time_wday(time.struct_time([2013, 4, 3, 10, 54, 0, 0, 0, 0])),
time.struct_time([2013, 4, 3, 10, 54, 0, 2, 0, 0]))
# same test but a bit more human readable for understanding:
# define a time-stamp with weekday == 0:
timestamp = time.struct_time([2013, 4, 3, 10, 54, 0, 0, 0, 0])
# OrgFormat.date(timestamp) ## '<2013-04-03 Mon>' -> Mon is wrong for April 3rd 2013
self.assertEqual(OrgFormat.date(OrgFormat.fix_struct_time_wday(timestamp), show_time=True, inactive=False),
'<2013-04-03 Wed 10:54>')
def test_date(self):
# NOTE: time.strptime() returns a time.struct_time
self.assertEqual(
OrgFormat.date(
time.strptime('2011-11-02T20:38:42', '%Y-%m-%dT%H:%M:%S')),
'<2011-11-02 Wed>')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38',
'%Y-%m-%dT%H:%M')),
'<2011-11-02 Wed>')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
show_time=True), '<2011-11-02 Wed 20:38>')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'),
show_time=True), '<2011-11-02 Wed 20:38>')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
inactive=True), '[2011-11-02 Wed]')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'),
inactive=True), '[2011-11-02 Wed]')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
inactive=True,
show_time=True), '[2011-11-02 Wed 20:38]')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'),
inactive=True,
show_time=True), '[2011-11-02 Wed 20:38]')
## testing repeater_or_delay:
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'),
inactive=False,
repeater_or_delay='+2w '), '<2011-11-02 Wed +2w>')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
inactive=True,
show_time=True,
repeater_or_delay=' ++1m '),
'[2011-11-02 Wed 20:38 ++1m]')
def test_apply_timedelta_to_org_timestamp(self):
self.assertEqual(
OrgFormat.apply_timedelta_to_org_timestamp(
'<2019-11-05 Tue 23:59>', 1), '<2019-11-06 Wed 00:59>')
self.assertEqual(
OrgFormat.apply_timedelta_to_org_timestamp(
'<2019-11-06 Wed 00:59>', -1), '<2019-11-05 Tue 23:59>')
self.assertEqual(
OrgFormat.apply_timedelta_to_org_timestamp(
'<2019-11-05 Tue 23:59>', 2.0), '<2019-11-06 Wed 01:59>')
self.assertEqual(
OrgFormat.apply_timedelta_to_org_timestamp(
'<2019-11-06 Wed 00:59>', -2.0), '<2019-11-05 Tue 22:59>')
self.assertEqual(
OrgFormat.apply_timedelta_to_org_timestamp(
'<2019-12-31 Tue 23:00>', 2.5), '<2020-01-01 Wed 01:30>')
self.assertEqual(
OrgFormat.apply_timedelta_to_org_timestamp(
'<2020-01-01 Wed 01:30>', -2.5), '<2019-12-31 Tue 23:00>')
self.assertEqual(
OrgFormat.apply_timedelta_to_org_timestamp(
'<2019-12-31 Tue 23:00>-<2019-12-31 Tue 23:30>', 2.5),
'<2020-01-01 Wed 01:30>--<2020-01-01 Wed 02:00>')
self.assertEqual(
OrgFormat.apply_timedelta_to_org_timestamp(
'<2020-01-01 Wed 01:30>--<2020-01-01 Wed 02:00>', -2.5),
'<2019-12-31 Tue 23:00>--<2019-12-31 Tue 23:30>')
def test_autotag(self):
test_filename = self.TMPFOLDER + "testautotag.org"
autotag_dict = {}
autotag_dict["programming"] = ["programming", "python", "java"]
autotag_dict["TUG"] = ["tugraz", "university"]
output = "Programming for my bachelor thesis at University"
# writing test output
writer = OrgOutputWriter(short_description="short-des",
tag="tag",
file_name=test_filename,
autotag_dict=autotag_dict)
timestamp = OrgFormat.date(time.gmtime(0), show_time=True)
properties = OrgProperties("data_for_hashing")
writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties)
writer.close()
# read and check the file_handler
file_handler = codecs.open(test_filename, "r", "utf-8")
input_handler = file_handler.readlines()
file_handler.close()
self.assertTrue(input_handler[4].startswith(
"** <1970-01-01 Thu 00:00> Programming for my " +
"bachelor thesis at University\t:"))
self.assertTrue(input_handler[4].endswith("programming:TUG:\n")
or input_handler[4].endswith("TUG:programming:\n"))
def __check_datestamp_correctness(self, datestamp):
"""
Checks a datestamp 'YYYY.MM.DD' if its components are a valid date.
"""
if len(datestamp) != 10:
return False
try:
year = int(datestamp[:4])
month = int(datestamp[5:7])
day = int(datestamp[8:10])
except ValueError:
logging.debug(
'__check_datestamp_correctness(' + str(datestamp) +
') does not follow YYYY-MM-DD with integers as components for year, month, day.'
)
return False
if year < 1900 or \
year > 2100 or \
month < 1 or \
month > 12 or \
day < 1 or \
day > 31:
logging.debug('__check_datestamp_correctness(' + str(datestamp) +
') NEGATIVE')
return False
else:
try:
orgdate = OrgFormat.strdate(datestamp)
except ValueError:
return False
return True
def startElement(self, name, attrs):
"""
at every <sms> tag write to orgfile
"""
logging.debug("Handler @startElement name=%s,attrs=%s", name, attrs)
if name == "sms":
#sms_subject = attrs['subject']
sms_date = int(attrs['date']) / 1000 # unix epoch
sms_body = attrs['body']
sms_address = attrs['address']
sms_time = attrs['time']
sms_service_center = attrs['service_center']
sms_type_incoming = int(attrs['type']) == 1
contact_name = attrs['name']
skip = False
if sms_type_incoming == True:
output = "SMS from "
if self._ignore_incoming:
skip = True
else:
output = "SMS to "
if self._ignore_outgoing:
skip = True
if not skip:
name_string = ""
if contact_name:
name_string = '[[contact:' + contact_name + '][' + contact_name + ']]'
else:
name_string = "Unknown"
output += name_string + ": "
#if sms_subject != "null":
# in case of MMS we have a subject
# output += sms_subject
# notes = sms_body
#else:
# output += sms_body
# notes = ""
notes = sms_body
timestamp = OrgFormat.date(time.gmtime(sms_date),
show_time=True)
data_for_hashing = output + timestamp + notes
properties = OrgProperties(data_for_hashing=data_for_hashing)
properties.add("NUMBER", sms_address)
properties.add("NAME", contact_name)
properties.add("SMS_SERVICE_CENTER", sms_service_center)
properties.add("TIME", sms_time)
self._writer.write_org_subitem(output=output,
timestamp=timestamp,
note=notes,
properties=properties)
def write_point(self, p):
"""write a point (including geocoding)"""
timestamp = OrgFormat.date(p.time, show_time=True)
geocode = self.reverse_geocode(p.latitude, p.longitude)
output = self._args.output_format.format(**geocode)
tags = []
properties = OrgProperties(data_for_hashing=timestamp)
if p.latitude:
properties.add('LATITUDE', p.latitude)
if p.longitude:
properties.add('LONGITUDE', p.longitude)
if p.source:
tags.append(p.source.lower())
if timestamp:
self._writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties,
tags=tags)
def format_timestamp(self, timestamp):
# show time with the timestamp format, but only
# if it contains at least hours and minutes
show_time = not self._args.timestamp_format or \
any(x in self._args.timestamp_format for x in ['%H', '%M'])
timestamp = OrgFormat.date(timestamp, show_time=show_time)
return timestamp
def _handle_message(self, msg):
"""parse a single message row"""
msg['number'] = '00' + msg['number'].split('@')[0]
msg['name'] = self._numberdict.get(msg['number'], msg['number'])
msg['verb'] = 'to' if msg['type'] else 'from'
msg['type'] = 'OUTGOING' if msg['type'] else 'INCOMING'
msg['handler'] = self._args.handler
if msg['text']:
if self._args.demojize:
msg['text'] = emoji.demojize(msg['text'])
if self._args.skip_emoji:
msg['text'] = re.sub(emoji.get_emoji_regexp(), '', msg['text'])
timestamp = datetime.datetime.fromtimestamp(msg['timestamp'] / 1000)
properties = OrgProperties(data_for_hashing=json.dumps(msg))
properties.add('NUMBER', msg['number'])
properties.add('TYPE', msg['type'])
output = self._args.output_format.format(**msg)
if msg['text'] and not self._is_ignored(msg):
self._writer.write_org_subitem(timestamp=OrgFormat.date(
timestamp, show_time=True),
output=output,
properties=properties)
def write_org_subitem(self,
timestamp,
output,
note="",
properties=OrgProperties(),
tags=None):
"""
Writes an org item line.
i.e:** <timestamp> <output> :<tags>:\n
:PROPERTIES:
<properties>
:ID: -generated id-
:END:
if an argument -a or --append is given,
then a desicion regarding the :ID: is made if the item has to be
written to file
@param timestamp: str/unicode
@param output: st tar/unicode
@param note: str/unicode
@param tags: list of tags
@param properties: OrgProperties object
"""
assert (timestamp.__class__ == str or timestamp.__class__ == unicode)
assert tags.__class__ == list or tags == None
assert properties.__class__ == OrgProperties
assert (output.__class__ == str or output.__class__ == unicode)
assert (note.__class__ == str or note.__class__ == unicode)
# count the entries we have written, if above our limit do not write
if self.__number_entries and \
self.__entries_count == self.__number_entries:
return
else:
self.__entries_count += 1
if tags == None:
tags = []
if self.__autotag_dict != {}:
self.__get_autotags(tags, output)
## fix time-stamps (if user wants to)
if self.__timestamp_delta:
timestamp = OrgFormat.apply_timedelta_to_Orgmode_timestamp(timestamp, int(self.__timestamp_delta))
if self.__append:
self.__append_org_subitem(timestamp,
output,
note,
properties,
tags)
else:
self.__write_org_subitem(timestamp,
output,
note,
properties,
tags)
def __get_datetime_range(self, dtstart, dtend):
"""
@return string: Datetime - Range in Org Format
"""
begin_tuple = OrgFormat.parse_basic_iso_datetime(dtstart)
end_tuple = OrgFormat.parse_basic_iso_datetime(dtend)
# handle "all-day" - events
if begin_tuple.tm_sec == 0 and \
begin_tuple.tm_min == 0 and \
begin_tuple.tm_hour == 0 and \
end_tuple.tm_sec == 0 and \
end_tuple.tm_min == 0 and \
end_tuple.tm_hour == 0:
# we have to subtract 1 day to get the correct dates
end_tuple = time.localtime(time.mktime(end_tuple) - 24 * 60 * 60)
return OrgFormat.daterange_autodetect_time(begin_tuple, end_tuple)
def test_daterange(self):
# NOTE: time.strptime() returns a time.struct_time
self.assertEqual(
OrgFormat.daterange(time.strptime('2011-11-29', '%Y-%m-%d'),
time.strptime('2011-11-30', '%Y-%m-%d')),
'<2011-11-29 Tue>--<2011-11-30 Wed>')
self.assertEqual(
OrgFormat.daterange(
time.strptime('2011-11-29T20:38:42', '%Y-%m-%dT%H:%M:%S'),
time.strptime('2011-11-30T23:59:58', '%Y-%m-%dT%H:%M:%S')),
'<2011-11-29 Tue>--<2011-11-30 Wed>')
self.assertEqual(
OrgFormat.daterange(time.strptime('2011-11-29T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
time.strptime('2011-11-30T23:59:58',
'%Y-%m-%dT%H:%M:%S'),
show_time=False),
'<2011-11-29 Tue>--<2011-11-30 Wed>')
self.assertEqual(
OrgFormat.daterange(time.strptime('2011-11-29T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
time.strptime('2011-11-30T23:59:58',
'%Y-%m-%dT%H:%M:%S'),
show_time=False,
inactive=True),
'[2011-11-29 Tue]--[2011-11-30 Wed]')
self.assertEqual(
OrgFormat.daterange(time.strptime('2011-11-29T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
time.strptime('2011-11-30T23:59:58',
'%Y-%m-%dT%H:%M:%S'),
show_time=True,
inactive=True),
'[2011-11-29 Tue 20:38]--[2011-11-30 Wed 23:59]')
self.assertEqual(
OrgFormat.daterange(time.strptime('2011-11-29T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
time.strptime('2011-11-30T23:59:58',
'%Y-%m-%dT%H:%M:%S'),
show_time=False,
inactive=False),
'<2011-11-29 Tue>--<2011-11-30 Wed>')
self.assertEqual(
OrgFormat.daterange(time.strptime('2011-11-29T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
time.strptime('2011-11-30T23:59:58',
'%Y-%m-%dT%H:%M:%S'),
inactive=True),
'[2011-11-29 Tue]--[2011-11-30 Wed]')
def test_properties_with_own_created(self):
p = OrgProperties()
p.add("CREATED", OrgFormat.date(time.gmtime(0), show_time=True))
properties = str(p).splitlines()
self.assertEqual(properties[0], " :PROPERTIES:")
self.assertEqual(properties[1], " :CREATED: <1970-01-0" + \
"1 Thu 00:00>")
self.assertEqual(properties[2], " :ID: fede47e9" + \
"f49e1b7f5c6599a6d607e9719ca98625")
self.assertEqual(properties[3], " :END:")
def __write_footer(self):
"""
Writes the footer of the file including calling python script and time
Don't call this function - call instead function close(),
close() does call this function
"""
self.writeln(u"* successfully parsed by " + \
sys.argv[0] + u" at " + \
OrgFormat.inactive_datetime(time.localtime()) + \
u" in ~" + self.__time + u".")
def _handle_row(self, row):
"""
handle a single row
"""
try:
# assume unix timestamp
if not self._args.timestamp_format:
timestamp = datetime.datetime.fromtimestamp(
int(row[self._args.timestamp_field]))
else:
timestamp = time.strptime(row[self._args.timestamp_field],
self._args.timestamp_format)
# show time with the timestamp format, but only
# if it contains at least hours and minutes
if not self._args.timestamp_format or \
any(x in self._args.timestamp_format for x in ['%H', '%M']):
timestamp = OrgFormat.date(timestamp, show_time=True)
else:
timestamp = OrgFormat.date(timestamp)
except ValueError as e:
logging.error("timestamp-format does not match: %s", e)
sys.exit(1)
except IndexError as e:
logging.error("did you specify the right delimiter?", e)
sys.exit(1)
properties = OrgProperties(data_for_hashing=json.dumps(row))
output = self._args.output_format.format(**row)
if self._args.properties:
for prop in self._args.properties.split(','):
properties.add(prop.upper().strip(), row[prop])
self._writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties)
def _main(self):
APP_KEY = self._get_config_option("APP_KEY")
APP_SECRET = self._get_config_option("APP_SECRET")
OAUTH_TOKEN = self._get_config_option("OAUTH_TOKEN")
OAUTH_TOKEN_SECRET = self._get_config_option("OAUTH_TOKEN_SECRET")
screen_name = self._get_config_option("screen_name")
count = self._get_config_option("count")
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
try:
home_timeline = twitter.get_home_timeline(screenname=screen_name,
count=count)
except TwythonError as e:
logging.error(e)
sys.exit(1)
for tweet in home_timeline:
# strptime doesn't support timezone info, so we are using dateutils.
date_object = parser.parse(tweet['created_at'])
timestamp = OrgFormat.date(date_object, show_time=True)
try:
# Data is already Unicode, so don't try to re-encode it.
output = tweet['text']
except:
logging.error(sys.exc_info()[0])
print("Error: ", sys.exc_info()[0])
data_for_hashing = output + timestamp + output
properties = OrgProperties(data_for_hashing=data_for_hashing)
properties.add("name", tweet['user']['name'])
properties.add("twitter_id", tweet['id'])
properties.add("contributors", tweet['contributors'])
properties.add("truncated", tweet['truncated'])
properties.add("in_reply_to_status_id",
tweet['in_reply_to_status_id'])
properties.add("favorite_count", tweet['favorite_count'])
properties.add("source", tweet['source'])
properties.add("retweeted", tweet['retweeted'])
properties.add("coordinates", tweet['coordinates'])
properties.add("entities", tweet['entities'])
self._writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties)
def __write_footer(self):
"""
Writes the footer of the file including calling python script and time
Don't call this function - call instead function close(),
close() does call this function
"""
self.writeln(u"* successfully parsed " +\
unicode(self.__entries_count) + \
" entries by " + \
sys.argv[0] + u" at " + \
OrgFormat.inactive_datetime(time.localtime()) + \
u" in ~" + self.__time + u".")
def __write_footer(self):
"""
Writes the footer of the file including calling python script and time
Don't call this function - call instead function close(),
close() does call this function
"""
self.writeln("* successfully parsed " +\
str(self.__entries_count) + \
" entries by " + \
sys.argv[0] + " at " + \
OrgFormat.date(time.localtime(), show_time=True, inactive=True) + \
" in ~" + self.__time.strip() + ".")
def __set_author_timestamp(self, line):
"""
extracts the date + time from line:
author Forename Lastname <mail> 1234567890 +0000
@param line
"""
self.__empty = False
date_info = line[-16:] # 1234567890 +0000
seconds_since_epoch = float(date_info[:10])
#timezone_info = date_info[11:]
self.__timestamp = OrgFormat.date(
time.localtime(seconds_since_epoch), show_time=True)
self.__author = line[7:line.find("<")].strip()
def __write(self):
"""
write attributes to writer (make an org_sub_item)
"""
logging.debug("msg:%s", self.__msg)
self.__msg = self.__msg.splitlines()
subject = ""
notes = ""
# idea: look for the first -nonempty- message
if len(self.__msg) > 0:
start_notes = 0
for i in range(len(self.__msg)):
if self.__msg[i].strip() != "":
subject = self.__msg[i].strip()
start_notes = i + 1
break
if len(self.__msg) > start_notes:
for n in self.__msg[start_notes:]:
if n != "":
notes += n + "\n"
output = "%s (r%d): %s" % (self.__author, self.__rev, subject)
properties = OrgProperties(data_for_hashing=self.__author + subject)
timestamp = OrgFormat.date(OrgFormat.parse_basic_iso_datetime(
self.__date),
show_time=True)
properties.add("REVISION", self.__rev)
if self.__grepauthor == None or \
(self.__author.strip() == self.__grepauthor.strip()):
self._writer.write_org_subitem(output=output,
timestamp=timestamp,
note=notes,
properties=properties)
def __write_file(self, file, link, timestamp):
"""
write entry to org file (omit replacement of spaces in file names)
"""
output = OrgFormat.link(link="file:" + link,
description=file,
replacespaces=False)
properties = None
if not self._args.omit_drawers:
# we need optional data for hashing due it can be, that more
# than one file have the same timestamp
properties = OrgProperties(data_for_hashing=output)
self._writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties)
def test_orgmode_timestamp_to_datetime(self):
self.assertEqual(OrgFormat.orgmode_timestamp_to_datetime(
'<1980-12-31 Wed 23:59>'),
datetime.datetime(1980, 12, 31, 23, 59, 0, tzinfo=None))
self.assertEqual(OrgFormat.orgmode_timestamp_to_datetime(
'[1980-12-31 Wed 23:59]'),
datetime.datetime(1980, 12, 31, 23, 59, 0, tzinfo=None))
with self.assertRaises(TimestampParseException):
OrgFormat.orgmode_timestamp_to_datetime('foobar')
with self.assertRaises(TimestampParseException):
OrgFormat.orgmode_timestamp_to_datetime('<1980-12-31 23:59>') # missing day of week
def _handle_recent_tracks(self, tracks):
"""parse recent tracks"""
logging.debug(tracks)
for t in tracks:
timestamp = datetime.datetime.fromtimestamp(int(t.timestamp))
output = self._args.output_format.format(title=t.track.title,
artist=t.track.artist,
album=t.album)
properties = OrgProperties(data_for_hashing=t.timestamp)
properties.add('ARTIST', t.track.artist)
properties.add('ALBUM', t.album)
self._writer.write_org_subitem(timestamp=OrgFormat.date(timestamp, show_time=True),
output=output,
properties=properties)
def write_org_subitem(self,
timestamp,
output,
note="",
properties=OrgProperties(),
tags=None):
"""
Writes an org item line.
i.e:** <timestamp> <output> :<tags>:\n
:PROPERTIES:
<properties>
:ID: -generated id-
:END:
if an argument -a or --append is given,
then a desicion regarding the :ID: is made if the item has to be
written to file
@param timestamp: str/unicode
@param output: st tar/unicode
@param note: str/unicode
@param tags: list of tags
@param properties: OrgProperties object
"""
assert (timestamp.__class__ == str or timestamp.__class__ == unicode)
assert tags.__class__ == list or tags == None
assert properties.__class__ == OrgProperties
assert (output.__class__ == str or output.__class__ == unicode)
assert (note.__class__ == str or note.__class__ == unicode)
# count the entries we have written, if above our limit do not write
if self.__number_entries and \
self.__entries_count == self.__number_entries:
return
else:
self.__entries_count += 1
if tags == None:
tags = []
if self.__autotag_dict != {}:
self.__get_autotags(tags, output)
## fix time-stamps (if user wants to)
if self.__timestamp_delta:
timestamp = OrgFormat.apply_timedelta_to_Orgmode_timestamp(timestamp, int(self.__timestamp_delta))
## a bit of a hack to get inactive time-stamps:
## FIXXME: use OrgFormat method to generate inactive time-stamps in the first place and remove asserts
if self.__inactive_timestamps:
assert(timestamp[0] == '<') ## at least try to find cases where this replace method fails
assert(timestamp[-1] == '>') ## at least try to find cases where this replace method fails
timestamp = '[' + timestamp[1:-1] + ']'
if self.__append:
self.__append_org_subitem(timestamp,
output,
note,
properties,
tags)
else:
self.__write_org_subitem(timestamp,
output,
note,
properties,
tags)
def parse_message(message, add_body=False):
"""
parses whole mail from string
@param message: mail message
@param add_body: if specified, body is added
@return values for OrgWriter.write_org_subitem
"""
msg = message_from_string(message)
# Read only these fields
use_headers = ["To",
"Date",
"From",
"Subject",
"Reply-To",
"Newsgroups",
"Cc",
]
# These fields are added, if found to :PROPERTIES: drawer
not_properties = ["Date",
"Subject",
"From"
]
properties = OrgProperties()
headers = {}
logging.debug("Message items:")
logging.debug(msg.items())
msg_id = None
# fill headers and properties
for key, value in msg.items():
value = value.replace("\r", "").decode('utf-8')
if key in use_headers:
headers[key] = value
if key not in not_properties:
properties.add(key, value.replace("\n", ""))
if key.upper() == "MESSAGE-ID":
msg_id = value
notes = ""
# look for payload
# if more than one payload, use text/plain payload
if add_body:
payload = msg.get_payload()
if payload.__class__ == list:
# default use payload[0]
payload_msg = payload[0].get_payload()
for payload_id in len(payload):
for param in payload[payload_id].get_params():
if param[0] == 'text/plain':
payload_msg = payload[payload_id].get_payload()
break
if payload_msg != payload[0].get_payload():
break
notes = payload_msg
else:
notes = payload
notes = notes.replace("\r", "").decode('utf-8')
output_from = MailParser.get_value_or_empty_str(headers, "From")
if output_from != "":
output_from = OrgFormat.contact_mail_mailto_link(output_from)
subject = MailParser.get_value_or_empty_str(headers, "Subject", True)
dt = MailParser.get_value_or_empty_str(headers, "Date", False)
timestamp = ""
if dt != "":
try:
time_tupel = time.localtime(time.mktime(parsedate(dt)))
timestamp = OrgFormat.datetime(time_tupel)
except TypeError:
logging.error("could not parse dateime from msg %s", dt)
properties.add_data_for_hashing(timestamp + "_" + msg_id)
if "Newsgroups" in headers:
ng_list = []
for ng in headers["Newsgroups"].split(","):
ng_list.append(OrgFormat.newsgroup_link(ng))
output_ng = ", ".join(map(str, ng_list))
output = output_from + u"@" + output_ng + ": " + subject
else:
output = output_from + u": " + subject
return timestamp, output, notes, properties
