def _handle_url(self, params):
timestamp = datetime.datetime.fromtimestamp(
int(params['timestamp'] / 1000000))
if not self._args.omit_drawer:
properties = OrgProperties()
if (params['title'] == ""):
params['title'] = params['url']
properties.add('URL', params['url'])
properties.add('VISIT_COUNT', params['visit_count'])
output = OrgFormat.link(params['url'], params['title'])
try:
output = self._args.output_format.decode('utf-8').format(**params)
except Exception:
pass
if self._args.omit_drawer:
self._writer.write_org_subitem(timestamp=OrgFormat.date(
timestamp, show_time=True),
output=output,
properties=None)
else:
self._writer.write_org_subitem(timestamp=OrgFormat.date(
timestamp, show_time=True),
output=output,
properties=properties)
def test_parse_basic_iso_datetime(self):
os.environ['TZ'] = "Europe/Vienna"
time.tzset()
self.assertEqual(
OrgFormat.date(
OrgFormat.parse_basic_iso_datetime('20111219T205510Z'), True
),
'<2011-12-19 Mon 21:55>'
)
self.assertEqual(
OrgFormat.date(
OrgFormat.parse_basic_iso_datetime('20111219T205510'),
True
),
'<2011-12-19 Mon 20:55>')
self.assertEqual(
OrgFormat.date(OrgFormat.parse_basic_iso_datetime('20111219'), False),
'<2011-12-19 Mon>'
)
self.assertEqual(
OrgFormat.date(OrgFormat.parse_basic_iso_datetime('18991230'), False),
'<1899-12-30 Sat>'
)
with self.assertRaises(TimestampParseException):
OrgFormat.parse_basic_iso_datetime('foobar')
with self.assertRaises(TimestampParseException):
OrgFormat.parse_basic_iso_datetime('20111219x205510Z')
with self.assertRaises(TimestampParseException):
OrgFormat.parse_basic_iso_datetime('20111319')
def __getTimestamp(self, time, onlyDate=False):
"""
converts xml timestamp into org readable timestamp
Do 6 Nov 21:22:17 2014
"""
time = time.strip()
mail_date = datetime.strptime(time,"%c")
if onlyDate is False:
return OrgFormat.date(mail_date, show_time=True)
return OrgFormat.date(mail_date)
def test_date(self):
# NOTE: time.strptime() returns a time.struct_time
self.assertEqual(
OrgFormat.date(
time.strptime('2011-11-02T20:38:42', '%Y-%m-%dT%H:%M:%S')),
'<2011-11-02 Wed>')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38',
'%Y-%m-%dT%H:%M')),
'<2011-11-02 Wed>')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
show_time=True), '<2011-11-02 Wed 20:38>')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'),
show_time=True), '<2011-11-02 Wed 20:38>')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
inactive=True), '[2011-11-02 Wed]')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'),
inactive=True), '[2011-11-02 Wed]')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
inactive=True,
show_time=True), '[2011-11-02 Wed 20:38]')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'),
inactive=True,
show_time=True), '[2011-11-02 Wed 20:38]')
## testing repeater_or_delay:
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'),
inactive=False,
repeater_or_delay='+2w '), '<2011-11-02 Wed +2w>')
self.assertEqual(
OrgFormat.date(time.strptime('2011-11-02T20:38:42',
'%Y-%m-%dT%H:%M:%S'),
inactive=True,
show_time=True,
repeater_or_delay=' ++1m '),
'[2011-11-02 Wed 20:38 ++1m]')
def test_autotag(self):
test_filename = self.TMPFOLDER + "testautotag.org"
autotag_dict = {}
autotag_dict["programming"] = ["programming", "python", "java"]
autotag_dict["TUG"] = ["tugraz", "university"]
output = "Programming for my bachelor thesis at University"
# writing test output
writer = OrgOutputWriter(short_description="short-des",
tag="tag",
file_name=test_filename,
autotag_dict=autotag_dict)
timestamp = OrgFormat.date(time.gmtime(0), show_time=True)
properties = OrgProperties("data_for_hashing")
writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties)
writer.close()
# read and check the file_handler
file_handler = codecs.open(test_filename, "r", "utf-8")
input_handler = file_handler.readlines()
file_handler.close()
self.assertTrue(input_handler[4].startswith(
"** <1970-01-01 Thu 00:00> Programming for my " +
"bachelor thesis at University\t:"))
self.assertTrue(input_handler[4].endswith("programming:TUG:\n")
or input_handler[4].endswith("TUG:programming:\n"))
def __handle_file(self, photo_file, filename):
"""
checks if file is an image, try to get exif data and
write to org file
"""
logging.debug("handling file %s", filename)
# check if file is an image:
if imghdr.what(filename) != None:
datetime = get_exif_datetime(filename)
if datetime == None:
logging.debug("skipping: %s has no EXIF information", filename)
else:
try:
datetime = time.strptime(datetime, "%Y:%m:%d %H:%M:%S")
timestamp = OrgFormat.date(datetime, show_time=True)
output = OrgFormat.link(filename, photo_file)
properties = OrgProperties(photo_file + timestamp)
self._writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties)
except ValueError as e:
logging.warning("skipping: Could not parse " + \
"timestamp for %s : %s", filename, e)
def write_point(self, p):
"""write a point (including geocoding)"""
timestamp = OrgFormat.date(p.time, show_time=True)
geocode = self.reverse_geocode(p.latitude, p.longitude)
output = self._args.output_format.format(**geocode)
tags = []
properties = OrgProperties(data_for_hashing=timestamp)
if p.latitude:
properties.add('LATITUDE', p.latitude)
if p.longitude:
properties.add('LONGITUDE', p.longitude)
if p.source:
tags.append(p.source.lower())
if timestamp:
self._writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties,
tags=tags)
def format_timestamp(self, timestamp):
# show time with the timestamp format, but only
# if it contains at least hours and minutes
show_time = not self._args.timestamp_format or \
any(x in self._args.timestamp_format for x in ['%H', '%M'])
timestamp = OrgFormat.date(timestamp, show_time=show_time)
return timestamp
def __get_datetime(self, mydate):
"""
@return string: Datetime - in Org Format
"""
mydate_tuple = OrgFormat.parse_basic_iso_datetime(mydate)
return OrgFormat.date(mydate_tuple)
def startElement(self, name, attrs):
"""
at every <sms> tag write to orgfile
"""
logging.debug("Handler @startElement name=%s,attrs=%s", name, attrs)
if name == "sms":
#sms_subject = attrs['subject']
sms_date = int(attrs['date']) / 1000 # unix epoch
sms_body = attrs['body']
sms_address = attrs['address']
sms_time = attrs['time']
sms_service_center = attrs['service_center']
sms_type_incoming = int(attrs['type']) == 1
contact_name = attrs['name']
skip = False
if sms_type_incoming == True:
output = "SMS from "
if self._ignore_incoming:
skip = True
else:
output = "SMS to "
if self._ignore_outgoing:
skip = True
if not skip:
name_string = ""
if contact_name:
name_string = '[[contact:' + contact_name + '][' + contact_name + ']]'
else:
name_string = "Unknown"
output += name_string + ": "
#if sms_subject != "null":
# in case of MMS we have a subject
# output += sms_subject
# notes = sms_body
#else:
# output += sms_body
# notes = ""
notes = sms_body
timestamp = OrgFormat.date(time.gmtime(sms_date),
show_time=True)
data_for_hashing = output + timestamp + notes
properties = OrgProperties(data_for_hashing=data_for_hashing)
properties.add("NUMBER", sms_address)
properties.add("NAME", contact_name)
properties.add("SMS_SERVICE_CENTER", sms_service_center)
properties.add("TIME", sms_time)
self._writer.write_org_subitem(output=output,
timestamp=timestamp,
note=notes,
properties=properties)
def _handle_message(self, msg):
"""parse a single message row"""
msg['number'] = '00' + msg['number'].split('@')[0]
msg['name'] = self._numberdict.get(msg['number'], msg['number'])
msg['verb'] = 'to' if msg['type'] else 'from'
msg['type'] = 'OUTGOING' if msg['type'] else 'INCOMING'
msg['handler'] = self._args.handler
if msg['text']:
if self._args.demojize:
msg['text'] = emoji.demojize(msg['text'])
if self._args.skip_emoji:
msg['text'] = re.sub(emoji.get_emoji_regexp(), '', msg['text'])
timestamp = datetime.datetime.fromtimestamp(msg['timestamp'] / 1000)
properties = OrgProperties(data_for_hashing=json.dumps(msg))
properties.add('NUMBER', msg['number'])
properties.add('TYPE', msg['type'])
output = self._args.output_format.format(**msg)
if msg['text'] and not self._is_ignored(msg):
self._writer.write_org_subitem(timestamp=OrgFormat.date(
timestamp, show_time=True),
output=output,
properties=properties)
def test_properties_with_own_created(self):
p = OrgProperties()
p.add("CREATED", OrgFormat.date(time.gmtime(0), show_time=True))
properties = str(p).splitlines()
self.assertEqual(properties[0], " :PROPERTIES:")
self.assertEqual(properties[1], " :CREATED: <1970-01-0" + \
"1 Thu 00:00>")
self.assertEqual(properties[2], " :ID: fede47e9" + \
"f49e1b7f5c6599a6d607e9719ca98625")
self.assertEqual(properties[3], " :END:")
def _handle_row(self, row):
"""
handle a single row
"""
try:
# assume unix timestamp
if not self._args.timestamp_format:
timestamp = datetime.datetime.fromtimestamp(
int(row[self._args.timestamp_field]))
else:
timestamp = time.strptime(row[self._args.timestamp_field],
self._args.timestamp_format)
# show time with the timestamp format, but only
# if it contains at least hours and minutes
if not self._args.timestamp_format or \
any(x in self._args.timestamp_format for x in ['%H', '%M']):
timestamp = OrgFormat.date(timestamp, show_time=True)
else:
timestamp = OrgFormat.date(timestamp)
except ValueError as e:
logging.error("timestamp-format does not match: %s", e)
sys.exit(1)
except IndexError as e:
logging.error("did you specify the right delimiter?", e)
sys.exit(1)
properties = OrgProperties(data_for_hashing=json.dumps(row))
output = self._args.output_format.format(**row)
if self._args.properties:
for prop in self._args.properties.split(','):
properties.add(prop.upper().strip(), row[prop])
self._writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties)
def _main(self):
APP_KEY = self._get_config_option("APP_KEY")
APP_SECRET = self._get_config_option("APP_SECRET")
OAUTH_TOKEN = self._get_config_option("OAUTH_TOKEN")
OAUTH_TOKEN_SECRET = self._get_config_option("OAUTH_TOKEN_SECRET")
screen_name = self._get_config_option("screen_name")
count = self._get_config_option("count")
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
try:
home_timeline = twitter.get_home_timeline(screenname=screen_name,
count=count)
except TwythonError as e:
logging.error(e)
sys.exit(1)
for tweet in home_timeline:
# strptime doesn't support timezone info, so we are using dateutils.
date_object = parser.parse(tweet['created_at'])
timestamp = OrgFormat.date(date_object, show_time=True)
try:
# Data is already Unicode, so don't try to re-encode it.
output = tweet['text']
except:
logging.error(sys.exc_info()[0])
print("Error: ", sys.exc_info()[0])
data_for_hashing = output + timestamp + output
properties = OrgProperties(data_for_hashing=data_for_hashing)
properties.add("name", tweet['user']['name'])
properties.add("twitter_id", tweet['id'])
properties.add("contributors", tweet['contributors'])
properties.add("truncated", tweet['truncated'])
properties.add("in_reply_to_status_id",
tweet['in_reply_to_status_id'])
properties.add("favorite_count", tweet['favorite_count'])
properties.add("source", tweet['source'])
properties.add("retweeted", tweet['retweeted'])
properties.add("coordinates", tweet['coordinates'])
properties.add("entities", tweet['entities'])
self._writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties)
def test_fix_struct_time_wday(self):
self.assertEqual(OrgFormat.fix_struct_time_wday(time.struct_time([2013, 4, 3, 10, 54, 0, 0, 0, 0])),
time.struct_time([2013, 4, 3, 10, 54, 0, 2, 0, 0]))
# same test but a bit more human readable for understanding:
# define a time-stamp with weekday == 0:
timestamp = time.struct_time([2013, 4, 3, 10, 54, 0, 0, 0, 0])
# OrgFormat.date(timestamp) ## '<2013-04-03 Mon>' -> Mon is wrong for April 3rd 2013
self.assertEqual(OrgFormat.date(OrgFormat.fix_struct_time_wday(timestamp), show_time=True, inactive=False),
'<2013-04-03 Wed 10:54>')
def __write_footer(self):
"""
Writes the footer of the file including calling python script and time
Don't call this function - call instead function close(),
close() does call this function
"""
self.writeln("* successfully parsed " +\
str(self.__entries_count) + \
" entries by " + \
sys.argv[0] + " at " + \
OrgFormat.date(time.localtime(), show_time=True, inactive=True) + \
" in ~" + self.__time.strip() + ".")
def __set_author_timestamp(self, line):
"""
extracts the date + time from line:
author Forename Lastname <mail> 1234567890 +0000
@param line
"""
self.__empty = False
date_info = line[-16:] # 1234567890 +0000
seconds_since_epoch = float(date_info[:10])
#timezone_info = date_info[11:]
self.__timestamp = OrgFormat.date(
time.localtime(seconds_since_epoch), show_time=True)
self.__author = line[7:line.find("<")].strip()
def _handle_recent_tracks(self, tracks):
"""parse recent tracks"""
logging.debug(tracks)
for t in tracks:
timestamp = datetime.datetime.fromtimestamp(int(t.timestamp))
output = self._args.output_format.format(title=t.track.title,
artist=t.track.artist,
album=t.album)
properties = OrgProperties(data_for_hashing=t.timestamp)
properties.add('ARTIST', t.track.artist)
properties.add('ALBUM', t.album)
self._writer.write_org_subitem(timestamp=OrgFormat.date(timestamp, show_time=True),
output=output,
properties=properties)
def test_date(self):
# NOTE: time.strptime() returns a time.struct_time
self.assertEqual(OrgFormat.date(time.strptime('2011-11-02T20:38:42', '%Y-%m-%dT%H:%M:%S')),
'<2011-11-02 Wed>')
self.assertEqual(OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M')),
'<2011-11-02 Wed>')
self.assertEqual(OrgFormat.date(time.strptime('2011-11-02T20:38:42', '%Y-%m-%dT%H:%M:%S'), show_time=True),
'<2011-11-02 Wed 20:38>')
self.assertEqual(OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'), show_time=True),
'<2011-11-02 Wed 20:38>')
self.assertEqual(OrgFormat.date(time.strptime('2011-11-02T20:38:42', '%Y-%m-%dT%H:%M:%S'), inactive=True),
'[2011-11-02 Wed]')
self.assertEqual(OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'), inactive=True),
'[2011-11-02 Wed]')
self.assertEqual(OrgFormat.date(time.strptime('2011-11-02T20:38:42', '%Y-%m-%dT%H:%M:%S'), inactive=True, show_time=True),
'[2011-11-02 Wed 20:38]')
self.assertEqual(OrgFormat.date(time.strptime('2011-11-02T20:38', '%Y-%m-%dT%H:%M'), inactive=True, show_time=True),
'[2011-11-02 Wed 20:38]')
def _handle_battery(self, bat):
"""
handle single battery, e.g. BAT0
"""
# calculate watt usage
consumption = float(bat.current_now / 1000000.0 * bat.voltage_now /
1000000.0)
timestamp = OrgFormat.date(datetime.datetime.now(), show_time=True)
output = self._args.output_format.format(battery=bat)
properties = OrgProperties(data_for_hashing=timestamp)
properties.add("CYCLE_COUNT", bat.cycle_count)
properties.add("CAPACITY", '%s%%' % bat.capacity)
properties.add("STATUS", bat.status.lower())
if consumption:
properties.add("CONSUMPTION", '%.1f W' % consumption)
self._writer.write_org_subitem(timestamp=timestamp,
output=output,
properties=properties)
def __write(self):
"""
write attributes to writer (make an org_sub_item)
"""
logging.debug("msg:%s", self.__msg)
self.__msg = self.__msg.splitlines()
subject = ""
notes = ""
# idea: look for the first -nonempty- message
if len(self.__msg) > 0:
start_notes = 0
for i in range(len(self.__msg)):
if self.__msg[i].strip() != "":
subject = self.__msg[i].strip()
start_notes = i + 1
break
if len(self.__msg) > start_notes:
for n in self.__msg[start_notes:]:
if n != "":
notes += n + "\n"
output = "%s (r%d): %s" % (self.__author, self.__rev, subject)
properties = OrgProperties(data_for_hashing=self.__author + subject)
timestamp = OrgFormat.date(OrgFormat.parse_basic_iso_datetime(
self.__date),
show_time=True)
properties.add("REVISION", self.__rev)
if self.__grepauthor == None or \
(self.__author.strip() == self.__grepauthor.strip()):
self._writer.write_org_subitem(output=output,
timestamp=timestamp,
note=notes,
properties=properties)
def parse_message(message, add_body=False):
"""
parses whole mail from string
@param message: mail message
@param add_body: if specified, body is added
@return values for OrgWriter.write_org_subitem
"""
msg = message_from_string(message)
# Read only these fields
use_headers = [
"To",
"Date",
"From",
"Subject",
"Reply-To",
"Newsgroups",
"Cc",
]
# These fields are added, if found to :PROPERTIES: drawer
not_properties = ["Date", "Subject", "From"]
properties = OrgProperties()
headers = {}
logging.debug("Message items:")
logging.debug(list(msg.items()))
msg_id = None
# fill headers and properties
for key, value in list(msg.items()):
value = value.replace("\r", "")
if key in use_headers:
headers[key] = value
if key not in not_properties:
properties.add(
key,
MailParser.get_value_or_empty_str(headers, key, True))
if key.upper() == "MESSAGE-ID":
msg_id = value
notes = ""
# look for payload
# if more than one payload, use text/plain payload
if add_body:
payload = msg.get_payload()
if payload.__class__ == list:
# default use payload[0]
payload_msg = payload[0].get_payload()
for payload_id in len(payload):
for param in payload[payload_id].get_params():
if param[0] == 'text/plain':
payload_msg = payload[payload_id].get_payload()
break
if payload_msg != payload[0].get_payload():
break
notes = payload_msg
else:
notes = payload
notes = notes.replace("\r", "")
output_from = MailParser.get_value_or_empty_str(headers, "From")
if output_from != "":
output_from = OrgFormat.mailto_link(output_from)
subject = MailParser.get_value_or_empty_str(headers, "Subject", True)
dt = MailParser.get_value_or_empty_str(headers, "Date", False)
timestamp = ""
if dt != "":
try:
time_tuple = time.localtime(time.mktime(parsedate(dt)))
timestamp = OrgFormat.date(time_tuple, show_time=True)
except TypeError:
logging.error("could not parse dateime from msg %s", dt)
properties.add_data_for_hashing(timestamp + "_" + msg_id)
if "Newsgroups" in headers:
ng_list = []
for ng in headers["Newsgroups"].split(","):
ng_list.append(OrgFormat.newsgroup_link(ng))
output_ng = ", ".join(map(str, ng_list))
output = output_from + "@" + output_ng + ": " + subject
else:
output = output_from + ": " + subject
return timestamp, output, notes, properties
def test_output_to_file(self):
"""
Simple Test
"""
test_filename = self.TMPFOLDER + "testfile.org"
properties = OrgProperties("data_for_hashing")
# writing test output
writer = OrgOutputWriter("short descript", "test-tag", test_filename)
writer.write("## abc\n")
writer.writeln("## abc")
writer.write_comment("abc\n")
writer.write_commentln("abc")
writer.write_org_item("begin")
timestamp = OrgFormat.date(time.gmtime(0), show_time=True)
writer.write_org_subitem(timestamp=timestamp,
output="sub",
properties=properties)
writer.write_org_subitem(timestamp=timestamp,
output="sub",
tags=["foo", "bar"],
properties=properties)
writer.write_org_subitem(timestamp=False,
output="no timestamp",
tags=["bar", "baz"],
properties=properties)
writer.close()
# read and check the file_handler
with codecs.open(test_filename, "r", "utf-8") as file_handler:
data = file_handler.readlines()
self.assertEqual(data[3],
"* short descript :Memacs:test-tag:\n")
self.assertEqual(data[4], "## abc\n")
self.assertEqual(data[5], "## abc\n")
self.assertEqual(data[6], "## abc\n")
self.assertEqual(data[7], "## abc\n")
self.assertEqual(data[8], "* begin\n")
self.assertEqual(data[9], "** <1970-01-01 Thu 00:00> sub\n")
self.assertEqual(data[10], " :PROPERTIES:\n")
self.assertEqual(
data[11],
" :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n")
self.assertEqual(data[12], " :END:\n")
self.assertEqual(data[13], "\n")
self.assertEqual(data[14],
"** <1970-01-01 Thu 00:00> sub\t:foo:bar:\n")
self.assertEqual(data[15], " :PROPERTIES:\n")
self.assertEqual(
data[16],
" :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n")
self.assertEqual(data[17], " :END:\n")
self.assertEqual(data[18], "\n")
self.assertEqual(data[19], "** no timestamp\t:bar:baz:\n")
self.assertEqual(data[20], " :PROPERTIES:\n")
self.assertEqual(
data[21],
" :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n")
self.assertEqual(data[22], " :END:\n")
def _main(self):
"""
get's automatically called from Memacs class
"""
# do all the stuff
# if you need something from config:
# attention: foo will be unicode
# foo = self._get_config_option("foo")
logging.info("foo started")
# how to handle config files ?
# sample config file:
# ---------8<-----------
# [memacs-example]
# foo = 0
# bar = 1
# --------->8-----------
# to read it out, just do following:
# foo = self._get_config_option("foo")
# bar = self._get_config_option("bar")
# use logging.debug() for debug messages
# use logging.error() for error messages
# use logging.info() instead of print for informing user
#
# on an fatal error:
# use logging.error() and sys.exit(1)
timestamp = OrgFormat.date(time.gmtime(0), show_time=True)
# note: timestamp has to be a struct_time object
# Orgproperties
# Option 1: no properties given, specify argument for hashing data
properties = OrgProperties("hashing data :ALKJ!@# should be unique")
# Option 2: add properties which are all-together unique
# properties.add("Category","fun")
# properties.add("from","*****@*****.**")
# properties.add("body","foo")
self._writer.write_org_subitem(timestamp=timestamp,
output="foo",
properties=properties)
# writes following:
#** <1970-01-01 Thu 00:00> foo
# :PROPERTIES:
# :ID: da39a3ee5e6b4b0d3255bfef95601890afd80709
# :END:
notes = "bar notes\nfoo notes"
p = OrgProperties(data_for_hashing="read comment below")
# if a hash is not unique only with its :PROPERTIES: , then
# set data_for_hasing string additional information i.e. the output
# , which then makes the hash really unique
#
# if you *really*, *really* have already a unique id,
# then you can call following method:
# p.set_id("unique id here")
p.add("DESCRIPTION", "foooo")
p.add("foo-property", "asdf")
tags = ["tag1", "tag2"]
self._writer.write_org_subitem(timestamp=timestamp,
output="bar",
note=notes,
properties=p,
tags=tags)
def startElement(self, name, attrs):
"""
at every <sms> tag write to orgfile
"""
logging.debug("Handler @startElement name=%s,attrs=%s", name, attrs)
htmlparser = html.parser.HTMLParser()
if name == "sms":
sms_subject = attrs.get('subject', '')
sms_date = int(attrs['date']) / 1000 # unix epoch
sms_body = attrs['body']
sms_address = attrs['address'].strip().replace('-', '').replace(
'/', '').replace(' ', '').replace('+', '00')
sms_type_incoming = int(attrs['type']) == 1
contact_name = False
if 'contact_name' in attrs:
## NOTE: older version of backup app did not insert contact_name into XML
contact_name = attrs['contact_name']
else:
if self._numberdict:
if sms_address in list(self._numberdict.keys()):
contact_name = self._numberdict[sms_address]
skip = False
if sms_type_incoming == True:
output = "SMS from "
if self._ignore_incoming:
skip = True
else:
output = "SMS to "
if self._ignore_outgoing:
skip = True
if not skip:
name_string = ""
if contact_name:
name_string = '[[contact:' + contact_name + '][' + contact_name + ']]'
else:
name_string = "Unknown"
output += name_string + ": "
## reverse encoding hack from just before:
sms_body = htmlparser.unescape(
sms_body.replace('EnCoDiNgHaCk42', '&#'))
for emoji in list(self.EMOJIS.keys()):
## FIXXME: this is a horrible dumb brute-force algorithm.
## In case of bad performance, this can be optimized dramtically
sms_body = sms_body.replace(emoji, self.EMOJI_ENCLOSING_CHARACTER + \
self.EMOJIS[emoji] + self.EMOJI_ENCLOSING_CHARACTER).replace('\n', '⏎')
if sms_subject != "null":
# in case of MMS we have a subject
output += sms_subject
notes = sms_body
else:
output += sms_body
notes = ""
timestamp = OrgFormat.date(time.gmtime(sms_date),
show_time=True)
data_for_hashing = output + timestamp + notes
properties = OrgProperties(data_for_hashing=data_for_hashing)
properties.add("NUMBER", sms_address)
properties.add("NAME", contact_name)
self._writer.write_org_subitem(output=output,
timestamp=timestamp,
note=notes,
properties=properties)
def startElement(self, name, attrs):
"""
at every <log> write to orgfile
"""
logging.debug("Handler @startElement name=%s,attrs=%s", name, attrs)
if name == "log":
call_number = attrs['number']
call_duration = int(attrs['dur'])
call_date = int(attrs['date']) / 1000 # unix epoch
call_type = int(attrs['type'])
call_incoming = call_type == 1
call_outgoing = call_type == 2
call_missed = call_type == 3
call_voicemail = call_type == 4
call_rejected = call_type == 5
call_refused = call_type == 6
call_name = attrs['name']
output = "Phonecall "
skip = False
if call_incoming:
output += "from "
if self._ignore_incoming:
skip = True
elif call_outgoing:
output += "to "
if self._ignore_outgoing:
skip = True
elif call_missed:
output += "missed "
if self._ignore_missed:
skip = True
elif call_voicemail:
output += "voicemail "
if self._ignore_voicemail:
skip = True
elif call_rejected:
output += "rejected "
if self._ignore_rejected:
skip = True
elif call_refused:
output += "refused "
if self._ignore_refused:
skip = True
else:
raise Exception("Invalid Phonecall Type: %d", call_type)
call_number_string = ""
if call_number != "-1":
call_number_string = call_number
else:
call_number_string = "Unknown Number"
name_string = ""
if call_name != "(Unknown)":
name_string = '[[contact:' + call_name + '][' + call_name + ']]'
else:
name_string = "Unknown"
output += name_string
if call_duration < self._minimum_duration:
skip = True
timestamp = OrgFormat.date(time.gmtime(call_date), show_time=True)
end_datetimestamp = datetime.datetime.utcfromtimestamp(
call_date + call_duration)
logging.debug(
"timestamp[%s] duration[%s] end[%s]" %
(str(timestamp), str(call_duration), str(end_datetimestamp)))
end_timestamp_string = OrgFormat.date(end_datetimestamp,
show_time=True)
logging.debug("end_time [%s]" % end_timestamp_string)
data_for_hashing = output + timestamp
properties = OrgProperties(data_for_hashing=data_for_hashing)
properties.add("NUMBER", call_number_string)
properties.add("DURATION", call_duration)
properties.add("NAME", call_name)
if not skip:
self._writer.write_org_subitem(output=output,
timestamp=timestamp + '-' +
end_timestamp_string,
properties=properties)
def __handle_file(self, file, rootdir):
"""
handles a file (except ending with a tilde)
"""
# don't handle emacs tmp files (file~)
if file[-1:] == '~':
return
link = os.path.join(rootdir, file)
logging.debug('__handle_file: ' + '#' * 50)
logging.debug('__handle_file: ' + link)
orgdate = False # set to default value
if self._args.force_filedate_extraction:
# in this case, skip any clever
# extraction mechanism to extract
# date/time from file name and use
# the mtime instead:
logging.debug(
'__handle_file: force_filedate_extraction: using datetime from mtime of file'
)
file_datetime = time.localtime(os.path.getmtime(link))
orgdate = OrgFormat.date(file_datetime, show_time=True)
self.__write_file(file, link, orgdate)
return
# very basic checks for correctness (e.g., month=20, hour=70)
# are part of these RegEx (and do not have to be checked
# below)
filename_timestamp_match = DATETIME_REGEX.match(file)
logging.debug('__handle_file: filename_timestamp_match? ' +
str(filename_timestamp_match is True))
if filename_timestamp_match:
# day1/2 are like 'YYYY-MM-DD' time1/2 like 'HH:MM':
has_1ymd, has_1ymdhm, has_2ymd, has_2ymdhm, \
day1, time1, day2, time2 = self.__extract_days_and_times(filename_timestamp_match)
# Note: following things are available for formatting:
# self._args.inactive_timestamps -> Bool
# OrgFormat.strdate('YYYY-MM-DD', inactive=False) -> <YYYY-MM-DD Sun>
# OrgFormat.strdate('YYYY-MM-DD HH:MM', inactive=False, show_time=True) -> <YYYY-MM-DD Sun HH:MM>
assert (has_1ymd)
try:
if has_1ymdhm:
if self.__check_datestamp_correctness(day1):
if self.__check_timestamp_correctness(time1):
orgdate = OrgFormat.strdate(
day1 + ' ' + time1,
inactive=self._args.inactive_timestamps,
show_time=True)
else:
logging.warning(
'File "' + file +
'" has an invalid timestamp (' + str(time1) +
'). Skipping this faulty time-stamp.')
orgdate = OrgFormat.strdate(
day1, inactive=self._args.inactive_timestamps)
else:
logging.warning('File "' + file +
'" has an invalid datestamp (' +
str(day1) + ').')
# omit optional second day if first has an issue:
has_2ymd = False
has_2ymdhm = False
orgdate = False
elif has_1ymd: # missing time-stamp for day1
if self.__check_datestamp_correctness(day1):
if not self._args.skip_filetime_extraction:
# we've got only a day but we're able to determine
# time from file mtime, if same as ISO day in file
# name:
logging.debug(
'__handle_file: try to get file time from mtime if days match between mtime and filename ISO ...'
)
file_datetime = time.localtime(
os.path.getmtime(link))
if self.__check_if_days_in_timestamps_are_same(
file_datetime, day1):
orgdate = OrgFormat.date(
file_datetime,
inactive=self._args.inactive_timestamps,
show_time=True)
else:
logging.debug(
'__handle_file: day of mtime and filename ISO differs, using filename ISO day'
)
orgdate = OrgFormat.strdate(
day1,
inactive=self._args.inactive_timestamps)
else:
# we've got only a day and determining mtime
# is not planned, so use the day as date-stamp
orgdate = OrgFormat.strdate(
day1, inactive=self._args.inactive_timestamps)
else:
logging.warning('File "' + file +
'" has an invalid datestamp (' +
str(day1) + ').')
orgdate = False
else:
logging.warning('File "' + file +
'" has an invalid datestamp (' +
str(day1) +
'). Skipping this faulty date.')
# omit optional second day if first has an issue:
has_2ymd = False
has_2ymdhm = False
# there is a time range:
if has_2ymdhm:
assert (day2)
if self.__check_datestamp_correctness(day2):
if self.__check_timestamp_correctness(time2):
orgdate += '--' + OrgFormat.strdate(
day2 + ' ' + time2,
inactive=self._args.inactive_timestamps,
show_time=True)
else:
logging.warning(
'File "' + file +
'" has an invalid timestamp (' + str(time2) +
'). Skipping this faulty time-stamp.')
orgdate += '--' + OrgFormat.strdate(
day2, inactive=self._args.inactive_timestamps)
else:
logging.warning('File "' + file +
'" has an invalid datestamp (' +
str(day2) +
'). Skipping this faulty date.')
elif has_2ymd:
assert (day2)
if self.__check_datestamp_correctness(day2):
orgdate += '--' + OrgFormat.strdate(
day2, inactive=self._args.inactive_timestamps)
else:
logging.warning('File "' + file +
'" has an invalid datestamp (' +
str(day2) +
'). Skipping this faulty date.')
except TimestampParseException:
logging.error(
'File "' + str(file) +
'" has in invalid date- or timestamp. OrgFormat of one of day1: "'
+ str(day1) + '" time1: "' + str(time1) + '" day2: "' +
str(day2) + '" time2: "' + str(time2) + '" ' +
'failed with TimestampParseException. Skipping this faulty date.'
)
orgdate = False
else:
logging.debug('__handle_file: no date- nor timestamp')
orgdate = False
if not orgdate and self._args.skip_notimestamp_files:
logging.debug(
'__handle_file: file had no or wrong time-stamp and you decided to skip them.'
)
return
self.__write_file(file, link, orgdate)
logging.debug('__handle_file: using orgdate: ' + str(orgdate))
return
def __get_item_data(self, item):
"""
gets information out of <item>..</item>
@return: output, note, properties, tags
variables for orgwriter.append_org_subitem
"""
try:
# logging.debug(item)
properties = OrgProperties()
guid = item['id']
if not guid:
logging.error("got no id")
unformatted_link = item['link']
short_link = OrgFormat.link(unformatted_link, "link")
# if we found a url in title
# then append the url in front of subject
if re.search("http[s]?://", item['title']) is not None:
output = short_link + ": " + item['title']
else:
output = OrgFormat.link(unformatted_link, item['title'])
note = item['description']
# converting updated_parsed UTC --> LOCALTIME
# Karl 2018-09-22 this might be changed due to:
# DeprecationWarning: To avoid breaking existing software
# while fixing issue 310, a temporary mapping has been
# created from `updated_parsed` to `published_parsed` if
# `updated_parsed` doesn't exist. This fallback will be
# removed in a future version of feedparser.
timestamp = OrgFormat.date(time.localtime(
calendar.timegm(item['updated_parsed'])),
show_time=True)
properties.add("guid", guid)
except KeyError:
logging.error("input is not a RSS 2.0")
sys.exit(1)
tags = []
# Karl 2018-09-22 this might be changed due to:
# DeprecationWarning: To avoid breaking existing software
# while fixing issue 310, a temporary mapping has been created
# from `updated_parsed` to `published_parsed` if
# `updated_parsed` doesn't exist. This fallback will be
# removed in a future version of feedparser.
dont_parse = [
'title', 'description', 'updated', 'summary', 'updated_parsed',
'link', 'links'
]
for i in item:
logging.debug(i)
if i not in dont_parse:
if (type(i) == str or type(i) == str) and \
type(item[i]) == str and item[i] != "":
if i == "id":
i = "guid"
properties.add(i, item[i])
else:
if i == "tags":
for tag in item[i]:
logging.debug("found tag: %s", tag['term'])
tags.append(tag['term'])
return output, note, properties, tags, timestamp
