def test_datetime(self): ## fixed time with seconds: self.assertEqual( OrgFormat.datetime( time.struct_time([1980, 12, 31, 23, 59, 58, 0, 0, 0])), '<1980-12-31 Wed 23:59>' ) ## seconds are not (yet) defined in Org-mode ## fixed time without seconds: self.assertEqual( OrgFormat.datetime( time.struct_time([1980, 12, 31, 23, 59, 0, 0, 0, 0])), '<1980-12-31 Wed 23:59>') YYYYMMDDwday = time.strftime('%Y-%m-%d %a', time.localtime()) hhmmss = time.strftime( '%H:%M', time.localtime()) ## seconds are not (yet) defined in Org-mode ## show_time parameter not named: self.assertEqual(OrgFormat.datetime(time.localtime()), '<' + YYYYMMDDwday + ' ' + hhmmss + '>') ## show_time parameter named: self.assertEqual(OrgFormat.datetime(time.localtime()), '<' + YYYYMMDDwday + ' ' + hhmmss + '>')
def _handle_url(self, params): epoch = datetime.datetime(1970, 1, 1) - datetime.datetime(1601, 1, 1) url_time = params['timestamp'] / 1000000 - epoch.total_seconds() if (url_time > 0): timestamp = datetime.datetime.fromtimestamp(int(url_time)) else: timestamp = datetime.datetime(1970, 1, 1) if not self._args.omit_drawer: properties = OrgProperties() if (params['title'] == ""): params['title'] = params['url'] properties.add('URL', params['url']) properties.add('VISIT_COUNT', params['visit_count']) output = "" try: output = self._args.output_format.decode('utf-8').format(**params) except Exception: pass if self._args.omit_drawer: self._writer.write_org_subitem( timestamp=OrgFormat.datetime(timestamp), output=output, properties=None) else: self._writer.write_org_subitem( timestamp=OrgFormat.datetime(timestamp), output=output, properties=properties)
def _handle_message(self, msg): """parse a single message row""" msg['number'] = '00' + msg['number'].split('@')[0] msg['name'] = self._numberdict.get(msg['number'], msg['number']) msg['verb'] = 'to' if msg['type'] else 'from' msg['type'] = 'OUTGOING' if msg['type'] else 'INCOMING' msg['handler'] = self._args.handler if msg['text']: if self._args.demojize: msg['text'] = emoji.demojize(msg['text']) if self._args.skip_emoji: msg['text'] = re.sub(emoji.get_emoji_regexp(), '', msg['text']) timestamp = datetime.datetime.fromtimestamp(msg['timestamp'] / 1000) properties = OrgProperties(data_for_hashing=json.dumps(msg)) properties.add('NUMBER', msg['number']) properties.add('TYPE', msg['type']) output = self._args.output_format.format(**msg) if msg['text'] and not self._is_ignored(msg): self._writer.write_org_subitem( timestamp=OrgFormat.datetime(timestamp), output=output, properties=properties)
def test_autotag(self): test_filename = self.TMPFOLDER + "testautotag.org" autotag_dict = {} autotag_dict["TUG"] = ["tugraz", "university"] autotag_dict["programming"] = ["programming", "python", "java"] output = "Programming for my bachelor thesis at University" # writing test output writer = OrgOutputWriter(short_description="short-des", tag="tag", file_name=test_filename, autotag_dict=autotag_dict) timestamp = OrgFormat.datetime(time.gmtime(0)) properties = OrgProperties("data_for_hashing") writer.write_org_subitem(timestamp=timestamp, output=output, properties=properties) writer.close() # read and check the file_handler file_handler = codecs.open(test_filename, "r", "utf-8") input_handler = file_handler.readlines() file_handler.close() self.assertEqual(input_handler[4], u"** <1970-01-01 Thu 00:00> Programming for my " + \ "bachelor thesis at University\t:programming:TUG:\n")
def test_autotag(self): test_filename = self.TMPFOLDER + "testautotag.org" autotag_dict = {} autotag_dict["TUG"] = ["tugraz", "university"] autotag_dict["programming"] = ["programming", "python", "java"] output = "Programming for my bachelor thesis at University" # writing test output writer = OrgOutputWriter(short_description="short-des", tag="tag", file_name=test_filename, autotag_dict=autotag_dict) timestamp = OrgFormat.datetime(time.gmtime(0)) properties = OrgProperties("data_for_hashing") writer.write_org_subitem(timestamp=timestamp, output=output, properties=properties) writer.close() # read and check the file_handler file_handler = codecs.open(test_filename, "r", "utf-8") input_handler = file_handler.readlines() file_handler.close() self.assertEqual(input_handler[4], u"** <1970-01-01 Thu 00:00> Programming for my " + \ "bachelor thesis at University\t:programming:TUG:\n") #cleaning up os.remove(self.TMPFOLDER + "testautotag.org")
def write_point(self, p): """write a point (including geocoding)""" timestamp = OrgFormat.datetime(p.time) geocode = self.reverse_geocode(p.latitude, p.longitude) output = self._args.output_format.format(**geocode) tags = [] properties = OrgProperties(data_for_hashing=timestamp) if p.latitude: properties.add('LATITUDE', p.latitude) if p.longitude: properties.add('LONGITUDE', p.longitude) if p.source: tags.append(p.source.lower()) if timestamp: self._writer.write_org_subitem(timestamp=timestamp, output=output, properties=properties, tags=tags)
def __handle_file(self, photo_file, filename): """ checks if file is an image, try to get exif data and write to org file """ logging.debug("handling file %s", filename) # check if file is an image: if imghdr.what(filename) != None: datetime = get_exif_datetime(filename) if datetime == None: logging.debug("skipping: %s has no EXIF information", filename) else: try: datetime = time.strptime(datetime, "%Y:%m:%d %H:%M:%S") timestamp = OrgFormat.datetime(datetime) output = OrgFormat.link(filename, photo_file) properties = OrgProperties(photo_file + timestamp) self._writer.write_org_subitem(timestamp=timestamp, output=output, properties=properties) except ValueError as e: logging.warning("skipping: Could not parse " + \ "timestamp for %s : %s", filename, e)
def startElement(self, name, attrs): """ at every <sms> tag write to orgfile """ logging.debug("Handler @startElement name=%s,attrs=%s", name, attrs) if name == "sms": #sms_subject = attrs['subject'] sms_date = int(attrs['date']) / 1000 # unix epoch sms_body = attrs['body'] sms_address = attrs['address'] sms_time = attrs['time'] sms_service_center = attrs['service_center'] sms_type_incoming = int(attrs['type']) == 1 contact_name = attrs['name'] skip = False if sms_type_incoming == True: output = "SMS from " if self._ignore_incoming: skip = True else: output = "SMS to " if self._ignore_outgoing: skip = True if not skip: name_string = "" if contact_name: name_string = '[[contact:' + contact_name + '][' + contact_name + ']]' else: name_string = "Unknown" output += name_string + ": " #if sms_subject != "null": # in case of MMS we have a subject # output += sms_subject # notes = sms_body #else: # output += sms_body # notes = "" notes = sms_body timestamp = OrgFormat.datetime(time.gmtime(sms_date)) data_for_hashing = output + timestamp + notes properties = OrgProperties(data_for_hashing=data_for_hashing) properties.add("NUMBER", sms_address) properties.add("NAME", contact_name) properties.add("SMS_SERVICE_CENTER", sms_service_center) properties.add("TIME", sms_time) self._writer.write_org_subitem(output=output, timestamp=timestamp, note=notes, properties=properties)
def test_properties_with_own_created(self): p = OrgProperties() p.add(u"CREATED", OrgFormat.datetime(time.gmtime(0))) properties = unicode(p).splitlines() self.assertEqual(properties[0], u" :PROPERTIES:") self.assertEqual(properties[1], u" :CREATED: <1970-01-0" + \ "1 Thu 00:00>") self.assertEqual(properties[2], u" :ID: fede47e9" + \ "f49e1b7f5c6599a6d607e9719ca98625") self.assertEqual(properties[3], u" :END:")
def __getTimestamp(self, time, onlyDate=False): """ converts xml timestamp into org readable timestamp Do 6 Nov 21:22:17 2014 """ time = time.strip().encode('utf-8') mail_date = datetime.strptime(time, "%c") if onlyDate is False: return OrgFormat.datetime(mail_date) return OrgFormat.date(mail_date)
def _main(self): APP_KEY = self._get_config_option("APP_KEY") APP_SECRET = self._get_config_option("APP_SECRET") OAUTH_TOKEN = self._get_config_option("OAUTH_TOKEN") OAUTH_TOKEN_SECRET = self._get_config_option("OAUTH_TOKEN_SECRET") screen_name = self._get_config_option("screen_name") count = self._get_config_option("count") twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) try: home_timeline = twitter.get_home_timeline(screenname=screen_name, count=count) except TwythonError as e: logging.error(e) sys.exit(1) for tweet in home_timeline: # strptime doesn't support timezone info, so we are using dateutils. date_object = parser.parse(tweet['created_at']) timestamp = OrgFormat.datetime(date_object) try: # Data is already Unicode, so don't try to re-encode it. output = tweet['text'] except: logging.error(sys.exc_info()[0]) print("Error: ", sys.exc_info()[0]) data_for_hashing = output + timestamp + output properties = OrgProperties(data_for_hashing=data_for_hashing) properties.add("name", tweet['user']['name']) properties.add("twitter_id", tweet['id']) properties.add("contributors", tweet['contributors']) properties.add("truncated", tweet['truncated']) properties.add("in_reply_to_status_id", tweet['in_reply_to_status_id']) properties.add("favorite_count", tweet['favorite_count']) properties.add("source", tweet['source']) properties.add("retweeted", tweet['retweeted']) properties.add("coordinates", tweet['coordinates']) properties.add("entities", tweet['entities']) self._writer.write_org_subitem(timestamp=timestamp, output=output, properties=properties)
def __set_author_timestamp(self, line): """ extracts the date + time from line: author Forename Lastname <mail> 1234567890 +0000 @param line """ self.__empty = False date_info = line[-16:] # 1234567890 +0000 seconds_since_epoch = float(date_info[:10]) #timezone_info = date_info[11:] self.__timestamp = OrgFormat.datetime( time.localtime(seconds_since_epoch)) self.__author = line[7:line.find("<")].strip()
def test_ouput_to_file(self): """ Simple Test """ test_filename = self.TMPFOLDER + "testfile.org" properties = OrgProperties("data_for_hashing") # writing test output writer = OrgOutputWriter("short descript", "test-tag", test_filename) writer.write("## abc\n") writer.writeln("## abc") writer.write_comment("abc\n") writer.write_commentln("abc") writer.write_org_item("begin") timestamp = OrgFormat.datetime(time.gmtime(0)) writer.write_org_subitem(timestamp=timestamp, output="sub", properties=properties) writer.write_org_subitem(timestamp=timestamp, output="sub", tags=["foo", "bar"], properties=properties) writer.close() # read and check the file_handler with codecs.open(test_filename, "r", "utf-8") as file_handler: data = file_handler.readlines() self.assertEqual(data[3], "* short descript :Memacs:test-tag:\n") self.assertEqual(data[4], "## abc\n") self.assertEqual(data[5], "## abc\n") self.assertEqual(data[6], "## abc\n") self.assertEqual(data[7], "## abc\n") self.assertEqual(data[8], "* begin\n") self.assertEqual(data[9], "** <1970-01-01 Thu 00:00> sub\n") self.assertEqual(data[10], " :PROPERTIES:\n") self.assertEqual( data[11], " :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n") self.assertEqual(data[12], " :END:\n") self.assertEqual(data[13], "\n") self.assertEqual(data[14], "** <1970-01-01 Thu 00:00> sub\t:foo:bar:\n") self.assertEqual(data[15], " :PROPERTIES:\n") self.assertEqual( data[16], " :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n") self.assertEqual(data[17], " :END:\n")
def test_datetime(self): ## fixed time with seconds: self.assertEqual( OrgFormat.datetime(time.struct_time([1980,12,31,23,59,58,0,0,0])), u'<1980-12-31 Mon 23:59:58>' ) ## however, it was a Wednesday ## fixed time without seconds: self.assertEqual( OrgFormat.datetime(time.struct_time([1980,12,31,23,59,0,0,0,0])), u'<1980-12-31 Mon 23:59>' ) ## however, it was a Wednesday YYYYMMDDwday = time.strftime('%Y-%m-%d %a', time.localtime()) hhmmss = time.strftime('%H:%M:%S', time.localtime()) ## show_time parameter not named: self.assertEqual( OrgFormat.datetime(time.localtime()), u'<' + YYYYMMDDwday + u' ' + hhmmss + u'>' ) ## show_time parameter named: self.assertEqual( OrgFormat.datetime(time.localtime()), u'<' + YYYYMMDDwday + u' ' + hhmmss + u'>' )
def test_datetime(self): ## fixed time with seconds: self.assertEqual( OrgFormat.datetime(time.struct_time([1980,12,31,23,59,58,0,0,0])), u'<1980-12-31 Wed 23:59>' ) ## seconds are not (yet) defined in Org-mode ## fixed time without seconds: self.assertEqual( OrgFormat.datetime(time.struct_time([1980,12,31,23,59,0,0,0,0])), u'<1980-12-31 Wed 23:59>' ) YYYYMMDDwday = time.strftime('%Y-%m-%d %a', time.localtime()) hhmmss = time.strftime('%H:%M', time.localtime()) ## seconds are not (yet) defined in Org-mode ## show_time parameter not named: self.assertEqual( OrgFormat.datetime(time.localtime()), u'<' + YYYYMMDDwday + u' ' + hhmmss + u'>' ) ## show_time parameter named: self.assertEqual( OrgFormat.datetime(time.localtime()), u'<' + YYYYMMDDwday + u' ' + hhmmss + u'>' )
def _handle_recent_tracks(self, tracks): """parse recent tracks""" logging.debug(tracks) for t in tracks: timestamp = datetime.datetime.fromtimestamp(int(t.timestamp)) output = self._args.output_format.format(title=t.track.title, artist=t.track.artist, album=t.album) properties = OrgProperties(data_for_hashing=t.timestamp) properties.add('ARTIST', t.track.artist) properties.add('ALBUM', t.album) self._writer.write_org_subitem( timestamp=OrgFormat.datetime(timestamp), output=output, properties=properties)
def _handle_row(self, row): """ handle a single row """ try: # assume unix timestamp if not self._args.timestamp_format: timestamp = datetime.datetime.fromtimestamp( int(row[self._args.timestamp_field])) else: timestamp = time.strptime(row[self._args.timestamp_field], self._args.timestamp_format) # show time with the timestamp format, but only # if it contains at least hours and minutes if not self._args.timestamp_format or \ any(x in self._args.timestamp_format for x in ['%H', '%M']): timestamp = OrgFormat.datetime(timestamp) else: timestamp = OrgFormat.date(timestamp) except ValueError as e: logging.error("timestamp-format does not match: %s", e) sys.exit(1) except IndexError as e: logging.error("did you specify the right delimiter?", e) sys.exit(1) properties = OrgProperties(data_for_hashing=json.dumps(row)) output = self._args.output_format.format(**row) if self._args.properties: for prop in self._args.properties.split(','): properties.add(prop.upper().strip(), row[prop]) self._writer.write_org_subitem(timestamp=timestamp, output=output, properties=properties)
def _handle_battery(self, bat): """ handle single battery, e.g. BAT0 """ # calculate watt usage consumption = float(bat.current_now / 1000000.0 * bat.voltage_now / 1000000.0) timestamp = OrgFormat.datetime(datetime.datetime.now()) output = self._args.output_format.format(battery=bat) properties = OrgProperties(data_for_hashing=timestamp) properties.add("CYCLE_COUNT", bat.cycle_count) properties.add("CAPACITY", '%s%%' % bat.capacity) properties.add("STATUS", bat.status.lower()) if consumption: properties.add("CONSUMPTION", '%.1f W' % consumption) self._writer.write_org_subitem(timestamp=timestamp, output=output, properties=properties)
def __write(self): """ write attributes to writer (make an org_sub_item) """ logging.debug("msg:%s", self.__msg) self.__msg = self.__msg.splitlines() subject = "" notes = "" # idea: look for the first -nonempty- message if len(self.__msg) > 0: start_notes = 0 for i in range(len(self.__msg)): if self.__msg[i].strip() != "": subject = self.__msg[i].strip() start_notes = i + 1 break if len(self.__msg) > start_notes: for n in self.__msg[start_notes:]: if n != "": notes += n + "\n" output = "%s (r%d): %s" % (self.__author, self.__rev, subject) properties = OrgProperties(data_for_hashing=self.__author + subject) timestamp = OrgFormat.datetime( OrgFormat.datetupelutctimestamp(self.__date)) properties.add("REVISION", self.__rev) if self.__grepauthor == None or \ (self.__author.strip() == self.__grepauthor.strip()): self._writer.write_org_subitem(output=output, timestamp=timestamp, note=notes, properties=properties)
def _main(self): """ get's automatically called from Memacs class """ # do all the stuff # if you need something from config: # attention: foo will be unicode # foo = self._get_config_option("foo") logging.info("foo started") # how to handle config files ? # sample config file: # ---------8<----------- # [memacs-example] # foo = 0 # bar = 1 # --------->8----------- # to read it out, just do following: # foo = self._get_config_option("foo") # bar = self._get_config_option("bar") # use logging.debug() for debug messages # use logging.error() for error messages # use logging.info() instead of print for informing user # # on an fatal error: # use logging.error() and sys.exit(1) timestamp = OrgFormat.datetime(time.gmtime(0)) # note: timestamp has to be a struct_time object # Orgproperties # Option 1: no properties given, specify argument for hashing data properties = OrgProperties("hashing data :ALKJ!@# should be unique") # Option 2: add properties which are all-together unique # properties.add("Category","fun") # properties.add("from","*****@*****.**") # properties.add("body","foo") self._writer.write_org_subitem(timestamp=timestamp, output="foo", properties=properties) # writes following: #** <1970-01-01 Thu 00:00> foo # :PROPERTIES: # :ID: da39a3ee5e6b4b0d3255bfef95601890afd80709 # :END: notes = "bar notes\nfoo notes" p = OrgProperties(data_for_hashing="read comment below") # if a hash is not unique only with its :PROPERTIES: , then # set data_for_hasing string additional information i.e. the output # , which then makes the hash really unique # # if you *really*, *really* have already a unique id, # then you can call following method: # p.set_id("unique id here") p.add("DESCRIPTION", "foooo") p.add("foo-property", "asdf") tags = ["tag1", "tag2"] self._writer.write_org_subitem(timestamp=timestamp, output="bar", note=notes, properties=p, tags=tags)
def test_ouput_to_file(self): """ Simple Test """ test_filename = self.TMPFOLDER + "testfile.org" properties = OrgProperties("data_for_hashing") # writing test output writer = OrgOutputWriter("short descript", "test-tag", test_filename) writer.write("## abc\n") writer.writeln("## abc") writer.write_comment("abc\n") writer.write_commentln("abc") writer.write_org_item("begin") timestamp = OrgFormat.datetime(time.gmtime(0)) writer.write_org_subitem(timestamp=timestamp, output="sub", properties=properties) writer.write_org_subitem(timestamp=timestamp, output="sub", tags=["foo", "bar"], properties=properties) writer.close() # read and check the file_handler file_handler = codecs.open(test_filename, "r", "utf-8") data = file_handler.readlines() #for d in range(len(data)): # print "self.assertEqual(\n\tdata[%d],\n\t\"%s\")" % \ # (d, data[d]) # self.assertEqual( # data[1], # "## this file is generated by "... # ") # self.assertEqual( # data[2], # "## To add this file to your org-agenda " ... # ") self.assertEqual(data[3], "* short descript :Memacs:test-tag:\n") self.assertEqual(data[4], "## abc\n") self.assertEqual(data[5], "## abc\n") self.assertEqual(data[6], "## abc\n") self.assertEqual(data[7], "## abc\n") self.assertEqual(data[8], "* begin\n") self.assertEqual(data[9], "** <1970-01-01 Thu 00:00> sub\n") self.assertEqual(data[10], " :PROPERTIES:\n") self.assertEqual( data[11], " :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n") self.assertEqual(data[12], " :END:\n") self.assertEqual(data[13], "\n") self.assertEqual(data[14], "** <1970-01-01 Thu 00:00> sub\t:foo:bar:\n") self.assertEqual(data[15], " :PROPERTIES:\n") self.assertEqual( data[16], " :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n") self.assertEqual(data[17], " :END:\n") #cleaning up file_handler.close() os.remove(self.TMPFOLDER + "testfile.org")
def test_ouput_to_file(self): """ Simple Test """ test_filename = self.TMPFOLDER + "testfile.org" properties = OrgProperties("data_for_hashing") # writing test output writer = OrgOutputWriter("short descript", "test-tag", test_filename) writer.write("## abc\n") writer.writeln("## abc") writer.write_comment("abc\n") writer.write_commentln("abc") writer.write_org_item("begin") timestamp = OrgFormat.datetime(time.gmtime(0)) writer.write_org_subitem(timestamp=timestamp, output="sub", properties=properties) writer.write_org_subitem(timestamp=timestamp, output="sub", tags=["foo", "bar"], properties=properties) writer.close() # read and check the file_handler file_handler = codecs.open(test_filename, "r", "utf-8") data = file_handler.readlines() #for d in range(len(data)): # print "self.assertEqual(\n\tdata[%d],\n\t\"%s\")" % \ # (d, data[d]) # self.assertEqual( # data[1], # "## this file is generated by "... # ") # self.assertEqual( # data[2], # "## To add this file to your org-agenda " ... # ") self.assertEqual( data[3], "* short descript :Memacs:test-tag:\n") self.assertEqual( data[4], "## abc\n") self.assertEqual( data[5], "## abc\n") self.assertEqual( data[6], "## abc\n") self.assertEqual( data[7], "## abc\n") self.assertEqual( data[8], "* begin\n") self.assertEqual( data[9], "** <1970-01-01 Thu 00:00> sub\n") self.assertEqual( data[10], " :PROPERTIES:\n") self.assertEqual( data[11], " :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n") self.assertEqual( data[12], " :END:\n") self.assertEqual( data[13], "\n") self.assertEqual( data[14], "** <1970-01-01 Thu 00:00> sub\t:foo:bar:\n") self.assertEqual( data[15], " :PROPERTIES:\n") self.assertEqual( data[16], " :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n") self.assertEqual( data[17], " :END:\n") #cleaning up file_handler.close() os.remove(self.TMPFOLDER + "testfile.org")
def test_ouput_to_file(self): """ Simple Test """ test_filename = self.TMPFOLDER + "testfile.org" properties = OrgProperties("data_for_hashing") # writing test output writer = OrgOutputWriter("short descript", "test-tag", test_filename) writer.write("## abc\n") writer.writeln("## abc") writer.write_comment("abc\n") writer.write_commentln("abc") writer.write_org_item("begin") timestamp = OrgFormat.datetime(time.gmtime(0)) writer.write_org_subitem(timestamp=timestamp, output="sub", properties=properties) writer.write_org_subitem(timestamp=timestamp, output="sub", tags=["foo", "bar"], properties=properties) writer.close() # read and check the file_handler with codecs.open(test_filename, "r", "utf-8") as file_handler: data = file_handler.readlines() self.assertEqual( data[3], "* short descript :Memacs:test-tag:\n") self.assertEqual( data[4], "## abc\n") self.assertEqual( data[5], "## abc\n") self.assertEqual( data[6], "## abc\n") self.assertEqual( data[7], "## abc\n") self.assertEqual( data[8], "* begin\n") self.assertEqual( data[9], "** <1970-01-01 Thu 00:00> sub\n") self.assertEqual( data[10], " :PROPERTIES:\n") self.assertEqual( data[11], " :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n") self.assertEqual( data[12], " :END:\n") self.assertEqual( data[13], "\n") self.assertEqual( data[14], "** <1970-01-01 Thu 00:00> sub\t:foo:bar:\n") self.assertEqual( data[15], " :PROPERTIES:\n") self.assertEqual( data[16], " :ID: 9cc53a63e13e18437401513316185f6f3b7ed703\n") self.assertEqual( data[17], " :END:\n")
def __handle_file(self, file, rootdir): """ handles a file (except ending with a tilde) """ # don't handle emacs tmp files (file~) if file[-1:] == '~': return link = os.path.join(rootdir, file) logging.debug('__handle_file: ' + '#' * 50) logging.debug('__handle_file: ' + link) orgdate = False # set to default value if self._args.force_filedate_extraction: # in this case, skip any clever # extraction mechanism to extract # date/time from file name and use # the mtime instead: logging.debug('__handle_file: force_filedate_extraction: using datetime from mtime of file') file_datetime = time.localtime(os.path.getmtime(link)) orgdate = OrgFormat.datetime(file_datetime) self.__write_file(file, link, orgdate) return # very basic checks for correctness (e.g., month=20, hour=70) # are part of these RegEx (and do not have to be checked # below) filename_timestamp_match = DATETIME_REGEX.match(file) logging.debug('__handle_file: filename_timestamp_match? ' + str(filename_timestamp_match is True)) if filename_timestamp_match: # day1/2 are like 'YYYY-MM-DD' time1/2 like 'HH:MM': has_1ymd, has_1ymdhm, has_2ymd, has_2ymdhm, \ day1, time1, day2, time2 = self.__extract_days_and_times(filename_timestamp_match) # Note: following things are available for formatting: # self._args.inactive_timestamps -> Bool # OrgFormat.strdate('YYYY-MM-DD', inactive=False) -> <YYYY-MM-DD Sun> # OrgFormat.strdatetime('YYYY-MM-DD HH:MM', inactive=False) -> <YYYY-MM-DD Sun HH:MM> assert(has_1ymd) try: if has_1ymdhm: if self.__check_datestamp_correctness(day1): if self.__check_timestamp_correctness(time1): orgdate = OrgFormat.strdatetime(day1 + ' ' + time1, inactive=self._args.inactive_timestamps) else: logging.warn('File "' + file + '" has an invalid timestamp (' + str(time1) + '). Skipping this faulty time-stamp.') orgdate = OrgFormat.strdate(day1, inactive=self._args.inactive_timestamps) else: logging.warn('File "' + file + '" has an invalid datestamp (' + str(day1) + '). Skipping this faulty date.') # omit optional second day if first has an issue: has_2ymd = False has_2ymdhm = False elif has_1ymd: # missing time-stamp for day1 if self.__check_datestamp_correctness(day1): if not self._args.skip_filetime_extraction: # we've got only a day but we're able to determine # time from file mtime, if same as ISO day in file # name: logging.debug('__handle_file: try to get file time from mtime if days match between mtime and filename ISO ...') file_datetime = time.localtime(os.path.getmtime(link)) if self.__check_if_days_in_timestamps_are_same(file_datetime, day1): orgdate = OrgFormat.datetime(file_datetime, inactive=self._args.inactive_timestamps) else: logging.debug('__handle_file: day of mtime and filename ISO differs, using filename ISO day') orgdate = OrgFormat.strdate(day1, inactive=self._args.inactive_timestamps) else: # we've got only a day and determining mtime # is not planned, so use the day as date-stamp orgdate = OrgFormat.strdate(day1, inactive=self._args.inactive_timestamps) else: logging.warn('File "' + file + '" has an invalid datestamp (' + str(day1) + '). Skipping this faulty date.') # omit optional second day if first has an issue: has_2ymd = False has_2ymdhm = False # there is a time range: if has_2ymdhm: assert(day2) if self.__check_datestamp_correctness(day2): if self.__check_timestamp_correctness(time2): orgdate += '--' + OrgFormat.strdatetime(day2 + ' ' + time2, inactive=self._args.inactive_timestamps) else: logging.warn('File "' + file + '" has an invalid timestamp (' + str(time2) + '). Skipping this faulty time-stamp.') orgdate += '--' + OrgFormat.strdate(day2, inactive=self._args.inactive_timestamps) else: logging.warn('File "' + file + '" has an invalid datestamp (' + str(day2) + '). Skipping this faulty date.') elif has_2ymd: assert(day2) if self.__check_datestamp_correctness(day2): orgdate += '--' + OrgFormat.strdate(day2, inactive=self._args.inactive_timestamps) else: logging.warn('File "' + file + '" has an invalid datestamp (' + str(day2) + '). Skipping this faulty date.') except TimestampParseException: logging.error('File "' + str(file) + '" has in invalid date- or timestamp. OrgFormat of one of day1: "' + str(day1) + '" time1: "' + str(time1) + '" day2: "' + str(day2) + '" time2: "' + str(time2) + '" ' + 'failed with TimestampParseException. Skipping this faulty date.') orgdate = False else: logging.debug('__handle_file: no date- nor timestamp') orgdate = False if not orgdate and self._args.skip_notimestamp_files: logging.debug('__handle_file: file had no or wrong time-stamp and you decided to skip them.') return self.__write_file(file, link, orgdate) logging.debug('__handle_file: using orgdate: ' + str(orgdate)) return
def __get_item_data(self, item): """ gets information out of <item>..</item> @return: output, note, properties, tags variables for orgwriter.append_org_subitem """ try: # logging.debug(item) properties = OrgProperties() guid = item['id'] if not guid: logging.error("got no id") unformatted_link = item['link'] short_link = OrgFormat.link(unformatted_link, "link") # if we found a url in title # then append the url in front of subject if re.search("http[s]?://", item['title']) is not None: output = short_link + ": " + item['title'] else: output = OrgFormat.link(unformatted_link, item['title']) note = item['description'] # converting updated_parsed UTC --> LOCALTIME # Karl 2018-09-22 this might be changed due to: # DeprecationWarning: To avoid breaking existing software # while fixing issue 310, a temporary mapping has been # created from `updated_parsed` to `published_parsed` if # `updated_parsed` doesn't exist. This fallback will be # removed in a future version of feedparser. timestamp = OrgFormat.datetime( time.localtime(calendar.timegm(item['updated_parsed']))) properties.add("guid", guid) except KeyError: logging.error("input is not a RSS 2.0") sys.exit(1) tags = [] # Karl 2018-09-22 this might be changed due to: # DeprecationWarning: To avoid breaking existing software # while fixing issue 310, a temporary mapping has been created # from `updated_parsed` to `published_parsed` if # `updated_parsed` doesn't exist. This fallback will be # removed in a future version of feedparser. dont_parse = [ 'title', 'description', 'updated', 'summary', 'updated_parsed', 'link', 'links' ] for i in item: logging.debug(i) if i not in dont_parse: if (type(i) == str or type(i) == str) and \ type(item[i]) == str and item[i] != "": if i == "id": i = "guid" properties.add(i, item[i]) else: if i == "tags": for tag in item[i]: logging.debug("found tag: %s", tag['term']) tags.append(tag['term']) return output, note, properties, tags, timestamp
def startElement(self, name, attrs): """ at every <log> write to orgfile """ logging.debug("Handler @startElement name=%s,attrs=%s", name, attrs) if name == "log": call_number = attrs['number'] call_duration = int(attrs['dur']) call_date = int(attrs['date']) / 1000 # unix epoch call_type = int(attrs['type']) call_incoming = call_type == 1 call_outgoing = call_type == 2 call_missed = call_type == 3 call_voicemail = call_type == 4 call_rejected = call_type == 5 call_refused = call_type == 6 call_name = attrs['name'] output = "Phonecall " skip = False if call_incoming: output += "from " if self._ignore_incoming: skip = True elif call_outgoing: output += "to " if self._ignore_outgoing: skip = True elif call_missed: output += "missed " if self._ignore_missed: skip = True elif call_voicemail: output += "voicemail " if self._ignore_voicemail: skip = True elif call_rejected: output += "rejected " if self._ignore_rejected: skip = True elif call_refused: output += "refused " if self._ignore_refused: skip = True else: raise Exception("Invalid Phonecall Type: %d", call_type) call_number_string = "" if call_number != "-1": call_number_string = call_number else: call_number_string = "Unknown Number" name_string = "" if call_name != "(Unknown)": name_string = '[[contact:' + call_name + '][' + call_name + ']]' else: name_string = "Unknown" output += name_string if call_duration < self._minimum_duration: skip = True timestamp = OrgFormat.datetime(time.gmtime(call_date)) end_datetimestamp = datetime.datetime.utcfromtimestamp(call_date + call_duration) logging.debug("timestamp[%s] duration[%s] end[%s]" % (str(timestamp), str(call_duration), str(end_datetimestamp))) end_timestamp_string = OrgFormat.datetime(end_datetimestamp) logging.debug("end_time [%s]" % end_timestamp_string) data_for_hashing = output + timestamp properties = OrgProperties(data_for_hashing=data_for_hashing) properties.add("NUMBER", call_number_string) properties.add("DURATION", call_duration) properties.add("NAME", call_name) if not skip: self._writer.write_org_subitem(output=output, timestamp=timestamp + '-' + end_timestamp_string, properties=properties )