def parse_date_field(date_field):
"""
Return a utcdatetime from `datetime_field`, which is of one of the
following formats:
YYYYMMDDhhmmssZ
YYYYMMDDhhmmss+hhmm
YYYYMMDDhhmmss-hhmm
See http://pyopenssl.sourceforge.net/pyOpenSSL.html/openssl-x509.html
"""
match = re.match(
'(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})'
'(?P<hour>\d{2})(?P<minute>\d{2})(?P<second>\d{2})'
'(?P<timezone>[+-]\d{4}|Z)', date_field.decode('utf-8'))
isodate = '{}-{}-{}T{}:{}:{}{}'.format(
match.group('year'),
match.group('month'),
match.group('day'),
match.group('hour'),
match.group('minute'),
match.group('second'),
match.group('timezone'))
return utcdatetime.utcdatetime.from_datetime(parse_datetime(isodate))
def test_parse_datetime():
d = iso8601.parse_datetime("2006-10-20T15:34:56Z")
assert d.year == 2006
assert d.month == 10
assert d.day == 20
assert d.hour == 15
assert d.minute == 34
assert d.second == 56
assert d.tzinfo == iso8601.UTC
def test_parse_datetime_tz():
d = iso8601.parse_datetime("2006-10-20T15:34:56.123+02:30")
assert d.year == 2006
assert d.month == 10
assert d.day == 20
assert d.hour == 15
assert d.minute == 34
assert d.second == 56
assert d.microsecond == 123000
assert d.tzinfo.tzname(None) == "+02:30"
offset = d.tzinfo.utcoffset(None)
assert offset.days == 0
assert offset.seconds == 60 * 60 * 2.5
def test_parse_datetime_fraction_2():
"""From issue 6, allow slightly looser date parsing
"""
d = iso8601.parse_datetime("2007-5-7T11:43:55.328Z'", strict=False)
assert d.year == 2007
assert d.month == 5
assert d.day == 7
assert d.hour == 11
assert d.minute == 43
assert d.second == 55
assert d.microsecond == 328000
assert d.tzinfo == iso8601.UTC
def test_space_separator():
"""Handle a separator other than T
"""
d = iso8601.parse_datetime("2007-06-23 06:40:34.00Z")
assert d.year == 2007
assert d.month == 6
assert d.day == 23
assert d.hour == 6
assert d.minute == 40
assert d.second == 34
assert d.microsecond == 0
assert d.tzinfo == iso8601.UTC
def test_parse_no_timezone_no_strict():
"""issue 4 - Handle datetime string without timezone
This tests what happens when you parse a date with no timezone. While not
strictly correct this is quite common. I'll assume UTC for the time zone
in this case.
"""
d = iso8601.parse_datetime("2007-01-01T08:00:00", strict=False)
assert d.year == 2007
assert d.month == 1
assert d.day == 1
assert d.hour == 8
assert d.minute == 0
assert d.second == 0
assert d.microsecond == 0
assert d.tzinfo == iso8601.UTC
def test_parse_invalid_datetime():
iso8601.parse_datetime(None)
def test_parse_issue_6_strict():
iso8601.parse_datetime("2007-5-7T11:43:55.328Z'")
def test_parse_datetime_different_timezone():
tz = iso8601.FixedOffset(1, 0, "IST")
d = iso8601.parse_datetime("2006-10-20T15:34:56", default_timezone=tz, strict=False)
assert d.tzinfo == tz
def test_parse_datetime_z():
d = iso8601.parse_datetime("2006-10-20T15:34:56Z", default_timezone=iso8601.FixedOffset(1, 0, "IST"))
assert d.tzinfo == iso8601.UTC
def _set(self, value):
if isinstance(value, (unicode, str)):
value = iso8601.parse_datetime(value)
super(DateTimeField, self)._set(value)
def test_parse_incorrect_date_digits():
iso8601.parse_datetime("2007-1-1T08:00:00Z")
def test_parse_no_timezine_strict():
"""Variation of issue 4, raise a ParseError when there is no time zone
"""
iso8601.parse_datetime("2007-01-01T08:00:00")
def create_badges(gpx_inst, con_mngr, cache_mngr, force_tb_update,
force_owned_update):
''' Generate the badges from statistical data.
Use the statistical data from the parser runs and the badge definitions to
create badges and set their status.
'''
all = gpx_inst.all_caches
##### LOGS ####
if Pers.ownfoundlogcount > 0:
avgwordcount = Pers.wordcount / Pers.ownfoundlogcount
else:
avgwordcount = 0
print "Average word count: " + str(avgwordcount)
badgeManager.setStatus('Author', avgwordcount)
#### OVERALL COUNT #####
badgeManager.setStatus('Geocacher', Pers.ownfoundlogcount)
print "Geocaches " + str(Pers.ownfoundlogcount)
##### TYPES #####
print '\n'
types = ['Traditional Cache', 'Multi-cache', 'Unknown Cache',
'Letterbox Hybrid', 'Earthcache', 'Wherigo Cache',
'CITO Event Cache','Event Cache','Virtual Cache',
'Mega Social Event Cache','Benchmark','Waymark','Webcam Cache',
'Project Ape Cache']
for type_ in types:
generate_type_badges(type_)
lostn = len([a for a in all if 'Lost and Found' in a.desc])
badgeManager.setStatus('Lost', lostn)
print '10 Years! Cache ' + str(lostn)
##### CONTAINERS #####
print '\n',
types = [u'Not chosen', u'Large', u'Micro', u'Regular', u'Small', u'Other']
found_types = [a.cache.container for a in all]
container_hist = {}
for con in found_types:
container_hist[con] = container_hist.get(con,0) + 1
for key in types:
try:
badgeManager.setStatus(key[:5], container_hist.get(key,0))
print key + ' ' + str(container_hist.get(key,0))
except NameError:
print key + " No Match"
######### D/T Matrix #############
print '\n\t',
dtm = [(float(a.cache.difficulty), float(a.cache.terrain))
for a in all]
matrix = defaultdict(lambda: defaultdict(lambda: 0))
for pair in dtm:
matrix[pair[0]][pair[1]] = matrix[pair[0]].get(pair[1],0) + 1
difficult = terrain = [1.0, 1.5 , 2.0 , 2.5 , 3.0, 3.5, 4.0, 4.5, 5.0]
mcount = 0
for dif in difficult:
for ter in terrain:
amount = matrix[dif][ter]
print("%3d" % amount),
if amount > 0:
mcount += 1
print "\n\n\t",
print "Found %d of 81 D/T combinations"% mcount
badgeManager.setStatus('Matrix', mcount)
####### OTHERS #####
print '\n',
try:
hccs = [wpt.cache for wpt in all
if wpt.cache.terrain == u'5' and wpt.cache.difficulty == u'5']
except AttributeError:
return all
badgeManager.setStatus('Adventur', len(hccs))
print('HCC Caches: ' + str(len(hccs)) +
" (" + str([a.name for a in hccs]) + ")")
ftfs = [a.cache for a in all if 'FTF' in a.cache.logs[0].text]
badgeManager.setStatus('FTF', len(ftfs))
print('FTF Caches: ' + str(len(ftfs)) +
" (" + str([a.name for a in ftfs]) + ")")
print('\n'),
if(not force_owned_update):
try:
owned = int(cache_mngr.get('OWNCACHES', 'caches_hidden'))
ownedcaches = cache_mngr.get('OWNCACHES','caches')
owned_events = int(cache_mngr.get('OWNCACHES', 'owned_events'))
except NoOptionError:
print "Cached data incomplete, please redownload with -o"
owned = 0
ownedcaches = []
owned_events = 0
else:
print 'No list of hidden caches cached, retrieving new data ...'
ownlist = con_mngr.get_owner_list()
ownlist = re.compile("<script([^>]*)>.*?</script>",
re.DOTALL).sub("", ownlist)
ownlist = re.compile("<span([^>]*)>.*?</span>",
re.DOTALL).sub("", ownlist)
ownparser = OwnParser()
ownparser.feed(ownlist)
owned = ownparser.owncount
owned_events = ownparser.own_event_count
ownedcaches = ownparser.owncaches
cache_mngr.set('OWNCACHES', 'caches_hidden', str(owned))
cache_mngr.set('OWNCACHES', 'owned_events', str(owned_events))
cache_mngr.set('OWNCACHES', 'caches', str(ownedcaches))
with open('cache.dat', 'ab') as cachefile:
cache_mngr.write(cachefile)
badgeManager.setStatus('Owner', owned)
print "Owner of " + str(owned) + ' Caches: ' + str(ownedcaches)
badgeManager.setStatus('Host', owned_events)
print "Hosted " + str(owned_events) + ' Events.'
scuba = [a.name for a in all
if (5,1) in [(b.id, b.inc) for b in a.cache.attributes]]
badgeManager.setStatus('Scuba', len(scuba))
##### COUNTRIES #######
print '\n',
travel = [a.cache.country for a in all]
travel_hist = {}
for country in travel:
travel_hist[country] = travel_hist.get(country, 0) + 1
badgeManager.setStatus('Travelling', len(travel_hist))
print 'Countries traveled ' + str(len(travel_hist)) + ': '+ str(travel_hist)
try:
with open("statelist.txt",'r') as filehandle:
statelist = filehandle.read()
except IOError:
# Couldn't read file, download new.
try:
statelist = con_mngr.getcountrylist()
except Exception:
# Failed, abort.
print "Not able to retrieve country list"
raise
else:
# New statelist downloaded, saving for further use.
try:
with open("statelist.txt",'w') as filehandle:
filehandle.write(statelist)
except IOError:
print("Could not write 'statelist.txt' file.\n"
"Continuing without saving")
if statelist:
# Only generate with valid statelist, else skip
badgeManager.setCountryList(statelist)
for country in travel_hist.keys():
cbadge = stateBadge(country)
cbadge.setStatus(len(Pers.stateList[country]))
badgeManager.addBadge(cbadge)
print('Visited ' + str(len(Pers.stateList[country])) +
' state(s) in ' + country + "\n\t" +
str(Pers.stateList[country].keys()))
##### GEOGRAPHY #######
print('\n'),
badgeManager.setStatus('Clouds', Pers.hMax)
print("Found cache above " + str(Pers.hMax) + "m N.N.")
badgeManager.setStatus('Gound', Pers.hMin)
print("Found cache below " + str(Pers.hMin) + "m N.N.")
badgeManager.setStatus('Distance', Pers.max_distance[1])
print("Found cache " + str(Pers.max_distance[0]) + " in " +
str(Pers.max_distance[1]) + "km distance")
#### COINS ##########
print('\n'),
if(cache_mngr.has_option('TRAVELITEMS', 'coins') and
cache_mngr.has_option('TRAVELITEMS', 'travelbugs') and
not force_tb_update):
coins = int(cache_mngr.get('TRAVELITEMS', 'coins'))
tbs = int(cache_mngr.get('TRAVELITEMS', 'travelbugs'))
else:
print 'No Coin list cached, retrieving new data ...'
coinlist = con_mngr.getmycoinlist()
coinlist = re.compile("<script([^>]*)>.*?</script>",
re.DOTALL).sub("", coinlist)
coinlist = re.compile("<span([^>]*)>.*?</span>",
re.DOTALL).sub("", coinlist)
coinparser = CoinParser()
coinparser.feed(coinlist)
coins = coinparser.CoinCount
tbs = coinparser.TBCount
cache_mngr.set('TRAVELITEMS', 'travelbugs', str(tbs))
cache_mngr.set('TRAVELITEMS', 'coins', str(coins))
with open('cache.dat', 'wb') as cachefile:
cache_mngr.write(cachefile)
badgeManager.setStatus('Coin', coins)
print "Coins " + str(coins)
badgeManager.setStatus('Travelbug', tbs)
print "Travelbugs " + str(tbs)
#### DATE ##########
print('\n'),
cachebyday = defaultdict(lambda: 0)
cachebydate = defaultdict(lambda: 0)
for wpt in all:
if 'Z' not in wpt.cache.logs[0].date:
wpt.cache.logs[0].date += 'Z'
found = wpt.cache.logs[0].date
cachebyday[str(parse_datetime(found).date())] += 1
cachebydate[parse_datetime(found).date().strftime('%m-%d')] += 1
maxfind = 43 #max(cachebyday.values())
for (key, value) in zip(cachebyday.keys(), cachebyday.values()):
if value == maxfind:
maxfinddate = key
badgeManager.setStatus('Busy', maxfind)
# print("Found %i caches on %s"% (maxfind, maxfinddate))
badgeManager.setStatus('Calendar', len(cachebydate))
print("Found caches on %d dates"% (len(cachebydate)))
days = cachebyday.keys()
days.sort()
maxdays = dayscount = 1
prev = "0000-00-00"
for date in days:
if int(date[-2:]) == int(prev[-2:]) + 1:
dayscount += 1
elif (int(date[-2:]) == 1 and
int(prev[-2:]) == monthrange(int(prev[:4]) , int(prev[5:7]))[1]):
dayscount += 1
else:
maxdays = max(dayscount, maxdays)
dayscount = 1
prev = date
badgeManager.setStatus('Daily', maxdays)
print "Found caches on %i consecutive days"% maxdays
def EndElementHandler(self, name) :
if self.level == 2 :
if name == "array" :
raise DoneException()
if self.level == 3 :
if name == "key" :
self.currentmode = "data"
self.currentdict[self.currentkey] = ""
elif name == "dict" :
self.level = 2
if not self.currentdict.has_key("Album") :
self.currentdict["Album"] = "*Unknown Album*"
if not self.currentdict.has_key("Artist") :
self.currentdict["Artist"] = "*Unknown Artist*"
albums = Album.objects.filter(album_library__id = self.lib.id, album_name = self.currentdict["Album"])
album = None
if len(albums) == 0 :
album = Album(album_library = self.lib,
album_name = self.currentdict["Album"],
album_synchronized = True)
album.save()
else :
album = albums[0]
album.album_synchronized = True
album.save()
songs = Song.objects.filter(song_filename = self.currentdict["Location"]) # assuming filename is a good enough primary key
song = None
d = self.currentdict
try :
modified = iso8601.parse_datetime(d.get("Date Modified", None))
except :
modified = None
if (not skipiTunesKind(d["Kind"])) and (modified is not None) :
if len(songs) == 0 :
song = Song(song_name = d.get("Name", None),
song_grouping = d.get("Grouping", None),
song_composer = d.get("Composer", None),
song_artist = d.get("Artist", None),
song_album = album,
song_genre = d.get("Genre", None),
song_time = d.get("Total Time", None),
song_tracknum = d.get("Track Number", None),
song_numbertracks = d.get("Track Count", None),
song_discnum = d.get("Disc Number", None),
song_numberdiscs = d.get("Disc Count", None),
song_filetype = iTunesKindToMime(d["Kind"]),
song_filesize = d.get("Size", None),
song_bitrate = d.get("Bit Rate", None),
song_filename = d.get("Location", None),
song_synchronized = True,
song_modified = modified)
song.save()
elif songs[0].song_modified < modified :
songs.update(song_name = d.get("Name", None),
song_grouping = d.get("Grouping", None),
song_composer = d.get("Composer", None),
song_artist = d.get("Artist", None),
song_album = album,
song_genre = d.get("Genre", None),
song_time = d.get("Total Time", None),
song_tracknum = d.get("Track Number", None),
song_numbertracks = d.get("Track Count", None),
song_discnum = d.get("Disc Number", None),
song_numberdiscs = d.get("Disc Count", None),
song_filetype = iTunesKindToMime(d["Kind"]),
song_filesize = d.get("Size", None),
song_bitrate = d.get("Bit Rate", None),
song_filename = d.get("Location", None),
song_synchronized = True,
song_modified = modified)
else :
songs.update(song_synchronized = True)
else :
if modified is None :
self.skippedSongs.append((d.get("Location"), "Bad last modified string"))
else :
self.skippedSongs.append((d.get("Location"), "Type \""+d["Kind"]+"\" is unsupported"))
else :
self.currentmode = None
def test_parse_invalid_datetime2():
iso8601.parse_datetime("23")
import json
import codecs
import iso8601
import time
col_names = ['date', 'size', 'activity', 'perms', 'uid', 'gid', 'inode', 'file']
mactime_file = open(sys.argv[1])
output_file = codecs.open(sys.argv[2], 'w', encoding = "utf-8")
content = mactime_file.read()
records = list(map(lambda x:x.split(','), content.strip().split('\n')))
for record in records:
json_dict = dict()
for index, value in enumerate(record):
if index == 0:
date = iso8601.parse_datetime(value)
timestamp = time.mktime(date.timetuple())
json_dict[col_names[index]] = int(timestamp)
elif index == 1: # convert the size from string to int for query selection purpose
json_dict[col_names[index]] = int(value)
else:
json_dict[col_names[index]] = value
json.dump(json_dict, output_file)
output_file.write('\n')
mactime_file.close()
output_file.close()
import iso8601
import time
col_names = [
'date', 'size', 'activity', 'perms', 'uid', 'gid', 'inode', 'file'
]
mactime_file = open(sys.argv[1])
output_file = codecs.open(sys.argv[2], 'w', encoding="utf-8")
content = mactime_file.read()
records = list(map(lambda x: x.split(','), content.strip().split('\n')))
for record in records:
json_dict = dict()
for index, value in enumerate(record):
if index == 0:
date = iso8601.parse_datetime(value)
timestamp = time.mktime(date.timetuple())
json_dict[col_names[index]] = int(timestamp)
elif index == 1: # convert the size from string to int for query selection purpose
json_dict[col_names[index]] = int(value)
else:
json_dict[col_names[index]] = value
json.dump(json_dict, output_file)
output_file.write('\n')
mactime_file.close()
output_file.close()
