def load_raw_file(self, filename):
checkin_id = 0
t = timestamp.Timestamp(str(self.para['preprocess']['startTime']))
max_ts = t.get_timestamp(str(self.para['preprocess']['endTime']),
'min')
with open(filename, 'r') as fin:
for line in fin:
# items = self.parse_checkin_file(line)
# if items is None:
# continue
items = line.strip().split('\x01')
time = (
int(items[5]) - 460162801 - 6048000 + 864000
) / 60 # -460162801-6048000 means Oct 10, +864000 means Sep 30
# time = t.get_timestamp(items[4].strip('"'), 'min')
if time < 0 or time > max_ts:
continue
user_id = int(items[1])
lat = float(items[2])
lng = float(items[3])
message = items[6]
raw_message = items[7]
if user_id in self.checkins:
self.checkins[user_id].append(
(checkin_id, time, user_id, lat, lng, message,
raw_message))
else:
self.checkins[user_id] = [(checkin_id, time, user_id, lat,
lng, message, raw_message)]
checkin_id += 1
if checkin_id % 10000 == 0:
print '%d checkins' % checkin_id
# break
print 'Finished processing %d checkins' % len(self.checkins)
def do_action(self, event_data, inventory):
time = timestamp.Timestamp(event_data[0])
try:
pet = inventory._inventory[event_data[2]]
pet.petcare(time)
self.result = ("SUCCESS",)
except:
self.result("FAIL", sys.exc_info()[1])
def __formatMessage(self, logType, msg):
"""__formatMessage(t,m) -> string
(Private method)
Builds a formatted message string that includes logType t, message m,
and a timestamp. The format is:
[#-00000327][2006-01-24 12:32:31][6776][INFO ] : This is a sample message.
[Counter ][Date ][Thrd][Level] : <message>
"""
return '[#-%08d][%s][%d][%-5s] : %s' % (
self.counter, timestamp.Timestamp().getLocalString(),
thread.get_ident(), logType, msg)
def parse_checkin_file(self, line):
terms = line.strip().split(',')
t = timestamp.Timestamp(str(self.para['preprocess']['startTime']))
ts = t.get_timestamp(terms[1].strip('"'), 'min')
max_ts = t.get_timestamp(str(self.para['preprocess']['endTime']),
'min')
if ts < 0 or ts > max_ts:
return None
user_id = terms[-2].strip('"') # user id
venue_id = terms[-1].strip('"') # venue id
lat, lng = self.places[venue_id][0:2]
message = self.parse_message(terms[4].strip('"'))
message.update(self.places[venue_id][-1])
return ts, user_id, lat, lng, message
def parse_checkin_db(self, tweet):
t = timestamp.Timestamp(str(self.para['preprocess']['startTime']))
ts = t.get_timestamp(tweet['created_at'], 'min')
maxTime = t.get_timestamp(str(self.para['preprocess']['endTime']),
'min')
if ts < 0 or ts > maxTime:
return None
user_id = tweet['user_id']
# Note: for the ny40k data set, lng comes first.
if type(tweet['location']) is list:
lng, lat = tweet['location']
else:
lat, lng = [float(e) for e in tweet['location'].split(',')]
message = self.parse_message(tweet['text'])
if len(message) < 3:
return None
return ts, user_id, lat, lng, message
def do_action(self, event_data, inventory):
self.timestamp = timestamp.Timestamp(event_data[0])
# Check for random pet deaths
deaths = list()
for item in inventory:
chance = random.random()
if chance <= item.attrition_rate and item.count > 0:
try:
amount = random.randrange(1, (item.count // 2))
except ValueError:
amount = 1
event = str(self.timestamp) + "|" + "PetLoss" +\
"|" + item.name + "|" + str(amount)
deaths.append(event)
if len(deaths) > 0:
self.result = ("SUCCESSWITHEVENTS", deaths)
else:
self.result = ("SUCCESS",)
def html(self, blue):
if blue:
td="<td valign=top bgcolor='#ccccee'"
else:
td="<td valign=top bgcolor='#ffffff'"
ret=[]
ret.append("<tr align=top>")
t=string.replace(timestamp.Timestamp(self.begin_date).getUTCString(),
" "," ")
ret.append("%s>%s</td>" % (td,t))
ret.append("%s>%s</td>" % (td, htmlquote(self.author)))
ret.append("%s>%s</td>" % (td, htmlquote(self.comment)))
files=[]
for f in self.files:
files.append(f.html())
ret.append("%s>%s</td>" % (td, string.join(files,"<br>")))
return string.join(ret,"")
def sox_info(filename):
# sox has a bug where --info won't let you specify type, so you can't
# have it read from stdin
info = {}
child = subprocess.Popen(['sox', '--info', filename],
stdout = subprocess.PIPE)
for line in line_reader(child.stdout):
colon = line.find(':')
if colon != -1:
key = line[:colon].strip()
value = line[colon+1:].strip()
if value.startswith("'") and value.endswith("'"):
value = value[1:-1]
if key == 'Channels' or key == 'Sample Rate':
info[key] = int(value)
elif key == 'Duration':
split_value = value.split(' = ')
for part in split_value:
split_part = part.split(' ', 1)
if len(split_part) == 1:
info['Duration'] = timestamp.Timestamp(split_part[0])
elif len(split_part) == 2:
if split_part[1] == 'samples':
info['Total Samples'] = int(split_part[0])
elif split_part[1] == 'CDDA sectors':
info['CDDA Sectors'] = float(split_part[0])
elif key == 'Precision':
info['Bit Precision'] = int(value.split('-', 1)[0])
elif key == 'Bit Rate':
split_value = value.split('k', 1)
if len(split_value) != 2 or split_value[1] != '':
raise RuntimeError(
'Unknown format for "Bit Rate": {}'.format(value))
info[key] = int(split_value[0])
elif key != 'Input File' and key != 'File Size':
info[key] = value;
elif key == 'Comments':
break; # not handling tags here
child.wait()
return info;
def getLineInfo(line):
atm = attempt.Attempt()
line_ips = re.findall(r'[0-9]+(?:\.[0-9]+){3}', line)
if len(line_ips) != 0:
spline = line.split()
month_hr = spline[0]
day = spline[1]
time = spline[2].split(":")
stamp = timestamp.Timestamp(timestamp.month_num(month_hr), day,
DATE.year, time[0], time[1], time[2])
atm.ip = line_ips[0]
atm.timestamp = stamp
else:
return None
#GeoIP Lookup
try:
atm.lookup = geolite2.lookup(line_ips[0])
except:
return None
if atm.lookup == None:
return None
#Determine if a failure or successful login
line_lower = line.lower()
fail_keywords = ["fail", "invalid", "preauth"]
if any(elm in line_lower for elm in fail_keywords):
# if ("fail" in line.lower()) or ("invalid" in line.lower()):
atm.success = 0
elif "accept" in line.lower():
atm.success = 1
else:
return None
return atm
def __init__(self):
self.id = ActionID.next()
self.name = "Action"
self.data = None
self.result = None
self.timestamp = timestamp.Timestamp('1492-01-01T00:00:00')
def place_order(self, pet, time_string): self.timestamp = timestamp.Timestamp(time_string) order = str(self.timestamp) + "|" + "OrderStock" + "|" +\ str(pet.__class__) + "|" + pet.name + "|" +\ pet.order_info() + "|" + str(50) return [order]
def __init__(self,
module_id,
source_branch,
source_tag,
source_date,
start_date,
end_date):
if start_date:
if not re.match("^[-a-zA-Z0-9:/ ]*$",start_date):
raise "Invalid start date"
if end_date:
if not re.match("^[-a-zA-Z0-9:/ ]*$",end_date):
raise "Invalid start date"
self.source_tag = source_tag
self.source_date = source_date
self.checkins = []
todo={}
if type(module_id) != types.ListType:
module_id=[ module_id ]
self.module_id = module_id
self.source_branch = source_branch
self.start_date = start_date
self.end_date = end_date
if self.start_date:
self.start_date_local=self.start_date
# FIXME: this was heinously broken. This call converts ticks
# to gmticks with the date_to_gmticks() call. Then converts that
# time a second time with the cvs_server_time() call.
# This will probably require deep investigation to make sure that
# diff reports are all using the correct times. No surprise that
# there is an open bug on the diff report timestamps.
#self.start_date = datelib.cvs_server_time(datelib.date_to_gmticks(self.start_date))
# Notice we are still propagating raw strings here instead of
# moving Timestamp objects around. Going to take a long time to
# clean this up completely.
startTS = timestamp.Timestamp( self.start_date )
self.start_date = startTS.getUTCString()
if self.end_date:
self.end_date_local=self.end_date
endTS = timestamp.Timestamp( self.end_date )
self.end_date = endTS.getUTCString()
#self.end_date = datelib.cvs_server_time(datelib.date_to_gmticks(self.end_date))
self.date_range = " -d '%s<%s'" % (self.start_date or "",
self.end_date or "")
if self.date_range == " -d '<'":
self.date_range = ""
## find the BIF file for the branch name
branch_list = branchlist.BranchList(source_tag, source_date)
old_filename = branch_list.file(source_branch)
if not old_filename:
raise "no BIF file for branch=\"%s\"" % (source_branch)
## parse old BIF file, and create the new BIF file
## Do not include shadows
print "parsing file=\"%s\"" % (old_filename)
bdata1 = bif.load_bif_data(old_filename, branch_list, 0)
## compute the dependancy list for the target
if module_id == [ "ALL" ]:
print "diffing all modules" % (string.capitalize(branch_or_tag))
deplist1_list = bdata1.module_hash.values()
else:
print "computing dependancy tree for target=%s" % (repr(module_id))
deplist1_list = dependlist.DependList(bdata1, module_id).list()
modules_to_diff = []
dists_done={}
for mod in deplist1_list:
if mod.type not in ["cvs", "distribution" ]:
continue
if mod.type == "distribution":
if dists_done.has_key(mod.cvs_root):
continue
dists_done[mod.cvs_root]=1
modules_to_diff.append(mod)
self.modules_to_diff = modules_to_diff
import timestamp
from conf import tokens
for each_token in tokens:
dropstamp = timestamp.Timestamp(token=each_token)
dropstamp.timestamp()
def time_from_last_petcare(self):
if not self._last_petcare:
return "They have not been fed yet."
else:
now = timestamp.Timestamp()
return now.difference(self._last_petcare)
def to_entries(self, add_tags=[], add_time=False, moments_only=False):
"""
convert log to a list of entry objects (essentially what a log is)
if moments_only is true, only Moments will be created
if add_time is false, and moments_only is true,
upon reaching an Entry only (*... ) (no timestamp)
that information will be added to the previous Moment
(this is useful when parsing data that was not originally intended to
be used as part of a moment... it may contain lines that start with '*')
"""
entries = []
entry_regex = "\*"
entry_search = re.compile(entry_regex)
cur_entry = Moment()
cur_entry.path = self.name
new_entry = None
try:
self.seek(0)
line = self.readline()
line = unicode(line)
except:
print "Problem reading file"
return entries
#first line of a log should have an entry... this is our check
if entry_search.match(line):
self.has_entries = True
while line:
#we might have found a new entry...
#see what kind, if any:
(ts, line_tags) = timestamp.parse_line_for_time(line)
if ts:
new_entry = Moment()
new_entry.created = timestamp.Timestamp(ts)
elif entry_search.match(line):
if not moments_only:
new_entry = Moment()
elif add_time and moments_only:
#ok to make a default time for the entry
new_entry = Moment()
print "no timestamp found in this entry"
else:
#must be moments only,
#but we don't want to add a timestamp
#just include the data with the previous moment
new_entry = None
if new_entry:
#finish up last entry...
#only need to add if it had information
if cur_entry.data or cur_entry.tags:
entries.append(cur_entry)
new_entry.path = self.name
current_tags = line_tags.strip().split()
if add_tags:
temp_tags = add_tags[:]
for t in current_tags:
if t not in temp_tags:
temp_tags.append(t)
current_tags = temp_tags
new_entry.tags.extend(current_tags)
cur_entry = new_entry
new_entry = None
else:
# only want to add the entry itself
cur_entry.data += line
line = unicode(self.readline())
#need to get the last entry from the file, if there is one.
if cur_entry.data:
entries.append(cur_entry)
#if not, don't scan
else:
print "File does not start with an entry: %s" % self.name
return entries