def main(args): if len(args) < 2: print "Usage: python %s {file1 file2 file3 ...}" % (args[0]) exit() ''' Sort is not the best way to go, since it doesn't account for month increments. Anyway, this can be easily corrected by renaming the output file after the merging process finishes. ''' filename_list = list(set(args[1:])) filename_list.sort() if not filename_list: print "No filenames provided" exit() dir_index = filename_list[0].rfind( os.sep, 0, -1) # os.sep is the folder separator (e.g. "/") ext_index = filename_list[0].rfind('.', dir_index, -1) output_filename = "%s%s_to_%s.gz" % ( filename_list[0][:dir_index + 1], filename_list[0][dir_index + 1:ext_index], filename_list[-1][dir_index + 1:ext_index]) files = [] output_file = open_file(output_filename, "w") try: for filename in filename_list: files.append(open_file(filename)) readings = [] ''' Initialize readings vector ''' for f in files: if file: parsed_line = parse_line(f.readline()) if parsed_line: readings.append((f, parsed_line)) while len(readings) > 0: readings.sort(key=get_timestamp) earliest = readings[0] output_file.write("%f %s\n" % (earliest[1][0], " ".join(earliest[1][1:]))) parsed_line = parse_line(earliest[0].readline()) if parsed_line and earliest[0]: readings[0] = (earliest[0], parsed_line) else: readings.pop(0) finally: if output_file: output_file.close() if files: for f in files: if f: f.close()
def main(args): if len(args) < 2: print "Usage: python %s {file1 file2 file3 ...}" % (args[0]) exit() ''' Sort is not the best way to go, since it doesn't account for month increments. Anyway, this can be easily corrected by renaming the output file after the merging process finishes. ''' filename_list = list(set(args[1:])) filename_list.sort() if not filename_list: print "No filenames provided" exit() dir_index = filename_list[0].rfind(os.sep, 0, -1) # os.sep is the folder separator (e.g. "/") ext_index = filename_list[0].rfind('.', dir_index, -1) output_filename = "%s%s_to_%s.gz" % (filename_list[0][:dir_index+1], filename_list[0][dir_index+1:ext_index], filename_list[-1][dir_index+1:ext_index]) files = [] output_file = open_file(output_filename, "w") try: for filename in filename_list: files.append(open_file(filename)) readings = [] ''' Initialize readings vector ''' for f in files: if file: parsed_line = parse_line(f.readline()) if parsed_line: readings.append((f, parsed_line)) while len(readings) > 0: readings.sort(key=get_timestamp) earliest = readings[0] output_file.write("%f %s\n" % (earliest[1][0], " ".join(earliest[1][1:]))) parsed_line = parse_line(earliest[0].readline()) if parsed_line and earliest[0]: readings[0] = (earliest[0], parsed_line) else: readings.pop(0) finally: if output_file: output_file.close() if files: for f in files: if f: f.close()
def read_intervals(input_filename, motes_to_ignore, max_latency): motes = {} if not input_filename: f = sys.stdin if f.closed: return motes else: f = open_file(input_filename, "r") try: for line in f: timestamp, mote_id, counter, temperature = parse_line(line) if mote_id not in motes_to_ignore: mote = motes.get(mote_id, None) if mote is None: motes[mote_id] = [[timestamp, timestamp]] if motes[mote_id][-1][1] - motes[mote_id][-1][0] > max_latency: ''' Close the interval ''' motes[mote_id].append([timestamp, timestamp]) else: ''' Update the end of the last interval ''' motes[mote_id][-1][1] = timestamp finally: if f: f.close() find_small_intervals(motes) return motes.values()
def read_latencies(input_filename, discard=20*60): ''' @param discard: number of seconds to discard out of the experiment beginning ''' motes = {} discard_time = 0 f = open_file(input_filename) try: for line in f: timestamp, mote_id, counter, temperature = parse_line(line) if not discard_time: discard_time = timestamp + discard if timestamp >= discard_time: mote = motes.get(mote_id, None) if mote is None: mote = {"last_timestamp": timestamp, "latencies": []} motes[mote_id] = mote else: latency = timestamp - mote["last_timestamp"] mote["latencies"].append(latency) mote["last_timestamp"] = timestamp finally: if f: f.close() return motes
def main(args): if len(args) < 3: print "Usage: python %s {filename} {start} {end} [-t: range is a timestamp (default is line numbers)]" % (args[0]) exit() filename = args[1] timestamp_mode = '-t' in args if timestamp_mode: range_start = float(args[2]) range_end = float(args[3]) else: range_start = int(args[2]) range_end = int(args[3]) f = open_file(filename) try: line_count = 0 for line in f: if timestamp_mode: timestamp, mote_id, counter, temperature = parse_line(line) #@UnusedVariable if timestamp > range_end: break if timestamp >= range_start: print line, else: if line_count > range_end: break if line_count >= range_start: print line, line_count += 1 finally: if f: f.close()
def read_latencies(input_filename, discard=20 * 60): ''' @param discard: number of seconds to discard out of the experiment beginning ''' motes = {} discard_time = 0 f = open_file(input_filename) try: for line in f: timestamp, mote_id, counter, temperature = parse_line(line) if not discard_time: discard_time = timestamp + discard if timestamp >= discard_time: mote = motes.get(mote_id, None) if mote is None: mote = {"last_timestamp": timestamp, "latencies": []} motes[mote_id] = mote else: latency = timestamp - mote["last_timestamp"] mote["latencies"].append(latency) mote["last_timestamp"] = timestamp finally: if f: f.close() return motes
def main(args): if len(args) < 3: print "Usage: python %s {filename} {start} {end} [-t: range is a timestamp (default is line numbers)]" % ( args[0]) exit() filename = args[1] timestamp_mode = '-t' in args if timestamp_mode: range_start = float(args[2]) range_end = float(args[3]) else: range_start = int(args[2]) range_end = int(args[3]) f = open_file(filename) try: line_count = 0 for line in f: if timestamp_mode: timestamp, mote_id, counter, temperature = parse_line( line) #@UnusedVariable if timestamp > range_end: break if timestamp >= range_start: print line, else: if line_count > range_end: break if line_count >= range_start: print line, line_count += 1 finally: if f: f.close()
def main(args): if len(args) < 2: print 'Usage: python %s {input file path}' % (args[0]) exit() input_file_path = args[1] output_file_path = "%s.filled.gz" % (input_file_path) #output_file_path = "%s.filled" % (input_file_path) input_file = open_file(input_file_path, 'r') output_file = open_file(output_file_path, "w") try: motes = {} # holds last reading for each mote for line in input_file: timestamp, mote_id, counter, temperature = parse_line(line) mote = motes.get(mote_id, None) if mote is not None: prev_timestamp, prev_line = mote if timestamp < prev_timestamp: print "Timestamps out of order: %f > %f" % (prev_timestamp, timestamp) exit() time_delta = timestamp - (prev_timestamp + TIME_RESOLUTION) while time_delta > MAX_TIME_DELTA: # needs to fill data prev_timestamp += TIME_RESOLUTION time_delta -= TIME_RESOLUTION output_file.write("%f %s" % (prev_timestamp, prev_line)) motes[mote_id] = (timestamp, line[line.index(" ") + 1:] ) # line minus the timestamp # writes original data output_file.write(line) finally: if input_file: input_file.close() if output_file: output_file.close()
def main(args): if len(args) < 2: print 'Usage: python %s {input file path}' % (args[0]) exit() input_file_path = args[1] output_file_path = "%s.filled.gz" % (input_file_path) #output_file_path = "%s.filled" % (input_file_path) input_file = open_file(input_file_path, 'r') output_file = open_file(output_file_path, "w") try: motes = {} # holds last reading for each mote for line in input_file: timestamp, mote_id, counter, temperature = parse_line(line) mote = motes.get(mote_id, None) if mote is not None: prev_timestamp, prev_line = mote if timestamp < prev_timestamp: print "Timestamps out of order: %f > %f" % (prev_timestamp, timestamp) exit() time_delta = timestamp - (prev_timestamp + TIME_RESOLUTION) while time_delta > MAX_TIME_DELTA: # needs to fill data prev_timestamp += TIME_RESOLUTION time_delta -= TIME_RESOLUTION output_file.write("%f %s" % (prev_timestamp, prev_line)) motes[mote_id] = (timestamp, line[line.index(" ") + 1:]) # line minus the timestamp # writes original data output_file.write(line) finally: if input_file: input_file.close() if output_file: output_file.close()
def _parse_arff_header(self): attributes = [] if not self.fp: self.fp = open_file(self.filename) for line in iter(self.fp): if line.startswith("@data"): break elif line.startswith("@attribute"): attr_name = line.split()[1] attributes.append(attr_name) if "timestamp" not in attributes and self.auto_timestamps: attributes.insert(0, "timestamp") self.use_auto_timestamps = True return attributes
def main(args): if len(args) < 2: print "Usage: python %s {input filename}" % (args[0]) return input_path = args[1] input_filename = os.path.basename(input_path) """ Create folder based on the filename """ dot_index = input_filename.find(".") if dot_index < 0: dot_index = len(input_filename) dir_path = os.path.join(os.path.dirname(input_path), input_filename[:dot_index]) if not os.path.exists(dir_path): os.makedirs(dir_path) output_files = {} # dict that maps mote_ids into files input_file = open_file(input_path, "r") try: for line in input_file: timestamp, mote_id, counter, value = parse_line(line) #@UnusedVariable if not mote_id: print "Mote id is missing!" exit() mote_file = output_files.get(mote_id, None) if mote_file is None: mote_filename = "%s/mote_%s.txt" % (dir_path, mote_id) mote_file = open(mote_filename, "w") output_files[mote_id] = mote_file mote_file.write(line) finally: if input_file: input_file.close() for f in output_files.values(): f.close()
def main(args): if "-h" in args: print "python %s [input filename] [-s (suppress zero temperature diffs)] [-j (join zero temperature diffs)]" % (args[0]) exit() suppress_zero_diff = '-s' in args if suppress_zero_diff: args.remove('-s') join_zero_temperature_diffs = '-j' in args if join_zero_temperature_diffs: args.remove('-j') f = sys.stdin if len(args) > 1: f = open_file(args[1]) motes = {} diff_temp = -1.0 try: for line in f: timestamp, mote_id, counter, temperature = parse_line(line) mote = motes.get(mote_id, None) if not mote: motes[mote_id] = mote = {} else: diff_timestamp = timestamp - mote['ts'] diff_temp = temperature - mote['temp'] if diff_timestamp > 0.00000001 and (not suppress_zero_diff or diff_temp != 0.0): print diff_timestamp, mote_id, diff_temp / diff_timestamp if not join_zero_temperature_diffs or (diff_temp != 0.0): mote['ts'] = timestamp mote['temp'] = temperature finally: if f: f.close()
def reset(self): self.close() self.fp = open_file(self.filename) if self.file_type == "arff": self.arff_attributes = self._parse_arff_header()
def main(args): if len(args) < 2: print "Usage: python %s {input filename} [-f (function to plot with file) mote_id]" % (args[0]) exit() filename = args[1] ts = TimeSeries() motes = set() print "Parsing..." f = None file_contents = read_file(filename) if not file_contents: f = open_file(filename) lines_iter = iter(f) else: lines_iter = iter(file_contents) try: for line in lines_iter: timestamp, mote_id, counter, temp = parse_line(line) ts.add(timestamp, (mote_id, temp)) motes.add(mote_id) finally: if f: f.close() if file_contents: del file_contents diffs = [] for mote_id1 in motes: for mote_id2 in motes: if mote_id1 < mote_id2 and is_opposed(mote_id1, mote_id2): diff_ts = mote_diffs(ts, mote_id1, mote_id2) diffs.append(( "%d-%d (rack %s)" % (mote_id1, mote_id2, MOTE_LOCATION[mote_id1]['rack']), diff_ts)) print "Plotting..." fig = pylab.figure() pylab.grid(True) ax1 = fig.add_subplot(111) i = 0 for diff in diffs: temps = [] dates = [] for t, temp in diff[1]: dates.append(datetime.datetime.fromtimestamp(t)) temps.append(temp) dates = matplotlib.dates.date2num(dates) h = float(i + 1) / len(diffs) color = hsv_to_rgb(h, 0.8, 0.8) # 'b' means default (plot with lines) ax1.plot_date(dates, temps, 'b', color=color) i += 1 pylab.legend( [mote_legend for mote_legend, diff_ts in diffs] ) ax1.set_ylabel("Temp difference (C)") ax1.set_xlabel("Time") pylab.show()
def main(args): ion() grid(True) figure(1) frame1 = plt.gca() frame1.axes.get_xaxis().set_visible(False) frame1.axes.get_yaxis().set_visible(False) motes = {} temp_matrix = [ [MIN_TEMP for i in range(7)] for j in range(9) ] temp_matrix[-1][-1] = MAX_TEMP positions = { \ # back 248: (1,1), 247: (2,1), 252: (3,1), 238: (1,5), 244: (2,5), 243: (3,5), # front 249: (6,1), 253: (7,1), 245: (8,1), 241: (6,5), 237: (7,5), 239: (8,5), } for mote_id, pos in positions.items(): plt.text(pos[1], pos[0], str(mote_id), fontsize=12, horizontalalignment='center', verticalalignment='center', color="black") plt.text(1, 0, "Back R1", fontsize=12, horizontalalignment='center', verticalalignment='bottom', color="white") plt.text(5, 0, "Back R2", fontsize=12, horizontalalignment='center', verticalalignment='bottom', color="white") plt.text(1, 5, "Front R1", fontsize=12, horizontalalignment='center', verticalalignment='bottom', color="white") plt.text(5, 5, "Front R2", fontsize=12, horizontalalignment='center', verticalalignment='bottom', color="white") f = open_file("/home/giulio/Dropbox/Projeto Sensores/experiments/temperatura/sala_servidores/temps_25_05_12_15h09m48s.txt.gz") loaded_color_bar = False try: for line in f: #A = rand(5,5) #imshow(A, interpolation='nearest') values = parse_line(line) timestamp = values[0] mote_id = values[1] temp = values[-1] mote = motes.get(mote_id, None) if mote is None or abs(mote["temp"] - temp) > TEMP_DRAW_THRESHOLD: if mote is None: mote = {"temp": temp, "timestamp": timestamp} motes[mote_id] = mote else: mote["temp"] = temp mote["timestamp"] = timestamp pos = positions.get(mote_id, None) if pos is not None: temp_matrix[pos[0]][pos[1]] = temp print "Drawing", mote_id, time.ctime(timestamp), temp imgplot = imshow(temp_matrix, interpolation='nearest') imgplot.set_cmap('jet') if not loaded_color_bar: plt.colorbar() loaded_color_bar = True draw() #time.sleep(.1) finally: if f: f.close()
def main(args): if len(args) < 2: print "Usage: python %s {input filename} [-f (function to plot with file) mote_id]" % ( args[0]) exit() filename = args[1] ts = TimeSeries() motes = set() print "Parsing..." f = None file_contents = read_file(filename) if not file_contents: f = open_file(filename) lines_iter = iter(f) else: lines_iter = iter(file_contents) try: for line in lines_iter: timestamp, mote_id, counter, temp = parse_line(line) ts.add(timestamp, (mote_id, temp)) motes.add(mote_id) finally: if f: f.close() if file_contents: del file_contents diffs = [] for mote_id1 in motes: for mote_id2 in motes: if mote_id1 < mote_id2 and is_opposed(mote_id1, mote_id2): diff_ts = mote_diffs(ts, mote_id1, mote_id2) diffs.append( ("%d-%d (rack %s)" % (mote_id1, mote_id2, MOTE_LOCATION[mote_id1]['rack']), diff_ts)) print "Plotting..." fig = pylab.figure() pylab.grid(True) ax1 = fig.add_subplot(111) i = 0 for diff in diffs: temps = [] dates = [] for t, temp in diff[1]: dates.append(datetime.datetime.fromtimestamp(t)) temps.append(temp) dates = matplotlib.dates.date2num(dates) h = float(i + 1) / len(diffs) color = hsv_to_rgb(h, 0.8, 0.8) # 'b' means default (plot with lines) ax1.plot_date(dates, temps, 'b', color=color) i += 1 pylab.legend([mote_legend for mote_legend, diff_ts in diffs]) ax1.set_ylabel("Temp difference (C)") ax1.set_xlabel("Time") pylab.show()
def main(args): if len(args) < 2: print "Usage: python %s {hidden_vars_file} [-b (blame_file)]" % ( args[0]) exit() hidden_vars = [[]] hv_file = open_file(args[1]) try: for line in hv_file: line = line.strip() if line: values = map(float, line.split()) for i in xrange(len(values) - len(hidden_vars) - 1): hidden_vars.append([]) for i in xrange(len(hidden_vars)): if i > len(values) - 2: hidden_vars[i].append((values[0], numpy.nan)) else: hidden_vars[i].append((values[0], values[i + 1])) finally: hv_file.close() blame_file = open_file(args[args.index('-b') + 1]) if '-b' in args else None weights = [] if blame_file: try: for line in blame_file: line = line.strip() if line: values = map(float, line.split()) if not weights: for i in xrange(len(values) - 1): # skip timestamp weights.append([]) for i in xrange(len(weights)): weights[i].append([values[0], values[i + 1]]) finally: blame_file.close() for w in weights: last = w[0][1] w[0][1] = 0 for i in xrange(1, len(w)): aux = w[i][1] w[i][1] = (w[i][1] - last)**2 last = aux size = 21 for w in weights: for i in xrange(0, len(w) - size, size): avg = sum(w[i + j][1] for j in xrange(size)) / size for j in xrange(size): w[i + j][1] = avg # filter poor motes cutoff = 5 * 1e-7 avgs = [] for w in weights: avg = sum(w[i][1] for i in xrange(len(w))) / len(w) avgs.append(avg) chosen_indexes = [] for i in xrange(len(avgs)): if avgs[i] > cutoff: chosen_indexes.append(i) # setup pylab #fig = pylab.figure() pylab.subplot(211) pylab.grid(True) for h in hidden_vars: x, y = zip(*h) pylab.plot(x, y, '-') pylab.legend(range(1, len(hidden_vars) + 1)) if weights: pylab.subplot(212) pylab.grid(True) for i in chosen_indexes: x, y = zip(*weights[i]) pylab.plot(x, y, '-') pylab.legend(chosen_indexes) pylab.show()
def main(args): if len(args) < 2 or '-h' in args: print "Usage: python %s {temp_file} [-t topology_file] [-b battery_file] [-at artificial timestamps] [-m naive (default) | additive {min, max, incr} | deadzone {delta, max_period}]" % (args[0]) exit() if '-at' in args and '-b' in args: print 'Impossible to use -at mode with -b' exit() artificial_timestamps = '-at' in args f_temps = open_file(args[1]) batt_filename = get_option_parameters(args, '-b') mode = NAIVE_MODE mode_args = None if '-m' in args: mode_str = get_option_parameters(args, '-m') if mode_str == 'additive': mode = ADDITIVE_MODE mode_args = get_option_parameters(args, '-m', 5) if mode_args is None: raise GenericException("Missing additive parameters") del mode_args[0] elif mode_str == 'deadzone': mode = DEADZONE_MODE mode_args = get_option_parameters(args, '-m', 4) if mode_args is None: raise GenericException("Missing deadzone parameters") del mode_args[0] elif mode_str is None: raise GenericException("Could not parse arguments of -m") else: raise GenericException("Unexpected mode %s" % mode_str) output = open("output.txt", "w") if batt_filename: f_batt = open_file(batt_filename) else: f_batt = None try: basestation = BaseStation(output) topology_filename = get_option_parameters(args, '-t') if topology_filename: nodes = read_topology(topology_filename, basestation, mode, mode_args) else: ''' If a topology is not specified, the network will include the nodes as they appear in the file. ''' nodes = {} batt_info = None if f_batt: batt_line = f_batt.readline() if batt_line: batt_info = parse_line(batt_line) artificial_ts = {} for temp_line in f_temps: timestamp, node_id, counter, temperature = parse_line(temp_line) if artificial_timestamps: timestamp = artificial_ts.get(node_id, 0) if timestamp == 0: artificial_ts[node_id] = 1 else: artificial_ts[node_id] += 1 temp_reading = ReadingMessage(timestamp, node_id, temperature, counter, TEMP_READING) temp_node = nodes.get(node_id, None) if temp_node is None: if not topology_filename: temp_node = auto_update_topology(node_id, nodes, basestation, mode, mode_args) else: raise GenericException("Node %d is outside topology!" % node_id) # handle battery while batt_info and batt_info[0] < timestamp: # send another batt_info now batt_reading = ReadingMessage(batt_info[1], batt_info[0], batt_info[3], batt_info[2], VOLTAGE_READING) batt_node = nodes[batt_info[1]] batt_node.feed_reading(batt_reading) # update batt_line = f_batt.readline() if batt_line: batt_info = parse_line(batt_line) temp_node.feed_reading(temp_reading) finally: if f_temps: f_temps.close() if f_batt: f_batt.close() if output: output.close() for node in nodes.values(): print "Node %03d :" % (node.node_id), node.get_stats() print "All nodes: Sent: %d (1 message every %f seconds) Received: %d" % \ (sum( (n.sent_count for n in nodes.values()) ), \ sum( ((float(n.send_end - n.send_start) / n.sent_count) / len(nodes.values()) for n in nodes.values()) ), \ sum( (n.received_count for n in nodes.values()) ) + \ sum( (n.snooped_count for n in nodes.values()) ) )
def main(args): if len(args) < 2: print "Usage: python %s {hidden_vars_file} [-b (blame_file)]" % (args[0]) exit() hidden_vars = [[]] hv_file = open_file(args[1]) try: for line in hv_file: line = line.strip() if line: values = map(float, line.split()) for i in xrange(len(values) - len(hidden_vars) - 1): hidden_vars.append([]) for i in xrange(len(hidden_vars)): if i > len(values) - 2: hidden_vars[i].append((values[0], numpy.nan)) else: hidden_vars[i].append((values[0], values[i + 1])) finally: hv_file.close() blame_file = open_file(args[args.index('-b') + 1]) if '-b' in args else None weights = [] if blame_file: try: for line in blame_file: line = line.strip() if line: values = map(float, line.split()) if not weights: for i in xrange(len(values) - 1): # skip timestamp weights.append([]) for i in xrange(len(weights)): weights[i].append([values[0], values[i + 1]]) finally: blame_file.close() for w in weights: last = w[0][1] w[0][1] = 0 for i in xrange(1, len(w)): aux = w[i][1] w[i][1] = (w[i][1] - last) ** 2 last = aux size = 21 for w in weights: for i in xrange(0, len(w) - size, size): avg = sum(w[i + j][1] for j in xrange(size)) / size for j in xrange(size): w[i + j][1] = avg # filter poor motes cutoff = 5 * 1e-7 avgs = [] for w in weights: avg = sum(w[i][1] for i in xrange(len(w))) / len(w) avgs.append(avg) chosen_indexes = [] for i in xrange(len(avgs)): if avgs[i] > cutoff: chosen_indexes.append(i) # setup pylab #fig = pylab.figure() pylab.subplot(211) pylab.grid(True) for h in hidden_vars: x, y = zip(*h) pylab.plot(x, y, '-') pylab.legend(range(1, len(hidden_vars) + 1)) if weights: pylab.subplot(212) pylab.grid(True) for i in chosen_indexes: x, y = zip(*weights[i]) pylab.plot(x, y, '-') pylab.legend(chosen_indexes) pylab.show()