def scan(sitelist):
i = 0
l = helpers.file_len(sitelist)
print('scan commencing...')
helpers.printProgress(i, l, prefix = "Scan Progress:", suffix = "Complete", barLength = 50)
with open(sitelist) as f:
for index, line in enumerate(f):
chomp_line = line.rstrip()
wp_cmd = "./wpscan/wpscan.rb --update --batch --url " + chomp_line + " --enumerate vp"
cmsmap_cmd = "./CMSmap/cmsmap.py -t " + chomp_line + " -o sites/cmsmap_" + chomp_line + "_log.txt"
run("wp", chomp_line, wp_cmd)
run("cms", chomp_line, cmsmap_cmd)
os.remove("./sites/" + chomp_line + "_cms.txt")
i += 1
helpers.printProgress(i, l, prefix = "Scan Progress:", suffix = "Complete", barLength = 50)
print('scan completed.')
def run(path):
logger.info("Starting execution of file_lengths.py!")
event_counts = []
file_sizes = []
files = [f for f in os.listdir(path) if ".log" in f]
T = len(files)
percent = T // 100 if T >= 100 else 1
for i, file_name in enumerate(files):
if (i + 1) % percent == 0 or (i + 1) == T:
hlp.print_progress_bar(i + 1, T, length=50)
event_counts.append(hlp.file_len(path + file_name))
file_sizes.append(os.path.getsize(path + file_name) / 1e9)
logger.info("Total Size: %sGB", sum(file_sizes))
fig = plt.figure()
plt.subplot(121)
plt.boxplot(event_counts, showmeans=True)
plt.title("Number of Events")
plt.subplot(122)
plt.boxplot(file_sizes, showmeans=True)
plt.title("Data Log Sizes (GB)")
fig.suptitle("Measurements on {} Logs".format(len(event_counts)))
fig.subplots_adjust(left=None,
bottom=None,
right=None,
top=None,
wspace=None,
hspace=0.3)
plt.savefig("log_sizes.png")
def __init__(self, file, printer, callback):
# this constructor could be a bit more elegant
super(GCodePrintThread, self).__init__()
self.file = file
self.printer = printer
self.callback = callback
#let's see how many lines the print file got and reset progress
self.lines_to_print = file_len(file)
self.lines_printed = 0
self.printing = False
def __init__(self, file, printer, callback):
# this constructor could be a bit more elegant
super(GCodePrintThread, self).__init__()
self.file = file
self.printer = printer
self.callback = callback
#let's see how many lines the print file got and reset progress
self.lines_to_print = file_len(file)
self.lines_printed = 0
self.printing = False
def read_events_from_file_old(file_name,
vehicle_id=None,
max_events=None,
create_timeline=False,
filter_vehicle="Bus",
group_by_id=True):
"""Functions that takes a file_name and returns events in the form of key-value objects.
Parameter "vehicle_id" is an optional parameter that is used to filter
results to only contain events for the given vehicle_id.
Returns a list of objects containing data in a key-value format.
"""
if group_by_id:
events = defaultdict(dict)
else:
events = defaultdict(list)
timeline_events = []
if max_events is None:
logger.info("Calculating number of events in file...")
T = file_len(file_name)
logger.info("File has %i events", T)
else:
T = max_events
percent = T // 100
with open(file_name, 'r', encoding="latin-1") as f:
for i in range(T):
if (i + 1) % percent == 0 or (i + 1) == T:
print_progress_bar(i + 1,
T,
prefix='Progress:',
suffix='Complete',
length=50)
event = parse_event(f.readline(), filter_vehicle)
if event is None:
continue
event_type = event["event.type"]
event_v_id = event["vehicle.id"]
if vehicle_id is None or vehicle_id == event_v_id:
if group_by_id:
if not event_type in events[event_v_id]:
events[event_v_id][event_type] = []
events[event_v_id][event_type].append(event)
else:
events[event_type].append(event)
if create_timeline:
if not timeline_events or timeline_events[len(
timeline_events) - 1]["event.type"] != event_type:
timeline_events.append(event)
return events, timeline_events
def read_events_from_file(file_name,
skip_n=0,
max_events=None,
geofence=None,
vehicle_id=None):
"""Opens a file containing events and parses them.
Checks if a journey has begun and saves all the position updates from the bus on that journey.
Bus stops stopped at or passed are also recorded.
"""
if max_events is None:
logger.info("Calculating number of events in file...")
T = file_len(file_name)
logger.info("File has %i events", T)
T -= skip_n
else:
T = max_events
percent = T // 100
if vehicle_id is not None and not isinstance(vehicle_id, list):
vehicle_id = [vehicle_id]
events = defaultdict(list)
with open(file_name, 'r', encoding="latin-1") as f:
logger.info("Skipping %s events", skip_n)
for _ in range(skip_n):
f.readline()
for i in range(T):
if (i + 1) % percent == 0 or (i + 1) == T:
print_progress_bar(i + 1,
T,
prefix='Progress:',
suffix='Complete',
length=50)
event = parse_event(f.readline())
if event is None:
continue
if vehicle_id is not None and event["vehicle.id"] not in vehicle_id:
continue
if geofence is not None and not is_inside(event, *geofence):
continue
assert isinstance(event["date"], datetime.datetime)
events[event["vehicle.id"]].append(event)
for k, v in events.items():
v.sort(key=lambda e: e["event.id"])
return events
def get_event_types_list(file_name, max_events=None):
if max_events is None:
logger.info("Calculating number of events in file...")
T = file_len(file_name)
logger.info("File has %i events", T)
else:
T = max_events
percent = T // 100
types = []
with open(file_name, 'r', encoding="latin-1") as f:
for i in range(T):
if (i + 1) % percent == 0 or (i + 1) == T:
print_progress_bar(i + 1,
T,
prefix='Progress:',
suffix='Complete',
length=50)
event_type = get_event_type(f.readline())
types.append(event_type)
return types
"../../data/prepared/artist_info.txt")
artists_file_path = os.path.join(dir, "../../data/prepared/artists.txt")
helpers.create_file_if_not_exists(covered_artists_file_path)
helpers.create_file_if_not_exists(artist_info_file_path)
artist_amount = 10000
counter = 0
covered_artists = helpers.read_lines_from_file(covered_artists_file_path)
artist_names = helpers.read_lines_from_file(artists_file_path)
covered_artists_file = open(covered_artists_file_path, "a")
artist_info_file = open(artist_info_file_path, "a")
covered_artists_amount = len(covered_artists)
found_artists_amount = helpers.file_len(artist_info_file_path)
helpers.startProgress(" Fetching next {0} Artists ".format(artist_amount),
" Artists covered: " + str(covered_artists_amount),
" Artists found: " + str(found_artists_amount))
for artist in artist_names:
if counter >= artist_amount:
break
if artist in covered_artists:
covered_artists.remove(artist)
continue
artist = artist.strip()
query = artist.replace(" ", "+")
query = urllib2.quote(query, safe='/')
url = helpers.get_spotify_api_url("search",
sys.setdefaultencoding('utf8')
# Filepaths
fetched_artists_from_lastfm_file_path = os.path.join(
dir, "../data/prepared/artists.txt")
fetched_artists_from_spotify_file_path = os.path.join(
dir, "../data/prepared/artist_info.txt")
fetched_top_tracks_artists_file_path = os.path.join(
dir, "../data/assist/covered_top_tracks_artists.txt")
fetched_artists_features_file_path = os.path.join(
dir, "../data/assist/covered_artists_features.txt")
cleaned_artists_file_path = os.path.join(
dir, "../data/prepared/cleaned_artists.txt")
# Read amounts
fetched_artists_last_fm = helpers.file_len(
fetched_artists_from_lastfm_file_path)
artists_spotify = helpers.file_len(fetched_artists_from_spotify_file_path)
artists_top_tracks = helpers.file_len(fetched_top_tracks_artists_file_path)
artists_tracks_features = helpers.file_len(fetched_artists_features_file_path)
artists_cleaned = helpers.file_len(cleaned_artists_file_path)
print "#" * 60
print ""
print " " * 23 + "\033[1;36m STATISTICS \033[0;0m"
print ""
print " Fetched Artists from Last FM Charts: \033[0;36m{0}\033[0;0m".format(
fetched_artists_last_fm)
print " Fetched Artists from Spotify (from last FM Charts + related): \033[0;36m{0}\033[0;0m".format(
artists_spotify)
print " Cleaned Artists from Spotify: \033[0;36m{0}\033[0;0m".format(