def fetch_html(self, date_item):
dpport = self.options['dpport']
arport = self.options['arport']
dpdate = date_item[0].strftime("%d/%m/%Y")
ardate = date_item[1].strftime("%d/%m/%Y")
cmpid = self.options['cmpid']
b2cpin = self.options['b2cpin']
html = fetch_html(dpport, arport, dpdate, ardate, cmpid, b2cpin)
rows = parse_html(html)
print("{0} ~ {1}".format(dpdate, ardate))
print("=========" * 5)
insert_rows = []
for row in rows:
print("{company_code:4s} {cabin:1s} {ticket_price:>8s} ({stay_day_min:>3s} ~ {stay_day_max:>3s}) ({valid_date_from:>10s} ~ {valid_date_to:>10s}) ({valid_buy_ticket_date_from:>10s} ~ {valid_buy_ticket_date_to:>10s}) {flight_info_link_cond_code:>15s}".format(
company_code=row['company_code'],
cabin=row['cabin'],
ticket_price=price_to_intenger(row['ticket_price']),
stay_day_min=row['stay_day_min'],
stay_day_max=row['stay_day_max'],
valid_date_from=row['valid_date_from'],
valid_date_to=row['valid_date_to'],
valid_buy_ticket_date_from=row['valid_buy_ticket_date_from'],
valid_buy_ticket_date_to=row['valid_buy_ticket_date_to'],
flight_info_link_cond_code=row['flight_info_link_cond_code']
))
insert_rows.append(dict(
departure_port=dpport,
arrival_port=arport,
company_code=row['company_code'],
cabin=row['cabin'],
ticket_price=price_to_intenger(row['ticket_price']),
stay_day_min=row['stay_day_min'],
stay_day_max=row['stay_day_max'],
valid_date_from=string_to_datetime(row['valid_date_from']),
valid_date_to=string_to_datetime(row['valid_date_to']),
valid_buy_ticket_date_from=string_to_datetime(row['valid_buy_ticket_date_from']),
valid_buy_ticket_date_to=string_to_datetime(row['valid_buy_ticket_date_to']),
flight_info_link=row['flight_info_link'],
flight_info_link_cond_code=row['flight_info_link_cond_code']
))
Flight.insert_many(insert_rows).execute()
sleep(2)
self.date_queue.task_done()
def plot_cpu(input_file, axes):
reader = csv.reader(input_file)
times = []
user = []
system = []
idle = []
waiting = []
for row in reader:
if len(row) == 0:
continue
if is_datetime(row[0]):
times.append(string_to_datetime(row[0]))
user.append(float(row[1]))
system.append(float(row[2]))
idle.append(float(row[3]))
waiting.append(float(row[4]))
del reader
first_time = times[0]
delta_times = [ (time - first_time).seconds for time in times]
user_sys = [ user[i] + system[i] for i in xrange(len(user)) ]
usw = [user_sys[i] + waiting[i] for i in xrange(len(user)) ]
#axes.plot(delta_times, user, color='#22AA22', linewidth=2)
axes.plot(delta_times, user_sys, color='#22AA22', linewidth=2)
axes.plot(delta_times, usw, color='#66FF66')
axes.set_ylabel('CPU Utilization (%)')
axes.set_xlabel('Time (s)')
axes.set_ylim(top=100)
def obtener_reportes_disponibles(retry=0):
print("Obteniendo reportes disponibles. Intento", retry)
try:
# Descargar página del SEREMI.
html = requests.get(URL_REPORTES)
# Cargar página a bs.
soup = BeautifulSoup(html.content, "html.parser")
# Buscar tag section que es parent de los links a los reportes.
section = soup.find("section", class_="body")
# Buscar todos los tags a
links = section.find_all("a")
# Iniciar lista para guardar los reportes disponibles.
reportes = []
for a in links:
# Obtener fecha del reporte
date = string_to_datetime(a.contents[0])
if date >= START_DATE:
link = URL_BASE + a["href"]
reportes.append(Reporte(date, link))
print(Reporte(date, link))
return reportes
except Exception as e:
print("Error al obtener la lista de informes")
print(e)
if retry < 10:
return obtener_reportes_disponibles(retry + 1)
else:
return []
def generate_new_event(self, sched):
busy_times = self.get_busy_times()
owner_obj = User.objects.get(email=self.catchup_owner)
time_zone = util.get_user_timezone(owner_obj)
event = StoredEvent.get_random_event()
event_dates = [
util.string_to_datetime(date_str, time_zone)
for date_str in event.preferred_times
]
start_date, end_date = event.find_free_date(event_dates, busy_times)
if start_date:
print(start_date)
print(end_date)
self.schedule_event(event, start_date, end_date, sched)
def plot_disk(input_file, axes):
reader = csv.reader(input_file)
times = []
read = []
write = []
for row in reader:
if len(row) == 0:
continue
if is_datetime(row[0]):
times.append(string_to_datetime(row[0]))
read.append(float(row[1]))
write.append(float(row[2]))
del reader
first_time = times[0]
delta_times = [ (time - first_time).seconds for time in times]
axes.plot(delta_times, read, color='#AAAAFF', linewidth=2)
axes.plot(delta_times, write, color='#000077', linewidth=2)
axes.set_ylabel('Disk Activity')
axes.set_xlabel('Time (s)')
def plot_net(input_file, axes):
reader = csv.reader(input_file)
times = []
recv = []
send = []
for row in reader:
if len(row) == 0:
continue
if is_datetime(row[0]):
times.append(string_to_datetime(row[0]))
recv.append(float(row[1]))
send.append(float(row[2]))
del reader
first_time = times[0]
delta_times = [ (time - first_time).seconds for time in times]
axes.plot(delta_times, recv, color='#CCCC00', linewidth=2)
axes.plot(delta_times, send, color='#FF0000', linewidth=2)
axes.set_ylabel('Network Activity')
axes.set_xlabel('Time (s)')
required=True)
parser.add_argument('--length',
type=int,
dest='length',
action='store',
required=True)
parser.add_argument('--output',
nargs='?',
type=argparse.FileType('w'),
default=sys.stdout,
dest='output',
action='store')
args = parser.parse_args()
start_time = string_to_datetime(args.start_time)
end_time = start_time + datetime.timedelta(seconds=args.length)
writer = csv.writer(args.output, delimiter=',', quotechar='"')
reader = csv.reader(args.input_file, delimiter=',', quotechar='"')
for row in reader:
if len(row) == 0:
continue
if row[0] in ['system', 'time', 'Host:']:
writer.writerow(row)
elif is_datetime(row[0]):
cur_date = string_to_datetime(row[0])
if cur_date >= start_time and cur_date <= end_time:
writer.writerow(row)
import csv
import re
import sys
import argparse
import datetime
from util import string_to_datetime, is_datetime
parser = argparse.ArgumentParser()
parser.add_argument('input_file', nargs='?', type=argparse.FileType('r'), default=sys.stdin, action='store')
parser.add_argument('--start-time', type=str, dest='start_time', action='store', required=True)
parser.add_argument('--length', type=int, dest='length', action='store', required=True)
parser.add_argument('--output', nargs='?', type=argparse.FileType('w'), default=sys.stdout, dest='output', action='store')
args = parser.parse_args()
start_time = string_to_datetime(args.start_time)
end_time = start_time + datetime.timedelta(seconds=args.length)
writer = csv.writer(args.output, delimiter=',', quotechar='"')
reader = csv.reader(args.input_file, delimiter=',', quotechar='"')
for row in reader:
if len(row) == 0:
continue
if row[0] in ['system', 'time', 'Host:']:
writer.writerow(row)
elif is_datetime(row[0]):
cur_date = string_to_datetime(row[0])
if cur_date >= start_time and cur_date <= end_time:
writer.writerow(row)
import matplotlib.pyplot as pyplot
import csv
import datetime
from util import string_to_datetime, is_datetime
reader = csv.reader(args.input_file)
times = []
recv = []
send = []
for row in reader:
if len(row) == 0:
continue
if is_datetime(row[0]):
times.append(string_to_datetime(row[0]))
recv.append(float(row[1]))
send.append(float(row[2]))
del reader
first_time = times[0]
delta_times = [ (time - first_time).seconds for time in times]
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot(delta_times, recv, color='#5555FF', linewidth=2)
ax.plot(delta_times, send, color='#00FF00', linewidth=2)
ax.set_ylabel('Network Activity')
ax.set_xlabel('Time (s)')
import matplotlib.pyplot as pyplot
import csv
import datetime
from util import string_to_datetime, is_datetime
reader = csv.reader(args.input_file)
times = []
recv = []
send = []
for row in reader:
if len(row) == 0:
continue
if is_datetime(row[0]):
times.append(string_to_datetime(row[0]))
recv.append(float(row[1]))
send.append(float(row[2]))
del reader
first_time = times[0]
delta_times = [(time - first_time).seconds for time in times]
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot(delta_times, recv, color='#5555FF', linewidth=2)
ax.plot(delta_times, send, color='#00FF00', linewidth=2)
ax.set_ylabel('Network Activity')
ax.set_xlabel('Time (s)')
for row in cur_reader: if len(row) == 0: continue if row[0] == 'time': #This row has the column headings! desired_col_idx = [ row.index(col_name) for col_name in desired_col_names ] break elif (not got_hostname) and row[0] == 'Host:': # This row tells us which computer this is host_list.append(row[1]) got_hostname = True # Read through the rest of the document and take out the data we want for row in cur_reader: if len(row) == 0: continue if is_datetime(row[0]): cur_time = string_to_datetime(row[0]) if cur_time not in time_buckets: time_buckets[cur_time] = [] time_buckets[cur_time] += [ row[idx] for idx in desired_col_idx ] host_headings = ['Hosts'] for hostname in host_list: host_headings += [hostname] + [None]*(len(desired_col_names)-1) writer.writerow(host_headings) writer.writerow(['system time'] + desired_col_names * len(host_list)) sorted_keys = time_buckets.keys() sorted_keys.sort() for key in sorted_keys: writer.writerow([key] + time_buckets[key])
desired_col_idx = [
row.index(col_name) for col_name in desired_col_names
]
break
elif (
not got_hostname
) and row[0] == 'Host:': # This row tells us which computer this is
host_list.append(row[1])
got_hostname = True
# Read through the rest of the document and take out the data we want
for row in cur_reader:
if len(row) == 0:
continue
if is_datetime(row[0]):
cur_time = string_to_datetime(row[0])
if cur_time not in time_buckets:
time_buckets[cur_time] = []
time_buckets[cur_time] += [row[idx] for idx in desired_col_idx]
host_headings = ['Hosts']
for hostname in host_list:
host_headings += [hostname] + [None] * (len(desired_col_names) - 1)
writer.writerow(host_headings)
writer.writerow(['system time'] + desired_col_names * len(host_list))
sorted_keys = time_buckets.keys()
sorted_keys.sort()
for key in sorted_keys:
writer.writerow([key] + time_buckets[key])
