def add_new_maint(self, msg, email):
maint = Maintenance()
provider_id = re.search('(?<=pw reference number: )\S+', msg.lower())
maint.provider_maintenance_id = provider_id.group()
start_time = re.search('(?<=start date and time: ).+', msg.lower())
end_time = re.search('(?<=end date and time: ).+', msg.lower())
reason = re.search('(?<=action and reason: ).+', msg.lower())
location = re.search('(?<=location of work: ).+', msg.lower())
cids = re.findall('service id: (.*)\r', msg.lower())
impact = re.findall('impact: (.*)\r', msg.lower())
if not all((provider_id, start_time, end_time, reason, location, cids,
impact)):
raise ParsingError(
'Unable to parse the maintenance notification from Telia: {}'.
format(msg))
start_dt = datetime.datetime.strptime(start_time.group().rstrip(),
'%Y-%b-%d %H:%M %Z')
start_dt = start_dt.replace(tzinfo=pytz.utc)
end_dt = datetime.datetime.strptime(end_time.group().rstrip(),
'%Y-%b-%d %H:%M %Z')
end_dt = end_dt.replace(tzinfo=pytz.utc)
maint.start = start_dt.time()
maint.end = end_dt.time()
maint.timezone = start_dt.tzname()
maint.reason = reason.group()
received = email['Received'].splitlines()[-1].strip()
maint.received_dt = parser.parse(received)
maint.location = location.group().rstrip()
self.add_and_commit(maint)
NEW_PARENT_MAINT.labels(provider=self.name).inc()
cids = re.findall('service id: (.*)\r', msg.lower())
impact = re.findall('impact: (.*)\r', msg.lower())
all_circuits = list(zip(cids, impact))
for cid, impact in all_circuits:
if not Circuit.query.filter_by(provider_cid=cid).first():
c = Circuit()
c.provider_cid = cid
this = Pro.query.filter_by(name=self.name,
type=self.type).first()
c.provider_id = this.id
self.add_and_commit(c)
circuit_row = Circuit.query.filter_by(provider_cid=cid).first()
maint_row = Maintenance.query.filter_by(
provider_maintenance_id=maint.provider_maintenance_id,
rescheduled=0).first()
mc = MaintCircuit(impact=impact, date=start_dt.date())
circuit_row.maintenances.append(mc)
mc.maint_id = maint_row.id
db.session.commit()
NEW_CID_MAINT.labels(cid=cid).inc()
return True
def add_new_maint(self, email, cal):
'''
add a new maintenance to the db
email: email.email object
cal: icalendar object
'''
maint = Maintenance()
maint.provider_maintenance_id = cal['X-MAINTNOTE-MAINTENANCE-ID'] \
.strip()
current_app.logger.info(
f'adding {maint.provider_maintenance_id} to db')
maint.start = cal['DTSTART'].dt.time()
maint.end = cal['DTEND'].dt.time()
maint.timezone = cal['DTSTART'].dt.tzname()
# not all ics attachments have descriptions
if cal.get('DESCRIPTION'):
maint.reason = cal['DESCRIPTION'].strip()
received = email['Received'].splitlines()[-1].strip()
maint.received_dt = parser.parse(received)
self.add_and_commit(maint)
current_app.logger.info(
f'added {maint.provider_maintenance_id} successfully')
NEW_PARENT_MAINT.labels(provider=self.name).inc()
cids = []
if type(cal['X-MAINTNOTE-OBJECT-ID']) == list:
# more than one circuit is affected
for cid in cal['X-MAINTNOTE-OBJECT-ID']:
cids.append(cid)
else:
# this is only for one circuit
cids.append(cal['X-MAINTNOTE-OBJECT-ID'])
for cid in cids:
if not Circuit.query.filter_by(provider_cid=cid).first():
self.add_circuit(cid)
circuit_row = Circuit.query.filter_by(provider_cid=cid).first()
maint_row = Maintenance.query.filter_by(
provider_maintenance_id=maint.provider_maintenance_id,
rescheduled=0).first()
mc = MaintCircuit(impact=cal['X-MAINTNOTE-IMPACT'].strip(),
date=cal['DTSTART'].dt.date())
circuit_row.maintenances.append(mc)
mc.maint_id = maint_row.id
db.session.commit()
NEW_CID_MAINT.labels(cid=cid).inc()
return True
def client():
a = create_app(TestConfig)
a.apscheduler.scheduler.shutdown()
db_fd, a.config['DATABASE_FILE'] = tempfile.mkstemp()
client = a.test_client()
a.before_first_request_funcs = []
with a.app_context():
db.create_all()
for provider in PROVIDERS:
p = provider()
circuit = Circuit(provider_cid='xxx',
a_side='a',
z_side='z',
provider_id=1)
db.session.add(circuit)
db.session.commit()
now = datetime.now()
maint = Maintenance(
provider_maintenance_id='pmaintid',
start=now.time(),
end=now.time(),
timezone='US/Eastern',
received_dt=now,
)
mc = MaintCircuit(impact='yuge', date=now.date())
circuit.maintenances.append(mc)
mc.maint_id = maint.id
db.session.add(maint)
db.session.commit()
yield client
with a.app_context():
db.drop_all()
os.close(db_fd)
os.unlink(a.config['DATABASE_FILE'])
def add_new_maint(self, soup, email):
maint_id = self.get_maint_id(email)
if not maint_id:
return False
maint = Maintenance()
maint.provider_maintenance_id = maint_id
received = email['Received'].splitlines()[-1].strip()
maint.received_dt = parser.parse(received)
start_re = re.search(r'Start: (.*)(\r|\n)', soup.text)
end_re = re.search(r'End: (.*)(\r|\n)', soup.text)
location_re = re.search(r'Location: (.*)(\r|\n)', soup.text)
reason_re = re.search(r'Reason: (.*)(\r|\n)', soup.text)
impact_re = re.search(r'Impact: (.*)(\r|\n)', soup.text)
impact = impact_re.groups()[0]
start_dt = parser.parse(start_re.groups()[0])
end_dt = parser.parse(end_re.groups()[0])
maint.start = start_dt.time()
maint.end = end_dt.time()
maint.timezone = start_dt.tzname()
maint.location = location_re.groups()[0]
maint.reason = reason_re.groups()[0]
if not all((start_re, end_re, location_re, reason_re, impact_re)):
raise ParsingError(
'Unable to parse the maintenance notification from GTT: {}'.
format(soup.text))
self.add_and_commit(maint)
NEW_PARENT_MAINT.labels(provider=self.name).inc()
# sometimes maint emails contain the same cid several times
cids = set()
a_side = set()
for line in soup.text.splitlines():
if 'gtt service' in line.lower():
cid = re.search(r'GTT Service = (.+);', line)
if cid:
cids.add(cid.groups()[0])
elif line.lower().startswith('site address'):
loc = re.search(r'= (.*)', line)
if loc:
a_side.add(loc.groups()[0])
# there is sometimes two location lines with
# = ostensibly being the circuit location
elif line.lower().startswith('location ='):
loc = re.search(r'= (.*)', line)
if loc:
a_side.add(loc.groups()[0])
if len(cids) == len(a_side):
for cid, a_side in zip(cids, a_side):
if not Circuit.query.filter_by(provider_cid=cid).first():
circuit = Circuit()
circuit.provider_cid = cid
circuit.a_side = a_side
this = Pro.query.filter_by(name=self.name,
type=self.type).first()
circuit.provider_id = this.id
self.add_and_commit(circuit)
circuit_row = Circuit.query.filter_by(provider_cid=cid).first()
maint_row = Maintenance.query.filter_by(
provider_maintenance_id=maint.provider_maintenance_id,
rescheduled=0).first()
mc = MaintCircuit(impact=impact, date=start_dt.date())
circuit_row.maintenances.append(mc)
mc.maint_id = maint_row.id
db.session.commit()
NEW_CID_MAINT.labels(cid=cid).inc()
return True
def add_new_maint(self, soup, email):
'''
zayo bolds the relevant fields so we use bs4 to search for those
and then get the next sibling
'''
current_app.logger.info(
f'attempting to add new maint from email {email["Subject"]}')
table = soup.find('table')
if not table:
subject = email['Subject']
return False
maint = Maintenance()
dates = []
for line in soup.find_all('b'):
if type(line) == bs4.element.Tag:
if line.text.lower().strip().endswith('activity date:'):
dt = parser.parse(self.clean_line(line.next_sibling))
t = datetime.date(dt.year, dt.month, dt.day)
dates.append(t)
if line.text.lower().strip().startswith('maintenance ticket'):
maint.provider_maintenance_id = self.clean_line(
line.next_sibling)
# elif 'urgency' in line.text.lower():
# row_insert['urgency'] = self.clean_line(line.next_sibling)
elif 'location of maintenance' in line.text.lower():
maint.location = self.clean_line(line.next_sibling)
elif 'maintenance window' in line.text.lower():
window = line.next_sibling.strip().split('-')
window = [time.strip() for time in window]
start = window.pop(0)
start = parser.parse(start)
maint.start = datetime.time(start.hour, start.minute)
window = window[0].split()
end = window.pop(0)
end = parser.parse(end)
maint.end = datetime.time(end.hour, end.minute)
if len(window) == 1:
if current_app.config['TZ_PREFIX']:
# zayo will send timezones such as "Eastern"
# instead of "US/Eastern" so the tzinfo
# may not be able to be parsed without a prefix
tz = window.pop()
pfx = current_app.config['TZ_PREFIX']
if tz != 'GMT':
maint.timezone = pfx + tz
else:
maint.timezone = tz
else:
maint.timezone = window.pop()
else:
# failsafe
maint.timezone = ' '.join(window)
elif 'reason for maintenance' in line.text.lower():
maint.reason = self.clean_line(line.next_sibling)
received = email['Received'].splitlines()[-1].strip()
maint.received_dt = parser.parse(received)
self.add_and_commit(maint)
current_app.logger.info(
f'maintenance {maint.provider_maintenance_id} added successfully')
NEW_PARENT_MAINT.labels(provider=self.name).inc()
cid_table = self.format_circuit_table(table)
for row in cid_table.values:
if not Circuit.query.filter_by(provider_cid=row[0]).first():
current_app.logger.info(f'adding circuit {row[0]}')
circuit = Circuit()
circuit.provider_cid = row[0]
if str(row[2]) == 'nan':
circuit.a_side = None
else:
circuit.a_side = row[2]
if str(row[3]) == 'nan':
circuit.z_side = None
else:
circuit.z_side = row[3]
this = Pro.query.filter_by(name=self.name,
type=self.type).first()
circuit.provider_id = this.id
self.add_and_commit(circuit)
current_app.logger.info(f'circuit {row[0]} added successfully')
circuit_row = Circuit.query.filter_by(provider_cid=row[0]).first()
maint_row = Maintenance.query.filter_by(
provider_maintenance_id=maint.provider_maintenance_id,
rescheduled=0).first()
for date in dates:
current_app.logger.info(
f'adding maint_circuit row for {maint_row.provider_maintenance_id}'
)
mc = MaintCircuit(impact=row[1], date=date)
circuit_row.maintenances.append(mc)
mc.maint_id = maint_row.id
db.session.commit()
current_app.logger.info(
f'maint_circuit row for {maint_row.provider_maintenance_id} added successfully'
)
NEW_CID_MAINT.labels(cid=circuit_row.provider_cid).inc()
return True
def add_new_maint(self, soup, email):
'''
create a new telstra maintenance
'''
maint = Maintenance()
maint.provider_maintenance_id = email['Subject'].split()[-1]
maint.location = 'n/a' # telstra doesn't give this info :(
maint.reason = ''
received = email['Received'].splitlines()[-1].strip()
maint.received_dt = parser.parse(received)
headers = soup.findAll('th')
impact = None
cid = None
date = None
for column in headers:
if 'expected impact' in self.clean_line(column.text.lower()):
impact = self.clean_line(column.next_sibling.next_sibling.text)
elif 'service(s) impacted' in self.clean_line(column.text.lower()):
cid = self.clean_line(column.next_sibling.next_sibling.text)
elif 'maintenance window' in self.clean_line(column.text.lower()):
date = self.clean_line(column.next_sibling.next_sibling.text)
if not all((impact, cid, date)):
raise ParsingError(
f'unable to parse telstra impact: {impact}, cid: {cid}, date: {date} subject: {email["Subject"]}'
)
fullstart, fullend = date.split(' to ')
datestart, timestart = fullstart.split()
dateend, timeend = fullend.split()
startdate = parser.parse(datestart).date()
timematch = re.compile(r'^\d+:\d+(?=[:00])?')
starttime = timematch.search(timestart).group()
endtime = timematch.search(timeend).group()
start = parser.parse(starttime).time()
end = parser.parse(endtime).time()
maint.start = start
maint.end = end
tzmatch = re.compile(r'\((\w+)\)')
tz = tzmatch.search(timestart).groups()[0]
maint.timezone = tz
# grab maintenance details. This is not pretty
details = []
for i in soup.find_all('tr'):
if 'maintenance details' in i.text.lower():
det = i
details = det.findNextSiblings('tr')
break
for line in details:
if 'service(s) impacted' in line.text.lower():
break
maint.reason += clean_line(line.text)
maint.reason += ' '
self.add_and_commit(maint)
NEW_PARENT_MAINT.labels(provider=self.name).inc()
# add the circuit if it doesn't exist
circuit = Circuit.query.filter_by(provider_cid=cid).first()
if not circuit:
circuit = Circuit()
circuit.provider_cid = cid
circuit.a_side = ''
circuit.z_side = ''
this = Pro.query.filter_by(name=self.name, type=self.type).first()
circuit.provider_id = this.id
self.add_and_commit(circuit)
# add the maint_circuit row
maint_row = Maintenance.query.filter_by(
provider_maintenance_id=maint.provider_maintenance_id,
rescheduled=0).first()
mc = MaintCircuit(impact=impact, date=startdate)
circuit.maintenances.append(mc)
mc.maint_id = maint_row.id
db.session.commit()
NEW_CID_MAINT.labels(cid=circuit.provider_cid).inc()
return True