def run(self):
if 'timer' in self.mission.trigger.keys():
interval = self.mission.trigger['timer']
while not self.lock.wait(timeout=interval):
m_id = self.mission.get_id()
time = datetime.datetime.now(datetime.timezone.utc).isoformat()
works = []
for sid, w in self.leader.work_cache:
if w.purpose != m_id:
continue
works.append({
"time": w.time,
"place": self.leader.subordinates[sid].place,
"values": w.values
})
report = Report(time, self.leader.place, self.mission.purpose,
works)
url = "{0}subordinates/{1}/report".\
format(self.leader.superior_ep, self.leader.id)
res, err = rest.post(url, json=report.to_dict())
if err is not None:
# TODO: エラー処理ちゃんとやる
# 本当に接続先がダウンしてる場合、ただのDoSになってしまう
logger.error('In WorkingThread, failed to post report')
logger.error('> err: {0}', err)
self.leader.work_cache = \
[(sid, w) for sid, w in self.leader.work_cache
if w.purpose != m_id]
else:
pass
def get_all_report(conn):
"""
This function returns a list of reports with one execution instance (the last one if any)
Args:
conn (Connection) : DB connection to use
Return:
[Report] : A list of reports (not all dependencies are instantiated)
"""
conn.row_factory = sqlite3.Row
cursor = conn.cursor()
reports = []
sql_get_all_report = 'select name, max(execution_date) as last_execution from Report left join Execution ' \
'on Execution.report_id = Report.id ;'
query_result = cursor.execute(sql_get_all_report).fetchall()
for row in query_result:
report = Report(str(row[0]))
if row[1]:
report.execution.append(Execution(parser.parse(row[1])))
reports.append(report)
return reports
def test_submit_report(self):
# add leader
leader = LeaderInfo(id='lxxx0',
name='lea_http',
place="desk",
endpoint='http://localhost:50000',
subordinates=[],
missions=[])
self.app.post('/commander/subordinates',
data=json.dumps(leader.to_dict()),
content_type='application/json')
# submit a report
report = Report(purpose="some app",
time=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
place="desk",
values="some values")
response = self.app.post('/commander/subordinates/lxxx0/report',
data=json.dumps(report.to_dict()),
content_type='application/json')
self.assertEqual(response.status_code, 200)
actual = json.loads(response.data.decode("utf-8"))
# assert
expected = {
"_status": {
'success': True,
'msg': "status is ok"
},
"accepted": report.to_dict()
}
self.assertEqual(actual, expected)
def populate_report_table(): # Will read from report list, if items are missing we will replace with items below after analysis # Deletes all table rows to assure no dublicates Report.query.delete() db.session.commit() scope1_total = 0 scope1_net = 0 scope1_offsets = 0 scope3_total = 0 scope3_net = 0 scope3_offsets = 0 scope2_market_total = 0 scope2_market_net = 0 scope2_market_offsets = 0 scope2_location_total = 0 scope2_location_net = 0 scope2_location_offsets = 0 for report in reports: name_of_company = report['name'] year = report['year'] co_id = Company.query.filter_by(name=name_of_company).first().co_id verification_body = report['verification_body'] level_of_assur = report['level_of_assur'] gwp_stand = report['gwp_stand'] reporting_protocol = report['reporting_protocol'] for scope in report['scopes']: if scope['scope'] == 'Scope 1': scope1_total = scope['scope_total'] scope1_net = scope['scope_net'] scope1_offsets = scope['scope_offsets'] elif scope['scope'] == 'Optional': scope3_total = scope['scope_total'] scope3_net = scope['scope_net'] scope3_offsets = scope['scope_offsets'] else: if "Market" in scope['name']: scope2_market_total = scope['scope_total'] scope2_market_net = scope['scope_net'] scope2_market_offsets = scope['scope_offsets'] elif "Location" in scope['name']: scope2_location_total = scope['scope_total'] scope2_location_net = scope['scope_net'] scope2_location_offsets = scope['scope_offsets'] rep = Report(year=year, co_id=co_id, verification_body=verification_body, level_assurance=level_of_assur, gwp_standard=gwp_stand, rep_protocol=reporting_protocol, scope1_total=scope1_total, scope1_net=scope1_net, scope1_offsets=scope1_offsets, scope2_market_total=scope2_market_total, scope2_market_net=scope2_market_net, scope2_market_offsets=scope2_market_offsets, scope2_location_total=scope2_location_total, scope2_location_net=scope2_location_net, scope2_location_offsets=scope2_location_offsets, scope3_total=scope3_total, scope3_net=scope3_net, scope3_offsets=scope3_offsets) db.session.add(rep) db.session.commit()
def get_report_by_name(report_name, conn, execution_id=None):
cursor = conn.cursor()
sql_report_by_name = "select id, name, report_query, mode, file_name, separator from Report where name = '%s';"
sql_report_columns_by_id = "select sql_name, business_name, is_used_for_compare, is_business_key " \
"from Report_Columns where report_id = '%s';"
# Fetch report information from persistence
print "Info : Retrieving report from the database " + now()
query_result = cursor.execute(sql_report_by_name % report_name).fetchone()
if not query_result:
print("Error : report with name '%s' not found" % report_name)
exit(1)
report_id = query_result["id"]
report_name = query_result["name"]
report_query = query_result["report_query"]
report_mode = query_result["mode"]
report_file_name = query_result["file_name"]
report_separator = query_result["separator"]
# Fetch column definitions
query_result = cursor.execute(sql_report_columns_by_id %
report_id).fetchall()
columns_mapping = {}
columns = []
for row in query_result:
columns.append(row["sql_name"])
columns_mapping[row["sql_name"]] = {
"business_name": row["business_name"]
}
columns_mapping[row["sql_name"]]["is_used_for_compare"] = str(
row["is_used_for_compare"])
columns_mapping[row["sql_name"]]["is_business_key"] = str(
row["is_business_key"])
report = Report(report_name, report_query, report_mode, columns,
columns_mapping, report_separator, report_file_name,
report_id)
# Retrieve last Execution or the one requested bw user
if not execution_id:
execution = get_execution_by_report_id(report_id, conn)
else:
execution = get_execution_by_report_id(report_id, conn, execution_id)
if execution:
report.execution = execution
else:
print "Warning : empty exectution"
report.execution = []
print "Info : Retrieval done " + now()
return report
def submit_error(self, msg):
time = datetime.datetime.now(datetime.timezone.utc).isoformat()
report = Report(time=time,
place="internal",
purpose="_error",
values=[{
"type": "error_msg",
"msg": msg
}])
url = "{0}subordinates/{1}/report".format(self.superior_ep, self.id)
rest.post(url, json=report.to_dict())
def report_bathroom_by_id(self, id, description):
if id is None or not description:
return False
b = Bathroom.query.filter_by(id=id).first()
if b is None:
return False
rep = Report()
rep.bathroom = b
rep.description = description
db.session.add(rep)
db.session.commit()
return True
def create_report(result):
"""Create and return a new user."""
url = result['url']
is_vulnerable = result['is_vulnerable']
if is_vulnerable == False:
report = Report(url=url, is_vulnerable=is_vulnerable)
else:
url = result['url']
result['is_vulnerable']
exploit = result['exploit']
field_name = result['field_name']
form_type = result['form_type']
method = result['method']
report = Report(url=url,
is_vulnerable=is_vulnerable,
exploit=exploit,
field_name=field_name,
form_type=form_type,
method=method)
db.session.add(report)
db.session.commit()
return Report.query.filter_by(url=url).first()
def track():
if request.method == "GET":
fake_reports = gen_fake_reports(65, 165, 1, 20)
return render_template('track.html', reports=fake_reports)
if request.method == "POST":
# get the JSON object with lat/long tracking data
info_json = request.get_json()
path_info = info_json['path']
#list of coordinates - i.e. [{u'lat': 65.5, u'lng': 165.6}, {u'lat': 65.5, u'lng': 165.7}, {u'lat': 65.5, u'lng': 165.9}]
# get user id from session
u_id = session.get("user_id", 1)
# get current datetime
uploaded_datetime = datetime.datetime.now()
new_tracking_entry = UserPath(user_id=u_id,
gps_path=path_info,
created_at=uploaded_datetime)
db.session.add(new_tracking_entry)
# list of reports made
reports_info = info_json['reports']
# go thru each report and grab the report made and coordinates
for report in reports_info:
# TODO: Change this so that instead of recieving text (i.e. "bear") of the type of report, we get the id to match what we have in db
if report['message'] == "bear":
rtype = 1
elif report['message'] == "hole":
rtype = 2
elif report['message'] == "thin ice":
rtype = 3
report_coordinates = {"lat": report['lat'], "lng": report['lng']}
new_report_entry = Report(rtype_id=rtype,
date_logged=uploaded_datetime,
geocoordinates=report_coordinates,
user_id=u_id)
db.session.add(new_report_entry)
db.session.commit()
return "ok"
def seed_reports(year):
"""Seed report data into Report instances."""
# **************************************************************************
# start timer, get data from nc file, clock data receipt, initiate variables
# **************************************************************************
start = datetime.datetime.now()
print("Opening dataset")
dataset = Dataset('static/data.nc')
print("Opened dataset")
np.seterr(invalid='ignore')
lons = dataset.variables['longitude']
lats = dataset.variables['latitude']
times = dataset.variables['time']
# land_mask = dataset.variables['land_mask']
# this is a masked array and cannot be made a list with tolist()
temps = dataset.variables['temperature']
climatology = dataset.variables['climatology']
# dataset.close()
got_data = datetime.datetime.now()
# sum_abs_temp = 0
# # coordinate_data = {}
# lat_data = {}
# temp_lat_data = {}
# for i, temp in enumerate(temps.flat):
# if (i >= 64800):
# break
# **************************************************************************
# create index trackers to calculate position in data
# **************************************************************************
# shape of temps: (2009, 180, 360) -> (time, lat, lng)
# lng_index: increments with i in range(0-359) lngs/i's for 1 row/lat_index
# lat_index: increments every 360i in range(0-179) lats/(360 i/lng_index) for 1 time_index/full map
# time_index: increments every 360*180*i in range(0-64799) for a full map/64800 locations
# month_index: increments every 64800, every map, range(0-11)
# year_index: increments every 64800*12 -- decade_index: increments every 64000*12*10
# lng_index = i % 360
# lat_index = (i / 360) % 180
# time_index = i / (64800)
# month_index = (i / 64800) % 12
# year_index = (time_index / 12) + 1850
# decade_index = i % (7776000)
# ***********************************************************************************
# calculate month from report time decimal to get monthly climate report for location
# calculate absolute temperature
# ***********************************************************************************
num_items = 0
for i, t in enumerate(times):
if (t < year or t >= year + 04166666667):
continue
for j, lon in enumerate(lons):
print "processed:", t, lon
for k, lat in enumerate(lats):
num_items += 1
temp = temps[i][k][j]
month = int(math.floor((times[i] % 1) * 12))
climate = climatology[month][k][j]
abs_temp = climate if str(temp) == '--' else temp + climate
report = Report(lng=float(lon),
lat=float(lat),
time=float(t),
time_index=i,
abs_temp=float(abs_temp))
db.session.add(report)
if num_items % 400 == 0:
db.session.commit()
# **********************************************************************
# iteration conditions
# escapes:
# 1) ignore first 2 and last 2 rows of latitudes for even area grids
# 2) time index to get decade data
# flags every:
# 1) 8 lngs averaged and processed (1 lat grid)
# 2) 45 entries of 8 complete grids (1 lat row)
# 3) 22 entires of 45 8x8 complete grids (1 full map or 990 entries)
# **********************************************************************
# if lat_index in GLOBAL_LAT_INDEX_SKIP:
# print 'skip current_lat {}, current_lng {}, i {}, temp {}, climate {}, abs_temp {}'.format(lats[lat_index], lons[lng_index], i, temp, climate, abs_temp)
# # continue
# if year_index == 0:
# print '**********************************************************************'
# print 'HOPEFULLY NEW YEAR JAN. month {}, time_index {}, month_index {}'.format(month, time_index, month_index)
# print '**********************************************************************'
# if decade_index == 0:
# print '**********************************************************************'
# print 'NEW DECADE HOPEFULLY NEW MAP JAN OF YEAR ENDING IN 0. month {}, time_index {}, month_index {}'.format(month, time_index, month_index)
# print '**********************************************************************'
# **********************************************************************
# flag for 8 lngs processed
# # **********************************************************************
# if (i % 8 == 7):
# entry_lng = lons[lng_index] - 3.5
# # print entry_lng
# if (entry_lng) in temp_lat_data.keys():
# temp_lat_data[entry_lng] += sum_abs_temp
# temp_lat_data[entry_lng] /= 8
# else:
# temp_lat_data[entry_lng] = sum_abs_temp
# temp_lat_data[entry_lng] /= 8
# sum_abs_temp = abs_temp
# # print 'temp_data dict, i {}'.format(i)
# for k, v in sorted(temp_lat_data.iteritems()):
# print 'temporary lngs -- key: {}, value: avg_abs_temp {}'.format(k, v)
# print '**START SET OF LNGS** lat/lng {},{} -- index {},{}, i {}'.format(lats[lat_index], lons[lng_index], lat_index, lng_index, i)
# else:
# # DOES THIS NEED TO HAPPEN IN IF ALSO TO ENSURE A DATA ENTRY ISN'T
# sum_abs_temp += abs_temp
# **********************************************************************
# flag 2880 data entries, 1 of 22 lat rows containing 45 8x8 lats/lngs
# grid of 64 averaged temperature data
# temp_lat_data can reset
# **********************************************************************
# if (i % COMPLETE_LAT_ROW == COMPLETE_LAT_ROW - 1):
# entry_lat = lats[lat_index] - 1.5
# for k, v in sorted(temp_lat_data.iteritems()):
# print k, v
# report = Report(lng=float(k),
# lat=float(entry_lat),
# time=float(times[time_index]),
# time_index=time_index,
# abs_temp=float(v - GLOBAL_JAN_1850_TEMP_MIN))
# print "processed: ", month, report.time, report.lat, report.lng, report.abs_temp
# db.session.add(report)
# db.session.commit()
# print '**********************************'
# print 'COMPLETE_LAT_ROW - 8 lats averaged'
# print '**********************************'
# temp_lat_data = {}
# **********************************************************************
# print '\tstatus: i {}, lng {}, lat {}, lng_index {}, lat_index {}, temp {}, abs_temp {}, sum_abs_temp {}'.format(i, lons[lng_index], lats[lat_index], lng_index, lat_index, temp, abs_temp, sum_abs_temp)
# if i % 1000 == 0:
# print 'processing: i {}, lng {}, lng_index {}, lat {}, lat_index {}, year {}, month {}, time_index {}'.format(i, lons[lng_index], lng_index, lats[lat_index], lat_index, times[time_index], month, time_index)
end = datetime.datetime.now()
print "start {},\tdata {},\tend {}".format(start, got_data, end)
print "data gathering {},\tother {}, total {}".format(
(got_data - start), (end - got_data), (end - start))
def report(self) -> Report:
entries = self.balance_registry.report()
cost: Money = reduce(lambda f, s: f + s.cost, entries, Decimal(0))
return Report(cost, entries)
