def show_tables(): """ creates a report for specified month """ month = request.args.get('month','') year = request.args.get('year','') day= request.args.get('day','') if not month or not year: abort(400) start = date(year=int(year), month=int(month), day=int(day)) r = Report(start=start, finished=False) r.put() assetid = request.args.get('assetid', '') month = request.args.get('fmonth','') year = request.args.get('fyear','') day= request.args.get('fday','') if assetid and month and year and day: r.end = date(year=int(year), month=int(month), day=int(day)) r.assetid = assetid r.finished = True r.put() deferred.defer(update_report_stats, str(r.key())) return r.as_json()
def setUp(self): # create resources self.polygon = [[[-63.154907226557498, -4.8118385341739005], [-64.143676757807498, -4.8118385341739005], [-64.132690429682498, -6.2879986723276584], [-62.814331054682498, -6.3535159310087908]]] self.inv_polygon = [[[y, x] for x, y in self.polygon[0]]] self.r = Report(start=date(year=2011, month=2, day=1), end=date(year=2011, month=3, day=1), finished=True) self.r.put() self.cell = Cell(x=0, y=0, z=2, report=self.r, ndfi_high=1.0, ndfi_low=0.0) self.cell.put() self.area = Area(geo=json.dumps(self.inv_polygon), added_by=None, type=1, cell=self.cell) self.area.put() self.area.create_fusion_tables() self.ndfi = NDFI(self.r.comparation_range(), self.r.range()) self.stats = Stats()
def setUp(self): app.config['TESTING'] = True self.app = app.test_client() self.login('*****@*****.**', 'testuser') # generate data for reports self.r = Report(start=date(year=2011, month=2, day=1), end=date(year=2011, month=3, day=1), finished=True) self.r.put() stats = { 'id': str(self.r.key()), 'stats': { '0000_01': { 'id': '01', 'table': '0000', 'def': 1, 'deg': 2 }, '0000_02': { 'id': '02', 'table': '0000', 'def': 1, 'deg': 2 }, '0001_02': { 'id': '02', 'table': '0001', 'def': 1, 'deg': 2 } } } StatsStore(report_id=str(self.r.key()), json=json.dumps(stats)).put()
def test_create_report(self): rv = self.app.post('/_ah/cmd/create_report?month=11&year=2010&day=2') self.assertEquals(200, rv.status_code) self.assertEquals(1, Report.all().count()) r = Report.all().fetch(1)[0] self.assertEquals(2, r.start.day) self.assertEquals(11, r.start.month) self.assertEquals(2010, r.start.year)
def setUp(self): app.config['TESTING'] = True self.app = app.test_client() r = Report(start=date.today(), finished=False) r.put() self.r = r self.cell = Cell(x=0, y=0, z=2, report=self.r, ndfi_high=1.0, ndfi_low=0.0) self.cell.put() self.area = Area(geo='[[[-61.5,-12],[-61.5,-11],[-60.5,-11],[-60.5,-12]]]', added_by=users.get_current_user(), type=1, cell=self.cell)
def setUp(self): for x in models.CELL_BLACK_LIST[:]: models.CELL_BLACK_LIST.pop() app.config['TESTING'] = True self.login('*****@*****.**', 'testuser') self.app = app.test_client() for x in Cell.all(): x.delete() r = Report(start=date.today(), finished=False) r.put() self.r = r
def setUp(self): r = Report(start=date.today(), finished=False) r.put() self.r = r self.cell = Cell(x=11, y=11, z=2, report=self.r, ndfi_high=1.0, ndfi_low=0.0) self.cell.put()
class ReportTest(unittest.TestCase): def setUp(self): for x in Report.all(): x.delete() self.r = Report(start=date(year=2011, month=2, day=1), finished=False) self.r.put() def test_range(self): r1 = self.r.comparation_range() self.assertEquals(1, datetime.fromtimestamp(r1[0] / 1000).month) self.assertEquals(1, datetime.fromtimestamp(r1[0] / 1000).day)
class ReportTest(unittest.TestCase): def setUp(self): for x in Report.all(): x.delete() self.r = Report(start=date(year=2011, month=2, day=1), finished=False) self.r.put() def test_range(self): r1 = self.r.comparation_range() self.assertEquals(1, datetime.fromtimestamp(r1[0]/1000).month) self.assertEquals(1, datetime.fromtimestamp(r1[0]/1000).day)
def show_tables(): """ creates a report for specified month """ month = request.args.get('month','') year = request.args.get('year','') day= request.args.get('day','') if not month or not year: abort(400) start = date(year=int(year), month=int(month), day=int(day)) r = Report(start=start, finished=False) r.put() return 'created'
def setUp(self): app.config['TESTING'] = True self.app = app.test_client() self.login('*****@*****.**', 'testuser') r = Report(start=date.today(), finished=False) r.put() self.r = r self.cell = Cell(x=0, y=0, z=2, report=self.r, ndfi_high=1.0, ndfi_low=0.0) self.cell.put() for x in Note.all(): x.delete() self.when = datetime.now() self.note = Note(msg='test msg', added_by=users.get_current_user(), cell=self.cell, added_on=self.when) self.note.put()
def setUp(self): app.config['TESTING'] = True self.app = app.test_client() self.login('*****@*****.**', 'testuser') for x in Area.all(): x.delete() for x in Cell.all(): x.delete() r = Report(start=date.today(), finished=False) r.put() self.r = r self.cell = Cell(x=0, y=0, z=2, report=self.r, ndfi_high=1.0, ndfi_low=0.0) self.cell.put() self.area = Area(geo='[]', added_by=users.get_current_user(), type=1, cell=self.cell) self.area.put()
def ndfi_change(self, report_id, id): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(r, x, y, z) ee = ndfi = NDFI('MOD09GA', r.comparation_range(), r.range()) bounds = cell.bounds(amazon_bounds) ne = bounds[0] sw = bounds[1] # spcify lon, lat F**K, MONKEY BALLS polygons = [[(sw[1], sw[0]), (sw[1], ne[0]), (ne[1], ne[0]), (ne[1], sw[0])]] cols = 1 rows = 1 if z < 2: cols = rows = 10 data = ndfi.ndfi_change_value(r.base_map(), [polygons], rows, cols) logging.info(data) ndfi = data['data'] #data['data']['properties']['ndfiSum']['values'] if request.args.get('_debug', False): ndfi['debug'] = { 'request': ee.ee.last_request, 'response': ee.ee.last_response } return Response(json.dumps(ndfi), mimetype='application/json')
def list(self, report_id): r = Report.get(Key(report_id)) cell = Cell.get_or_default(r, 0, 0, 0) return self._as_json([ x.as_dict() for x in iter(cell.children()) if not self.is_in_backlist(x) ])
def landsat(self, report_id, id): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(r, x, y, z) bounds = cell.bounds(amazon_bounds) bounds = "%f,%f,%f,%f" % (bounds[1][1], bounds[1][0], bounds[0][1], bounds[0][0]) ee = EELandsat(LANDSAT7) d = ee.list(bounds=bounds) data = {} if len(d) >= 1: x = d[-1] img_info = x.split('/')[2][3:] path = img_info[:3] row = img_info[3:6] year = int(img_info[6:10]) julian_date = img_info[10:13] date = date_from_julian(int(julian_date), year) data = { 'info': img_info, 'path': path, 'row': row, 'year': year, 'timestamp': timestamp(date), 'date': date.isoformat() } return Response(json.dumps(data), mimetype='application/json')
def setUp(self): app.config['TESTING'] = True self.app = app.test_client() self.login('*****@*****.**', 'testuser') # generate data for reports self.r = Report(start=date(year=2011, month=2, day=1), end=date(year=2011, month=3, day=1), finished=True) self.r.put(); stats = { 'id': str(self.r.key()), 'stats': { '0000_01': { 'id': '01', 'table': '0000', 'def': 1, 'deg': 2 }, '0000_02': { 'id': '02', 'table': '0000', 'def': 1, 'deg': 2 }, '0001_02': { 'id': '02', 'table': '0001', 'def': 1, 'deg': 2 } } } StatsStore(report_id=str(self.r.key()), json=json.dumps(stats)).put();
def landsat(self, report_id, id): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(r, x, y, z) bounds = cell.bounds(amazon_bounds) bounds = "%f,%f,%f,%f" % (bounds[1][1], bounds[1][0], bounds[0][1], bounds[0][0]) ee = EELandsat(LANDSAT7) d = ee.list(bounds=bounds) data = {} if len(d) >= 1: x = d[-1] img_info = x.split("/")[2][3:] path = img_info[:3] row = img_info[3:6] year = int(img_info[6:10]) julian_date = img_info[10:13] date = date_from_julian(int(julian_date), year) data = { "info": img_info, "path": path, "row": row, "year": year, "timestamp": timestamp(date), "date": date.isoformat(), } return Response(json.dumps(data), mimetype="application/json")
def children(self, report_id, id): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(r, x, y, z) cells = cell.children() return self._as_json( [x.as_dict() for x in cells if not self.is_in_backlist(x)])
def update_cells_ndfi(): r = Report.current() cell = Cell.get_or_default(r, 0, 0, 0) for c in iter(cell.children()): c.put() deferred.defer(ndfi_value_for_cells, str(c.key()), _queue="ndfichangevalue") return 'working'
def update_main_cell_ndfi(z, x, y): r = Report.current() cell = Cell.get_or_create(r, x, y, z) deferred.defer(ndfi_value_for_cells, str(cell.key()), _queue="ndfichangevalue") return 'working'
def ndfi_change(self, report_id, id): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(r, x, y, z) ee = ndfi = NDFI('MOD09GA', r.comparation_range(), r.range()) bounds = cell.bounds(amazon_bounds) ne = bounds[0] sw = bounds[1] # spcify lon, lat polygons = [ (sw[1], sw[0]), (sw[1], ne[0]), (ne[1], ne[0]), (ne[1], sw[0]) ] cols = 1 rows = 1 if z < 2: cols = rows = 5 data = ndfi.ndfi_change_value(r.base_map(), {"type":"Polygon","coordinates":[polygons]}, rows, cols) logging.info(data) ndfi = data['data'] #data['data']['properties']['ndfiSum']['values'] if request.args.get('_debug', False): ndfi['debug'] = { 'request': ee.ee.last_request, 'response': ee.ee.last_response } return Response(json.dumps(ndfi), mimetype='application/json')
def landsat(self, report_id, id): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(r, x, y, z) bounds = cell.bounds(amazon_bounds) bounds = "%f,%f,%f,%f" % (bounds[1][1], bounds[1][0], bounds[0][1], bounds[0][0]) ee = EELandsat(LANDSAT7) d = ee.list(bounds=bounds) data = {} if len(d) >= 1: x = d[-1] img_info = x.split('/')[2][3:] path = img_info[:3] row = img_info[3:6] year = int(img_info[6: 10]) julian_date = img_info[10: 13] date = date_from_julian(int(julian_date), year) data = { 'info': img_info, 'path': path, 'row': row, 'year': year, 'timestamp': timestamp(date), 'date': date.isoformat() } return Response(json.dumps(data), mimetype='application/json')
def list(self, report_id, cell_pos): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(cell_pos) cell = Cell.get_cell(r, x, y, z) notes = [] if cell: return self._as_json([x.as_dict() for x in cell.note_set]) return self._as_json([])
def list(self, report_id, cell_pos): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(cell_pos) cell = Cell.get_cell(r, x, y, z) if not cell: return self._as_json([]) else: return self._as_json([x.as_dict() for x in cell.area_set])
def generate_report_pdf(*, report: Report=None): pdf = FPDF() pdf.add_page() pdf.set_font('Arial', 'B', 14) pdf.cell(100, 100, report.content) filename = f'{uuid.uuid4().hex}.pdf' pdf.output(filename, 'F') with open(filename, 'rb') as f: new_file = ContentFile(f.read(), filename) report.document = new_file report.save() os.remove(filename) return report.document.url
def update_cells_ndfi_dummy(): r = Report.current() if not r: return 'create a report first' cell = Cell.get_or_default(r, 0, 0, 0) for c in iter(cell.children()): c.put() deferred.defer(ndfi_value_for_cells_dummy, str(c.key()), _queue="ndfichangevalue") return 'working DUMMY'
def setUp(self): app.config['TESTING'] = True self.app = app.test_client() r = Report(start=date.today(), finished=False) r.put() self.r = r self.cell = Cell(x=0, y=0, z=2, report=self.r, ndfi_high=1.0, ndfi_low=0.0) self.cell.put() self.area = Area( geo='[[[-61.5,-12],[-61.5,-11],[-60.5,-11],[-60.5,-12]]]', added_by=users.get_current_user(), type=1, cell=self.cell)
def close(self, report_id): """ close current and create new one """ r = Report.get(Key(report_id)) if not r.finished: ndfi = NDFI(r.comparation_range(), r.range()) data = ndfi.freeze_map(r.base_map(), int(settings.FT_TABLE_ID), r.key().id()) logging.info(data) if 'data' not in data: abort(400) data = data['data']['id'] r.close(data) cache_key = NDFIMapApi._cache_key(report_id) memcache.delete(cache_key) # open new report new_report = Report(start=date.today()) new_report.put() return str(new_report.key()) return "already finished"
def rgb_mapid(self, report_id, id, r, g, b): report = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(report, x, y, z) ndfi = NDFI("MOD09GA", report.comparation_range(), report.range()) poly = cell.bbox_polygon(amazon_bounds) mapid = ndfi.rgb_strech(poly, tuple(map(int, (r, g, b)))) if "data" not in mapid: abort(404) return Response(json.dumps(mapid["data"]), mimetype="application/json")
def create(self, report_id, cell_pos): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(cell_pos) cell = Cell.get_or_create(r, x, y, z) data = json.loads(request.data) a = Area(geo=json.dumps(data["paths"]), type=data["type"], added_by=users.get_current_user(), cell=cell) a.save() cell.last_change_by = users.get_current_user() cell.put() return Response(a.as_json(), mimetype="application/json")
def create(self, report_id, cell_pos): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(cell_pos) cell = Cell.get_or_create(r, x, y, z) data = json.loads(request.data) if "msg" not in data: abort(400) a = Note(msg=data["msg"], added_by=users.get_current_user(), cell=cell) a.save() return Response(a.as_json(), mimetype="application/json")
def rgb_mapid(self, report_id, operation, id, r, g, b, sensor): report = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(report, operation, x, y, z) ndfi = NDFI(report.comparation_range(), report.range()) poly = cell.bbox_polygon(amazon_bounds) mapid = ndfi.rgb_stretch(poly, sensor, tuple(map(int, (r, g, b)))) if not mapid: abort(404) return Response(json.dumps(mapid), mimetype='application/json')
def create(self, report_id, cell_pos): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(cell_pos) cell = Cell.get_or_create(r, x, y, z) data = json.loads(request.data) if 'msg' not in data: abort(400) a = Note(msg=data['msg'], added_by=users.get_current_user(), cell=cell) a.save() return Response(a.as_json(), mimetype='application/json')
def get(self, report_id, id): r = Report.get(Key(report_id)) s = self.stats_for(str(r.key().id()), r.assetid, int(id)) if request.args.get('_debug', False): s['debug'] = { 'request': self.ee.ee.last_request, 'response': self.ee.ee.last_response } data = json.dumps(s) return Response(data, mimetype='application/json')
def rgb_mapid(self, report_id, id, r, g, b): report = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(report, x, y, z) ndfi = NDFI('MOD09GA', report.comparation_range(), report.range()) poly = cell.bbox_polygon(amazon_bounds) mapid = ndfi.rgb_strech(poly, tuple(map(int, (r, g, b)))) if 'data' not in mapid: abort(404) return Response(json.dumps(mapid['data']), mimetype='application/json')
def list(self, report_id): cache_key = self._cache_key(report_id) data = memcache.get(cache_key) if not data: r = Report.get(Key(report_id)) ndfi = NDFI(r.comparation_range(), r.range()) data = ndfi.mapid2(r.base_map()) if not data: abort(404) memcache.add(key=cache_key, value=data, time=3600) return jsonify(data)
def list(self, report_id, sensor): cache_key = self._cache_key(report_id, sensor) data = memcache.get(cache_key) if not data: r = Report.get(Key(report_id)) ndfi = NDFI(r.comparation_range(), r.range()) logging.info('((((( Report Id: ' + str(report_id) +', Sensor:'+ str(sensor) +' )))))') data = ndfi.mapid2(r.base_map(), sensor) if not data: abort(404) memcache.add(key=cache_key, value=data, time=3600) return jsonify(data)
def update_total_stats_for_report(report_id): r = Report.get(Key(report_id)) stats = StatsStore.get_for_report(report_id) if stats: s = stats.table_accum(tables_map['Legal Amazon']) logging.info("stats for %s" % s) if s: r.degradation = s['deg'] r.deforestation = s['def'] r.put() else: logging.error("can't find stats for %s" % report_id)
def create(self, report_id, cell_pos): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(cell_pos) cell = Cell.get_or_create(r, x, y, z) data = json.loads(request.data) a = Area(geo=json.dumps(data['paths']), type=data['type'], added_by=users.get_current_user(), cell=cell) a.save() cell.last_change_by = users.get_current_user() cell.put() return Response(a.as_json(), mimetype='application/json')
def list(self, report_id): cache_key = self._cache_key(report_id) data = memcache.get(cache_key) if not data: r = Report.get(Key(report_id)) ee_resource = 'MOD09GA' ndfi = NDFI(ee_resource, r.comparation_range(), r.range()) data = ndfi.mapid2(r.base_map()) logging.info(data) if 'data' not in data: abort(404) data = data['data'] memcache.add(key=cache_key, value=data, time=3600) return jsonify(data)
def update(self, report_id, id): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(r, x, y, z) cell.report = r data = json.loads(request.data) cell.ndfi_low = float(data['ndfi_low']) cell.ndfi_high = float(data['ndfi_high']) cell.done = data['done'] cell.last_change_by = users.get_current_user() cell.put() return Response(cell.as_json(), mimetype='application/json')
def list(self, report_id): cache_key = self._cache_key(report_id) data = memcache.get(cache_key) if not data: r = Report.get(Key(report_id)) ee_resource = "MOD09GA" ndfi = NDFI(ee_resource, r.comparation_range(), r.range()) data = ndfi.mapid2(r.base_map()) logging.info(data) if "data" not in data: abort(404) data = data["data"] memcache.add(key=cache_key, value=data, time=3600) return jsonify(data)
def export_areas(): compounddate = request.args.get('compounddate','') if not compounddate: abort(400) month = compounddate[4:6] year = compounddate[0:4] start = datetime.date(int(year), int(month), 1) start = datetime.datetime.combine(start, datetime.time()) end = datetime.date(int(year), int(month), calendar.monthrange(int(year), int(month))[1]) end = datetime.datetime.combine(end, datetime.time()) report = Report.find_by_period(start, end) polygons = Cell.polygon_by_report(report) return json.dumps(polygons)
def update_report_stats(report_id): r = Report.get(Key(report_id)) stats = {'id': report_id, 'stats': {}} for desc, table, name in tables: stats['stats'].update(stats_for(str(r.key().id()), r.assetid, table)) # sleep for some time to avoid problems with FT time.sleep(4) data = json.dumps(stats) s = StatsStore.get_for_report(report_id) if s: s.json = data s.put() else: StatsStore(report_id=report_id, json=data).put() # wait a little bit to allow app store saves the data time.sleep(1.0) update_total_stats_for_report(report_id)
def update(self, report_id, operation, id): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(r, operation, x, y, z) cell.report = r data = json.loads(request.data) cell.ndfi_low = float(data['ndfi_low']) cell.ndfi_high = float(data['ndfi_high']) cell.compare_view = str(data['compare_view']) cell.map_one_layer_status = str(data['map_one_layer_status']) cell.map_two_layer_status = str(data['map_two_layer_status']) cell.map_three_layer_status = str(data['map_three_layer_status']) cell.map_four_layer_status = str(data['map_four_layer_status']) cell.done = data['done'] cell.last_change_by = users.get_current_user() cell.put() return Response(cell.as_json(), mimetype='application/json')
def polygon(self): """ return stats for given polygon """ data = json.loads(request.data) polygon = data['polygon'] reports = data['reports'] try: reports = [Report.get(Key(x)) for x in reports] except ValueError: logging.error("can't find some report") abort(404) #TODO: test if polygon is ccw # exchange lat, lon -> lon, lat normalized_poly = [(coord[1], coord[0]) for coord in polygon] stats = self.ee.get_stats_for_polygon([(str(r.key().id()), r.assetid) for r in reports], [normalized_poly]) try: # aggregate data['def'] = sum(s['def'] for s in stats) data['deg'] = sum(s['deg'] for s in stats) data['total_area'] = stats[0]['total_area'] return Response(json.dumps(data), mimetype='application/json') except (KeyError, ValueError, IndexError): abort(404)
def ndfi_change(self, report_id, id): r = Report.get(Key(report_id)) z, x, y = Cell.cell_id(id) cell = Cell.get_or_default(r, x, y, z) ee = ndfi = NDFI(r.comparation_range(), r.range()) bounds = cell.bounds(amazon_bounds) ne = bounds[0] sw = bounds[1] # spcify lon, lat polygons = [(sw[1], sw[0]), (sw[1], ne[0]), (ne[1], ne[0]), (ne[1], sw[0])] cols = 1 rows = 1 if z < 2: cols = rows = 5 data = ndfi.ndfi_change_value(r.base_map(), { "type": "Polygon", "coordinates": [polygons] }, rows, cols) logging.info(data) ndfi = data #data['properties']['ndfiSum']['values'] return Response(json.dumps(ndfi), mimetype='application/json')