def run_migration():
if type(db) == peewee.SqliteDatabase:
migrator = SqliteMigrator(db)
elif type(db) == peewee.MySQLDatabase:
migrator = MySQLMigrator(db)
elif type(db) == peewee.PostgresqlDatabase:
migrator = PostgresqlMigrator(db)
else:
return
migrate(
migrator.add_column('report', 'first_report_datetime',
peewee.DateTimeField(default=UTC_now)),
migrator.drop_column('report', 'is_open'),
)
query = Report.select()
for report in query:
report.first_report_datetime = report.datetime
report.save()
def run_migration():
if type(db) == peewee.SqliteDatabase:
migrator = SqliteMigrator(db)
elif type(db) == peewee.MySQLDatabase:
migrator = MySQLMigrator(db)
elif type(db) == peewee.PostgresqlDatabase:
migrator = PostgresqlMigrator(db)
else:
return
migrate(
migrator.add_column('report', 'source',
peewee.CharField(max_length=255, default='')),
migrator.add_column('report', 'shape_geojson',
peewee.TextField(default=None, null=True)),
)
query = Report.select()
for report in query:
report.source = 'unknown'
report.save()
def process_opendata(name, data, report_type=REPORT_TYPE):
# Coordinates of current reports in db, as shapely Points in Lambert93
current_reports_points = []
active_reports_from_db = Report.select().where(
# Load reports from db of the current type
(Report.type == report_type) & (
# Either with an expiration_datetime in the future
((Report.expiration_datetime is not None) &
(Report.expiration_datetime > UTC_now())) |
# Or without expiration_datetime but which are still active (shown
# on the map)
((Report.expiration_datetime is None) &
(Report.downvotes < REPORT_DOWNVOTES_THRESHOLD))))
for report in active_reports_from_db:
current_reports_points.append(
transform(project, Point(report.lng, report.lat)))
for item in data:
try:
fields = item['fields']
# Check that the work is currently being done
now = arrow.now('Europe/Paris')
if fields['date_debut']:
start_date = arrow.get(fields['date_debut'].replace('/', '-'))
else:
# Defaults to now if start date is unknown
start_date = arrow.get(now)
if fields['date_fin']:
end_date = arrow.get(fields['date_fin'].replace('/', '-'))
else:
# Defaults to in a week if start date is unknown
end_date = arrow.get(now).shift(days=+7)
if not (start_date < now < end_date):
logging.info(
'Ignoring record %s, work not currently in progress.',
item['recordid'])
continue
# Report geographical shape
if 'geo_shape' in fields:
maybe_multi_geo_shape = shape(fields['geo_shape'])
else:
maybe_multi_geo_shape = shape(item['geometry'])
geo_shapes = []
if (isinstance(maybe_multi_geo_shape, MultiPolygon)
or isinstance(maybe_multi_geo_shape, MultiPoint)):
# Split MultiPolygon into multiple Polygon
# Same for MultiPoint
positions = [p.centroid for p in maybe_multi_geo_shape]
geo_shapes = [p for p in maybe_multi_geo_shape]
elif isinstance(maybe_multi_geo_shape, MultiLineString):
# Split MultiLineString into multiple LineString
positions = [
p.interpolate(0.5, normalized=True)
for p in maybe_multi_geo_shape
]
geo_shapes = [p for p in maybe_multi_geo_shape]
elif isinstance(maybe_multi_geo_shape, LineString):
# LineString, interpolate midpoint
positions = [
maybe_multi_geo_shape.interpolate(0.5, normalized=True)
]
geo_shapes = [maybe_multi_geo_shape]
else:
# Polygon or Point
positions = [maybe_multi_geo_shape.centroid]
geo_shapes = [maybe_multi_geo_shape]
for (geo_shape, position) in zip(geo_shapes, positions):
# Check if this precise position is already in the database
if transform(project, position) in current_reports_points:
logging.info(
('Ignoring record %s, a similar report is already in '
'the database.'), item['recordid'])
continue
# Check no similar reports is within the area of the report, up
# to the report distance.
overlap_area = transform(
project, geo_shape).buffer(MIN_DISTANCE_REPORT_DETAILS)
is_already_inserted = False
for report_point in current_reports_points:
if report_point.within(overlap_area):
# A similar report is already there
is_already_inserted = True
logging.info(
('Ignoring record %s, a similar report is already '
'in the database.'), item['recordid'])
break
if is_already_inserted:
# Skip this report if a similar one is nearby
continue
# Get the position of the center of the item
lng, lat = position.x, position.y
# Compute expiration datetime
expiration_datetime = end_date.replace(microsecond=0).naive
# Add the report to the db
logging.info('Adding record %s to the database.',
item['recordid'])
Report.create(type=report_type,
expiration_datetime=expiration_datetime,
lat=lat,
lng=lng,
source=item['source'],
shape_geojson=json.dumps(mapping(geo_shape)))
except KeyError as exc:
logging.warning('Invalid record %s in %s, missing key: %s',
item.get('recordid', '?'), name, exc)
def process_opendata(name, data, report_type=REPORT_TYPE):
# Coordinates of current reports in db, as shapely Points in Lambert93
current_reports_points = []
active_reports_from_db = Report.select().where(
# Load reports from db of the current type
(Report.type == report_type) & (
# Either with an expiration_datetime in the future
((Report.expiration_datetime is not None) &
(Report.expiration_datetime > UTC_now())) |
# Or without expiration_datetime but which are still active (shown
# on the map)
((Report.expiration_datetime is None) &
(Report.downvotes < REPORT_DOWNVOTES_THRESHOLD))))
for report in active_reports_from_db:
current_reports_points.append(
transform(project, Point(report.lng, report.lat)))
# TODO: Remove reports which are no longer valid
# Filter out unknown states and roads without traffic
data = [
x for x in data
if x['fields'].get('state') not in ['FLUIDE', 'INCONNU']
]
for item in data:
try:
fields = item['fields']
# Get geometry and position
geometry = shape(item['geometry'])
position = geometry.centroid
lng, lat = position.x, position.y
# Check if this precise position is already in the database
if transform(project, position) in current_reports_points:
logging.info(
('Ignoring record %s, a similar report is already in '
'the database.'), item['recordid'])
continue
# Check no similar reports is within the area of the report, up
# to the report distance.
overlap_area = transform(
project, geometry).buffer(MIN_DISTANCE_REPORT_DETAILS)
is_already_inserted = False
for report_point in current_reports_points:
if report_point.within(overlap_area):
# A similar report is already there
is_already_inserted = True
logging.info(
('Ignoring record %s, a similar report is already '
'in the database.'), item['recordid'])
break
if is_already_inserted:
# Skip this report if a similar one is nearby
continue
# Expires in an hour
expiration_datetime = (
# TODO: Check the datetime value in the opendata file
arrow.get(fields['datetime']).shift(hours=+1).naive)
# Add the report to the db
logging.info('Adding record %s to the database.', item['recordid'])
Report.create(type=report_type,
expiration_datetime=expiration_datetime,
lat=lat,
lng=lng,
source=item['source'],
shape_geojson=json.dumps(mapping(geometry)))
except KeyError as exc:
logging.warning('Invalid record %s in %s, missing key: %s',
item.get('recordid', '?'), name, exc)