def handle(self, *args, **options):
try_seeding = True
#asyncio.get_event_loop().run_until_complete(start_headless_browser())
#x = DBManager()
#c = '/usr/bin/chromium --headless --disable-gpu --no-sandbox --remote-debugging-port=9222 http://localhost/gsvpanoramacollector/link_browser/default/'.split(' ')
#c = '/usr/bin/google-chrome-stable --headless --disable-gpu --no-sandbox --remote-debugging-port=9222 http://localhost/gsvpanoramacollector/link_browser/default/'
#process = subprocess.Popen(c, stdout=subprocess.PIPE)
#self.stdout.write('process opened, waiting 5 seconds to start...')
#sleep(5)
self.stdout.write('Starting!')
iterations = 1
while True:
self.stdout.write(f'Collecting...')
try:
self.stdout.write('Starting DBManager')
x = DBManager()
self.stdout.write('Starting headless browser')
asyncio.get_event_loop().run_until_complete(self.start_headless_browser())
self.stdout.write('x = DBManager()')
if try_seeding:
try_seeding = False
self.stdout.write('Trying to seed')
x._seed_panorama()
self.stdout.write('x._update_panorama_references()')
self.stdout.write(f'{datetime.now().time()}')
x._update_panorama_references()
self.stdout.write(f'Success: {iterations}')
iterations += 1
if iterations % 20 == 0:
self.stdout.write(f'20 successes, restarting neo4j to avoid the memory leak issue...')
os.system('neo4j restart')
sleep(5)
self.stdout.write('x.close()')
x.close()
self.stdout.write('asyncio.get_event_loop().run_until_complete(stop_headless_browser())')
asyncio.get_event_loop().run_until_complete(self.stop_headless_browser())
except Exception as e:
self.stdout.write(f'Failed (iteration {iterations}) updating trying to reset chromium')
#print(f'e.__str__(): {e.__str__()}')
traceback.print_exc(file=sys.stdout)
#process.kill()
asyncio.get_event_loop().run_until_complete(self.stop_headless_browser())
if x is not None:
x.close()
#process = subprocess.Popen(c, stdout=subprocess.PIPE)
asyncio.get_event_loop().run_until_complete(self.start_headless_browser())
#sleep(5)
finally:
#process.kill()
asyncio.get_event_loop().run_until_complete(self.stop_headless_browser())
def get_comments_view(request):
jsondata = request.data
geoimage = jsondata['geoimage']
geoimage = json.loads(geoimage)
geoimage = GeoImage.fromJSON(geoimage)
filteruserid = jsondata.get('filteruserid')
dbmanager = DBManager()
comments = dbmanager.get_comments_for_view(geoImage=geoimage,
user_id=filteruserid)
comments = {"comments": comments}
#write_to_log(comments)
return JsonResponse(comments, CustomJSONEncoder)
def comment_view(request):
jsondata = request.data
geoimage = jsondata['geoimage']
geoimage = json.loads(geoimage)
geoimage = GeoImage.fromJSON(geoimage)
comment = jsondata['comment']
dbmanager = DBManager()
view = dbmanager.get_view_from_geoimage(geoImage=geoimage, include_id=True)
if not view:
return HttpResponseNotFound('View associated with GeoImage not found!')
view_id = view[0]
UserViewComments.objects.create(user=request.user,
last_update=datetime.date.today(),
viewid=view_id,
comment=comment)
dbmanager.insert_comment_for_view(geoImage=geoimage,
user_id=request.user.id,
comment=comment)
return HttpResponse(comment, status=201)
def _setOutput(cls, geoImage, featureLeaf, index):
try:
write_to_log('_setOutput')
dbmanager = DBManager()
result = dbmanager.load_processed_data_for_geoImage(
geoImage, cls.filterId)
if result:
filter_result = result[0]
geoImage.setProcessedData(cls.filterId,
'base64',
result[1],
density=filter_result['density'])
else:
if geoImage.dataType == 'data:image/jpeg;base64':
base64decoded = base64.b64decode(geoImage.data)
ndarrayImage = img_as_float(imageio.imread(base64decoded))
elif geoImage.dataType == 'URL':
ndarrayImage = img_as_float(imageio.imread(geoImage.data))
mask = mt_li_espectral(ndarrayImage)
density = np.count_nonzero(mask) / mask.size
mask = img_as_ubyte(overlay_mask(ndarrayImage, mask))
geoImage.setProcessedData(cls.filterId,
'ndarray',
mask,
density=density)
dbmanager.store_geoImage_as_view(geoImage)
featureLeaf[index] = geoImage
#dbmanager.store_geoImage_as_view(geoImage)
except HTTPError:
traceback.print_exc()
write_to_log(f"Http error - Have the quota been achieved?")
except ValueError:
traceback.print_exc()
write_to_log(f'Image bytes: {ndarrayImage[:100]}')
except Exception as e:
traceback.print_exc()
write_to_log(f"Unexpected error: {sys.exc_info()[0]}")
write_to_log(f"Error message: {e.args}")
write_to_log(f'Offending url: {geoImage.data[:300]}')
def _getPanoramasInBoundingBoxes(
regions: FeatureCollection) -> MultiLineString:
"""Collect a set of Ways (from GSVPanoramaManager)
and convert them to a MultiLineString"""
neo4jDB = DBManager()
featuresList = []
for feature in regions['features']:
geom = feature['geometry']
assert type(geom is Polygon)
singleLineString = geom.get('coordinates')[0]
bottom_left, top_right = GSVPanoramaMiner._get_bottom_left_top_right_corners(
singleLineString)
result = neo4jDB.retrieve_panorama_subgraphs_in_bounding_box(
bottom_left, top_right)
addresses = dict.fromkeys(
[d['r'].get('description') for d in result])
addresses.pop(None, 'None')
leftNodes = set()
rightNodes = set()
for record in result:
try:
leftNode = record['left']
rightNode = record['right']
arrow = record['r']
if (arrow['description'] in leftNode['description'])\
and (arrow['description'] in rightNode['description']):
if leftNode['pano'] in leftNodes:
continue
address = addresses[arrow['description']]
if address is None:
address = addresses[arrow['description']] = {}
linestrings = address.get('linestrings')
leftNodeProps = leftNode.__dict__['_properties'].copy()
leftNodeProps['heading'] = arrow.get('heading')
rightNodeProps = rightNode.__dict__[
'_properties'].copy()
# Temporarily sets the heading of the last node from a
# linestring. If a 'new' last node is found then
# this heading will be updated (following the arrow)
rightNodeProps['heading'] = arrow.get('heading')
if linestrings is None:
address['linestrings'] = []
linestrings = address.get('linestrings')
linestrings.append([leftNodeProps, rightNodeProps])
leftNodes.add(leftNodeProps['pano'])
rightNodes.add(rightNodeProps['pano'])
else:
# This segment merges two linestrings already
# added
if (leftNodeProps['pano'] in rightNodes)\
and (rightNodeProps['pano'] in leftNodes):
leftLinestringIdx = None
rightLinestringIdx = None
counter = -1
for linestring in linestrings:
counter = counter + 1
# Notice that only one direction
# can be extended at once
# so if backward arrows are found
# they will be discarded since they
# require both directions to be treated
# in a single check.
if linestring[-1]['pano'] == leftNodeProps[
'pano']:
leftLinestringIdx = counter
elif linestring[0][
'pano'] == rightNodeProps['pano']:
rightLinestringIdx = counter
else:
continue
if (leftLinestringIdx is not None)\
and (rightLinestringIdx is not None):
linestrings[leftLinestringIdx] = \
linestrings[leftLinestringIdx] +\
linestrings[rightLinestringIdx]
linestrings.pop(rightLinestringIdx)
#leftLinestring = leftLinestring + rightLinestring
break
# The line can be extended to the right
elif leftNodeProps['pano'] in rightNodes:
for linestring in linestrings:
if linestring[-1]['pano'] == leftNodeProps[
'pano']:
print(
f"extending (->) {linestring[-1]['pano']} with {rightNodeProps['pano']}"
)
# this updates the heading
linestring[-1] = leftNodeProps
# Updating the nodes that have been found as
# left nodes. This keeps the navigation
# consistently following a single direction.
leftNodes.add(leftNodeProps['pano'])
rightNodes.add(rightNodeProps['pano'])
linestring.append(rightNodeProps)
# The line can be extended to the left
elif rightNodeProps['pano'] in leftNodes:
for linestring in linestrings:
if linestring[0]['pano'] == rightNodeProps[
'pano']:
print(
f"extending (<-) {linestring[0]['pano']} with {leftNodeProps['pano']}"
)
# This time the heading of the existing node doesn't
# need to be updated.
rightNodes.add(rightNodeProps['pano'])
leftNodes.add(leftNodeProps['pano'])
linestring.insert(0, leftNodeProps)
# A new linestring must be created
else:
linestrings.append(
[leftNodeProps, rightNodeProps])
leftNodes.add(leftNodeProps['pano'])
rightNodes.add(rightNodeProps['pano'])
# Pegar a lista de endereços presentes na consulta.
# Cada dois nós de um mesmo endereço definem um segmento.
# Um segmento pode ser extendido.
# Somente um sentido de cada rua é exibido.
# Se dois segmentos de uma mesma rua forem desconexos
# então esta rua será representada por 2 linestrings
# cada linestring tem sua própria orientação
except TypeError:
if (arrow['description'] is None)\
or (rightNode['description'] is None)\
or (leftNode['description'] is None):
continue
else:
print("Unexpected error:", sys.exc_info()[0])
raise
except:
print("Unexpected error:", sys.exc_info()[0])
raise
for address in addresses:
MLS = []
geoImageMLS = []
if addresses[address] is None:
continue
for linestring in addresses[address]['linestrings']:
LS = []
geoImageLS = []
for node in linestring:
geoJsonPoint = Point(
[node['location'].x, node['location'].y])
LS.append(geoJsonPoint)
geoImage = GeoImage.fromJSON({
'id':
node['pano'],
'location':
geoJsonPoint,
'heading':
node['heading'],
'pitch':
node.get('pitch', 0)
})
geoImage.data=GSVPanoramaMiner.\
_imageURLBuilderForGeoImage(geoImage)
geoImage.dataType = 'URL'
geoImageLS.append(geoImage)
MLS.append(LineString(LS))
geoImageMLS.append(geoImageLS)
newFeature = Feature(id=address,
properties={
'name': address,
'geoImages': geoImageMLS
},
geometry=MultiLineString(MLS))
featuresList.append(newFeature)
return FeatureCollection(featuresList, crs=GSVPanoramaMiner._crs)
def getImageForFeature(feature: Feature) -> Feature:
"""
Receives a single Feature from a
Feature Collection of point/line or their
multi equivalents and inserts a "geoimages"
property with images for the coordinates of this
Feature.
Works in-place.
The geoimages property will have the same structure
of the geometry property.
The coordinates without a corresponding panorama will
be represented by a string "NOT FOUND" in the geoimages
correspondent position.
"""
write_to_log(f'getImageForFeature')
dbmanager = DBManager()
def ffunction(feature, clonedTree):
"""
ffunction - Feature's function
Defines how each feature (e.g. street) must be
processed.
"""
feature['properties']['geoImages'] = clonedTree
pass
def cfunction(coordinates, pointAtCoordinate=False):
"""
cfunction - Coordinates's function
Defines how each coordinate must be processed.
pointAtCoordinate: If true then the heading
will be defined to be one that points to the
coordinate rather than the frontal direction
from the camera.
"""
#Try to retrieve or collect the panorama
panorama = dbmanager.retrieve_nearest_panorama(coordinates)\
or dbmanager.collect_panorama_by_location(coordinates)
if panorama:
#Get view and consequently the stored image or its url
pano_id = panorama['pano']
if pointAtCoordinate:
originP = (panorama['location']['lng'],
panorama['location']['lat'])
destinationP = (coordinates[0], coordinates[1])
heading = calculate_initial_compass_bearing(originP, destinationP)
else:
heading = panorama['centerHeading']
pitch = panorama['originPitch']
# Optional: Just used to keep track of which views
# were already collected.
view = dbmanager.retrieve_panorama_view(
pano_id,
target_heading=heading,
heading_tolerance=10,
target_pitch=pitch,
pitch_tolerance=1
) or dbmanager.create_update_view(
pano_id,
heading,
pitch
)
#img_filename = dbmanager.image_filename_from_panorama_parameters(
# pano_id,
# heading,
# pitch
#)
#img_path = os.path.join(
# settings.PICTURES_FOLDER,
# img_filename
#)
#if os.path.exists(img_path):
local_img = dbmanager._retrieve_local_image(
pano_id,
view
)
if not local_img:
dbmanager._store_image_local(
pano_id,
view
)
local_img = dbmanager._retrieve_local_image(
pano_id,
view
)
pass
return GoogleStreetViewProvider.createGeoImage(
pano_id,
panorama['location'],
view,
local_img
).__dict__
#).toJSON()
else:
return "NOT FOUND"
pointAtCoordinate = (feature['geometry']['type'].lower() == 'multipoint')
clonedTree = ImageProvider.traverseFeature(feature, cfunction, pointAtCoordinate)
ffunction(feature, clonedTree)
return True
def getImageForFeatureCollection(featureCollection: FeatureCollection) -> FeatureCollection:
"""
Receives a feature collection of point/line or their
multi equivalents and returns a list of GeoImage's
"""
write_to_log(f'getImageForFeatureCollection')
dbmanager = DBManager()
def ffunction(feature, clonedTree):
"""
ffunction - Feature's function
Defines how each feature (e.g. street) must be
processed.
"""
feature['properties']['geoImages'] = clonedTree
pass
def cfunction(coordinates):
"""
cfunction - Coordinates's function
Defines how each coordinate must be processed.
"""
#Try to retrieve or collect the panorama
panorama = dbmanager.retrieve_nearest_panorama(coordinates)\
or dbmanager.collect_panorama_by_location(coordinates)
if panorama:
#Get view and consequently the stored image or its url
pano_id = panorama['pano']
heading = panorama['centerHeading']
pitch = panorama['originPitch']
# Optional: Just used to keep track of which views
# were already collected.
view = dbmanager.retrieve_panorama_view(
pano_id,
target_heading=heading,
heading_tolerance=10,
target_pitch=pitch,
pitch_tolerance=1
) or dbmanager.create_update_view(
pano_id,
heading,
pitch
)
#img_filename = dbmanager.image_filename_from_panorama_parameters(
# pano_id,
# heading,
# pitch
#)
#img_path = os.path.join(
# settings.PICTURES_FOLDER,
# img_filename
#)
#if os.path.exists(img_path):
local_img = dbmanager._retrieve_local_image(
pano_id,
view
)
if not local_img:
dbmanager._store_image_local(
pano_id,
view
)
local_img = dbmanager._retrieve_local_image(
pano_id,
view
)
pass
return GoogleStreetViewProvider.createGeoImage(
pano_id,
panorama['location'],
view,
local_img
).__dict__
#).toJSON()
else:
return "NOT FOUND"
ImageProvider.traverseFeatureCollection(featureCollection, ffunction, cfunction)
return True