def findProcess(imageName, filterString):
pid = Popen([
'wmic', 'path', 'win32_process', 'where',
"Name='%s'" % imageName, 'get', 'CommandLine,ProcessId', '/VALUE'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out = pid.communicate()
out = out[0].split('\r\r\n\r\r\n')
out = filter(lambda x: 'CommandLine' in x, out)
out = map(lambda x: x.strip().split('\r\r\n'), out)
if out:
if out[0][0].startswith('CommandLine'):
commandLineIndex = 0
processIndex = 1
else:
commandLineIndex = 1
processIndex = 0
out = map(
lambda x: [int(x[processIndex][10:]), x[commandLineIndex][12:]],
out)
out = filter(lambda x: filterString in x[1], out)
return map(lambda x: x[0], out)
return []
def zoomify_add_image(self, filename, width, height, bands, pixel_format):
from hashlib import sha256
import urllib
import voxel_globe.meta.models
#Create database entry
img = voxel_globe.meta.models.Image(
name="%s Upload %s (%s) Frame %s" %
(self.meta_name, self.upload_session.name, self.upload_session.id,
os.path.basename(filename)),
image_width=width,
image_height=height,
number_bands=bands,
pixel_format=pixel_format,
file_format='zoom',
service_id=self.task.request.id)
img.filename_path = filename
img.save()
self.image_set.images.add(img)
zoomify_path = img.zoomify_path
pid = Popen(
['vips', 'dzsave', filename, zoomify_path, '--layout', 'zoomify'])
pid.wait()
def convert_ply_to_potree(ply_filename, potree_dirname):
from voxel_globe.tools.subprocessbg import Popen
potree_ply = os.path.join(potree_dirname, 'potree.ply')
with open(ply_filename, 'r') as fid_in, open(potree_ply, 'w') as fid_out:
line = 'None'
while not line.startswith('end_header') and line != '':
line = fid_in.readline()
if line == 'property float prob\n':
line = 'property float nx\n'
if line == 'property float le\n':
line = 'property float ny\n'
if line == 'property float ce\n':
line = 'property float nz\n'
fid_out.write(line)
chunk = None
while not chunk == '':
chunk = fid_in.read(64*1024*1024)
fid_out.write(chunk)
pid = Popen(['PotreeConverter', '--source', potree_ply,
'-a', 'RGB', 'INTENSITY', 'CLASSIFICATION', 'REAL_NORMAL',
'-o', potree_dirname, 'potree'])
pid.wait()
def zoomify_add_image(self, filename, width, height, bands, pixel_format):
from hashlib import sha256
import urllib
import voxel_globe.meta.models
hasher = sha256()
chunk = 1024 * 1024 * 16
with open(filename, 'rb') as fid:
data = fid.read(chunk)
while data:
hasher.update(data)
data = fid.read(chunk)
zoomify_name = os.path.join(
os.path.split(filename)[0],
hasher.hexdigest() + '_zoomify')
#zoomify_name = os.path.splitext(filename)[0] + '_zoomify'
pid = Popen(
['vips', 'dzsave', filename, zoomify_name, '--layout', 'zoomify'])
pid.wait()
#convert the slashes to URL slashes
relative_file_path = urllib.pathname2url(
os.path.relpath(filename, env['VIP_IMAGE_SERVER_ROOT']))
basename = os.path.split(filename)[-1]
relative_zoom_path = urllib.pathname2url(
os.path.relpath(zoomify_name, env['VIP_IMAGE_SERVER_ROOT']))
img = voxel_globe.meta.models.Image.create(
name="%s Upload %s (%s) Frame %s" %
(self.meta_name, self.upload_session.name, self.upload_session.id,
basename),
imageWidth=width,
imageHeight=height,
numberColorBands=bands,
pixelFormat=pixel_format,
fileFormat='zoom',
imageUrl='%s://%s:%s/%s/%s/' %
(env['VIP_IMAGE_SERVER_PROTOCOL'], env['VIP_IMAGE_SERVER_HOST'],
env['VIP_IMAGE_SERVER_PORT'], env['VIP_IMAGE_SERVER_URL_PATH'],
relative_zoom_path),
originalImageUrl='%s://%s:%s/%s/%s' %
(env['VIP_IMAGE_SERVER_PROTOCOL'], env['VIP_IMAGE_SERVER_HOST'],
env['VIP_IMAGE_SERVER_PORT'], env['VIP_IMAGE_SERVER_URL_PATH'],
relative_file_path),
service_id=self.task.request.id,
original_filename=basename)
img.save()
self.image_collection.images.add(img)
def zoomify_add_image(self, filename, width, height, bands, pixel_format):
from hashlib import sha256
import urllib
import voxel_globe.meta.models
hasher = sha256()
chunk = 1024*1024*16
with open(filename, 'rb') as fid:
data = fid.read(chunk)
while data:
hasher.update(data)
data = fid.read(chunk)
zoomify_name = os.path.join(os.path.split(filename)[0], hasher.hexdigest()+'_zoomify')
#zoomify_name = os.path.splitext(filename)[0] + '_zoomify'
pid = Popen(['vips', 'dzsave', filename, zoomify_name, '--layout',
'zoomify'])
pid.wait()
#convert the slashes to URL slashes
relative_file_path = urllib.pathname2url(os.path.relpath(filename,
env['VIP_IMAGE_SERVER_ROOT']))
basename = os.path.split(filename)[-1]
relative_zoom_path = urllib.pathname2url(os.path.relpath(zoomify_name,
env['VIP_IMAGE_SERVER_ROOT']))
img = voxel_globe.meta.models.Image.create(
name="%s Upload %s (%s) Frame %s" % (self.meta_name,
self.upload_session.name,
self.upload_session.id,
basename),
imageWidth=width, imageHeight=height,
numberColorBands=bands, pixelFormat=pixel_format, fileFormat='zoom',
imageUrl='%s://%s:%s/%s/%s/' % (env['VIP_IMAGE_SERVER_PROTOCOL'],
env['VIP_IMAGE_SERVER_HOST'],
env['VIP_IMAGE_SERVER_PORT'],
env['VIP_IMAGE_SERVER_URL_PATH'],
relative_zoom_path),
originalImageUrl='%s://%s:%s/%s/%s' % (
env['VIP_IMAGE_SERVER_PROTOCOL'],
env['VIP_IMAGE_SERVER_HOST'],
env['VIP_IMAGE_SERVER_PORT'],
env['VIP_IMAGE_SERVER_URL_PATH'],
relative_file_path),
service_id=self.task.request.id,
original_filename=basename)
img.save()
self.image_collection.images.add(img)
def findProcess(imageName, filterString):
pid = Popen(['wmic', 'path', 'win32_process', 'where', "Name='%s'" % imageName, 'get', 'CommandLine,ProcessId', '/VALUE'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out = pid.communicate()
out=out[0].split('\r\r\n\r\r\n')
out=filter(lambda x:'CommandLine' in x, out)
out = map(lambda x:x.strip().split('\r\r\n'), out)
if out:
if out[0][0].startswith('CommandLine'):
commandLineIndex = 0
processIndex = 1
else:
commandLineIndex = 1
processIndex = 0
out = map(lambda x:[int(x[processIndex][10:]), x[commandLineIndex][12:]], out)
out = filter(lambda x:filterString in x[1], out)
return map(lambda x:x[0], out)
return []
def convert_ply_to_potree(ply_filename, potree_dirname):
from voxel_globe.tools.subprocessbg import Popen
potree_ply = os.path.join(potree_dirname, "potree.ply")
with open(ply_filename, "r") as fid_in, open(potree_ply, "w") as fid_out:
line = "None"
while not line.startswith("end_header") and line != "":
line = fid_in.readline()
if line == "property float prob\n":
line = "property float nx\n"
if line == "property float le\n":
line = "property float ny\n"
if line == "property float ce\n":
line = "property float nz\n"
fid_out.write(line)
chunk = None
while not chunk == "":
chunk = fid_in.read(64 * 1024 * 1024)
fid_out.write(chunk)
pid = Popen(
[
"PotreeConverter",
"--source",
potree_ply,
"-a",
"RGB",
"INTENSITY",
"CLASSIFICATION",
"REAL_NORMAL",
"-o",
potree_dirname,
"potree",
]
)
pid.wait()
def zoomify_add_image(self, filename, width, height, bands, pixel_format):
from hashlib import sha256
import urllib
import voxel_globe.meta.models
#Create database entry
img = voxel_globe.meta.models.Image(
name="%s Upload %s (%s) Frame %s" % (self.meta_name,
self.upload_session.name,
self.upload_session.id,
os.path.basename(filename)),
image_width=width, image_height=height,
number_bands=bands, pixel_format=pixel_format, file_format='zoom',
service_id=self.task.request.id)
img.filename_path=filename
img.save()
self.image_set.images.add(img)
zoomify_path = img.zoomify_path
pid = Popen(['vips', 'dzsave', filename, zoomify_path, '--layout',
'zoomify'])
pid.wait()
def zoomify_image(filename, zoomify_dir):
pid = Popen(['vips', 'dzsave', filename, zoomify_dir, '--layout',
'zoomify'])
pid.wait()
def ingest_data(self, uploadSession_id, imageDir):
''' task for the ingest route, to ingest the data an upload sessions points to '''
import voxel_globe.ingest.models as IngestModels
from .tools import loadAdjTaggedMetadata
import numpy
from voxel_globe.tools.camera import save_krt
from PIL import Image
uploadSession = IngestModels.UploadSession.objects.get(id=uploadSession_id);
#directories = uploadSession.directory.all();
#imageDirectory = directories.filter(name='image')
#metaDirectory = directories.filter(name='meta')
metadataFilename = glob(os.path.join(imageDir, '*', '*_adj_tagged_images.txt'), False);
if not len(metadataFilename) == 1:
logger.error('Only one metadatafile should have been found, found %d instead', len(metadataFilename));
try:
metadataFilename = metadataFilename[0]
(day, timeOfDay) = os.path.split(metadataFilename)[1].split(' ');
timeOfDay = timeOfDay.split('_', 1)[0];
except:
metadataFilename = os.devnull;
day = 'NYA'
timeOfDay = 'NYA'
imageCollection = voxel_globe.meta.models.ImageCollection.create(name="Arducopter Upload %s %s %s (%s)" % (uploadSession.name, day, timeOfDay, uploadSession_id), service_id = self.request.id);
imageCollection.save();
for d in glob(os.path.join(imageDir, '*'+os.path.sep), False):
files = glob(os.path.join(d, '*.jpg'), False);
files.sort()
for f in files:
self.update_state(state='PROCESSING',
meta={'stage':'File %s of %d' % (f, len(files))})
logger.debug('Processing %s of %s', f, len(files))
zoomifyName = f[:-4] + '_zoomify'
pid = Popen(['vips', 'dzsave', f, zoomifyName, '--layout', 'zoomify'])
pid.wait();
#convert the slashes to URL slashes
relFilePath = urllib.pathname2url(os.path.relpath(f, env['VIP_IMAGE_SERVER_ROOT']));
basename = os.path.split(f)[-1]
relZoomPath = urllib.pathname2url(os.path.relpath(zoomifyName, env['VIP_IMAGE_SERVER_ROOT']));
image = Image.open(f)
if image.bits == 8:
pixel_format = 'b';
if image.bits == 16:
pixel_format = 's';
if image.bits == 32:
if image.mode == "I":
pixel_format = 'i';
elif image.mode == "F":
pixel_format = 'f'
img = voxel_globe.meta.models.Image.create(
name="Arducopter Upload %s (%s) Frame %s" % (uploadSession.name, uploadSession_id, basename),
imageWidth=image.size[0], imageHeight=image.size[1],
numberColorBands=image.layers, pixelFormat=pixel_format, fileFormat='zoom',
imageUrl='%s://%s:%s/%s/%s/' % (env['VIP_IMAGE_SERVER_PROTOCOL'],
env['VIP_IMAGE_SERVER_HOST'],
env['VIP_IMAGE_SERVER_PORT'],
env['VIP_IMAGE_SERVER_URL_PATH'],
relZoomPath),
originalImageUrl='%s://%s:%s/%s/%s' % (env['VIP_IMAGE_SERVER_PROTOCOL'],
env['VIP_IMAGE_SERVER_HOST'],
env['VIP_IMAGE_SERVER_PORT'],
env['VIP_IMAGE_SERVER_URL_PATH'],
relFilePath),
service_id = self.request.id);
img.save();
imageCollection.images.add(img);
self.update_state(state='Processing', meta={'stage':'metadata'})
metadata = loadAdjTaggedMetadata(metadataFilename);
for meta in metadata:
try:
img = imageCollection.images.get(name__icontains='Frame %s'%meta.filename)
k = numpy.eye(3);
k[0,2] = img.imageWidth/2;
k[1,2] = img.imageHeight/2;
r = numpy.eye(3);
t = [0, 0, 0];
origin = meta.llh_xyz;
save_krt(self.request.id, img, k, r, t, origin, srid=7428);
except Exception as e:
logger.warning('%s', e)
logger.error('Could not match metadata entry for %s' % meta.filename)
averageGps = numpy.mean(numpy.array(map(lambda x:x.llh_xyz, metadata)), 0);
voxel_globe.meta.models.Scene.create(name="Arducopter origin %s (%s)" % (uploadSession.name, uploadSession_id),
service_id = self.request.id,
origin='SRID=%d;POINT(%0.12f %0.12f %0.12f)' % \
(7428, averageGps[0], averageGps[1], averageGps[2])).save()
uploadSession.delete()
def zoomify_image(filename, zoomify_dir):
pid = Popen(
['vips', 'dzsave', filename, zoomify_dir, '--layout', 'zoomify'])
pid.wait()
def ingest_data(self, uploadSession_id, imageDir):
''' task for the ingest route, to ingest the data an upload sessions points
to '''
import voxel_globe.ingest.models as IngestModels
from voxel_globe.tools.camera import save_krt
from PIL import Image
from datetime import datetime, timedelta
from .tools import split_clif
uploadSession = IngestModels.UploadSession.objects.get(id=uploadSession_id);
metadataFilenames = glob(os.path.join(imageDir, '*.txt'), False);
metadataFilenames = sorted(metadataFilenames, key=lambda s:s.lower())
metadataBasenames = map(lambda x:os.path.basename(x).lower(), metadataFilenames)
#In case none of them succeeded...
date = 'NYA'
timeOfDay = 'NYA'
for metadata_filename in metadataFilenames:
#Loop through until one succeeds
try:
with open(metadata_filename, 'r') as fid:
data = fid.readline().split(',')
imu_time = float(data[6])
imu_week = int(data[7])
timestamp = datetime(1980, 1, 6) + timedelta(weeks=imu_week,
seconds=imu_time)
date = '%04d-%02d-%02d' % (timestamp.year, timestamp.month,
timestamp.day)
timeOfDay = '%02d:%02d:%02d.%06d' % (timestamp.hour, timestamp.minute,
timestamp.second,
timestamp.microsecond)
break #Break on first success
except:
pass
imageCollection = voxel_globe.meta.models.ImageCollection.create(
name="CLIF Upload %s %s %s (%s)" % (uploadSession.name, date,
timeOfDay, uploadSession_id),
service_id = self.request.id);
imageCollection.save();
llhs_xyz = []
#for d in glob(os.path.join(imageDir, '*'+os.path.sep), False):
if 1:
files = glob(os.path.join(imageDir, '*'+os.extsep+'raw'), False);
files.sort()
for index,f in enumerate(files):
self.update_state(state='PROCESSING',
meta={'stage':'File %s (%d of %d)' % (f, index+1, len(files))})
logger.debug('Processing %s (%d of %d)', f, index+1, len(files))
basename = os.path.basename(f)
img_filename = os.extsep.join([os.path.splitext(f)[0], 'png'])
with open(f, 'rb') as fid:
data = fid.read();
img = np.fromstring(data,
dtype=CLIF_DATA[CLIF_VERSION]['dtype']).reshape(
(CLIF_DATA[CLIF_VERSION]['width'],
CLIF_DATA[CLIF_VERSION]['height'])).T
img2 = Image.fromarray(img)
img2.save(img_filename)
zoomifyName = os.path.splitext(f)[0] + '_zoomify'
pid = Popen(['vips', 'dzsave', img_filename, zoomifyName, '--layout', 'zoomify'])
pid.wait();
#convert the slashes to URL slashes
relFilePath = urllib.pathname2url(os.path.relpath(img_filename, env['VIP_IMAGE_SERVER_ROOT']));
basename = os.path.split(f)[-1]
relZoomPath = urllib.pathname2url(os.path.relpath(zoomifyName, env['VIP_IMAGE_SERVER_ROOT']));
pixel_format = CLIF_DATA[CLIF_VERSION]['pixel_format']
width = CLIF_DATA[CLIF_VERSION]['width']
height = CLIF_DATA[CLIF_VERSION]['height']
bands = CLIF_DATA[CLIF_VERSION]['bands']
img = voxel_globe.meta.models.Image.create(
name="CLIF Upload %s (%s) Frame %s" % (uploadSession.name, uploadSession_id, basename),
imageWidth=width, imageHeight=height,
numberColorBands=bands, pixelFormat=pixel_format, fileFormat='zoom',
imageUrl='%s://%s:%s/%s/%s/' % (env['VIP_IMAGE_SERVER_PROTOCOL'],
env['VIP_IMAGE_SERVER_HOST'],
env['VIP_IMAGE_SERVER_PORT'],
env['VIP_IMAGE_SERVER_URL_PATH'],
relZoomPath),
originalImageUrl='%s://%s:%s/%s/%s' % (env['VIP_IMAGE_SERVER_PROTOCOL'],
env['VIP_IMAGE_SERVER_HOST'],
env['VIP_IMAGE_SERVER_PORT'],
env['VIP_IMAGE_SERVER_URL_PATH'],
relFilePath),
service_id = self.request.id);
img.save();
imageCollection.images.add(img);
metadata_filename_desired = split_clif(f)
metadata_filename_desired = '%06d-%s.txt' % (0, metadata_filename_desired[2])
if 1:
# try:
metadata_index = metadataBasenames.index(metadata_filename_desired)
metadata_filename = metadataFilenames[metadata_index]
with open(metadata_filename, 'r') as fid:
metadata = fid.readline().split(',')
llh_xyz = [float(metadata[4]), float(metadata[3]),
float(metadata[5])*CLIF_DATA[CLIF_VERSION]['altitude_conversion']]
llhs_xyz.append(llh_xyz)
k = np.eye(3);
k[0,2] = width/2;
k[1,2] = height/2;
r = np.eye(3);
t = [0, 0, 0];
origin = llh_xyz;
save_krt(self.request.id, img, k, r, t, origin, srid=4326);
# except Exception as e:
pass
averageGps = np.mean(np.array(llhs_xyz), 0);
voxel_globe.meta.models.Scene.create(name="CLIF origin %s (%s)" % (uploadSession.name, uploadSession_id),
service_id = self.request.id,
origin='SRID=%d;POINT(%0.12f %0.12f %0.12f)' % \
(4326, averageGps[0], averageGps[1], averageGps[2])).save()
uploadSession.delete()
def ingest_data(self, uploadSession_id, imageDir):
''' task for the ingest route, to ingest the data an upload sessions points to '''
import voxel_globe.ingest.models as IngestModels
import numpy
from voxel_globe.tools.camera import save_krt
uploadSession = IngestModels.UploadSession.objects.get(id=uploadSession_id);
#directories = uploadSession.directory.all();
#imageDirectory = directories.filter(name='image')
#metaDirectory = directories.filter(name='meta')
imageCollection = voxel_globe.meta.models.ImageCollection.create(name="Generic Upload %s (%s)" % (uploadSession.name, uploadSession_id), service_id = self.request.id);
imageCollection.save();
r = numpy.eye(3);
t = [0, 0, 0];
gpsList = []
gpsList2 = []
for d in glob(os.path.join(imageDir, '*'+os.path.sep), False):
files = glob(os.path.join(d, '*'), False);
files.sort()
for f in files:
self.update_state(state='PROCESSING',
meta={'stage':'File %s of %d' % (f, len(files))})
zoomifyName = f[:-4] + '_zoomify'
pid = Popen(['vips', 'dzsave', f, zoomifyName, '--layout', 'zoomify'])
pid.wait();
#convert the slashes to URL slashes
relFilePath = urllib.pathname2url(os.path.relpath(f, env['VIP_IMAGE_SERVER_ROOT']));
basename = os.path.split(f)[-1]
relZoomPath = urllib.pathname2url(os.path.relpath(zoomifyName, env['VIP_IMAGE_SERVER_ROOT']));
with open(f, 'rb') as fid:
magic = fid.read(4)
image_info = {}
if magic == '49492A00'.decode('hex') or \
magic == '4D4D002A'.decode('hex'):
logger.debug('Tifffile: %s', f)
from tifffile import TiffFile
with TiffFile(f) as image:
if image.pages[0].dtype == 's':
image_info['dtype'] = numpy.dtype('S')
else:
image_info['dtype'] = numpy.dtype(image.pages[0].dtype)
image_info['bps'] = image.pages[0].bits_per_sample
image_info['height'] = image.pages[0].shape[0] #Yep, y,x,z order
image_info['width'] = image.pages[0].shape[1]
try:
image_info['bands'] = image.pages[0].shape[2]
except IndexError:
image_info['bands'] = 1
else:
logger.debug('Pil: %s', f)
from PIL import Image
with Image.open(f) as image:
#The getmode* commands do not give you the REAL datatypes. I need the
#REAL (numpy in this case) bps, not some random PIL designation
image_info['dtype'] = numpy.dtype(Image._MODE_CONV[image.mode][0])
#probably doesn't work well for bool... Oh well
image_info['bps'] = image_info['dtype'].itemsize*8
image_info['width'] = image.size[0] #Yep, x,y order
image_info['height'] = image.size[1]
image_info['bands'] = Image.getmodebands(image.mode)
img = voxel_globe.meta.models.Image.create(
name="Generic Upload %s (%s) Frame %s" % (uploadSession.name, uploadSession_id, basename),
imageWidth=image_info['width'],
imageHeight=image_info['height'],
numberColorBands=image_info['bands'],
pixelFormat=image_info['dtype'].char,
fileFormat='zoom',
imageUrl='%s://%s:%s/%s/%s/' % (env['VIP_IMAGE_SERVER_PROTOCOL'],
env['VIP_IMAGE_SERVER_HOST'],
env['VIP_IMAGE_SERVER_PORT'],
env['VIP_IMAGE_SERVER_URL_PATH'],
relZoomPath),
originalImageUrl='%s://%s:%s/%s/%s' % (env['VIP_IMAGE_SERVER_PROTOCOL'],
env['VIP_IMAGE_SERVER_HOST'],
env['VIP_IMAGE_SERVER_PORT'],
env['VIP_IMAGE_SERVER_URL_PATH'],
relFilePath),
service_id = self.request.id);
img.save();
imageCollection.images.add(img);
origin = [0,0,0];
logger.debug('Origin is: %s' % origin)
k = numpy.eye(3);
k[0,2] = image_info['width']/2;
k[1,2] = image_info['height']/2;
save_krt(self.request.id, img, k, r, t, origin);
voxel_globe.meta.models.Scene.create(name="Generic origin %s (%s)" % (uploadSession.name, uploadSession_id),
service_id = self.request.id,
geolocated=False,
origin='POINT(%0.12f %0.12f %0.12f)' % \
(0,0,0)).save()
uploadSession.delete()
