def catreal(image):
global generation_pk
generation, created = RealPointGeneration.objects.get_or_create(pk=generation_pk)
points = parse_cat(os.path.join(outdir, "cat", image.get_file() + ".cat"))
if points is None:
return
realpoints = []
for realpoint in image.realpoint_set.filter(generation=copy_generation_pk):
point = points.ix[realpoint.idx]
realpoint.pk = None
realpoint.id = None
realpoint.flux = point["flux"]
realpoint.flux_error = point["flux_error"]
realpoint.generation = generation
realpoints.append(realpoint)
RealPoint.objects.bulk_create(realpoints)
reset_queries()
def catmatch(image):
global generation_pk
generation, created = RealPointGeneration.objects.get_or_create(pk=generation_pk)
points = parse_cat(os.path.join(outdir, 'cat', image.get_file()+'.cat'))
if points is None:
return
realpoints = []
for i, point in points.iterrows():
if not in_mask(point):
continue
realpoint = RealPoint(
x=point['x'], y=point['y'],
x_min=point['x_min'], y_min=point['y_min'],
width=point['width'], height=point['height'],
flux=point['flux'], flux_error=point['flux_error'],
idx=i, image=image, generation=generation)
realpoints.append(realpoint)
RealPoint.objects.bulk_create(realpoints)
cursor = connection.cursor()
cursor.execute('UPDATE clouds_realpoint r SET sidpoint_id=(SELECT id FROM clouds_sidpoint WHERE sidtime_id=%s AND x>r.x-5 AND x<r.x+5 AND y>r.y-5 AND y<r.y+5 AND POW(x-r.x,2)+POW(y-r.y,2) < 9 ORDER BY POW(x-r.x,2)+POW(y-r.y,2) ASC LIMIT 1) WHERE r.image_id=%s', (image.sidtime_id,image.id,))
cursor.execute('SELECT r.id, sidpoint_id, POW(s.x-r.x,2)+POW(s.y-r.y,2) AS d FROM clouds_realpoint r JOIN clouds_sidpoint s ON s.id=r.sidpoint_id WHERE sidpoint_id IN (SELECT sidpoint_id FROM clouds_realpoint GROUP BY sidpoint_id HAVING COUNT(sidpoint_id)>1) AND r.image_id=%s ORDER BY d ASC', (image.id,))
done = {}
null = []
while True:
row = cursor.fetchone()
if not row: break
if row[1] in done:
null.append(str(row[0]))
else:
done[row[1]] = 0
if null:
cursor.execute('UPDATE clouds_realpoint SET sidpoint_id=NULL WHERE id IN ({0})'.format(','.join(null)))
cursor.execute('UPDATE clouds_realpoint r SET line_id=(SELECT line_id FROM clouds_sidpoint WHERE id=r.sidpoint_id) WHERE r.image_id=%s', (image.id,))
reset_queries()
import dateutil.parser
import os, shutil
dp = DataProcessor()
dp.outdir = 'test/out'
dp.verbose = 1
st = '0429'
path_end = os.path.join(st[0:2], st[2:4])
path = os.path.join('sid', path_end)
night = os.listdir(path)
dp.do_total = True
dp.indir = 'sid'
dp.do_filter = False
dp.do_output = False
dp.process_night(path, night)
from django.template import Context, Template
t = Template(open(os.path.join('clouds','templates','clouds','image.html')).read())
from catlib import parse_cat
point_list = map(lambda (i,row):row, parse_cat(os.path.join('test', 'out', 'cat', path, 'total.cat')).iterrows())
print len(point_list)
with open(os.path.join('test',st+'.html'), 'w') as out:
out.write(t.render(Context({'point_list': point_list,
'point_pk': -1,
'object': {'get_url': path+'/total' }
})))
def catsid(sidtime, prev_points_list, rerun=False):
siddir = os.path.join(outdir, 'cat', sidtime.get_dir())
path = os.path.join(siddir, 'total.cat')
if os.path.exists(path):
points = parse_cat(path)
else:
print "{0} does not exist.".format(path)
return
for i, point in points.iterrows():
if not in_mask(point):
continue
with transaction.commit_manually():
try:
#if i > 30:
# break
if rerun:
sidpoint = SidPoint.objects.get(sidtime=sidtime, idx=i)
if sidpoint.prev:
continue
matched = False
step = 0
for prev_sidtime, prev_points in prev_points_list:
step += 1
dsquared = (prev_points-point)**2
distances = dsquared['x'] + dsquared['y']
# Maybe fall back to second closest etc.
distance = distances.min()
if distance < 3**2:
idx = distances.idxmin()
# This name is confusing
try:
prev_point = SidPoint.objects.select_for_update().get(sidtime=prev_sidtime, idx=idx)
except SidPoint.DoesNotExist, e:
# Break if previous point is not in database
# Should only happen due to not being within the mask
break
#raise e
competing_point = prev_point.next
if competing_point: # then we have a collision
if competing_point.step >= i:
distance1 = ( (prev_point.x - competing_point.x)**2 +
(prev_point.y - competing_point.y)**2 )
if distance < distance1:
new_line = Line()
new_line.save()
competing_point.line = new_line
competing_point.prev = None
competing_point.save()
line = prev_point.line
matched = True
else:
line = prev_point.line
matched = True
break
if not matched:
if rerun:
continue
prev_point = None
step = None
line = Line()
line.save()
if rerun:
sidpoint.prev = prev_point
if sidpoint.line != line:
sidpoint.line.sidpoint_set.update(line=line)
sidpoint.line.delete()
sidpoint.line = line
else:
sidpoint = SidPoint(
x=point['x'], y=point['y'],
x_min=point['x_min'], y_min=point['y_min'],
width=point['width'], height=point['height'],
flux=point['flux'], flux_error=point['flux_error'],
line=line, idx=i, sidtime=sidtime, prev=prev_point, step=step)
sidpoint.save()
finally:
