def create_source_list(df, tempfiles=None):
img = None
fits = None
source_type = None
path = df.get_path()
print('path:', path, type(path))
try:
# see if disk file is a fits list
fits = fits_table(str(df.get_path()))
source_type = 'fits'
except:
logmsg('file is not a fits table')
# otherwise, check to see if it is a text list
try:
fitsfn = get_temp_file(tempfiles=tempfiles)
text_file = open(str(df.get_path()))
text = text_file.read()
text_file.close()
# add x y header
# potential hack, assumes it doesn't exist...
text = "# x y\n" + text
text_table = text_table_fields("", text=text)
text_table.write_to(fitsfn)
logmsg("Creating fits table from text list")
fits = fits_table(fitsfn)
source_type = 'text'
except Exception as e:
#logmsg('Traceback:\n' + traceback.format_exc())
logmsg('fitsfn: %s' % fitsfn)
logmsg(e)
logmsg('file is not a text list')
if fits:
try:
img = SourceList(disk_file=df, source_type=source_type)
# w = fits.x.max()-fits.x.min()
# h = fits.y.max()-fits.y.min()
# w = int(w)
# h = int(h)
w = int(math.ceil(fits.x.max()))
h = int(math.ceil(fits.y.max()))
logmsg('w %i, h %i' % (w, h))
if w < 1 or h < 1:
raise RuntimeError(
'Source list must contain POSITIVE x,y coordinates')
img.width = w
img.height = h
img.save()
except Exception as e:
logmsg(e)
img = None
raise e
return img
def handle_upload(file=None, url=None, tempfiles=None):
#logmsg('handle_uploaded_file: req=' + str(req))
#logmsg('handle_uploaded_file: req.session=' + str(req.session))
#logmsg('handle_uploaded_file: req.session.user='******'handle_uploaded_file: req.user='******'wb+') as uploaded_file:
original_filename = ''
if file:
for chunk in file.chunks():
uploaded_file.write(chunk)
file_hash.update(chunk)
original_filename = file.name
elif url:
logmsg('handling url upload')
f = urlopen(url) # nosec
CHUNK_SIZE = 4096
while True:
chunk = f.read(CHUNK_SIZE)
if not chunk:
break
uploaded_file.write(chunk)
file_hash.update(chunk)
p = urlparse(url)
p = p.path
if p:
s = p.split('/')
original_filename = s[-1]
else:
return None
df = DiskFile.from_file(temp_file_path,
collection=Image.ORIG_COLLECTION,
hashkey=file_hash.hexdigest())
return df, original_filename
def dosub(sub, tempfiles=None, tempdirs=None):
print('dosub: tempdir:', tempfile.gettempdir())
sub.set_processing_started()
sub.save()
print('Submission disk file:', sub.disk_file)
if sub.disk_file is None:
logmsg('Sub %i: retrieving URL' % (sub.id), sub.url)
(fn, headers) = urllib.request.urlretrieve(sub.url)
logmsg('Sub %i: wrote URL to file' % (sub.id), fn)
df = DiskFile.from_file(fn, Image.ORIG_COLLECTION)
logmsg('Created DiskFile', df)
# Try to split the URL into a filename component and save it
p = urlparse(sub.url)
p = p.path
if p:
s = p.split('/')
origname = s[-1]
sub.original_filename = origname
df.save()
sub.disk_file = df
sub.save()
logmsg('Saved DiskFile', df)
else:
logmsg('uploaded disk file for this submission is ' +
str(sub.disk_file))
df = sub.disk_file
fn = df.get_path()
logmsg('DiskFile path ' + fn)
original_filename = sub.original_filename
# check if file is a gzipped file
try:
with gzip.open(fn) as gzip_file:
tempfn = get_temp_file(tempfiles=tempfiles)
with open(tempfn, 'wb') as f:
# should fail on the following line if not a gzip file
f.write(gzip_file.read())
df = DiskFile.from_file(tempfn, 'uploaded-gunzip')
i = original_filename.find('.gz')
if i != -1:
original_filename = original_filename[:i]
logmsg('extracted gzip file %s' % original_filename)
#fn = tempfn
fn = df.get_path()
except:
# not a gzip file
pass
is_tar = False
try:
is_tar = tarfile.is_tarfile(fn)
except:
pass
if is_tar:
logmsg('File %s: tarball' % fn)
tar = tarfile.open(fn)
dirnm = tempfile.mkdtemp()
if tempdirs is not None:
tempdirs.append(dirnm)
for tarinfo in tar.getmembers():
if tarinfo.isfile():
logmsg('extracting file %s' % tarinfo.name)
tar.extract(tarinfo, dirnm)
tempfn = os.path.join(dirnm, tarinfo.name)
df = DiskFile.from_file(tempfn, 'uploaded-untar')
# create Image object
img = get_or_create_image(df, tempfiles=tempfiles)
# create UserImage object.
if img:
create_user_image(sub, img, tarinfo.name)
tar.close()
shutil.rmtree(dirnm, ignore_errors=True)
else:
# assume file is single image
logmsg('File %s: single file' % fn)
# create Image object
img = get_or_create_image(df, tempfiles=tempfiles)
logdebug('File %s: created Image %s' % (fn, str(img)))
# create UserImage object.
if img:
uimg = create_user_image(sub, img, original_filename)
logmsg('File %s: Image id %i, UserImage id %i' %
(fn, img.id, uimg.id))
sub.set_processing_finished()
sub.save()
return sub.id
'propagate': False,
}
tempdir = os.path.join(settings.TEMPDIR, 'proc-subs')
try:
os.makedirs(tempdir)
except:
pass
settings.TEMPDIR = tempdir
os.environ['TMP'] = tempdir
os.environ['TMPDIR'] = tempdir
import tempfile
from astrometry.net.tmpfile import get_temp_file
print('get_temp_file():', get_temp_file())
from astrometry.util import image2pnm
from astrometry.util.filetype import filetype_short
from astrometry.util.run_command import run_command
from astrometry.util.util import Tan
from astrometry.util import util as anutil
from astrometry.util.fits import *
from astrometry.net.models import *
from astrometry.net.log import logmsg
from django.db.models import Count
from django.db import DatabaseError
from django.db.models import Q
#import logging