def publisher_upload(request, pub_name):
"""Display the publishers upload.html template on a GET of the page.
If the view is POSTed to, the pass the uploaded data to the publisher.
"""
pub = models.Publisher.objects.get(name=pub_name)
if request.method == 'POST':
meta = request.POST.dict()
files = request.FILES.values()
if len(files) == 0:
return render_to_json({"error": "Error: No file selected for upload"})
else:
try:
upload, async = edit_upload(pub, files[0], json.dumps(meta))
return render_to_json({"status": upload.status, "id": upload.id})
except Exception as e:
return render_to_json({"error": str(e)})
else:
action = request.get_full_path()
error = ''
contents = ''
try:
path = os.path.join(pub.path, 'upload.html')
with open(path, 'r') as f:
contents = f.read()
except:
error = "Error: Unable to read %s" % path
ctx = RequestContext(request, {"contents": contents, "action": action, "error": error})
return render_to_response('rundb/configure/modal_publisher_upload.html', context_instance=ctx)
def publisher_upload(request, pub_name):
"""Display the publishers upload.html template on a GET of the page.
If the view is POSTed to, the pass the uploaded data to the publisher.
"""
pub = models.Publisher.objects.get(name=pub_name)
if request.method == 'POST':
meta = request.POST.dict()
files = request.FILES.values()
if len(files) == 0:
return render_to_json({"error": "Error: No file selected for upload"})
else:
try:
meta['username'] = request.user.username
upload, async = edit_upload(pub, files[0], json.dumps(meta))
return render_to_json({"status": upload.status, "id": upload.id})
except Exception as e:
return render_to_json({"error": str(e)})
else:
action = request.get_full_path()
error = ''
contents = ''
try:
path = os.path.join(pub.path, 'upload.html')
with open(path, 'r') as f:
contents = f.read()
except:
error = "Error: Unable to read %s" % path
ctx = RequestContext(request, {"contents": contents, "action": action, "error": error})
return render_to_response('rundb/configure/modal_publisher_upload.html', context_instance=ctx)
def genome_status(request, reference_id):
"""Provide a way for the index creator to let us know when the index has been created"""
if request.method == "POST":
rg = get_object_or_404(ReferenceGenome, pk=reference_id)
status = request.POST.get('status', False)
enabled = request.POST.get('enabled', False)
verbose_error = request.POST.get('verbose_error', "")
index_version = request.POST.get('index_version', "")
if not status:
return render_to_json({"status": "error genome status not given"})
rg.status = status
rg.enabled = enabled
rg.verbose_error = verbose_error
rg.index_version = index_version
rg.reference_path = os.path.join(settings.TMAP_DIR, rg.short_name)
rg.save()
return render_to_json({
"status": "genome status updated",
"enabled": enabled
})
if request.method == "GET":
rg = get_object_or_404(ReferenceGenome, pk=reference_id)
return render_to_json({"status": rg.status})
def add_custom_genome(request):
''' Import custom genome via file upload or URL '''
if request.method == "POST":
url = request.POST.get("reference_url", None)
target_file = request.POST.get('target_file', False)
reference_args = {
"short_name": request.POST.get("short_name"),
"name": request.POST.get("name"),
"version": request.POST.get("version", ""),
"notes": request.POST.get("notes", ""),
"index_version": ""
}
if target_file:
# import via file upload
reference_path = os.path.join(settings.TEMP_PATH, target_file)
reference_args["source"] = reference_path
# check expected file size
failed = False
reported_file_size = request.POST.get('reported_file_size', False)
try:
uploaded_file_size = str(os.path.getsize(reference_path))
if reported_file_size and (reported_file_size !=
uploaded_file_size):
failed = "Upload error: uploaded file size is incorrect"
except OSError:
failed = "Upload error: temporary file not found"
try:
new_reference_genome(reference_args, None, target_file)
except Exception as e:
failed = str(e)
if failed:
try:
os.remove(reference_path)
except OSError:
failed += " The FASTA file could not be deleted."
logger.error("Failed uploading genome file: " + failed)
return render_to_json({"status": failed, "error": True})
else:
return render_to_json({"error": False})
else:
# import via URL
reference_args["source"] = url
try:
new_reference_genome(reference_args, url, None)
except Exception as e:
return render_to_json({"status": str(e), "error": True})
else:
return HttpResponseRedirect(
urlresolvers.reverse("configure_references"))
elif request.method == "GET":
return render_to_response(
"rundb/configure/modal_references_new_genome.html",
context_instance=RequestContext(request, {}))
def write_plupload(request, pub_name):
"""file upload for plupload"""
logger.info("Starting write plupload")
pub = models.Publisher.objects.get(name=pub_name)
meta = request.POST.get('meta','{}')
logger.debug("%s" % meta)
if request.method == 'POST':
name = request.REQUEST.get('name', '')
uploaded_file = request.FILES['file']
if not name:
name = uploaded_file.name
logger.debug("plupload name = '%s'" % name)
try:
validate_plupload(request, pub_name, name)
except Exception as err:
return HttpResponseBadRequest(str(err))
upload_dir = "/results/referenceLibrary/temp"
if not os.path.exists(upload_dir):
return render_to_json({"error": "upload path does not exist"})
dest_path = os.path.join(upload_dir, name)
logger.debug("plupload destination = '%s'" % dest_path)
chunk = request.REQUEST.get('chunk', '0')
chunks = request.REQUEST.get('chunks', '0')
logger.debug("plupload chunk %s %s of %s" % (str(type(chunk)), str(chunk), str(chunks)))
debug = [chunk, chunks]
with open(dest_path, ('wb' if chunk == '0' else 'ab')) as f:
for content in uploaded_file.chunks():
logger.debug("content chunk = '%d'" % len(content))
f.write(content)
my_contentupload_id = None
if int(chunk) + 1 >= int(chunks):
try:
meta = json.loads(meta)
meta['username'] = request.user.username
upload = move_upload(pub, dest_path, name, json.dumps(meta))
async_upload = run_pub_scripts.delay(pub, upload)
my_contentupload_id = upload.id
except Exception as err:
logger.exception("There was a problem during upload of a file for a publisher.")
else:
logger.info("Successfully pluploaded %s" % name)
logger.debug("plupload done")
return render_to_json({"chunk posted": debug, "contentupload_id": my_contentupload_id})
else:
return render_to_json({"method": "only post here"})
def write_plupload(request, pub_name):
"""file upload for plupload"""
logger.info("Starting write plupload")
pub = models.Publisher.objects.get(name=pub_name)
logger.debug("%s %s" % (str(type(request.REQUEST['meta'])), request.REQUEST['meta']))
logger.debug("Publisher Plupload started")
if request.method == 'POST':
name = request.REQUEST.get('name', '')
uploaded_file = request.FILES['file']
if not name:
name = uploaded_file.name
logger.debug("plupload name = '%s'" % name)
#check to see if a user has uploaded a file before, and if they have
#not, make them a upload directory
upload_dir = "/results/referenceLibrary/temp"
if not os.path.exists(upload_dir):
return render_to_json({"error": "upload path does not exist"})
dest_path = os.path.join(upload_dir, name)
logger.debug("plupload destination = '%s'" % dest_path)
chunk = request.REQUEST.get('chunk', '0')
chunks = request.REQUEST.get('chunks', '0')
logger.debug("plupload chunk %s %s of %s" % (str(type(chunk)), str(chunk), str(chunks)))
debug = [chunk, chunks]
with open(dest_path, ('wb' if chunk == '0' else 'ab')) as f:
for content in uploaded_file.chunks():
logger.debug("content chunk = '%d'" % len(content))
f.write(content)
my_contentupload_id = None
if int(chunk) + 1 >= int(chunks):
try:
upload = move_upload(pub, dest_path, name, request.REQUEST['meta'])
async_upload = run_pub_scripts.delay(pub, upload)
my_contentupload_id = upload.id
except Exception as err:
logger.exception("There was a problem during upload of a file for a publisher.")
else:
logger.info("Successfully pluploaded %s" % name)
logger.debug("plupload done")
return render_to_json({"chunk posted": debug, "contentupload_id": my_contentupload_id})
else:
return render_to_json({"method": "only post here"})
def add_custom_genome(request):
''' Import custom genome via file upload or URL '''
if request.method == "POST":
url = request.POST.get("reference_url", None)
target_file = request.POST.get('target_file', False)
reference_args = {
"short_name": request.POST.get("short_name"),
"name": request.POST.get("name"),
"version": request.POST.get("version", ""),
"notes": request.POST.get("notes", ""),
"index_version": ""
}
if target_file:
# import via file upload
reference_path = os.path.join(settings.TEMP_PATH, target_file)
reference_args["source"] = reference_path
# check expected file size
failed = False
reported_file_size = request.POST.get('reported_file_size', False)
try:
uploaded_file_size = str(os.path.getsize(reference_path))
if reported_file_size and (reported_file_size != uploaded_file_size):
failed = "Upload error: uploaded file size is incorrect"
except OSError:
failed = "Upload error: temporary file not found"
try:
new_reference_genome(reference_args, None, target_file)
except Exception as e:
failed = str(e)
if failed:
try:
os.remove(reference_path)
except OSError:
failed += " The FASTA file could not be deleted."
logger.error("Failed uploading genome file: " + failed)
return render_to_json({"status": failed, "error": True})
else:
return render_to_json({"error": False})
else:
# import via URL
reference_args["source"] = url
try:
new_reference_genome(reference_args, url, None)
except Exception as e:
return render_to_json({"status": str(e), "error": True})
else:
return HttpResponseRedirect(urlresolvers.reverse("configure_references"))
elif request.method == "GET":
return render_to_response("rundb/configure/modal_references_new_genome.html", context_instance=RequestContext(request, {}))
def readTimezone(request):
# read /etc/timezone into the string timezone
timezone = ''
with open('/etc/timezone', 'rb') as fh:
timezone = fh.read()
fh.close()
return render_to_json({"timezone": timezone.strip()})
def download_genome(request):
# called by "Import Preloaded Ion References"
if request.method == "POST":
reference_meta = request.POST.get("reference_meta", None)
ref_annot_update = request.POST.get("ref_annot_update", None)
reference_args = json.loads(base64.b64decode(reference_meta))
annotation_meta = request.POST.get("missingAnnotation_meta", None)
reference_mask_info = request.POST.get("reference_mask", None)
if annotation_meta:
annotation_data = json.loads(base64.b64decode(annotation_meta))
annotation_lists = annotation_data
# Download and register only the Ref Annotation file if Reference Genome is already Imported
# If not, download refAnnot file and Reference Genome asynchronously
if annotation_data and ref_annot_update:
logger.debug(
"Downloading Annotation File {0} with meta {1}".format(
annotation_data, reference_meta))
if annotation_lists:
download_genome_annotation(annotation_lists, reference_args)
else:
url = request.POST.get("reference_url", None)
logger.debug("Downloading {0} with meta {1}".format(
url, reference_meta))
if url is not None:
if annotation_lists:
download_genome_annotation(annotation_lists,
reference_args)
try:
new_reference_genome(
reference_args,
url,
reference_mask_filename=reference_mask_info)
except Exception as e:
return render_to_json({"status": str(e), "error": True})
return HttpResponseRedirect(
urlresolvers.reverse("references_genome_download"))
elif request.method == "GET":
references = get_references() or []
downloads = FileMonitor.objects.filter(
tags__contains="reference").order_by('-created')
downloads_annot = FileMonitor.objects.filter(
tags__contains="reference_annotation").order_by('-created')
(references, downloads_annot) = get_annotation(references,
downloads_annot)
ctx = {
'downloads': downloads,
'downloads_annot': downloads_annot,
'references': references
}
return render_to_response("rundb/configure/reference_download.html",
ctx,
context_instance=RequestContext(request))
def download_genome(request):
# called by "Import Preloaded Ion References"
if request.method == "POST":
reference_meta = request.POST.get("reference_meta", None)
reference_args = json.loads(base64.b64decode(reference_meta))
reference_mask_info = request.POST.get("reference_mask", None)
url = request.POST.get("reference_url", None)
logger.debug("Downloading {0} with meta {1}".format(url, reference_meta))
if url is not None:
try:
new_reference_genome(reference_args, url, reference_mask_filename=reference_mask_info)
except Exception as e:
return render_to_json({"status": str(e), "error": True})
return HttpResponseRedirect(urlresolvers.reverse("references_genome_download"))
elif request.method == "GET":
references = get_references() or []
downloads = FileMonitor.objects.filter(tags__in=["reference","bedfile","annotation"]).order_by('-created')
# update BED files available for pre-loaded references
for ref in references:
if ref['installed'] or ref.get('preInstalled'):
bedfiles = []
for bedfile in ref.get('bedfiles',[]):
status = get_bedfile_status(bedfile, ref['meta']['short_name'])
if status != "_installed":
bedfile['status'] = status
bedfiles.append(bedfile)
ref['bedfiles'] = bedfiles
annotationfiles = []
#if ref['meta']['short_name'] is "hg19":
#import pdb;pdb.set_trace()
for annotationfile in ref.get('annotation', []):
status = get_annotationfile_status(annotationfile, ref['meta']['short_name'])
if status != "_installed":
annotationfile['status'] = status
annotationfiles.append(annotationfile)
ref['annotationfiles'] = annotationfiles
# set up page to refresh
_in_progress_status = ["Queued", "Starting", "Downloading", "Preprocessing", "Indexing"]
_in_progress_status += [s.lower() for s in _in_progress_status]
downloading = downloads.filter(status__in=_in_progress_status)
id_hashes = [r['meta']['identity_hash'] for r in references if r['meta']['identity_hash']]
processing = ReferenceGenome.objects.filter(identity_hash__in=id_hashes, status__in=_in_progress_status)
ctx = {
'downloads': downloads,
'references': references,
'refresh_progress': downloading.count() > 0 or processing.count() > 0
}
return render_to_response("rundb/configure/reference_download.html", ctx, context_instance=RequestContext(request))
def delete_genome(request, pk):
"""delete a reference genome
the filesystem file deletions should be done with a method on the model"""
if request.method == "POST":
ref_genome = get_object_or_404(ReferenceGenome, pk=pk)
#delete dir by default
try_delete = ref_genome.delete()
if not try_delete:
#the file could not be deleted, present the user with an error message.
return render_to_json({"status": " <strong>Error</strong> <p>Genome could not be deleted.</p> \
<p>Check the file permissions for the genome on the file system at: </p> \
<p><strong>" + str(ref_genome.reference_path) + "</p></strong> "})
return render_to_json({"status": "Genome was deleted successfully"})
if request.method == "GET":
return render_to_json({"status": "This must be accessed via post"})
def delete_genome(request, pk):
"""delete a reference genome
the filesystem file deletions should be done with a method on the model"""
if request.method == "POST":
ref_genome = get_object_or_404(ReferenceGenome, pk=pk)
#delete dir by default
try_delete = ref_genome.delete()
if not try_delete:
#the file could not be deleted, present the user with an error message.
return render_to_json({"status": " <strong>Error</strong> <p>Genome could not be deleted.</p> \
<p>Check the file permissions for the genome on the file system at: </p> \
<p><strong>" + str(ref_genome.reference_path) + "</p></strong> "})
return render_to_json({"status": "Genome was deleted successfully"})
if request.method == "GET":
return render_to_json({"status": "This must be accessed via post"})
def plupload_file_upload(
request,
dest_dir,
request_filename_attr='name',
request_files_attr='file',
):
""" A shared file upload to save an uploaded file to disk when using plupload.
request - the HTTPRequest object
dest_dir - the destination directory to save the file within
"""
if request.method == 'POST':
name = request.POST.get(request_filename_attr, '')
uploaded_file = request.FILES[request_files_attr]
if not name:
name = uploaded_file.name
name, ext = os.path.splitext(name)
#check to see if a user has uploaded a file before, and if they have
#not, make them a upload directory
if not os.path.exists(dest_dir):
return render_to_json({"error": "upload path does not exist"})
dest_path = '%s%s%s%s' % (dest_dir, os.sep, name, ext)
chunk = request.POST.get('chunk', '0')
chunks = request.POST.get('chunks', '0')
debug = [chunk, chunks]
with open(dest_path, ('wb' if chunk == 0 else 'ab')) as outfile:
for content in uploaded_file.chunks():
outfile.write(content)
if int(chunk) + 1 >= int(chunks):
#the upload has finished
pass
return render_to_json({"chuck posted": debug})
else:
return render_to_json({"method": "only POST here"})
def plupload_file_upload(request, dest_dir, request_filename_attr='name', request_files_attr='file', ):
""" A shared file upload to save an uploaded file to disk when using plupload.
request - the HTTPRequest object
dest_dir - the destination directory to save the file within
"""
if request.method == 'POST':
name = request.POST.get(request_filename_attr, '')
uploaded_file = request.FILES[request_files_attr]
if not name:
name = uploaded_file.name
name, ext = os.path.splitext(name)
#check to see if a user has uploaded a file before, and if they have
#not, make them a upload directory
if not os.path.exists(dest_dir):
return render_to_json({"error": "upload path does not exist"})
dest_path = '%s%s%s%s' % (dest_dir, os.sep, name, ext)
chunk = request.POST.get('chunk', '0')
chunks = request.POST.get('chunks', '0')
debug = [chunk, chunks]
with open(dest_path, ('wb' if chunk == 0 else 'ab')) as outfile:
for content in uploaded_file.chunks():
outfile.write(content)
if int(chunk) + 1 >= int(chunks):
#the upload has finished
pass
return render_to_json({"chuck posted": debug})
else:
return render_to_json({"method": "only POST here"})
def genome_status(request, reference_id):
"""Provide a way for the index creator to let us know when the index has been created"""
if request.method == "POST":
rg = get_object_or_404(ReferenceGenome, pk=reference_id)
status = request.POST.get('status', False)
enabled = request.POST.get('enabled', False)
verbose_error = request.POST.get('verbose_error', "")
index_version = request.POST.get('index_version', "")
if not status:
return render_to_json({"status": "error genome status not given"})
rg.status = status
rg.enabled = enabled
rg.verbose_error = verbose_error
rg.index_version = index_version
rg.reference_path = os.path.join(settings.TMAP_DIR, rg.short_name)
rg.save()
return render_to_json({"status": "genome status updated", "enabled": enabled})
if request.method == "GET":
rg = get_object_or_404(ReferenceGenome, pk=reference_id)
return render_to_json({"status": rg.status})
def download_genome(request):
# called by "Import Preloaded Ion References"
if request.method == "POST":
reference_meta = request.POST.get("reference_meta", None)
ref_annot_update = request.POST.get("ref_annot_update", None)
reference_args = json.loads(base64.b64decode(reference_meta))
annotation_meta = request.POST.get("missingAnnotation_meta", None)
reference_mask_info = request.POST.get("reference_mask", None)
if annotation_meta:
annotation_data = json.loads(base64.b64decode(annotation_meta))
annotation_lists = annotation_data
# Download and register only the Ref Annotation file if Reference Genome is already Imported
# If not, download refAnnot file and Reference Genome asynchronously
if annotation_data and ref_annot_update:
logger.debug("Downloading Annotation File {0} with meta {1}".format(annotation_data, reference_meta))
if annotation_lists:
download_genome_annotation(annotation_lists, reference_args)
else:
url = request.POST.get("reference_url", None)
logger.debug("Downloading {0} with meta {1}".format(url, reference_meta))
if url is not None:
if annotation_lists:
download_genome_annotation(annotation_lists, reference_args)
try:
new_reference_genome(reference_args, url, reference_mask_filename=reference_mask_info)
except Exception as e:
return render_to_json({"status": str(e), "error": True})
return HttpResponseRedirect(urlresolvers.reverse("references_genome_download"))
elif request.method == "GET":
references = get_references() or []
downloads = FileMonitor.objects.filter(tags__contains="reference").order_by('-created')
downloads_annot = FileMonitor.objects.filter(tags__contains="reference_annotation").order_by('-created')
(references, downloads_annot) = get_annotation(references, downloads_annot)
ctx = {
'downloads': downloads,
'downloads_annot': downloads_annot,
'references': references
}
return render_to_response("rundb/configure/reference_download.html", ctx, context_instance=RequestContext(request))
def new_genome(request):
"""This is the page to create a new genome. The XML-RPC server is ionJobServer.
"""
if request.method == "POST":
# parse the data sent in
#required
name = request.POST.get('name', False)
short_name = request.POST.get('short_name', False)
fasta = request.POST.get('target_file', False)
version = request.POST.get('version', False)
notes = request.POST.get('notes', "")
#optional
read_exclude_length = request.POST.get('read_exclude_length', False)
#URL download
url = request.POST.get('url', False)
error_status = ""
reference_path = REFERENCE_LIBRARY_TEMP_DIR + fasta
why_delete = ""
#if any of those were false send back a failed message
if not all((name, short_name, fasta, version)):
return render_to_json({
"status": "Form validation failed",
"error": True
})
if not set(short_name).issubset(
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
):
return render_to_json({
"status":
"The short name has invalid characters. The valid values are letters, numbers, and underscores.",
"error": True
})
#TODO: check to make sure the zip file only has one fasta or fa
if not url:
#check to ensure the size on the OS the same as the reported.
reported_file_size = request.POST.get('reported_file_size', False)
try:
uploaded_file_size = str(os.path.getsize(reference_path))
except OSError:
return render_to_json({
"status": "The FASTA temporary files was not found",
"error": True
})
if reported_file_size != uploaded_file_size:
why_delete = "The file you uploaded differs from the expected size. This is due to an error uploading."
if not (fasta.lower().endswith(".fasta")
or fasta.lower().endswith(".zip")):
why_delete = "The file you uploaded does not have a .fasta or .zip extension. It must be a plain text fasta file or a Zip compressed fasta."
if why_delete:
try:
os.remove(reference_path)
except OSError:
why_delete += " The FASTA file could not be deleted."
return render_to_json({"status": why_delete, "error": True})
#Make an genome ref object
if ReferenceGenome.objects.filter(short_name=short_name,
index_version=settings.TMAP_VERSION):
#check to see if the genome already exists in the database with the same version
return render_to_json({
"status":
"Failed - Genome with this short name and index version already exist.",
"error": True
})
ref_genome = ReferenceGenome()
ref_genome.name = name
ref_genome.short_name = short_name
ref_genome.version = version
ref_genome.date = datetime.datetime.now()
ref_genome.notes = notes
ref_genome.status = "queued"
ref_genome.enabled = False
ref_genome.index_version = settings.TMAP_VERSION
#before the object is saved we should ping the xml-rpc server to see if it is alive.
try:
conn = client.connect(JOBSERVER_HOST, settings.JOBSERVER_PORT)
#just check uptime to make sure the call does not fail
conn.uptime()
logger.debug('Connected to ionJobserver process.')
except (socket.error, xmlrpclib.Fault):
return render_to_json({
"status":
"Unable to connect to ionJobserver process. You may need to restart ionJobserver",
"error": True
})
#if the above didn't fail then we can save the object
#this object must be saved before the tmap call is made
ref_genome.save()
logger.debug('Saved ReferenceGenome %s' % ref_genome.__dict__)
#kick off the anaserve tmap xmlrpc call
import traceback
try:
conn = client.connect(JOBSERVER_HOST, settings.JOBSERVER_PORT)
tmap_bool, tmap_status = conn.tmap(str(ref_genome.id), fasta,
short_name, name, version,
read_exclude_length,
settings.TMAP_VERSION)
logger.debug('ionJobserver process reported %s %s' %
(tmap_bool, tmap_status))
except (socket.error, xmlrpclib.Fault):
#delete the genome object, because it was not sucessful
ref_genome.delete()
return render_to_json({
"status": "Error with index creation",
"error": traceback.format_exc()
})
if not tmap_bool:
ref_genome.delete()
return render_to_json({"status": tmap_status, "error": True})
return render_to_json({
"status":
"The genome index is being created. This might take a while, check the status on the references tab. \
You are being redirected there now.",
"error": False
})
elif request.method == "GET":
ctx = RequestContext(request, {})
return render_to_response(
"rundb/configure/modal_references_new_genome.html",
context_instance=ctx)
def new_genome(request):
"""This is the page to create a new genome.
"""
def is_fasta(filename):
ext = os.path.splitext(filename)[1]
return ext.lower() in ['.fasta', '.fas', '.fa', '.seq']
if request.method == "POST":
# parse the data sent in
#required
name = request.POST.get('name', False)
short_name = request.POST.get('short_name', False)
fasta = request.POST.get('target_file', False)
version = request.POST.get('version', "")
notes = request.POST.get('notes', "")
#optional
read_exclude_length = request.POST.get('read_exclude_length', False)
#URL download
url = request.POST.get('url', False)
reference_path = os.path.join(settings.TEMP_PATH, fasta)
why_delete = ""
#if any of those were false send back a failed message
if not all((name, short_name, fasta)):
return render_to_json({
"status": "Form validation failed",
"error": True
})
if not set(short_name).issubset(
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
):
return render_to_json({
"status":
"The short name has invalid characters. The valid values are letters, numbers, and underscores.",
"error": True
})
#TODO: check to make sure the zip file only has one fasta or fa
if not url:
#check to ensure the size on the OS the same as the reported.
reported_file_size = request.POST.get('reported_file_size', False)
try:
uploaded_file_size = str(os.path.getsize(reference_path))
except OSError:
return render_to_json({
"status": "The FASTA temporary files was not found",
"error": True
})
if reported_file_size != uploaded_file_size:
why_delete = "The file you uploaded differs from the expected size. This is due to an error uploading."
if not (is_fasta(fasta) or fasta.lower().endswith(".zip")):
why_delete = "The file you uploaded does not have a FASTA or ZIP extension. It must be a plain text a Zip compressed fasta file."
is_zip = zipfile.is_zipfile(reference_path)
if is_zip:
zip_file = zipfile.ZipFile(reference_path, 'r')
files = zip_file.namelist()
# MAC OS zip is being compressed with __MACOSX folder Ex: '__MACOSX/', '__MACOSX/._contigs_2.fasta'.
# Filter out those files and Upload only FASTA file
files = [x for x in files if not 'MACOSX' in x]
zip_file.close()
else:
files = [fasta]
fasta_files = filter(lambda x: is_fasta(x), files)
if len(fasta_files) != 1:
why_delete = "Error: upload must contain exactly one fasta file"
else:
target_fasta_file = fasta_files[0]
if why_delete:
try:
os.remove(reference_path)
except OSError:
why_delete += " The FASTA file could not be deleted."
logger.warning("User uploaded bad fasta file: " + str(why_delete))
return render_to_json({"status": why_delete, "error": True})
#Make an genome ref object
if ReferenceGenome.objects.filter(short_name=short_name,
index_version=settings.TMAP_VERSION):
#check to see if the genome already exists in the database with the same version
return render_to_json({
"status":
"Failed - Genome with this short name and index version already exist.",
"error": True
})
ref_genome = ReferenceGenome()
ref_genome.name = name
ref_genome.short_name = short_name
ref_genome.version = version
ref_genome.notes = notes
ref_genome.status = "preprocessing"
ref_genome.enabled = False
ref_genome.index_version = settings.TMAP_VERSION
ref_genome.save()
logger.debug("Created new reference: %d/%s" %
(ref_genome.pk, ref_genome))
temp_dir = tempfile.mkdtemp(suffix=short_name, dir=settings.TEMP_PATH)
temp_upload_path = os.path.join(temp_dir, fasta)
os.chmod(temp_dir, 0777)
os.rename(reference_path, temp_upload_path)
monitor = FileMonitor(local_dir=temp_dir, name=fasta)
monitor.save()
ref_genome.file_monitor = monitor
ref_genome.reference_path = temp_upload_path
ref_genome.save()
index_task = tasks.build_tmap_index.subtask((ref_genome.id, ),
immutable=True)
if is_zip:
result = tasks.unzip_reference.apply_async(
args=(ref_genome.id, target_fasta_file), link=index_task)
else:
result = tasks.copy_reference.apply_async(args=(ref_genome.id, ),
link=index_task)
ref_genome.status = "queued"
ref_genome.celery_task_id = result.task_id
ref_genome.save()
return render_to_json({
"status":
"The genome index is being created. This might take a while, check the status on the references tab. \
You are being redirected there now.",
"error": False
})
elif request.method == "GET":
ctx = RequestContext(request, {})
return render_to_response(
"rundb/configure/modal_references_new_genome.html",
context_instance=ctx)
def new_genome(request):
"""This is the page to create a new genome.
"""
if request.method == "POST":
# parse the data sent in
#required
name = request.POST.get('name', False)
short_name = request.POST.get('short_name', False)
fasta = request.POST.get('target_file', False)
version = request.POST.get('version', "")
notes = request.POST.get('notes', "")
#optional
read_exclude_length = request.POST.get('read_exclude_length', False)
#URL download
url = request.POST.get('url', False)
reference_path = os.path.join(settings.TEMP_PATH, fasta)
why_delete = ""
#if any of those were false send back a failed message
if not all((name, short_name, fasta)):
return render_to_json({"status": "Form validation failed", "error": True})
if not set(short_name).issubset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"):
return render_to_json({"status": "The short name has invalid characters. The valid values are letters, numbers, and underscores.", "error": True})
#TODO: check to make sure the zip file only has one fasta or fa
if not url:
#check to ensure the size on the OS the same as the reported.
reported_file_size = request.POST.get('reported_file_size', False)
try:
uploaded_file_size = str(os.path.getsize(reference_path))
except OSError:
return render_to_json({"status": "The FASTA temporary files was not found", "error": True})
if reported_file_size != uploaded_file_size:
why_delete = "The file you uploaded differs from the expected size. This is due to an error uploading."
if not (fasta.lower().endswith(".fasta") or fasta.lower().endswith(".zip")):
why_delete = "The file you uploaded does not have a .fasta or .zip extension. It must be a plain text fasta file or a Zip compressed fasta."
is_zip = zipfile.is_zipfile(reference_path)
if is_zip:
zip_file = zipfile.ZipFile(reference_path, 'r')
files = zip_file.namelist()
# MAC OS zip is being compressed with __MACOSX folder Ex: '__MACOSX/', '__MACOSX/._contigs_2.fasta'.
# Filter out those files and Upload only FASTA file
files = [x for x in files if not 'MACOSX' in x]
zip_file.close()
else:
files = [fasta]
fasta_files = filter(lambda x: x.endswith('.fa') or x.endswith('.fasta'), files)
if len(fasta_files) != 1:
why_delete = "Error: upload must contain exactly one fasta file"
else:
target_fasta_file = fasta_files[0]
if why_delete:
try:
os.remove(reference_path)
except OSError:
why_delete += " The FASTA file could not be deleted."
logger.warning("User uploaded bad fasta file: " + str(why_delete))
return render_to_json({"status": why_delete, "error": True})
#Make an genome ref object
if ReferenceGenome.objects.filter(short_name=short_name, index_version=settings.TMAP_VERSION):
#check to see if the genome already exists in the database with the same version
return render_to_json({"status": "Failed - Genome with this short name and index version already exist.", "error": True})
ref_genome = ReferenceGenome()
ref_genome.name = name
ref_genome.short_name = short_name
ref_genome.version = version
ref_genome.notes = notes
ref_genome.status = "preprocessing"
ref_genome.enabled = False
ref_genome.index_version = settings.TMAP_VERSION
ref_genome.save()
logger.debug("Created new reference: %d/%s" % (ref_genome.pk, ref_genome))
temp_dir = tempfile.mkdtemp(suffix=short_name, dir=settings.TEMP_PATH)
temp_upload_path = os.path.join(temp_dir, fasta)
os.chmod(temp_dir, 0777)
os.rename(reference_path, temp_upload_path)
monitor = FileMonitor(
local_dir=temp_dir,
name=fasta
)
monitor.save()
ref_genome.file_monitor = monitor
ref_genome.reference_path = temp_upload_path
ref_genome.save()
index_task = tasks.build_tmap_index.subtask((ref_genome.id,), immutable=True)
if is_zip:
result = tasks.unzip_reference.apply_async(
args=(ref_genome.id, target_fasta_file),
link=index_task
)
else:
result = tasks.copy_reference.apply_async(
args=(ref_genome.id,),
link=index_task
)
ref_genome.status = "queued"
ref_genome.celery_task_id = result.task_id
ref_genome.save()
return render_to_json({"status": "The genome index is being created. This might take a while, check the status on the references tab. \
You are being redirected there now.", "error": False})
elif request.method == "GET":
ctx = RequestContext(request, {})
return render_to_response("rundb/configure/modal_references_new_genome.html", context_instance=ctx)
def write_plupload(request, pub_name):
"""file upload for plupload"""
logger.info("Starting write plupload")
pub = models.Publisher.objects.get(name=pub_name)
meta = request.POST.get("meta", "{}")
logger.debug("%s" % meta)
if request.method == "POST":
name = request.REQUEST.get("name", "")
uploaded_file = request.FILES["file"]
if not name:
name = uploaded_file.name
logger.debug("plupload name = '%s'" % name)
try:
labelsContent = labels.TargetRegionsContent # default case
if RepresentsJSON(meta):
metaJson = json.loads(meta)
labelsContent = (labels.HotspotsContent
if metaJson and metaJson["hotspot"] else
labels.TargetRegionsContent)
file_name = validate_plupload(request, pub_name, name,
labelsContent)
except Exception as err:
return HttpResponseBadRequest(unicode(err))
upload_dir = "/results/referenceLibrary/temp"
if not os.path.exists(upload_dir):
return render_to_json(
{"error": validation.missing_error(upload_dir)})
dest_path = os.path.join(upload_dir, file_name)
logger.debug("plupload destination = '%s'" % dest_path)
chunk = request.REQUEST.get("chunk", "0")
chunks = request.REQUEST.get("chunks", "0")
logger.debug("plupload chunk %s %s of %s" %
(str(type(chunk)), str(chunk), str(chunks)))
debug = [chunk, chunks]
with open(dest_path, ("wb" if chunk == "0" else "ab")) as f:
for content in uploaded_file.chunks():
logger.debug("content chunk = '%d'" % len(content))
f.write(content)
my_contentupload_id = None
if int(chunk) + 1 >= int(chunks):
try:
meta = json.loads(meta)
meta["username"] = request.user.username
upload = move_upload(pub, dest_path, file_name,
json.dumps(meta))
async_upload = run_pub_scripts.delay(pub, upload)
my_contentupload_id = upload.id
except Exception as err:
logger.exception(
"There was a problem during upload of a file for a publisher."
)
else:
logger.info("Successfully pluploaded %s" % name)
logger.debug("plupload done")
return render_to_json({
"chunk posted": debug,
"contentupload_id": my_contentupload_id
})
else:
return render_to_json({
"method":
i18n_errors.fatal_unsupported_http_method_expected(
request.method, "POST")
})
def write_plupload(request, pub_name):
"""file upload for plupload"""
logger.info("Starting write plupload")
pub = models.Publisher.objects.get(name=pub_name)
logger.debug("%s %s" %
(str(type(request.REQUEST['meta'])), request.REQUEST['meta']))
logger.debug("Publisher Plupload started")
if request.method == 'POST':
name = request.REQUEST.get('name', '')
uploaded_file = request.FILES['file']
if not name:
name = uploaded_file.name
logger.debug("plupload name = '%s'" % name)
#check to see if a user has uploaded a file before, and if they have
#not, make them a upload directory
upload_dir = "/results/referenceLibrary/temp"
if not os.path.exists(upload_dir):
return render_to_json({"error": "upload path does not exist"})
dest_path = os.path.join(upload_dir, name)
logger.debug("plupload destination = '%s'" % dest_path)
chunk = request.REQUEST.get('chunk', '0')
chunks = request.REQUEST.get('chunks', '0')
logger.debug("plupload chunk %s %s of %s" %
(str(type(chunk)), str(chunk), str(chunks)))
debug = [chunk, chunks]
with open(dest_path, ('wb' if chunk == '0' else 'ab')) as f:
for content in uploaded_file.chunks():
logger.debug("content chunk = '%d'" % len(content))
f.write(content)
my_contentupload_id = None
if int(chunk) + 1 >= int(chunks):
try:
upload = move_upload(pub, dest_path, name,
request.REQUEST['meta'])
async_upload = run_pub_scripts.delay(pub, upload)
my_contentupload_id = upload.id
except Exception as err:
logger.exception(
"There was a problem during upload of a file for a publisher."
)
else:
logger.info("Successfully pluploaded %s" % name)
logger.debug("plupload done")
return render_to_json({
"chunk posted": debug,
"contentupload_id": my_contentupload_id
})
else:
return render_to_json({"method": "only post here"})
def download_genome(request):
# called by "Import Preloaded Ion References"
if request.method == "POST":
reference_meta = request.POST.get("reference_meta", None)
ref_annot_update = request.POST.get("ref_annot_update", None)
reference_args = json.loads(base64.b64decode(reference_meta))
annotation_meta = request.POST.get("missingAnnotation_meta", None)
reference_mask_info = request.POST.get("reference_mask", None)
if annotation_meta:
annotation_data = json.loads(base64.b64decode(annotation_meta))
annotation_lists = annotation_data
# Download and register only the Ref Annotation file if Reference Genome is already Imported
# If not, download refAnnot file and Reference Genome asynchronously
if annotation_data and ref_annot_update:
logger.debug(
"Downloading Annotation File {0} with meta {1}".format(
annotation_data, reference_meta))
if annotation_lists:
download_genome_annotation(annotation_lists, reference_args)
else:
url = request.POST.get("reference_url", None)
logger.debug("Downloading {0} with meta {1}".format(
url, reference_meta))
if url is not None:
if annotation_lists:
download_genome_annotation(annotation_lists,
reference_args)
try:
new_reference_genome(
reference_args,
url,
reference_mask_filename=reference_mask_info)
except Exception as e:
return render_to_json({"status": str(e), "error": True})
return HttpResponseRedirect(
urlresolvers.reverse("references_genome_download"))
elif request.method == "GET":
references = get_references() or []
downloads = FileMonitor.objects.filter(
tags__in=["reference", "bedfile"]).order_by('-created')
downloads_annot = FileMonitor.objects.filter(
tags__contains="reference_annotation").order_by('-created')
(references, downloads_annot) = get_annotation(references,
downloads_annot)
# update BED files available for pre-loaded references
for ref in references:
if ref['installed']:
bedfiles = []
for bedfile in ref.get('bedfiles', []):
status = get_bedfile_status(bedfile,
ref['meta']['short_name'])
if status != "_installed":
bedfile['status'] = status
bedfiles.append(bedfile)
ref['bedfiles'] = bedfiles
# set up page to refresh
_in_progress_status = [
"Queued", "Starting", "Downloading", "Preprocessing", "Indexing"
]
_in_progress_status += [s.lower() for s in _in_progress_status]
downloading = downloads.filter(
status__in=_in_progress_status) | downloads_annot.filter(
status__in=_in_progress_status)
id_hashes = [
r['meta']['identity_hash'] for r in references
if r['meta']['identity_hash']
]
processing = ReferenceGenome.objects.filter(
identity_hash__in=id_hashes, status__in=_in_progress_status)
ctx = {
'downloads': downloads,
'downloads_annot': downloads_annot,
'references': references,
'refresh_progress': downloading.count() > 0
or processing.count() > 0
}
return render_to_response("rundb/configure/reference_download.html",
ctx,
context_instance=RequestContext(request))
def new_genome(request):
"""This is the page to create a new genome. The XML-RPC server is ionJobServer.
"""
if request.method == "POST":
# parse the data sent in
#required
name = request.POST.get('name', False)
short_name = request.POST.get('short_name', False)
fasta = request.POST.get('target_file', False)
version = request.POST.get('version', False)
notes = request.POST.get('notes', "")
#optional
read_sample_size = request.POST.get('read_sample_size', False)
read_exclude_length = request.POST.get('read_exclude_length', False)
#URL download
url = request.POST.get('url', False)
error_status = ""
reference_path = REFERENCE_LIBRARY_TEMP_DIR + fasta
why_delete = ""
#if any of those were false send back a failed message
if not all((name, short_name, fasta, version)):
return render_to_json({"status": "Form validation failed", "error": True})
if not set(short_name).issubset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"):
return render_to_json({"status": "The short name has invalid characters. The valid values are letters, numbers, and underscores.", "error": True})
#TODO: check to make sure the zip file only has one fasta or fa
if not url:
#check to ensure the size on the OS the same as the reported.
reported_file_size = request.POST.get('reported_file_size', False)
try:
uploaded_file_size = str(os.path.getsize(reference_path))
except OSError:
return render_to_json({"status": "The FASTA temporary files was not found", "error": True})
if reported_file_size != uploaded_file_size:
why_delete = "The file you uploaded differs from the expected size. This is due to an error uploading."
if not (fasta.lower().endswith(".fasta") or fasta.lower().endswith(".zip")):
why_delete = "The file you uploaded does not have a .fasta or .zip extension. It must be a plain text fasta file or a Zip compressed fasta."
if why_delete:
try:
os.remove(reference_path)
except OSError:
why_delete += " The FASTA file could not be deleted."
return render_to_json({"status": why_delete, "error": True})
#Make an genome ref object
if ReferenceGenome.objects.filter(short_name=short_name, index_version=settings.TMAP_VERSION):
#check to see if the genome already exists in the database with the same version
return render_to_json({"status": "Failed - Genome with this short name and index version already exist.", "error": True})
ref_genome = ReferenceGenome()
ref_genome.name = name
ref_genome.short_name = short_name
ref_genome.version = version
ref_genome.date = datetime.datetime.now()
ref_genome.notes = notes
ref_genome.status = "queued"
ref_genome.enabled = False
ref_genome.index_version = settings.TMAP_VERSION
#before the object is saved we should ping the xml-rpc server to see if it is alive.
try:
conn = client.connect(JOBSERVER_HOST, settings.JOBSERVER_PORT)
#just check uptime to make sure the call does not fail
conn.uptime()
logger.debug('Connected to ionJobserver process.')
except (socket.error, xmlrpclib.Fault):
return render_to_json({"status": "Unable to connect to ionJobserver process. You may need to restart ionJobserver", "error": True})
#if the above didn't fail then we can save the object
#this object must be saved before the tmap call is made
ref_genome.save()
logger.debug('Saved ReferenceGenome %s' % ref_genome.__dict__)
#kick off the anaserve tmap xmlrpc call
import traceback
try:
conn = client.connect(JOBSERVER_HOST, settings.JOBSERVER_PORT)
tmap_bool, tmap_status = conn.tmap(str(ref_genome.id), fasta, short_name, name, version,
read_sample_size, read_exclude_length, settings.TMAP_VERSION)
logger.debug('ionJobserver process reported %s %s' % (tmap_bool, tmap_status))
except (socket.error, xmlrpclib.Fault):
#delete the genome object, because it was not sucessful
ref_genome.delete()
return render_to_json({"status": "Error with index creation", "error": traceback.format_exc()})
if not tmap_bool:
ref_genome.delete()
return render_to_json({"status": tmap_status, "error": True})
return render_to_json({"status": "The genome index is being created. This might take a while, check the status on the references tab. \
You are being redirected there now.", "error": False})
elif request.method == "GET":
ctx = RequestContext(request, {})
return render_to_response("rundb/configure/modal_references_new_genome.html", context_instance=ctx)
