def start_ampliseq_fixed_solution_download(design_id, meta, auth): url = urlparse.urljoin(settings.AMPLISEQ_URL, "ws/tmpldesign/{0}/download/results".format(design_id)) monitor = FileMonitor(url=url, tags="ampliseq_template", status="Queued") monitor.save() t = tasks.download_something.apply_async( (url, monitor.id), {"auth": auth}, link=tasks.ampliseq_zip_upload.subtask((meta,)))
def start_reference_download(url, reference): monitor = FileMonitor(url=url, tags="reference") monitor.save() reference.file_monitor = monitor reference.save() try: download_args = (url, monitor.id, settings.TEMP_PATH) install_callback = tasks.install_reference.subtask((reference.id,)) t = tasks.download_something.apply_async(download_args, link=install_callback) except Exception as err: monitor.status = "System Error: " + str(err) monitor.save()
def start_reference_download(url, reference, callback=None, reference_mask_filename=None): monitor = FileMonitor(url=url, tags="reference") monitor.save() reference.status = "downloading" reference.file_monitor = monitor reference.save() try: download_args = (url, monitor.id, settings.TEMP_PATH) install_callback = tasks.install_reference.subtask((reference.id, reference_mask_filename)) if callback: install_callback.link(callback) async_result = tasks.download_something.apply_async(download_args, link=install_callback) return async_result except Exception as err: monitor.status = "System Error: " + str(err) monitor.save()
def start_update_product(name, url, updateVersion, callback=None): tagsInfo = "offCycleRel_{0}".format(updateVersion) monitor = FileMonitor(name=name, url=url, tags=tagsInfo) monitor.status = "downloading" monitor.save() return monitor.id
def new_genome(request): """This is the page to create a new genome. """ if request.method == "POST": # parse the data sent in #required name = request.POST.get('name', False) short_name = request.POST.get('short_name', False) fasta = request.POST.get('target_file', False) version = request.POST.get('version', "") notes = request.POST.get('notes', "") #optional read_exclude_length = request.POST.get('read_exclude_length', False) #URL download url = request.POST.get('url', False) reference_path = os.path.join(settings.TEMP_PATH, fasta) why_delete = "" #if any of those were false send back a failed message if not all((name, short_name, fasta)): return render_to_json({"status": "Form validation failed", "error": True}) if not set(short_name).issubset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"): return render_to_json({"status": "The short name has invalid characters. The valid values are letters, numbers, and underscores.", "error": True}) #TODO: check to make sure the zip file only has one fasta or fa if not url: #check to ensure the size on the OS the same as the reported. reported_file_size = request.POST.get('reported_file_size', False) try: uploaded_file_size = str(os.path.getsize(reference_path)) except OSError: return render_to_json({"status": "The FASTA temporary files was not found", "error": True}) if reported_file_size != uploaded_file_size: why_delete = "The file you uploaded differs from the expected size. This is due to an error uploading." if not (fasta.lower().endswith(".fasta") or fasta.lower().endswith(".zip")): why_delete = "The file you uploaded does not have a .fasta or .zip extension. It must be a plain text fasta file or a Zip compressed fasta." is_zip = zipfile.is_zipfile(reference_path) if is_zip: zip_file = zipfile.ZipFile(reference_path, 'r') files = zip_file.namelist() # MAC OS zip is being compressed with __MACOSX folder Ex: '__MACOSX/', '__MACOSX/._contigs_2.fasta'. # Filter out those files and Upload only FASTA file files = [x for x in files if not 'MACOSX' in x] zip_file.close() else: files = [fasta] fasta_files = filter(lambda x: x.endswith('.fa') or x.endswith('.fasta'), files) if len(fasta_files) != 1: why_delete = "Error: upload must contain exactly one fasta file" else: target_fasta_file = fasta_files[0] if why_delete: try: os.remove(reference_path) except OSError: why_delete += " The FASTA file could not be deleted." logger.warning("User uploaded bad fasta file: " + str(why_delete)) return render_to_json({"status": why_delete, "error": True}) #Make an genome ref object if ReferenceGenome.objects.filter(short_name=short_name, index_version=settings.TMAP_VERSION): #check to see if the genome already exists in the database with the same version return render_to_json({"status": "Failed - Genome with this short name and index version already exist.", "error": True}) ref_genome = ReferenceGenome() ref_genome.name = name ref_genome.short_name = short_name ref_genome.version = version ref_genome.notes = notes ref_genome.status = "preprocessing" ref_genome.enabled = False ref_genome.index_version = settings.TMAP_VERSION ref_genome.save() logger.debug("Created new reference: %d/%s" % (ref_genome.pk, ref_genome)) temp_dir = tempfile.mkdtemp(suffix=short_name, dir=settings.TEMP_PATH) temp_upload_path = os.path.join(temp_dir, fasta) os.chmod(temp_dir, 0777) os.rename(reference_path, temp_upload_path) monitor = FileMonitor( local_dir=temp_dir, name=fasta ) monitor.save() ref_genome.file_monitor = monitor ref_genome.reference_path = temp_upload_path ref_genome.save() index_task = tasks.build_tmap_index.subtask((ref_genome.id,), immutable=True) if is_zip: result = tasks.unzip_reference.apply_async( args=(ref_genome.id, target_fasta_file), link=index_task ) else: result = tasks.copy_reference.apply_async( args=(ref_genome.id,), link=index_task ) ref_genome.status = "queued" ref_genome.celery_task_id = result.task_id ref_genome.save() return render_to_json({"status": "The genome index is being created. This might take a while, check the status on the references tab. \ You are being redirected there now.", "error": False}) elif request.method == "GET": ctx = RequestContext(request, {}) return render_to_response("rundb/configure/modal_references_new_genome.html", context_instance=ctx)
def new_genome(request): """This is the page to create a new genome. """ def is_fasta(filename): ext = os.path.splitext(filename)[1] return ext.lower() in ['.fasta', '.fas', '.fa', '.seq'] if request.method == "POST": # parse the data sent in #required name = request.POST.get('name', False) short_name = request.POST.get('short_name', False) fasta = request.POST.get('target_file', False) version = request.POST.get('version', "") notes = request.POST.get('notes', "") #optional read_exclude_length = request.POST.get('read_exclude_length', False) #URL download url = request.POST.get('url', False) reference_path = os.path.join(settings.TEMP_PATH, fasta) why_delete = "" #if any of those were false send back a failed message if not all((name, short_name, fasta)): return render_to_json({ "status": "Form validation failed", "error": True }) if not set(short_name).issubset( "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" ): return render_to_json({ "status": "The short name has invalid characters. The valid values are letters, numbers, and underscores.", "error": True }) #TODO: check to make sure the zip file only has one fasta or fa if not url: #check to ensure the size on the OS the same as the reported. reported_file_size = request.POST.get('reported_file_size', False) try: uploaded_file_size = str(os.path.getsize(reference_path)) except OSError: return render_to_json({ "status": "The FASTA temporary files was not found", "error": True }) if reported_file_size != uploaded_file_size: why_delete = "The file you uploaded differs from the expected size. This is due to an error uploading." if not (is_fasta(fasta) or fasta.lower().endswith(".zip")): why_delete = "The file you uploaded does not have a FASTA or ZIP extension. It must be a plain text a Zip compressed fasta file." is_zip = zipfile.is_zipfile(reference_path) if is_zip: zip_file = zipfile.ZipFile(reference_path, 'r') files = zip_file.namelist() # MAC OS zip is being compressed with __MACOSX folder Ex: '__MACOSX/', '__MACOSX/._contigs_2.fasta'. # Filter out those files and Upload only FASTA file files = [x for x in files if not 'MACOSX' in x] zip_file.close() else: files = [fasta] fasta_files = filter(lambda x: is_fasta(x), files) if len(fasta_files) != 1: why_delete = "Error: upload must contain exactly one fasta file" else: target_fasta_file = fasta_files[0] if why_delete: try: os.remove(reference_path) except OSError: why_delete += " The FASTA file could not be deleted." logger.warning("User uploaded bad fasta file: " + str(why_delete)) return render_to_json({"status": why_delete, "error": True}) #Make an genome ref object if ReferenceGenome.objects.filter(short_name=short_name, index_version=settings.TMAP_VERSION): #check to see if the genome already exists in the database with the same version return render_to_json({ "status": "Failed - Genome with this short name and index version already exist.", "error": True }) ref_genome = ReferenceGenome() ref_genome.name = name ref_genome.short_name = short_name ref_genome.version = version ref_genome.notes = notes ref_genome.status = "preprocessing" ref_genome.enabled = False ref_genome.index_version = settings.TMAP_VERSION ref_genome.save() logger.debug("Created new reference: %d/%s" % (ref_genome.pk, ref_genome)) temp_dir = tempfile.mkdtemp(suffix=short_name, dir=settings.TEMP_PATH) temp_upload_path = os.path.join(temp_dir, fasta) os.chmod(temp_dir, 0777) os.rename(reference_path, temp_upload_path) monitor = FileMonitor(local_dir=temp_dir, name=fasta) monitor.save() ref_genome.file_monitor = monitor ref_genome.reference_path = temp_upload_path ref_genome.save() index_task = tasks.build_tmap_index.subtask((ref_genome.id, ), immutable=True) if is_zip: result = tasks.unzip_reference.apply_async( args=(ref_genome.id, target_fasta_file), link=index_task) else: result = tasks.copy_reference.apply_async(args=(ref_genome.id, ), link=index_task) ref_genome.status = "queued" ref_genome.celery_task_id = result.task_id ref_genome.save() return render_to_json({ "status": "The genome index is being created. This might take a while, check the status on the references tab. \ You are being redirected there now.", "error": False }) elif request.method == "GET": ctx = RequestContext(request, {}) return render_to_response( "rundb/configure/modal_references_new_genome.html", context_instance=ctx)