def generate_file_attributes(file_id): chunked_upload = ChunkedUpload.objects.get(id=int(file_id)) header_class = "Headingx" cell_class = "Cellx" row_class = "Rowx" size = u.filesize_toString(chunked_upload.offset) uploaded_on = (str(chunked_upload.completed_on)).split(".")[0] dt_format1 = '%Y-%m-%d %H:%M:%S' dt_format2 = '%d-%m-%Y %H:%M:%S' uploaded_on = datetime.datetime.strptime(str(uploaded_on), dt_format1).strftime(dt_format2) hash = chunked_upload.hash html_tag = "<div class='Tablex'>" # row html_tag += "<div class='{row_class!s}'>".format(**locals()) html_tag += "<div class='{cell_class!s}'><p><strong>Size:</strong></p></div>".format(**locals()) html_tag += "<div class='{cell_class!s}'><p>{size!s}</p></div>".format(**locals()) html_tag += "</div>" # row html_tag += "<div class='{row_class!s}'>".format(**locals()) html_tag += "<div class='{cell_class!s}'><p><strong>Uploaded on:</strong></p></div>".format(**locals()) html_tag += "<div class='{cell_class!s}'><p>{uploaded_on!s}</p></div>".format(**locals()) html_tag += "</div>" # row html_tag += "<div class='{row_class!s}'>".format(**locals()) html_tag += "<div class='{cell_class!s}'><p><strong>Hash:</strong></p></div>".format(**locals()) html_tag += "<div class='{cell_class!s}'><p>{hash!s}</p></div>".format(**locals()) html_tag += "</div>" html_tag += "</div>" return html_tag
def populate_exp_modal(request): # this method gets the current files associated with an ENA experiment or group of ENA experiments # and populates a table in the upload modal dialogue along with delete functionality data_modal_id = request.GET.get('data_modal_id') # get experiments exps = EnaCollection().get_experiments_by_modal_id(data_modal_id) output_files = [] for exp in exps: # for each experiment get a list of the associated files files = EnaCollection().get_files_by_experiment_id(exp["experiments"][0]["_id"]) for file in files: # get chunked upload object ch = ChunkedUpload.objects.get(id=file['files']["chunked_upload_id"]) # now populate output object f = {} f['id'] = str(ch.id) f['name'] = ch.filename f['size'] = u.filesize_toString(ch.offset) f['md5'] = file['files']["hash"] f['data_modal_id'] = data_modal_id f['panel_id'] = exp["experiments"][0]["panel_id"] f['experiment_id'] = str(exp["experiments"][0]["_id"]) output_files.append(f) return HttpResponse(jsonpickle.encode(output_files), content_type='text/plain')
def create_transfer(self, submission_id, file_path=None): # before creating a new transfer record for this submission, remove all others remote_record = self.get_by_sub_id(submission_id) if remote_record: self.delete_transfer(str(remote_record["_id"])) fields = data_utils.json_to_pytype( DB_TEMPLATES['REMOTE_FILE_COLLECTION']) fields['submission_id'] = submission_id fields['profile_id'] = self.profile_id fields['file_path'] = file_path transfer_time = datetime.now().strftime("%d-%m-%Y %H:%M:%S") fields["commenced_on"] = transfer_time fields["current_time"] = transfer_time fields["transfer_rate"] = "" if file_path: d = DataFile().GET(submission_id) chunked_upload = ChunkedUpload.objects.get(id=int(d['file_id'])) fields["file_size_bytes"] = u.filesize_toString( chunked_upload.offset) doc = self.RemoteFileCollection.insert(fields) # return inserted record df = self.GET(str(doc)) return df
def zip_file(request): # need to get a reference to the file to zip file_id = request.GET['file_id'] print("zip started " + file_id) file_obj = ChunkedUpload.objects.get(pk=file_id) # get the name of the file to zip and change its suffix to .gz output_file_location = os.path.join(settings.MEDIA_ROOT, file_obj.file.name) output_file_name = file_obj.filename + '.gz' try: # open the file as gzip acrchive...set compression level temp_name = os.path.join(settings.MEDIA_ROOT, str(uuid.uuid4()) + '.tmp') myzip = gzip.open(temp_name, 'wb', compresslevel=1) src = open(output_file_location, 'r') # write input file to gzip archive in n byte chunks n = 100000000 for chunk in iter(lambda: src.read(n), ''): myzip.write(bytes(chunk, 'UTF-8')) finally: myzip.close() src.close() print('zip complete ' + file_id) # now need to delete the old file and update the file record with the new file new_file_name = output_file_location + '.gz' os.rename(temp_name, new_file_name) os.remove(output_file_location) # calculate new file size stats = os.stat(new_file_name) new_file_size = stats.st_size / 1000 / 1000 # update filename file_obj.filename = output_file_name file_obj.file.name = new_file_name # update file size file_obj.offset = stats.st_size file_obj.save() out = {'zipped': True, 'file_name': output_file_name, 'file_size': new_file_size} # update record in mongo record_object = DataFile().get_by_file_id(file_id) auto_fields = dict() auto_fields[DataFile().get_qualified_field("file_size")] = u.filesize_toString(file_obj.offset) auto_fields[DataFile().get_qualified_field("name")] = output_file_name auto_fields[DataFile().get_qualified_field("file_location")] = new_file_name BrokerDA(target_id=str(record_object.get("_id", str())), component="datafile", auto_fields=auto_fields ).do_save_edit() out = jsonpickle.encode(out) return HttpResponse(out, content_type='text/plain')
def inspect_file(request): # utillity method to examine a file and return meta-data to the frontend output_dict = {'file_type': 'unknown', 'do_compress': False} # get reference to file file_id = request.GET['file_id'] chunked_upload = ChunkedUpload.objects.get(id=int(file_id)) file_name = os.path.join(MEDIA_ROOT, chunked_upload.file.name) # size threshold to determine if a file should be compressed zip_threshold = 200000000 # size in bytes # check if file is compressed is_zipped = u.is_gzipped(file_name) if chunked_upload.offset >= zip_threshold and not is_zipped: output_dict['do_compress'] = True # check for file type if u.is_pdf_file(file_name): output_dict['file_type'] = 'pdf' else: try: if (u.is_fastq_file(file_name)): output_dict['file_type'] = 'fastq' elif (u.is_sam_file(file_name)): output_dict['file_type'] = 'sam' elif (u.is_bam_file(file_name)): output_dict['file_type'] = 'bam' except: output_dict['file_type'] = '' # add datafile schema chunked_upload.type = output_dict['file_type'] chunked_upload.save() # ...and obtain the inserted record profile_id = request.session['profile_id'] df = DataFile(profile_id).save_datafile(file_id) output_dict['data_file_id'] = str(df["_id"]) output_dict['table_data'] = htags.generate_copo_datafiles_data(profile_id, df) out = jsonpickle.encode(output_dict) return HttpResponse(out, content_type='json')
def save_ena_sample_callback(request): # get sample form list, attribute list, and the collection id # collection_id = request.GET['collection_id'] details_id = request.GET['study_id'] sample_id = request.GET['sample_id'] # get details of user enetered sample sample = jsonpickle.decode(request.GET['sample_details']) attr = jsonpickle.decode(request.GET['sample_attr']) if sample_id == '': EnaCollection().add_sample_to_study(sample, attr, details_id) else: EnaCollection().update_sample_in_study(sample, attr, details_id, sample_id) # now clear attributes and readd the new set out = u.get_sample_html_from_details_id(details_id) return HttpResponse(out, content_type='html')
def create_transfer(self, submission_id, file_path=None): fields = data_utils.json_to_pytype(DB_TEMPLATES['REMOTE_FILE_COLLECTION']) fields['submission_id'] = submission_id fields['profile_id'] = self.profile_id fields['file_path'] = file_path transfer_time = datetime.now().strftime("%d-%m-%Y %H:%M:%S") fields["commenced_on"] = transfer_time fields["current_time"] = transfer_time fields["transfer_rate"] = [] if file_path: d = DataFile().GET(submission_id) chunked_upload = ChunkedUpload.objects.get(id=int(d['file_id'])) fields["file_size_bytes"] = u.filesize_toString(chunked_upload.offset) doc = self.RemoteFileCollection.insert(fields) # return inserted record df = self.GET(str(doc)) return df
def zip_file(request): # need to get a reference to the file to zip file_id = request.GET['file_id'] print("zip started " + file_id) file_obj = ChunkedUpload.objects.get(pk=file_id) # get the name of the file to zip and change its suffix to .gz output_file_location = os.path.join(settings.MEDIA_ROOT, file_obj.file.name) output_file_name = file_obj.filename + '.gz' try: # open the file as gzip acrchive...set compression level temp_name = os.path.join(settings.MEDIA_ROOT, str(uuid.uuid4()) + '.tmp') myzip = gzip.open(temp_name, 'wb', compresslevel=1) src = open(output_file_location, 'r') # write input file to gzip archive in n byte chunks n = 100000000 for chunk in iter(lambda: src.read(n), ''): myzip.write(bytes(chunk, 'UTF-8')) finally: myzip.close() src.close() print('zip complete ' + file_id) # now need to delete the old file and update the file record with the new file new_file_name = output_file_location + '.gz' os.rename(temp_name, new_file_name) os.remove(output_file_location) # calculate new file size stats = os.stat(new_file_name) new_file_size = stats.st_size / 1000 / 1000 # update filename file_obj.filename = output_file_name file_obj.file.name = new_file_name # update file size file_obj.offset = stats.st_size file_obj.save() out = { 'zipped': True, 'file_name': output_file_name, 'file_size': new_file_size } # update record in mongo record_object = DataFile().get_by_file_id(file_id) auto_fields = dict() auto_fields[DataFile().get_qualified_field( "file_size")] = u.filesize_toString(file_obj.offset) auto_fields[DataFile().get_qualified_field("name")] = output_file_name auto_fields[DataFile().get_qualified_field( "file_location")] = new_file_name profile_id = request.session['profile_id'] component = "datafile" BrokerDA(target_id=str(record_object.get("_id", str())), component=component, auto_fields=auto_fields).do_save_edit() out = jsonpickle.encode(out) return HttpResponse(out, content_type='json')
def inspect_file(request): # utility method to examine a file and return meta-data to the frontend output_dict = {'file_type': 'unknown', 'do_compress': False} # get reference to file file_id = request.GET['file_id'] chunked_upload = ChunkedUpload.objects.get(id=int(file_id)) file_name = os.path.join(settings.MEDIA_ROOT, chunked_upload.file.name) # size threshold to determine if a file should be compressed zip_threshold = 200000000 # size in bytes # check if file is compressed is_zipped = u.is_gzipped(file_name) if chunked_upload.offset >= zip_threshold and not is_zipped: output_dict['do_compress'] = True # check for file type if u.is_pdf_file(file_name): output_dict['file_type'] = 'pdf' else: try: if u.is_fastq_file(file_name): output_dict['file_type'] = 'fastq' if not is_zipped: output_dict['do_compress'] = True elif u.is_sam_file(file_name): output_dict['file_type'] = 'sam' if not is_zipped: output_dict['do_compress'] = True elif u.is_bam_file(file_name): output_dict['file_type'] = 'bam' if not is_zipped: output_dict['do_compress'] = True else: # make file type same as extension output_dict['file_type'] = chunked_upload.filename.rsplit( '.')[1] except: output_dict['file_type'] = 'unknown' # add datafile schema chunked_upload.type = output_dict['file_type'] chunked_upload.save() # ...and obtain the inserted record profile_id = request.session['profile_id'] component = "datafile" auto_fields = dict() auto_fields[DataFile().get_qualified_field("file_id")] = file_id auto_fields[DataFile().get_qualified_field( "file_type")] = output_dict['file_type'] auto_fields[DataFile().get_qualified_field("file_location")] = file_name auto_fields[DataFile().get_qualified_field( "file_size")] = u.filesize_toString(chunked_upload.offset) auto_fields[DataFile().get_qualified_field( "name")] = chunked_upload.filename # get default type from schema type = [ f for f in d_utils.get_copo_schema(component) if f.get("id").split(".")[-1] == "type" ] if type: type = type[0]["default_value"] auto_fields[DataFile().get_qualified_field("type")] = type df = BrokerDA(context=dict(), profile_id=profile_id, component=component, auto_fields=auto_fields, visualize="last_record").do_save_edit().get( "record_object", dict()) out = jsonpickle.encode(output_dict) return HttpResponse(out, content_type='json')
def inspect_file(request): # utility method to examine a file and return meta-data to the frontend output_dict = {'file_type': 'unknown', 'do_compress': False} # get reference to file file_id = request.GET['file_id'] chunked_upload = ChunkedUpload.objects.get(id=int(file_id)) file_name = os.path.join(settings.MEDIA_ROOT, chunked_upload.file.name) # size threshold to determine if a file should be compressed zip_threshold = 200000000 # size in bytes # check if file is compressed is_zipped = u.is_gzipped(file_name) if chunked_upload.offset >= zip_threshold and not is_zipped: output_dict['do_compress'] = True # check for file type if u.is_pdf_file(file_name): output_dict['file_type'] = 'pdf' else: try: if u.is_fastq_file(file_name): output_dict['file_type'] = 'fastq' if not is_zipped: output_dict['do_compress'] = True elif u.is_sam_file(file_name): output_dict['file_type'] = 'sam' if not is_zipped: output_dict['do_compress'] = True elif u.is_bam_file(file_name): output_dict['file_type'] = 'bam' if not is_zipped: output_dict['do_compress'] = True else: # make file type same as extension output_dict['file_type'] = chunked_upload.filename.rsplit('.')[1] except: output_dict['file_type'] = 'unknown' # add datafile schema chunked_upload.type = output_dict['file_type'] chunked_upload.save() # ...and obtain the inserted record profile_id = request.session['profile_id'] component = "datafile" auto_fields = dict() auto_fields[DataFile().get_qualified_field("file_id")] = file_id auto_fields[DataFile().get_qualified_field("file_type")] = output_dict['file_type'] auto_fields[DataFile().get_qualified_field("file_location")] = file_name auto_fields[DataFile().get_qualified_field("file_size")] = u.filesize_toString(chunked_upload.offset) auto_fields[DataFile().get_qualified_field("name")] = chunked_upload.filename # get default type from schema type = [f for f in d_utils.get_copo_schema(component) if f.get("id").split(".")[-1] == "type"] if type: type = type[0]["default_value"] auto_fields[DataFile().get_qualified_field("type")] = type df = BrokerDA(context=dict(), profile_id=profile_id, component=component, auto_fields=auto_fields, visualize="last_record" ).do_save_edit().get("record_object", dict()) # do visualise table_data = BrokerVisuals( profile_id=profile_id, context=output_dict, component=component, record_object=df ).do_row_data().get("table_data", dict()) output_dict['table_data'] = table_data out = jsonpickle.encode(output_dict) return HttpResponse(out, content_type='json')
def populate_samples_form(request): collection_id = request.GET['collection_id'] collection_id = 1 out = u.get_sample_html_from_collection_id(collection_id) return HttpResponse(out, content_type='html')