def gen_samples_db(self): if not self.samples_order_file_path: self.samples_order_file_path = self.gen_samples_order_file(self.view_data_path) samples_db_output_path = self.get_output_file_path('samples.db') s = dbops.SamplesInformationDatabase(samples_db_output_path, run=self.run, progress=self.progress, quiet=True) s.create(self.samples_info_file_path, self.samples_order_file_path)
def gen_samples_db(self): samples_info_file_path = self.gen_samples_info_file() samples_order_file_path = self.gen_samples_order_file() samples_db_output_path = self.get_output_file_path(self.project_name + '-SAMPLES.db', delete_if_exists=True) s = dbops.SamplesInformationDatabase(samples_db_output_path, run=self.run, progress=self.progress, quiet=True) s.create(samples_info_file_path, samples_order_file_path)
def receive_additional_upload_file(request, userdb, response): set_default_headers(response) if not request.files.get('uploadFile'): return '{ "status": "error", "message": "you did not upload a file", "data": null }' if not request.forms.get('project'): return '{ "status": "error", "message": "you did not specify a project", "data": null }' user = get_user(request, userdb, response) if not user: return '{ "status": "error", "message": "you need to be logged in to upload additional data", "data": null }' user = user['data'] project = userdb.get_project(user, request.forms.get('project')) if not project['status'] == 'ok': return json.dumps(project) basepath = userdb.users_data_dir + '/userdata/' + user[ 'path'] + '/' + project['data']['path'] + '/' fileType = 'additionalFile' validFileType = { 'additionalFile': True, 'dataFile': True, 'treeFile': True, 'fastaFile': True, 'samplesOrderFile': True, 'samplesInformationFile': True } if request.forms.get('type'): fileType = request.forms.get('type') if not validFileType[fileType]: return '{ "status": "error", "message": "Invalid file type selected for upload", "data": null }' message = 'added' filePath = basepath + fileType if os.path.isfile(filePath): os.remove(filePath) message = 'updated' request.files.get('uploadFile').save(filePath) # check if this is a samples order or samples info file in which case the samples.db needs to be # created / updated if fileType == 'samplesOrderFile' or fileType == 'samplesInformationFile': # if there was a previous samples.db, it needs to be removed samplesDBPath = basepath + 'samples.db' if os.path.isfile(samplesDBPath): os.remove(samplesDBPath) # create a new samples.db samplesInfoPath = None samplesOrderPath = None if os.path.isfile(basepath + 'samplesInformationFile'): samplesInfoPath = basepath + 'samplesInformationFile' if os.path.isfile(basepath + 'samplesOrderFile'): samplesOrderPath = basepath + 'samplesOrderFile' if samplesInfoPath and samplesOrderPath: sample = dbops.SamplesInformationDatabase(samplesDBPath) try: sample.create(samplesInfoPath, samplesOrderPath) except Exception as e: return json.dumps({ 'status': 'error', 'message': "That one did not go as expected. Here is the error: %s" % e, "data": None }) return '{ "status": "ok", "message": "file ' + message + '", "data": null }'
def receive_upload_file(request, userdb, response): set_default_headers(response) user = get_user(request, userdb, response) # check mandatory values if not user: return '{ "status": "error", "message": "you need to be logged in to create a project", "data": null }' if not request.files.get('treeFile'): return '{ "status": "error", "message": "you need to upload a tree file", "data": null }' if not request.forms.get('title'): return '{ "status": "error", "message": "a title is required to create a project", "data": null }' # create the project retval = userdb.create_project(user['data'], request.forms.get('title'), request.forms.get('description')) if not retval['status'] == 'ok': return json.dumps(retval) project = retval['data'] # set user project to the new one retval = userdb.set_project(user['data'], project['name']) if not retval['status'] == 'ok': return json.dumps(retval) # save the uploaded files to the project directory basepath = userdb.users_data_dir + '/userdata/' + user['data'][ 'path'] + '/' + project['path'] + '/' # tree and data fiels are mandatory request.files.get('treeFile').save(basepath + 'treeFile') if request.files.get('dataFile'): request.files.get('dataFile').save(basepath + 'dataFile') if request.files.get('fastaFile'): request.files.get('fastaFile').save(basepath + 'fastaFile') # check if we have samples information createSamplesDB = False samplesOrderPath = None if request.files.get('samplesOrderFile'): request.files.get('samplesOrderFile').save(basepath + 'samplesOrderFile') samplesOrderPath = basepath + 'samplesOrderFile' createSamplesDB = True samplesInfoPath = None if request.files.get('samplesInformationFile'): request.files.get('samplesInformationFile').save( basepath + 'samplesInformationFile') samplesInfoPath = basepath + 'samplesInformationFile' createSamplesDB = True # create a samples database if needed if createSamplesDB: sample = dbops.SamplesInformationDatabase(basepath + 'samples.db') try: sample.create(samplesInfoPath, samplesOrderPath) except Exception as e: return json.dumps({ 'status': 'error', 'message': "That one did not go as expected. Here is the error: %s" % e, "data": None }) # all files are uploaded, do a sanity check retval = userdb.get_the_interactive_object(basepath, read_only=False) if not retval['status'] == 'ok': # if the files are not ok, the project needs to be deleted userdb.delete_project(user['data'], project['name']) # return the error to the ui return json.dumps(retval) return '{ "status": "ok", "message": "project created", "data": null }'
def gen_samples_db_for_the_merged_profile(self): """Geenrate a samples db for the merged profile. We use the ProfileSuperclass to load all the views we added into the meged profile, and generate clusterings of samples for each view to generate a default samples database.""" self.run.info_single("SAMPLES.db stuff...", nl_before=1, nl_after=1, mc="blue") essential_fields = [ f for f in self.atomic_data_fields if constants.IS_ESSENTIAL_FIELD(f) ] class Args: pass args = Args() args.profile_db = self.merged_profile_db_path # initialize views. profile_db_super = dbops.ProfileSuperclass(args) profile_db_super.load_views(omit_parent_column=True) # figure out sample orders dictionary sample_orders = {} failed_attempts = [] self.progress.new('Working on SAMPLES.db') for essential_field in essential_fields: self.progress.update('recovering samples order for "%s"' % (essential_field)) try: sample_orders[essential_field] = \ clustering.get_newick_tree_data_for_dict(profile_db_super.views[essential_field]['dict'], distance=self.distance, linkage=self.linkage, transpose=True) except: failed_attempts.append(essential_field) self.progress.end() if not len(sample_orders): self.run.warning( "This may or may not be important: anvi'o attempted to generate a samples\ database for this merged profile, however, all attempts to cluster samples\ based on view data available in the merged profile failed. No samples db\ for you :/") return if len(failed_attempts): self.run.warning("While anvi'o was trying to generate clusterings of samples based on view data\ available in the merged profile, clustering of some of the essential data\ failed. It is likely not a very big deal, but you shall be the judge of it.\ Anvi'o now proceeds to generate a samples db with clusterings it generated\ using the view data that worked. Here is the list of stuff that failed: '%s'"\ % (', '.join(failed_attempts))) # generate the samples order file samples_order_file_path = filesnpaths.get_temp_file_path() samples_order_file = open(samples_order_file_path, 'w') samples_order_file.write('attributes\tbasic\tnewick\n') for sample_order in sample_orders: samples_order_file.write( '%s\t%s\t%s\n' % (sample_order, '', sample_orders[sample_order])) samples_order_file.close() # figure out samples information stuff samples_information = {} headers = [] for sample_name in self.sample_ids_found_in_input_dbs: samples_information[sample_name] = {} self.progress.new('Working on SAMPLES.db') self.progress.update('...') # figure out num reads mapped per sample: for sample_name in self.sample_ids_found_in_input_dbs: samples_information[sample_name][ 'num_mapped_reads'] = self.total_reads_mapped_per_sample[ sample_name] self.progress.end() # generate the samples information file samples_information_file_path = filesnpaths.get_temp_file_path() utils.store_dict_as_TAB_delimited_file(samples_information, samples_information_file_path, headers=headers) # generate the samples database samples_db = dbops.SamplesInformationDatabase(self.samples_db_path, quiet=False) samples_db.create( samples_order_path=samples_order_file_path, samples_information_path=samples_information_file_path) os.remove(samples_order_file_path) os.remove(samples_information_file_path) self.run.info('Samples database', self.samples_db_path)
def handle(self, *args, **options): conn = sqlite3.connect(options['userdb_path'][0]) """ CREATE TABLE users ( 0 login TEXT PRIMARY KEY, 1 firstname TEXT, 2 lastname TEXT, 3 email TEXT, 4 password TEXT, 5 path TEXT, 6 token TEXT, 7 accepted INTEGER, 8 project TEXT, 9 affiliation TEXT, 10 ip TEXT, 11 clearance TEXT, 12 date TEXT, 13 visit TEXT) """ print("Migrating users table...") user_paths = {} for row in conn.execute('SELECT * FROM users;'): username = sanitize_username(row[0]) password = "******" + row[4] email = row[3] is_active = True is_superuser = True if row[11] == 'admin' else False user_paths[row[5]] = username date_joined = datetime.strptime( row[12], "%Y-%m-%d").replace(tzinfo=timezone.utc) newuser = User(username=username, password=password, email=email, is_active=is_active, is_superuser=is_superuser, is_staff=is_superuser, date_joined=date_joined) newuser.save() fullname = "%s %s" % (row[1], row[2]) institution = row[9] if len(fullname) < 2: fullname = None newuser_profile = UserProfile(user=newuser, fullname=fullname, orcid=None, institution=institution) newuser_profile.save() print(" - Successful.") print("Moving project files... ") for path in user_paths: username = user_paths[path] # old user project dir src = os.path.join(options['userfiles_path'][0], path) #new user project dir dst = os.path.join(settings.USER_DATA_DIR, username) try: shutil.copytree(src, dst) except FileNotFoundError: # if user path does not exists create empty dir for user os.makedirs(dst) print(" - Successful") print("Migratins project table... ") """ CREATE TABLE projects ( 0 name TEXT PRIMARY KEY, 1 path TEXT, 2 user TEXT, 3 description TEXT) """ for row in conn.execute('SELECT * FROM projects;'): name = row[0] slug = slugify(name).replace('-', '_') path = row[1] username = sanitize_username(row[2]) description = row[3] #rename project files fileTypes_old = [ 'treeFile', 'dataFile', 'fastaFile', 'samplesOrderFile', 'samplesInformationFile' ] fileTypes_new = [ 'tree.txt', 'data.txt', 'fasta.fa', 'samples-order.txt', 'samples-info.txt' ] for i in range(5): try: os.rename( os.path.join(settings.USER_DATA_DIR, username, path, fileTypes_old[i]), os.path.join(settings.USER_DATA_DIR, username, path, fileTypes_new[i])) except: pass try: if not description or not len(description) > 0: description = "" project = Project(name=name, slug=slug, user=User.objects.get(username=username), secret=path) samples_info = project.get_file_path('samples-order.txt', default=None) samples_order = project.get_file_path('samples-info.txt', default=None) if (samples_info or samples_order) and not project.get_file_path( 'samples.db', default=None): s = dbops.SamplesInformationDatabase(project.get_file_path( 'samples.db', dont_check_exists=True), quiet=True) s.create(samples_order, samples_info) interactive = project.get_interactive() # try to get number of leaves try: leaves = get_names_order_from_newick_tree( project.get_file_path('tree.txt', default=None)) project.num_leaves = len(leaves) if leaves != [''] else 0 except: project.num_leaves = 0 # try to get number of layers try: project.num_layers = len( interactive.views['single'] [0]) - 1 # <- -1 because first column is contigs except: project.num_layers = 0 # store description dbops.update_description_in_db( project.get_file_path('profile.db', default=None), description or '') project.synchronize_num_states() project.synchronize_num_collections() project.save() # try to migrate old links. for row_links in conn.execute( 'SELECT * FROM views WHERE project LIKE \'%s\';' % (name)): old_link = OldLinks( name=row_links[0], user=sanitize_username(row_links[1]), project=Project.objects.filter( name=row_links[2], user__username=sanitize_username(row_links[1]))[0], is_public=True if row_links[3] == 1 else False, token=row_links[4]) old_link.save() project_link = ProjectLink(project=old_link.project, link=old_link.token) project_link.save() except Exception as e: print(username + " " + name + " " + path + " failed to create project, here is the exception " + str(e)) shutil.rmtree( os.path.join(settings.USER_DATA_DIR, username, path)) print(" - Successful")