def gen_samples_db(self):
if not self.samples_order_file_path:
self.samples_order_file_path = self.gen_samples_order_file(self.view_data_path)
samples_db_output_path = self.get_output_file_path('samples.db')
s = dbops.SamplesInformationDatabase(samples_db_output_path, run=self.run, progress=self.progress, quiet=True)
s.create(self.samples_info_file_path, self.samples_order_file_path)
def gen_samples_db(self):
samples_info_file_path = self.gen_samples_info_file()
samples_order_file_path = self.gen_samples_order_file()
samples_db_output_path = self.get_output_file_path(self.project_name + '-SAMPLES.db', delete_if_exists=True)
s = dbops.SamplesInformationDatabase(samples_db_output_path, run=self.run, progress=self.progress, quiet=True)
s.create(samples_info_file_path, samples_order_file_path)
def receive_additional_upload_file(request, userdb, response):
set_default_headers(response)
if not request.files.get('uploadFile'):
return '{ "status": "error", "message": "you did not upload a file", "data": null }'
if not request.forms.get('project'):
return '{ "status": "error", "message": "you did not specify a project", "data": null }'
user = get_user(request, userdb, response)
if not user:
return '{ "status": "error", "message": "you need to be logged in to upload additional data", "data": null }'
user = user['data']
project = userdb.get_project(user, request.forms.get('project'))
if not project['status'] == 'ok':
return json.dumps(project)
basepath = userdb.users_data_dir + '/userdata/' + user[
'path'] + '/' + project['data']['path'] + '/'
fileType = 'additionalFile'
validFileType = {
'additionalFile': True,
'dataFile': True,
'treeFile': True,
'fastaFile': True,
'samplesOrderFile': True,
'samplesInformationFile': True
}
if request.forms.get('type'):
fileType = request.forms.get('type')
if not validFileType[fileType]:
return '{ "status": "error", "message": "Invalid file type selected for upload", "data": null }'
message = 'added'
filePath = basepath + fileType
if os.path.isfile(filePath):
os.remove(filePath)
message = 'updated'
request.files.get('uploadFile').save(filePath)
# check if this is a samples order or samples info file in which case the samples.db needs to be
# created / updated
if fileType == 'samplesOrderFile' or fileType == 'samplesInformationFile':
# if there was a previous samples.db, it needs to be removed
samplesDBPath = basepath + 'samples.db'
if os.path.isfile(samplesDBPath):
os.remove(samplesDBPath)
# create a new samples.db
samplesInfoPath = None
samplesOrderPath = None
if os.path.isfile(basepath + 'samplesInformationFile'):
samplesInfoPath = basepath + 'samplesInformationFile'
if os.path.isfile(basepath + 'samplesOrderFile'):
samplesOrderPath = basepath + 'samplesOrderFile'
if samplesInfoPath and samplesOrderPath:
sample = dbops.SamplesInformationDatabase(samplesDBPath)
try:
sample.create(samplesInfoPath, samplesOrderPath)
except Exception as e:
return json.dumps({
'status':
'error',
'message':
"That one did not go as expected. Here is the error: %s" %
e,
"data":
None
})
return '{ "status": "ok", "message": "file ' + message + '", "data": null }'
def receive_upload_file(request, userdb, response):
set_default_headers(response)
user = get_user(request, userdb, response)
# check mandatory values
if not user:
return '{ "status": "error", "message": "you need to be logged in to create a project", "data": null }'
if not request.files.get('treeFile'):
return '{ "status": "error", "message": "you need to upload a tree file", "data": null }'
if not request.forms.get('title'):
return '{ "status": "error", "message": "a title is required to create a project", "data": null }'
# create the project
retval = userdb.create_project(user['data'], request.forms.get('title'),
request.forms.get('description'))
if not retval['status'] == 'ok':
return json.dumps(retval)
project = retval['data']
# set user project to the new one
retval = userdb.set_project(user['data'], project['name'])
if not retval['status'] == 'ok':
return json.dumps(retval)
# save the uploaded files to the project directory
basepath = userdb.users_data_dir + '/userdata/' + user['data'][
'path'] + '/' + project['path'] + '/'
# tree and data fiels are mandatory
request.files.get('treeFile').save(basepath + 'treeFile')
if request.files.get('dataFile'):
request.files.get('dataFile').save(basepath + 'dataFile')
if request.files.get('fastaFile'):
request.files.get('fastaFile').save(basepath + 'fastaFile')
# check if we have samples information
createSamplesDB = False
samplesOrderPath = None
if request.files.get('samplesOrderFile'):
request.files.get('samplesOrderFile').save(basepath +
'samplesOrderFile')
samplesOrderPath = basepath + 'samplesOrderFile'
createSamplesDB = True
samplesInfoPath = None
if request.files.get('samplesInformationFile'):
request.files.get('samplesInformationFile').save(
basepath + 'samplesInformationFile')
samplesInfoPath = basepath + 'samplesInformationFile'
createSamplesDB = True
# create a samples database if needed
if createSamplesDB:
sample = dbops.SamplesInformationDatabase(basepath + 'samples.db')
try:
sample.create(samplesInfoPath, samplesOrderPath)
except Exception as e:
return json.dumps({
'status':
'error',
'message':
"That one did not go as expected. Here is the error: %s" % e,
"data":
None
})
# all files are uploaded, do a sanity check
retval = userdb.get_the_interactive_object(basepath, read_only=False)
if not retval['status'] == 'ok':
# if the files are not ok, the project needs to be deleted
userdb.delete_project(user['data'], project['name'])
# return the error to the ui
return json.dumps(retval)
return '{ "status": "ok", "message": "project created", "data": null }'
def gen_samples_db_for_the_merged_profile(self):
"""Geenrate a samples db for the merged profile.
We use the ProfileSuperclass to load all the views we added into the meged profile,
and generate clusterings of samples for each view to generate a default samples database."""
self.run.info_single("SAMPLES.db stuff...",
nl_before=1,
nl_after=1,
mc="blue")
essential_fields = [
f for f in self.atomic_data_fields
if constants.IS_ESSENTIAL_FIELD(f)
]
class Args:
pass
args = Args()
args.profile_db = self.merged_profile_db_path
# initialize views.
profile_db_super = dbops.ProfileSuperclass(args)
profile_db_super.load_views(omit_parent_column=True)
# figure out sample orders dictionary
sample_orders = {}
failed_attempts = []
self.progress.new('Working on SAMPLES.db')
for essential_field in essential_fields:
self.progress.update('recovering samples order for "%s"' %
(essential_field))
try:
sample_orders[essential_field] = \
clustering.get_newick_tree_data_for_dict(profile_db_super.views[essential_field]['dict'],
distance=self.distance,
linkage=self.linkage,
transpose=True)
except:
failed_attempts.append(essential_field)
self.progress.end()
if not len(sample_orders):
self.run.warning(
"This may or may not be important: anvi'o attempted to generate a samples\
database for this merged profile, however, all attempts to cluster samples\
based on view data available in the merged profile failed. No samples db\
for you :/")
return
if len(failed_attempts):
self.run.warning("While anvi'o was trying to generate clusterings of samples based on view data\
available in the merged profile, clustering of some of the essential data\
failed. It is likely not a very big deal, but you shall be the judge of it.\
Anvi'o now proceeds to generate a samples db with clusterings it generated\
using the view data that worked. Here is the list of stuff that failed: '%s'"\
% (', '.join(failed_attempts)))
# generate the samples order file
samples_order_file_path = filesnpaths.get_temp_file_path()
samples_order_file = open(samples_order_file_path, 'w')
samples_order_file.write('attributes\tbasic\tnewick\n')
for sample_order in sample_orders:
samples_order_file.write(
'%s\t%s\t%s\n' %
(sample_order, '', sample_orders[sample_order]))
samples_order_file.close()
# figure out samples information stuff
samples_information = {}
headers = []
for sample_name in self.sample_ids_found_in_input_dbs:
samples_information[sample_name] = {}
self.progress.new('Working on SAMPLES.db')
self.progress.update('...')
# figure out num reads mapped per sample:
for sample_name in self.sample_ids_found_in_input_dbs:
samples_information[sample_name][
'num_mapped_reads'] = self.total_reads_mapped_per_sample[
sample_name]
self.progress.end()
# generate the samples information file
samples_information_file_path = filesnpaths.get_temp_file_path()
utils.store_dict_as_TAB_delimited_file(samples_information,
samples_information_file_path,
headers=headers)
# generate the samples database
samples_db = dbops.SamplesInformationDatabase(self.samples_db_path,
quiet=False)
samples_db.create(
samples_order_path=samples_order_file_path,
samples_information_path=samples_information_file_path)
os.remove(samples_order_file_path)
os.remove(samples_information_file_path)
self.run.info('Samples database', self.samples_db_path)
def handle(self, *args, **options):
conn = sqlite3.connect(options['userdb_path'][0])
"""
CREATE TABLE users (
0 login TEXT PRIMARY KEY,
1 firstname TEXT,
2 lastname TEXT,
3 email TEXT,
4 password TEXT,
5 path TEXT,
6 token TEXT,
7 accepted INTEGER,
8 project TEXT,
9 affiliation TEXT,
10 ip TEXT,
11 clearance TEXT,
12 date TEXT,
13 visit TEXT)
"""
print("Migrating users table...")
user_paths = {}
for row in conn.execute('SELECT * FROM users;'):
username = sanitize_username(row[0])
password = "******" + row[4]
email = row[3]
is_active = True
is_superuser = True if row[11] == 'admin' else False
user_paths[row[5]] = username
date_joined = datetime.strptime(
row[12], "%Y-%m-%d").replace(tzinfo=timezone.utc)
newuser = User(username=username,
password=password,
email=email,
is_active=is_active,
is_superuser=is_superuser,
is_staff=is_superuser,
date_joined=date_joined)
newuser.save()
fullname = "%s %s" % (row[1], row[2])
institution = row[9]
if len(fullname) < 2:
fullname = None
newuser_profile = UserProfile(user=newuser,
fullname=fullname,
orcid=None,
institution=institution)
newuser_profile.save()
print(" - Successful.")
print("Moving project files... ")
for path in user_paths:
username = user_paths[path]
# old user project dir
src = os.path.join(options['userfiles_path'][0], path)
#new user project dir
dst = os.path.join(settings.USER_DATA_DIR, username)
try:
shutil.copytree(src, dst)
except FileNotFoundError:
# if user path does not exists create empty dir for user
os.makedirs(dst)
print(" - Successful")
print("Migratins project table... ")
"""
CREATE TABLE projects (
0 name TEXT PRIMARY KEY,
1 path TEXT,
2 user TEXT,
3 description TEXT)
"""
for row in conn.execute('SELECT * FROM projects;'):
name = row[0]
slug = slugify(name).replace('-', '_')
path = row[1]
username = sanitize_username(row[2])
description = row[3]
#rename project files
fileTypes_old = [
'treeFile', 'dataFile', 'fastaFile', 'samplesOrderFile',
'samplesInformationFile'
]
fileTypes_new = [
'tree.txt', 'data.txt', 'fasta.fa', 'samples-order.txt',
'samples-info.txt'
]
for i in range(5):
try:
os.rename(
os.path.join(settings.USER_DATA_DIR, username, path,
fileTypes_old[i]),
os.path.join(settings.USER_DATA_DIR, username, path,
fileTypes_new[i]))
except:
pass
try:
if not description or not len(description) > 0:
description = ""
project = Project(name=name,
slug=slug,
user=User.objects.get(username=username),
secret=path)
samples_info = project.get_file_path('samples-order.txt',
default=None)
samples_order = project.get_file_path('samples-info.txt',
default=None)
if (samples_info
or samples_order) and not project.get_file_path(
'samples.db', default=None):
s = dbops.SamplesInformationDatabase(project.get_file_path(
'samples.db', dont_check_exists=True),
quiet=True)
s.create(samples_order, samples_info)
interactive = project.get_interactive()
# try to get number of leaves
try:
leaves = get_names_order_from_newick_tree(
project.get_file_path('tree.txt', default=None))
project.num_leaves = len(leaves) if leaves != [''] else 0
except:
project.num_leaves = 0
# try to get number of layers
try:
project.num_layers = len(
interactive.views['single']
[0]) - 1 # <- -1 because first column is contigs
except:
project.num_layers = 0
# store description
dbops.update_description_in_db(
project.get_file_path('profile.db', default=None),
description or '')
project.synchronize_num_states()
project.synchronize_num_collections()
project.save()
# try to migrate old links.
for row_links in conn.execute(
'SELECT * FROM views WHERE project LIKE \'%s\';' %
(name)):
old_link = OldLinks(
name=row_links[0],
user=sanitize_username(row_links[1]),
project=Project.objects.filter(
name=row_links[2],
user__username=sanitize_username(row_links[1]))[0],
is_public=True if row_links[3] == 1 else False,
token=row_links[4])
old_link.save()
project_link = ProjectLink(project=old_link.project,
link=old_link.token)
project_link.save()
except Exception as e:
print(username + " " + name + " " + path +
" failed to create project, here is the exception " +
str(e))
shutil.rmtree(
os.path.join(settings.USER_DATA_DIR, username, path))
print(" - Successful")
