def download_project(self):
directory = self.get_downloads_directory()
self.download_project_report()
self.download_project_datasets()
utils.compress_directory(directory,
directory,
compression_format='zip')
def serve_static(value):
directory = os.path.join(cwd, 'apps/templates/')
filename = os.path.join(directory, value)
url = filename + '.zip'
if not os.path.isfile(url):
utils.compress_directory(filename,
os.path.join(directory, 'files'),
compression_format='zip')
return flask.send_file(url,
attachment_filename=value + '.zip',
as_attachment=True)
def run_processing(n_clicks, project_id):
message = None
style = {'display': 'none'}
table = None
if n_clicks > 0:
session_cookie = flask.request.cookies.get('custom-auth-session')
destDir = os.path.join(experimentDir, project_id)
builder_utils.checkDirectory(destDir)
temporaryDirectory = os.path.join(tmpDirectory,
session_cookie + "upload")
datasets = builder_utils.listDirectoryFoldersNotEmpty(
temporaryDirectory)
res_n = dataUpload.check_samples_in_project(driver, project_id)
if 'experimental_design' in datasets:
dataset = 'experimental_design'
directory = os.path.join(temporaryDirectory, dataset)
experimental_files = os.listdir(directory)
if config['file_design'].replace('PROJECTID',
project_id) in experimental_files:
experimental_filename = config['file_design'].replace(
'PROJECTID', project_id)
designData = builder_utils.readDataset(
os.path.join(directory, experimental_filename))
designData = designData.astype(str)
if 'subject external_id' in designData.columns and 'biological_sample external_id' in designData.columns and 'analytical_sample external_id' in designData.columns:
if (res_n > 0).any().values.sum() > 0:
res = dataUpload.remove_samples_nodes_db(
driver, project_id)
res_n = dataUpload.check_samples_in_project(
driver, project_id)
if (res_n > 0).any().values.sum() > 0:
message = 'ERROR: There is already an experimental design loaded into the database and there was an error when trying to delete it. Contact your administrator.'.format(
experimental_filename, ','.join([
'subject external_id',
'biological_sample external_id',
'analytical_sample external_id'
]))
return message, style, table
res_n = None
result = create_new_identifiers.apply_async(
args=[
project_id,
designData.to_json(), directory,
experimental_filename
],
task_id='data_upload_' + session_cookie +
datetime.now().strftime('%Y%m-%d%H-%M%S-'))
result_output = result.wait(timeout=None,
propagate=True,
interval=0.2)
res_n = pd.DataFrame.from_dict(result_output['res_n'])
else:
message = 'ERROR: The Experimental design file provided ({}) is missing some of the required fields: {}'.format(
experimental_filename, ','.join([
'subject external_id',
'biological_sample external_id',
'analytical_sample external_id'
]))
builder_utils.remove_directory(directory)
return message, style, table
if 'clinical' in datasets:
dataset = 'clinical'
directory = os.path.join(temporaryDirectory, dataset)
clinical_files = os.listdir(directory)
if config['file_clinical'].replace('PROJECTID',
project_id) in clinical_files:
clinical_filename = config['file_clinical'].replace(
'PROJECTID', project_id)
data = builder_utils.readDataset(
os.path.join(directory, clinical_filename))
external_ids = {}
if 'subject external_id' in data and 'biological_sample external_id' in data and 'analytical_sample external_id' in data:
external_ids['subjects'] = data[
'subject external_id'].astype(str).unique().tolist()
external_ids['biological_samples'] = data[
'biological_sample external_id'].astype(
str).unique().tolist()
external_ids['analytical_samples'] = data[
'analytical_sample external_id'].astype(
str).unique().tolist()
dataUpload.create_mapping_cols_clinical(
driver,
data,
directory,
clinical_filename,
separator=separator)
if 0 in res_n.values:
samples = ', '.join([k for (k, v) in res_n if v == 0])
message = 'ERROR: No {} for project {} in the database. Please upload first the experimental design (ExperimentalDesign_{}.xlsx)'.format(
samples, project_id, project_id)
builder_utils.remove_directory(directory)
return message, style, table
else:
db_ids = dataUpload.check_external_ids_in_db(
driver, project_id).to_dict()
message = ''
intersections = {}
differences_in = {}
differences_out = {}
for col in external_ids:
intersect = list(
set(db_ids[col].values()).intersection(
external_ids[col]))
difference_in = list(
set(db_ids[col].values()).difference(
external_ids[col]))
difference_out = list(
set(external_ids[col]).difference(
set(db_ids[col].values())))
if len(difference_in) > 0 or len(
difference_out) > 0:
intersections[col] = intersect
differences_in[col] = difference_in
differences_out[col] = difference_out
for col in intersections:
message += 'WARNING: Some {} identifiers were not matched:\n Matching: {}\n No information provided: {} \n Non-existing in the database: {}\n'.format(
col, len(intersections[col]),
','.join(differences_in[col]),
','.join(differences_out[col]))
else:
message = 'ERROR: Format of the Clinical Data file is not correct. Check template in the documentation. Check columns: subject external_id, biological_sample external_id and analytical_sample external_id'
builder_utils.remove_directory(directory)
return message, style, table
try:
for dataset in datasets:
source = os.path.join(temporaryDirectory, dataset)
destination = os.path.join(destDir, dataset)
builder_utils.copytree(source, destination)
datasetPath = os.path.join(
os.path.join(experimentsImportDir, project_id), dataset)
if dataset != "experimental_design":
eh.generate_dataset_imports(project_id, dataset,
datasetPath)
loader.partialUpdate(imports=['project', 'experiment'],
specific=[project_id])
filename = os.path.join(tmpDirectory,
'Uploaded_files_' + project_id)
utils.compress_directory(filename,
temporaryDirectory,
compression_format='zip')
style = {'display': 'block'}
message = 'Files successfully uploaded.'
table = dataUpload.get_project_information(driver, project_id)
except Exception as err:
style = {'display': 'block'}
message = str(err)
return message, style, table