def checkUsersKey(usersdict, galaxyWeb):
#~ inputAbsolutPath="/nas_Dir/workflow"
for userinfo in usersdict:
#~ #Check if the user is not the admin
logger.debug(str(userinfo['email']))
logger.debug("checkUsersKey function what the pb")
if (str(userinfo['email']) != "*****@*****.**"):
logger.debug("check if the email already on the DB")
try:
user = GalaxyUsers.objects.get(user_email=userinfo['email'])
except GalaxyUsers.DoesNotExist:
user = None
logger.debug("user info:" + str(userinfo['id']))
#~ #if none create a new user
if user == None:
logger.debug("a none value create a new user")
user = GalaxyUsers(
user_id=userinfo['id'],
user_email=userinfo['email'],
user_aphid=userinfo['username'],
user_apikey=GalaxyCommunication_data_manager.
createUserApikey(galaxyWeb, str(userinfo['id'])))
logger.info("Add WOrkflow to users")
gi = GalaxyCommunication_data_manager.galaxyConnection(
galaxy_base_url, user.user_apikey)
GalaxyCommunication_data_manager.addAllWorkflow(
gi, workflowPath)
user.save()
#~ #else print user apikey
else:
logger.info("user.apikey:" + str(user.user_apikey))
def getGalaxyUsers(request):
gi = GalaxyCommunication_data_manager.galaxyConnection(
galaxy_base_url, apiKey)
users = GalaxyCommunication_data_manager.returnGalaxyUsers(
gi) #return a list of dictionnary
logger.info("##########################")
logger.info("checkusers from getGalaxyUsers")
checkUsersKey(users, gi)
djangoUsers(GalaxyUsers.objects.all())
logger.info("##########################")
#~ # Always return an HttpResponseRedirect after successfully dealing
#~ with POST data. This prevents data from being posted twice if a
# user hits the Back button.
return HttpResponseRedirect(reverse('sequencer:savedData'))
def showSavedDataLogs(request, user_name):
logger.info("##########################")
logger.info("START showSavedData() view")
logger.info("##########################")
logger.debug("Load Template sequencer/ngsData.html ")
template = loader.get_template('sequencer/ngsData.html')
logger.info("Return all Experiments ")
#~ latest_question_list = Experiments.objects.all()
#~ experiments_list = Experiments.objects.all()
todownload_list = savedNGSData.objects.all()
##############################################
gi = GalaxyCommunication_data_manager.galaxyConnection(
galaxy_base_url, apiKey)
users = GalaxyUsers.objects.all() #return a list of dictionnary
currentUser = GalaxyUsers.objects.get(user_id=user_name)
logger.debug("##########################")
logger.debug("checkusers")
logger.debug("##########################")
user = authenticate(username=currentUser.user_email,
password=currentUser.user_email)
if user is not None:
login(request, user)
context = {
#~ 'latest_question_list': latest_question_list,
'todownload_list': todownload_list,
'users': users,
'user_name': currentUser.user_email,
'current_user_name': currentUser.user_email,
}
logger.debug("##########################")
logger.debug("job done getExperiments")
logger.debug("##########################")
return HttpResponse(template.render(context, request))
def getExperimentsLogs(request, user_name):
experimentLimit = 10
logger.info("##########################")
logger.debug("start work getExperiments")
logger.debug("create dict getExperiments")
template = loader.get_template('sequencer/projects.html')
#~ latest_question_list = Experiments.objects.all()
experiments_list = Experiments.objects.all()
##############################################
gi = GalaxyCommunication_data_manager.galaxyConnection(
galaxy_base_url, apiKey)
users = GalaxyUsers.objects.all() #return a list of dictionnary
currentUser = GalaxyUsers.objects.get(user_id=user_name)
logger.debug("##########################")
logger.debug("checkusers")
logger.debug("##########################")
user = authenticate(username=currentUser.user_email,
password=currentUser.user_email)
if user is not None:
login(request, user)
context = {
#~ 'latest_question_list': latest_question_list,
'experiments_list': experiments_list,
'users': users,
'user_name': currentUser.user_email,
'current_user_name': currentUser.user_email,
}
logger.debug("##########################")
logger.debug("job done getExperiments")
logger.debug("##########################")
return HttpResponse(template.render(context, request))
def Download_NGS_RawData():
logger.info("##########################")
ticket_histories = GalaxyJobs.objects.all()
for job in ticket_histories:
#~ if job.progression=="suspendu":
#~ if job.progression=="suspendu":
if job.history_download == False:
if job.progression == "suspendu":
job.progression = "Download_in_progress"
job.save()
bamtoDl = []
logger.info("Build the bamtoDL list for : %s", job.tag_id)
for bam in job.list_experimentRawData.all():
bamtoDl.append(bam.bam_path)
#~ do not forgot to close the ssh connection()
#~ the data are downloaded, now you can create a new
#~ Galaxy history
job.history_download = True
job.progression = "Download_complete"
#~ job.galaxy_dictionnary=json.dumps(z)
job.save()
job.progression = "chargement_dans_galaxy"
job.save()
logger.info("##########################")
logger.info("start Galaxy job")
experimentsDict = json.loads(job.galaxy_dictionnary)
#~ z = experimentsDict.copy()
#~ z.update(json.loads(job.galaxy_dictionnary))
experimentsDict['resultsName'] = job.resultsName
experimentsDict['bamForPlasma'] = bamtoDl
job.galaxy_dictionnary = json.dumps(experimentsDict)
#~ confirmation telechargement
#~ sendmailwrapper(fromAddrOfficial,job.history_user_email.,subject,body)
historyPlasma = GalaxyCommunication_data_manager.mainSamtools_fromNGSData(
experimentsDict, galaxy_base_url,
job.history_user_email.user_apikey, plasmaFolderName)
#~ historyPlasma=GalaxyCommunication_data_manager.mainPlasma_fromNGSData(experimentsDict,galaxy_base_url,job.history_user_email.user_apikey,plasmaFolderName)
job.history_id = historyPlasma['id']
job.history_name = historyPlasma['name']
job.history_today = historyPlasma['today']
job.progression = "chargement_dans_galaxy_complet"
#~ confirmation lancement de galaxy
job.save()
body = """<p>Votre analyse à été ajoutée à l'historique:\n</p><p style="color:#2e1cb2";> """ + job.history_name + """.\n</p>
""" + str(
job.list_experimentRawData.count()
) + """ échantillon(s) est(sont) en cours de traitement. Vous pourrez les trouver en cliquant sur le lien
<p><a href='""" + galaxy_base_url + """/history/view_multiple'> <img src="http://www.carpem.fr/wp-content/themes/carpem/img/logo.gif" /></a></p>
"""
sendmailwrapper(fromAddrOfficial,
str(job.history_user_email.user_email),
" Analyse Plasma dans Galaxy", body)
#~ will be put in other function
else:
logger.info("##########################")
logger.info(
"Ce Job est deja pris en charge dans une task de telechargement de donnees"
)
continue
def sendReminder_JobIsDone():
logger.info("##########################")
ticket_histories = GalaxyJobs.objects.all()
for job in ticket_histories:
if job.progression == "chargement_dans_galaxy_complet":
logger.info("##########################")
logger.info("Ce Job est en cours d'analyse")
gi = GalaxyCommunication_data_manager.galaxyConnection(
galaxy_base_url, job.history_user_email.user_apikey)
if (gi.histories.get_status(str(
job.history_id))['percent_complete'] == 100):
logger.info("##########################")
logger.info("Un mail sera envoye a l'utilisateur")
body = """<p>Votre analyse est terminée. Les résultats sont disponibles dans votre historique:\n</p><p style="color:#2e1cb2";> """ + job.history_name + """.\n</p>
<p><b>Bilan:</b></p><table style="width:100%" border="1">
<tr>
<th>Information</th><th>Résultats</th><th>Résultats attendu</th>
</tr>
<tr>
<td>Fichier(s) complet(s) </td><td>""" + str(
gi.histories.get_status(str(job.history_id))
['state_details']['ok']) + """</td><td>""" + str(
(6 * job.list_experimentRawData.count()) +
job.list_experimentRawData.count()) + """</td>
</tr>
<tr>
<td>Fichier(s) en erreur </td><td>""" + str(
gi.histories.get_status(str(job.history_id))
['state_details']['error']) + """</td><td>0</td>
</tr>
<tr>
<td>Fichier(s) vide </td><td>""" + str(
gi.histories.get_status(str(
job.history_id))['state_details']
['empty']) + """</td><td>0</td>
</tr>
</table>
<p> Vous pourrez les trouver en cliquant sur le lien
</p>
<p><a href='""" + galaxy_base_url + """/history/view_multiple'> <img src="http://www.carpem.fr/wp-content/themes/carpem/img/logo.gif" /></a></p>
"""
sendmailwrapper(fromAddrOfficial,
str(job.history_user_email.user_email),
"Plasma Complet disponible dans Galaxy", body)
job.progression = "Complet"
job.save()
def download_data_Histories():
logger.info("##########################")
histories = GalaxyJobs.objects.all()
base_url = galaxy_base_url
for job in histories:
if job.history_download == False:
currentuser = GalaxyUsers.objects.get(
user_email=job.history_user_email)
gi = GalaxyCommunication_data_manager.galaxyConnection(
base_url, currentuser.user_apikey)
status = gi.histories.get_status(job.history_id)
history_info = gi.histories.show_history(job.history_id,
contents=True)
datasetslist = ""
for dataset in history_info:
datasetslist = datasetslist + str(dataset['dataset_id']) + ";"
job.history_state = status['state']
job.history_percent_complete = str(status['percent_complete'])
job.history_datasets_id = datasetslist
job.save()
logger.info(str(job.history_percent_complete))
logger.info(str(job.history_state))
logger.info(str(datasetslist))
if job.history_percent_complete == "100":
logger.info("job found at 100")
for dataset in job.history_datasets_id.split(';'):
if dataset != '':
logger.info("dataset is downloading")
logger.info(str(dataset))
gi.datasets.download_dataset(
dataset,
file_path="/nas_Dir/results/",
use_default_filename=True,
wait_for_completion=True)
job.history_download = True
def updateJobStatus(userGalaxy):
gi = GalaxyCommunication_data_manager.galaxyConnection(
galaxy_base_url, str(userGalaxy.user_apikey))
jobsList = gi.jobs.get_jobs()
for currentJob in jobsList:
#check if the job exist if not create a new job
logger.info("#######################")
newJob = gi.jobs.show_job(str(currentJob['id']), full_details=True)
print(newJob)
tryGetJob = None
try:
tryGetJob = UserCommonJobs.objects.get(job_id=str(newJob['id']))
logger.info(" the job exist already")
except UserCommonJobs.DoesNotExist:
tryGetJob = None
if (tryGetJob == None):
logger.info("Add a new Galaxy Job")
if str(newJob['tool_id']) not in tools_default:
logger.info("add a new tool")
this_tool = gi.tools.get_tools(newJob['tool_id'])
AddaWorkflowTool(this_tool)
else:
logger.info(
"Add a default tool which is not linked to edam Ontology. They are not treated by Regate"
)
this_tool = gi.tools.get_tools(newJob['tool_id'])
try:
tryexp = WorkflowsTools.objects.get(
primary_name=str(this_tool[0]['id'] + "_" +
this_tool[0]['version'] +
".json"))
except WorkflowsTools.DoesNotExist:
tryexp = None
logger.info(
"tool found was not added to the DB. We Add now this new tool"
)
newtool = WorkflowsTools(
primary_name=str(this_tool[0]['id'] + "_" +
this_tool[0]['version'] +
".json"),
name=str(this_tool[0]['id']),
version=str(this_tool[0]['version']))
newtool.save()
theTool = None
this_tool = gi.tools.get_tools(newJob['tool_id'])
try:
theTool = WorkflowsTools.objects.get(
primary_name=str(this_tool[0]['id'] + "_" +
this_tool[0]['version'] + ".json"))
except WorkflowsTools.DoesNotExist:
logger.info("No tool here")
continue
logger.info("Add a new Galaxy Job")
job_local = UserCommonJobs(
job_user_email=userGalaxy,
job_id=str(newJob['id']),
job_create_time=str(newJob['create_time']),
job_state=str(newJob['state']),
job_exit_code=str(newJob['exit_code']),
job_tool_id=theTool,
job_stderr=str(newJob['stderr']),
job_stdout=str(newJob['stdout']),
job_params=str(newJob['params']),
job_outputs=str(newJob['outputs']),
job_inputs=str(newJob['inputs']))
job_local.save()
def buildJobHistory(currentuser, historyCNV):
logger.info("##########################")
base_url = galaxy_base_url
gi = GalaxyCommunication_data_manager.galaxyConnection(
base_url, currentuser.user_apikey)
status = gi.histories.get_status(historyCNV['id'])
history_info = gi.histories.show_history(historyCNV['id'], contents=True)
#~ print str(gi.histories.get_status(historyCNV['id']))
#~ print str(gi.histories.show_history(historyCNV['id'], contents=True))
datasetslist = ""
for dataset in history_info:
datasetslist = datasetslist + str(dataset['dataset_id']) + ";"
try:
local_history = GalaxyJobs.objects.get(history_id=historyCNV['id'])
except GalaxyJobs.DoesNotExist:
local_history = None
#~ #if none create a new user
if local_history == None:
logger.info("a none value create a new local_history")
local_history = GalaxyJobs(
history_id=historyCNV['id'],
history_name=historyCNV['name'],
history_today=historyCNV['today'],
history_user_email=currentuser.user_email,
history_analyse_type="CNV",
history_state=status['state'],
history_percent_complete=status['percent_complete'],
history_datasets_id=datasetslist)
local_history.save()
if local_history.history_percent_complete != 100:
buildJobHistory(currentuser, historyCNV)
else:
for dataset in local_history.history_datasets_id.split(';'):
if dataset != '':
gi.datasets.download_dataset(dataset,
file_path="/nas_Dir/results/",
use_default_filename=True,
wait_for_completion=True)
else:
local_history = GalaxyJobs(
history_state=status['state'],
history_percent_complete=status['percent_complete'],
history_datasets_id=datasetslist)
local_history.save()
if local_history.history_percent_complete != 100:
buildJobHistory(currentuser, historyCNV)
else:
for dataset in local_history.history_datasets_id.split(';'):
if dataset != '':
gi.datasets.download_dataset(dataset,
file_path="/nas_Dir/results/",
use_default_filename=True,
wait_for_completion=True)
logger.info("##########################")
logger.info("checkusers from getGalaxyUsers")
logger.info("##########################")
#~ # Always return an HttpResponseRedirect after successfully dealing
#~ with POST data. This prevents data from being posted twice if a
# user hits the Back button.
return (1)