Exemple #1
0
def build_report(report_data, request_data, report_property, sections):
    """ Assembles report home folder, configures DRMAA and R related files
		and spawns a new process for reports DRMAA job on cluster.

	:param report_data: report info dictionary
	:type report_data: dict
	:param request_data: a copy of request object
	:type request_data: HTTPrequest
	:param report_property: report property form
	:type report_property: breezeForms.ReportPropsForm
	:param sections: a list of 'Rscripts' db objects
	:type sections: list
	:return: True
	:rtype: bool
	"""

    from breeze.models import Project, UserProfile, ReportType, Report, ComputeTarget
    from django.contrib.auth.models import User
    log = logger.getChild('build_report')
    assert isinstance(log, logging.getLoggerClass())
    assert isinstance(request_data.user, User)

    # get the request ReportType
    rt = ReportType.objects.get(type=report_data['report_type'])
    # list of users that will have access to this report
    shared_users = aux.extract_users(request_data.POST.get('Groups'),
                                     request_data.POST.get('Individuals'))
    if shared_users == list() and request_data.POST.get('shared'):
        shared_users = request_data.POST.getlist('shared')
    # author
    the_user = request_data.user
    the_user.prof = UserProfile.objects.get(user=the_user)
    assert isinstance(the_user.prof, UserProfile)

    # target profile :
    target = ComputeTarget.objects.get(pk=request_data.POST.get('target'))
    assert target.id in rt.ready_id_list  # TODO make a validator in the form section

    # create initial instance so that we can use its db id
    dbitem = Report(
        _type=rt,
        _name=report_data['instance_name'],
        _author=the_user,
        project=Project.objects.get(id=request_data.POST.get('project')),
        _institute=the_user.prof.institute_info,
        _breeze_stat=JobStat.INIT,
        target=target,
        rora_id=report_data['instance_id'])
    dbitem.assemble(request_data=request_data,
                    shared_users=shared_users,
                    sections=sections)
    dbitem.submit_to_cluster()

    return True
Exemple #2
0
def build_report(report_data, request_data, report_property, sections):
    """ Assembles report home folder, configures DRMAA and R related files
		and spawns a new process for reports DRMAA job on cluster.

	:param report_data: report info dictionary
	:type report_data: dict
	:param request_data: a copy of request object
	:type request_data: HTTPrequest
	:param report_property: report property form
	:type report_property: breezeForms.ReportPropsForm
	:param sections: a list of 'Rscripts' db objects
	:type sections: list
	:return: True
	:rtype: bool
	"""

    from breeze.models import Project, UserProfile, ReportType, Report
    from django.contrib.auth.models import User

    log = logger.getChild("build_report")
    assert isinstance(log, logging.getLoggerClass())
    assert isinstance(request_data.user, User)

    # get the request ReportType
    rt = ReportType.objects.get(type=report_data["report_type"])
    # list of users that will have access to this report
    shared_users = aux.extract_users(request_data.POST.get("Groups"), request_data.POST.get("Individuals"))
    if shared_users == list() and request_data.POST.get("shared"):
        shared_users = request_data.POST.getlist("shared")
        # author
    the_user = request_data.user
    the_user.prof = UserProfile.objects.get(user=the_user)
    assert isinstance(the_user.prof, UserProfile)

    # create initial instance so that we can use its db id
    dbitem = Report(
        _type=rt,
        _name=report_data["instance_name"],
        _author=the_user,
        project=Project.objects.get(id=request_data.POST.get("project")),
        _institute=the_user.prof.institute_info,
        _breeze_stat=JobStat.INIT,
        rora_id=report_data["instance_id"],
    )
    dbitem.assemble(request_data=request_data, shared_users=shared_users, sections=sections)
    dbitem.submit_to_cluster()

    return True
Exemple #3
0
def build_report(report_data, request_data, report_property, sections):
    """ Assembles report home folder, configures DRMAA and R related files
		and spawns a new process for reports DRMAA job on cluster.

	:param report_data: report info dictionary
	:type report_data: dict
	:param request_data: a copy of request object
	:type request_data: HTTPrequest
	:param report_property: report property form
	:type report_property: breezeForms.ReportPropsForm
	:param sections: a list of 'Rscripts' db objects
	:type sections: list
	:return: True
	:rtype: bool
	"""

    from breeze.models import Project, UserProfile, ReportType, Report, ComputeTarget
    from django.contrib.auth.models import User

    log = logger.getChild("build_report")
    assert isinstance(log, logging.getLoggerClass())
    assert isinstance(request_data.user, User)

    # get the request ReportType
    rt = ReportType.objects.get(type=report_data["report_type"])
    # list of users that will have access to this report
    shared_users = aux.extract_users(request_data.POST.get("Groups"), request_data.POST.get("Individuals"))
    if shared_users == list() and request_data.POST.get("shared"):
        shared_users = request_data.POST.getlist("shared")
        # author
    the_user = request_data.user
    the_user.prof = UserProfile.objects.get(user=the_user)
    assert isinstance(the_user.prof, UserProfile)

    # target profile :
    from django.core.exceptions import ObjectDoesNotExist

    target_id = request_data.POST.get("target")
    try:
        # target = ComputeTarget.objects.get_ready().filter(pk=target_id)
        target = ComputeTarget.objects.get(pk=target_id)
    except ObjectDoesNotExist:
        from django.contrib import messages

        messages.add_message(request_data, messages.ERROR, "No such target %s" % target_id)
        return False
    if not target.is_ready:
        from django.contrib import messages

        messages.add_message(request_data, messages.ERROR, "target %s is either disable or not ready" % target)
        return False
        # target = ComputeTarget.objects.get(pk=request_data.POST.get('target'))
        # if target.id not in rt.ready_id_list: # TODO make a validator in the form section
        # if target not in ComputeTarget.objects.ready():
        # 	from django.contrib import messages
        # 	messages.add_message(request_data, messages.INFO, 'target %s is either disable or not ready' % target)
        # 	return False

        # create initial instance so that we can use its db id
    dbitem = Report(
        _type=rt,
        _name=report_data["instance_name"],
        _author=the_user,
        project=Project.objects.get(id=request_data.POST.get("project")),
        _institute=the_user.prof.institute_info,
        _breeze_stat=JobStat.INIT,
        target=target,
        rora_id=report_data["instance_id"],
    )
    dbitem.assemble(request_data=request_data, shared_users=shared_users, sections=sections)
    dbitem.submit_to_cluster()

    return True
Exemple #4
0
def build_report(report_data, request_data, report_property, sections):
    """ Assembles report home folder, configures DRMAA and R related files
        and spawns a new process for reports DRMAA job on cluster.

    Arguments:
    report_data      -- report info dictionary
    request_data     -- a copy of request object
    report_property  -- report property form
    sections         -- a list of 'Rscripts' db objects

    """
    # 'report_name' - report's headline
    rt = breeze.models.ReportType.objects.get(type=report_data['report_type'])
    report_name = report_data['report_type'] + ' Report' + ' :: ' + report_data[
        'instance_name'] + '  <br>  ' + str(rt.description)

    # This trick is to extract users's names from the form
    # buddies = list()
    # for e in list(report_property.cleaned_data['share']):
    #     buddies.append( str(e) )

    # shared_users = breeze.models.User.objects.filter(username__in=buddies)

    shared_users = aux.extract_users(request_data.POST.get('Groups'),
                                     request_data.POST.get('Individuals'))
    insti = breeze.models.UserProfile.objects.get(
        user=request_data.user).institute_info
    # create initial instance so that we can use its db id
    dbitem = breeze.models.Report(
        type=breeze.models.ReportType.objects.get(
            type=report_data['report_type']),
        name=str(report_data['instance_name']),
        author=request_data.user,
        progress=0,
        project=breeze.models.Project.objects.get(
            id=request_data.POST.get('project')),
        institute=insti
        # project=breeze.models.Project.objects.get(name=report_property.cleaned_data['Project'])
    )
    dbitem.save()

    if shared_users:
        dbitem.shared = shared_users

    # define location: that is report's folder name
    path = slugify(
        str(dbitem.id) + '_' + dbitem.name + '_' + dbitem.author.username)
    loc = str(settings.MEDIA_ROOT) + str("reports/") + path
    dochtml = loc + '/report'
    dbitem.home = str("reports/") + path
    dbitem.save()

    # BUILD R-File
    script_string = 'setwd(\"%s\")\n' % loc
    script_string += 'require( Nozzle.R1 )\n\n'
    script_string += 'path <- \"%s\"\n' % loc
    script_string += 'report_name <- \"%s\"\n' % report_name
    # define a function for exception handler
    script_string += 'failed_fun_print <- function(section_name, error_report){\n'
    script_string += '  Error_report_par  <- newParagraph("<br>", asStrong( "Error Log Details: " ),"<br><br>",asCode(paste(error_report,collapse=""))); \n'
    script_string += '  section_name      <- addTo( section_name, newParagraph( "This section FAILED! Contact the development team... " ), Error_report_par )\n'
    script_string += '  return (section_name)\n}\n\n'

    script_string += dump_project_parameters(dbitem.project, dbitem)
    script_string += dump_pipeline_config(rt, report_data['instance_id'])

    script_string += 'REPORT <- newCustomReport(report_name)\n'

    dummy_flag = False
    for tag in sections:
        secID = 'Section_dbID_' + str(tag.id)
        if secID in request_data.POST and request_data.POST[secID] == '1':
            tree = xml.parse(str(settings.MEDIA_ROOT) + str(tag.docxml))
            script_string += '##### TAG: %s #####\n' % tag.name
            if tag.name == "Import to FileMaker":
                dummy_flag = True

            # source main code segment
            code_path = str(settings.MEDIA_ROOT) + str(tag.code)
            script_string += '# <----------  body  ----------> \n' + open(
                code_path, 'r').read() + '\n'
            script_string += '# <------- end of body --------> \n'
            # input parameters definition
            script_string += '# <----------  parameters  ----------> \n'
            script_string += gen_params_string(
                tree, request_data.POST,
                str(settings.MEDIA_ROOT) + dbitem.home, request_data.FILES)
            script_string += '# <------- end of parameters --------> \n'
            # final step - fire header
            header_path = str(settings.MEDIA_ROOT) + str(tag.header)
            script_string += '# <----------  header  ----------> \n' + open(
                header_path, 'r').read() + '\n\n'
            script_string += 'new_section <- newSection( section_name )\n'
            script_string += 'tag_section <- tryCatch({section_body(new_section)}, error = function(e){ failed_fun_print(new_section,e) })\n'
            script_string += 'REPORT <- addTo( REPORT, tag_section )\n'
            script_string += '# <------- end of header --------> \n'
            script_string += '##### END OF TAG #####\n\n\n'
            script_string += 'setwd(\"%s\")\n' % loc

        else:  # if tag disabled - do nothing
            pass
    # render report to file
    script_string += '# Render the report to a file\n' + 'writeReport( REPORT, filename=toString(\"%s\"))\n' % dochtml
    script_string += 'system("chmod -R 770 .")'

    # save r-file
    dbitem.rexec.save('script.r', base.ContentFile(script_string))
    dbitem.save()

    # configure shell-file
    config_path = loc + '/sgeconfig.sh'
    config = open(config_path, 'w')

    # config should be executble
    st = os.stat(config_path)
    os.chmod(config_path, st.st_mode | stat.S_IEXEC)

    command = '#!/bin/bash \n' + str(
        settings.R_ENGINE_PATH) + 'CMD BATCH --no-save ' + str(
            settings.MEDIA_ROOT) + str(dbitem.rexec)
    config.write(command)
    config.close()

    # open report's folder for others
    st = os.stat(loc)
    os.chmod(loc, st.st_mode | stat.S_IRWXG)

    # submit r-code
    p = Process(target=run_report, args=(dbitem, dummy_flag))
    #print(dbitem)
    #run_report(dbitem,dummy_flag)
    p.start()

    return True
Exemple #5
0
def build_report(report_data, request_data, report_property, sections):
    """ Assembles report home folder, configures DRMAA and R related files
        and spawns a new process for reports DRMAA job on cluster.

    Arguments:
    report_data      -- report info dictionary
    request_data     -- a copy of request object
    report_property  -- report property form
    sections         -- a list of 'Rscripts' db objects

    """
    # 'report_name' - report's headline
    rt = breeze.models.ReportType.objects.get(type=report_data['report_type'])
    report_name = report_data['report_type'] + ' Report' + ' :: ' + report_data['instance_name'] + '  <br>  ' + str(rt.description)

    # This trick is to extract users's names from the form
    # buddies = list()
    # for e in list(report_property.cleaned_data['share']):
    #     buddies.append( str(e) )

    # shared_users = breeze.models.User.objects.filter(username__in=buddies)

    shared_users = aux.extract_users(request_data.POST.get('Groups'), request_data.POST.get('Individuals'))
    insti = breeze.models.UserProfile.objects.get(user=request_data.user).institute_info
    # create initial instance so that we can use its db id
    dbitem = breeze.models.Report(
                type=breeze.models.ReportType.objects.get(type=report_data['report_type']),
                name=str(report_data['instance_name']),
                author=request_data.user,
                progress=0,
                project=breeze.models.Project.objects.get(id=request_data.POST.get('project')), 
                institute=insti
                # project=breeze.models.Project.objects.get(name=report_property.cleaned_data['Project'])
            )
    dbitem.save()

    if shared_users:
        dbitem.shared = shared_users

    # define location: that is report's folder name
    path = slugify(str(dbitem.id) + '_' + dbitem.name + '_' + dbitem.author.username)
    loc = str(settings.MEDIA_ROOT) + str("reports/") + path
    dochtml = loc + '/report'
    dbitem.home = str("reports/") + path
    dbitem.save()

    # BUILD R-File
    script_string = 'setwd(\"%s\")\n' % loc
    script_string += 'require( Nozzle.R1 )\n\n'
    script_string += 'path <- \"%s\"\n' % loc
    script_string += 'report_name <- \"%s\"\n' % report_name
    # define a function for exception handler
    script_string += 'failed_fun_print <- function(section_name){\n'
    script_string += '  section_name <- addTo( section_name, newParagraph( "This section FAILED! Contact the development team... " ) )\n'
    script_string += '  return (section_name)\n}\n\n'

    script_string += dump_project_parameters(dbitem.project, dbitem)
    script_string += dump_pipeline_config(rt, report_data['instance_id'])

    script_string += 'REPORT <- newCustomReport(report_name)\n'

    for tag in sections:
        secID = 'Section_dbID_' + str(tag.id)
        if secID in request_data.POST and request_data.POST[secID] == '1':
            tree = xml.parse(str(settings.MEDIA_ROOT) + str(tag.docxml))
            script_string += '##### TAG: %s #####\n' % tag.name

            # source main code segment
            code_path = str(settings.MEDIA_ROOT) + str(tag.code)
            script_string += '# <----------  body  ----------> \n' + open(code_path, 'r').read() + '\n'
            script_string += '# <------- end of body --------> \n'
            # input parameters definition
            script_string += '# <----------  parameters  ----------> \n'
            script_string += gen_params_string(tree, request_data.POST, str(settings.MEDIA_ROOT) + dbitem.home, request_data.FILES)
            script_string += '# <------- end of parameters --------> \n'
            # final step - fire header
            header_path = str(settings.MEDIA_ROOT) + str(tag.header)
            script_string += '# <----------  header  ----------> \n' + open(header_path, 'r').read() + '\n\n'
            script_string += 'new_section <- newSection( section_name )\n'
            script_string += 'tag_section <- tryCatch({section_body(new_section)}, error = function(e){ failed_fun_print(new_section) })\n'
            script_string += 'REPORT <- addTo( REPORT, tag_section )\n'
            script_string += '# <------- end of header --------> \n'
            script_string += '##### END OF TAG #####\n\n\n'
            script_string += 'setwd(\"%s\")\n' % loc
            
            
        else:  # if tag disabled - do nothing
            pass

    # render report to file
    script_string += '# Render the report to a file\n' + 'writeReport( REPORT, filename=toString(\"%s\"))\n' % dochtml
    script_string += 'system("chmod -R 770 .")'

    # save r-file
    dbitem.rexec.save('script.r', base.ContentFile(script_string))
    dbitem.save()

    # configure shell-file
    config_path = loc + '/sgeconfig.sh'
    config = open(config_path, 'w')

    # config should be executble
    st = os.stat(config_path)
    os.chmod(config_path, st.st_mode | stat.S_IEXEC)

    command = '#!/bin/bash \n' + str(settings.R_ENGINE_PATH) + 'CMD BATCH --no-save ' + str(settings.MEDIA_ROOT) + str(dbitem.rexec)
    config.write(command)
    config.close()

    # open report's folder for others
    st = os.stat(loc)
    os.chmod(loc, st.st_mode | stat.S_IRWXG)

    # submit r-code
    p = Process(target=run_report, args=(dbitem,))
    p.start()

    return True