Example #1
0
def file_transfer(request):
    '''
    Triggers VBL file transfer which triggers tranny transfer. This
    url is typcially called my another MyTARDIS instance which has
    received a METS file.
    '''

    if request.method == 'POST':

        form = FileTransferRequestForm(request.POST)
        if form.is_valid():

            site_settings_url = request.POST['site_settings_url']
            # remote experiment id
            expid = request.POST['eid']
            # local experiment id
            originid = request.POST['originid']

            try:
                experiment = models.Experiment.objects.get(pk=int(originid))
            except models.Experiment.DoesNotExist:
                logger.error('experiment %s does not exist' % originid)
                return HttpResponse('ERROR', status=404)

            # get EPN (needed to kick-off vbl gridftp transfer)
            epn = models.ExperimentParameter.objects.get(parameterset__experiment=experiment,
                                                         name__name='EPN').string_value
            logger.info('=== file transfer evoked: epn %s' % epn)
            logger.debug(site_settings_url)

            # checking MyTARDIS sites
            url = settings.MYTARDIS_SITES_URL
            logger.debug('fetching mytardis sites from %s' % url)
            try:
                sites_username = settings.MYTARDIS_SITES_USERNAME
                sites_password = settings.MYTARDIS_SITES_PASSWORD
            except AttributeError:
                sites_username = ''
                sites_password = ''

            # check if the requesting site is known (very basic
            # security)
            transfer = False
            sp = SiteParser(url, sites_username, sites_password)
            for name in sp.getSiteNames():
                if site_settings_url == sp.getSiteSettingsURL(name):
                    transfer = True
                    username = sp.getSiteSettingsUsername(name)
                    password = sp.getSiteSettingsPassword(name)

            if transfer == False:
                logger.error('site %s not resgistered in %s' % (site_settings_url, url))
                logger.error('file transfer request not allowed!')
                logger.error('=== file transfer for epn %s FAILED!' % epn)
                return HttpResponse('ERROR', status=403)
            else:
                logger.debug('found site %s in %s' % (site_settings_url, url))

            logger.debug('fetching site config for %s from %s' % (name, site_settings_url))

            # read remote MyTARDIS config
            ssp = SiteSettingsParser(site_settings_url, username, password)
            client = Client(settings.VBLSTORAGEGATEWAY, cache=None)
            x509 = ssp.getTransferSetting('password')

            # build destination path
            dirname = os.path.abspath(
                os.path.join(ssp.getTransferSetting('serversite'), expid)
                )
            logger.debug('destination url:  %s' % site_settings_url)
            logger.debug('destination path: %s' % dirname)

            # contact VBL
            key = client.service.VBLstartTransferGridFTP(
                ssp.getTransferSetting('user'),
                x509,
                epn,
                '/Frames\\r\\nTARDIS\\r\\n',
                ssp.getTransferSetting('sl'),
                dirname,
                ssp.getTransferSetting('optionFast'),
                ssp.getTransferSetting('optionPenable'),
                ssp.getTransferSetting('optionP'),
                ssp.getTransferSetting('optionBSenable'),
                ssp.getTransferSetting('optionBS'),
                ssp.getTransferSetting('optionTCPBenable'),
                ssp.getTransferSetting('optionTCPBS'))

            if key.startswith('Error:'):
                logger.error('[vbl] %s: epn %s' % (key, epn))
                return HttpResponse('ERROR', status=403)
            else:
                logger.info('[vbl] %s: pn %s' % (key, epn))

            return HttpResponse('OK', status=200)

    else:
        form = FileTransferRequestForm()

    c = Context({'header': 'Register File Transfer Request',
                 'form': form})
    return render_to_response('tardis_portal/form_template.html', c)
Example #2
0
def _register_file_transfer(request, cleaned_data, expid):
    """
    ingests meta-data and transfers data to experiment owner's
    mytardis instance at the home institute

    :keyword request: Django request
    :keyword cleaned_data: cleaned form data
    :keyword expid: id of experiment to be transfered
    """

    # sites which will receive a copy of the data
    sites = []
    # check if the experiment exists locally at all
    try:
        experiment = models.Experiment.objects.get(pk=expid)
    except models.Experiment.DoesNotExist:
        logger.error('experiment %i does not exist' % expid)
        return

    # register at home institute
    owners = re.compile(r'\s+').split(cleaned_data['experiment_owner'])
    for owner in owners:
        if owner == '':
            continue

        # get list of site from master tardis instance (Monash)
        # TODO: might be better to store these sites in Django's
        # sites table!
        url = settings.MYTARDIS_SITES_URL
        logger.debug('fetching mytardis sites from %s' % url)
        try:
            sites_username = settings.MYTARDIS_SITES_USERNAME
            sites_password = settings.MYTARDIS_SITES_PASSWORD
        except AttributeError:
            # maybe we get the site list from a file...
            sites_username = ''
            sites_password = ''

        # username and password must be set if another site is contacted
        sp = SiteParser(url, sites_username, sites_password)

        # loop over sites
        for name in sp.getSiteNames():
            url = sp.getSiteSettingsURL(name)
            # fetch a MyTARDIS site's config
            logger.debug('fetching site config for %s from %s' % (name, url))
            try:
                ssp = SiteSettingsParser(url,
                                         sp.getSiteSettingsUsername(name),
                                         sp.getSiteSettingsPassword(name))
            except:
                logger.exception('fetching site config from %s FAILED' % url)
                continue

            # is the email domain of the experiment's owner registered
            # by any site?
            siteOwners = []
            for owner in owners:
                for domain in ssp.getEmailEndswith():
                    if owner.endswith(domain):
                        siteOwners.append(owner)

            # register meta-data and file transfer request at another
            # MyTARDIS instance
            if siteOwners:
                # Create the form with simple fields
                mpform = MultiPartForm()
                mpform.add_field('username', ssp.getRegisterSetting('username'))
                mpform.add_field('password', ssp.getRegisterSetting('password'))
                mpform.add_field('from_url', request.build_absolute_uri())
                mpform.add_field('originid', str(expid))

                for siteOwner in siteOwners:
                    mpform.add_field('experiment_owner', siteOwner)

                protocol = ssp.getRegisterSetting('fileProtocol')

                # export METS file
                filename = 'mets_expid_%i_%s' % (experiment.id, protocol)
                logger.debug('=== extracting mets file for experiment %i ' % expid)
                from tardis.tardis_portal.metsexporter import MetsExporter
                exporter = MetsExporter()
                if protocol:
                    # translate vbl:// into tardis:// url for datafiles
                    metsfile = exporter.export(experimentId=experiment.id,
                                               replace_protocols={'vbl': protocol},
                                               filename=filename,
                                               export_images=False)
                else:
                    metsfile = exporter.export(experimentId=experiment.id,
                                               filename=filename,
                                               export_images=False)
                logger.debug('=== extraction done, filename = %s' % metsfile)

                f = open(metsfile, "r")
                mpform.add_file('xmldata', 'METS.xml', fileHandle=f)

                ws = ssp.getRegisterSetting('url')
                logger.debug('about to send register request to site %s' % ws)
                # build the request
                requestmp = urllib2.Request(ws)
                requestmp.add_header('User-agent', 'PyMOTW (http://www.doughellmann.com/PyMOTW/)')
                body = str(mpform)
                requestmp.add_header('Content-type', mpform.get_content_type())
                requestmp.add_header('Content-length', len(body))

                # logger.debug('OUTGOING DATA: ' + body)
                logger.debug('SERVER RESPONSE: ' + urllib2.urlopen(requestmp, body, 99999).read())

                f.close()
                sites.append(url)

        # return a list of registered sites
        return sites

    return []