def file_transfer(request): ''' Triggers VBL file transfer which triggers tranny transfer. This url is typcially called my another MyTARDIS instance which has received a METS file. ''' if request.method == 'POST': form = FileTransferRequestForm(request.POST) if form.is_valid(): site_settings_url = request.POST['site_settings_url'] # remote experiment id expid = request.POST['eid'] # local experiment id originid = request.POST['originid'] try: experiment = models.Experiment.objects.get(pk=int(originid)) except models.Experiment.DoesNotExist: logger.error('experiment %s does not exist' % originid) return HttpResponse('ERROR', status=404) # get EPN (needed to kick-off vbl gridftp transfer) epn = models.ExperimentParameter.objects.get(parameterset__experiment=experiment, name__name='EPN').string_value logger.info('=== file transfer evoked: epn %s' % epn) logger.debug(site_settings_url) # checking MyTARDIS sites url = settings.MYTARDIS_SITES_URL logger.debug('fetching mytardis sites from %s' % url) try: sites_username = settings.MYTARDIS_SITES_USERNAME sites_password = settings.MYTARDIS_SITES_PASSWORD except AttributeError: sites_username = '' sites_password = '' # check if the requesting site is known (very basic # security) transfer = False sp = SiteParser(url, sites_username, sites_password) for name in sp.getSiteNames(): if site_settings_url == sp.getSiteSettingsURL(name): transfer = True username = sp.getSiteSettingsUsername(name) password = sp.getSiteSettingsPassword(name) if transfer == False: logger.error('site %s not resgistered in %s' % (site_settings_url, url)) logger.error('file transfer request not allowed!') logger.error('=== file transfer for epn %s FAILED!' % epn) return HttpResponse('ERROR', status=403) else: logger.debug('found site %s in %s' % (site_settings_url, url)) logger.debug('fetching site config for %s from %s' % (name, site_settings_url)) # read remote MyTARDIS config ssp = SiteSettingsParser(site_settings_url, username, password) client = Client(settings.VBLSTORAGEGATEWAY, cache=None) x509 = ssp.getTransferSetting('password') # build destination path dirname = os.path.abspath( os.path.join(ssp.getTransferSetting('serversite'), expid) ) logger.debug('destination url: %s' % site_settings_url) logger.debug('destination path: %s' % dirname) # contact VBL key = client.service.VBLstartTransferGridFTP( ssp.getTransferSetting('user'), x509, epn, '/Frames\\r\\nTARDIS\\r\\n', ssp.getTransferSetting('sl'), dirname, ssp.getTransferSetting('optionFast'), ssp.getTransferSetting('optionPenable'), ssp.getTransferSetting('optionP'), ssp.getTransferSetting('optionBSenable'), ssp.getTransferSetting('optionBS'), ssp.getTransferSetting('optionTCPBenable'), ssp.getTransferSetting('optionTCPBS')) if key.startswith('Error:'): logger.error('[vbl] %s: epn %s' % (key, epn)) return HttpResponse('ERROR', status=403) else: logger.info('[vbl] %s: pn %s' % (key, epn)) return HttpResponse('OK', status=200) else: form = FileTransferRequestForm() c = Context({'header': 'Register File Transfer Request', 'form': form}) return render_to_response('tardis_portal/form_template.html', c)
def _register_file_transfer(request, cleaned_data, expid): """ ingests meta-data and transfers data to experiment owner's mytardis instance at the home institute :keyword request: Django request :keyword cleaned_data: cleaned form data :keyword expid: id of experiment to be transfered """ # sites which will receive a copy of the data sites = [] # check if the experiment exists locally at all try: experiment = models.Experiment.objects.get(pk=expid) except models.Experiment.DoesNotExist: logger.error('experiment %i does not exist' % expid) return # register at home institute owners = re.compile(r'\s+').split(cleaned_data['experiment_owner']) for owner in owners: if owner == '': continue # get list of site from master tardis instance (Monash) # TODO: might be better to store these sites in Django's # sites table! url = settings.MYTARDIS_SITES_URL logger.debug('fetching mytardis sites from %s' % url) try: sites_username = settings.MYTARDIS_SITES_USERNAME sites_password = settings.MYTARDIS_SITES_PASSWORD except AttributeError: # maybe we get the site list from a file... sites_username = '' sites_password = '' # username and password must be set if another site is contacted sp = SiteParser(url, sites_username, sites_password) # loop over sites for name in sp.getSiteNames(): url = sp.getSiteSettingsURL(name) # fetch a MyTARDIS site's config logger.debug('fetching site config for %s from %s' % (name, url)) try: ssp = SiteSettingsParser(url, sp.getSiteSettingsUsername(name), sp.getSiteSettingsPassword(name)) except: logger.exception('fetching site config from %s FAILED' % url) continue # is the email domain of the experiment's owner registered # by any site? siteOwners = [] for owner in owners: for domain in ssp.getEmailEndswith(): if owner.endswith(domain): siteOwners.append(owner) # register meta-data and file transfer request at another # MyTARDIS instance if siteOwners: # Create the form with simple fields mpform = MultiPartForm() mpform.add_field('username', ssp.getRegisterSetting('username')) mpform.add_field('password', ssp.getRegisterSetting('password')) mpform.add_field('from_url', request.build_absolute_uri()) mpform.add_field('originid', str(expid)) for siteOwner in siteOwners: mpform.add_field('experiment_owner', siteOwner) protocol = ssp.getRegisterSetting('fileProtocol') # export METS file filename = 'mets_expid_%i_%s' % (experiment.id, protocol) logger.debug('=== extracting mets file for experiment %i ' % expid) from tardis.tardis_portal.metsexporter import MetsExporter exporter = MetsExporter() if protocol: # translate vbl:// into tardis:// url for datafiles metsfile = exporter.export(experimentId=experiment.id, replace_protocols={'vbl': protocol}, filename=filename, export_images=False) else: metsfile = exporter.export(experimentId=experiment.id, filename=filename, export_images=False) logger.debug('=== extraction done, filename = %s' % metsfile) f = open(metsfile, "r") mpform.add_file('xmldata', 'METS.xml', fileHandle=f) ws = ssp.getRegisterSetting('url') logger.debug('about to send register request to site %s' % ws) # build the request requestmp = urllib2.Request(ws) requestmp.add_header('User-agent', 'PyMOTW (http://www.doughellmann.com/PyMOTW/)') body = str(mpform) requestmp.add_header('Content-type', mpform.get_content_type()) requestmp.add_header('Content-length', len(body)) # logger.debug('OUTGOING DATA: ' + body) logger.debug('SERVER RESPONSE: ' + urllib2.urlopen(requestmp, body, 99999).read()) f.close() sites.append(url) # return a list of registered sites return sites return []