def _post_experiment(self, experiment, site_owners, site_settings): uid = self.generate_exp_uid(experiment) url = site_settings['url'] username = site_settings['username'] password = site_settings['password'] protocol = site_settings['fileProtocol'] # Create the form with simple fields mpform = MultiPartForm() mpform.add_field('username', username) mpform.add_field('password', password) mpform.add_field('from_url', settings.MYTARDIS_SITE_URL) mpform.add_field('originid', uid) for owner in site_owners: mpform.add_field('experiment_owner', owner) # export METS file filename = 'mets_expid_%i_%s' % (experiment.id, protocol) logger.debug('=== extracting mets file for experiment %s ' % uid) from tardis.tardis_portal.metsexporter import MetsExporter exporter = MetsExporter() if protocol: # translate vbl:// into tardis:// url for datafiles metsfile = exporter.export(experimentId=experiment.id, replace_protocols={'vbl': protocol}, filename=filename, export_images=False) else: metsfile = exporter.export(experimentId=experiment.id, filename=filename, export_images=False) logger.debug('=== extraction done, filename = %s' % metsfile) f = open(metsfile, "r") mpform.add_file('xmldata', 'METS.xml', fileHandle=f) body = str(mpform) logger.debug('about to send register request to site %s' % url) # build the request headers = { 'User-agent': 'MyTardis', 'Content-type': mpform.get_content_type(), } # This should be made into a background task. # Or rather- the processing on the other end should be. h = Http(timeout=9999, disable_ssl_certificate_validation=True) h.force_exception_to_status_code = True resp, content = h.request(url, 'POST', headers=headers, body=body) f.close() if resp.status != 200: logger.error('Posting experiment to %s failed:' % url) logger.error('%s: %s' % (resp.status, resp.reason)) logger.debug('SERVER RESPONSE: ' + content) return False return True
def _post_experiment(self, experiment, site_owners, site_settings): uid = self.generate_exp_uid(experiment) url = site_settings['url'] username = site_settings['username'] password = site_settings['password'] protocol = site_settings['fileProtocol'] # Create the form with simple fields mpform = MultiPartForm() mpform.add_field('username', username) mpform.add_field('password', password) mpform.add_field('from_url', settings.MYTARDIS_SITE_URL) mpform.add_field('originid', uid) for owner in site_owners: mpform.add_field('experiment_owner', owner) # export METS file filename = 'mets_expid_%i_%s' % (experiment.id, protocol) logger.debug('=== extracting mets file for experiment %s ' % uid) from tardis.tardis_portal.metsexporter import MetsExporter exporter = MetsExporter() if protocol: # translate vbl:// into tardis:// url for datafiles metsfile = exporter.export(experimentId=experiment.id, replace_protocols={'vbl': protocol}, filename=filename, export_images=False) else: metsfile = exporter.export(experimentId=experiment.id, filename=filename, export_images=False) logger.debug('=== extraction done, filename = %s' % metsfile) f = open(metsfile, "r") mpform.add_file('xmldata', 'METS.xml', fileHandle=f) body = str(mpform) logger.debug('about to send register request to site %s' % url) # build the request headers = { 'User-agent': 'MyTardis', 'Content-type': mpform.get_content_type(), } # This should be made into a background task. # Or rather- the processing on the other end should be. h = Http(timeout=9999, disable_ssl_certificate_validation=True) h.force_exception_to_status_code = True resp, content = h.request(url, 'POST', headers=headers, body=body) f.close() if resp.status != 200: logger.error('Posting experiment to %s failed:' % url) logger.error('%s: %s' % (resp.status, resp.reason)) logger.debug('SERVER RESPONSE: ' + content) return False return True
def poll_feed_sub(mb, sub, config): if sub.subscription_type != FEED_SUBSCRIPTION_TYPE: return False # sweet, go ahead... http = Http(config.get('http_cache', None)) http.timeout = 15 http.force_exception_to_status_code = True poll_feed(mb, sub, http) return True
def index_feed_polling(url, context, timeout=15, request_info=None): """ poll the feed at the url given and index it immediately on the calling thread. """ if request_info is None: request_info = {} feed = RemoteFeed.get_by_url(url, context) if feed is None: feed = RemoteFeed.create_from_url(url, context) if check_request_approved(feed, request_info, context) == False: log.warn("Rejected index request for %s" % url) return reschedule = not request_info.get('skip_reschedule', False) http_cache = context.config.get('http', {}).get('cache', None) # fetch http = Http(cache=http_cache, timeout=timeout) http.force_exception_to_status_code = True response, content = http.request(url, 'GET') updated_docs = [] if response.fromcache: feed.record_update_info(success=True, updates=0, method=METHOD_POLL) elif response.status != 200: feed.record_update_info(success=False, updates=0, reason=response.reason, method=METHOD_POLL) else: # 200 status code, not from cache, do update... feed.update_from_feed(content, method=METHOD_POLL) # compute the next time to check... next_interval = compute_next_fetch_interval(feed.update_history) log.debug("next update interval for %s = %s" % (feed.url, next_interval)) feed.next_poll_time = datetime.utcnow() + next_interval feed.poll_in_progress = False feed.save() log.info("Updated feed %s success: %s, %d new items" % (feed.url, feed.update_history[0].success, feed.update_history[0].updates)) # whee... request at the next time ! if reschedule: message_id = 'periodic_index_%s' % RemoteFeed.id_for_url(feed.url) schedule_feed_index(feed.url, feed.next_poll_time, context, message_id=message_id) run_post_index_hooks(feed, context)
def _open_file(self, url, username, password): if self.url.startswith('http'): # This should pull in and use HttpClient or at least httplib2. body = urllib.urlencode({'username': username, 'password': password}) h = Http(timeout=10, disable_ssl_certificate_validation=True) h.force_exception_to_status_code = True resp, content = h.request(url, 'POST', body=body) if resp.status != 200: logger.error('URLParser: POST for url %s failed: %s %s' % (url, resp.status, resp.reason)) return None f = StringIO(content) else: try: f = open(url) except IOError: logger.error('URLParser: File %s does not exist' % (url)) return None return f
def get_http(): ''' Get an http object This is just an instance of the httplib2.Http class, properly customized, extended and configured. Note in most cases, handlers and other code should use the get_http method of a MobileSite instance, rather than using this function directly, in case the desktop source of a particular mobile site needs further customization. @return : http object @rtype : httplib2.Http ''' from httplib2 import Http http = Http() http.follow_redirects = False http.force_exception_to_status_code = True return http
def create_client(cfg): """ create an http client according to the configuration given. """ cfg = config_section('http', cfg) kw = {} if 'timeout' in cfg: kw['timeout'] = cfg['timeout'] if 'cache' in cfg: kw['cache'] = cfg['cache'] if cfg.get('allow_local') == True: client = UnrestrictedHttp(**kw) else: client = RestrictedHttp(**kw) client.force_exception_to_status_code = True return client
def _open_file(self, url, username, password): if self.url.startswith('http'): # This should pull in and use HttpClient or at least httplib2. body = urllib.urlencode({ 'username': username, 'password': password }) h = Http(timeout=10, disable_ssl_certificate_validation=True) h.force_exception_to_status_code = True resp, content = h.request(url, 'POST', body=body) if resp.status != 200: logger.error('URLParser: POST for url %s failed: %s %s' % (url, resp.status, resp.reason)) return None f = StringIO(content) else: try: f = open(url) except IOError: logger.error('URLParser: File %s does not exist' % (url)) return None return f
sqlhub.processConnection = connectionForURI('postgresql://*****:*****@localhost/onebusaway') targetServer = "http://novalis.org/cgi/env.cgi" class LocationRecord(SQLObject): class sqlmeta: table = "oba_nyc_raw_location" style = MixedCaseStyle() timeReceived = IntCol() rawData = StringCol() lastCheck = time() * 1000 while 1: sleep(1) for row in LocationRecord.select(LocationRecord.q.timeReceived > lastCheck).orderBy("timeReceived"): print "sending row" sys.stdout.flush() h = Http() h.force_exception_to_status_code = True resp, content = h.request(targetServer, "POST", body=row.rawData) if not resp['status'] == '200': print resp, content print row.rawData print if lastCheck < row.timeReceived: lastCheck = row.timeReceived
def get_http(cls): """Send http request without validade ssl certificate""" http = Http() http.force_exception_to_status_code = True http.disable_ssl_certificate_validation = True return http