def _post_experiment(self, experiment, site_owners, site_settings):
uid = self.generate_exp_uid(experiment)
url = site_settings['url']
username = site_settings['username']
password = site_settings['password']
protocol = site_settings['fileProtocol']
# Create the form with simple fields
mpform = MultiPartForm()
mpform.add_field('username', username)
mpform.add_field('password', password)
mpform.add_field('from_url', settings.MYTARDIS_SITE_URL)
mpform.add_field('originid', uid)
for owner in site_owners:
mpform.add_field('experiment_owner', owner)
# export METS file
filename = 'mets_expid_%i_%s' % (experiment.id, protocol)
logger.debug('=== extracting mets file for experiment %s ' % uid)
from tardis.tardis_portal.metsexporter import MetsExporter
exporter = MetsExporter()
if protocol:
# translate vbl:// into tardis:// url for datafiles
metsfile = exporter.export(experimentId=experiment.id,
replace_protocols={'vbl': protocol},
filename=filename,
export_images=False)
else:
metsfile = exporter.export(experimentId=experiment.id,
filename=filename,
export_images=False)
logger.debug('=== extraction done, filename = %s' % metsfile)
f = open(metsfile, "r")
mpform.add_file('xmldata', 'METS.xml', fileHandle=f)
body = str(mpform)
logger.debug('about to send register request to site %s' % url)
# build the request
headers = {
'User-agent': 'MyTardis',
'Content-type': mpform.get_content_type(),
}
# This should be made into a background task.
# Or rather- the processing on the other end should be.
h = Http(timeout=9999, disable_ssl_certificate_validation=True)
h.force_exception_to_status_code = True
resp, content = h.request(url, 'POST', headers=headers, body=body)
f.close()
if resp.status != 200:
logger.error('Posting experiment to %s failed:' % url)
logger.error('%s: %s' % (resp.status, resp.reason))
logger.debug('SERVER RESPONSE: ' + content)
return False
return True
def _post_experiment(self, experiment, site_owners, site_settings):
uid = self.generate_exp_uid(experiment)
url = site_settings['url']
username = site_settings['username']
password = site_settings['password']
protocol = site_settings['fileProtocol']
# Create the form with simple fields
mpform = MultiPartForm()
mpform.add_field('username', username)
mpform.add_field('password', password)
mpform.add_field('from_url', settings.MYTARDIS_SITE_URL)
mpform.add_field('originid', uid)
for owner in site_owners:
mpform.add_field('experiment_owner', owner)
# export METS file
filename = 'mets_expid_%i_%s' % (experiment.id, protocol)
logger.debug('=== extracting mets file for experiment %s ' % uid)
from tardis.tardis_portal.metsexporter import MetsExporter
exporter = MetsExporter()
if protocol:
# translate vbl:// into tardis:// url for datafiles
metsfile = exporter.export(experimentId=experiment.id,
replace_protocols={'vbl': protocol},
filename=filename,
export_images=False)
else:
metsfile = exporter.export(experimentId=experiment.id,
filename=filename,
export_images=False)
logger.debug('=== extraction done, filename = %s' % metsfile)
f = open(metsfile, "r")
mpform.add_file('xmldata', 'METS.xml', fileHandle=f)
body = str(mpform)
logger.debug('about to send register request to site %s' % url)
# build the request
headers = {
'User-agent': 'MyTardis',
'Content-type': mpform.get_content_type(),
}
# This should be made into a background task.
# Or rather- the processing on the other end should be.
h = Http(timeout=9999, disable_ssl_certificate_validation=True)
h.force_exception_to_status_code = True
resp, content = h.request(url, 'POST', headers=headers, body=body)
f.close()
if resp.status != 200:
logger.error('Posting experiment to %s failed:' % url)
logger.error('%s: %s' % (resp.status, resp.reason))
logger.debug('SERVER RESPONSE: ' + content)
return False
return True
def poll_feed_sub(mb, sub, config):
if sub.subscription_type != FEED_SUBSCRIPTION_TYPE:
return False
# sweet, go ahead...
http = Http(config.get('http_cache', None))
http.timeout = 15
http.force_exception_to_status_code = True
poll_feed(mb, sub, http)
return True
def index_feed_polling(url, context, timeout=15, request_info=None):
"""
poll the feed at the url given and index it immediately on
the calling thread.
"""
if request_info is None:
request_info = {}
feed = RemoteFeed.get_by_url(url, context)
if feed is None:
feed = RemoteFeed.create_from_url(url, context)
if check_request_approved(feed, request_info, context) == False:
log.warn("Rejected index request for %s" % url)
return
reschedule = not request_info.get('skip_reschedule', False)
http_cache = context.config.get('http', {}).get('cache', None)
# fetch
http = Http(cache=http_cache, timeout=timeout)
http.force_exception_to_status_code = True
response, content = http.request(url, 'GET')
updated_docs = []
if response.fromcache:
feed.record_update_info(success=True, updates=0, method=METHOD_POLL)
elif response.status != 200:
feed.record_update_info(success=False, updates=0,
reason=response.reason, method=METHOD_POLL)
else:
# 200 status code, not from cache, do update...
feed.update_from_feed(content, method=METHOD_POLL)
# compute the next time to check...
next_interval = compute_next_fetch_interval(feed.update_history)
log.debug("next update interval for %s = %s" % (feed.url, next_interval))
feed.next_poll_time = datetime.utcnow() + next_interval
feed.poll_in_progress = False
feed.save()
log.info("Updated feed %s success: %s, %d new items" %
(feed.url, feed.update_history[0].success, feed.update_history[0].updates))
# whee... request at the next time !
if reschedule:
message_id = 'periodic_index_%s' % RemoteFeed.id_for_url(feed.url)
schedule_feed_index(feed.url, feed.next_poll_time, context, message_id=message_id)
run_post_index_hooks(feed, context)
def _open_file(self, url, username, password):
if self.url.startswith('http'):
# This should pull in and use HttpClient or at least httplib2.
body = urllib.urlencode({'username': username, 'password': password})
h = Http(timeout=10, disable_ssl_certificate_validation=True)
h.force_exception_to_status_code = True
resp, content = h.request(url, 'POST', body=body)
if resp.status != 200:
logger.error('URLParser: POST for url %s failed: %s %s' % (url, resp.status, resp.reason))
return None
f = StringIO(content)
else:
try:
f = open(url)
except IOError:
logger.error('URLParser: File %s does not exist' % (url))
return None
return f
def get_http():
'''
Get an http object
This is just an instance of the httplib2.Http class, properly
customized, extended and configured.
Note in most cases, handlers and other code should use the
get_http method of a MobileSite instance, rather than using this
function directly, in case the desktop source of a particular
mobile site needs further customization.
@return : http object
@rtype : httplib2.Http
'''
from httplib2 import Http
http = Http()
http.follow_redirects = False
http.force_exception_to_status_code = True
return http
def create_client(cfg):
"""
create an http client according to
the configuration given.
"""
cfg = config_section('http', cfg)
kw = {}
if 'timeout' in cfg:
kw['timeout'] = cfg['timeout']
if 'cache' in cfg:
kw['cache'] = cfg['cache']
if cfg.get('allow_local') == True:
client = UnrestrictedHttp(**kw)
else:
client = RestrictedHttp(**kw)
client.force_exception_to_status_code = True
return client
def _open_file(self, url, username, password):
if self.url.startswith('http'):
# This should pull in and use HttpClient or at least httplib2.
body = urllib.urlencode({
'username': username,
'password': password
})
h = Http(timeout=10, disable_ssl_certificate_validation=True)
h.force_exception_to_status_code = True
resp, content = h.request(url, 'POST', body=body)
if resp.status != 200:
logger.error('URLParser: POST for url %s failed: %s %s' %
(url, resp.status, resp.reason))
return None
f = StringIO(content)
else:
try:
f = open(url)
except IOError:
logger.error('URLParser: File %s does not exist' % (url))
return None
return f
sqlhub.processConnection = connectionForURI('postgresql://*****:*****@localhost/onebusaway')
targetServer = "http://novalis.org/cgi/env.cgi"
class LocationRecord(SQLObject):
class sqlmeta:
table = "oba_nyc_raw_location"
style = MixedCaseStyle()
timeReceived = IntCol()
rawData = StringCol()
lastCheck = time() * 1000
while 1:
sleep(1)
for row in LocationRecord.select(LocationRecord.q.timeReceived > lastCheck).orderBy("timeReceived"):
print "sending row"
sys.stdout.flush()
h = Http()
h.force_exception_to_status_code = True
resp, content = h.request(targetServer,
"POST", body=row.rawData)
if not resp['status'] == '200':
print resp, content
print row.rawData
print
if lastCheck < row.timeReceived:
lastCheck = row.timeReceived
def get_http(cls):
"""Send http request without validade ssl certificate"""
http = Http()
http.force_exception_to_status_code = True
http.disable_ssl_certificate_validation = True
return http
