def test_param_guard(self):
r = route(
"/news/{id:int}/", "target",
guards=qs(q=opt(String), page=opt(Int)))
req = Request.blank("/news/42/")
tr = r(req)
self.assertEqual(
(tr.args, tr.kwargs, tr.target),
((42,), {}, "target"))
req = Request.blank("/news/42/?q=search")
tr = r(req)
self.assertEqual(
(tr.args, tr.kwargs.items(), tr.target),
((42,), [("q", "search")], "target"))
req = Request.blank("/news/42/?q=search&page=100")
tr = r(req)
self.assertEqual(
(tr.args, tr.kwargs.items(), tr.target),
((42,), [("q", "search"), ("page", 100)], "target"))
self.assertRaises(
exc.HTTPBadRequest,
r, Request.blank("/news/42/?q=search&page=aa"))
def test_GET_limit_marker_json(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
self.controller.PUT(req)
for c in xrange(5):
req = Request.blank('/sda1/p/a/c%d' % c,
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Put-Timestamp': str(c + 1),
'X-Delete-Timestamp': '0',
'X-Object-Count': '2',
'X-Bytes-Used': '3',
'X-Timestamp': normalize_timestamp(0)})
self.controller.PUT(req)
req = Request.blank('/sda1/p/a?limit=3&format=json',
environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 200)
self.assertEquals(simplejson.loads(resp.body),
[{'count': 2, 'bytes': 3, 'name': 'c0'},
{'count': 2, 'bytes': 3, 'name': 'c1'},
{'count': 2, 'bytes': 3, 'name': 'c2'}])
req = Request.blank('/sda1/p/a?limit=3&marker=c2&format=json',
environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 200)
self.assertEquals(simplejson.loads(resp.body),
[{'count': 2, 'bytes': 3, 'name': 'c3'},
{'count': 2, 'bytes': 3, 'name': 'c4'}])
def test_POST_reset_container_sync(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'x-timestamp': '1',
'x-container-sync-to': 'http://127.0.0.1:12345/v1/a/c'})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 201)
db = self.controller._get_container_broker('sda1', 'p', 'a', 'c')
info = db.get_info()
self.assertEquals(info['x_container_sync_point1'], -1)
self.assertEquals(info['x_container_sync_point2'], -1)
db.set_x_container_sync_points(123, 456)
info = db.get_info()
self.assertEquals(info['x_container_sync_point1'], 123)
self.assertEquals(info['x_container_sync_point2'], 456)
# Set to same value
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'x-timestamp': '1',
'x-container-sync-to': 'http://127.0.0.1:12345/v1/a/c'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 204)
db = self.controller._get_container_broker('sda1', 'p', 'a', 'c')
info = db.get_info()
self.assertEquals(info['x_container_sync_point1'], 123)
self.assertEquals(info['x_container_sync_point2'], 456)
# Set to new value
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'x-timestamp': '1',
'x-container-sync-to': 'http://127.0.0.1:12345/v1/a/c2'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 204)
db = self.controller._get_container_broker('sda1', 'p', 'a', 'c')
info = db.get_info()
self.assertEquals(info['x_container_sync_point1'], -1)
self.assertEquals(info['x_container_sync_point2'], -1)
def test_group_inexact_pattern(self):
r = route("news",
route("{id:int}",
route("comments", "view")))
req = Request.blank("/news/42/comments")
tr = r(req)
self.assertEqual(
(tr.args, tr.kwargs, tr.endpoint.target),
((42,), {}, "view"))
r = route("news/{id:int}",
route("comments", "view"))
req = Request.blank("/news/42/comments")
tr = r(req)
self.assertEqual(
(tr.args, tr.kwargs, tr.endpoint.target),
((42,), {}, "view"))
r = route("news",
route("{id:int}/comments", "view"))
req = Request.blank("/news/42/comments")
tr = r(req)
self.assertEqual(
(tr.args, tr.kwargs, tr.endpoint.target),
((42,), {}, "view"))
def test_GET_empty_account_plain(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
self.controller.PUT(req)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 204)
def test_cdn_get_regex(self):
prev_data = json.dumps({'account': 'acc', 'container': 'cont',
'ttl': 1234, 'logs_enabled': True, 'cdn_enabled': True})
def check_urls(req):
vrs, acc, cont, obj = utils.split_path(req.path, 1, 4)
self.assertEquals(acc, 'acc')
self.assertEquals(cont, 'cont')
self.assertEquals(obj, 'obj1.jpg')
self.test_origin.app = FakeApp(iter([
('204 No Content', {}, prev_data), # call to _get_cdn_data
('304 No Content', {}, '', check_urls)])) #call to get obj
req = Request.blank('http://1234.r3.origin_cdn.com:8080/obj1.jpg',
environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.test_origin)
self.assertEquals(resp.status_int, 304)
self.test_origin.app = FakeApp(iter([
('204 No Content', {}, prev_data), # call to _get_cdn_data
('304 No Content', {}, '', check_urls)])) #call to get obj
req = Request.blank('http://r3.origin_cdn.com:8080/nohash/obj1.jpg',
environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.test_origin)
self.assertEquals(resp.status_int, 404)
def make_pre_authed_request(env, method, path, body=None, headers=None, agent="Swift"):
"""
Makes a new webob.Request based on the current env but with the
parameters specified. Note that this request will be preauthorized.
:param env: Current WSGI environment dictionary
:param method: HTTP method of new request
:param path: HTTP path of new request
:param body: HTTP body of new request; None by default
:param headers: Extra HTTP headers of new request; None by default
:returns: webob.Request object
(Stolen from Swauth: https://github.com/gholt/swauth)
"""
newenv = {"REQUEST_METHOD": method, "HTTP_USER_AGENT": agent}
for name in ("swift.cache", "swift.trans_id"):
if name in env:
newenv[name] = env[name]
newenv["swift.authorize"] = lambda req: None
if not headers:
headers = {}
if body:
return Request.blank(path, environ=newenv, body=body, headers=headers)
else:
return Request.blank(path, environ=newenv, headers=headers)
def test_origin_db_post_fail(self):
self.test_origin.app = FakeApp(iter([
('204 No Content', {}, ''), # call to _get_cdn_data
('404 Not Found', {}, ''), # put to .hash
]))
req = Request.blank('http://origin_db.com:8080/v1/acc/cont',
environ={'REQUEST_METHOD': 'PUT'})
resp = req.get_response(self.test_origin)
self.assertEquals(resp.status_int, 500)
self.test_origin.app = FakeApp(iter([
('204 No Content', {}, ''), # call to _get_cdn_data
('204 No Content', {}, ''), # put to .hash
('404 Not Found', {}, ''), # HEAD check to list container
('404 Not Found', {}, ''), # PUT to list container
]))
req = Request.blank('http://origin_db.com:8080/v1/acc/cont',
environ={'REQUEST_METHOD': 'PUT'})
resp = req.get_response(self.test_origin)
self.assertEquals(resp.status_int, 500)
self.test_origin.app = FakeApp(iter([
('204 No Content', {}, ''), # call to _get_cdn_data
('204 No Content', {}, ''), # put to .hash
('204 No Content', {}, ''), # HEAD check to list container
('404 Not Found', {}, ''), # PUT to list container
]))
req = Request.blank('http://origin_db.com:8080/v1/acc/cont',
environ={'REQUEST_METHOD': 'PUT'})
resp = req.get_response(self.test_origin)
self.assertEquals(resp.status_int, 500)
def test_cdn_get_no_content(self):
prev_data = json.dumps({'account': 'acc', 'container': 'cont',
'ttl': 1234, 'logs_enabled': True, 'cdn_enabled': True})
self.test_origin.app = FakeApp(iter([
('204 No Content', {}, prev_data), # call to _get_cdn_data
('304 No Content', {}, '')])) #call to get obj
req = Request.blank('http://1234.r34.origin_cdn.com:8080/obj1.jpg',
environ={'REQUEST_METHOD': 'HEAD',
'swift.cdn_hash': 'abcd',
'swift.cdn_object_name': 'obj1.jpg'})
resp = req.get_response(self.test_origin)
self.assertEquals(resp.status_int, 304)
self.test_origin.app = FakeApp(iter([
('204 No Content', {}, prev_data), # call to _get_cdn_data
('404 No Content', {}, '')])) #call to get obj
req = Request.blank('http://1234.r34.origin_cdn.com:8080/obj1.jpg',
environ={'REQUEST_METHOD': 'HEAD',
'swift.cdn_hash': 'abcd',
'swift.cdn_object_name': 'obj1.jpg'})
resp = req.get_response(self.test_origin)
self.assertEquals(resp.status_int, 404)
self.test_origin.app = FakeApp(iter([
('204 No Content', {}, prev_data), # call to _get_cdn_data
('416 No Content', {}, '')])) #call to get obj
req = Request.blank('http://1234.r34.origin_cdn.com:8080/obj1.jpg',
environ={'REQUEST_METHOD': 'HEAD',
'swift.cdn_hash': 'abcd',
'swift.cdn_object_name': 'obj1.jpg'})
resp = req.get_response(self.test_origin)
self.assertEquals(resp.status_int, 416)
def test_admin_setup(self):
# PUTs for account and 16 .hash's
self.test_origin.app = FakeApp(iter(
[('204 No Content', {}, '') for i in xrange(102)]))
resp = Request.blank('/origin/.prep',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Origin-Admin-User': '******',
'X-Origin-Admin-Key': 'unittest'}).get_response(
self.test_origin)
self.assertEquals(resp.status_int, 204)
self.assertEquals(self.test_origin.app.calls, 101)
self.test_origin.app = FakeApp(iter(
[('404 Not Found', {}, '')]))
req = Request.blank('/origin/.prep',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Origin-Admin-User': '******',
'X-Origin-Admin-Key': 'unittest'})
self.assertRaises(Exception, req.get_response, self.test_origin)
self.test_origin.app = FakeApp(iter(
[('204 No Content', {}, ''), ('404 Not Found', {}, '')]))
req = Request.blank('/origin/.prep',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Origin-Admin-User': '******',
'X-Origin-Admin-Key': 'unittest'})
self.assertRaises(Exception, req.get_response, self.test_origin)
def test_origin_db_valid_setup(self):
fake_conf = FakeConf(data=['[sos]',
'origin_cdn_host_suffixes = origin_cdn.com'])
test_origin = origin.filter_factory(
{'sos_conf': fake_conf})(FakeApp())
resp = Request.blank('http://origin_db.com:8080/v1/acc/cont',
environ={'REQUEST_METHOD': 'POST'}).get_response(test_origin)
self.assertEquals(resp.status_int, 404)
fake_conf = FakeConf(data='''[sos]
origin_admin_key = unittest
origin_cdn_host_suffixes = origin_cdn.com
origin_db_hosts = origin_db.com
origin_account = .origin
max_cdn_file_size = 0
'''.split('\n'))
test_origin = origin.filter_factory(
{'sos_conf': fake_conf})(FakeApp())
resp = Request.blank('http://origin_db.com:8080/v1/acc/cont',
environ={'REQUEST_METHOD': 'POST'}).get_response(test_origin)
self.assertEquals(resp.status_int, 500)
fake_conf = FakeConf(data='''[sos]
origin_admin_key = unittest
origin_db_hosts = origin_db.com
origin_account = .origin
max_cdn_file_size = 0
'''.split('\n'))
factory = origin.filter_factory(
{'sos_conf': fake_conf})
self.assertRaises(origin.InvalidConfiguration, factory, FakeApp())
def dispatch_wsgi(self, environ, start_response):
"""
Dispatches a WSGI request.
:param environ: WSGI request environment.
:param start_response: WSGI response callback
:returns: A valid WSGI response content.
"""
req = Request(environ)
req.response = Response()
try:
resp = self.dispatch_request(req)
except HTTPException as e:
resp = e
if inspect.isgenerator(resp):
resp = Response(app_iter=resp)
elif resp is None:
resp = req.response
if isinstance(resp, text_type):
resp = bytes_(resp, req.charset)
if isinstance(resp, bytes):
body = resp
resp = req.response
resp.write(body)
if resp is not req.response:
resp = req.response.merge_cookies(resp)
return resp(environ, start_response)
def test_bucket_GET_max_keys(self):
class FakeApp(object):
def __call__(self, env, start_response):
self.query_string = env['QUERY_STRING']
start_response('200 OK', [])
return '[]'
fake_app = FakeApp()
local_app = swift3.filter_factory({})(fake_app)
bucket_name = 'junk'
req = Request.blank('/%s' % bucket_name,
environ={'REQUEST_METHOD': 'GET',
'QUERY_STRING': 'max-keys=5'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, lambda *args: None)
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.getElementsByTagName('MaxKeys')[0].
childNodes[0].nodeValue, '5')
args = dict(cgi.parse_qsl(fake_app.query_string))
self.assert_(args['limit'] == '6')
req = Request.blank('/%s' % bucket_name,
environ={'REQUEST_METHOD': 'GET',
'QUERY_STRING': 'max-keys=5000'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, lambda *args: None)
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.getElementsByTagName('MaxKeys')[0].
childNodes[0].nodeValue, '1000')
args = dict(cgi.parse_qsl(fake_app.query_string))
self.assertEquals(args['limit'], '1001')
def test_translation_fails(self):
# No language
request = Request.blank('')
response = self.item_descriptor.studio_transcript(request=request, dispatch='translation')
self.assertEqual(response.status, '400 Bad Request')
# No filename in request.GET
request = Request.blank('')
response = self.item_descriptor.studio_transcript(request=request, dispatch='translation/uk')
self.assertEqual(response.status, '400 Bad Request')
# Correct case:
filename = os.path.split(self.non_en_file.name)[1]
_upload_file(self.non_en_file, self.item_descriptor.location, filename)
self.non_en_file.seek(0)
request = Request.blank(u'translation/uk?filename={}'.format(filename))
response = self.item_descriptor.studio_transcript(request=request, dispatch='translation/uk')
self.assertEqual(response.body, self.non_en_file.read())
self.assertEqual(response.headers['Content-Type'], 'application/x-subrip; charset=utf-8')
self.assertEqual(
response.headers['Content-Disposition'],
'attachment; filename="{}"'.format(filename)
)
self.assertEqual(response.headers['Content-Language'], 'uk')
# Non ascii file name download:
self.non_en_file.seek(0)
_upload_file(self.non_en_file, self.item_descriptor.location, u'塞.srt')
self.non_en_file.seek(0)
request = Request.blank('translation/zh?filename={}'.format(u'塞.srt'.encode('utf8')))
response = self.item_descriptor.studio_transcript(request=request, dispatch='translation/zh')
self.assertEqual(response.body, self.non_en_file.read())
self.assertEqual(response.headers['Content-Type'], 'application/x-subrip; charset=utf-8')
self.assertEqual(response.headers['Content-Disposition'], 'attachment; filename="塞.srt"')
self.assertEqual(response.headers['Content-Language'], 'zh')
def DELETE(self, env, start_response):
"""
Handle DELETE both container and data object removal.
"""
path = "/v1/" + self.account_name + "/" + self.container_name
query_string = "delimiter=/"
if self.object_name:
query_string += "&prefix=" + concat_parts(self.parent_name, self.object_name) + "/"
exists, dummy, body = check_resource(env, "GET", path, self.logger, True, query_string)
# Not even the top container exist, so there is no such resource.
if not exists:
return get_err_response("NoSuchKey")
# Top container exists, check if there is anything under.
else:
try:
children = json.loads(body)
# there are some children under
if len(children) > 0:
return get_err_response("ContainerNotEmpty")
except ValueError:
return get_err_response("InconsistantState")
# Create a new WebOb Request object according to the current request
req = Request(env)
# Now send the request over.
return req.get_response(self.app)
def test_translation_static_transcript(self):
"""
Set course static_asset_path and ensure we get redirected to that path
if it isn't found in the contentstore
"""
self.course.static_asset_path = 'dummy/static'
self.course.save()
store = modulestore()
with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, self.course.id):
store.update_item(self.course, self.user.id)
# Test youtube style en
request = Request.blank('/translation/en?videoId=12345')
response = self.item.transcript(request=request, dispatch='translation/en')
self.assertEqual(response.status, '307 Temporary Redirect')
self.assertIn(
('Location', '/static/dummy/static/subs_12345.srt.sjson'),
response.headerlist
)
# Test HTML5 video style
self.item.sub = 'OEoXaMPEzfM'
request = Request.blank('/translation/en')
response = self.item.transcript(request=request, dispatch='translation/en')
self.assertEqual(response.status, '307 Temporary Redirect')
self.assertIn(
('Location', '/static/dummy/static/subs_OEoXaMPEzfM.srt.sjson'),
response.headerlist
)
# Test different language to ensure we are just ignoring it since we can't
# translate with static fallback
request = Request.blank('/translation/uk')
response = self.item.transcript(request=request, dispatch='translation/uk')
self.assertEqual(response.status, '404 Not Found')
def test_xml_transcript(self):
"""
Set data_dir and remove runtime modulestore to simulate an XMLModuelStore course.
Then run the same tests as static_asset_path run.
"""
# Simulate XMLModuleStore xmodule
self.item_descriptor.data_dir = 'dummy/static'
del self.item_descriptor.runtime.modulestore
self.assertFalse(self.course.static_asset_path)
# Test youtube style en
request = Request.blank('/translation/en?videoId=12345')
response = self.item.transcript(request=request, dispatch='translation/en')
self.assertEqual(response.status, '307 Temporary Redirect')
self.assertIn(
('Location', '/static/dummy/static/subs_12345.srt.sjson'),
response.headerlist
)
# Test HTML5 video style
self.item.sub = 'OEoXaMPEzfM'
request = Request.blank('/translation/en')
response = self.item.transcript(request=request, dispatch='translation/en')
self.assertEqual(response.status, '307 Temporary Redirect')
self.assertIn(
('Location', '/static/dummy/static/subs_OEoXaMPEzfM.srt.sjson'),
response.headerlist
)
# Test different language to ensure we are just ignoring it since we can't
# translate with static fallback
request = Request.blank('/translation/uk')
response = self.item.transcript(request=request, dispatch='translation/uk')
self.assertEqual(response.status, '404 Not Found')
def test_translation_fails(self):
# No language
request = Request.blank("")
response = self.item_descriptor.studio_transcript(request=request, dispatch="translation")
self.assertEqual(response.status, "400 Bad Request")
# No filename in request.GET
request = Request.blank("")
response = self.item_descriptor.studio_transcript(request=request, dispatch="translation/uk")
self.assertEqual(response.status, "400 Bad Request")
# Correct case:
filename = os.path.split(self.non_en_file.name)[1]
_upload_file(self.non_en_file, self.item_descriptor.location, filename)
self.non_en_file.seek(0)
request = Request.blank(u"translation/uk?filename={}".format(filename))
response = self.item_descriptor.studio_transcript(request=request, dispatch="translation/uk")
self.assertEqual(response.body, self.non_en_file.read())
self.assertEqual(response.headers["Content-Type"], "application/x-subrip; charset=utf-8")
self.assertEqual(response.headers["Content-Disposition"], 'attachment; filename="{}"'.format(filename))
self.assertEqual(response.headers["Content-Language"], "uk")
# Non ascii file name download:
self.non_en_file.seek(0)
_upload_file(self.non_en_file, self.item_descriptor.location, u"塞.srt")
self.non_en_file.seek(0)
request = Request.blank("translation/zh?filename={}".format(u"塞.srt".encode("utf8")))
response = self.item_descriptor.studio_transcript(request=request, dispatch="translation/zh")
self.assertEqual(response.body, self.non_en_file.read())
self.assertEqual(response.headers["Content-Type"], "application/x-subrip; charset=utf-8")
self.assertEqual(response.headers["Content-Disposition"], 'attachment; filename="塞.srt"')
self.assertEqual(response.headers["Content-Language"], "zh")
def test_GET_json_last_modified(self):
# make a container
req = Request.blank('/sda1/p/a/jsonc', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
for i, d in [(0, 1.5),
(1, 1.0),]:
req = Request.blank('/sda1/p/a/jsonc/%s'%i, environ=
{'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': d,
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
'HTTP_X_SIZE': 0})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 201)
# test format
# last_modified format must be uniform, even when there are not msecs
json_body = [{"name":"0",
"hash":"x",
"bytes":0,
"content_type":"text/plain",
"last_modified":"1970-01-01T00:00:01.500000"},
{"name":"1",
"hash":"x",
"bytes":0,
"content_type":"text/plain",
"last_modified":"1970-01-01T00:00:01.000000"},]
req = Request.blank('/sda1/p/a/jsonc?format=json',
environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(eval(resp.body), json_body)
self.assertEquals(resp.charset, 'utf-8')
def _do_proxy_call(self, environ, orig_response, url, prefix):
# TODO: wrap it all into try/except and display main app page with
# traceback in it
log.debug('SSI proxy call to "%s"'%url)
proxy = WSGIProxyApp(url)
o = urlparse(url)
middleware = WSGIProxyMiddleware(proxy,
scheme=o.scheme, domain=o.hostname, port=(o.port or '80'))
# after parse includes process reads input restore file pointer so proxy
# can still read all post data
# environ['wsgi.input'].seek(0)
reqenv = environ.copy()
if reqenv['PATH_INFO'].startswith(prefix):
reqenv['PATH_INFO'] = reqenv['PATH_INFO'][len(prefix):]
reqenv['RAW_URI'] = reqenv['RAW_URI'][len(prefix):]
proxy_req = Request(reqenv)
# tweak proxy request headers a bit
self._copy_user_headers(orig_response, proxy_req)
self._purge_cache_headers(proxy_req)
proxy_resp = proxy_req.get_response(middleware)
# ignore redirects
# TODO: redirect only when location is within proxy_url
proxy_resp.location = None
return proxy_resp
def test_translation_static_transcript(self):
"""
Set course static_asset_path and ensure we get redirected to that path
if it isn't found in the contentstore
"""
self.course.static_asset_path = "dummy/static"
self.course.save()
store = editable_modulestore()
store.update_item(self.course, "OEoXaMPEzfM")
# Test youtube style en
request = Request.blank("/translation/en?videoId=12345")
response = self.item.transcript(request=request, dispatch="translation/en")
self.assertEqual(response.status, "307 Temporary Redirect")
self.assertIn(("Location", "/static/dummy/static/subs_12345.srt.sjson"), response.headerlist)
# Test HTML5 video style
self.item.sub = "OEoXaMPEzfM"
request = Request.blank("/translation/en")
response = self.item.transcript(request=request, dispatch="translation/en")
self.assertEqual(response.status, "307 Temporary Redirect")
self.assertIn(("Location", "/static/dummy/static/subs_OEoXaMPEzfM.srt.sjson"), response.headerlist)
# Test different language to ensure we are just ignoring it since we can't
# translate with static fallback
request = Request.blank("/translation/uk")
response = self.item.transcript(request=request, dispatch="translation/uk")
self.assertEqual(response.status, "404 Not Found")
def test_container4unknown_memcache(self):
fake_memcache = FakeMemcache()
self.assertEquals(fake_memcache.store, {})
resp = Request.blank("/v1/a/c4", environ={"swift.cache": fake_memcache}).get_response(self.test_staticweb)
self.assertEquals(resp.status_int, 301)
self.assertEquals(fake_memcache.store, {"/staticweb/v1/a/c4": ("index.html", "error.html", "t", "listing.css")})
self.assert_(self.test_staticweb.app.get_c4_called)
self.test_staticweb.app.get_c4_called = False
resp = Request.blank("/v1/a/c4", environ={"swift.cache": fake_memcache}).get_response(self.test_staticweb)
self.assertEquals(resp.status_int, 301)
self.assert_(not self.test_staticweb.app.get_c4_called)
self.assertEquals(fake_memcache.store, {"/staticweb/v1/a/c4": ("index.html", "error.html", "t", "listing.css")})
resp = Request.blank("/v1/a/c4", environ={"swift.cache": fake_memcache, "REQUEST_METHOD": "PUT"}).get_response(
self.test_staticweb
)
self.assertEquals(resp.status_int, 200)
self.assertEquals(fake_memcache.store, {})
resp = Request.blank("/v1/a/c4", environ={"swift.cache": fake_memcache}).get_response(self.test_staticweb)
self.assertEquals(resp.status_int, 301)
self.assertEquals(fake_memcache.store, {"/staticweb/v1/a/c4": ("index.html", "error.html", "t", "listing.css")})
resp = Request.blank("/v1/a/c4", environ={"swift.cache": fake_memcache, "REQUEST_METHOD": "POST"}).get_response(
self.test_staticweb
)
self.assertEquals(resp.status_int, 200)
self.assertEquals(fake_memcache.store, {})
def __call__(self,environ,start_response):
req = Request(environ)
session = environ.get('pybald.session', None)
if session and session.user and session.user.can_login:
environ['REMOTE_USER'] = session.user
else:
environ['REMOTE_USER'] = None
if environ['REMOTE_USER']:
# Continuously validate user sessions
# TODO: Clean up our app-level login code a lot...
# Look for a "valid" method on User
try:
valid = getattr(environ['REMOTE_USER'], "valid")
except AttributeError:
# (If this method isn't defined, do nothing.)
pass
else:
# If available, call it, and expect a Boolean:
# If False, end the session right now.
if not valid():
environ['REMOTE_USER'] = None
# Otherwise, do nothing
# update or create the pybald.extension to populate controller instances
environ['pybald.extension'] = environ.get('pybald.extension', {})
environ['pybald.extension']['user'] = environ['REMOTE_USER']
# call the next part of the pipeline
resp = req.get_response(self.application)
return resp(environ,start_response)
def test_domain_remap_account(self):
req = Request.blank("/", environ={"REQUEST_METHOD": "GET"}, headers={"Host": "AUTH_a.example.com"})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, "/v1/AUTH_a")
req = Request.blank("/", environ={"REQUEST_METHOD": "GET"}, headers={"Host": "AUTH-uuid.example.com"})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, "/v1/AUTH_uuid")
def test_method_inner(self):
r = route(
route('news', 'news'),
route(GET, 'comments', 'comments_get'),
route(POST, 'comments', 'comments_post'))
req = Request.blank('/news', {'REQUEST_METHOD': 'GET'})
tr = r(req)
self.assertEqual(
(tr.args, tr.kwargs, tr.target),
((), {}, 'news'))
req = Request.blank('/news', {'REQUEST_METHOD': 'POST'})
self.assertRaises(
RouteGuarded,
r, req)
req = Request.blank('/comments', {'REQUEST_METHOD': 'POST'})
tr = r(req)
self.assertEqual(
(tr.args, tr.kwargs, tr.target),
((), {}, 'comments_post'))
req = Request.blank('/comments', {'REQUEST_METHOD': 'DELETE'})
self.assertRaises(
RouteGuarded,
r, req)
self.assertNoMatch(r, '/newsweeek')
self.assertNoMatch(r, '/ne')
def __call__(self, environ, start_response):
req = Request(environ)
if self._no_redirect_re.match(environ['PATH_INFO']):
return req.get_response(self.app)(environ, start_response)
resp = None
try:
request_uri = req.url
request_uri.decode('ascii')
except UnicodeError:
resp = exc.HTTPNotFound()
secure = req.url.startswith('https://')
srv_path = req.url.split('://', 1)[-1]
# This SFUSER check is SourceForge-specific (to require all logged-in users to use https)
# BUT has the additional affect of not forcing SSL for regular Allura instances
# This is important for local development, at least. When we remove SFUSER (perhaps by requiring SSL everywhere),
# we can use `no_redirect.pattern = .` for local development to work without SSL
force_ssl = req.cookies.get('SFUSER') or self._force_ssl_re.match(environ['PATH_INFO'])
if not secure and force_ssl:
resp = exc.HTTPFound(location='https://' + srv_path)
elif secure and not force_ssl:
resp = exc.HTTPFound(location='http://' + srv_path)
if not resp:
resp = self.app
return resp(environ, start_response)
def test_domain_remap_passthrough(self):
req = Request.blank("/", environ={"REQUEST_METHOD": "GET"}, headers={"Host": "example.com"})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, "/")
req = Request.blank("/", environ={"REQUEST_METHOD": "GET"}, headers={"Host": "example.com:8080"})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, "/")
def __call__(self, env, start_response):
req = Request(env)
if 'AWSAccessKeyId' in req.GET:
try:
req.headers['Date'] = req.GET['Expires']
req.headers['Authorization'] = \
'AWS %(AWSAccessKeyId)s:%(Signature)s' % req.GET
except KeyError:
return get_err_response('InvalidArgument')(env, start_response)
if not 'Authorization' in req.headers:
return self.app(env, start_response)
try:
account, signature = \
req.headers['Authorization'].split(' ')[-1].rsplit(':', 1)
except Exception:
return get_err_response('InvalidArgument')(env, start_response)
try:
controller, path_parts = self.get_controller(req.path)
except ValueError:
return get_err_response('InvalidURI')(env, start_response)
token = base64.urlsafe_b64encode(canonical_string(req))
controller = controller(env, self.app, account, token, **path_parts)
if hasattr(controller, req.method):
res = getattr(controller, req.method)(env, start_response)
else:
return get_err_response('InvalidURI')(env, start_response)
return res(env, start_response)
def test_method_inner(self):
r = route(
route("news", "news"),
route(GET, "comments", "comments_get"),
route(POST, "comments", "comments_post"))
req = Request.blank("/news", {"REQUEST_METHOD": "GET"})
tr = r(req)
self.assertEqual(
(tr.args, tr.kwargs, tr.target),
((), {}, "news"))
req = Request.blank("/news", {"REQUEST_METHOD": "POST"})
self.assertRaises(
RouteGuarded,
r, req)
req = Request.blank("/comments", {"REQUEST_METHOD": "POST"})
tr = r(req)
self.assertEqual(
(tr.args, tr.kwargs, tr.target),
((), {}, "comments_post"))
req = Request.blank("/comments", {"REQUEST_METHOD": "DELETE"})
self.assertRaises(
RouteGuarded,
r, req)
self.assertNoMatch(r, "/newsweeek")
self.assertNoMatch(r, "/ne")
def __call__(self, environ, start_response):
req = Request(environ)
req.remove_conditional_headers()
resp = req.get_response(self.app)
if resp.content_type == 'text/html':
resp.body = jumble_words(resp.body)
return resp(environ, start_response)
def test_limiter_offset_bad(self):
""" Test offset key works with a BAD offset. """
req = Request.blank(u'/?offset=\u0020aa')
self.assertRaises(webob.exc.HTTPBadRequest, common.limited, self.tiny,
req)
def test_domain_remap_account_with_path_root(self):
req = Request.blank('/v1',
environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'AUTH_a.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/v1/AUTH_a')
def test_domain_remap_extra_subdomains(self):
req = Request.blank('/',
environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'x.y.c.AUTH_a.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, ['Bad domain in host header'])
def test_domain_remap_account_matching_ending_not_domain(self):
req = Request.blank('/dontchange',
environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'c.aexample.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/dontchange')
def test_index(self):
request = Request.blank('/api/regex/')
response = request.get_response(app)
self.assertEqual(404, response.status_int)
def build_request(url, query_dict=None, *args, **kwargs):
if query_dict:
url = "%s?%s" % (url, urlencode(query_dict))
r = Request.blank(url, *args, **kwargs)
r.headers['Accept'] = '*/*'
return r
def get(self, path): req = Request.blank(path, cookies=self.cookies) resp = req.get_response(self.app) assert resp.status_int == 200 self._update_cookies(resp) return resp
def test_valid_limit(self):
""" Test valid limit param. """
req = Request.blank('/?limit=10')
self.assertEqual(common.get_pagination_params(req), {'limit': 10})
def test_valid_marker(self):
""" Test valid marker param. """
req = Request.blank('/?marker=1')
self.assertEqual(common.get_pagination_params(req), {'marker': 1})
def test_invalid_limit(self):
""" Test invalid limit param. """
req = Request.blank('/?limit=-2')
self.assertRaises(webob.exc.HTTPBadRequest,
common.get_pagination_params, req)
def test_limiter_negative_offset(self):
""" Test a negative offset. """
req = Request.blank('/?offset=-30')
self.assertRaises(webob.exc.HTTPBadRequest, common.limited, self.tiny,
req)
def test_invalid_marker(self):
""" Test invalid marker param. """
req = Request.blank('/?marker=-2')
self.assertRaises(webob.exc.HTTPBadRequest,
common.get_pagination_params, req)
def __init__(self, env):
self.request = Request(env)
self.response = Response()
self.session = Session(self.request, self.response, \
self.config.session_timeout, self.config.secret)
def test_no_params(self):
""" Test no params. """
req = Request.blank('/')
self.assertEqual(common.get_pagination_params(req), {})
def test_catcherrors(self):
app = catch_errors.CatchErrorMiddleware(FakeApp(True), {})
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'})
resp = app(req.environ, start_response)
self.assertEquals(resp, ['An error occurred'])
def test_simulate_without_body():
req = Request.blank('/api/1/simulate', headers = (('Content-Type', 'application/json'),), method = 'POST')
res = req.get_response(common.app)
assert_equal(res.status_code, 400)
def worker(self):
from allura import model as M
name = '%s pid %s' % (os.uname()[1], os.getpid())
wsgi_app = loadapp('config:%s#task' % self.args[0],
relative_to=os.getcwd())
poll_interval = asint(tg.config.get('monq.poll_interval', 10))
only = self.options.only
if only:
only = only.split(',')
def start_response(status, headers, exc_info=None):
if status != '200 OK':
log.warn(
'Unexpected http response from taskd request: %s. Headers: %s',
status, headers)
def waitfunc_noq():
time.sleep(poll_interval)
def check_running(func):
def waitfunc_checks_running():
if self.keep_running:
return func()
else:
raise StopIteration
return waitfunc_checks_running
waitfunc = waitfunc_noq
waitfunc = check_running(waitfunc)
while self.keep_running:
try:
while self.keep_running:
self.task = M.MonQTask.get(process=name,
waitfunc=waitfunc,
only=only)
if self.task:
with (proctitle("taskd:{0}:{1}".format(
self.task.task_name, self.task._id))):
# Build the (fake) request
request_path = '/--%s--/%s/' % (
self.task.task_name, self.task._id)
r = Request.blank(
request_path,
base_url=tg.config['base_url'].rstrip('/') +
request_path,
environ={
'task': self.task,
'nocapture': self.options.nocapture,
})
list(wsgi_app(r.environ, start_response))
self.task = None
except Exception as e:
if self.keep_running:
base.log.exception(
'taskd error %s; pausing for 10s before taking more tasks'
% e)
time.sleep(10)
else:
base.log.exception('taskd error %s' % e)
base.log.info('taskd pid %s stopping gracefully.' % os.getpid())
if self.restart_when_done:
base.log.info('taskd pid %s restarting itself' % os.getpid())
os.execv(sys.argv[0], sys.argv)
class View(object):
__metaclass__ = ViewBuilder
_path_ = ''
config = ViewConfig(os.getcwd(), 'thebestwaytoservewhiskey', 3600)
def __init__(self, env):
self.request = Request(env)
self.response = Response()
self.session = Session(self.request, self.response, \
self.config.session_timeout, self.config.secret)
@classmethod
def setup(cls, file_name=None, secret=None, session_timeout=None):
if file_name is None:
working_directory = cls.config.working_directory
else:
working_directory = os.path.dirname(os.path.realpath(file_name))
if secret is None:
secret = cls.config.secret
if session_timeout is None:
session_timeout = cls.config.session_timeout
cls.config = ViewConfig(working_directory, secret, session_timeout)
def template_dir(self):
return os.path.join(self.config.working_directory, 'template')
def static_dir(self):
return os.path.join(self.config.working_directory, 'static')
def static_file(self, fname):
file_app = static.FileApp(os.path.join(self.static_dir(), fname))
self.response = self.request.get_response(file_app)
raise Interrupt()
def error(self, code, message='', interrupt=True):
self.response.status = code
self.response.body = message
if interrupt: raise Interrupt()
def url(self, view, *args, **kwargs):
if type(view) == str:
if view in Route.path:
path = os.path.join(Route.path[view], *[str(a) for a in args])
view = Route.view[view]
elif view.startswith('http://') or view.startswith('https://'):
return view
else:
self.error(500, 'Unknown view: ' + view)
elif type(view) == type(View):
path = os.path.join(Route.path[view.__name__],
*[str(a) for a in args])
else:
self.error(500, 'redirect: url must be string or View instance')
qs = urlencode(kwargs)
if view.client_side_target is None:
frag = kwargs.pop('fragment', '')
else:
frag = 'rendered_by_client'
return urlunsplit(
(self.request.scheme, self.request.host, path, qs, frag))
def redirect(self, view, *args, **kwargs):
self.response.location = self.url(view, *args, **kwargs)
self.response.status = 303
raise Interrupt()
def render(self, **kw):
if self._templater_ is None:
template = Route.get_template_name(self.__class__.__name__)
try:
p = os.path.join(self.config.working_directory, 'template')
if self.client_side_target is None:
l = Environment(loader=jj2_loader(p))
self._templater_ = l.get_template(template)
else:
self._templater_ = open(os.path.join(p, template)).read()
except TemplateNotFound:
self.error(500, 'template not found: ' + template)
except:
self.error(500, 'Error processing ' + template)
self.response.charset = 'utf8'
if self.client_side_target is not None:
self.response.content_type = 'application/json'
t = dict(template=self._templater_,
target=self.client_side_target,
data=kw)
return json.dumps(t, default=self.serializer)
else:
self.response.content_type = 'text/html'
kw.update(url=self.url)
return self._templater_.render(**kw)
def __call__(self, *args, **kwargs):
method = getattr(self, self.request.method.lower())
o = method(*args, **kwargs)
if type(o) not in (str, unicode):
self.error(500, 'View must return str or unicode')
self.response.text = o if type(o) is unicode else unicode(o)
def get(self, *args, **kwargs):
self.error(404, 'Unknown handler')
def post(self, *args, **kwargs):
self.error(404, 'Unknown handler')
def put(self, *args, **kwargs):
self.error(404, 'Unknown handler')
def delete(self, *args, **kwargs):
self.error(404, 'Unknown handler')
def head(self, *args, **kwargs):
self.error(404, 'Unknown handler')
def serializer(self, obj):
if hasattr(obj, 'isoformat'):
return obj.isoformat()
return obj
def __call__(self, environ, start_response):
req = Request(environ)
try:
resp = self.process(req)
except ValueError, e:
resp = exc.HTTPBadRequest(str(e))
def test_catcherrors_passthrough(self):
app = catch_errors.CatchErrorMiddleware(FakeApp(), {})
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'})
resp = app(req.environ, start_response)
self.assertEquals(resp, ['FAKE APP'])
def wsgi_app(self, environ, start_response):
"""通过 webob 将请求的环境信息转为request对象"""
request = Request(environ)
response = self.handle_request(request)
return response(environ, start_response)
def application(environment, start_response):
from webob import Request, Response
request = Request(environment)
params = request.params
post = request.POST
res = Response()
import pyadmin.conn, pyadmin.login
importlib.reload(pyadmin.conn)
importlib.reload(pyadmin.login)
# Get the session object from the environ
session = environment['beaker.session']
# Set some other session variable
#session['user_id'] = 10
#user_id = 'user_id' in session
if not 'username' in session:
page = pyadmin.login.loginform
elif not 'password' in session:
page = pyadmin.login.loginform
else:
user = session['username']
passwd = session['password']
import psycopg2, pyadmin.module, re
try:
con = psycopg2.connect(pyadmin.conn.conn)
except:
page = "Can not access databases"
cur = con.cursor()
cur.execute(
"select username,account_password,account_level from account where username=%s and account_password=%s ",
(
user,
passwd,
))
ps = cur.fetchall()
if len(ps) == 0:
page = pyadmin.login.login_again
else:
if int(ps[0][2]) == 2:
importlib.reload(pyadmin.module)
from pyadmin.module import head, headlink, menuadmin, menuuser, load, save, menuhead, menufoot
from datetime import datetime, date
year_today = date.today().year
month_today = date.today().month
day_today = date.today().day
if not 'display' in post:
display = 200
else:
display = post['display']
if not 'hidecols' in post:
hidecols = []
else:
hidecols = post.getall('hidecols')
if not 'hidefils' in post:
hidefils = []
else:
hidefils = post.getall('hidefils')
if not 'orderby' in post:
orderby = ['1']
else:
orderby = post.getall('orderby')
if not 'by' in post:
by = 'asc'
else:
by = post['by']
que = ""
if 'table' in post:
if post['table'] != "":
table = post['table']
cur.execute(
"select count(*) from information_schema.tables where table_name='%s' "
% post['table'])
check = cur.fetchone()
if check[0] > 0:
# table exists in database
cur.execute("select count(*) from %s limit 1" %
table)
que = "select *from " + table
countrows = cur.fetchone()
if countrows[
0] == 0: # no records in table, table is empty
cur.execute(
"select column_name, data_type from information_schema.columns where table_name = '"
+ post['table'] + "'")
rws = cur.fetchall()
cols = [
desc[0] for desc in rws
if desc[0] not in hidecols
]
#colHeaders =[co.title().replace("_"," ") for co in cols]
data_types = [
desc[1] for desc in rws
if desc[0] not in hidecols
]
colslist = [
desc[0] for desc in cur.description
]
filcols = [
fils for fils in cols
if fils not in hidefils
]
if 'movecols' in post:
if len(post.getall('movecols')) != 0:
movecols = ",movecols:%s" % str(
post.getall('movecols'))
cols = [
co
for co in post.getall('movecols')
if co not in hidecols
]
que = "select " + ",".join(
cols) + " from " + table
else:
movecols = ""
# records in table, table is not empty
else:
cur.execute("select * from %s limit 1" % table)
que = "select *from " + table
cols = [
desc[0] for desc in cur.description
if desc[0] not in hidecols
]
daty = cur.fetchone()
data_types = [
type(val).__name__
for index, val in enumerate(daty)
]
colslist = [
desc[0] for desc in cur.description
]
filcols = [
fils for fils in cols
if fils not in hidefils
]
if 'movecols' in post:
if len(post.getall('movecols')) != 0:
movecols = ",movecols:%s" % str(
post.getall('movecols'))
cols = [
co
for co in post.getall('movecols')
if co not in hidecols
]
cur.execute("select " +
",".join(cols) +
" from %s limit 1" % table)
que = "select " + ",".join(
cols) + " from " + table
colslist = [
desc[0] for desc in cur.description
]
filcols = [
fils for fils in cols
if fils not in hidefils
]
else:
movecols = ""
else: # table does not exits in database, only for query sql
table = post['table']
cur.execute(
"select count(*) from settings where tablename='"
+ table + "'")
check2 = cur.fetchone()
if check2[0] > 0:
cur.execute(
"select query from settings where tablename='"
+ table + "' limit 1")
check3 = cur.fetchone()
cur.execute("select count(*) from (" +
check3[0] +
") as jotikaandmendaka")
check5 = cur.fetchone()
if check5[0] > 0:
cur.execute(
"select * from (" + check3[0] +
") as jotikaandmendaka limit 1")
que = check3[0]
cols = [
desc[0] for desc in cur.description
if desc[0] not in hidecols
]
daty = cur.fetchone()
data_types = [
type(val).__name__
for index, val in enumerate(daty)
]
colslist = [
desc[0] for desc in cur.description
]
filcols = [
fils for fils in cols
if fils not in hidefils
]
if 'movecols' in post:
if len(post.getall('movecols')) > 0:
movecols = ",movecols:%s" % str(
post.getall('movecols'))
cols = [
co for co in post.getall(
'movecols')
if co not in hidecols
]
cur.execute(
"select " + ",".join(cols) +
" from (" + check3[0] +
") as jotikaandmendaka limit 1"
)
que = check3[0]
colslist = [
desc[0]
for desc in cur.description
]
filcols = [
fils for fils in cols
if fils not in hidefils
]
else: # no row in query
movecols = ""
else:
cols = []
data_types = []
table = ""
colslist = []
filcols = []
movecols = ""
que = ""
else:
cols = []
data_types = []
table = ""
colslist = []
filcols = []
movecols = ""
que = ""
else: # table =""
cols = []
data_types = []
table = ""
colslist = []
filcols = []
movecols = ""
que = ""
else: # no table in post
cols = []
data_types = []
table = ""
colslist = []
filcols = []
movecols = ""
que = ""
colHeaders = [co.title().replace("_", " ") for co in cols]
cur.execute("select tablename,query from settings order by id")
lst = cur.fetchall()
ops = ""
for l in lst:
#ops += "<option value='%s'>"%l[0]
if l[0] == table:
ops += """<option value="%s" selected="selected">%s</option>""" % (
l[0], l[0])
else:
ops += """<option value="%s">%s</option>""" % (l[0],
l[0])
types = {}
for i in range(len(cols)):
types[cols[i]] = data_types[i]
data = []
grofil = ""
for fil in filcols:
if types[fil] == 'int':
grofil += fil.title().replace(
"_", " "
) + """ > <input class="input-mini" name='mor""" + fil + """' value=''/> and """
grofil += fil.title().replace(
"_", " "
) + """ < <input class="input-mini" name='les""" + fil + """' value=''/> || """
elif types[fil] == 'datetime':
grofil += fil.title().replace(
"_", " "
) + """ > <input class="input-mini" name='mor""" + fil + """' value=''/> and """
grofil += fil.title().replace(
"_", " "
) + """ < <input class="input-mini" name='les""" + fil + """' value=''/> || """
else:
grofil += fil.title().replace(
"_", " "
) + """ ilike <input class="input-mini" name='fil""" + fil + """' value=''/>|| """
grofil += fil.title().replace(
"_", " "
) + """ not ilike <input class="input-mini" name='notfil""" + fil + """' value=''/>|| """
grofil += fil.title().replace(
"_", " ") + """ <select name="%snull">
<option value=""></option>
<option value="null">Empty</option>
<option value="not_null">not empty</option>
</select> """ % fil
if 'mor%s' % fil in post:
data.append(""" mor%s:"%s" """ %
(fil, post["mor%s" % fil]))
if 'les%s' % fil in post:
data.append(""" les%s:"%s" """ %
(fil, post["les%s" % fil]))
if 'fil%s' % fil in post:
data.append(""" fil%s:"%s" """ %
(fil, post["fil%s" % fil]))
if '%snull' % fil in post:
data.append(""" %snull:"%s" """ %
(fil, post["%snull" % fil]))
if len(data) > 0:
send_data = "," + ",".join(data)
else:
send_data = ""
hidefilter = ""
hidefilter += """
<div class="btn-group">
<button data-toggle="dropdown" class="btn dropdown-toggle" data-placeholder="Hide filter">
Hide filter <span class="caret"></span>
</button>
<ul id="sortable3" class="connectedSortable dropdown-menu">
"""
for colsname in colslist:
if colsname in hidefils:
hidefilter += """<li class="ui-state-default"><input type="checkbox" id="fil%s" name="hidefils" value="%s"><label for="fil%s" name="hidefils" value="%s" checked>%s</label></li>""" % (
colsname, colsname, colsname, colsname,
colsname.title().replace("_", " "))
else:
hidefilter += """<li class="ui-state-default"><input type="checkbox" id="fil%s" name="hidefils" value="%s"><label for="fil%s" name="hidefils" value="%s" >%s</label></li>""" % (
colsname, colsname, colsname, colsname,
colsname.title().replace("_", " "))
hidefilter += """
<!-- Other items -->
</ul>
</div>"""
columns = []
for colname in cols:
if colname == 'id':
columns.append({'readOnly': 'true'})
elif colname == 'account_password':
columns.append({"type": "password"})
elif colname == 'update_time':
columns.append({'readOnly': 'true'})
elif colname == 'account_level':
columns.append({
'type': 'numeric',
'allowEmpty': 'false'
})
elif colname == 'username':
columns.append({'allowEmpty': 'false'})
elif colname == 'fid':
columns.append({
'type': 'numeric',
'allowEmpty': 'false'
})
else:
columns.append({})
saveurl = """'%s/save_admin_create'""" % save
loadurl = """'%s/load_admin_create'""" % load
page = ""
page += head
page += "<title>Account manager</title>"
page += headlink
page += """
</head>
<body>"""
page += menuhead
if int(ps[0][2]) == 2:
page += menuadmin
else:
page += menuuser
page += menufoot
page += """<br />
<br />"""
page += """<ul class="nav nav-tabs">
<li class="active"><a href="%s/account_manager">%s</a></li>
</ul>""" % (pyadmin.module.control, table)
page += """<h2>Table %s</h2>""" % (table)
page += """Order by: %s. Sort by: %s. Hide columns: %s | Hide filter: %s + %s <br />
<nav class='navbar navbar-default'>
<form method="post" action="">
<div class="btn-group col-sm-2">
<label>Table:
<input list="table" name="table" value="%s" onchange='if(this.value != 0) { this.form.submit(); }'>
<datalist id="table">
%s
</datalist>
</label>
</div>
<select id="example-single-selected">
<option value="">test</option>
<option value="">test</option>
</select>
<div class="btn-group">
<button data-toggle="dropdown" class="btn dropdown-toggle" data-placeholder="Move columns">
Move columns <span class="caret"></span>
</button>
<ul id="sortable6" class="connectedSortable dropdown-menu">""" % (
",".join(orderby), by, ",".join(hidecols),
",".join(hidefils), movecols, table, ops)
for colsname in colslist:
page += """<li class="ui-state-default"><input type="checkbox" id="move%s" name="movecols" value="%s"><label for="move%s" name="movecols" value="%s" >%s</label></li>""" % (
colsname, colsname, colsname, colsname,
colsname.title().replace("_", " "))
page += """
<!-- Other items -->
</ul>
</div>
<div class="btn-group">
<button data-toggle="dropdown" class="btn dropdown-toggle" data-placeholder="Hide column">
Hide columns <span class="caret"></span>
</button>
<ul id="sortable1" class="connectedSortable dropdown-menu">"""
for colsname in colslist:
page += """<li class="ui-state-default"><input type="checkbox" id="%s" name="hidecols" value="%s"><label for="%s" name="hidecols" value="%s" >%s</label></li>""" % (
colsname, colsname, colsname, colsname,
colsname.title().replace("_", " "))
page += """
<!-- Other items -->
</ul>
</div>
<div class="btn-group">
<button data-toggle="dropdown" class="btn dropdown-toggle" data-placeholder="Order by">
Order by <span class="caret"></span>
</button>
<ul id="sortable2" class="connectedSortable dropdown-menu">"""
for colsname in colslist:
page += """<li class="ui-state-default"><input type="checkbox" id="by%s" name="orderby" value="%s"><label for="by%s" name="orderby" value="%s" >%s</label></li>""" % (
colsname, colsname, colsname, colsname,
colsname.title().replace("_", " "))
page += """
<!-- Other items -->
</ul>
</div>
<div class="btn-group">
<button data-toggle="dropdown" class="btn dropdown-toggle" data-placeholder="Sort by">
Sort by <span class="caret"></span>
</button>
<ul class="dropdown-menu">
<li class="ui-state-default"><input type="radio" id="asc" name="by" value="asc"><label for="asc" name="by" value="asc" >asc</label></li>
<li class="ui-state-default"><input type="radio" id="desc" name="by" value="desc"><label for="desc" name="by" value="desc" >desc</label></li>
<!-- Other items -->
</ul>
</div>"""
page += "<div id='filadv' style='display:none'>"
page += hidefilter
page += grofil
page += "</div>"
page += """Show filter <input class="btn-group" type="checkbox" onclick="myFunction()"/>"""
page += """
<div class="btn-group col-sm-1">
<input class="btn-group form-control col-sm-1" type="number" name ="display" value = %s />
</div>
""" % (display)
page += """ <input type="submit" id ="chon" value="Chon" />
</div>
</form>
</nav>
<p>
<button name="load" id="load_dog">Load</button>
<button name="reset">Reset</button>
<label><input id="autosave" type="checkbox" name="autosave" checked="checked" autocomplete="off"> Autosave</label>
</p>
<div>
<span id="exampleConsole" class="console">Click "Load" to load data from server </span> |
<span class="page2">No page selected</span>
</div>
<div id="example1" style="width:100%; height: 500px; overflow: hidden"></div>
<div class="demo2"></div>
<script>"""
for hids in hidefils:
page += """$("#fil%s").click();""" % hids
#for c in cols:
#page +="""$("#move%s").click();"""%c
page += """
$(document).ready(function() {
$('#example-dropDown, #example-single-selected').multiselect({
enableFiltering: true,
includeSelectAllOption: false,
disableIfEmpty: true,
multiple:false,
maxHeight: 400,
dropDown: true
});
});
function myFunction() {
var x = document.getElementById('filadv');
if (x.style.display === 'none') {
x.style.display = 'inline';
} else {
x.style.display = 'none';
}
}
$( function() {
$( "#sortable1, #sortable2,#sortable3,#sortable5 ,#sortable6").sortable({
connectWith: ".connectedSortable"
}).disableSelection();
$( "ul, li" ).disableSelection();
} );
"""
page += """
var colu = %s;""" % cols
page += """
emailValidator = function (value, callback) {
setTimeout(function(){
if (/.+@.+/.test(value)) {
callback(true);
}
else {
callback(false);
}
}, 1000);
};
emptyValidator = function(value, callback) {
setTimeout(function(){
if (isEmpty(value)) { // isEmpty is a function that determines emptiness, you should define it
callback(true);
} else {
callback(fasle);
}
}, 1000);
}
var
$$ = function(id) {
return document.getElementById(id);
},
autosave = $$('autosave'),
$container = $("#example1"),
$console = $("#exampleConsole"),
$parent = $container.parent(),
autosaveNotification,
hot;
hot = new Handsontable($container[0], {
columnSorting: true,
startRows: 8,
startCols: 3,
currentRowClassName: 'currentRow',
currentColClassName: 'currentCol',
autoWrapRow: true,
rowHeaders: true,"""
page += """ colHeaders: %s,
columns: %s,""" % (str(colHeaders), str(columns))
page += """
//colWidths: [0.1,50,200,50,50,50,50],
manualColumnResize: true,
autoColumnSize : true,
//stretchH: 'all',
hiddenColumns: true,
minSpareCols: 0,
minSpareRows: 1,
contextMenu: true,
beforeRemoveRow: function(index, amount) {
var dellist=[];
for(var i=0; i<amount; i++){
dellist.push(hot.getData()[index +i][colu.indexOf("id")]);
}
//alert(dellist);
$.ajax({
url: """ + saveurl + """,
data: {delete:dellist,table:"%s"}, // returns all cells' data
dataType: 'json',
type: 'POST',
success: function(res) {//alert(res);
if (res.result === 'ok') {
$console.text('Data saved');
document.getElementById("load_dog").click();
}
else {
$console.text('Save error');
}
},
error: function () {
$console.text('Save error');
}
});
},
afterChange: function (change, source) {
var data;
if (source === 'loadData' || !$parent.find('input[name=autosave]').is(':checked')) {
return;
}
data = change[0];""" % table
page += """ var update = [],insert=[],rows=[],unique=[];
for (var i=0;i<change.length;i++){
if (hot.getData()[change[i][0]][colu.indexOf("id")] == null){
rows.push(change[i][0]);
}
else{
update.push({"id":hot.getData()[change[i][0]][colu.indexOf("id")],"column":colu[change[i][1]],"value":change[i][3]});
}
}
if (rows.length >0) {
for(var i in rows){
if(unique.indexOf(rows[i]) === -1){
unique.push(rows[i]);
}
}
for (var i in unique){
var son = {};
for (var k in colu){
son[colu[k]] = hot.getData()[unique[i]][k]
}
insert.push(son);
}
}
// transform sorted row to original row
data[0] = hot.sortIndex[data[0]] ? hot.sortIndex[data[0]][0] : data[0];
clearTimeout(autosaveNotification);
$.ajax({
url: """ + saveurl + """,
dataType: 'json',
type: 'POST',
//data: {"changes": change}, // contains changed cells' data
data: {update:update,insert:insert,lenupdate:update.length,leninsert:insert.length,table:"%s",cols:%s},
success: function (res) {
if (res.result === 'ok') {
//alert(res);
//$console.text('Autosaved (' + change.length + ' cell' + (change.length > 1 ? 's' : '') + ')');
document.getElementById("load_dog").click();
autosaveNotification = setTimeout(function () {
$console.text('Changes will be autosaved ');
}, 1000);
}else{$console.html("<font color='red'>Data save error</font>");}
},
error: function (res) {
autosaveNotification = setTimeout(function () {
$console.html("<font color='red'>Data save error:</font>");
}, 1000);
}
});
}
});
$parent.find('button[name=load]').click(function () {
$.ajax({
url: """ % (table, str(cols)) + loadurl + ""","""
page += """ data: JSON.parse(JSON.stringify({types:%s,"display":%s,table:"%s",cols:%s,orderby:%s,by:"%s",filcols:%s%s%s,que:"%s"})),""" % (
str(types), display, table, str(cols), str(orderby), by,
str(filcols), send_data, movecols,
re.sub('\s+', ' ', que.replace('"', '\\"')))
page += """ dataType: 'json',
type: 'POST',
success: function (res) {
var data = [], row;
for (var i = 0, ilen = res.product.length; i < ilen; i++) {
row = [];
for(var m in colu){
row[m] = res.product[i][colu[m]];
}
data[res.product[i].idha - 1] = row;
}
$console.text('Data loaded');
hot.loadData(data);
$(".page2").html("<strong>Page 1/ " + Math.round(res.sum_page)+"</strong>");
$('.demo2').bootpag({
total: res.sum_page,
page: 1,
maxVisible: 10,
//href:'../demo/account_manager.py?page={{number}}',
leaps: false,
firstLastUse: true,
first: '←',
last: '→',
wrapClass: 'pagination',
activeClass: 'active',
disabledClass: 'disabled',
nextClass: 'next',
prevClass: 'prev',
lastClass: 'last',
firstClass: 'first'
}).on('page', function(event, num){
$(".page2").html("<strong>Page " + num + '/' + Math.round(res.sum_page)+"</strong>"/* + res.test */);
$.ajax({
url: """ + loadurl + ""","""
page += """ data: JSON.parse(JSON.stringify({types:%s,"page":num,"display":%s,table:"%s",cols:%s,orderby:%s,by:"%s",filcols:%s%s%s,que:"%s"})),""" % (
str(types), display, table, str(cols), str(orderby), by,
str(filcols), send_data, movecols,
re.sub('\s+', ' ', que.replace('"', '\\"')))
page += """ dataType: 'json',
type: 'POST',
success: function (res) {
var data = [], row;
for (var i = 0, ilen = res.product.length; i < ilen; i++) {
row = [];
for(var m in colu){
row[m] = res.product[i][colu[m]];
}
data[res.product[i].idha - 1] = row;
}
$console.text('Data loaded');
hot.loadData(data);
}
});
});
}
});
}).click(); // execute immediately
$parent.find('button[name=reset]').click(function () {
$.ajax({
url: 'php/reset.php',
success: function () {
$parent.find('button[name=load]').click();
},
error: function () {
$console.text('Data reset failed');
}
});
});
$parent.find('input[name=autosave]').click(function () {
if ($(this).is(':checked')) {
$console.text('Changes will be autosaved');
}
else {
$console.text('Changes will not be autosaved');
}
});
hot.selectCell(3,3);
//hot.updateSettings({columns: [{data:1},{data:2,type:"password"},{data:3},{data:4},{data:5},{data:6}] });
</script>
</body>
</html>
"""
else:
page = pyadmin.login.login_again
con.commit()
cur.close()
con.close()
#request.headers['Cookie']
response = Response(body=page,
content_type="text/html",
charset="utf8",
status="200 OK")
return response(environment, start_response)
def test_translation_fails(self, url, dispatch, status_code):
request = Request.blank(url)
response = self.item.transcript(request=request, dispatch=dispatch)
self.assertEqual(response.status, status_code)
def test_url(url):
req = Request.blank(url)
response = req.get_response(application)
if not response.body.find('<!--end footer-->') > -1:
print response.body
raise Exception("Could not find '<!--end footer-->' in response body.")
def test_download_srt_exist(self, __):
request = Request.blank('/download')
response = self.item.transcript(request=request, dispatch='download')
self.assertEqual(response.body, 'Subs!')
self.assertEqual(response.headers['Content-Type'], 'application/x-subrip; charset=utf-8')
self.assertEqual(response.headers['Content-Language'], 'en')
def application(environ, start_response):
import cgi, os, sys
from datetime import datetime
#import postgresql.driver as pg_driver
from webob import Request, Response
#db = pg_driver.connect(host='localhost',user='******',password='******',database='Google_qc',port=5432)
request = Request(environ)
import psycopg2
import psycopg2.extras
import psycopg2.extensions
conn = psycopg2.connect("dbname=omnivore user=postgres password=12345678")
cur = conn.cursor()
#params = request.params
try: # Windows needs stdio set for binary mode.
import msvcrt
msvcrt.setmode(0, os.O_BINARY) # stdin = 0
msvcrt.setmode(1, os.O_BINARY) # stdout = 1
except ImportError:
pass
def fbuffer(f, chunk_size=100000):
while True:
chunk = f.read(chunk_size)
if not chunk: break
#page = chunk
if environ['REQUEST_METHOD'] == 'POST':
#post = cgi.FieldStorage(
# fp=environ['wsgi.input'],
# environ=environ,
# keep_blank_values=True
#)
post = request.POST
fileitem = post['file']
account = request.headers["account"] #.replace("A","")
time = request.headers["time"]
table_name = time.replace(" ", "_").replace(":", "_")
year = table_name.split("_")[3]
day = table_name.split("_")[2]
month = int(datetime.strptime(time, '%a %b %d %Y %H:%M:%S').month)
#year = int(datetime.strptime(time,'%a %b %d %Y %H:%M:%S').year)
#day = int(datetime.strptime(time,'%a %b %d %Y %H:%M:%S').day)
#hour= int(datetime.strptime(time,'%a %b %d %Y %H:%M:%S').hour)
#minute = int(datetime.strptime(time,'%a %b %d %Y %H:%M:%S').minute)
#second = int(datetime.strptime(time,'%a %b %d %Y %H:%M:%S').second)
if 'file' in post:
filefield = post['file']
if not isinstance(filefield, list):
filefield = [filefield]
for fileitem in filefield:
# #account = request.headers["account"]
# #time = request.headers["time"]
if fileitem.filename:
page = ""
# strip leading path from file name to avoid directory traversal attacks
fn = os.path.basename(fileitem.filename)
open(
'c:/wsgi_app/file_upload/' + fn + account + '%s' %
(table_name), 'wb').write(fileitem.file.read())
cur.execute("""create table if not exists omnivore_%s_%s_%s
(
id serial8 primary key,
no text,agent text, module text,domain text,title text,link text, quest text, answer text, time timestamp, process_time numeric,img_id text,
update_time timestamp default now() )""" % (year, month, day))
# page += """copy qc_project_csv%s%s(no ,agent , module ,domain ,title ,link , quest , answer , time, process_time) from 'c:/wsgi_app/file_upload/%s%s%s_%s_%s_%s_%s_%s' delimiter ';' CSV HEADER escape '\\' quote '"' """%(year,month,fn,account,year,month,day,hour,minute,second)
try:
cur.copy_expert(
"""copy omnivore_%s_%s_%s(no ,agent , module ,domain ,title ,link , quest , answer , time , process_time,img_id ) from 'c:/wsgi_app/file_upload/%s%s%s' delimiter ';' CSV HEADER quote '"' """
% (year, month, day, fn, account, table_name),
sys.stdout)
#cur.copy_expert("""copy omnivore_%s_%s_%s(no ,agent , module ,domain ,title ,link , quest , answer , time , process_time,img_id ) from 'c:/wsgi_app/file_upload/%s%s%s' delimiter ';' CSV HEADER escape '\\' quote '"' """%(year,month,day,fn,account,table_name),sys.stdout)
#db.execute("""delete from qc_project_csv%s%s where agent='%s' and not update_time =(select max(update_time) from qc_project_csv%s%s) """%(year,month,account,year,month))
#db.execute("""update qc_project_csv%s%s set status =null ,price =null,currency=null,condition=null,availability=null where status ='' or price ='' or currency='' or condition ='' or availability=''"""%(year,month))
#cur.execute("select count(*) from qc_project_csv%s%s where process_time='NaN'"%(year,month))
#ps3 = cur.fetchone()
#if int(ps3[0]) > 0:
#3 cur.execute("""update qc_project_csv%s%s set process_time='10' where process_time='NaN'"""%(year,month))
page += ' file was uploaded %s%s%s' % (fn, account,
table_name)
#page += 'The file "' + fn + '" was uploaded and import successfully'
#db.execute("""insert into log_import_csv(filename,agent) values('%s','%s')"""%(fn,username))
#page += "Upload file sucessfull"
# #xoa file vua gui len
#try:
# os.remove('/usr/local/www/apache24/wsgi-scripts/file_upload/' + fn + account + '%s_%s_%s_%s_%s_%s.csv'%(year,month,day,hour,minute,second))
#except OSError:
# pass
except IOError as err:
page += "I/O error: {0}".format(err)
except ValueError:
page += "Could not import data file csv to database"
raise
else:
page = u"""
<html>
<head><title>Upload</title></head>
<body>
<form name="test" method="post" action="upload_file2a" enctype="multipart/form-data">
Import file csv : <input type="file" name="file" multiple/> <br />
<p>--------------</p>
<!--Upload file anh va excel:<br />
<input type="file" name="file2" multiple/><br />
<input type="file" name="file3" multiple/><br />-->
<input type="submit" name="submit" value="Submit" />
</form>
<p>Note: files with the same name with overwrite any existing files.</p>
</body>
</html>
"""
conn.commit()
cur.close()
conn.close()
response = Response(body=page,
content_type="text/html",
charset="utf8",
status="200 OK")
return response(environ, start_response)
def test_available_translation_non_en(self):
_upload_file(self.srt_file, self.item_descriptor.location, os.path.split(self.srt_file.name)[1])
request = Request.blank('/available_translations')
response = self.item.transcript(request=request, dispatch='available_translations')
self.assertEqual(json.loads(response.body), ['uk'])
def test_download_non_en_non_ascii_filename(self, __):
request = Request.blank('/download')
response = self.item.transcript(request=request, dispatch='download')
self.assertEqual(response.body, 'Subs!')
self.assertEqual(response.headers['Content-Type'], 'application/x-subrip; charset=utf-8')
self.assertEqual(response.headers['Content-Disposition'], 'attachment; filename="塞.srt"')
def application(environment, start_response):
from webob import Request, Response
request = Request(environment)
params = request.params
post = request.POST
res = Response()
import agora.login
importlib.reload(agora.login)
# Get the session object from the environ
session = environment['beaker.session']
# Check to see if a value is in the session
#user = '******' in session
if not 'username' in session:
page = agora.login.loginform
response = Response(body = page,
content_type = "text/html",
charset = "utf8",
status = "200 OK")
elif not 'password' in session:
page = agora.login.loginform
response = Response(body = page,
content_type = "text/html",
charset = "utf8",
status = "200 OK")
else:
user = session['username']
passwd = session['password']
import psycopg2,agora.conn
importlib.reload(agora.conn)
try:
con = psycopg2.connect(agora.conn.conn)
except:
page ="Can not access databases"
cur = con.cursor()
cur.execute("select username,account_password,account_level from account where username=%s and account_password=%s ",(user,passwd,))
ps = cur.fetchall()
if len(ps) == 0:
page = agora.login.login_again
response = Response(body = page,
content_type = "text/html",
charset = "utf8",
status = "200 OK")
else:
if ps[0][2] == 2:
if not 'menu' in post:
menu = 'admin_first_menu'
else:
menu = post['menu']
if not 'display' in post:
display = 200
else:
display = int(post['display'])
if not 'page' in post:
page = 1
else:
page = post['page']
start = (int(page)-1)*display
if menu == 'admin_first_menu' or menu =='first_menu':
query_count = """select count(*) from %s"""%menu
query_rows= """select id,menu1,link from %s order by id limit %s offset %s """%(menu,display,start)
elif menu == 'admin_second_menu' or menu == 'second_menu' :
query_count = """select count(*) from %s"""%menu
query_rows= """select id,first_menu_id,menu2,link from %s order by id limit %s offset %s """%(menu,display,start)
cur.execute(query_count)
rows_count = cur.fetchall()
cur.execute(query_rows)
rows= cur.fetchall()
sum_page = (int(rows_count[0][0])/display)+1
row = []
for ro in rows:
row.append(list(ro))
page = '{"product":'
objects_list = []
import json,collections
if menu == 'admin_first_menu' or menu =='first_menu':
for i in range(len(row)):
d = collections.OrderedDict()
d['index'] = i+1 #row[0]
d['id']=row[i][0]
d['menu1']=row[i][1]
d['link'] = row[i][2]
objects_list.append(d)
elif menu == 'admin_second_menu' or menu == 'second_menu' :
for i in range(len(row)):
d = collections.OrderedDict()
d['index'] = i+1 #row[0]
d['id']=row[i][0]
d['first_menu_id']=row[i][1]
d['menu2'] = row[i][2]
d['link'] = row[i][3]
objects_list.append(d)
#print(objects_list)
page += json.dumps(objects_list)
page +=""","sum_page":%s}"""%sum_page
response = Response(body = page,
content_type = "application/json",
charset = "utf8",
status = "200 OK")
else:
page = agora.login.login_again
response = Response(body = page,
content_type = "text/html",
charset = "utf8",
status = "200 OK")
con.commit()
cur.close()
con.close()
return response(environment, start_response)
def test_download_transcript_not_exist(self):
request = Request.blank('/download')
response = self.item.transcript(request=request, dispatch='download')
self.assertEqual(response.status, '404 Not Found')
