def visit_uri(self, uri, depth):
parsed_uri = _urlparse(uri)
env = {
"REQUEST_METHOD": "GET",
"REQUEST_URI": uri,
"PATH_INFO": parsed_uri.path,
"QUERY_STRING": parsed_uri.query,
}
response_result = [None, None]
def response(_status, _headers):
response_result[0] = _status
response_result[1] = dict(_headers)
content = self.app._receive_request(env, response)
content = "".join(content)
status, headers = response_result
if status.startswith("303"):
return self.visit_uri(headers["Location"], depth + 1)
return status, headers, content
def urlparse(url: str) -> Url:
if on_win and url.startswith('file:'):
url.replace('\\', '/')
# Allows us to pass in strings like 'example.com:8080/path/1'.
if not has_scheme(url):
url = "//" + url
return Url.from_parse_result(_urlparse(url))
def test_sasl(session):
sasl_user = "******"
sasl_password = "******"
additional_server_args = []
additional_server_args.append("--key={}".format(TSERVER_PRIVATE_KEY_PEM))
additional_server_args.append("--key-password={}".format("password"))
additional_server_args.append("--cert={}".format(TSERVER_CERTIFICATE_PEM))
additional_server_args.append("--sasl-user={}".format(sasl_user))
additional_server_args.append("--sasl-password={}".format(sasl_password))
with _TestServer(additional_server_args=additional_server_args,
scheme="amqp") as server:
server_url = _urlparse(server.url)
client_url = "{}://{}:{}@{}{}".format(server_url.scheme, sasl_user,
sasl_password, server_url.netloc,
server_url.path)
for impl in AMQP_ARROW_IMPLS:
if not impl_available(impl):
continue
call("quiver-arrow send {} --impl {} --count 1 --verbose",
client_url, impl)
call("quiver-arrow receive {} --impl {} --count 1 --verbose",
client_url, impl)
def parse_address_url(self, address):
url = _urlparse(address)
if url.path is None:
self.fail("The URL has no path")
scheme = url.scheme
host = url.hostname
port = url.port
path = url.path
default_scheme = "amqps" if self.tls_enabled else "amqp"
try:
default_host, default_port = self.server.split(":", 1)
except ValueError:
default_host, default_port = self.server, 5672
if not scheme:
scheme = default_scheme
if host is None:
host = default_host
if port is None:
port = default_port
port = str(port)
if path.startswith("/"):
path = path[1:]
return scheme, host, port, path
def test_sasl(session):
raise TestSkipped("Disabled: https://github.com/ssorj/quiver/issues/75")
sasl_user = "******"
sasl_password = "******"
extra_server_args = []
extra_server_args.append("--key={}".format(TSERVER_PRIVATE_KEY_PEM))
extra_server_args.append("--key-password={}".format("password"))
extra_server_args.append("--cert={}".format(TSERVER_CERTIFICATE_PEM))
extra_server_args.append("--sasl-user={}".format(sasl_user))
extra_server_args.append("--sasl-password={}".format(sasl_password))
with _TestServer(extra_server_args=extra_server_args,
scheme="amqp") as server:
server_url = _urlparse(server.url)
client_url = "{}://{}:{}@{}{}".format(server_url.scheme, sasl_user,
sasl_password, server_url.netloc,
server_url.path)
for impl in AMQP_ARROW_IMPLS:
if not impl_available(impl):
continue
call("quiver-arrow send {} --impl {} --count 1 --verbose",
client_url, impl)
call("quiver-arrow receive {} --impl {} --count 1 --verbose",
client_url, impl)
def schemizeUrl(url, color=True, quiet=False):
"""
Returns url string after adding scheme if not present, and modofying scheme if not http.
Parameters:
url (str): URL string
color (bool): Shall color be printed while function is running
Returns:
url (str): URL string after prepending or modifying scheme to http
"""
parsedUrl = _urlparse(url)
if not parsedUrl.scheme:
url = 'http' + '://' + url
elif parsedUrl.scheme not in ('http', 'https'):
if not quiet:
print(
bad(
f'Scheme `{parsedUrl.scheme}` does not looks like of `http`',
color))
print(info('Scheme modified to `http`', color))
url = url.split('://')
url[0] = 'http'
url = '://'.join(url)
return url
def delete(urls):
"""
Delete a shared file.
Args:
urls (iterable of str): Shareable URL of files to delete.
"""
# TODO: Fix unmounted storage not found because URL starts with https://
for url in urls:
url = _urlparse(url)
_remove(f"{url.scheme}://{url.netloc}{url.path}")
def _parts_mother_id_typename(cls, url):
"""
>>> murl = "qpfer://mother/qfid#module_and_type_hint"
>>> qfurl._parts_mother_id_typename(murl)
('mother', 'qfid', 'module_and_type_hint')
"""
scheme, mother, qfid, _ign, _ore, typname = _urlparse(url)
if scheme != QFURL_SCHEME:
raise QfurlError("Wrong scheme: %s" % scheme)
qfid = qfid.lstrip("/")
return mother, qfid, typname
def urlparse(d, keys=None):
"""Return a copy of the given dictionary with url values parsed."""
d = d.copy()
if keys is None:
keys = d.keys()
for key in keys:
d[key] = _urlparse(d[key])
return d
def _parts_mother_id_typename(cls, url):
"""
>>> murl = "qpfer://mother/qfid#module_and_type_hint"
>>> qfurl._parts_mother_id_typename(murl)
('mother', 'qfid', 'module_and_type_hint')
"""
scheme, mother, qfid, _ign, _ore, typname = _urlparse(url)
if scheme != QFURL_SCHEME:
raise QfurlError("Wrong scheme: %s" % scheme)
qfid = qfid.lstrip("/")
return mother, qfid, typname
def urlparse(d, keys=None):
"""Returns a copy of the given dictionary with url values parsed."""
d = d.copy()
if keys is None:
keys = d.keys()
for key in keys:
d[key] = _urlparse(d[key])
return d
def _open_seed_url(self, base, branch, name):
path = os.path.join(base, branch)
if not path.endswith('/'):
path += '/'
url = urljoin(path, name)
if not _urlparse(url).scheme:
fullpath = os.path.join(path, name)
_logger.info("Using %s", fullpath)
return open(fullpath)
_logger.info("Downloading %s", url)
req = Request(url)
req.add_header('Cache-Control', 'no-cache')
req.add_header('Pragma', 'no-cache')
return urlopen(req)
def _open_seed_url(self, base, branch, name):
path = os.path.join(base, branch)
if not path.endswith('/'):
path += '/'
url = urljoin(path, name)
if not _urlparse(url).scheme:
fullpath = os.path.join(path, name)
_logger.info("Using %s", fullpath)
return open(fullpath)
_logger.info("Downloading %s", url)
req = Request(url)
req.add_header('Cache-Control', 'no-cache')
req.add_header('Pragma', 'no-cache')
return urlopen(req)
def _get_domain(self, url):
"""Return the domain name of the server in the passed url"""
from urllib.parse import urlparse as _urlparse
try:
domain = _urlparse(url).netloc.split(":")[0]
except:
domain = None
if domain is None or len(domain) == 0:
if self.is_test_registry():
return "testing"
else:
raise ValueError("Cannot extract the domain from '%s'"
% url)
return domain
def __init__(
self,
url=None,
timeout=30 * 60,
user_id=None,
password=None,
token=None,
ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://kbase.us/services/authorization/Sessions/Login',
lookup_url=False,
async_job_check_time_ms=100,
async_job_check_time_scale_percent=150,
async_job_check_max_time_ms=300000):
if url is None:
raise ValueError('A url is required')
scheme, _, _, _, _, _ = _urlparse(url)
if scheme not in _URL_SCHEME:
raise ValueError(url + " isn't a valid http url")
self.url = url
self.timeout = int(timeout)
self._headers = dict()
self.trust_all_ssl_certificates = trust_all_ssl_certificates
self.lookup_url = lookup_url
self.async_job_check_time = async_job_check_time_ms / 1000.0
self.async_job_check_time_scale_percent = (
async_job_check_time_scale_percent)
self.async_job_check_max_time = async_job_check_max_time_ms / 1000.0
# token overrides user_id and password
if token is not None:
self._headers['AUTHORIZATION'] = token
elif user_id is not None and password is not None:
self._headers['AUTHORIZATION'] = _get_token(
user_id, password, auth_svc)
elif 'KB_AUTH_TOKEN' in _os.environ:
self._headers['AUTHORIZATION'] = _os.environ.get('KB_AUTH_TOKEN')
elif not ignore_authrc:
authdata = _read_inifile()
if authdata is not None:
if authdata.get('token') is not None:
self._headers['AUTHORIZATION'] = authdata['token']
elif (authdata.get('user_id') is not None
and authdata.get('password') is not None):
self._headers['AUTHORIZATION'] = _get_token(
authdata['user_id'], authdata['password'], auth_svc)
if self.timeout < 1:
raise ValueError('Timeout value must be at least 1 second')
def __init__(
self,
url=None,
timeout=30 * 60,
user_id=None,
password=None,
token=None,
ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc="https://kbase.us/services/auth/api/legacy/KBase/Sessions/Login",
lookup_url=False,
async_job_check_time_ms=100,
async_job_check_time_scale_percent=150,
async_job_check_max_time_ms=300000,
):
if url is None:
raise ValueError("A url is required")
scheme, _, _, _, _, _ = _urlparse(url)
if scheme not in _URL_SCHEME:
raise ValueError(url + " isn't a valid http url")
self.url = url
self.timeout = int(timeout)
self._headers = dict()
self.trust_all_ssl_certificates = trust_all_ssl_certificates
self.lookup_url = lookup_url
self.async_job_check_time = async_job_check_time_ms / 1000.0
self.async_job_check_time_scale_percent = async_job_check_time_scale_percent
self.async_job_check_max_time = async_job_check_max_time_ms / 1000.0
# token overrides user_id and password
if token is not None:
self._headers["AUTHORIZATION"] = token
elif user_id is not None and password is not None:
self._headers["AUTHORIZATION"] = _get_token(
user_id, password, auth_svc)
elif "KB_AUTH_TOKEN" in _os.environ:
self._headers["AUTHORIZATION"] = _os.environ.get("KB_AUTH_TOKEN")
elif not ignore_authrc:
authdata = _read_inifile()
if authdata is not None:
if authdata.get("token") is not None:
self._headers["AUTHORIZATION"] = authdata["token"]
elif (authdata.get("user_id") is not None
and authdata.get("password") is not None):
self._headers["AUTHORIZATION"] = _get_token(
authdata["user_id"], authdata["password"], auth_svc)
if self.timeout < 1:
raise ValueError("Timeout value must be at least 1 second")
def __init__(
self,
url=None,
timeout=30 * 60,
user_id=None,
password=None,
token=None,
ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc="https://kbase.us/services/authorization/Sessions/Login",
lookup_url=False,
async_job_check_time_ms=100,
async_job_check_time_scale_percent=150,
async_job_check_max_time_ms=300000,
):
if url is None:
raise ValueError("A url is required")
scheme, _, _, _, _, _ = _urlparse(url)
if scheme not in _URL_SCHEME:
raise ValueError(url + " isn't a valid http url")
self.url = url
self.timeout = int(timeout)
self._headers = dict()
self.trust_all_ssl_certificates = trust_all_ssl_certificates
self.lookup_url = lookup_url
self.async_job_check_time = async_job_check_time_ms / 1000.0
self.async_job_check_time_scale_percent = async_job_check_time_scale_percent
self.async_job_check_max_time = async_job_check_max_time_ms / 1000.0
# token overrides user_id and password
if token is not None:
self._headers["AUTHORIZATION"] = token
elif user_id is not None and password is not None:
self._headers["AUTHORIZATION"] = _get_token(user_id, password, auth_svc)
elif "KB_AUTH_TOKEN" in _os.environ:
self._headers["AUTHORIZATION"] = _os.environ.get("KB_AUTH_TOKEN")
elif not ignore_authrc:
authdata = _read_inifile()
if authdata is not None:
if authdata.get("token") is not None:
self._headers["AUTHORIZATION"] = authdata["token"]
elif authdata.get("user_id") is not None and authdata.get("password") is not None:
self._headers["AUTHORIZATION"] = _get_token(authdata["user_id"], authdata["password"], auth_svc)
if self.timeout < 1:
raise ValueError("Timeout value must be at least 1 second")
def __init__(
self, url=None, timeout=30 * 60, user_id=None,
password=None, token=None, ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://kbase.us/services/auth/api/legacy/KBase/Sessions/Login',
lookup_url=False,
async_job_check_time_ms=100,
async_job_check_time_scale_percent=150,
async_job_check_max_time_ms=300000):
if url is None:
raise ValueError('A url is required')
scheme, _, _, _, _, _ = _urlparse(url)
if scheme not in _URL_SCHEME:
raise ValueError(url + " isn't a valid http url")
self.url = url
self.timeout = int(timeout)
self._headers = dict()
self.trust_all_ssl_certificates = trust_all_ssl_certificates
self.lookup_url = lookup_url
self.async_job_check_time = async_job_check_time_ms / 1000.0
self.async_job_check_time_scale_percent = (
async_job_check_time_scale_percent)
self.async_job_check_max_time = async_job_check_max_time_ms / 1000.0
# token overrides user_id and password
if token is not None:
self._headers['AUTHORIZATION'] = token
elif user_id is not None and password is not None:
self._headers['AUTHORIZATION'] = _get_token(
user_id, password, auth_svc)
elif 'KB_AUTH_TOKEN' in _os.environ:
self._headers['AUTHORIZATION'] = _os.environ.get('KB_AUTH_TOKEN')
elif not ignore_authrc:
authdata = _read_inifile()
if authdata is not None:
if authdata.get('token') is not None:
self._headers['AUTHORIZATION'] = authdata['token']
elif(authdata.get('user_id') is not None and
authdata.get('password') is not None):
self._headers['AUTHORIZATION'] = _get_token(
authdata['user_id'], authdata['password'], auth_svc)
if self.timeout < 1:
raise ValueError('Timeout value must be at least 1 second')
def get(url, output=".", extract=False):
"""
Get a file or archive from an URL.
Args:
output (str): Output file or directory path.
url (str): Input URL.
extract (bool): If True, extract archive.
"""
response = _request("GET", url, stream=True)
response.raise_for_status()
if extract:
with _tarfile_open(fileobj=response.raw) as archive:
archive.extractall(output)
else:
if not _isfile(output):
output = _join(output, _basename(_urlparse(url).path))
with open(output, "wb") as dest:
for chunk in response.iter_content():
dest.write(chunk)
def test_sasl(session):
sasl_user = "******"
sasl_password = "******"
additional_server_args = []
additional_server_args.append("--key={}".format(TSERVER_PRIVATE_KEY_PEM))
additional_server_args.append("--key-password={}".format("password"))
additional_server_args.append("--cert={}".format(TSERVER_CERTIFICATE_PEM))
additional_server_args.append("--sasl-user={}".format(sasl_user))
additional_server_args.append("--sasl-password={}".format(sasl_password))
with _TestServer(additional_server_args = additional_server_args, scheme="amqp") as server:
server_url = _urlparse(server.url)
client_url = "{}://{}:{}@{}{}".format(server_url.scheme,
sasl_user, sasl_password,
server_url.netloc, server_url.path)
for impl in AMQP_ARROW_IMPLS:
if not impl_available(impl):
continue
call("quiver-arrow send {} --impl {} --count 1 --verbose", client_url, impl)
call("quiver-arrow receive {} --impl {} --count 1 --verbose", client_url, impl)
def init_url_attributes(self):
self.url = self.args.url
url = _urlparse(self.url)
if url.path is None:
raise CommandError("The URL has no path")
self.host = url.hostname
self.port = url.port
self.path = url.path
if self.host is None:
self.host = "localhost"
if self.port is None:
self.port = "-"
self.port = str(self.port)
if self.path.startswith("/"):
self.path = self.path[1:]
def escape_channel_url(channel):
if channel.startswith("file:"):
if "%" in channel: # it's escaped already
return channel
if on_win:
channel = channel.replace("\\", "/")
parts = _urlparse(channel)
if parts.scheme:
components = parts.path.split("/")
if on_win:
if len(parts.netloc) == 2 and parts.netloc[1] == ":":
# with absolute paths (e.g. C:/something), C:, D:, etc might get parsed as netloc
path = "/".join([parts.netloc] +
[quote(p) for p in components])
parts = parts._replace(netloc="")
else:
path = "/".join(components[:2] +
[quote(p) for p in components[2:]])
else:
path = "/".join([quote(p) for p in components])
parts = parts._replace(path=path)
return _urlunparse(parts)
return channel
def _open_seed(self, base, branch, name, bzr=False):
path = os.path.join(base, branch)
if not path.endswith('/'):
path += '/'
if bzr:
global _bzr_cache_dir
if _bzr_cache_dir is None:
_bzr_cache_dir = tempfile.mkdtemp(prefix='germinate-')
atexit.register(
shutil.rmtree, _bzr_cache_dir, ignore_errors=True)
checkout = os.path.join(_bzr_cache_dir, branch)
if not os.path.isdir(checkout):
command = ['bzr']
# https://bugs.launchpad.net/bzr/+bug/39542
if path.startswith('http:'):
command.append('branch')
_logger.info("Fetching branch of %s", path)
else:
command.extend(['checkout', '--lightweight'])
_logger.info("Checking out %s", path)
command.extend([path, checkout])
status = subprocess.call(command)
if status != 0:
raise SeedError("Command failed with exit status %d:\n"
" '%s'" % (status, ' '.join(command)))
return open(os.path.join(checkout, name))
else:
url = urljoin(path, name)
if not _urlparse(url).scheme:
fullpath = os.path.join(path, name)
_logger.info("Using %s", fullpath)
return open(fullpath)
_logger.info("Downloading %s", url)
req = Request(url)
req.add_header('Cache-Control', 'no-cache')
req.add_header('Pragma', 'no-cache')
return urlopen(req)
def _returnDomain(url):
l_domain = _urlparse(url).netloc.split('.')
domain = '.'.join(l_domain[1:])
return domain
def send_password(self, url, username=None, password=None,
otpcode=None, remember_password=True,
remember_device=None, dryrun=None):
"""Send a password and one-time code to the supplied login url"""
if not remember_password:
remember_device = False
# the login URL is http[s]://something.com?id=XXXX/YY.YY.YY.YY
# where XXXX is the service_uid of the service we should
# connect with, and YY.YY.YY.YY is the short_uid of the login
try:
from urllib.parse import urlparse as _urlparse
from urllib.parse import parse_qs as _parse_qs
idcode = _parse_qs(_urlparse(url).query)["id"][0]
except Exception as e:
from Acquire.Client import LoginError
raise LoginError(
"Cannot identify the session or service information from "
"the login URL '%s'. This should have id=XX-XX/YY.YY.YY.YY "
"as a query parameter. <%s> %s" %
(url, e.__class__.__name__, str(e)))
try:
(service_uid, short_uid) = idcode.split("/")
except:
from Acquire.Client import LoginError
raise LoginError(
"Cannot extract the service_uid and short_uid from the "
"login ID code '%s'. This should be in the format "
"XX-XX/YY.YY.YY.YY" % idcode)
# now get the service
try:
service = self.get_service(service_uid=service_uid)
except Exception as e:
from Acquire.Client import LoginError
raise LoginError(
"Cannot find the service with UID %s: <%s> %s" %
(service_uid, e.__class__.__name__, str(e)))
if not service.can_identify_users():
from Acquire.Client import LoginError
raise LoginError(
"Service '%s' is unable to identify users! "
"You cannot log into something that is not "
"a valid identity service!" % (service))
userinfo = self._find_userinfo(username=username,
password=password,
service_uid=service_uid)
if username is None:
username = userinfo["username"]
if "user_uid" in userinfo:
user_uid = userinfo["user_uid"]
else:
user_uid = None
_output("Logging in using username '%s'" % username)
try:
device_uid = userinfo["device_uid"]
except:
device_uid = None
if password is None:
password = self._get_user_password(userinfo=userinfo)
if otpcode is None:
otpcode = self._get_otpcode(userinfo=userinfo)
else:
# user is providing the primary OTP, so this is not a device
device_uid = None
_output("\nLogging in to '%s', session '%s'..." % (
service.canonical_url(), short_uid), end="")
_flush_output()
if dryrun:
print("Calling %s with username=%s, password=%s, otpcode=%s, "
"remember_device=%s, device_uid=%s, short_uid=%s "
"user_uid=%s" %
(service.canonical_url(), username, password, otpcode,
remember_device, device_uid, short_uid, user_uid))
return
try:
from Acquire.Client import Credentials as _Credentials
creds = _Credentials(username=username, password=password,
otpcode=otpcode, short_uid=short_uid,
device_uid=device_uid)
args = {"credentials": creds.to_data(identity_uid=service.uid()),
"user_uid": user_uid,
"remember_device": remember_device,
"short_uid": short_uid}
response = service.call_function(function="login", args=args)
_output("SUCCEEDED!")
_flush_output()
except Exception as e:
_output("FAILED!")
_flush_output()
from Acquire.Client import LoginError
raise LoginError("Failed to log in. %s" % e.args)
if not remember_password:
return
try:
returned_user_uid = response["user_uid"]
if returned_user_uid != user_uid:
# change of user?
userinfo = {}
user_uid = returned_user_uid
except:
# no user_uid, so nothing to save
return
if user_uid is None:
# can't save anything
return
userinfo["username"] = username
userinfo["password"] = password
try:
userinfo["device_uid"] = response["device_uid"]
except:
pass
try:
userinfo["otpsecret"] = response["otpsecret"]
except:
pass
self._set_userinfo(userinfo=userinfo,
user_uid=user_uid,
service_uid=service.uid())
def create(source: str,
name: str = None,
description: str = None,
propose_path: str = None,
**kwargs) -> _model.AbstractFile:
"""Create a file object from a local or remote file
"""
# Create temporary file
tmp_file_d, tmp_file_path = _util.mk_tmp_file()
tmp_file = open(tmp_file_d, 'wb')
# Remote file
try:
_validation.rule.Url(source).validate()
# Copy remote file to the temporary local file
req = _urllib_request(
_util.url_quote(source, safe='/:?&%'),
headers={
'User-Agent':
'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0',
})
with _urlopen(req) as src:
data = src.read()
tmp_file.write(data)
if not name:
name = _urlparse(source).path.split('/')[-1]
if description is None:
description = 'Downloaded from ' + source
# Local file
except _validation.error.RuleError:
with open(source, 'rb') as f:
data = f.read()
tmp_file.write(data)
if not name:
name = _os.path.basename(source)
if description is None:
description = 'Created from local file ' + source
# Close temporary file
tmp_file.close()
# Validate file's size (in megabytes)
file_size = _os.stat(tmp_file_path).st_size
max_file_size_mb = float(_reg.get('file.upload_max_size', '10'))
if file_size > int(max_file_size_mb * 1048576):
raise RuntimeError('File size exceeds {} MB'.format(max_file_size_mb))
# Determining file's MIME type
mime = _magic.from_file(tmp_file_path, True)
# Add an extension to the name
if not _os.path.splitext(name)[1]:
name += _guess_extension(mime)
# Ask driver to store file content into its storage
file_object = get_driver().create(tmp_file_path, mime, name, description,
propose_path, **kwargs)
# Delete temporary file
_os.unlink(tmp_file_path)
return file_object
def urlparse(url, scheme='', allow_fragments=True):
_presult = _urlparse(url, scheme=scheme, allow_fragments=allow_fragments)
_lst = list(_presult)
_lst[4] = dict(parse_qsl(_lst[4]))
return URL(*_lst)
"class": "raven.handlers.logging.SentryHandler",
"dsn": SENTRY_DSN
}
LOGGING["root"]["handlers"].append("sentry_handler")
# Provides the ability to deactivate file logging on the development
# and testing environment by setting LOGS_LOCATION="False"
if LOGS_LOCATION not in (False, "False"):
LOGGING["handlers"]["standard_handler"] = {
"formatter": "standard_formatter",
"level": "INFO",
"class": "logging.handlers.TimedRotatingFileHandler",
"filename": LOGS_LOCATION + "/log.log",
"when": "D",
"interval": 1,
"backupCount": 10,
"encoding": "utf-8",
"delay": False,
"utc": True
}
LOGGING["root"]["handlers"].append("standard_handler")
_dictConfig(LOGGING)
_db_url = _urlparse(TRANSLATIONS_SERVER_DB_URL)
DB_PARAMS = {
"database": _db_url.path[1:],
"user": _db_url.username,
"password": _db_url.password,
"host": _db_url.hostname
}
def init(self):
super(QuiverArrowCommand, self).init()
try:
self.impl = _impls_by_name[self.args.impl]
except KeyError:
self.impl = self.args.impl
m = "Warning: Implementation '{}' is unknown".format(self.args.impl)
eprint(m)
self.operation = self.args.operation
self.id_ = self.args.id
self.connection_mode = "client"
self.channel_mode = "active"
self.prelude = _shlex.split(self.args.prelude)
if self.id_ is None:
self.id_ = "quiver-{}".format(_unique_id())
if self.args.server:
self.connection_mode = "server"
if self.args.passive:
self.channel_mode = "passive"
url = _urlparse(self.address)
if url.path is None:
raise QuiverError("The address URL has no path")
self.host = url.hostname
self.port = url.port
self.path = url.path[1:]
if self.host is None:
self.host = "localhost"
if self.port is None:
self.port = "-"
self.port = str(self.port)
self.impl_file = "{}/exec/arrow-{}".format(self.home_dir, self.impl)
if not _os.path.exists(self.impl_file):
m = "No impl at '{}'".format(self.impl_file)
raise QuiverError(m)
if self.operation == "send":
self.snapshots_file = _join(self.output_dir, "sender-snapshots.csv")
self.summary_file = _join(self.output_dir, "sender-summary.json")
self.transfers_file = _join(self.output_dir, "sender-transfers.csv")
self.transfers_parse_func = _parse_send
elif self.operation == "receive":
self.snapshots_file = _join(self.output_dir, "receiver-snapshots.csv")
self.summary_file = _join(self.output_dir, "receiver-summary.json")
self.transfers_file = _join(self.output_dir, "receiver-transfers.csv")
self.transfers_parse_func = _parse_receive
else:
raise Exception()
self.first_send_time = None
self.last_send_time = None
self.first_receive_time = None
self.last_receive_time = None
self.message_count = None
self.message_rate = None
self.latency_average = None
self.latency_quartiles = None
self.latency_nines = None
def parse_url(self):
self.url = _urlparse(self.path)
self.queries = _parse_qs(self.url.query)
from os.path import abspath
from random import choice
from subprocess import Popen
from urllib.parse import urlparse as _urlparse
import config
from translations_server.lib import postgres_db, db
_migrated_test_db = False
# TEST_DB is not actually in config because it is not needed ever for prod
# but if you set it in config_local it will be imported into config.
# If not and you run the tests this will tell you to set it.
try:
_test_db_url = _urlparse(config.TRANSLATIONS_SERVER_TEST_DB_URL)
TEST_DB_PARAMS = {
"database": _test_db_url.path[1:],
"user": _test_db_url.username,
"password": _test_db_url.password,
"host": _test_db_url.hostname
}
except AttributeError:
raise AttributeError("Please set TEST_DB in config_local")
def _migrate_test_db():
global _migrated_test_db
if not _migrated_test_db:
env = dict(environ)
