def _reset_databases(self):
for key, engine in self.engines.items():
conn_string = self.test_databases[key]
conn_pieces = urlutils.urlparse(conn_string)
engine.dispose()
if conn_string.startswith("sqlite"):
# We can just delete the SQLite database, which is
# the easiest and cleanest solution
db_path = conn_pieces.path.strip("/")
if os.path.exists(db_path):
os.unlink(db_path)
# No need to recreate the SQLite DB. SQLite will
# create it for us if it's not there...
elif conn_string.startswith("mysql"):
# We can execute the MySQL client to destroy and re-create
# the MYSQL database, which is easier and less error-prone
# than using SQLAlchemy to do this via MetaData...trust me.
(user, password, database, host) = utils.get_db_connection_info(conn_pieces)
sql = ("drop database if exists %(db)s; " "create database %(db)s;") % {"db": database}
cmd = ('mysql -u "%(user)s" -p"%(password)s" -h %(host)s ' '-e "%(sql)s"') % {
"user": user,
"password": password,
"host": host,
"sql": sql,
}
self.execute_cmd(cmd)
elif conn_string.startswith("postgresql"):
self._reset_pg(conn_pieces)
def _reset_databases(self):
for key, engine in self.engines.items():
conn_string = self.test_databases[key]
conn_pieces = urlutils.urlparse(conn_string)
engine.dispose()
if conn_string.startswith('sqlite'):
# We can just delete the SQLite database, which is
# the easiest and cleanest solution
db_path = conn_pieces.path.strip('/')
if os.path.exists(db_path):
os.unlink(db_path)
# No need to recreate the SQLite DB. SQLite will
# create it for us if it's not there...
elif conn_string.startswith('mysql'):
# We can execute the MySQL client to destroy and re-create
# the MYSQL database, which is easier and less error-prone
# than using SQLAlchemy to do this via MetaData...trust me.
(user, password, database, host) = \
utils.get_db_connection_info(conn_pieces)
sql = ("drop database if exists %(db)s; "
"create database %(db)s;") % {
'db': database
}
cmd = ("mysql -u \"%(user)s\" -p\"%(password)s\" -h %(host)s "
"-e \"%(sql)s\"") % {
'user': user,
'password': password,
'host': host,
'sql': sql
}
self.execute_cmd(cmd)
elif conn_string.startswith('postgresql'):
self._reset_pg(conn_pieces)
def get(url, max_size, chunk_size=None, allowed_schemes=('http', 'https')):
"""Get the data at the specified URL.
The URL must use the http: or https: schemes.
The file: scheme is also supported if you override
the allowed_schemes argument.
The max_size represents the total max byte of your file.
The chunk_size is by default set at max_size, it represents the size
of your chunk.
Raise an IOError if getting the data fails and if max_size is exceeded.
"""
LOG.info(_('Fetching data from %s') % url)
components = urlutils.urlparse(url)
if components.scheme not in allowed_schemes:
raise IOError(_('Invalid URL scheme %s') % components.scheme)
if chunk_size is None:
chunk_size = max_size
if max_size < 1:
raise IOError("max_size should be greater than 0")
if chunk_size < 1:
raise IOError("chunk_size should be greater than 0")
if components.scheme == 'file':
try:
return urlutils.urlopen(url).read()
except urlutils.URLError as uex:
raise IOError(_('Failed to read file: %s') % str(uex))
try:
resp = requests.get(url, stream=True)
resp.raise_for_status()
# We cannot use resp.text here because it would download the
# entire file, and a large enough file would bring down the
# engine. The 'Content-Length' header could be faked, so it's
# necessary to download the content in chunks to until
# max_size is reached. The chunk_size we use needs
# to balance CPU-intensive string concatenation with accuracy
# (eg. it's possible to fetch 1000 bytes greater than
# max_size with a chunk_size of 1000).
reader = resp.iter_content(chunk_size=chunk_size)
result = ""
for chunk in reader:
result += chunk
if len(result) > max_size:
raise IOError("File exceeds maximum allowed size (%s "
"bytes)" % max_size)
return result
except exceptions.RequestException as ex:
raise IOError(_('Failed to retrieve file: %s') % str(ex))
def table_args():
engine_name = urlutils.urlparse(cfg.CONF.database.connection).scheme
if engine_name == 'mysql':
return {'mysql_engine': 'InnoDB',
'mysql_charset': "utf8"}
return None
