def check_postgres_extensions(deps, username, password, host, port, database):
psql_cmd = f'psql '
flags = f'-U {username} '
if password:
psql_cmd = f'PGPASSWORD="******" {psql_cmd}'
flags += f' --no-password'
if host:
flags += f' -h {host}'
if port:
flags += f' -p {port}'
def get_version(v):
lines = v.split('\n')
for i, line in enumerate(lines):
if '1 row' in line.strip():
return lines[i - 1].strip()
fail = []
for dep, min_version in deps:
query = f'{dep} >= {min_version}'
clause = f"SELECT max(extversion) FROM pg_extension WHERE extname = '{dep}';"
cmd = psql_cmd + f' {flags} {database} -c "{clause}"'
try:
with status(query):
success, out = output(cmd, shell=True)
if not success:
raise ValueError(out.decode("utf-8").strip())
try:
version = get_version(out.decode('utf-8').strip())
print(f'[{version.rjust(8)}]'.rjust(40 - len(query)),
end='')
except:
raise ValueError('Could not parse version')
if not (Version(version) >= Version(min_version)):
raise RuntimeError(
f'Required {min_version}, found {version}')
except ValueError:
print(f'\n[!] Sorry, but our script could not parse the output of '
f'`{cmd.replace(password, "***") if password else cmd}`; '
f'please file a bug, or see `zuds/core.py`\n')
raise
except Exception as e:
fail.append((dep, e, cmd, min_version))
if fail:
failstr = ''
for (pkg, exc, cmd, min_version) in fail:
repcmd = cmd
if password is not None:
repcmd = repcmd.replace(password, '***')
failstr += f' - {pkg}: `{repcmd}`\n'
failstr += ' ' + str(exc).replace(password, '***') + '\n'
msg = f'''
[!] Some system dependencies seem to be unsatisfied
The failed checks were:
{failstr}
'''
raise DependencyError(msg)
def galaxy_validate(serializer, attrs):
"""Ansible Galaxy config options have mutual exclusivity rules, these rules
are enforced here on serializer validation so that users will not be able
to save settings which obviously break all project updates.
"""
prefix = 'PRIMARY_GALAXY_'
from awx.main.constants import GALAXY_SERVER_FIELDS
if not any('{}{}'.format(prefix, subfield.upper()) in attrs
for subfield in GALAXY_SERVER_FIELDS):
return attrs
def _new_value(setting_name):
if setting_name in attrs:
return attrs[setting_name]
elif not serializer.instance:
return ''
return getattr(serializer.instance, setting_name, '')
galaxy_data = {}
for subfield in GALAXY_SERVER_FIELDS:
galaxy_data[subfield] = _new_value('{}{}'.format(
prefix, subfield.upper()))
errors = {}
if not galaxy_data['url']:
for k, v in galaxy_data.items():
if v:
setting_name = '{}{}'.format(prefix, k.upper())
errors.setdefault(setting_name, [])
errors[setting_name].append(
_('Cannot provide field if PRIMARY_GALAXY_URL is not set.')
)
for k in GALAXY_SERVER_FIELDS:
if galaxy_data[k]:
setting_name = '{}{}'.format(prefix, k.upper())
if (not serializer.instance) or (not getattr(
serializer.instance, setting_name, '')):
# new auth is applied, so check if compatible with version
from awx.main.utils import get_ansible_version
current_version = get_ansible_version()
min_version = '2.9'
if Version(current_version) < Version(min_version):
errors.setdefault(setting_name, [])
errors[setting_name].append(
_('Galaxy server settings are not available until Ansible {min_version}, '
'you are running {current_version}.').format(
min_version=min_version,
current_version=current_version))
if (galaxy_data['password']
or galaxy_data['username']) and (galaxy_data['token']
or galaxy_data['auth_url']):
for k in ('password', 'username', 'token', 'auth_url'):
setting_name = '{}{}'.format(prefix, k.upper())
if setting_name in attrs:
errors.setdefault(setting_name, [])
errors[setting_name].append(
_('Setting Galaxy token and authentication URL is mutually exclusive with username and password.'
))
if bool(galaxy_data['username']) != bool(galaxy_data['password']):
msg = _(
'If authenticating via username and password, both must be provided.'
)
for k in ('username', 'password'):
setting_name = '{}{}'.format(prefix, k.upper())
errors.setdefault(setting_name, [])
errors[setting_name].append(msg)
if bool(galaxy_data['token']) != bool(galaxy_data['auth_url']):
msg = _(
'If authenticating via token, both token and authentication URL must be provided.'
)
for k in ('token', 'auth_url'):
setting_name = '{}{}'.format(prefix, k.upper())
errors.setdefault(setting_name, [])
errors[setting_name].append(msg)
if errors:
raise serializers.ValidationError(errors)
return attrs
def __init__(self, common, is_gui, mode='share'):
self.common = common
self.common.log('Web', '__init__',
'is_gui={}, mode={}'.format(is_gui, mode))
# The flask app
self.app = Flask(
__name__,
static_folder=self.common.get_resource_path('static'),
template_folder=self.common.get_resource_path('templates'))
self.app.secret_key = self.common.random_string(8)
# Verbose mode?
if self.common.verbose:
self.verbose_mode()
# Are we running in GUI mode?
self.is_gui = is_gui
# If the user stops the server while a transfer is in progress, it should
# immediately stop the transfer. In order to make it thread-safe, stop_q
# is a queue. If anything is in it, then the user stopped the server
self.stop_q = queue.Queue()
# Are we using receive mode?
self.mode = mode
if self.mode == 'receive':
# Use custom WSGI middleware, to modify environ
self.app.wsgi_app = ReceiveModeWSGIMiddleware(
self.app.wsgi_app, self)
# Use a custom Request class to track upload progess
self.app.request_class = ReceiveModeRequest
# Starting in Flask 0.11, render_template_string autoescapes template variables
# by default. To prevent content injection through template variables in
# earlier versions of Flask, we force autoescaping in the Jinja2 template
# engine if we detect a Flask version with insecure default behavior.
if Version(flask_version) < Version('0.11'):
# Monkey-patch in the fix from https://github.com/pallets/flask/commit/99c99c4c16b1327288fd76c44bc8635a1de452bc
Flask.select_jinja_autoescape = self._safe_select_jinja_autoescape
self.security_headers = [(
'Content-Security-Policy',
'default-src \'self\'; style-src \'self\'; script-src \'self\'; img-src \'self\' data:;'
), ('X-Frame-Options', 'DENY'), ('X-Xss-Protection', '1; mode=block'),
('X-Content-Type-Options', 'nosniff'),
('Referrer-Policy', 'no-referrer'),
('Server', 'OnionShare')]
self.q = queue.Queue()
self.slug = None
self.error404_count = 0
self.done = False
# shutting down the server only works within the context of flask, so the easiest way to do it is over http
self.shutdown_slug = self.common.random_string(16)
# Keep track if the server is running
self.running = False
# Define the web app routes
self.define_common_routes()
# Create the mode web object, which defines its own routes
self.share_mode = None
self.receive_mode = None
if self.mode == 'receive':
self.receive_mode = ReceiveModeWeb(self.common, self)
elif self.mode == 'share':
self.share_mode = ShareModeWeb(self.common, self)
def make_request(self, method, endpoint, *args, **kwargs):
# In case someone is calling us directly; make sure we were given a method, let's not just assume a GET
if not method:
raise Exception("The HTTP method must be defined")
if method in ['POST', 'PUT', 'PATCH']:
url = self.build_url(endpoint)
else:
url = self.build_url(endpoint, query_params=kwargs.get('data'))
# Extract the headers, this will be used in a couple of places
headers = kwargs.get('headers', {})
# Authenticate to Tower (if we don't have a token and if not already done so)
if not self.oauth_token and not self.authenticated:
# This method will set a cookie in the cookie jar for us and also an oauth_token
self.authenticate(**kwargs)
if self.oauth_token:
# If we have a oauth token, we just use a bearer header
headers['Authorization'] = 'Bearer {0}'.format(self.oauth_token)
if method in ['POST', 'PUT', 'PATCH']:
headers.setdefault('Content-Type', 'application/json')
kwargs['headers'] = headers
data = None # Important, if content type is not JSON, this should not be dict type
if headers.get('Content-Type', '') == 'application/json':
data = dumps(kwargs.get('data', {}))
try:
response = self.session.open(method,
url.geturl(),
headers=headers,
validate_certs=self.verify_ssl,
follow_redirects=True,
data=data)
except (SSLValidationError) as ssl_err:
self.fail_json(
msg=
"Could not establish a secure connection to your host ({1}): {0}."
.format(url.netloc, ssl_err))
except (ConnectionError) as con_err:
self.fail_json(
msg=
"There was a network error of some kind trying to connect to your host ({1}): {0}."
.format(url.netloc, con_err))
except (HTTPError) as he:
# Sanity check: Did the server send back some kind of internal error?
if he.code >= 500:
self.fail_json(
msg=
'The host sent back a server error ({1}): {0}. Please check the logs and try again later'
.format(url.path, he))
# Sanity check: Did we fail to authenticate properly? If so, fail out now; this is always a failure.
elif he.code == 401:
self.fail_json(
msg=
'Invalid Tower authentication credentials for {0} (HTTP 401).'
.format(url.path))
# Sanity check: Did we get a forbidden response, which means that the user isn't allowed to do this? Report that.
elif he.code == 403:
self.fail_json(
msg="You don't have permission to {1} to {0} (HTTP 403).".
format(url.path, method))
# Sanity check: Did we get a 404 response?
# Requests with primary keys will return a 404 if there is no response, and we want to consistently trap these.
elif he.code == 404:
if kwargs.get('return_none_on_404', False):
return None
self.fail_json(
msg='The requested object could not be found at {0}.'.
format(url.path))
# Sanity check: Did we get a 405 response?
# A 405 means we used a method that isn't allowed. Usually this is a bad request, but it requires special treatment because the
# API sends it as a logic error in a few situations (e.g. trying to cancel a job that isn't running).
elif he.code == 405:
self.fail_json(
msg=
"The Tower server says you can't make a request with the {0} method to this endpoing {1}"
.format(method, url.path))
# Sanity check: Did we get some other kind of error? If so, write an appropriate error message.
elif he.code >= 400:
# We are going to return a 400 so the module can decide what to do with it
page_data = he.read()
try:
return {'status_code': he.code, 'json': loads(page_data)}
# JSONDecodeError only available on Python 3.5+
except ValueError:
return {'status_code': he.code, 'text': page_data}
elif he.code == 204 and method == 'DELETE':
# A 204 is a normal response for a delete function
pass
else:
self.fail_json(
msg="Unexpected return code when calling {0}: {1}".format(
url.geturl(), he))
except (Exception) as e:
self.fail_json(
msg=
"There was an unknown error when trying to connect to {2}: {0} {1}"
.format(type(e).__name__, e, url.geturl()))
if not self.version_checked:
# In PY2 we get back an HTTPResponse object but PY2 is returning an addinfourl
# First try to get the headers in PY3 format and then drop down to PY2.
try:
tower_type = response.getheader('X-API-Product-Name', None)
tower_version = response.getheader('X-API-Product-Version',
None)
except Exception:
tower_type = response.info().getheader('X-API-Product-Name',
None)
tower_version = response.info().getheader(
'X-API-Product-Version', None)
parsed_collection_version = Version(
self._COLLECTION_VERSION).version
parsed_tower_version = Version(tower_version).version
if tower_type == 'AWX':
collection_compare_ver = parsed_collection_version[0]
tower_compare_ver = parsed_tower_version[0]
else:
collection_compare_ver = "{0}.{1}".format(
parsed_collection_version[0], parsed_collection_version[1])
tower_compare_ver = '{0}.{1}'.format(parsed_tower_version[0],
parsed_tower_version[1])
if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[
self._COLLECTION_TYPE] != tower_type:
self.warn(
"You are using the {0} version of this collection but connecting to {1}"
.format(self._COLLECTION_TYPE, tower_type))
elif collection_compare_ver != tower_compare_ver:
self.warn(
"You are running collection version {0} but connecting to {2} version {1}"
.format(self._COLLECTION_VERSION, tower_version,
tower_type))
self.version_checked = True
response_body = ''
try:
response_body = response.read()
except (Exception) as e:
self.fail_json(msg="Failed to read response body: {0}".format(e))
response_json = {}
if response_body and response_body != '':
try:
response_json = loads(response_body)
except (Exception) as e:
self.fail_json(
msg="Failed to parse the response json: {0}".format(e))
if PY2:
status_code = response.getcode()
else:
status_code = response.status
return {'status_code': status_code, 'json': response_json}
def _list_policies(self):
if self._version and self._version >= Version('3.7.9'):
# Remove first header line from policies list for version > 3.7.9
return self._exec(['list_policies'], True)[1:]
return self._exec(['list_policies'], True)
# -*- encoding: utf-8 -*-
from __future__ import unicode_literals
import operator
import pkg_resources
from distutils.version import LooseVersion as Version
HTML5LIB_VERSION = Version(pkg_resources.get_distribution("html5lib").version)
HTML5LIB_SIMPLETREE = Version("0.95")
from isso.compat import reduce
import html5lib
from html5lib.sanitizer import HTMLSanitizer
from html5lib.serializer import HTMLSerializer
import misaka
def Sanitizer(elements, attributes):
class Inner(HTMLSanitizer):
# attributes found in Sundown's HTML serializer [1] except for <img> tag,
# because images are not generated anyways.
#
# [1] https://github.com/vmg/sundown/blob/master/html/html.c
allowed_elements = [
"a", "p", "hr", "br", "ol", "ul", "li", "pre", "code",
"blockquote", "del", "ins", "strong", "em", "h1", "h2", "h3", "h4",
all_data = pd.read_csv('wine.csv')
col_num = all_data.shape[1]
#データインポート
X = all_data.iloc[1:, 0:col_num - 1]
Y = all_data.iloc[1:, col_num - 1]
#features name
X_name = all_data.iloc[0, 0:col_num - 1]
#学習結果検証用 ratingがあるデータを、トレーニングデータと検証データに分解して利用。
from sklearn import __version__ as sklearn_version
from distutils.version import LooseVersion as Version
if Version(sklearn_version) < '0.18':
from sklearn.cross_validation import train_test_split
else:
from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
test_size=0.3,
random_state=0)
#データの標準化
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = np.array(X_train)
X_test = np.array(X_test)
def __init__(self, common, is_gui, mode_settings, mode="share"):
self.common = common
self.common.log("Web", "__init__", f"is_gui={is_gui}, mode={mode}")
self.settings = mode_settings
# The flask app
self.app = Flask(
__name__,
static_folder=self.common.get_resource_path("static"),
static_url_path=
f"/static_{self.common.random_string(16)}", # randomize static_url_path to avoid making /static unusable
template_folder=self.common.get_resource_path("templates"),
)
self.app.secret_key = self.common.random_string(8)
self.generate_static_url_path()
# Verbose mode?
if self.common.verbose:
self.verbose_mode()
# Are we running in GUI mode?
self.is_gui = is_gui
# If the user stops the server while a transfer is in progress, it should
# immediately stop the transfer. In order to make it thread-safe, stop_q
# is a queue. If anything is in it, then the user stopped the server
self.stop_q = queue.Queue()
# Are we using receive mode?
self.mode = mode
if self.mode == "receive":
# Use custom WSGI middleware, to modify environ
self.app.wsgi_app = ReceiveModeWSGIMiddleware(
self.app.wsgi_app, self)
# Use a custom Request class to track upload progress
self.app.request_class = ReceiveModeRequest
# Starting in Flask 0.11, render_template_string autoescapes template variables
# by default. To prevent content injection through template variables in
# earlier versions of Flask, we force autoescaping in the Jinja2 template
# engine if we detect a Flask version with insecure default behavior.
if Version(flask_version) < Version("0.11"):
# Monkey-patch in the fix from https://github.com/pallets/flask/commit/99c99c4c16b1327288fd76c44bc8635a1de452bc
Flask.select_jinja_autoescape = self._safe_select_jinja_autoescape
self.security_headers = [
("X-Frame-Options", "DENY"),
("X-Xss-Protection", "1; mode=block"),
("X-Content-Type-Options", "nosniff"),
("Referrer-Policy", "no-referrer"),
("Server", "OnionShare"),
]
self.q = queue.Queue()
self.done = False
# shutting down the server only works within the context of flask, so the easiest way to do it is over http
self.shutdown_password = self.common.random_string(16)
# Keep track if the server is running
self.running = False
# Define the web app routes
self.define_common_routes()
# Create the mode web object, which defines its own routes
self.share_mode = None
self.receive_mode = None
self.website_mode = None
self.chat_mode = None
if self.mode == "share":
self.share_mode = ShareModeWeb(self.common, self)
elif self.mode == "receive":
self.receive_mode = ReceiveModeWeb(self.common, self)
elif self.mode == "website":
self.website_mode = WebsiteModeWeb(self.common, self)
elif self.mode == "chat":
self.socketio = SocketIO()
self.socketio.init_app(self.app)
self.chat_mode = ChatModeWeb(self.common, self)
self.cleanup_filenames = []
import re
import subprocess
import sys
from distutils.version import StrictVersion as Version
import buildconfig
from mozbuild.util import memoize
from mozpack.executables import (
get_type,
ELF,
MACHO,
UNKNOWN,
)
STDCXX_MAX_VERSION = Version('3.4.17')
GLIBC_MAX_VERSION = Version('2.12')
LIBGCC_MAX_VERSION = Version('4.8')
HOST = {
'MOZ_LIBSTDCXX_VERSION':
buildconfig.substs.get('MOZ_LIBSTDCXX_HOST_VERSION'),
'platform': buildconfig.substs['HOST_OS_ARCH'],
'readelf': 'readelf',
}
TARGET = {
'MOZ_LIBSTDCXX_VERSION':
buildconfig.substs.get('MOZ_LIBSTDCXX_TARGET_VERSION'),
'platform': buildconfig.substs['OS_TARGET'],
'readelf':
import numpy as np
from skimage.segmentation import random_walker
from skimage.transform import resize
from skimage._shared._warnings import expected_warnings
from skimage._shared import testing
from skimage._shared.testing import xfail, arch32
import scipy
from distutils.version import LooseVersion as Version
# older versions of scipy raise a warning with new NumPy because they use
# numpy.rank() instead of arr.ndim or numpy.linalg.matrix_rank.
SCIPY_RANK_WARNING = r'numpy.linalg.matrix_rank|\A\Z'
PYAMG_MISSING_WARNING = r'pyamg|\A\Z'
PYAMG_OR_SCIPY_WARNING = SCIPY_RANK_WARNING + '|' + PYAMG_MISSING_WARNING
if Version(scipy.__version__) < '1.3':
NUMPY_MATRIX_WARNING = 'matrix subclass'
else:
NUMPY_MATRIX_WARNING = None
def make_2d_syntheticdata(lx, ly=None):
if ly is None:
ly = lx
np.random.seed(1234)
data = np.zeros((lx, ly)) + 0.1 * np.random.randn(lx, ly)
small_l = int(lx // 5)
data[lx // 2 - small_l:lx // 2 + small_l,
ly // 2 - small_l:ly // 2 + small_l] = 1
data[lx // 2 - small_l + 1:lx // 2 + small_l - 1,
ly // 2 - small_l + 1:ly // 2 + small_l -
import numpy as np
from skimage import restoration, data, color, img_as_float, measure
from skimage.measure import compare_psnr
from skimage.restoration._denoise import _wavelet_threshold
import pywt
from skimage._shared import testing
from skimage._shared.testing import (assert_equal, assert_almost_equal,
assert_warns, assert_)
from skimage._shared._warnings import expected_warnings
from distutils.version import LooseVersion as Version
if (Version(np.__version__) >= '1.15.0' and
Version(pywt.__version__) <= '0.5.2'):
PYWAVELET_ND_INDEXING_WARNING = 'non-tuple sequence for multidimensional'
else:
PYWAVELET_ND_INDEXING_WARNING = None
np.random.seed(1234)
astro = img_as_float(data.astronaut()[:128, :128])
astro_gray = color.rgb2gray(astro)
checkerboard_gray = img_as_float(data.checkerboard())
checkerboard = color.gray2rgb(checkerboard_gray)
def test_denoise_tv_chambolle_2d():
def read(self,count=-1,warn_version=True):
"""Read objects from a pyFormex Geometry File.
This function reads objects from a Geometry File until the file
ends, or until `count` objects have been read.
The File should have been opened for reading.
A count may be specified to limit the number of objects read.
Returns a dict with the objects read. The keys of the dict are the
object names found in the file. If the file does not contain
object names, they will be autogenerated from the file name.
Note that PGF files of version 1.0 are no longer supported.
The use of formats 1.1 to 1.5 is deprecated, and users are
urged to upgrade these files to a newer format. Support for
these formats may be removed in future.
"""
if self.writing:
print("File is opened for writing, not reading.")
return {}
self.results = OrderedDict()
self.geometry = None # used to make sure fields follow geom block
if Version(self.version) < Version('1.6'):
if warn_version:
pf.warning("This is an old PGF format (%s). We recommend you to convert it to a newer format. The geometry import menu contains an item to upgrade a PGF file to the latest format (%s)." % (self.version,GeometryFile._version_))
return self.readLegacy(count)
while True:
s = self.fil.readline()
if len(s) == 0: # end of file
break
if s.startswith('#'):
# Remove the leading '#' and space
s = s[1:].strip()
if s.startswith('objtype'):
if count > 0 and len(self.results) >= count:
break
self.readGeometry(**self.decode(s))
elif s.startswith('field'):
self.readField(**self.decode(s))
elif s.startswith('attrib'):
self.readAttrib(**self.decode(s))
elif s.startswith('pyFormex Geometry File'):
# we have a new header line
self.readHeader(s)
# Unrecognized lines are silently ignored, whether starting
# with a '#' or not.
# We recommend to start all comments lines with a '#' though.
self.file.close()
return self.results
def get_osutil(distro_name=DISTRO_NAME,
distro_code_name=DISTRO_CODE_NAME,
distro_version=DISTRO_VERSION,
distro_full_name=DISTRO_FULL_NAME):
if distro_name == "arch":
return ArchUtil()
if distro_name == "clear linux os for intel architecture" \
or distro_name == "clear linux software for intel architecture":
return ClearLinuxUtil()
if distro_name == "ubuntu":
if Version(distro_version) in [Version("12.04"), Version("12.10")]:
return Ubuntu12OSUtil()
elif Version(distro_version) in [Version("14.04"), Version("14.10")]:
return Ubuntu14OSUtil()
elif Version(distro_version) in [
Version('16.04'),
Version('16.10'),
Version('17.04')
]:
return Ubuntu16OSUtil()
elif Version(distro_version) in [Version('18.04')]:
return Ubuntu18OSUtil()
elif distro_full_name == "Snappy Ubuntu Core":
return UbuntuSnappyOSUtil()
else:
return UbuntuOSUtil()
if distro_name == "alpine":
return AlpineOSUtil()
if distro_name == "kali":
return DebianOSUtil()
if distro_name == "coreos" or distro_code_name == "coreos":
return CoreOSUtil()
if distro_name in ("suse", "sles", "opensuse"):
if distro_full_name == 'SUSE Linux Enterprise Server' \
and Version(distro_version) < Version('12') \
or distro_full_name == 'openSUSE' and Version(distro_version) < Version('13.2'):
return SUSE11OSUtil()
else:
return SUSEOSUtil()
elif distro_name == "debian":
return DebianOSUtil()
elif distro_name == "redhat" \
or distro_name == "centos" \
or distro_name == "oracle":
if Version(distro_version) < Version("7"):
return Redhat6xOSUtil()
else:
return RedhatOSUtil()
elif distro_name == "euleros":
return RedhatOSUtil()
elif distro_name == "freebsd":
return FreeBSDOSUtil()
elif distro_name == "openbsd":
return OpenBSDOSUtil()
elif distro_name == "bigip":
return BigIpOSUtil()
elif distro_name == "gaia":
return GaiaOSUtil()
else:
logger.warn(
"Unable to load distro implementation for {0}. Using "
"default distro implementation instead.", distro_name)
return DefaultOSUtil()
print('Checking system dependencies:')
fail = []
for dep, (cmd, get_version, min_version) in deps.items():
try:
query = f'{dep} >= {min_version}'
with status(query):
success, out = output(cmd)
try:
version = get_version(out.decode('utf-8').strip())
print(f'[{version.rjust(8)}]'.rjust(40 - len(query)), end='')
except:
raise ValueError('Could not parse version')
if not (Version(version) >= Version(min_version)):
raise RuntimeError(f'Required {min_version}, found {version}')
except ValueError:
print(
f'\n[!] Sorry, but our script could not parse the output of '
f'`{" ".join(cmd)}`; please file a bug, or see '
f'`check_app_environment.py`\n'
)
raise
except Exception as e:
fail.append((dep, e))
if fail:
print()
print('[!] Some system dependencies seem to be unsatisfied')
print()
def connect(
self,
custom_settings=None,
config=None,
tor_status_update_func=None,
connect_timeout=120,
local_only=False,
):
if local_only:
self.common.log("Onion", "connect",
"--local-only, so skip trying to connect")
return
self.common.log("Onion", "connect")
# Either use settings that are passed in, or use them from common
if custom_settings:
self.settings = custom_settings
elif config:
self.common.load_settings(config)
self.settings = self.common.settings
else:
self.common.load_settings()
self.settings = self.common.settings
# The Tor controller
self.c = None
if self.settings.get("connection_type") == "bundled":
if not self.bundle_tor_supported:
raise BundledTorNotSupported(
# strings._("settings_error_bundled_tor_not_supported")
"Using the Tor version that comes with OnionShare does not work in developer mode on Windows or macOS."
)
# Create a torrc for this session
if self.use_tmp_dir:
self.tor_data_directory = tempfile.TemporaryDirectory(
dir=self.common.build_tmp_dir())
self.tor_data_directory_name = self.tor_data_directory.name
else:
self.tor_data_directory_name = self.common.build_tor_dir()
self.common.log(
"Onion",
"connect",
f"tor_data_directory_name={self.tor_data_directory_name}",
)
# Create the torrc
with open(self.common.get_resource_path("torrc_template")) as f:
torrc_template = f.read()
self.tor_cookie_auth_file = os.path.join(
self.tor_data_directory_name, "cookie")
try:
self.tor_socks_port = self.common.get_available_port(
1000, 65535)
except:
raise OSError("OnionShare port not available")
self.tor_torrc = os.path.join(self.tor_data_directory_name,
"torrc")
if self.common.platform == "Windows" or self.common.platform == "Darwin":
# Windows doesn't support unix sockets, so it must use a network port.
# macOS can't use unix sockets either because socket filenames are limited to
# 100 chars, and the macOS sandbox forces us to put the socket file in a place
# with a really long path.
torrc_template += "ControlPort {{control_port}}\n"
try:
self.tor_control_port = self.common.get_available_port(
1000, 65535)
except:
raise OSError("OnionShare port not available")
self.tor_control_socket = None
else:
# Linux and BSD can use unix sockets
torrc_template += "ControlSocket {{control_socket}}\n"
self.tor_control_port = None
self.tor_control_socket = os.path.join(
self.tor_data_directory_name, "control_socket")
torrc_template = torrc_template.replace(
"{{data_directory}}", self.tor_data_directory_name)
torrc_template = torrc_template.replace("{{control_port}}",
str(self.tor_control_port))
torrc_template = torrc_template.replace(
"{{control_socket}}", str(self.tor_control_socket))
torrc_template = torrc_template.replace("{{cookie_auth_file}}",
self.tor_cookie_auth_file)
torrc_template = torrc_template.replace("{{geo_ip_file}}",
self.tor_geo_ip_file_path)
torrc_template = torrc_template.replace(
"{{geo_ipv6_file}}", self.tor_geo_ipv6_file_path)
torrc_template = torrc_template.replace("{{socks_port}}",
str(self.tor_socks_port))
with open(self.tor_torrc, "w") as f:
f.write(torrc_template)
# Bridge support
if self.settings.get("tor_bridges_use_obfs4"):
f.write(
f"ClientTransportPlugin obfs4 exec {self.obfs4proxy_file_path}\n"
)
with open(
self.common.get_resource_path(
"torrc_template-obfs4")) as o:
for line in o:
f.write(line)
elif self.settings.get("tor_bridges_use_meek_lite_azure"):
f.write(
f"ClientTransportPlugin meek_lite exec {self.obfs4proxy_file_path}\n"
)
with open(
self.common.get_resource_path(
"torrc_template-meek_lite_azure")) as o:
for line in o:
f.write(line)
if self.settings.get("tor_bridges_use_custom_bridges"):
if "obfs4" in self.settings.get(
"tor_bridges_use_custom_bridges"):
f.write(
f"ClientTransportPlugin obfs4 exec {self.obfs4proxy_file_path}\n"
)
elif "meek_lite" in self.settings.get(
"tor_bridges_use_custom_bridges"):
f.write(
f"ClientTransportPlugin meek_lite exec {self.obfs4proxy_file_path}\n"
)
f.write(
self.settings.get("tor_bridges_use_custom_bridges"))
f.write("\nUseBridges 1")
# Make sure the tor path is accurate
if not os.path.exists(self.tor_path):
raise BundledTorNotSupported(
f"Cannot find tor binary: {self.tor_path}")
# Execute a tor subprocess
start_ts = time.time()
if self.common.platform == "Windows":
# In Windows, hide console window when opening tor.exe subprocess
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
self.tor_proc = subprocess.Popen(
[self.tor_path, "-f", self.tor_torrc],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
startupinfo=startupinfo,
)
else:
self.tor_proc = subprocess.Popen(
[self.tor_path, "-f", self.tor_torrc],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
# Wait for the tor controller to start
time.sleep(2)
# Connect to the controller
try:
if (self.common.platform == "Windows"
or self.common.platform == "Darwin"):
self.c = Controller.from_port(port=self.tor_control_port)
self.c.authenticate()
else:
self.c = Controller.from_socket_file(
path=self.tor_control_socket)
self.c.authenticate()
except Exception as e:
raise BundledTorBroken(
# strings._("settings_error_bundled_tor_broken").format(e.args[0])
"OnionShare could not connect to Tor:\n{}".format(e.args[0]
))
while True:
try:
res = self.c.get_info("status/bootstrap-phase")
except SocketClosed:
raise BundledTorCanceled()
res_parts = shlex.split(res)
progress = res_parts[2].split("=")[1]
summary = res_parts[4].split("=")[1]
# "\033[K" clears the rest of the line
print(
f"\rConnecting to the Tor network: {progress}% - {summary}\033[K",
end="",
)
if callable(tor_status_update_func):
if not tor_status_update_func(progress, summary):
# If the dialog was canceled, stop connecting to Tor
self.common.log(
"Onion",
"connect",
"tor_status_update_func returned false, canceling connecting to Tor",
)
print()
return False
if summary == "Done":
print("")
break
time.sleep(0.2)
# If using bridges, it might take a bit longer to connect to Tor
if (self.settings.get("tor_bridges_use_custom_bridges")
or self.settings.get("tor_bridges_use_obfs4") or
self.settings.get("tor_bridges_use_meek_lite_azure")):
# Only override timeout if a custom timeout has not been passed in
if connect_timeout == 120:
connect_timeout = 150
if time.time() - start_ts > connect_timeout:
print("")
try:
self.tor_proc.terminate()
raise BundledTorTimeout(
# strings._("settings_error_bundled_tor_timeout")
"Taking too long to connect to Tor. Maybe you aren't connected to the Internet, or have an inaccurate system clock?"
)
except FileNotFoundError:
pass
elif self.settings.get("connection_type") == "automatic":
# Automatically try to guess the right way to connect to Tor Browser
# Try connecting to control port
found_tor = False
# If the TOR_CONTROL_PORT environment variable is set, use that
env_port = os.environ.get("TOR_CONTROL_PORT")
if env_port:
try:
self.c = Controller.from_port(port=int(env_port))
found_tor = True
except:
pass
else:
# Otherwise, try default ports for Tor Browser, Tor Messenger, and system tor
try:
ports = [9151, 9153, 9051]
for port in ports:
self.c = Controller.from_port(port=port)
found_tor = True
except:
pass
# If this still didn't work, try guessing the default socket file path
socket_file_path = ""
if not found_tor:
try:
if self.common.platform == "Darwin":
socket_file_path = os.path.expanduser(
"~/Library/Application Support/TorBrowser-Data/Tor/control.socket"
)
self.c = Controller.from_socket_file(
path=socket_file_path)
found_tor = True
except:
pass
# If connecting to default control ports failed, so let's try
# guessing the socket file name next
if not found_tor:
try:
if self.common.platform == "Linux" or self.common.platform == "BSD":
socket_file_path = (
f"/run/user/{os.geteuid()}/Tor/control.socket")
elif self.common.platform == "Darwin":
socket_file_path = (
f"/run/user/{os.geteuid()}/Tor/control.socket")
elif self.common.platform == "Windows":
# Windows doesn't support unix sockets
raise TorErrorAutomatic(
# strings._("settings_error_automatic")
"Could not connect to the Tor controller. Is Tor Browser (available from torproject.org) running in the background?"
)
self.c = Controller.from_socket_file(path=socket_file_path)
except:
raise TorErrorAutomatic(
# strings._("settings_error_automatic")
"Could not connect to the Tor controller. Is Tor Browser (available from torproject.org) running in the background?"
)
# Try authenticating
try:
self.c.authenticate()
except:
raise TorErrorAutomatic(
# strings._("settings_error_automatic")
"Could not connect to the Tor controller. Is Tor Browser (available from torproject.org) running in the background?"
)
else:
# Use specific settings to connect to tor
# Try connecting
try:
if self.settings.get("connection_type") == "control_port":
self.c = Controller.from_port(
address=self.settings.get("control_port_address"),
port=self.settings.get("control_port_port"),
)
elif self.settings.get("connection_type") == "socket_file":
self.c = Controller.from_socket_file(
path=self.settings.get("socket_file_path"))
else:
raise TorErrorInvalidSetting(
# strings._("settings_error_unknown")
"Can't connect to Tor controller because your settings don't make sense."
)
except:
if self.settings.get("connection_type") == "control_port":
raise TorErrorSocketPort(
# strings._("settings_error_socket_port")
"Can't connect to the Tor controller at {}:{}.".format(
self.settings.get("control_port_address"),
self.settings.get("control_port_port"),
))
else:
raise TorErrorSocketFile(
# strings._("settings_error_socket_file")
"Can't connect to the Tor controller using socket file {}."
.format(self.settings.get("socket_file_path")))
# Try authenticating
try:
if self.settings.get("auth_type") == "no_auth":
self.c.authenticate()
elif self.settings.get("auth_type") == "password":
self.c.authenticate(self.settings.get("auth_password"))
else:
raise TorErrorInvalidSetting(
# strings._("settings_error_unknown")
"Can't connect to Tor controller because your settings don't make sense."
)
except MissingPassword:
raise TorErrorMissingPassword(
# strings._("settings_error_missing_password")
"Connected to Tor controller, but it requires a password to authenticate."
)
except UnreadableCookieFile:
raise TorErrorUnreadableCookieFile(
# strings._("settings_error_unreadable_cookie_file")
"Connected to the Tor controller, but password may be wrong, or your user is not permitted to read the cookie file."
)
except AuthenticationFailure:
raise TorErrorAuthError(
# strings._("settings_error_auth")
"Connected to {}:{}, but can't authenticate. Maybe this isn't a Tor controller?"
.format(
self.settings.get("control_port_address"),
self.settings.get("control_port_port"),
))
# If we made it this far, we should be connected to Tor
self.connected_to_tor = True
# Get the tor version
self.tor_version = self.c.get_version().version_str
self.common.log("Onion", "connect",
f"Connected to tor {self.tor_version}")
# Do the versions of stem and tor that I'm using support ephemeral onion services?
list_ephemeral_hidden_services = getattr(
self.c, "list_ephemeral_hidden_services", None)
self.supports_ephemeral = (callable(list_ephemeral_hidden_services)
and self.tor_version >= "0.2.7.1")
# Do the versions of stem and tor that I'm using support stealth onion services?
try:
res = self.c.create_ephemeral_hidden_service(
{1: 1},
basic_auth={"onionshare": None},
await_publication=False,
key_type="NEW",
key_content="RSA1024",
)
tmp_service_id = res.service_id
self.c.remove_ephemeral_hidden_service(tmp_service_id)
self.supports_stealth = True
except:
# ephemeral stealth onion services are not supported
self.supports_stealth = False
# Does this version of Tor support next-gen ('v3') onions?
# Note, this is the version of Tor where this bug was fixed:
# https://trac.torproject.org/projects/tor/ticket/28619
self.supports_v3_onions = self.tor_version >= Version("0.3.5.7")
def export_attributes(cls, state, buf, mesh_name, base_vert_list,
vert_list):
gltf_attrs = {}
is_skinned = mesh_name in state['skinned_meshes']
color_size = 4 if state['settings']['meshes_vertex_color_alpha'] else 3
num_uv_layers = len(vert_list[0].uvs)
num_col_layers = len(vert_list[0].colors)
vertex_size = (3 + 3 + num_uv_layers * 2 +
num_col_layers * color_size) * 4
if is_skinned:
vertex_size += (4 + 4) * 4
num_verts = len(vert_list)
if state['settings']['meshes_interleave_vertex_data']:
view = buf.add_view(vertex_size * num_verts, vertex_size,
Buffer.ARRAY_BUFFER)
else:
view = None
offset = OffsetTracker()
def create_attr_accessor(name, component_type, component_count):
component_size = 1 if component_type == Buffer.UNSIGNED_BYTE else 4
if not view:
stride = component_size * component_count
buffer = Buffer(mesh_name + '_' + name)
state['buffers'].append(buffer)
state['input']['buffers'].append(SimpleID(buffer.name))
_view = buffer.add_view(stride * num_verts, stride,
Buffer.ARRAY_BUFFER)
interleaved = False
else:
_view = view
buffer = buf
stride = vertex_size
interleaved = True
data_type = [Buffer.SCALAR, Buffer.VEC2, Buffer.VEC3,
Buffer.VEC4][component_count - 1]
_offset = offset.get()
acc = buffer.add_accessor(_view, _offset, stride, component_type,
num_verts, data_type)
if interleaved:
offset.add(component_size * component_count)
return acc
def add_attribute(name, num_components, component_type=Buffer.FLOAT):
acc = create_attr_accessor(name, component_type, num_components)
gltf_attrs[name] = Reference('accessors', acc.name, gltf_attrs,
name)
state['references'].append(gltf_attrs[name])
return acc
vdata = add_attribute('POSITION', 3)
ndata = add_attribute('NORMAL', 3)
if not base_vert_list:
tdata = [
add_attribute('TEXCOORD_' + str(i), 2)
for i in range(num_uv_layers)
]
cdata = [
add_attribute('COLOR_' + str(i), color_size)
for i in range(num_col_layers)
]
# Copy vertex data
if base_vert_list:
fill_data(vdata, ([a - b for a, b in zip(v.co, b.co)]
for v, b in zip(vert_list, base_vert_list)))
fill_data(ndata, ([a - b for a, b in zip(v.normal, b.normal)]
for v, b in zip(vert_list, base_vert_list)))
else:
fill_data(vdata, (v.co for v in vert_list))
fill_data(ndata, (v.normal for v in vert_list))
for i, accessor in enumerate(cdata):
if state['settings']['meshes_vertex_color_alpha']:
fill_data(accessor,
(v.colors[i] + [1.0] for v in vert_list))
else:
fill_data(accessor, (v.colors[i] for v in vert_list))
for i, accessor in enumerate(tdata):
if state['settings']['asset_profile'] == 'WEB':
fill_data(accessor, ((v.uvs[i][0], 1.0 - v.uvs[i][1])
for v in vert_list))
else:
fill_data(accessor, (v.uvs[i] for v in vert_list))
for i, vtx in enumerate(vert_list):
vtx.index = i
state['buffers'].append(buf)
state['input']['buffers'].append(SimpleID(buf.name))
if is_skinned:
if state['version'] < Version('2.0'):
joint_key = 'JOINT'
weight_key = 'WEIGHT'
else:
joint_key = 'JOINTS_0'
weight_key = 'WEIGHTS_0'
jdata = add_attribute(joint_key, 4, Buffer.UNSIGNED_BYTE)
fill_data(jdata, (v.joint_indexes for v in vert_list))
wdata = add_attribute(weight_key, 4)
fill_data(wdata, (v.weights for v in vert_list))
return gltf_attrs
def export_gltf(scene_delta, settings=None):
# Fill in any missing settings with defaults
if not settings:
settings = {}
for key, value in DEFAULT_SETTINGS.items():
settings.setdefault(key, value)
res_x = bpy.context.scene.render.resolution_x
res_y = bpy.context.scene.render.resolution_y
# Initialize export state
state = {
'version': Version(settings['asset_version']),
'settings': settings,
'animation_dt': 1.0 / bpy.context.scene.render.fps,
'mod_meshes_obj': {},
'aspect_ratio': res_x / res_y,
'mod_meshes': {},
'shape_keys': {},
'dupli_nodes': [],
'bone_children': {},
'extensions_used': [],
'gl_extensions_used': [],
'buffers': [],
'samplers': [],
'input': {
'buffers': [],
'accessors': [],
'bufferViews': [],
'objects': [],
'bones': [],
'anim_samplers': [],
'samplers': [],
'scenes': [],
'skins': [],
'materials': [],
'dupli_ids': [],
},
'output': {
'extensions': [],
},
'references': [],
'files': {},
'decompose_fn': _decompose,
'decompose_mesh_fn': _decompose,
}
state['input'].update({key: list(value) for key, value in scene_delta.items()})
# Filter out empty meshes
if 'meshes' in state['input']:
state['input']['meshes'] = [mesh for mesh in state['input']['meshes'] if mesh.loops]
if 'objects' in state['input']:
state['input']['objects'] = [
obj for obj in state['input']['objects']
if obj.type != 'MESH' or obj.data in state['input']['meshes']
]
# Make sure any temporary meshes do not have animation data baked in
default_scene = bpy.context.scene
if not settings['hacks_streaming']:
saved_pose_positions = [armature.pose_position for armature in bpy.data.armatures]
for armature in bpy.data.armatures:
armature.pose_position = 'REST'
if saved_pose_positions:
for obj in bpy.data.objects:
if obj.type == 'ARMATURE':
obj.update_tag()
default_scene.frame_set(default_scene.frame_current)
mesh_list = []
mod_obs = [
ob for ob in state['input']['objects']
if [mod for mod in ob.modifiers if mod.type != 'ARMATURE']
]
for mesh in state['input'].get('meshes', []):
if mesh.shape_keys and mesh.shape_keys.use_relative:
relative_key = mesh.shape_keys.key_blocks[0].relative_key
keys = [key for key in mesh.shape_keys.key_blocks if key != relative_key]
# Gather weight values
weights = [key.value for key in keys]
# Clear weight values
for key in keys:
key.value = 0.0
mesh_users = [obj for obj in state['input']['objects'] if obj.data == mesh]
# Mute modifiers if necessary
muted_modifiers = []
original_modifier_states = []
if not settings['meshes_apply_modifiers']:
muted_modifiers = itertools.chain.from_iterable(
[obj.modifiers for obj in mesh_users]
)
original_modifier_states = [mod.show_viewport for mod in muted_modifiers]
for modifier in muted_modifiers:
modifier.show_viewport = False
for user in mesh_users:
base_mesh = user.to_mesh(default_scene, True, 'PREVIEW')
mesh_name = base_mesh.name
state['mod_meshes_obj'][user.name] = base_mesh
if mesh_name not in state['shape_keys']:
key_meshes = []
for key, weight in zip(keys, weights):
key.value = key.slider_max
key_meshes.append((
weight,
user.to_mesh(default_scene, True, 'PREVIEW')
))
key.value = 0.0
state['shape_keys'][mesh_name] = key_meshes
# Reset weight values
for key, weight in zip(keys, weights):
key.value = weight
# Unmute modifiers
for modifier, state in zip(muted_modifiers, original_modifier_states):
modifier.show_viewport = state
elif settings['meshes_apply_modifiers']:
mod_users = [ob for ob in mod_obs if ob.data == mesh]
# Only convert meshes with modifiers, otherwise each non-modifier
# user ends up with a copy of the mesh and we lose instancing
state['mod_meshes_obj'].update(
{ob.name: ob.to_mesh(default_scene, True, 'PREVIEW') for ob in mod_users}
)
# Add unmodified meshes directly to the mesh list
if len(mod_users) < mesh.users:
mesh_list.append(mesh)
else:
mesh_list.append(mesh)
mesh_list.extend(state['mod_meshes_obj'].values())
state['input']['meshes'] = mesh_list
apply_global_matrix = (
settings['nodes_global_matrix'] != mathutils.Matrix.Identity(4)
and settings['nodes_global_matrix_apply']
)
if apply_global_matrix:
global_mat = settings['nodes_global_matrix']
global_scale_mat = mathutils.Matrix([[abs(j) for j in i] for i in global_mat])
def decompose_apply(matrix):
loc, rot, scale = matrix.decompose()
loc.rotate(global_mat)
loc = loc.to_tuple()
rot.rotate(global_mat)
rot = (rot.x, rot.y, rot.z, rot.w)
scale.rotate(global_scale_mat)
scale = scale.to_tuple()
return loc, rot, scale
def decompose_mesh_apply(matrix):
loc, rot, scale = matrix.decompose()
loc.rotate(global_mat)
loc = loc.to_tuple()
rot = mathutils.Vector(list(rot.to_euler()))
rot.rotate(global_mat)
rot = mathutils.Euler(rot, 'XYZ').to_quaternion()
rot = (rot.x, rot.y, rot.z, rot.w)
scale.rotate(global_scale_mat)
scale = scale.to_tuple()
return loc, rot, scale
state['decompose_fn'] = decompose_apply
state['decompose_mesh_fn'] = decompose_mesh_apply
transformed_meshes = [mesh.copy() for mesh in mesh_list]
for mesh in transformed_meshes:
mesh.transform(global_mat, shape_keys=False)
state['mod_meshes'].update(
{mesh.name: xformed_mesh for xformed_mesh, mesh in zip(transformed_meshes, mesh_list)}
)
for shape_key_list in state['shape_keys'].values():
for shape_key in shape_key_list:
shape_key[1].transform(global_mat, shape_keys=False)
# Restore armature pose positions
for i, armature in enumerate(bpy.data.armatures):
armature.pose_position = saved_pose_positions[i]
state['input']['action_pairs'] = AnimationExporter.gather(state)
exporters = [
CameraExporter,
ImageExporter,
NodeExporter,
MaterialExporter,
# Make sure meshes come after nodes to detect which meshes are skinned
MeshExporter,
SceneExporter,
SkinExporter,
TextureExporter,
AnimationExporter,
]
state['output'] = {
exporter.gltf_key: [
exporter.export(state, data)
if exporter.check(state, data)
else exporter.default(state, data)
for data in state['input'].get(exporter.blender_key, [])
] for exporter in exporters
}
# Export top level data
gltf = {
'asset': {
'version': settings['asset_version'],
'generator': 'blendergltf v1.2.0',
'copyright': settings['asset_copyright'],
}
}
if state['version'] < Version('2.0'):
gltf['asset']['profile'] = PROFILE_MAP[settings['asset_profile']]
# Export samplers
state['output']['samplers'] = state['samplers']
# Export animations
state['output']['nodes'].extend([
export_joint(state, sid.data) for sid in state['input']['bones']
])
# Move bones to nodes for updating references
state['input']['objects'].extend(state['input']['bones'])
state['input']['bones'] = []
# Export dupli-groups
state['output']['nodes'].extend(state['dupli_nodes'])
state['input']['objects'].extend(state['input']['dupli_ids'])
state['input']['dupli_ids'] = []
# Export default scene
default_scene = None
for scene in state['input']['scenes']:
if scene == bpy.context.scene:
default_scene = scene
if default_scene:
scene_ref = Reference('scenes', bpy.context.scene.name, gltf, 'scene')
scene_ref.value = 0
state['references'].append(scene_ref)
# Export extensions
state['refmap'] = build_int_refmap(state['input'])
for ext_exporter in settings['extension_exporters']:
ext_exporter.export(state)
# Insert root nodes if axis conversion is needed
root_node_needed = (
settings['nodes_global_matrix'] != mathutils.Matrix.Identity(4)
and not settings['nodes_global_matrix_apply']
)
if root_node_needed:
insert_root_nodes(state, togl(settings['nodes_global_matrix']))
if state['buffers']:
state['output'].update(export_buffers(state))
state['output'] = {key: value for key, value in state['output'].items() if value != []}
if state['extensions_used']:
gltf.update({'extensionsUsed': state['extensions_used']})
if state['version'] < Version('2.0'):
gltf.update({'glExtensionsUsed': state['gl_extensions_used']})
# Convert lists to dictionaries
if state['version'] < Version('2.0'):
extensions = state['output'].get('extensions', [])
state['output'] = {
key: {
'{}_{}'.format(key, data['name']): data for data in value
} for key, value in state['output'].items()
if key != 'extensions'
}
if extensions:
state['output']['extensions'] = extensions
gltf.update(state['output'])
# Gather refmap inputs
reference_inputs = state['input']
if settings['hacks_streaming']:
reference_inputs.update({
'actions': list(bpy.data.actions),
'cameras': list(bpy.data.cameras),
'lamps': list(bpy.data.lamps),
'images': list(bpy.data.images),
'materials': list(bpy.data.materials),
'meshes': list(bpy.data.meshes),
'objects': list(bpy.data.objects),
'scenes': list(bpy.data.scenes),
'textures': list(bpy.data.textures),
})
# Resolve references
if state['version'] < Version('2.0'):
refmap = build_string_refmap(reference_inputs)
ref_default = 'INVALID'
else:
refmap = build_int_refmap(reference_inputs)
ref_default = -1
for ref in state['references']:
ref.source[ref.prop] = refmap.get((ref.blender_type, ref.blender_name), ref_default)
if ref.source[ref.prop] == ref_default:
print(
'Warning: {} contains an invalid reference to {}'
.format(ref.source, (ref.blender_type, ref.blender_name))
)
# Remove any temporary meshes
temp_mesh_collections = (
state['mod_meshes'].values(),
state['mod_meshes_obj'].values(),
[
shape_key_pair[1] for shape_key_pair
in itertools.chain.from_iterable(state['shape_keys'].values())
]
)
for mesh in itertools.chain(*temp_mesh_collections):
bpy.data.meshes.remove(mesh)
# Transform gltf data to binary
if settings['gltf_export_binary']:
json_data = json.dumps(gltf, sort_keys=True, check_circular=False).encode()
json_length = len(json_data)
json_pad = (' ' * (4 - json_length % 4)).encode()
json_pad = json_pad if len(json_pad) != 4 else b''
json_length += len(json_pad)
json_format = '<II{}s{}s'.format(len(json_data), len(json_pad))
chunks = [struct.pack(json_format, json_length, 0x4e4f534a, json_data, json_pad)]
if settings['buffers_embed_data']:
buffers = [data for path, data in state['files'].items() if path.endswith('.bin')]
# Get padded lengths
lengths = [len(buffer) for buffer in buffers]
lengths = [
length + ((4 - length % 4) if length % 4 != 0 else 0)
for length in lengths
]
chunks.extend([
struct.pack('<II{}s'.format(length), length, 0x004E4942, buffer)
for buffer, length in zip(buffers, lengths)
])
state['files'] = {
path: data for path, data in state['files'].items()
if not path.endswith('.bin')
}
version = 2
size = 12
for chunk in chunks:
size += len(chunk)
header = struct.pack('<4sII', b'glTF', version, size)
gltf = bytes(0).join([header, *chunks])
# Write secondary files
for path, data in state['files'].items():
with open(path, 'wb') as fout:
fout.write(data)
return gltf
import matplotlib.pyplot as plt
import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn import datasets
import numpy as np
from distutils.version import LooseVersion as Version
from sklearn import __version__ as sklearn_version
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import Perceptron
from matplotlib.colors import ListedColormap
import matplotlib.pyplot as plt
from libscikitlearn import plot_decision_regions
if Version(sklearn_version) < "0.18":
from sklearn.grid_search import train_test_split
else:
from sklearn.model_selection import train_test_split
iris = datasets.load_iris()
X = iris.data[:, [2, 3]]
y = iris.target
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=0)
sc = StandardScaler()
sc.fit(X_train)
X_train_std = sc.transform(X_train)
X_test_std = sc.transform(X_test)
def check(self, force=False, config=False):
common.log('UpdateChecker', 'check', 'force={}'.format(force))
# Load the settings
settings = Settings(config)
settings.load()
# If force=True, then definitely check
if force:
check_for_updates = True
else:
check_for_updates = False
# See if it's been 1 day since the last check
autoupdate_timestamp = settings.get('autoupdate_timestamp')
if autoupdate_timestamp:
last_checked = datetime.datetime.fromtimestamp(
autoupdate_timestamp)
now = datetime.datetime.now()
one_day = datetime.timedelta(days=1)
if now - last_checked > one_day:
check_for_updates = True
else:
check_for_updates = True
# Check for updates
if check_for_updates:
common.log('UpdateChecker', 'check', 'checking for updates')
# Download the latest-version file over Tor
try:
# User agent string includes OnionShare version and platform
user_agent = 'OnionShare {}, {}'.format(
common.get_version(), platform.system())
# If the update is forced, add '?force=1' to the URL, to more
# accurately measure daily users
path = '/latest-version.txt'
if force:
path += '?force=1'
if Version(self.onion.tor_version) >= Version('0.3.2.9'):
onion_domain = 'lldan5gahapx5k7iafb3s4ikijc4ni7gx5iywdflkba5y2ezyg6sjgyd.onion'
else:
onion_domain = 'elx57ue5uyfplgva.onion'
common.log('UpdateChecker', 'check',
'loading http://{}{}'.format(onion_domain, path))
(socks_address, socks_port) = self.onion.get_tor_socks_port()
socks.set_default_proxy(socks.SOCKS5, socks_address,
socks_port)
s = socks.socksocket()
s.settimeout(15) # 15 second timeout
s.connect((onion_domain, 80))
http_request = 'GET {} HTTP/1.0\r\n'.format(path)
http_request += 'Host: {}\r\n'.format(onion_domain)
http_request += 'User-Agent: {}\r\n'.format(user_agent)
http_request += '\r\n'
s.sendall(http_request.encode('utf-8'))
http_response = s.recv(1024)
latest_version = http_response[
http_response.find(b'\r\n\r\n'):].strip().decode('utf-8')
common.log(
'UpdateChecker', 'check',
'latest OnionShare version: {}'.format(latest_version))
except Exception as e:
common.log('UpdateChecker', 'check', '{}'.format(e))
self.update_error.emit()
raise UpdateCheckerCheckError
# Validate that latest_version looks like a version string
# This regex is: 1-3 dot-separated numeric components
version_re = r"^(\d+\.)?(\d+\.)?(\d+)$"
if not re.match(version_re, latest_version):
self.update_invalid_version.emit()
raise UpdateCheckerInvalidLatestVersion(latest_version)
# Update the last checked timestamp (dropping the seconds and milliseconds)
timestamp = datetime.datetime.now().replace(microsecond=0).replace(
second=0).timestamp()
# Re-load the settings first before saving, just in case they've changed since we started our thread
settings.load()
settings.set('autoupdate_timestamp', timestamp)
settings.save()
# Do we need to update?
update_url = 'https://github.com/micahflee/onionshare/releases/tag/v{}'.format(
latest_version)
installed_version = common.get_version()
if installed_version < latest_version:
self.update_available.emit(update_url, installed_version,
latest_version)
return
# No updates are available
self.update_not_available.emit()
class QRS(object):
'''Qlik Sense Repository Service REST API'''
VERSION_API = Version('2.1.0')
def __init__(self,
proxy='localhost',
port=4242,
vproxy='',
certificate=None,
verify=False,
userDirectory='internal',
userID='sa_repository',
verbosity=Verbose.INFO):
self.driver = _Controller(proxy, port, vproxy, certificate, verify,
userDirectory, userID, verbosity)
self.VERSION_SERVER = self.getServerVersion()
if self.VERSION_API > self.VERSION_SERVER:
raise Exception(
'<server version mismatch, API:{0} > Server:{1}'.format(
self.VERSION_API, self.VERSION_SERVER))
else:
if self.driver.verbose.isApi():
print(' Server version: {0}'.format(self.VERSION_SERVER))
def ping(self):
'''
@return: “Ping successful”, if there are no problems contacting the Qlik Sense Repository Service (QRS).
'''
return self.driver.call('GET', '/ssl/ping')
def getServerVersion(self):
'''
@Function: retrieve the server version
'''
return Version(
self.driver.call('GET', '/qrs/about').json().get('buildVersion'))
def getAbout(self):
'''
@Function: Get information on the Qlik Sense repository, including version, database provider, and whether the node is the central node of the site or not.
'''
return self.driver.get('/qrs/about')
def count(self, pType, pFilter=None):
'''
@Function: generic purpose call
@param pType: entity to count
@param pFilter: filter the entities before calculating the number of entities.
@return : integer from json response
'''
return self.driver.get('/qrs/{0}/count'.format(pType),
param={
'filter': pFilter
}).json()['value']
def getDescription(self, extended='False', method=None, outformat='JSON'):
'''@Function : List all paths available in the Qlik Sense Repository Service (QRS) API. Optionally, return extended information, endpoints that use a specific HTTP verb, or the return values in JSON format.
@param extended: If true, returns the following:
The type (if any) that needs to be included in the body.
The type of return value (if any).
If the endpoint is automatically generated or not.
@param method: If set to an HTTP verb (GET, PUT, POST, or DELETE), only endpoints that use the verb are returned.
@param outformat: If set to "JSON", the return value is given in JSON format.
'''
param = {
'extended': extended in ('True', 'true', True),
'method': method,
'format': outformat
}
return self.driver.get('/qrs/about/api/description', param).json()
def getEnum(self):
'''@Function: Get all enums that are used by the public part of the Qlik Sense Repository Service (QRS) API.
'''
return self.driver.get('/qrs/about/api/enums').json()
def AppDictAttributes(self, guid='full', key='name', attr='id'):
'''@Function: retrieve a mapping of apps attributes
@param pId: limmit the scope to the App {GUID}
@param key: the attribute to be the key
@param attr: the attribute value to retrieve
@return: dict(key:attr)
'''
apipath = '/qrs/app/{guid}'.format(guid=guid)
s = self.driver.get(apipath)
r = {}
if s.ok:
j = s.json()
if guid != "full":
r[j.get(key)] = j.get(attr)
else:
for x in j:
r[x.get(key)] = x.get(attr)
return (r)
def AppCopy(self, pId, name=None):
'''
@Function: Copy an existing app, identified by {id}. Optionally, provide a name for the copy.
@param pId: app identifier
@param name: Name of the app
'''
param = {'name': name}
return self.driver.post('/qrs/app/{id}/copy'.format(id=pId),
param).json()
def AppExport(self, pId, filename):
'''
@Function: Get an export qvf for an existing app, identified by {id}.
@param pId: app GUI
@param filename: target path filename
@return : stored application
'''
pUUID = uuid.uuid4()
r = self.driver.post('/qrs/app/{id}/export/{token}'.format(
id=pId, token=pUUID))
if r.ok:
app = self.driver.get(r.json()['downloadPath']).content
with open(filename, 'wb') as qvf:
return qvf.write(app)
def AppGet(self, pId='full', pFilter=None):
'''
@Function: retrieve App information
@param pId: App GUI
@param pFilter: filter the entities before calculating the number of entities.
@return : json response
'''
return self.driver.get('/qrs/app/{id}'.format(id=pId),
param={
'filter': pFilter
}).json()
def AppMigrate(self, pId):
'''
@Function: Migrate an app so that it can be used in the currently installed version of Qlik Sense. Normally, this is done automatically
@param pId: app identifier
'''
return self.driver.put('/qrs/app/{id}/migrate'.format(id=pId))
def AppUpload(self, filename, pName):
'''
@Function: Upload a filename.qvf into Central Node.
@param filename: target path filename
@param pName: target app name
@return : json response
'''
file_type = '' # passing empty file to get correct ContentType in header
with open(filename, 'rb') as qvf:
return self.driver.post(
'/qrs/app/upload?name={name}'.format(name=pName),
files=file_type,
data=qvf)
def AppReload(self, pId):
'''
@Function: Reload an app
@param pId: app identifier
'''
return self.driver.post('/qrs/app/{id}/reload'.format(id=pId))
def AppPublish(self, pId, stream=None):
'''
@Function: Publish an app to a stream
@param pId: app identifier
@param stream: stream identifier
'''
param = {'stream': stream}
return self.driver.put('/qrs/app/{id}/publish'.format(id=pId),
param).json()
def AppReplace(self, pId, rId):
'''
@Function: Replace an app, identified by {rId}, with the app identified by {pId}.
@param pId: identifier of app to be published
@param rId: identifier of app to be replaced
'''
return self.driver.put('/qrs/app/{id}/replace?app={appid}'.format(
id=pId, appid=rId))
def AppDelete(self, pId):
'''
@Function: Delete an app
@param pId: app identifier
'''
return self.driver.delete('/qrs/app/{id}/delete'.format(id=pId))
def StreamGet(self, pId='full', pFilter=None):
'''
@Function: retrieve Stream information
@param pId: Stream GUI
@param pFilter: filter the entities before calculating the number of entities.
@return : json response
'''
return self.driver.get('/qrs/stream/{id}'.format(id=pId),
param={
'filter': pFilter
}).json()
#TODO: generalizar, es lo mismo que AppDict
def UserDictAttributes(self, pUserID='full', key='name', attr='id'):
'''@Function: retrieve a mapping of user attributes
@param pUserID: limmit the scope to the User {UID}
@param key: the attribute to be the key
@param attr: the attribute value to retrieve
@return: dict(key:attr)
'''
apipath = '/qrs/user/{uid}'.format(uid=pUserID)
s = self.driver.get(apipath)
r = {}
if s.ok:
j = s.json()
if pUserID != "full":
r[j.get(key)] = j.get(attr)
else:
for x in j:
r[x.get(key)] = x.get(attr)
return (r)
def UserGet(self, pUserID='full', pFilter=None):
'''
@Function: retrieve user information
@param pUserID: User id
@param pFilter: filter the entities before calculating the number of entities.
@return : json response
'''
return self.driver.get('/qrs/user/{id}'.format(id=pUserID),
param={
'filter': pFilter
}).json()
def UserUpdate(self, pUserID, pData):
'''
@Function: update user information
@param pUserID: User id
@param pData: json with user information.
@return : json response
'''
return self.driver.put('/qrs/user/{id}'.format(id=pUserID), data=pData)
def UserDelete(self, pUserID):
'''
@Function: retrieve user information
@param pUserID: User id
@param pFilter: filter the entities before calculating the number of entities.
@return : json response
'''
return self.driver.delete('/qrs/user/{id}'.format(id=pUserID))
def SystemRules(self, pFilter=None):
'''
@Function: Get the system rules
'''
return self.driver.get('/qrs/systemrule/full', {
'filter': pFilter
}).json()
#TODO: Complete Rules methods
def PropertiesGet(self, pFilter=None):
'''
@Function: Get the system rules
'''
return self.driver.get('/qrs/custompropertydefinition/full', {
'filter': pFilter
}).json()
def url(self):
if _get_git_version() >= Version("2.7"):
return self.repo.git.remote.get_url(self._name).strip()
return self.repo.git.ls_remote(self._name, get_url=True).strip()
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradient_checker
from tensorflow.python.ops import manip_ops
from tensorflow.python.platform import test as test_lib
# pylint: disable=g-import-not-at-top
try:
from distutils.version import StrictVersion as Version
# numpy.roll for multiple shifts was introduced in numpy version 1.12.0
NP_ROLL_CAN_MULTISHIFT = Version(np.version.version) >= Version("1.12.0")
except ImportError:
NP_ROLL_CAN_MULTISHIFT = False
# pylint: enable=g-import-not-at-top
class RollTest(test_util.TensorFlowTestCase):
def _testRoll(self, np_input, shift, axis):
expected_roll = np.roll(np_input, shift, axis)
with self.test_session():
roll = manip_ops.roll(np_input, shift, axis)
self.assertAllEqual(roll.eval(), expected_roll)
def _testGradient(self, np_input, shift, axis):
with self.test_session():
inx = constant_op.constant(np_input.tolist())
def dependencies_ok(check_python_requirements: bool = True):
print("Checking system dependencies:")
unsatisfied_system_dependencies = []
for dep, (cmd, get_version, min_version) in system_dependencies.items():
try:
query = f"{dep} >= {min_version}"
with status(query):
success, out = output(cmd)
try:
version = get_version(out.decode("utf-8").strip())
print(f"[{version.rjust(8)}]".rjust(40 - len(query)),
end="")
except Exception:
raise ValueError("Could not parse version")
if not (Version(version) >= Version(min_version)):
raise RuntimeError(
f"Required {min_version}, found {version}")
except ValueError:
print(f"\n[!] Sorry, but our script could not parse the output of "
f'`{" ".join(cmd)}`; please file a bug, or see '
f"`check_app_environment.py`\n")
raise
except Exception as e:
unsatisfied_system_dependencies.append((dep, e))
if unsatisfied_system_dependencies:
print()
print("[!] Some system dependencies seem to be unsatisfied")
print()
print(" The failed checks were:")
print()
for (pkg, exc) in unsatisfied_system_dependencies:
cmd, get_version, min_version = system_dependencies[pkg]
print(f' - {pkg}: `{" ".join(cmd)}`')
print(" ", exc)
print()
print(" Please refer to https://docs.fritz.science "
"for installation instructions.")
print()
unsatisfied_python_requirements = []
if check_python_requirements:
print("\nChecking python requirements:")
for requirement in get_python_requirements():
try:
with status(requirement):
pkg_resources.require(requirement)
except (
pkg_resources.DistributionNotFound,
pkg_resources.VersionConflict,
) as e:
unsatisfied_python_requirements.append(e.report())
if unsatisfied_python_requirements:
print()
print(
"[!] Some python package requirements seem to be unsatisfied")
print()
print(" The failed requirements were:")
print()
for requirement in unsatisfied_python_requirements:
print(requirement)
print()
print(" Please refer to https://docs.fritz.science "
"for installation instructions.")
print()
if unsatisfied_system_dependencies:
return False
if unsatisfied_python_requirements:
attempt_resolving = input(
"Would you like to attempt to resolve unsatisfied "
"Python package requirements in your current environment? [y/N] "
).lower()
if attempt_resolving in ("y", "yes", "yup", "yea", "yeah"):
p = subprocess.run(
[
"python",
"-m",
"pip",
"install",
"-r",
pathlib.Path(__file__).parent.parent.absolute() /
"requirements.txt",
],
check=True,
)
if p.returncode != 0:
print("\nAttempt failed.\n")
return False
else:
return False
print("-" * 20)
return True
class QRS(object):
'''Qlik Sense Repository Service REST API'''
VERSION_API = Version(__version__)
def __init__(self, schema='https', proxy='localhost', port=4242, vproxy=None, certificate=None, verify=False, \
user={'userDirectory':'internal', 'userID':'sa_repository', 'password': None}, \
verbosity='INFO', logger='qsapi'):
schema, proxy, port = _Controller.normalize(schema, proxy, port,
certificate)
p_vproxy = {
'preffix': vproxy,
'path': '^/qrs/',
'template': '/{}/qrs/'
} if vproxy else None
self.driver = _Controller(schema, proxy, port, p_vproxy, certificate,
verify, user, verbosity, logger)
self.VERSION_SERVER = self.getServerVersion()
if self.VERSION_API > self.VERSION_SERVER:
raise Exception(
'<server version mismatch, API:{0} > Server:{1}'.format(
self.VERSION_API, self.VERSION_SERVER))
else:
self.driver.log.info('Server version: {0}'.format(
self.VERSION_SERVER))
def _toDict(self, response, uid='full', key='name', attr='id'):
r = {}
if response.ok:
j = response.json()
if uid != "full":
if isinstance(attr, str):
r[j.get(key)] = j.get(attr)
elif isinstance(attr, list):
ra = {}
for a in attr:
ra[a] = (j.get(a))
r[j.get(key)] = ra
else:
raise TypeError('attr argument must be a str or list')
else:
for x in j:
if isinstance(attr, str):
r[x.get(key)] = x.get(attr)
elif isinstance(attr, list):
ra = {}
for a in attr:
ra[a] = (x.get(a))
r[x.get(key)] = ra
else:
raise TypeError('attr argument must be a str or list')
return (r)
def ping(self):
'''
@return: "Ping successful", if there are no problems contacting the Qlik Sense Repository Service (QRS).
'''
return self.driver.call('GET', '/qrs/ssl/ping')
def getServerVersion(self):
'''
@Function: retrieve the server version
'''
return Version(
self.driver.call('GET', '/qrs/about').json().get('buildVersion'))
def getAbout(self):
'''
@Function: Get information on the Qlik Sense repository, including version, database provider, and whether the node is the central node of the site or not.
'''
return self.driver.get('/qrs/about').json()
def count(self, pType, pFilter=None):
'''
@Function: generic purpose call
@param pType: entity to count
@param pFilter: filter the entities before calculating the number of entities.
@return : integer from json response
'''
return self.driver.get('/qrs/{0}/count'.format(pType),
param={
'filter': pFilter
}).json()['value']
def getDescription(self, extended='False', method=None, outformat='JSON'):
'''@Function : List all paths available in the Qlik Sense Repository Service (QRS) API. Optionally, return extended information, endpoints that use a specific HTTP verb, or the return values in JSON format.
@param extended: If true, returns the following:
The type (if any) that needs to be included in the body.
The type of return value (if any).
If the endpoint is automatically generated or not.
@param method: If set to an HTTP verb (GET, PUT, POST, or DELETE), only endpoints that use the verb are returned.
@param outformat: If set to "JSON", the return value is given in JSON format.
'''
param = {
'extended': extended in ('True', 'true', True),
'method': method,
'format': outformat
}
return self.driver.get('/qrs/about/api/description', param).json()
def getEnum(self):
'''@Function: Get all enums that are used by the public part of the Qlik Sense Repository Service (QRS) API.
'''
return self.driver.get('/qrs/about/api/enums').json()
#=========================================================================================
def AppDictAttributes(self, guid='full', key='name', attr='id'):
'''@Function: retrieve a mapping of apps attributes
@param pId: limmit the scope to the App {GUID}
@param key: the attribute to be the key
@param attr: the attribute value to retrieve (single value or list)
@return: dict(key:attr)
'''
apipath = '/qrs/app/{guid}'.format(guid=guid)
return self._toDict(self.driver.get(apipath), guid, key, attr)
def AppCopy(self, pId, name=None):
'''
@Function: Copy an existing app, identified by {id}. Optionally, provide a name for the copy.
@param pId: app identifier
@param name: Name of the app
'''
param = {'name': name}
return self.driver.post('/qrs/app/{id}/copy'.format(id=pId),
param).json()
def AppExport(self, pId, filename=None):
'''
@Function: Get an export qvf for an existing app, identified by {id}.
@param pId: app GUI
@param filename: target path filename
@return : stored application
'''
file = filename if filename else pId + '.qvf'
if self.VERSION_SERVER < "17.0":
#DEPRECATED API since November-2017
self.driver.log.info('Server version: %s, using legacy API',
self.VERSION_SERVER)
r = self.driver.get('/qrs/app/{id}/export'.format(id=pId))
if r.ok:
r = self.driver.download(
'/qrs/download/app/{appId}/{TicketId}/{fileName}'.format(
appId=pId, TicketId=r.json()['value'], fileName=file),
file)
return (r)
#Current API method
r = self.driver.post('/qrs/app/{id}/export/{token}'.format(
id=pId, token=uuid.uuid4()))
if r.ok:
r = self.driver.download(r.json()['downloadPath'], file)
return (r)
def AppUpload(self, filename, pName, keepdata=None):
'''
@Function: Upload a filename.qvf into Central Node.
@param filename: target path filename
@param name: target app name
@param keepdata: Exclude the app data when uploading the app (when it is implemented)
'''
param = {'name': pName, 'keepdata': keepdata}
return self.driver.upload('/qrs/app/upload', filename, param)
def AppGet(self, pId='full', pFilter=None):
'''
@Function: retrieve App information
@param pId: App GUID
@param pFilter: filter the entities before calculating the number of entities.
@return : json response
'''
return self.driver.get('/qrs/app/{id}'.format(id=pId),
param={
'filter': pFilter
}).json()
def AppMigrate(self, pId):
'''
@Function: Migrate an app so that it can be used in the currently installed version of Qlik Sense.
Normally, this is done automatically
@param pId: app identifier
'''
return self.driver.put('/qrs/app/{id}/migrate'.format(id=pId))
def AppReload(self, pId):
'''
@Function: Reload an app
@param pId: app identifier
'''
return self.driver.post('/qrs/app/{id}/reload'.format(id=pId))
def AppPublish(self, pId, streamId, name=None):
'''
@Function: Publish an existing app, identified by {id}, to the stream identified by {streamid}.
@param pId: app identifier
@param streamId: stream identifier
@param name: optional alternate name
'''
param = {'stream': streamId, 'name': name}
return self.driver.put('/qrs/app/{id}/publish'.format(id=pId), param)
def AppUpdate(self, pId, pData):
'''
@Function: update App info referenced
@param pId: App GUID
'''
return self.driver.put('/qrs/app/{id}'.format(id=pId), data=pData)
def AppReplace(self, pId, pAppId):
'''
@Function: Replace an app, identified by {appid}, with the app identified by {id}.
@param pId: App GUID
@param pAppId: target App GUID
If the replaced app is published, only the sheets that were originally published with the app are replaced.
If the replaced app is not published, the entire app is replaced.
'''
param = {'app': pAppId}
return self.driver.put('/qrs/app/{id}/replace'.format(id=pId), param)
def AppDelete(self, pId):
'''
@Function: delete App referenced
@param pId: App GUID
'''
return self.driver.delete('/qrs/app/{id}'.format(id=pId))
#=========================================================================================
def StreamCreate(self, pName, pProperties=[], pTags=[], pUUID=None):
'''
@Function: create a Stream
@param pName: Stream Name
@param pUID: Stream UUID
@param pProperties: list of dict with properties definitions.
@param pTags: list of dict with tag definitions
@return : json response
'''
param = {'name': pName, 'customProperties': pProperties, 'tags': pTags}
if pUUID is not None:
param['id'] = pUUID
return self.driver.post('/qrs/stream', data=param).json()
def StreamGet(self, pId='full', pFilter=None):
'''
@Function: retrieve Stream information
@param pId: Stream GUID
@param pFilter: filter the entities before calculating the number of entities.
@return : json response
'''
return self.driver.get('/qrs/stream/{id}'.format(id=pId),
param={
'filter': pFilter
}).json()
def StreamUpdate(self, pId, pData):
'''
@Function: update Stream info referenced
@param pId: Stream GUID
'''
return self.driver.put('/qrs/stream/{id}'.format(id=pId), data=pData)
def StreamDelete(self, pId):
'''
@Function: delete Stream referenced
@param pId: Stream GUID
@return : json response
'''
return self.driver.delete('/qrs/stream/{id}'.format(id=pId))
def StreamDictAttributes(self, pStreamID='full', key='name', attr='id'):
'''@Function: retrieve a mapping of Stream attributes
@param pStreamID: limmit the scope to the Stream {UID}
@param key: the attribute to be the key
@param attr: the attribute value to retrieve (single value or list)
@return: dict(key:attr)
'''
apipath = '/qrs/stream/{uid}'.format(uid=pStreamID)
return self._toDict(self.driver.get(apipath), pStreamID, key, attr)
#=========================================================================================
def UserGet(self, pUserID='full', pFilter=None):
'''
@Function: retrieve user information
@param pUserID: User id
@param pFilter: filter the entities before calculating the number of entities.
@return : json response
'''
return self.driver.get('/qrs/user/{id}'.format(id=pUserID),
param={
'filter': pFilter
}).json()
def UserUpdate(self, pUserID, pData):
'''
@Function: update user information
@param pUserID: User id
@param pData: json with user information.
@return : json response
'''
return self.driver.put('/qrs/user/{id}'.format(id=pUserID), data=pData)
def UserDelete(self, pUserID):
'''
@Function: retrieve user information
@param pUserID: User id
@param pFilter: filter the entities before calculating the number of entities.
@return : json response
'''
return self.driver.delete('/qrs/user/{id}'.format(id=pUserID))
def UserDictAttributes(self, pUserID='full', key='name', attr='id'):
'''@Function: retrieve a mapping of user attributes
@param pUserID: limmit the scope to the User {UID}
@param key: the attribute to be the key
@param attr: the attribute value to retrieve (single value or list)
@return: dict(key:attr)
'''
apipath = '/qrs/user/{uid}'.format(uid=pUserID)
return self._toDict(self.driver.get(apipath), pUserID, key, attr)
#=========================================================================================
def TaskGet(self, pFilter=None):
'''
@Function: retrieve Task information
@param pFilter: filter the entities
@return : json response
'''
return self.driver.get('/qrs/task/full', param={
'filter': pFilter
}).json()
def TaskStart(self, taskid):
'''
@Function: Starts a task by id and waits until a slave starts to execute a task
@param taskid: taskid of the task to start
'''
return self.driver.post(
'/qrs/task/{taskid}/start'.format(taskid=taskid))
def TaskStartSynchronous(self, taskid):
'''
@Function: Starts a task by id and waits until a slave starts to execute a task
@param taskid: taskid of the task to start
'''
return self.driver.post(
'/qrs/task/{taskid}/start/synchronous'.format(taskid=taskid))
def TaskStartByName(self, taskname):
'''
@Function: Starts a task by name
@param taskname: Name of the task to start
'''
return self.driver.post('/qrs/task/start', param={'name': taskname})
def TaskStartMany(self, taskids):
'''
@Function: Starts multiple tasks
@param taskids: list of id's of the task to start
Sample list: ["6ca1c5f2-2742-44d5-8adf-d6cba3701a4e","965ca0cf-952f-4502-a65e-2a82e3de4803"]
'''
return self.driver.post('/qrs/task/start/many', data=taskids)
def TaskStartByNameSynchronous(self, taskname):
'''
@Function: Starts a task and waits until a slave starts to execute a task
@param taskname: Name of the task to start
'''
return self.driver.post('/qrs/task/start/synchronous',
param={'name': taskname})
def TaskStop(self, taskid):
'''
@Function: Stops a task
@param taskid: id of the task to stop
'''
return self.driver.post(
'/qrs/task/{taskid}/stop'.format(taskid=taskid))
def TaskStopMany(self, taskids):
'''
@Function: Stops multiple tasks
@param taskname: list of id's of the task to stop
Sample list: ["6ca1c5f2-2742-44d5-8adf-d6cba3701a4e","965ca0cf-952f-4502-a65e-2a82e3de4803"]
'''
return self.driver.post('/qrs/task/stop/many', data=taskids)
#=========================================================================================
def SystemRules(self, pFilter=None):
'''
@Function: Get the system rules
'''
return self.driver.get('/qrs/systemrule/full', {
'filter': pFilter
}).json()
def SystemRulesDictAttributes(self, pRuleID='full', key='name', attr='id'):
'''@Function: retrieve a mapping of rules attributes
@param pRuleID: limmit the scope to the Rule {UID}
@param key: the attribute to be the key
@param attr: the attribute value to retrieve (single value or list)
@return: dict(key:attr)
'''
apipath = '/qrs/systemrule/{uid}'.format(uid=pRuleID)
return self._toDict(self.driver.get(apipath), pRuleID, key, attr)
#=========================================================================================
def PropertiesGet(self, pFilter=None):
'''
@Function: Get the system rules
'''
return self.driver.get('/qrs/custompropertydefinition/full', {
'filter': pFilter
}).json()
import argparse
import os
import subprocess
import sys
from distutils.version import StrictVersion as Version
import buildconfig
from mozbuild.util import memoize
from mozpack.executables import (
get_type,
ELF,
MACHO,
)
STDCXX_MAX_VERSION = Version('3.4.16')
GLIBC_MAX_VERSION = Version('2.12')
HOST = {
'MOZ_LIBSTDCXX_VERSION':
buildconfig.substs.get('MOZ_LIBSTDCXX_HOST_VERSION'),
'platform': buildconfig.substs['HOST_OS_ARCH'],
'readelf': 'readelf',
'nm': 'nm',
}
TARGET = {
'MOZ_LIBSTDCXX_VERSION':
buildconfig.substs.get('MOZ_LIBSTDCXX_TARGET_VERSION'),
'platform': buildconfig.substs['OS_TARGET'],
'readelf':
def getServerVersion(self):
'''
@Function: retrieve the server version
'''
return Version(
self.driver.call('GET', '/qrs/about').json().get('buildVersion'))
def check(self, force=False):
self.common.log("UpdateChecker", "check", f"force={force}")
# Load the settings
settings = Settings(self.common)
settings.load()
# If force=True, then definitely check
if force:
check_for_updates = True
else:
check_for_updates = False
# See if it's been 1 day since the last check
autoupdate_timestamp = settings.get("autoupdate_timestamp")
if autoupdate_timestamp:
last_checked = datetime.datetime.fromtimestamp(
autoupdate_timestamp)
now = datetime.datetime.now()
one_day = datetime.timedelta(days=1)
if now - last_checked > one_day:
check_for_updates = True
else:
check_for_updates = True
# Check for updates
if check_for_updates:
self.common.log("UpdateChecker", "check", "checking for updates")
# Download the latest-version file over Tor
try:
# User agent string includes OnionShare version and platform
user_agent = f"OnionShare {self.common.version}, {self.common.platform}"
# If the update is forced, add '?force=1' to the URL, to more
# accurately measure daily users
path = "/latest-version.txt"
if force:
path += "?force=1"
if Version(self.onion.tor_version) >= Version("0.3.2.9"):
onion_domain = (
"lldan5gahapx5k7iafb3s4ikijc4ni7gx5iywdflkba5y2ezyg6sjgyd.onion"
)
else:
onion_domain = "elx57ue5uyfplgva.onion"
self.common.log("UpdateChecker", "check",
f"loading http://{onion_domain}{path}")
(socks_address, socks_port) = self.onion.get_tor_socks_port()
socks.set_default_proxy(socks.SOCKS5, socks_address,
socks_port)
s = socks.socksocket()
s.settimeout(15) # 15 second timeout
s.connect((onion_domain, 80))
http_request = f"GET {path} HTTP/1.0\r\n"
http_request += f"Host: {onion_domain}\r\n"
http_request += f"User-Agent: {user_agent}\r\n"
http_request += "\r\n"
s.sendall(http_request.encode("utf-8"))
http_response = s.recv(1024)
latest_version = (
http_response[http_response.find(b"\r\n\r\n"):].strip(
).decode("utf-8"))
self.common.log(
"UpdateChecker",
"check",
f"latest OnionShare version: {latest_version}",
)
except Exception as e:
self.common.log("UpdateChecker", "check", str(e))
self.update_error.emit()
raise UpdateCheckerCheckError
# Validate that latest_version looks like a version string
# This regex is: 1-3 dot-separated numeric components
version_re = r"^(\d+\.)?(\d+\.)?(\d+)$"
if not re.match(version_re, latest_version):
self.update_invalid_version.emit(latest_version)
raise UpdateCheckerInvalidLatestVersion(latest_version)
# Update the last checked timestamp (dropping the seconds and milliseconds)
timestamp = (datetime.datetime.now().replace(
microsecond=0).replace(second=0).timestamp())
# Re-load the settings first before saving, just in case they've changed since we started our thread
settings.load()
settings.set("autoupdate_timestamp", timestamp)
settings.save()
# Do we need to update?
update_url = f"https://github.com/micahflee/onionshare/releases/tag/v{latest_version}"
installed_version = self.common.version
if installed_version < latest_version:
self.update_available.emit(update_url, installed_version,
latest_version)
return
# No updates are available
self.update_not_available.emit()
version = getattr(mod, "__version__", 0) or getattr(mod, "VERSION", 0)
if Version(version) < min_ver:
print(
FAIL, "%s version %s or higher required, but %s installed." %
(lib, min_ver, version))
else:
print(OK, '%s version %s' % (pkg, version))
except ImportError:
print(FAIL, '%s not installed. %s' % (pkg, fail_msg))
return mod
# first check the python version
print('Using python in', sys.prefix)
print(sys.version)
pyversion = Version(sys.version)
if pyversion >= "3":
if pyversion < "3.4":
print(
FAIL,
"Python version 3.4 (or 2.7) is required, but %s is installed." %
sys.version)
elif pyversion >= "2":
if pyversion < "2.7":
print(
FAIL, "Python version 2.7 is required, but %s is installed." %
sys.version)
else:
print(FAIL, "Unknown Python version: %s" % sys.version)
print()
# Use legacy numpy printing. This fix is made to keep doctests functional.
# For more info, see https://github.com/scikit-image/scikit-image/pull/2935 .
# TODO: remove this workaround once minimal required numpy is set to 1.14.0
from distutils.version import LooseVersion as Version
import numpy as np
if Version(np.__version__) >= Version('1.14'):
np.set_printoptions(legacy='1.13')
# List of files that pytest should ignore
collect_ignore = [
"io/_plugins",
]
try:
import visvis
except ImportError:
collect_ignore.append("measure/mc_meta/visual_test.py")
# importing skimage.novice issues some warnings. Without these lines,
# pytest issues numerous warnings when crawling the package.
import warnings
warnings.filterwarnings('ignore', message='The `skimage.novice` module')
def __init__(self, version):
Version.__init__(self, version)
self.versionstr = version
self.isBranch = CraftVersion.isBranch_re.match(
self.versionstr) if version else None
