def create_introducer_clients(config, main_tub, _introducer_factory=None):
"""
Read, validate and parse any 'introducers.yaml' configuration.
:param _introducer_factory: for testing; the class to instantiate instead
of IntroducerClient
:returns: a list of IntroducerClient instances
"""
if _introducer_factory is None:
_introducer_factory = IntroducerClient
# we return this list
introducer_clients = []
introducers = config.get_introducer_configuration()
for petname, (furl, cache_path) in list(introducers.items()):
ic = _introducer_factory(
main_tub,
furl.encode("ascii"),
config.nickname,
str(allmydata.__full_version__),
str(_Client.OLDEST_SUPPORTED_VERSION),
partial(_sequencer, config),
cache_path,
)
introducer_clients.append(ic)
return introducer_clients
def _renderHTTP_exception(request, failure):
try:
text, code = humanize_failure(failure)
except:
log.msg("exception in humanize_failure")
log.msg("argument was %s" % (failure, ))
log.err()
text = str(failure)
code = None
if code is not None:
return _renderHTTP_exception_simple(request, text, code)
accept = request.getHeader("accept")
if not accept:
accept = "*/*"
if "*/*" in accept or "text/*" in accept or "text/html" in accept:
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
return template.renderElement(
request,
tags.html(
tags.head(tags.title(u"Exception"), ),
tags.body(FailureElement(failure), ),
),
)
# use plain text
traceback = failure.getTraceback()
return _renderHTTP_exception_simple(
request,
traceback,
http.INTERNAL_SERVER_ERROR,
)
def get_arg(
req,
argname,
default=None,
multiple=False
): # type: (IRequest, Union[bytes,str], Any, bool) -> Union[bytes,Tuple[bytes],Any]
"""Extract an argument from either the query args (req.args) or the form
body fields (req.fields). If multiple=False, this returns a single value
(or the default, which defaults to None), and the query args take
precedence. If multiple=True, this returns a tuple of arguments (possibly
empty), starting with all those in the query args.
:param TahoeLAFSRequest req: The request to consider.
:return: Either bytes or tuple of bytes.
"""
if isinstance(argname, str):
argname = argname.encode("utf-8")
if isinstance(default, str):
default = default.encode("utf-8")
results = []
if argname in req.args:
results.extend(req.args[argname])
argname_unicode = str(argname, "utf-8")
if req.fields and argname_unicode in req.fields:
value = req.fields[argname_unicode].value
if isinstance(value, str):
value = value.encode("utf-8")
results.append(value)
if multiple:
return tuple(results)
if results:
return results[0]
return default
def _generate_name(self):
"""Fold up ranges. e.g. 1,2,3,4 -> 1:4"""
port_start = None
line_start, prev_line = None, None
name = ''
def get_num(name):
return int(name[4:])
for port, line in self.line_pairs:
lineno = get_num(line)
if port_start is None:
port_start = get_num(port)
name += '+port{}.'.format(port_start)
if line_start is None:
line_start = lineno
name += str(line_start)
elif lineno != prev_line + 1:
# This line doesn't continue the existing range
line_start = lineno
if lineno != line_start:
name += ':{}'.format(prev_line)
name += '+port{}.{}'.format(port_start, line_start)
prev_line = lineno
if lineno != line_start:
name += ':{}'.format(prev_line)
name = name.replace('.0:7', '')
return name[1:]
def readdir(self, source, compressed=False, encoding='UTF-8'):
"""Iterates over a dir in s3 split on newline.
Yields line in file in dir.
"""
for f in self.listdir(source, keys=True):
for l in self.read(str(f), compressed=compressed, encoding=encoding):
yield l
def render_GET(self, req):
t = str(get_arg(req, b"t", b"").strip(), "ascii")
# t=info contains variable ophandles, so is not allowed an ETag.
FIXED_OUTPUT_TYPES = ["", "json", "uri", "readonly-uri"]
if not self.node.is_mutable() and t in FIXED_OUTPUT_TYPES:
# if the client already has the ETag then we can
# short-circuit the whole process.
si = self.node.get_storage_index()
if si and req.setETag(b'%s-%s' % (base32.b2a(si), t.encode("ascii") or b"")):
return b""
if not t:
# just get the contents
# the filename arrives as part of the URL or in a form input
# element, and will be sent back in a Content-Disposition header.
# Different browsers use various character sets for this name,
# sometimes depending upon how language environment is
# configured. Firefox sends the equivalent of
# urllib.quote(name.encode("utf-8")), while IE7 sometimes does
# latin-1. Browsers cannot agree on how to interpret the name
# they see in the Content-Disposition header either, despite some
# 11-year old standards (RFC2231) that explain how to do it
# properly. So we assume that at least the browser will agree
# with itself, and echo back the same bytes that we were given.
filename = get_arg(req, "filename", self.name) or "unknown"
d = self.node.get_best_readable_version()
d.addCallback(lambda dn: FileDownloader(dn, filename))
return d
if t == "json":
# We do this to make sure that fields like size and
# mutable-type (which depend on the file on the grid and not
# just on the cap) are filled in. The latter gets used in
# tests, in particular.
#
# TODO: Make it so that the servermap knows how to update in
# a mode specifically designed to fill in these fields, and
# then update it in that mode.
if self.node.is_mutable():
d = self.node.get_servermap(MODE_READ)
else:
d = defer.succeed(None)
if self.parentnode and self.name:
d.addCallback(lambda ignored:
self.parentnode.get_metadata_for(self.name))
else:
d.addCallback(lambda ignored: None)
d.addCallback(lambda md: _file_json_metadata(req, self.node, md))
return d
if t == "info":
return MoreInfo(self.node)
if t == "uri":
return _file_uri(req, self.node)
if t == "readonly-uri":
return _file_read_only_uri(req, self.node)
raise WebError("GET file: bad t=%s" % t)
def list_instruments(server=None, module=None):
"""Returns a list of info about available instruments.
May take a few seconds because it must poll hardware devices.
It actually returns a list of specialized dict objects that contain
parameters needed to create an instance of the given instrument. You can
then get the actual instrument by passing the dict to
:py:func:`~instrumental.drivers.instrument`.
>>> inst_list = get_instruments()
>>> print(inst_list)
[<NIDAQ 'Dev1'>, <TEKTRONIX 'TDS 3032'>, <TEKTRONIX 'AFG3021B'>]
>>> inst = instrument(inst_list[0])
Parameters
----------
server : str, optional
The remote Instrumental server to query. It can be an alias from your instrumental.conf
file, or a str of the form `(hostname|ip-address)[:port]`, e.g. '192.168.1.10:12345'. Is
None by default, meaning search on the local machine.
"""
if server is not None:
from . import remote
session = remote.client_session(server)
return session.list_instruments()
try:
import visa
try:
inst_list = list_visa_instruments()
except visa.VisaIOError:
inst_list = [] # Hide visa errors
except (ImportError, ConfigError):
inst_list = [] # Ignore if PyVISA not installed or configured
for mod_name in _acceptable_params:
if module and module not in mod_name:
continue
try:
log.info("Importing driver module '%s'", mod_name)
mod = import_module('.' + mod_name, __package__)
except Exception as e:
# Module not supported
log.info("Error when importing module %s: <<%s>>", mod_name,
str(e))
continue
try:
inst_list.extend(mod.list_instruments())
except AttributeError:
# Module doesn't have a list_instruments() function
continue
return inst_list
def listglob(self, glb, **kwargs):
""" Returns a list of the files matching a glob.
Name must be in the form of s3://bucket/glob
"""
r = []
regex = re.compile('[*?\[]')
for file in self.list(regex.split(glb, 1)[0], **kwargs):
if fnmatch.fnmatch(str(file), glb):
r.append(file)
return r
def render_PUT(self, req):
t = get_arg(req, b"t", b"").strip()
replace = parse_replace_arg(get_arg(req, "replace", "true"))
assert self.parentnode and self.name
if req.getHeader("content-range"):
raise WebError("Content-Range in PUT not yet supported",
http.NOT_IMPLEMENTED)
if not t:
return self.replace_me_with_a_child(req, self.client, replace)
if t == b"uri":
return self.replace_me_with_a_childcap(req, self.client, replace)
raise WebError("PUT to a file: bad t=%s" % str(t, "utf-8"))
def get_format(req, default="CHK"):
arg = get_arg(req, "format", None)
if not arg:
if boolean_of_arg(get_arg(req, "mutable", "false")):
return "SDMF"
return default
if arg.upper() == b"CHK":
return "CHK"
elif arg.upper() == b"SDMF":
return "SDMF"
elif arg.upper() == b"MDMF":
return "MDMF"
else:
raise WebError(
"Unknown format: %s, I know CHK, SDMF, MDMF" % str(arg, "ascii"),
http.BAD_REQUEST)
def render_POST(self, req):
t = get_arg(req, b"t", b"").strip()
replace = boolean_of_arg(get_arg(req, b"replace", b"true"))
if t == b"upload":
# like PUT, but get the file data from an HTML form's input field.
# We could get here from POST /uri/mutablefilecap?t=upload,
# or POST /uri/path/file?t=upload, or
# POST /uri/path/dir?t=upload&name=foo . All have the same
# behavior, we just ignore any name= argument
d = self.replace_me_with_a_formpost(req, self.client, replace)
else:
# t=mkdir is handled in DirectoryNodeHandler._POST_mkdir, so
# there are no other t= values left to be handled by the
# placeholder.
raise WebError("POST to a file: bad t=%s" % str(t, "utf-8"))
return handle_when_done(req, d)
def close(self):
"""Close the camera and release associated image memory.
Should be called when you are done using the camera. Alternatively, you
can use the camera as a context manager--see the documentation for
__init__.
"""
self._dev.ExitEvent(lib.SET_EVENT_SEQ)
self._dev.ExitEvent(lib.SET_EVENT_FRAME)
try:
self._dev.ExitCamera()
UC480_Camera._open_cameras.remove(self)
self._in_use = False
except Exception as e:
log.error("Failed to close camera")
log.error(str(e))
def convert_children_json(nodemaker, children_json):
"""I convert the JSON output of GET?t=json into the dict-of-nodes input
to both dirnode.create_subdirectory() and
client.create_directory(initial_children=). This is used by
t=mkdir-with-children and t=mkdir-immutable"""
children = {}
if children_json:
data = json.loads(children_json)
for (namex, (ctype, propdict)) in list(data.items()):
namex = str(namex)
writecap = to_bytes(propdict.get("rw_uri"))
readcap = to_bytes(propdict.get("ro_uri"))
metadata = propdict.get("metadata", {})
# name= argument is just for error reporting
childnode = nodemaker.create_from_cap(writecap,
readcap,
name=namex)
children[namex] = (childnode, metadata)
return children
def _getGitShaString(dist=None, sha=None):
"""If generic==True then returns empty __git_sha__ string
"""
shaStr = 'n/a'
if dist is not None:
proc = subprocess.Popen('git rev-parse --short HEAD',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd='.', shell=True)
repo_commit, _ = proc.communicate()
del proc # to get rid of the background process
if repo_commit:
shaStr = str(repo_commit.strip()) # remove final linefeed
else:
shaStr = 'n/a'
#this looks neater but raises errors on win32
# output = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).split()[0]
# if output:
# shaStr = output
return shaStr
def _getGitShaString(dist=None, sha=None):
"""If generic==True then returns empty __git_sha__ string
"""
shaStr = 'n/a'
if dist is not None:
proc = subprocess.Popen('git rev-parse --short HEAD',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd='.', shell=True)
repo_commit, _ = proc.communicate()
del proc # to get rid of the background process
if repo_commit:
shaStr = str(repo_commit.strip()) # remove final linefeed
else:
shaStr = 'n/a'
#this looks neater but raises errors on win32
# output = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).split()[0]
# if output:
# shaStr = output
return shaStr
def render_PUT(self, req):
t = get_arg(req, b"t", b"").strip()
replace = parse_replace_arg(get_arg(req, b"replace", b"true"))
offset = parse_offset_arg(get_arg(req, b"offset", None))
if not t:
if not replace:
# this is the early trap: if someone else modifies the
# directory while we're uploading, the add_file(overwrite=)
# call in replace_me_with_a_child will do the late trap.
raise ExistingChildError()
if self.node.is_mutable():
# Are we a readonly filenode? We shouldn't allow callers
# to try to replace us if we are.
if self.node.is_readonly():
raise WebError("PUT to a mutable file: replace or update"
" requested with read-only cap")
if offset is None:
return self.replace_my_contents(req)
if offset >= 0:
return self.update_my_contents(req, offset)
raise WebError("PUT to a mutable file: Invalid offset")
else:
if offset is not None:
raise WebError("PUT to a file: append operation invoked "
"on an immutable cap")
assert self.parentnode and self.name
return self.replace_me_with_a_child(req, self.client, replace)
if t == b"uri":
if not replace:
raise ExistingChildError()
assert self.parentnode and self.name
return self.replace_me_with_a_childcap(req, self.client, replace)
raise WebError("PUT to a file: bad t=%s" % str(t, "utf-8"))
def render_POST(self, req):
t = get_arg(req, b"t", b"").strip()
replace = boolean_of_arg(get_arg(req, b"replace", b"true"))
if t == b"check":
d = self._POST_check(req)
elif t == b"upload":
# like PUT, but get the file data from an HTML form's input field
# We could get here from POST /uri/mutablefilecap?t=upload,
# or POST /uri/path/file?t=upload, or
# POST /uri/path/dir?t=upload&name=foo . All have the same
# behavior, we just ignore any name= argument
if self.node.is_mutable():
d = self.replace_my_contents_with_a_formpost(req)
else:
if not replace:
raise ExistingChildError()
assert self.parentnode and self.name
d = self.replace_me_with_a_formpost(req, self.client, replace)
else:
raise WebError("POST to file: bad t=%s" % str(t, "ascii"))
return handle_when_done(req, d)
def convert_string_value_to_type_value(string_value, data_type):
"""Helper function to convert a given string to a given data type
:param str string_value: the string to convert
:param type data_type: the target data type
:return: the converted value
"""
from ast import literal_eval
try:
if data_type in (str, type(None)):
converted_value = str(string_value)
elif data_type == int:
converted_value = int(string_value)
elif data_type == float:
converted_value = float(string_value)
elif data_type == bool:
converted_value = bool(literal_eval(string_value))
elif data_type in (list, dict, tuple):
converted_value = literal_eval(string_value)
if type(converted_value) != data_type:
raise ValueError("Invalid syntax: {0}".format(string_value))
elif data_type == object:
try:
converted_value = literal_eval(string_value)
except (ValueError, SyntaxError):
converted_value = literal_eval('"' + string_value + '"')
elif isinstance(data_type, type): # Try native type conversion
converted_value = data_type(string_value)
elif isclass(data_type): # Call class constructor
converted_value = data_type(string_value)
else:
raise ValueError("No conversion from string '{0}' to data type '{0}' defined".format(
string_value, data_type.__name__))
except (ValueError, SyntaxError, TypeError) as e:
raise AttributeError("Can't convert '{0}' to type '{1}': {2}".format(string_value, data_type.__name__, e))
return converted_value
def render(self, req):
"""
Dispatch to a renderer for a particular format, as selected by a query
argument.
A renderer for the format given by the query argument matching
``formatArgument`` will be selected and invoked. render_HTML will be
used as a default if no format is selected (either by query arguments
or by ``formatDefault``).
:return: The result of the selected renderer.
"""
t = get_arg(req, self.formatArgument, self.formatDefault)
# It's either bytes or None.
if isinstance(t, bytes):
t = str(t, "ascii")
renderer = self._get_renderer(t)
result = renderer(req)
# On Python 3, json.dumps() returns Unicode for example, but
# twisted.web expects bytes. Instead of updating every single render
# method, just handle Unicode one time here.
if isinstance(result, str):
result = result.encode("utf-8")
return result
def unpack_extension(data):
d = {}
while data:
colon = data.index(b':')
key = data[:colon]
data = data[colon + 1:]
colon = data.index(b':')
number = data[:colon]
length = int(number)
data = data[colon + 1:]
value = data[:length]
assert data[length:length + 1] == b','
data = data[length + 1:]
d[str(key, "utf-8")] = value
# convert certain things to numbers
for intkey in ('size', 'segment_size', 'num_segments', 'needed_shares',
'total_shares'):
if intkey in d:
d[intkey] = int(d[intkey])
return d
def __repr__(self):
return str(self.to_dict()) #self.cmd
def _make_key():
private_key, _ = ed25519.create_signing_keypair()
# Config values are always unicode:
return str(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8")
def instrument(inst=None, **kwargs):
"""
Create any Instrumental instrument object from an alias, parameters,
or an existing instrument.
>>> inst1 = instrument('MYAFG')
>>> inst2 = instrument(visa_address='TCPIP::192.168.1.34::INSTR')
>>> inst3 = instrument({'visa_address': 'TCPIP:192.168.1.35::INSTR'})
>>> inst4 = instrument(inst1)
"""
alias = None
if inst is None:
params = {}
elif isinstance(inst, Instrument):
return inst
elif isinstance(inst, dict):
params = inst
elif isinstance(inst, str):
name = inst
params = conf.instruments.get(name, None)
if params is None:
# Try looking for the string in the output of list_instruments()
test_str = name.lower()
for inst_params in list_instruments():
if test_str in str(inst_params).lower():
params = inst_params
break
else:
alias = name
if params is None:
raise Exception("Instrument with alias `{}` not ".format(name) +
"found in config file")
params = params.copy() # Make sure we don't modify any existing dicts
params.update(kwargs)
if 'server' in params:
from . import remote
host = params['server']
session = remote.client_session(host)
return session.instrument(params)
if 'module' in params:
# We've already been given the name of the module
# SHOULD PROBABLY INTEGRATE THIS WITH THE OTHER CASE
try:
mod = import_module('.' + params['module'], __package__)
except Exception as e:
msg = (
"\n\nSpecified module '{}' could not be imported. Make sure you have all of "
"this driver module's dependencies installed.".format(
params['module']))
e.args = (e.args[0] + msg, ) + e.args[1:]
raise
try:
new_inst = mod._instrument(params)
except InstrumentTypeError:
raise Exception(
"Instrument is not compatible with the given module")
new_inst._alias = alias
# HACK to allow 'parent' modules to do special initialization of instruments
# We may get rid of this in the future by having each class's __init__ method directly
# handle params, getting rid of the _instrument() middleman.
parent_mod = import_module('.' + params['module'].rsplit('.', 1)[0],
__package__)
try:
parent_mod._init_instrument(new_inst, params)
except AttributeError:
pass
return new_inst
# Find the right type of Instrument to create
acceptable_modules = [
mod_name for mod_name, acc_params in list(_acceptable_params.items())
if _has_acceptable_params(acc_params, params)
]
for mod_name in acceptable_modules:
# Try to import module, skip it if optional deps aren't met
try:
log.info("Trying to import module '{}'".format(mod_name))
mod = import_module('.' + mod_name, __package__)
except Exception as e:
if len(acceptable_modules) == 1: raise
log.info("Module {} not supported, skipping".format(mod_name),
exc_info=e)
continue
# Try to create an instance of this instrument type
try:
log.info("Trying to create instrument using module '{}'".format(
mod_name))
new_inst = mod._instrument(params)
except AttributeError:
if len(acceptable_modules) == 1: raise
log.info(
"Module {} missing _instrument(), skipping".format(mod_name))
continue
except InstrumentTypeError:
if len(acceptable_modules) == 1: raise
log.info("Not the right type")
continue
except InstrumentNotFoundError:
if len(acceptable_modules) == 1: raise
log.info("Instrument not found")
continue
new_inst._alias = alias
# HACK to allow 'parent' modules to do special initialization of instruments
# We may get rid of this in the future by having each class's __init__ method directly
# handle params, getting rid of the _instrument() middleman.
parent_mod = import_module('.' + mod_name.rsplit('.', 1)[0],
__package__)
try:
parent_mod._init_instrument(new_inst, params)
except AttributeError:
pass
return new_inst
# If we reach this point, we haven't been able to create a valid instrument
if not acceptable_modules:
raise Exception(
"Parameters {} match no existing driver module".format(params))
else:
raise Exception("No instrument matching {} was found".format(params))
def spaces_to_nbsp(text):
return str(text).replace(u' ', u'\u00A0')
def list_visa_instruments():
"""Returns a list of info about available VISA instruments.
May take a few seconds because it must poll the network.
It actually returns a list of specialized dict objects that contain
parameters needed to create an instance of the given instrument. You can
then get the actual instrument by passing the dict to
:py:func:`~instrumental.drivers.instrument`.
>>> inst_list = get_visa_instruments()
>>> print(inst_list)
[<TEKTRONIX 'TDS 3032'>, <TEKTRONIX 'AFG3021B'>]
>>> inst = instrument(inst_list[0])
"""
import visa
instruments, skipped = [], []
prev_addr = 'START'
rm = visa.ResourceManager()
visa_list = rm.list_resources()
for addr in visa_list:
if not addr.startswith(prev_addr):
prev_addr = addr
try:
log.info("Opening VISA resource '{}'".format(addr))
i = rm.open_resource(addr, open_timeout=50, timeout=200)
except visa.VisaIOError as e:
# Could not create visa instrument object
skipped.append(addr)
log.info("Skipping this resource due to VisaIOError")
log.info(e)
continue
except socket.timeout:
skipped.append(addr)
log.info("Skipping this resource due to socket.timeout")
continue
try:
idn = i.ask("*IDN?")
log.info("*IDN? gives '{}'".format(idn.strip()))
try:
manufac, model, rest = idn.split(',', 2)
except ValueError as e:
skipped.append(addr)
log.info("Invalid response to IDN query")
log.info(str(e))
continue
module_name = _find_visa_inst_type(manufac, model)
params = _ParamDict("<{} '{}'>".format(manufac, model))
params['visa_address'] = addr
if module_name:
params.module = module_name
instruments.append(params)
except UnicodeDecodeError as e:
skipped.append(addr)
log.info(
"UnicodeDecodeError while getting IDN. Probably a non-Visa Serial device"
)
log.info(str(e))
continue
except visa.VisaIOError as e:
skipped.append(addr)
log.info("Getting IDN failed due to VisaIOError")
log.info(str(e))
continue
except socket.timeout:
skipped.append(addr)
log.info("Getting IDN failed due to socket.timeout")
continue
finally:
i.close()
return instruments
def should_create_intermediate_directories(req):
t = str(get_arg(req, "t", "").strip(), "ascii")
return bool(req.method in (b"PUT", b"POST")
and t not in ("delete", "rename", "rename-form", "check"))
def run(self, date=None):
# NOTE(willkg): This lets us have a dry-run app that doesn't run as
# a backfill app. In the normal case, this will get passed a date.
date = date or utc_now()
db_class = self.config.primary_destination.database_class
primary_database = db_class(self.config.primary_destination)
tx_class = self.config.primary_destination.transaction_executor_class
primary_transaction = tx_class(
self.config,
primary_database,
)
transactions = [primary_transaction]
db_class = self.config.secondary_destination.database_class
# The reason for checking if this is anything at all is
# because one way of disabling the secondary destination
# is to set the database_class to an empty string.
if db_class:
secondary_database = db_class(self.config.secondary_destination)
if secondary_database.config != primary_database.config:
# The secondary really is different from the first one.
# By default, if not explicitly set, it'll pick up the same
# resource values as the first one.
tx_class = (
self.config.secondary_destination
.transaction_executor_class
)
secondary_transaction = tx_class(
self.config,
secondary_database,
)
transactions.append(secondary_transaction)
target_date = (date - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
raw_adi_logs_pathname = os.path.join(
tempfile.gettempdir(),
"%s.raw_adi_logs.TEMPORARY%s" % (
target_date,
'.txt'
)
)
try:
with codecs.open(raw_adi_logs_pathname, 'w', 'utf-8') as f:
hive = pyhs2.connect(
host=self.config.hive_host,
port=self.config.hive_port,
authMechanism=self.config.hive_auth_mechanism,
user=self.config.hive_user,
password=self.config.hive_password,
database=self.config.hive_database,
# the underlying TSocket setTimeout() wants milliseconds
timeout=self.config.timeout * 1000
)
cur = hive.cursor()
query = self.config.query % target_date
cur.execute(query)
rows_written = 0
for row in cur:
if None in row:
continue
f.write(
"\t"
.join(
self.remove_control_characters(
urllib2.unquote(v)
).replace('\\', '\\\\')
if isinstance(v, basestring) else str(v)
for v in row
)
)
f.write("\n")
rows_written += 1
if not rows_written:
raise NoRowsWritten('hive yielded no rows to write')
self.config.logger.info(
'Wrote %d rows from doing hive query' % rows_written
)
self.persist_data(transactions, raw_adi_logs_pathname, target_date)
finally:
if os.path.isfile(raw_adi_logs_pathname):
os.remove(raw_adi_logs_pathname)
# This should not happen very often.
pass
full_version = "unknown"
branch = "unknown"
try:
# type ignored as it fails in CI
# (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972)
from allmydata._version import full_version, branch # type: ignore
except ImportError:
# We're running in a tree that hasn't run update_version, and didn't
# come with a _version.py, so we don't know what our full version or
# branch is. This should not happen very often.
pass
__appname__ = "tahoe-lafs"
# __full_version__ is the one that you ought to use when identifying yourself
# in the "application" part of the Tahoe versioning scheme:
# https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Versioning
__full_version__ = __appname__ + '/' + str(__version__)
# Install Python 3 module locations in Python 2:
from future import standard_library
standard_library.install_aliases()
# Monkey-patch 3rd party libraries:
from ._monkeypatch import patch
patch()
del patch
def humanize_exception(exc):
"""
Like ``humanize_failure`` but for an exception.
:param Exception exc: The exception to describe.
:return: See ``humanize_failure``.
"""
if isinstance(exc, EmptyPathnameComponentError):
return ("The webapi does not allow empty pathname components, "
"i.e. a double slash", http.BAD_REQUEST)
if isinstance(exc, ExistingChildError):
return ("There was already a child by that name, and you asked me "
"to not replace it.", http.CONFLICT)
if isinstance(exc, NoSuchChildError):
quoted_name = quote_output(exc.args[0],
encoding="utf-8",
quotemarks=False)
return ("No such child: %s" % quoted_name, http.NOT_FOUND)
if isinstance(exc, NotEnoughSharesError):
t = ("NotEnoughSharesError: This indicates that some "
"servers were unavailable, or that shares have been "
"lost to server departure, hard drive failure, or disk "
"corruption. You should perform a filecheck on "
"this object to learn more.\n\nThe full error message is:\n"
"%s") % str(exc)
return (t, http.GONE)
if isinstance(exc, NoSharesError):
t = ("NoSharesError: no shares could be found. "
"Zero shares usually indicates a corrupt URI, or that "
"no servers were connected, but it might also indicate "
"severe corruption. You should perform a filecheck on "
"this object to learn more.\n\nThe full error message is:\n"
"%s") % str(exc)
return (t, http.GONE)
if isinstance(exc, UnrecoverableFileError):
t = ("UnrecoverableFileError: the directory (or mutable file) could "
"not be retrieved, because there were insufficient good shares. "
"This might indicate that no servers were connected, "
"insufficient servers were connected, the URI was corrupt, or "
"that shares have been lost due to server departure, hard drive "
"failure, or disk corruption. You should perform a filecheck on "
"this object to learn more.")
return (t, http.GONE)
if isinstance(exc, MustNotBeUnknownRWError):
quoted_name = quote_output(exc.args[1], encoding="utf-8")
immutable = exc.args[2]
if immutable:
t = (
"MustNotBeUnknownRWError: an operation to add a child named "
"%s to a directory was given an unknown cap in a write slot.\n"
"If the cap is actually an immutable readcap, then using a "
"webapi server that supports a later version of Tahoe may help.\n\n"
"If you are using the webapi directly, then specifying an immutable "
"readcap in the read slot (ro_uri) of the JSON PROPDICT, and "
"omitting the write slot (rw_uri), would also work in this "
"case.") % quoted_name
else:
t = (
"MustNotBeUnknownRWError: an operation to add a child named "
"%s to a directory was given an unknown cap in a write slot.\n"
"Using a webapi server that supports a later version of Tahoe "
"may help.\n\n"
"If you are using the webapi directly, specifying a readcap in "
"the read slot (ro_uri) of the JSON PROPDICT, as well as a "
"writecap in the write slot if desired, would also work in this "
"case.") % quoted_name
return (t, http.BAD_REQUEST)
if isinstance(exc, MustBeDeepImmutableError):
quoted_name = quote_output(exc.args[1], encoding="utf-8")
t = ("MustBeDeepImmutableError: a cap passed to this operation for "
"the child named %s, needed to be immutable but was not. Either "
"the cap is being added to an immutable directory, or it was "
"originally retrieved from an immutable directory as an unknown "
"cap.") % quoted_name
return (t, http.BAD_REQUEST)
if isinstance(exc, MustBeReadonlyError):
quoted_name = quote_output(exc.args[1], encoding="utf-8")
t = (
"MustBeReadonlyError: a cap passed to this operation for "
"the child named '%s', needed to be read-only but was not. "
"The cap is being passed in a read slot (ro_uri), or was retrieved "
"from a read slot as an unknown cap.") % quoted_name
return (t, http.BAD_REQUEST)
if isinstance(exc, blacklist.FileProhibited):
t = "Access Prohibited: %s" % quote_output(
exc.reason, encoding="utf-8", quotemarks=False)
return (t, http.FORBIDDEN)
if isinstance(exc, WebError):
return (exc.text, exc.code)
if isinstance(exc, FileTooLargeError):
return ("FileTooLargeError: %s" % (exc, ),
http.REQUEST_ENTITY_TOO_LARGE)
return (str(exc), None)
def __repr__(self):
return str(self.to_dict()) #self.cmd
def sww2dem(
name_in,
name_out,
quantity=None, # defaults to elevation
reduction=None,
cellsize=10,
number_of_decimal_places=None,
NODATA_value=-9999.0,
easting_min=None,
easting_max=None,
northing_min=None,
northing_max=None,
verbose=False,
origin=None,
datum='WGS84',
block_size=None):
"""Read SWW file and convert to Digitial Elevation model format
(.asc or .ers)
Example (ASC):
ncols 3121
nrows 1800
xllcorner 722000
yllcorner 5893000
cellsize 25
NODATA_value -9999
138.3698 137.4194 136.5062 135.5558 ..........
The number of decimal places can be specified by the user to save
on disk space requirements by specifying in the call to sww2dem.
Also write accompanying file with same basename_in but extension .prj
used to fix the UTM zone, datum, false northings and eastings.
The prj format is assumed to be as
Projection UTM
Zone 56
Datum WGS84
Zunits NO
Units METERS
Spheroid WGS84
Xshift 0.0000000000
Yshift 10000000.0000000000
Parameters
The parameter quantity must be the name of an existing quantity or
an expression involving existing quantities. The default is
'elevation'. Quantity is not a list of quantities.
If reduction is given and it's an index, sww2dem will output the quantity at that time-step.
If reduction is given and it's a built in function (eg max, min, mean), then that
function is used to reduce the quantity over all time-steps. If reduction is not given,
reduction is set to "max" by default.
datum
format can be either 'asc' or 'ers'
block_size - sets the number of slices along the non-time axis to
process in one block.
"""
import sys
import types
from anuga.geometry.polygon import inside_polygon, outside_polygon
from anuga.abstract_2d_finite_volumes.util import \
apply_expression_to_dictionary
basename_in, in_ext = os.path.splitext(name_in)
basename_out, out_ext = os.path.splitext(name_out)
out_ext = out_ext.lower()
if in_ext != '.sww':
raise IOError('Input format for %s must be .sww' % name_in)
if out_ext not in ['.asc', '.ers']:
raise IOError('Format for %s must be either asc or ers.' % name_out)
false_easting = 500000
false_northing = 10000000
if quantity is None:
quantity = 'elevation'
if reduction is None:
reduction = max
if quantity in quantity_formula:
quantity = quantity_formula[quantity]
if number_of_decimal_places is None:
number_of_decimal_places = 3
if block_size is None:
block_size = DEFAULT_BLOCK_SIZE
assert (isinstance(block_size, (int, int, float)))
# Read sww file
if verbose:
log.critical('Reading from %s' % name_in)
log.critical('Output directory is %s' % name_out)
from anuga.file.netcdf import NetCDFFile
fid = NetCDFFile(name_in)
#Get extent and reference
x = num.array(fid.variables['x'], num.float)
y = num.array(fid.variables['y'], num.float)
volumes = num.array(fid.variables['volumes'], num.int)
if type(reduction) is not types.BuiltinFunctionType:
times = fid.variables['time'][reduction]
else:
times = fid.variables['time'][:]
number_of_timesteps = fid.dimensions['number_of_timesteps']
number_of_points = fid.dimensions['number_of_points']
if origin is None:
# Get geo_reference
# sww files don't have to have a geo_ref
try:
geo_reference = Geo_reference(NetCDFObject=fid)
except AttributeError as e:
geo_reference = Geo_reference() # Default georef object
xllcorner = geo_reference.get_xllcorner()
yllcorner = geo_reference.get_yllcorner()
zone = geo_reference.get_zone()
else:
zone = origin[0]
xllcorner = origin[1]
yllcorner = origin[2]
# FIXME: Refactor using code from Interpolation_function.statistics
# (in interpolate.py)
# Something like print swwstats(swwname)
if verbose:
log.critical('------------------------------------------------')
log.critical('Statistics of SWW file:')
log.critical(' Name: %s' % name_in)
log.critical(' Reference:')
log.critical(' Lower left corner: [%f, %f]' %
(xllcorner, yllcorner))
if type(reduction) is not types.BuiltinFunctionType:
log.critical(' Time: %f' % times)
else:
log.critical(' Start time: %f' % fid.starttime[0])
log.critical(' Extent:')
log.critical(' x [m] in [%f, %f], len(x) == %d' %
(num.min(x), num.max(x), len(x.flat)))
log.critical(' y [m] in [%f, %f], len(y) == %d' %
(num.min(y), num.max(y), len(y.flat)))
if type(reduction) is not types.BuiltinFunctionType:
log.critical(' t [s] = %f, len(t) == %d' % (times, 1))
else:
log.critical(' t [s] in [%f, %f], len(t) == %d' %
(min(times), max(times), len(times)))
log.critical(' Quantities [SI units]:')
# Comment out for reduced memory consumption
for name in ['stage', 'xmomentum', 'ymomentum']:
q = fid.variables[name][:].flatten()
if type(reduction) is not types.BuiltinFunctionType:
q = q[reduction * len(x):(reduction + 1) * len(x)]
if verbose:
log.critical(' %s in [%f, %f]' % (name, min(q), max(q)))
for name in ['elevation']:
q = fid.variables[name][:].flatten()
if verbose:
log.critical(' %s in [%f, %f]' % (name, min(q), max(q)))
# Get the variables in the supplied expression.
# This may throw a SyntaxError exception.
var_list = get_vars_in_expression(quantity)
# Check that we have the required variables in the SWW file.
missing_vars = []
for name in var_list:
try:
_ = fid.variables[name]
except KeyError:
missing_vars.append(name)
if missing_vars:
msg = (
"In expression '%s', variables %s are not in the SWW file '%s'" %
(quantity, str(missing_vars), name_in))
raise_(Exception, msg)
# Create result array and start filling, block by block.
result = num.zeros(number_of_points, num.float)
if verbose:
msg = 'Slicing sww file, num points: ' + str(number_of_points)
msg += ', block size: ' + str(block_size)
log.critical(msg)
for start_slice in range(0, number_of_points, block_size):
# Limit slice size to array end if at last block
end_slice = min(start_slice + block_size, number_of_points)
# Get slices of all required variables
q_dict = {}
for name in var_list:
# check if variable has time axis
if len(fid.variables[name].shape) == 2:
q_dict[name] = fid.variables[name][:, start_slice:end_slice]
else: # no time axis
q_dict[name] = fid.variables[name][start_slice:end_slice]
# Evaluate expression with quantities found in SWW file
res = apply_expression_to_dictionary(quantity, q_dict)
if len(res.shape) == 2:
new_res = num.zeros(res.shape[1], num.float)
for k in range(res.shape[1]):
if type(reduction) is not types.BuiltinFunctionType:
new_res[k] = res[reduction, k]
else:
new_res[k] = reduction(res[:, k])
res = new_res
result[start_slice:end_slice] = res
# Post condition: Now q has dimension: number_of_points
assert len(result.shape) == 1
assert result.shape[0] == number_of_points
if verbose:
log.critical('Processed values for %s are in [%f, %f]' %
(quantity, min(result), max(result)))
# Create grid and update xll/yll corner and x,y
# Relative extent
if easting_min is None:
xmin = min(x)
else:
xmin = easting_min - xllcorner
if easting_max is None:
xmax = max(x)
else:
xmax = easting_max - xllcorner
if northing_min is None:
ymin = min(y)
else:
ymin = northing_min - yllcorner
if northing_max is None:
ymax = max(y)
else:
ymax = northing_max - yllcorner
msg = 'xmax must be greater than or equal to xmin.\n'
msg += 'I got xmin = %f, xmax = %f' % (xmin, xmax)
assert xmax >= xmin, msg
msg = 'ymax must be greater than or equal to xmin.\n'
msg += 'I got ymin = %f, ymax = %f' % (ymin, ymax)
assert ymax >= ymin, msg
if verbose: log.critical('Creating grid')
ncols = int(old_div((xmax - xmin), cellsize)) + 1
nrows = int(old_div((ymax - ymin), cellsize)) + 1
# New absolute reference and coordinates
newxllcorner = xmin + xllcorner
newyllcorner = ymin + yllcorner
x = x + xllcorner - newxllcorner
y = y + yllcorner - newyllcorner
vertex_points = num.concatenate((x[:, num.newaxis], y[:, num.newaxis]),
axis=1)
assert len(vertex_points.shape) == 2
def calc_grid_values_old(vertex_points, volumes, result):
grid_points = num.zeros((ncols * nrows, 2), num.float)
for i in range(nrows):
if out_ext == '.asc':
yg = i * cellsize
else:
# this will flip the order of the y values for ers
yg = (nrows - i) * cellsize
for j in range(ncols):
xg = j * cellsize
k = i * ncols + j
grid_points[k, 0] = xg
grid_points[k, 1] = yg
# Interpolate
from anuga.fit_interpolate.interpolate import Interpolate
# Remove loners from vertex_points, volumes here
vertex_points, volumes = remove_lone_verts(vertex_points, volumes)
# export_mesh_file('monkey.tsh',{'vertices':vertex_points, 'triangles':volumes})
interp = Interpolate(vertex_points, volumes, verbose=verbose)
bprint = 0
# Interpolate using quantity values
if verbose: log.critical('Interpolating')
grid_values = interp.interpolate(bprint, result, grid_points).flatten()
outside_indices = interp.get_outside_poly_indices()
for i in outside_indices:
#print 'change grid_value',NODATA_value
grid_values[i] = NODATA_value
return grid_values
def calc_grid_values(vertex_points, volumes, result):
grid_points = num.zeros((ncols * nrows, 2), num.float)
for i in range(nrows):
if out_ext == '.asc':
yg = i * cellsize
else:
#this will flip the order of the y values for ers
yg = (nrows - i) * cellsize
for j in range(ncols):
xg = j * cellsize
k = i * ncols + j
grid_points[k, 0] = xg
grid_points[k, 1] = yg
grid_values = num.zeros(ncols * nrows, num.float)
eval_grid(nrows, ncols, NODATA_value, grid_points,
vertex_points.flatten(), volumes, result, grid_values)
return grid_values.flatten()
grid_values = calc_grid_values(vertex_points, volumes, result)
if verbose:
log.critical('Interpolated values are in [%f, %f]' %
(num.min(grid_values), num.max(grid_values)))
# Assign NODATA_value to all points outside bounding polygon (from interpolation mesh)
# P = interp.mesh.get_boundary_polygon()
# outside_indices = outside_polygon(grid_points, P, closed=True)
if out_ext == '.ers':
# setup ERS header information
grid_values = num.reshape(grid_values, (nrows, ncols))
header = {}
header['datum'] = '"' + datum + '"'
# FIXME The use of hardwired UTM and zone number needs to be made optional
# FIXME Also need an automatic test for coordinate type (i.e. EN or LL)
header['projection'] = '"UTM-' + str(zone) + '"'
header['coordinatetype'] = 'EN'
if header['coordinatetype'] == 'LL':
header['longitude'] = str(newxllcorner)
header['latitude'] = str(newyllcorner)
elif header['coordinatetype'] == 'EN':
header['eastings'] = str(newxllcorner)
header['northings'] = str(newyllcorner)
header['nullcellvalue'] = str(NODATA_value)
header['xdimension'] = str(cellsize)
header['ydimension'] = str(cellsize)
header['value'] = '"' + quantity + '"'
#header['celltype'] = 'IEEE8ByteReal' #FIXME: Breaks unit test
#Write
if verbose:
log.critical('Writing %s' % name_out)
import ermapper_grids
ermapper_grids.write_ermapper_grid(name_out, grid_values, header)
fid.close()
else:
#Write to Ascii format
#Write prj file
prjfile = basename_out + '.prj'
if verbose: log.critical('Writing %s' % prjfile)
prjid = open(prjfile, 'w')
prjid.write('Projection %s\n' % 'UTM')
prjid.write('Zone %d\n' % zone)
prjid.write('Datum %s\n' % datum)
prjid.write('Zunits NO\n')
prjid.write('Units METERS\n')
prjid.write('Spheroid %s\n' % datum)
prjid.write('Xshift %d\n' % false_easting)
prjid.write('Yshift %d\n' % false_northing)
prjid.write('Parameters\n')
prjid.close()
if verbose: log.critical('Writing %s' % name_out)
ascid = open(name_out, 'w')
ascid.write('ncols %d\n' % ncols)
ascid.write('nrows %d\n' % nrows)
ascid.write('xllcorner %d\n' % newxllcorner)
ascid.write('yllcorner %d\n' % newyllcorner)
ascid.write('cellsize %f\n' % cellsize)
ascid.write('NODATA_value %d\n' % NODATA_value)
#Get bounding polygon from mesh
#P = interp.mesh.get_boundary_polygon()
#inside_indices = inside_polygon(grid_points, P)
# change printoptions so that a long string of zeros in not
# summarized as [0.0, 0.0, 0.0, ... 0.0, 0.0, 0.0]
#printoptions = num.get_printoptions()
#num.set_printoptions(threshold=sys.maxint)
format = '%.' + '%g' % number_of_decimal_places + 'e'
for i in range(nrows):
if verbose and i % (old_div((nrows + 10), 10)) == 0:
log.critical('Doing row %d of %d' % (i, nrows))
base_index = (nrows - i - 1) * ncols
slice = grid_values[base_index:base_index + ncols]
num.savetxt(ascid, slice.reshape(1, ncols), format, ' ')
#Close
ascid.close()
fid.close()
return basename_out
def __repr__(self):
return str(self)
