def insertScalar(self,
nHistoryId,
strMeasName,
strUnits,
val,
dtMeasTime=None):
if isFloat(val) or isInt(val) or isLong(val):
strStoredProc = "insertScalarFloat"
elif isString(val):
strStoredProc = "insertScalarString"
else:
raise TypeError("EXPECTED FLOAT, INT, OR STRING FOR VALUE")
# SANITIZE THE ARGUMENTS
if not (isInt(nHistoryId) or isLong(nHistoryId)):
raise TypeError("EXPECTED HISTORY ID INTEGER")
elif nHistoryId <= 0:
raise ValueError("EXPECTED HISTORY ID >= 0")
cursor = self._execStoredProc(strStoredProc, nHistoryId, \
strMeasName, strUnits, val, dtMeasTime)
# expect a scalar result
return self._checkResult(cursor)
def get_selected_pubs(conf_name=None, year=None):
"""
Get pub records from selected conferences in selected years.
"""
# Check parameter types
if isinstance(conf_name, basestring):
conf_name_str = "('%s')" % str(conf_name)
elif hasattr(
conf_name, '__iter__'
): # length of tuple should be larger than 1, otherwise use string
conf_name_str = str(tuple(conf_name))
else:
raise TypeError(
"Parameter 'conf_name' is of unsupported type. String or iterable needed."
)
if isinstance(year, basestring):
year_str = "(%s)" % str(year)
elif hasattr(
year, '__iter__'
): # length of tuple should be larger than 1, otherwise use string
year_str = str(tuple(year))
else:
raise TypeError(
"Parameter 'year' is of unsupported type. String or iterable needed."
)
year_cond = "selected_papers.year IN %s" % year_str if year else ""
conf_name_cond = "selected_papers.venue_abbr_name IN %s" % conf_name_str if conf_name else ""
if year_cond != '' and conf_name_cond != '':
where_cond = '%s AND %s' % (year_cond, conf_name_cond)
elif year_cond == '' and conf_name_cond != '':
where_cond = conf_name_cond
elif year_cond != '' and conf_name_cond == '':
where_cond = year_cond
else:
where_cond = None
rst = db.select(['selected_papers.id', 'paper_author_affils.author_id', 'paper_author_affils.affil_id'], \
['selected_papers', 'paper_author_affils'], join_on=['id', 'paper_id'], \
where=where_cond)
# re-pack data to this format: {paper_id: {author_id:[affil_id,],},}
pub_records = defaultdict()
for paper_id, author_id, affil_id in rst:
if pub_records.has_key(paper_id):
if pub_records[paper_id].has_key(author_id):
pub_records[paper_id][author_id].append(affil_id)
else:
pub_records[paper_id][author_id] = [affil_id]
else:
pub_records[paper_id] = {author_id: [affil_id]}
return pub_records
def _checkInputs(self):
if self._dtStart is not None:
if not isinstance(self._dtStart, datetime.datetime):
raise TypeError("start_date MUST BE datetime.datetime")
if self._dtStop is not None:
if not isinstance(self._dtStop, datetime.datetime):
raise TypeError("start_date MUST BE datetime.datetime")
def save(self):
if self.distribution is None:
raise TypeError('distribution')
if not isinstance(self.distribution, pd.DataFrame):
raise TypeError('distribution', allow=[pd.DataFrame])
self.distribution.to_csv(self.csv_save_path)
return self
def get_selected_docs(conf_name=None, year=None):
"""
Get pub records from selected conferences in selected years.
"""
# Check parameter types
if isinstance(conf_name, basestring):
conf_name_str = "('%s')" % str(conf_name)
elif hasattr(conf_name, '__iter__'):
if len(conf_name) == 0:
return []
elif len(conf_name) == 1:
conf_name_str = "(%s)" % conf_name[0]
else:
conf_name_str = str(tuple(conf_name))
else:
raise TypeError(
"Parameter 'conf_name' is of unsupported type. String or iterable needed."
)
if isinstance(year, basestring):
year_str = "(%s)" % str(year)
elif hasattr(year, '__iter__'):
if len(year) == 0:
return []
elif len(year) == 1:
year_str = "(%s)" % year[0]
else:
year_str = str(tuple(year))
else:
raise TypeError(
"Parameter 'year' is of unsupported type. String or iterable needed."
)
year_cond = "selected_papers.year IN %s" % year_str if year else ""
conf_name_cond = "selected_papers.venue_abbr_name IN %s" % conf_name_str if conf_name else ""
if year_cond != '' and conf_name_cond != '':
where_cond = '%s AND %s' % (year_cond, conf_name_cond)
elif year_cond == '' and conf_name_cond != '':
where_cond = conf_name_cond
elif year_cond != '' and conf_name_cond == '':
where_cond = year_cond
else:
where_cond = None
rst = db.select('id', 'selected_papers', where=where_cond)
return rst
def ldmsd_pid_cmd(xprt, port):
"""Return the command to get the pid of an ldmsd
@return: Command string <tt>pgrep -f 'ldmsd.+<xprt>:<port>'</tt>
"""
if xprt is None:
raise TypeError("xprt must be string, not 'NoneType")
if port is None:
raise TypeError("port must be number string or number, not 'NoneType'")
return "pgrep -f ^ldmsd.+{0}:{1}".format(xprt, port)
def kill_9_ldmsd_cmd(xprt, port):
"""Construct the command pkill -9 -f ^ldmsd.+<xprt>:<port>
@param xprt: ldmsd transport
@param port: ldmsd listener port
@return: String "pkill -9 -f ^ldmsd.+<xprt>:<port>"
"""
if xprt is None:
raise TypeError("xprt must be string, not 'NoneType")
if port is None:
raise TypeError("port must be number string or number, not 'NoneType'")
return "pkill -9 -f ^ldmsd.+{0}:{1}".format(xprt, port)
def _validate_sequence(self, sequence):
"""validates the given sequence value
"""
from oyProjectManager.models.sequence import Sequence
if sequence is None:
raise TypeError("Shot.sequence can not be None")
if not isinstance(sequence, Sequence):
raise TypeError("Shot.sequence should be an instance of "
"oyProjectManager.models.sequence.Sequence")
return sequence
def _validate_description(self, key, description):
"""validates the given description value
"""
if not isinstance(description, (str, unicode)):
raise TypeError("Asset.description should be an instance of "
"string or unicode")
return description
def create(self, POST=None, path=None, instance=None):
if not issubclass(self, forms.BaseForm):
raise TypeError(
"pirate_core.FormMixin should be used in multiple inheritance "
"in combination with a django.forms.BaseForm or subclass thereof."
)
Form = type(self.__name__, (self, ), {})
form_id = self.form_id(instance)
Form.base_fields["form_id"] = self.form_id_field(form_id)
if POST is not None:
if not self.post_has_id(POST, form_id):
form = Form() if instance is None else Form(instance=instance)
else:
form = Form(POST) if instance is None else Form(
POST, instance=instance)
if form.is_valid():
commit = False if path is None else True
form.save(commit=commit)
if path is not None:
e = HttpRedirectException(path)
e.form = form
raise e
else:
form = Form() if instance is None else Form(instance=instance)
return form
def _validate_input(self, inputs):
if isinstance(inputs, unicode):
inputs = str(inputs)
if not isinstance(inputs, dict) and not isinstance(inputs, str):
raise TypeError('''Your input value must be a dictionary or
string. Got: %s''' % inputs)
return inputs
def set(self, reference, cim_namespace, remote=None, properties={}, raw=False):
"""
Sets the properties of an instance using the properties argument.
@param reference: CIM reference response object
@type reference: L{Reference}
@param cim_namespace: Namespace of the CIM class
@type cim_namespace: String
@param remote: Remote configuration object
@type remote: L{Remote}
@param properties: The properties and values to set
@type properties: dict
@return: Instance object or Fault
@rtype: L{Response}
"""
# Get the keys reference for this instance (This needs to be set prior to the get)
if reference and\
isinstance(reference, Reference):
# Important:
# Form the query from the reference and not from the enumerated instance
pairs = []
for (key, value) in reference.items:
# Assemble the intermediate value since it is stored as a list
# TODO: What if the value is an array? How do we assemble an array for a get query?
value_ = value[0] if value else ''
value_ = value_ if value_ else ''
# JCT: used to wrap values with paces in double quotes - now we wrap the entire uri
safe_value = value_
pairs.append('%s=%s' % (key, safe_value))
# Get the query parameter for command construction
query = ','.join(pairs)
classname = reference.classname
# Construct the command
get_command = 'winrm put \"%s?%s\" ' % (reference.resource_uri, query)
get_command += self.remote_options(remote)
for k,v in properties.items():
get_command += '-k \"%s=%s\" ' % (k,v)
# Use the transport and execute the command
output = self.get_transport().execute(get_command)
if raw:
return output
else:
return self.parse(output)
else:
raise TypeError("reference argument must be of type Reference. Use values from enumerate_keys instead?")
# Generate a Fault
return Fault('WSManCLI',\
'Get is not supported on this instance or the reference is not set or it returned an error.',\
'WinRM provider for WSMAN returned an error.')
def _validate_name(self, key, name):
"""validates the given name value
"""
if name is None or not isinstance(name, (str, unicode)):
raise TypeError('Client.name should be a string or unicode, not '
'%s' % name.__class__.__name__)
return name
def _validates_number(self, key, number):
"""validates the given number value
"""
if not isinstance(number, (int, str, unicode)):
raise TypeError("Shot.number should be and instance of integer, "
"string or unicode")
# first convert it to a string
number = str(number)
# then format it
# remove anything which is not a number or letter
number = re.sub(r"[^0-9a-zA-Z\-_\ ]+", "", number)
number = re.sub(r"[\ ]+", "_", number)
#number = number.upper()
if number == "":
raise ValueError("Shot.number is not in good format, please "
"supply something like 1, 2, 3A, 10B")
# now check if the number is present for the current Sequence
shot_instance = db.session.query(Shot).\
filter(Shot.number==number).\
filter(Shot.sequence_id==self.sequence.id).\
first()
if shot_instance is not None:
raise ValueError("Shot.number already exists for the given "
"sequence please give a unique shot code")
return number
def _execStoredProc(self, strProcName, *args):
# generate the sql statement.
# other databases microsoft T-SQL will have a slightly diffent syntax.
# so I want to isolate the differences here so that I can easily
# deal with them later
strPrefix = "call"
lstArgs = []
for arg in args:
if isinstance(arg, str):
strArg = "'%s'" % arg
elif isinstance(arg, float) or isinstance(
arg, int) or isLong(arg) or isUnicode(arg):
strArg = str(arg)
elif arg is None:
strArg = "NULL"
else:
raise TypeError("UNHANDLED ARG TYPE: %s" % str(type(arg)))
lstArgs.append(strArg)
strSql = "%s %s(%s);" % (strPrefix, strProcName, ", ".join(lstArgs))
if self.__blnPrintSQL:
print(strSql)
# now execute the SQL statement and return the cursor
return self.queryCursor(strSql)
def get_venues_layer(self, papers):
"""
Returns the venues' ids and edges from publications to venues according
to the venues used in the publications.
"""
if hasattr(papers, '__iter__'):
if len(papers) == 0:
return [], []
else:
paper_str = ",".join(
["'%s'" % paper_id for paper_id in papers])
else:
raise TypeError(
"Parameter 'papers' is of unsupported type. Iterable needed.")
venues = set()
pub_venue_edges = list()
rows = db.select(fields=["id", "jornal_id", "conf_id"], table="papers", \
where="id IN (%s)"%paper_str)
for pub, jornal_id, conf_id in rows:
if jornal_id:
venues.add(jornal_id)
pub_venue_edges.append((pub, jornal_id, 1.0))
elif conf_id:
venues.add(conf_id)
pub_venue_edges.append((pub, conf_id, 1.0))
return list(venues), pub_venue_edges
def FQTN(phone_number, default_country_code):
"""
Assures phone numbers are in the form of a E164 Fully Qualified Telephone Number
without the leading + sign.
The alternative would be the Python port of Google's libphonenumber:
https://github.com/daviddrysdale/python-phonenumbers
"""
phone_number = phone_number.replace(' ', '').replace('-', '').replace(
'+', '').replace('/', '')
## number starting with 00 (so it's an international format)
if re.compile("^00[1-9][0-9]*$").match(phone_number):
return phone_number[2:]
## number starting with your country code (so it was already a FQTN):
if re.compile("^" + default_country_code +
"[1-9][0-9]*$").match(phone_number):
return phone_number
if re.compile("^0[1-9]*$").match(phone_number):
return default_country_code + phone_number[1:]
if re.compile("^[1-9]*$").match(phone_number):
return phone_number
raise TypeError("Couldn't parse this phone number: " + phone_number)
def references(self, reference, cim_namespace, remote=None, raw=False, uri_host=""):
"""
Do a references operation for an instance
@param reference: CIM reference(key) response object
@type reference: L{Reference}
@param cim_namespace: Namespace of the CIM class
@type cim_namespace: String
@param remote: Remote configuration object
@type remote: L{Remote}
@param uri_host: The host portion of the resource URI
@type uri_host: L{String}
@return: Response objects after enumerating the keys
@rtype: List of L{Reference} objects/ L{Fault}
"""
# Get the keys reference for this instance (This needs to be set prior to the get)
if reference and\
isinstance(reference, Reference):
pairs = []
for (key, value) in reference.items:
# Assemble the intermediate value since it is stored as a list
# TODO: What if the value is an array? How do we assemble an array for a get query?
value_ = value[0] if value else ''
value_ = value_ if value_ else ''
# JCT: used to wrap values with paces in double quotes - now we wrap the entire uri
safe_value = value_
pairs.append('%s=%s' % (key, safe_value))
# Get the query parameter for command construction
query = '+'.join(pairs)
classname = reference.classname
get_command = 'winrm e %s/wbem/wscim/1/cim-schema/2/* ' % (uri_host)
get_command += '-dialect:association -associations -filter:{object=%s?%s} ' % (classname, query)
get_command += self.remote_options(remote)
get_command += '-SkipCNcheck -SkipCAcheck -format:Pretty'
# Use the transport and execute the command
output = self.get_transport().execute(get_command)
if raw:
return output
else:
# Parse the output into a response object
return self.parse(output)
else:
raise TypeError("reference argument must be of type Reference. Use values from enumerate_keys instead?")
# Generate a Fault
return Fault('WinRM',\
'Get is not supported on this instance or the reference is not set or it returned an error.',\
'WinRM provider for WSMAN returned an error.')
def __init__(self, response_dict):
try:
self.StatusCode, self.StatusString = int(
response_dict['StatusCode']), response_dict['StatusString']
self.success = self.StatusCode == 200
except:
raise TypeError(RESPONSE_NOT_A_DICTIONARY % response_dict)
dict.__init__(self, response_dict)
def __init__(self, name, output, input, mat_level='P1'):
"""Constructor"""
if not ListType == type(input):
raise TypeError('input must be a list')
if not StringType == type(name):
raise TypeError('name must be a string')
if not IntType == type(output):
raise TypeError('output must be an integer')
if not StringType == type(mat_level):
raise TypeError('mat_level must be a string')
self._name = name
self._input = input
self._output = output
self._mat_level = mat_level
def extract(self):
if self.result is None:
raise TypeError('result')
return [
{
'property': binding['property']['value'],
'propertyLabel': binding['propertyLabel']['value']
} for binding in self.result['results']['bindings']
]
def fit(self, timeout=None, iterations=None):
# type: (float, int) -> None
"""
Fit the random forest distance to a set of labeled data by sampling and fitting
to pairwise constraints.
"""
# state/input checking
if self.fitted:
return
if not hasattr(self, 'X') or not hasattr(self, 'y'):
raise ValueError("Missing training data.")
if (not (isinstance(self.X, np.ndarray)
and isinstance(self.y, np.ndarray))):
raise TypeError(
'Training inputs and outputs must be numpy arrays.')
# store state in case of timeout
if hasattr(self, 'd'):
dtemp = self.d
else:
dtemp = None
rftemp = copy.deepcopy(self.rf)
# do fitting with timeout
with stopit.ThreadingTimeout(timeout) as to_ctx_mgr:
n = self.X.shape[0]
self.d = self.X.shape[1]
assert n > 0
assert self.d > 0
assert n == self.y.shape[0]
constraints = get_random_constraints(self.y,
int(self.class_cons / 3),
2 * int(self.class_cons / 3))
c1 = self.X[constraints[:, 0], :]
c2 = self.X[constraints[:, 1], :]
rfdfeat = np.empty(dtype=np.float32,
shape=(constraints.shape[0], self.d * 2))
rfdfeat[:, :self.d] = np.abs(c1 - c2)
rfdfeat[:, self.d:] = (c1 + c2) / 2
self.rf.fit(rfdfeat, constraints[:, 2])
self.fitted = True
# if we completed on time, return. Otherwise reset state and raise error.
if to_ctx_mgr.state == to_ctx_mgr.EXECUTED:
return
else:
self.d = dtemp
self.rf = rftemp
self.fitted = False
raise TimeoutError("RFD fitting timed out.")
def update(self, *args, **kwargs):
"""Overridden so our __setitem__ is not avoided."""
if args:
if len(args) > 1:
raise TypeError("update expected at most 1 arguments, got %d" % len(args))
other = dict(args[0])
for key in other:
self[key] = other[key]
for key in kwargs:
self[key] = kwargs[key]
def insertMeasurement(self, strName, strUnits):
# check the inputs
# expect both to be strings.
if not (isString(strName) and isString(strUnits)):
raise TypeError("EXPECTED STRING FOR NAME AND UNITS")
cursor = self._execStoredProc("insertMeasurement", strName, strUnits)
# expect a scalar result
return self._checkResult(cursor)
def process_bind_param(self, dq, engine):
if not dq:
#raise TypeError('IPV4 can not be None')
#FIX ME: this is a quick fix to accomodate Nullable field.
return None
dq = str(dq)
q = dq.split('.')
if len(q) != 4:
msg = "%r: IPv4 address invalid: should contain 4 bytes" % (dq)
raise TypeError(msg)
for x in q:
if 0 > int(x) > 255:
msg = (dq, " : bytes should be between 0 and 255")
raise TypeError(msg)
return dq_to_int(dq)
def _colToIdx(self, col):
if isinstance(col, str):
if col in self.header():
return self._header.index(col)
else:
raise ValueError("COULD NOT FIND COLUMN %s" % col)
elif isInt(col) or isLong(col):
return col
else:
raise TypeError("EXPECTED A STRING OR AN INT FOR COLUMN")
def _validate_description(self, key, description):
"""validates the given description value
"""
if description is None:
description = ""
if not isinstance(description, (str, unicode)):
raise TypeError("Shot.description should be an instance of str "
"or unicode")
return description
def get_relevant_topics(self, doc_topics, ntop=None, above=None):
"""
Get the most important topics for the given document by either:
* Taking the 'ntop' values if 'ntop' id provided or
* Taking all topics with contributions greater than 'above'.
"""
if ntop:
return np.argsort(doc_topics)[::-1][:ntop]
if above:
return np.where(doc_topics > above)[0]
raise TypeError("Arguments 'ntop' and 'above' cannot be both None.")
def read_image(filename):
"""
Read an image (in tiff, fits, png or jpg format) from a file.
:param filename: Path name of the input image.
:return: RGB or monochrome image.
"""
name, suffix = splitext(filename)
# Make sure files with extensions written in large print can be read as well.
suffix = suffix.lower()
# Case FITS format:
if suffix in ('.fit', '.fits'):
image = fits.getdata(filename)
# FITS output file from AS3 is 16bit depth file, even though BITPIX
# has been set to "-32", which would suggest "numpy.float32"
# https://docs.astropy.org/en/stable/io/fits/usage/image.html
# To process this data in PSS, do "round()" and convert numpy array to "np.uint16"
if image.dtype == '>f4':
image = image.round().astype(uint16)
# If color image, move axis to be able to process the content
if len(image.shape) == 3:
image = moveaxis(image, 0, -1).copy()
# Flip image horizontally to recover original orientation
image = flip(image, 0)
# Case other supported image formats:
elif suffix in ('.tiff', '.tif', '.png', '.jpg'):
input_image = imread(filename, IMREAD_UNCHANGED)
if input_image is None:
raise IOError(
"Cannot read image file. Possible cause: Path contains non-ascii characters"
)
# If color image, convert to RGB mode.
if len(input_image.shape) == 3:
image = cvtColor(input_image, COLOR_BGR2RGB)
else:
image = input_image
else:
raise TypeError(
"Attempt to read image format other than 'tiff', 'tif',"
" '.png', '.jpg' or 'fit', 'fits'")
return image
def produce(self, inputs, timeout=None, iterations=None):
# type: (Tuple[(Inputs, Inputs)], float, int) -> Outputs
"""
Compute the distance matrix between vector arrays inputs[0] and
inputs[1], yielding an output of shape n by m (where n and m are
the number of instances in inputs[0] and inputs[1] respectively).
Both inputs must match the dimensionality of the training data.
The same array may be input twice in order to generate an
(inverted) kernel matrix for use in clustering, etc.
Raises:
AssertError: thrown if the model has not yet been fit to data
TypeError: thrown if X or Y are not both numpy arrays
ValueError: thrown if the dimensionality of instance(s) in
X or Y is not equal to the dimensionality of the
training data (or the matrices are otherwise the wrong
shape)
"""
# first do assorted error checking and initialization
assert self.fitted == True
X = inputs[0]
Y = inputs[1]
if (not (isinstance(X, np.ndarray) and (isinstance(Y, np.ndarray)))):
raise TypeError('Both inputs must be numpy arrays.')
if (X.shape[1] != self.d or Y.shape[1] != self.d):
raise ValueError('Input has the wrong dimensionality.')
n1 = X.shape[0]
n2 = Y.shape[0]
# start timeout counter
with stopit.ThreadingTimeout(timeout) as to_ctx_mgr:
# compute distance from each instance in X to all instances in Y
dist = np.empty(dtype=np.float32, shape=(n1, n2))
for i in xrange(0, n1):
data = np.empty(dtype=np.float32, shape=(n2, self.d * 2))
data[:, :self.d] = np.abs(X[i, :] - Y)
data[:, self.d:] = (X[i, :] + Y) / 2
dist[i, :] = self.rf.predict(data)
# return distance
return dist
# if we did not finish in time, raise error.
if to_ctx_mgr.state != to_ctx_mgr.EXECUTED:
raise TimeoutError("RFD produce timed out.")
def __init__(self, name, needed, given):
TypeError.__init__(self, 'the command \'{0}\' takes exactly {1} argument{3} ({2} given)'.format(name, needed, given,'s' if needed > 1 else ''))