def test_stats(): table = (('foo', 'bar', 'baz'), ('A', 1, 2), ('B', '2', '3.4'), (u'B', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None)) result = stats(table, 'bar') assert result['min'] == 1.0 assert result['max'] == 3.0 assert result['sum'] == 6.0 assert result['count'] == 3 assert result['errors'] == 2 assert result['mean'] == 2.0
def test_stats(): table = (('foo', 'bar', 'baz'), ('A', 1, 2), ('B', '2', '3.4'), ('B', '3', '7.8', True), ('D', 'xyz', 9.0), ('E', None)) result = stats(table, 'bar') assert result['min'] == 1.0 assert result['max'] == 3.0 assert result['sum'] == 6.0 assert result['count'] == 3 assert result['errors'] == 2 assert result['mean'] == 2.0
def test_stats(): table = ( ("foo", "bar", "baz"), ("A", 1, 2), ("B", "2", "3.4"), (u"B", u"3", u"7.8", True), ("D", "xyz", 9.0), ("E", None), ) result = stats(table, "bar") assert result["min"] == 1.0 assert result["max"] == 3.0 assert result["sum"] == 6.0 assert result["count"] == 3 assert result["errors"] == 2 assert result["mean"] == 2.0
def upload_scores(self, tbl, config, url_type, id_type='vanid', email=None, auto_approve=True, approve_tolerance=.1, **url_kwargs): """ Upload scores. Use to create or overwrite scores. Multiple score loads should be configured in a single call. [1]_ `Args:` tbl: object A parsons.Table object. The table must contain the scores and first column in the table must contain the primary key (e.g. vanid). config: list The score configuration. A list of dictionaries in which you specify the following .. list-table:: :widths: 20 80 :header-rows: 0 * - ``score_column`` - The name of the column where the score is housed. * - ``score_id`` - The score slot id. Example: .. highlight:: python .. code-block:: python [{'score1_id' : int, score1_column': str} {'score2_id' : int, score2_column': str}] url_type: str The cloud file storage to use to post the file (``S3`` or ``GCS``). See :ref:`Cloud Storage <cloud-storage>` for more details. email: str An email address to send job load status updates. auto_approve: boolean If the scores are within the expected tolerance of deviation from the average values provided, then score will be automatically approved. approve_tolderance: float The deviation from the average scores allowed in order to automatically approve the score. Maximum of .1. **url_kwargs: kwargs Arguments to configure your cloud storage url type. See :ref:`Cloud Storage <cloud-storage>` for more details. `Returns:` int The score load job id. .. [1] NGPVAN asks that you load multiple scores in a single call to reduce the load on their servers. """ # Move to cloud storage file_name = str(uuid.uuid1()) url = cloud_storage.post_file(tbl, url_type, file_path=file_name + '.zip', **url_kwargs) logger.info(f'Table uploaded to {url_type}.') # Generate shell request json = { "description": 'A description', "file": { "columnDelimiter": 'csv', "columns": [{ 'name': c } for c in tbl.columns], "fileName": file_name + '.csv', "hasHeader": "True", "hasQuotes": "False", "sourceUrl": url }, "actions": [] } # Configure each score for i in config: action = { "actionType": "score", "personIdColumn": tbl.columns[0], "personIdType": id_type, "scoreColumn": i['score_column'], "scoreId": i['score_id'] } if auto_approve: average = petl.stats(tbl.table, i['score_column']).mean action['approvalCriteria'] = { "average": average, "tolerance": approve_tolerance } json['actions'].append(action) # Add email listener if email: json['listeners'] = [{"type": "EMAIL", 'value': email}] # Upload scores r = self.connection.post_request('fileLoadingJobs', json=json) logger.info(f"Scores job {r['jobId']} created.") return r['jobId']
from __future__ import division, print_function, absolute_import # limits() ########## import petl as etl table = [['foo', 'bar'], ['a', 1], ['b', 2], ['b', 3]] minv, maxv = etl.limits(table, 'bar') minv maxv # stats() ######### import petl as etl table = [['foo', 'bar', 'baz'], ['A', 1, 2], ['B', '2', '3.4'], [u'B', u'3', u'7.8', True], ['D', 'xyz', 9.0], ['E', None]] etl.stats(table, 'bar')