def test_timestamp_from_string(self):
a_str = "2016-10-02T03:40:21.019793+00:00"
print(datastore_helpers.timestamp_from_string(a_str))
now = datastore_helpers.utcnow()
a_str = "last 2s"
expected = now - datetime.timedelta(seconds=2)
parsed = datastore_helpers.timestamp_from_string(a_str)
assert parsed.replace(microsecond=0) == expected.replace(microsecond=0)
a_str = "last 10m"
expected = now - datetime.timedelta(minutes=10)
parsed = datastore_helpers.timestamp_from_string(a_str)
assert parsed.replace(microsecond=0) == expected.replace(microsecond=0)
a_str = "last 10m,5s"
expected = now - datetime.timedelta(seconds=5, minutes=10)
parsed = datastore_helpers.timestamp_from_string(a_str)
assert parsed.replace(microsecond=0) == expected.replace(microsecond=0)
a_str = "last 10m,1h,5s"
expected = now - datetime.timedelta(hours=1, seconds=5, minutes=10)
parsed = datastore_helpers.timestamp_from_string(a_str)
assert parsed.replace(microsecond=0) == expected.replace(microsecond=0)
a_str = "last 10d,1h,5s"
expected = now - datetime.timedelta(hours=1, seconds=5, days=10)
parsed = datastore_helpers.timestamp_from_string(a_str)
assert parsed.replace(microsecond=0) == expected.replace(microsecond=0)
def test_timestamp_from_string(self):
a_str = "2016-10-02T03:40:21.019793+00:00"
print(datastore_helpers.timestamp_from_string(a_str))
now = datastore_helpers.utcnow()
a_str = "last 2s"
expected = now - datetime.timedelta(seconds=2)
parsed = datastore_helpers.timestamp_from_string(a_str)
assert parsed.replace(microsecond=0) == expected.replace(microsecond=0)
a_str = "last 10m"
expected = now - datetime.timedelta(minutes=10)
parsed = datastore_helpers.timestamp_from_string(a_str)
assert parsed.replace(microsecond=0) == expected.replace(microsecond=0)
a_str = "last 10m,5s"
expected = now - datetime.timedelta(seconds=5, minutes=10)
parsed = datastore_helpers.timestamp_from_string(a_str)
assert parsed.replace(microsecond=0) == expected.replace(microsecond=0)
a_str = "last 10m,1h,5s"
expected = now - datetime.timedelta(hours=1, seconds=5, minutes=10)
parsed = datastore_helpers.timestamp_from_string(a_str)
assert parsed.replace(microsecond=0) == expected.replace(microsecond=0)
a_str = "last 10d,1h,5s"
expected = now - datetime.timedelta(hours=1, seconds=5, days=10)
parsed = datastore_helpers.timestamp_from_string(a_str)
assert parsed.replace(microsecond=0) == expected.replace(microsecond=0)
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('timeseries')
helpers.reset_db()
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
cls.dataset = model.Package.get('annakarenina')
cls.resource = cls.dataset.resources[0]
cls.data = {
'resource_id': cls.resource.id,
'force': True,
'aliases': 'books3',
'fields': [{'id': 'author', 'type': 'text'},
{'id': 'published'}],
'records': [{'author': 'tolstoy1',
'published': '2005-03-01'},
{'author': 'tolstoy2'}
]
}
cls.data2 = {
'resource_id': cls.resource.id,
'force': True,
'method': 'insert',
'records': [{'author': 'tolstoy3',
'published': '2005-03-03'},
{'author': 'tolstoy4'}
]
}
cls.data3 = {
'resource_id': cls.resource.id,
'force': True,
'method': 'insert',
'records': [{'author': 'tolstoy5',
'published': '2005-03-05'},
{'author': 'tolstoy6'},
{'author': 'tolstoy7',
'published': '2005-03-05'}
]
}
cls.startdata = utcnow()
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_ts_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
cls.enddata = utcnow()
cls.startdata2 = utcnow()
time.sleep(2)
postparams = '%s=1' % json.dumps(cls.data2)
res = cls.app.post('/api/action/datastore_ts_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
cls.enddata2 = utcnow()
cls.startdata3 = utcnow()
time.sleep(2)
postparams = '%s=1' % json.dumps(cls.data3)
res = cls.app.post('/api/action/datastore_ts_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
cls.enddata3 = utcnow()
# Make an organization, because private datasets must belong to one.
cls.organization = tests.call_action_api(
cls.app, 'organization_create',
name='test_org',
apikey=cls.sysadmin_user.apikey)
cls.expected_records = [{u'published': u'2005-03-01T00:00:00',
u'_id': 1,
u'author': u'tolstoy1'},
{u'published': None,
u'_id': 2,
u'author': u'tolstoy2'},
{u'published': u'2005-03-03T00:00:00',
u'_id': 3,
u'author': u'tolstoy3'},
{u'published': None,
u'_id': 4,
u'author': u'tolstoy4'},
{u'published': u'2005-03-05T00:00:00',
u'_id': 5,
u'author': u'tolstoy5'},
{u'published': None,
u'_id': 6,
u'author': u'tolstoy6'},
{u'published': u'2005-03-05T00:00:00',
u'_id': 7,
u'author': u'tolstoy7'}]
cls.expected_records1 = [{u'published': u'2005-03-01T00:00:00',
u'_id': 1,
u'author': u'tolstoy1'},
{u'published': None,
u'_id': 2,
u'author': u'tolstoy2'}]
cls.expected_records12 = [{u'published': u'2005-03-01T00:00:00',
u'_id': 1,
u'author': u'tolstoy1'},
{u'published': None,
u'_id': 2,
u'author': u'tolstoy2'},
{u'published': u'2005-03-03T00:00:00',
u'_id': 3,
u'author': u'tolstoy3'},
{u'published': None,
u'_id': 4,
u'author': u'tolstoy4'}]
cls.expected_records23 = [{u'published': u'2005-03-03T00:00:00',
u'_id': 3,
u'author': u'tolstoy3'},
{u'published': None,
u'_id': 4,
u'author': u'tolstoy4'},
{u'published': u'2005-03-05T00:00:00',
u'_id': 5,
u'author': u'tolstoy5'},
{u'published': None,
u'_id': 6,
u'author': u'tolstoy6'},
{u'published': u'2005-03-05T00:00:00',
u'_id': 7,
u'author': u'tolstoy7'}]
cls.expected_records3 = [{u'published': u'2005-03-05T00:00:00',
u'_id': 5,
u'author': u'tolstoy5'},
{u'published': None,
u'_id': 6,
u'author': u'tolstoy6'},
{u'published': u'2005-03-05T00:00:00',
u'_id': 7,
u'author': u'tolstoy7'}]
engine = db._get_engine(
{'connection_url': pylons.config['ckan.datastore.write_url']}
)
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('timeseries')
helpers.reset_db()
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
cls.dataset = model.Package.get('annakarenina')
cls.resource = cls.dataset.resources[0]
cls.data = {
'resource_id':
cls.resource.id,
'force':
True,
'aliases':
'books3',
'fields': [{
'id': 'author',
'type': 'text'
}, {
'id': 'published'
}],
'records': [{
'author': 'tolstoy1',
'published': '2005-03-01'
}, {
'author': 'tolstoy2'
}]
}
cls.data2 = {
'resource_id':
cls.resource.id,
'force':
True,
'method':
'insert',
'records': [{
'author': 'tolstoy3',
'published': '2005-03-03'
}, {
'author': 'tolstoy4'
}]
}
cls.data3 = {
'resource_id':
cls.resource.id,
'force':
True,
'method':
'insert',
'records': [{
'author': 'tolstoy5',
'published': '2005-03-05'
}, {
'author': 'tolstoy6'
}, {
'author': 'tolstoy7',
'published': '2005-03-05'
}]
}
cls.startdata = utcnow()
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_ts_create',
params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
cls.enddata = utcnow()
cls.startdata2 = utcnow()
postparams = '%s=1' % json.dumps(cls.data2)
res = cls.app.post('/api/action/datastore_ts_create',
params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
cls.enddata2 = utcnow()
cls.startdata3 = utcnow()
postparams = '%s=1' % json.dumps(cls.data3)
res = cls.app.post('/api/action/datastore_ts_create',
params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
cls.enddata3 = utcnow()
# Make an organization, because private datasets must belong to one.
cls.organization = tests.call_action_api(
cls.app,
'organization_create',
name='test_org',
apikey=cls.sysadmin_user.apikey)
cls.expected_records = [{
u'published': u'2005-03-01T00:00:00',
u'_id': 1,
u'author': u'tolstoy1'
}, {
u'published': None,
u'_id': 2,
u'author': u'tolstoy2'
}, {
u'published': u'2005-03-03T00:00:00',
u'_id': 3,
u'author': u'tolstoy3'
}, {
u'published': None,
u'_id': 4,
u'author': u'tolstoy4'
}, {
u'published': u'2005-03-05T00:00:00',
u'_id': 5,
u'author': u'tolstoy5'
}, {
u'published': None,
u'_id': 6,
u'author': u'tolstoy6'
}, {
u'published': u'2005-03-05T00:00:00',
u'_id': 7,
u'author': u'tolstoy7'
}]
cls.expected_records1 = [{
u'published': u'2005-03-01T00:00:00',
u'_id': 1,
u'author': u'tolstoy1'
}, {
u'published': None,
u'_id': 2,
u'author': u'tolstoy2'
}]
cls.expected_records12 = [{
u'published': u'2005-03-01T00:00:00',
u'_id': 1,
u'author': u'tolstoy1'
}, {
u'published': None,
u'_id': 2,
u'author': u'tolstoy2'
}, {
u'published': u'2005-03-03T00:00:00',
u'_id': 3,
u'author': u'tolstoy3'
}, {
u'published': None,
u'_id': 4,
u'author': u'tolstoy4'
}]
cls.expected_records23 = [{
u'published': u'2005-03-03T00:00:00',
u'_id': 3,
u'author': u'tolstoy3'
}, {
u'published': None,
u'_id': 4,
u'author': u'tolstoy4'
}, {
u'published': u'2005-03-05T00:00:00',
u'_id': 5,
u'author': u'tolstoy5'
}, {
u'published': None,
u'_id': 6,
u'author': u'tolstoy6'
}, {
u'published': u'2005-03-05T00:00:00',
u'_id': 7,
u'author': u'tolstoy7'
}]
cls.expected_records3 = [{
u'published': u'2005-03-05T00:00:00',
u'_id': 5,
u'author': u'tolstoy5'
}, {
u'published': None,
u'_id': 6,
u'author': u'tolstoy6'
}, {
u'published': u'2005-03-05T00:00:00',
u'_id': 7,
u'author': u'tolstoy7'
}]
engine = db._get_engine(
{'connection_url': pylons.config['ckan.datastore.write_url']})
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
'%', '%%') for name in field_names] + ['"_full_text"','"_autogen_timestamp"'])
if method == _INSERT:
rows = []
for num, record in enumerate(records):
_validate_record(record, num, field_names)
row = []
for field in fields:
value = record.get(field['id'])
if value and field['type'].lower() == 'nested':
# a tuple with an empty second value
value = (json.dumps(value), '')
row.append(value)
row.append(_to_full_text(fields, record))
row.append(datastore_helpers.utcnow())
rows.append(row)
# another %s after to_tsvector(%s) for _autogen_timestamp
sql_string = u'''INSERT INTO "{res_id}" ({columns})
VALUES ({values}, to_tsvector(%s), %s);'''.format(
res_id=data_dict['resource_id'],
columns=sql_columns,
values=', '.join(['%s' for field in field_names])
)
try:
context['connection'].execute(sql_string, rows)
except sqlalchemy.exc.DataError as err:
raise InvalidDataError(
toolkit._("The data was invalid (for example: a numeric value "
for name in field_names] + ['"_full_text"', '"_autogen_timestamp"'])
if method == _INSERT:
rows = []
for num, record in enumerate(records):
_validate_record(record, num, field_names)
row = []
for field in fields:
value = record.get(field['id'])
if value and field['type'].lower() == 'nested':
# a tuple with an empty second value
value = (json.dumps(value), '')
row.append(value)
row.append(_to_full_text(fields, record))
row.append(datastore_helpers.utcnow())
rows.append(row)
# another %s after to_tsvector(%s) for _autogen_timestamp
sql_string = u'''INSERT INTO "{res_id}" ({columns})
VALUES ({values}, to_tsvector(%s), %s);'''.format(
res_id=data_dict['resource_id'],
columns=sql_columns,
values=', '.join(['%s' for field in field_names]))
try:
context['connection'].execute(sql_string, rows)
except sqlalchemy.exc.DataError as err:
raise InvalidDataError(
toolkit._(
"The data was invalid (for example: a numeric value "