def delete_multi(self, owner_id, report_id, tags, min_report_instance_id,
max_report_instance_id, limit, update_counters,
use_insertion_datetime):
columns = ['report_instance_id', 'all_tags_repr', 'day']
if update_counters:
columns.append('input_string')
ris = self.select_multi(report_id, tags, min_report_instance_id,
max_report_instance_id, columns, 'asc', limit)
log.info('Selected %d report instances to delete', len(ris))
if use_insertion_datetime:
days = util.uniq_sameorder(ri['day'] for ri in ris)
rim_rows = c.cass.execute(
"""SELECT report_instance_id, inserted
FROM mqe.report_instance_metadata
WHERE report_id=? AND day IN ?
AND report_instance_id > ? AND report_instance_id < ?""",
[
report_id, days, min_report_instance_id,
max_report_instance_id
])
rim_rows = list(rim_rows)
assert len(rim_rows) == len(ris)
ri_by_rid = {ri['report_instance_id']: ri for ri in ris}
from_dt = util.datetime_from_uuid1(min_report_instance_id)
to_dt = util.datetime_from_uuid1(max_report_instance_id)
for rim in rim_rows:
if not (from_dt <= rim['inserted'] <= to_dt):
del ri_by_rid[rim['report_instance_id']]
ris = ri_by_rid.values()
return self._delete_ris(owner_id, report_id, tags, ris,
update_counters)
def process_events_csv(self, filename):
with open(filename) as f:
csvreader = csv.DictReader(f)
counter = 0
batch = []
for row in csvreader:
# extract time from UUID for indexing
event_uuid = uuid.UUID(row['event_id'])
event_dt = datetime_from_uuid1(event_uuid)
#row['started_at'] = event_dt.strftime('%Y-%m-%d %H:%M:%S.%f')
row['started_at'] = datetime_from_uuid1(event_uuid)
# add cluster name to doc
row['cluster'] = self.cluster_name
batch.append({
'_index': self.es_index_name,
'_type': self.es_event_doc_type,
'_parent': row['session_id'],
'_routing': row['session_id'],
'_id': row['event_id'],
'_source': row
})
if len(batch) >= self.args.batch_size:
counter = counter + 1
r = helpers.bulk(self.es, batch)
print(counter, r)
batch = []
def day_text(dt_or_uuid):
if isinstance(dt_or_uuid, uuid.UUID):
dt = util.datetime_from_uuid1(dt_or_uuid)
else:
dt = dt_or_uuid
if dt.year < 2000:
return '000000'
return dt.strftime('%y%m%d')
def to_object(self):
return {
'chat_id': self.chat_id,
'message_id': str(self.message_id),
'author_id': self.author_id,
'time': datetime_from_uuid1(self.message_id).isoformat(),
'type': self.type,
'text': self.text,
'asset_name': self.asset_name,
'meta_data': self.meta_data
}
def test_times_from_uuid1(self):
node = uuid.getnode()
now = time.time()
u = uuid.uuid1(node, 0)
t = util.unix_time_from_uuid1(u)
self.assertAlmostEqual(now, t, 3)
dt = util.datetime_from_uuid1(u)
t = calendar.timegm(dt.timetuple()) + dt.microsecond / 1e6
self.assertAlmostEqual(now, t, 3)
def test_times_from_uuid1(self):
node = uuid.getnode()
now = time.time()
u = uuid.uuid1(node, 0)
t = util.unix_time_from_uuid1(u)
self.assertAlmostEqual(now, t, 2)
dt = util.datetime_from_uuid1(u)
t = calendar.timegm(dt.timetuple()) + dt.microsecond / 1e6
self.assertAlmostEqual(now, t, 2)
def main():
args = parse_arguments()
ArgUtil.setArgs(args)
newMostResentSnapshot = min_uuid_from_time(
datetime.datetime.now().timestamp())
prevSnapshot = parsePrevSnapshot()
print('prevSnapshot:' + str(datetime_from_uuid1(prevSnapshot)))
print('newMostResentSnapshot:' +
str(datetime_from_uuid1(newMostResentSnapshot)))
applicationsPath = args.applicationIdsPath
applicationUUIIdsFromFile = parseApplicationUUIDsFromFile(applicationsPath)
if args.verbose:
appIdsStringList = [
str(appUUID) for appUUID in applicationUUIIdsFromFile
]
print('Application UUIDs from file: ' + str(appIdsStringList))
contactPoints = args.contactPoints
cluster = Cluster(contactPoints)
totalStatusDocumentSplited = 0
totalResponsesCreated = 0
try:
session = cluster.connect()
print('connecting to cluster: [' + ', '.join(contactPoints) + '] ...')
session.execute('use unifiedpush_server')
for appUUID in applicationUUIIdsFromFile:
totalResponsesCreated, totalStatusDocumentSplited = \
workOnApplication(appUUID, session, totalResponsesCreated, totalStatusDocumentSplited, prevSnapshot)
updateMostResentSnapshot(newMostResentSnapshot)
finally:
print('Total number of STATUS document found: ' +
str(totalStatusDocumentSplited))
print('Total number of RESPONSE document created: ' +
str(totalResponsesCreated))
print('closing cluster connections...')
cluster.shutdown()
async def get_user_info(user_id: int):
response_status = 200
try:
future = db.query("select id, toTimestamp(time), rollcall_score, class_activity, discipline, total_quizzes_avg FROM users_info where id=? allow filtering", [user_id])
user_ = future.result()[0]
user = User.objects(id=user_id).allow_filtering()[0]
user.time = datetime_from_uuid1(user.time)
except Exception as e:
print(f"[Exception] ::: {e}")
response_status = 500
user_ = []
user = []
return {"db.query()": user_, "schema": user, "response_status": response_status}
async def get_user_raw_position(user_id: int):
response_status = 200
try:
future = db.query(f"select user_id, toTimestamp(time), position_raw from users_position_raw where user_id = ? allow filtering;", [user_id])
position_ = future.result()[0]
position = PositionRaw.objects(user_id=user_id).allow_filtering()[0]
position.time = datetime_from_uuid1(position.time)
except Exception as e:
print(f"[Exception] ::: {e}")
response_status = 500
position_ = []
position = []
return {"db.query()": position_, "schema": position, "response_status": response_status}
def testTimeuuid(cql, test_keyspace):
with create_table(cql, test_keyspace,
"(k int, t timeuuid, PRIMARY KEY(k, t))") as table:
assert_invalid_throw(
cql, table, SyntaxException,
"INSERT INTO %s (k, t) VALUES (0, 2012-11-07 18:18:22-0800)")
for i in range(4):
execute(cql, table, "INSERT INTO %s (k, t) VALUES (0, now())")
rows = list(execute(cql, table, "SELECT * FROM %s"))
assert 4 == len(rows)
assert_row_count(
execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t >= ?",
rows[0][1]), 4)
assert_empty(
execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t < ?",
rows[0][1]))
assert_row_count(
execute(cql, table,
"SELECT * FROM %s WHERE k = 0 AND t > ? AND t <= ?",
rows[0][1], rows[2][1]), 2)
assert_row_count(
execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t = ?",
rows[0][1]), 1)
assert_invalid(cql, table,
"SELECT dateOf(k) FROM %s WHERE k = 0 AND t = ?",
rows[0][1])
for i in range(4):
uuid = rows[i][1]
datetime = datetime_from_uuid1(uuid)
timestamp = round(
datetime.replace(tzinfo=timezone.utc).timestamp() * 1000)
assert_rows(
execute(
cql, table,
"SELECT dateOf(t), unixTimestampOf(t) FROM %s WHERE k = 0 AND t = ?",
rows[i][1]), [datetime, timestamp])
assert_empty(
execute(
cql, table,
"SELECT t FROM %s WHERE k = 0 AND t > maxTimeuuid(1234567) AND t < minTimeuuid('2012-11-07 18:18:22-0800')"
))
def uploadQAs(email, setname, qas):
cloud_config= {
'secure_connect_bundle': '/mnt/c/Users/Rufus Behr/Downloads/secure-connect-oxfordhack.zip'
}
auth_provider = PlainTextAuthProvider()
cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider)
session = cluster.connect("Flashcards")
foo = uuid.uuid1()
dt_foo = datetime_from_uuid1(foo)
for i in range(len(qas)):
session.execute(
"""
INSERT INTO cards (email, setname, question, answer)
VALUES (%(email)s, %(setname)s, %(question)s, %(answer)s)
""",
{'email': email, 'setname': setname, 'question': qas[i][0], 'answer': qas[i][1]}
)
async def get_users_info(limit: int):
users_list_ = []
users_list = []
response_status = 200
try: # for timeuuid ops refer to UUID and timeuuid functions on datastax docs
future = db.query(f"select id, toTimestamp(time), rollcall_score, class_activity, discipline, total_quizzes_avg from users_info limit {limit};", [])
users_ = future.result()
for user_ in users_:
users_list_.append(user_)
users = User.objects().limit(limit)
for user in users:
user_dict = dict(user)
user_dict["time"] = datetime_from_uuid1(user_dict["time"]) # convert uuid1 to datetime
users_list.append(user_dict)
except Exception as e:
print(f"[Exception] ::: {e}")
response_status = 500
return {"db.query()": users_list_, "schema": users_list, "response_status": response_status}
async def get_users_position_raw(position: str):
positions_raw_list_ = []
positions_raw_list = []
response_status = 200
try:
future = db.query(f"select user_id, toTimestamp(time), position_raw from users_position_raw where position_raw = ? allow filtering;", [position])
positions_ = future.result()
for position_ in positions_:
positions_raw_list_.append(position_)
positions = PositionRaw.objects.filter(position_raw=position).allow_filtering()
for position in positions:
position_dict = dict(position)
position_dict["time"] = datetime_from_uuid1(position_dict["time"])
positions_raw_list.append(position_dict)
except Exception as e:
response_status = 500
print(f"[Exception] ::: {e}")
return {"db.query()": positions_raw_list_, "schema": positions_raw_list, "response_status": response_status}
async def get_all_users_latent_position(limit: int):
positions_list_ = []
positions_list = []
response_status = 200
try:
future = db.query(f"select user_id, toTimestamp(time), position_latent from users_position_latent limit {limit};", [])
positions_ = future.result()
for position_ in positions_:
positions_list_.append(position_)
positions = PositionLatent.objects().limit(limit)
for position in positions:
position_dict = dict(position)
position_dict["time"] = datetime_from_uuid1(position_dict["time"]) # convert uuid1 to datetime
positions_list.append(position_dict)
except Exception as e:
response_status = 500
print(f"[Exception] ::: {e}")
return {"db.query()": positions_list_, "schema": positions_list, "response_status": response_status}
def get_all_intraday_data(self):
results = []
prepared_stmt = session.prepare(
"select * from intraday_spots where symbol = ?")
bound_stmt = prepared_stmt.bind([self.symbol])
db_results = session.execute(bound_stmt)
for row in db_results:
results.append({
'timestamp':
util.datetime_from_uuid1(row.time_uuid).__str__(),
'open':
row.open,
'close':
row.close,
'low':
row.low,
'high':
row.high,
'volume':
row.volume
})
return results
def testTimeuuid(cql, test_keyspace):
with create_table(cql, test_keyspace,
"(k int, t timeuuid, PRIMARY KEY(k, t))") as table:
assert_invalid_throw(
cql, table, SyntaxException,
"INSERT INTO %s (k, t) VALUES (0, 2012-11-07 18:18:22-0800)")
for i in range(4):
execute(cql, table, "INSERT INTO %s (k, t) VALUES (0, now())")
rows = list(execute(cql, table, "SELECT * FROM %s"))
assert 4 == len(rows)
assert_row_count(
execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t >= ?",
rows[0][1]), 4)
assert_empty(
execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t < ?",
rows[0][1]))
assert_row_count(
execute(cql, table,
"SELECT * FROM %s WHERE k = 0 AND t > ? AND t <= ?",
rows[0][1], rows[2][1]), 2)
assert_row_count(
execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t = ?",
rows[0][1]), 1)
assert_invalid(cql, table,
"SELECT dateOf(k) FROM %s WHERE k = 0 AND t = ?",
rows[0][1])
for i in range(4):
uuid = rows[i][1]
datetime = datetime_from_uuid1(uuid)
# Before comparing this datetime to the result of dateOf(), we
# must truncate the resolution of datetime to milliseconds.
# he problem is that the dateOf(timeuuid) CQL function converts a
# timeuuid to CQL's "timestamp" type, which has millisecond
# resolution, but datetime *may* have finer resolution. It will
# usually be whole milliseconds, because this is what the now()
# implementation usually does, but when now() is called more than
# once per millisecond, it *may* start incrementing the sub-
# millisecond part.
datetime = datetime.replace(microsecond=datetime.microsecond //
1000 * 1000)
timestamp = round(
datetime.replace(tzinfo=timezone.utc).timestamp() * 1000)
assert_rows(
execute(
cql, table,
"SELECT dateOf(t), unixTimestampOf(t) FROM %s WHERE k = 0 AND t = ?",
rows[i][1]), [datetime, timestamp])
assert_empty(
execute(
cql, table,
"SELECT t FROM %s WHERE k = 0 AND t > maxTimeuuid(1234567) AND t < minTimeuuid('2012-11-07 18:18:22-0800')"
))
def splitConvertToResponsesAndPutIfAbsent(session, userUUID, content,
minSnapshotUUID, maxSnapshotUUID,
appUUID):
minSnapshotDocument = datetime_from_uuid1(minSnapshotUUID)
maxSnapshotDocument = datetime_from_uuid1(maxSnapshotUUID)
now = datetime.datetime.now()
dateDiff = now - minSnapshotDocument
isPassed30days = dateDiff.days > 30
isLogVerbose = ArgUtil.args.verbose
if isLogVerbose:
print(
'-----------------------------------------------------------------------------'
)
print('max snapshot date : ' +
maxSnapshotDocument.strftime('%d/%b/%Y'))
print('min snapshot date : ' +
minSnapshotDocument.strftime('%d/%b/%Y'))
print('now : ' + now.strftime('%d/%b/%Y'))
print('Date diff from now : days = %d' % (dateDiff.days))
print('isPassed30days : ' + str(isPassed30days))
data = json.loads(content)
if data.get('version'):
version = data['version']
else:
version = 'MISSING'
deployId = data['deployId']
alias = data['alias']
actions = data['actions']
totalResponsesDocumentCreated = 0
for actionId, statesMap in actions.items():
statesList = statesMap['states']
if isPassed30days:
lifeCycle = 'EXPIRED'
else:
lifeCycle = 'ACTIVE'
for state in statesList:
update_date = state['updateDate']
try:
remark = state['remark']
except:
remark = ""
status_state = state['state']
if status_state == 'done':
status_state = 'DONE'
else:
status_state = 'NOT_DONE'
reasonId = state['reasonId']
responseObject = {
'version': version,
'alias': alias,
'deployId': deployId,
'taskId': actionId,
'reasonId': reasonId,
'state': status_state,
'remark': remark,
'viewed': True,
'updateDate': update_date,
'lifeCycle': lifeCycle,
}
totalResponsesDocumentCreated += 1
if isLogVerbose:
print(
json.dumps(responseObject,
sort_keys=True,
indent=4,
separators=(',', ': ')))
putResponseIfAbsent(session, responseObject, userUUID, appUUID,
maxSnapshotUUID)
return totalResponsesDocumentCreated
def published(self):
return datetime_from_uuid1(self.id)
def insert(self, owner_id, report_id, report_instance_id, tags, ri_data,
input_string, extra_ri_data, custom_created):
created = util.datetime_from_uuid1(report_instance_id)
day = day_text(created)
all_tags_repr = tags_repr_from_tags(tags)
qs = []
metadata_row = dict(
report_id=report_id,
day=day,
report_instance_id=report_instance_id,
all_tags_repr=all_tags_repr,
inserted=datetime.datetime.utcnow(),
)
if extra_ri_data:
metadata_row['extra_ri_data'] = extra_ri_data
qs.append(insert('mqe.report_instance_metadata', metadata_row))
first_row = None
tags_powerset = util.powerset(tags[:mqeconfig.MAX_TAGS])
for tags_subset in tags_powerset:
tags_repr = tags_repr_from_tags(tags_subset)
row = dict(report_id=report_id,
day=day,
tags_repr=tags_repr,
report_instance_id=report_instance_id,
ri_data=ri_data,
input_string=input_string,
all_tags_repr=all_tags_repr)
if first_row is None:
first_row = row
qs.append(
insert('mqe.report_instance', row,
COLUMN_RENAMES['report_instance']))
if not c.cass.execute(
"""SELECT day FROM mqe.report_instance_day
WHERE report_id=? AND tags_repr=? AND day=?""",
[report_id, tags_repr, day]):
qs.append(
insert(
'mqe.report_instance_day',
dict(report_id=report_id, tags_repr=tags_repr,
day=day)))
qs.append(
bind(
"""UPDATE mqe.report_instance_count SET count=count+1
WHERE report_id=? AND tags_repr=?""",
[report_id, tags_repr]))
diskspace = self._compute_ri_diskspace(row)
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace SET bytes=bytes+?
WHERE report_id=? AND tags_repr=?""",
[diskspace, report_id, tags_repr]))
### queries for all tags
qs.append(
bind(
"""UPDATE mqe.report_instance_count_for_owner SET count=count+1
WHERE owner_id=?""", [owner_id]))
if first_row:
diskspace = self._compute_ri_diskspace(first_row)
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace_for_owner SET bytes=bytes+?
WHERE owner_id=?""", [diskspace, owner_id]))
# avoid reinserting the same tag multiple times
tag_rows = c.cass.execute(
"""SELECT tag FROM mqe.report_tag
WHERE report_id=? AND tag_prefix='' AND tag IN ?""",
[report_id, tags])
tags_from_rows = {row['tag'] for row in tag_rows}
for tag in tags:
if tag in tags_from_rows:
continue
for p in util.iter_prefixes(tag, include_empty=True):
qs.append(
insert('mqe.report_tag',
dict(report_id=report_id, tag_prefix=p, tag=tag)))
c.cass.execute_parallel(qs)
return postprocess_tags(first_row)
