def delete_multi(self, owner_id, report_id, tags, min_report_instance_id, max_report_instance_id, limit, update_counters, use_insertion_datetime): columns = ['report_instance_id', 'all_tags_repr', 'day'] if update_counters: columns.append('input_string') ris = self.select_multi(report_id, tags, min_report_instance_id, max_report_instance_id, columns, 'asc', limit) log.info('Selected %d report instances to delete', len(ris)) if use_insertion_datetime: days = util.uniq_sameorder(ri['day'] for ri in ris) rim_rows = c.cass.execute( """SELECT report_instance_id, inserted FROM mqe.report_instance_metadata WHERE report_id=? AND day IN ? AND report_instance_id > ? AND report_instance_id < ?""", [ report_id, days, min_report_instance_id, max_report_instance_id ]) rim_rows = list(rim_rows) assert len(rim_rows) == len(ris) ri_by_rid = {ri['report_instance_id']: ri for ri in ris} from_dt = util.datetime_from_uuid1(min_report_instance_id) to_dt = util.datetime_from_uuid1(max_report_instance_id) for rim in rim_rows: if not (from_dt <= rim['inserted'] <= to_dt): del ri_by_rid[rim['report_instance_id']] ris = ri_by_rid.values() return self._delete_ris(owner_id, report_id, tags, ris, update_counters)
def process_events_csv(self, filename): with open(filename) as f: csvreader = csv.DictReader(f) counter = 0 batch = [] for row in csvreader: # extract time from UUID for indexing event_uuid = uuid.UUID(row['event_id']) event_dt = datetime_from_uuid1(event_uuid) #row['started_at'] = event_dt.strftime('%Y-%m-%d %H:%M:%S.%f') row['started_at'] = datetime_from_uuid1(event_uuid) # add cluster name to doc row['cluster'] = self.cluster_name batch.append({ '_index': self.es_index_name, '_type': self.es_event_doc_type, '_parent': row['session_id'], '_routing': row['session_id'], '_id': row['event_id'], '_source': row }) if len(batch) >= self.args.batch_size: counter = counter + 1 r = helpers.bulk(self.es, batch) print(counter, r) batch = []
def day_text(dt_or_uuid): if isinstance(dt_or_uuid, uuid.UUID): dt = util.datetime_from_uuid1(dt_or_uuid) else: dt = dt_or_uuid if dt.year < 2000: return '000000' return dt.strftime('%y%m%d')
def to_object(self): return { 'chat_id': self.chat_id, 'message_id': str(self.message_id), 'author_id': self.author_id, 'time': datetime_from_uuid1(self.message_id).isoformat(), 'type': self.type, 'text': self.text, 'asset_name': self.asset_name, 'meta_data': self.meta_data }
def test_times_from_uuid1(self): node = uuid.getnode() now = time.time() u = uuid.uuid1(node, 0) t = util.unix_time_from_uuid1(u) self.assertAlmostEqual(now, t, 3) dt = util.datetime_from_uuid1(u) t = calendar.timegm(dt.timetuple()) + dt.microsecond / 1e6 self.assertAlmostEqual(now, t, 3)
def test_times_from_uuid1(self): node = uuid.getnode() now = time.time() u = uuid.uuid1(node, 0) t = util.unix_time_from_uuid1(u) self.assertAlmostEqual(now, t, 2) dt = util.datetime_from_uuid1(u) t = calendar.timegm(dt.timetuple()) + dt.microsecond / 1e6 self.assertAlmostEqual(now, t, 2)
def main(): args = parse_arguments() ArgUtil.setArgs(args) newMostResentSnapshot = min_uuid_from_time( datetime.datetime.now().timestamp()) prevSnapshot = parsePrevSnapshot() print('prevSnapshot:' + str(datetime_from_uuid1(prevSnapshot))) print('newMostResentSnapshot:' + str(datetime_from_uuid1(newMostResentSnapshot))) applicationsPath = args.applicationIdsPath applicationUUIIdsFromFile = parseApplicationUUIDsFromFile(applicationsPath) if args.verbose: appIdsStringList = [ str(appUUID) for appUUID in applicationUUIIdsFromFile ] print('Application UUIDs from file: ' + str(appIdsStringList)) contactPoints = args.contactPoints cluster = Cluster(contactPoints) totalStatusDocumentSplited = 0 totalResponsesCreated = 0 try: session = cluster.connect() print('connecting to cluster: [' + ', '.join(contactPoints) + '] ...') session.execute('use unifiedpush_server') for appUUID in applicationUUIIdsFromFile: totalResponsesCreated, totalStatusDocumentSplited = \ workOnApplication(appUUID, session, totalResponsesCreated, totalStatusDocumentSplited, prevSnapshot) updateMostResentSnapshot(newMostResentSnapshot) finally: print('Total number of STATUS document found: ' + str(totalStatusDocumentSplited)) print('Total number of RESPONSE document created: ' + str(totalResponsesCreated)) print('closing cluster connections...') cluster.shutdown()
async def get_user_info(user_id: int): response_status = 200 try: future = db.query("select id, toTimestamp(time), rollcall_score, class_activity, discipline, total_quizzes_avg FROM users_info where id=? allow filtering", [user_id]) user_ = future.result()[0] user = User.objects(id=user_id).allow_filtering()[0] user.time = datetime_from_uuid1(user.time) except Exception as e: print(f"[Exception] ::: {e}") response_status = 500 user_ = [] user = [] return {"db.query()": user_, "schema": user, "response_status": response_status}
async def get_user_raw_position(user_id: int): response_status = 200 try: future = db.query(f"select user_id, toTimestamp(time), position_raw from users_position_raw where user_id = ? allow filtering;", [user_id]) position_ = future.result()[0] position = PositionRaw.objects(user_id=user_id).allow_filtering()[0] position.time = datetime_from_uuid1(position.time) except Exception as e: print(f"[Exception] ::: {e}") response_status = 500 position_ = [] position = [] return {"db.query()": position_, "schema": position, "response_status": response_status}
def testTimeuuid(cql, test_keyspace): with create_table(cql, test_keyspace, "(k int, t timeuuid, PRIMARY KEY(k, t))") as table: assert_invalid_throw( cql, table, SyntaxException, "INSERT INTO %s (k, t) VALUES (0, 2012-11-07 18:18:22-0800)") for i in range(4): execute(cql, table, "INSERT INTO %s (k, t) VALUES (0, now())") rows = list(execute(cql, table, "SELECT * FROM %s")) assert 4 == len(rows) assert_row_count( execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t >= ?", rows[0][1]), 4) assert_empty( execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t < ?", rows[0][1])) assert_row_count( execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t > ? AND t <= ?", rows[0][1], rows[2][1]), 2) assert_row_count( execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t = ?", rows[0][1]), 1) assert_invalid(cql, table, "SELECT dateOf(k) FROM %s WHERE k = 0 AND t = ?", rows[0][1]) for i in range(4): uuid = rows[i][1] datetime = datetime_from_uuid1(uuid) timestamp = round( datetime.replace(tzinfo=timezone.utc).timestamp() * 1000) assert_rows( execute( cql, table, "SELECT dateOf(t), unixTimestampOf(t) FROM %s WHERE k = 0 AND t = ?", rows[i][1]), [datetime, timestamp]) assert_empty( execute( cql, table, "SELECT t FROM %s WHERE k = 0 AND t > maxTimeuuid(1234567) AND t < minTimeuuid('2012-11-07 18:18:22-0800')" ))
def uploadQAs(email, setname, qas): cloud_config= { 'secure_connect_bundle': '/mnt/c/Users/Rufus Behr/Downloads/secure-connect-oxfordhack.zip' } auth_provider = PlainTextAuthProvider() cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider) session = cluster.connect("Flashcards") foo = uuid.uuid1() dt_foo = datetime_from_uuid1(foo) for i in range(len(qas)): session.execute( """ INSERT INTO cards (email, setname, question, answer) VALUES (%(email)s, %(setname)s, %(question)s, %(answer)s) """, {'email': email, 'setname': setname, 'question': qas[i][0], 'answer': qas[i][1]} )
async def get_users_info(limit: int): users_list_ = [] users_list = [] response_status = 200 try: # for timeuuid ops refer to UUID and timeuuid functions on datastax docs future = db.query(f"select id, toTimestamp(time), rollcall_score, class_activity, discipline, total_quizzes_avg from users_info limit {limit};", []) users_ = future.result() for user_ in users_: users_list_.append(user_) users = User.objects().limit(limit) for user in users: user_dict = dict(user) user_dict["time"] = datetime_from_uuid1(user_dict["time"]) # convert uuid1 to datetime users_list.append(user_dict) except Exception as e: print(f"[Exception] ::: {e}") response_status = 500 return {"db.query()": users_list_, "schema": users_list, "response_status": response_status}
async def get_users_position_raw(position: str): positions_raw_list_ = [] positions_raw_list = [] response_status = 200 try: future = db.query(f"select user_id, toTimestamp(time), position_raw from users_position_raw where position_raw = ? allow filtering;", [position]) positions_ = future.result() for position_ in positions_: positions_raw_list_.append(position_) positions = PositionRaw.objects.filter(position_raw=position).allow_filtering() for position in positions: position_dict = dict(position) position_dict["time"] = datetime_from_uuid1(position_dict["time"]) positions_raw_list.append(position_dict) except Exception as e: response_status = 500 print(f"[Exception] ::: {e}") return {"db.query()": positions_raw_list_, "schema": positions_raw_list, "response_status": response_status}
async def get_all_users_latent_position(limit: int): positions_list_ = [] positions_list = [] response_status = 200 try: future = db.query(f"select user_id, toTimestamp(time), position_latent from users_position_latent limit {limit};", []) positions_ = future.result() for position_ in positions_: positions_list_.append(position_) positions = PositionLatent.objects().limit(limit) for position in positions: position_dict = dict(position) position_dict["time"] = datetime_from_uuid1(position_dict["time"]) # convert uuid1 to datetime positions_list.append(position_dict) except Exception as e: response_status = 500 print(f"[Exception] ::: {e}") return {"db.query()": positions_list_, "schema": positions_list, "response_status": response_status}
def get_all_intraday_data(self): results = [] prepared_stmt = session.prepare( "select * from intraday_spots where symbol = ?") bound_stmt = prepared_stmt.bind([self.symbol]) db_results = session.execute(bound_stmt) for row in db_results: results.append({ 'timestamp': util.datetime_from_uuid1(row.time_uuid).__str__(), 'open': row.open, 'close': row.close, 'low': row.low, 'high': row.high, 'volume': row.volume }) return results
def testTimeuuid(cql, test_keyspace): with create_table(cql, test_keyspace, "(k int, t timeuuid, PRIMARY KEY(k, t))") as table: assert_invalid_throw( cql, table, SyntaxException, "INSERT INTO %s (k, t) VALUES (0, 2012-11-07 18:18:22-0800)") for i in range(4): execute(cql, table, "INSERT INTO %s (k, t) VALUES (0, now())") rows = list(execute(cql, table, "SELECT * FROM %s")) assert 4 == len(rows) assert_row_count( execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t >= ?", rows[0][1]), 4) assert_empty( execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t < ?", rows[0][1])) assert_row_count( execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t > ? AND t <= ?", rows[0][1], rows[2][1]), 2) assert_row_count( execute(cql, table, "SELECT * FROM %s WHERE k = 0 AND t = ?", rows[0][1]), 1) assert_invalid(cql, table, "SELECT dateOf(k) FROM %s WHERE k = 0 AND t = ?", rows[0][1]) for i in range(4): uuid = rows[i][1] datetime = datetime_from_uuid1(uuid) # Before comparing this datetime to the result of dateOf(), we # must truncate the resolution of datetime to milliseconds. # he problem is that the dateOf(timeuuid) CQL function converts a # timeuuid to CQL's "timestamp" type, which has millisecond # resolution, but datetime *may* have finer resolution. It will # usually be whole milliseconds, because this is what the now() # implementation usually does, but when now() is called more than # once per millisecond, it *may* start incrementing the sub- # millisecond part. datetime = datetime.replace(microsecond=datetime.microsecond // 1000 * 1000) timestamp = round( datetime.replace(tzinfo=timezone.utc).timestamp() * 1000) assert_rows( execute( cql, table, "SELECT dateOf(t), unixTimestampOf(t) FROM %s WHERE k = 0 AND t = ?", rows[i][1]), [datetime, timestamp]) assert_empty( execute( cql, table, "SELECT t FROM %s WHERE k = 0 AND t > maxTimeuuid(1234567) AND t < minTimeuuid('2012-11-07 18:18:22-0800')" ))
def splitConvertToResponsesAndPutIfAbsent(session, userUUID, content, minSnapshotUUID, maxSnapshotUUID, appUUID): minSnapshotDocument = datetime_from_uuid1(minSnapshotUUID) maxSnapshotDocument = datetime_from_uuid1(maxSnapshotUUID) now = datetime.datetime.now() dateDiff = now - minSnapshotDocument isPassed30days = dateDiff.days > 30 isLogVerbose = ArgUtil.args.verbose if isLogVerbose: print( '-----------------------------------------------------------------------------' ) print('max snapshot date : ' + maxSnapshotDocument.strftime('%d/%b/%Y')) print('min snapshot date : ' + minSnapshotDocument.strftime('%d/%b/%Y')) print('now : ' + now.strftime('%d/%b/%Y')) print('Date diff from now : days = %d' % (dateDiff.days)) print('isPassed30days : ' + str(isPassed30days)) data = json.loads(content) if data.get('version'): version = data['version'] else: version = 'MISSING' deployId = data['deployId'] alias = data['alias'] actions = data['actions'] totalResponsesDocumentCreated = 0 for actionId, statesMap in actions.items(): statesList = statesMap['states'] if isPassed30days: lifeCycle = 'EXPIRED' else: lifeCycle = 'ACTIVE' for state in statesList: update_date = state['updateDate'] try: remark = state['remark'] except: remark = "" status_state = state['state'] if status_state == 'done': status_state = 'DONE' else: status_state = 'NOT_DONE' reasonId = state['reasonId'] responseObject = { 'version': version, 'alias': alias, 'deployId': deployId, 'taskId': actionId, 'reasonId': reasonId, 'state': status_state, 'remark': remark, 'viewed': True, 'updateDate': update_date, 'lifeCycle': lifeCycle, } totalResponsesDocumentCreated += 1 if isLogVerbose: print( json.dumps(responseObject, sort_keys=True, indent=4, separators=(',', ': '))) putResponseIfAbsent(session, responseObject, userUUID, appUUID, maxSnapshotUUID) return totalResponsesDocumentCreated
def published(self): return datetime_from_uuid1(self.id)
def insert(self, owner_id, report_id, report_instance_id, tags, ri_data, input_string, extra_ri_data, custom_created): created = util.datetime_from_uuid1(report_instance_id) day = day_text(created) all_tags_repr = tags_repr_from_tags(tags) qs = [] metadata_row = dict( report_id=report_id, day=day, report_instance_id=report_instance_id, all_tags_repr=all_tags_repr, inserted=datetime.datetime.utcnow(), ) if extra_ri_data: metadata_row['extra_ri_data'] = extra_ri_data qs.append(insert('mqe.report_instance_metadata', metadata_row)) first_row = None tags_powerset = util.powerset(tags[:mqeconfig.MAX_TAGS]) for tags_subset in tags_powerset: tags_repr = tags_repr_from_tags(tags_subset) row = dict(report_id=report_id, day=day, tags_repr=tags_repr, report_instance_id=report_instance_id, ri_data=ri_data, input_string=input_string, all_tags_repr=all_tags_repr) if first_row is None: first_row = row qs.append( insert('mqe.report_instance', row, COLUMN_RENAMES['report_instance'])) if not c.cass.execute( """SELECT day FROM mqe.report_instance_day WHERE report_id=? AND tags_repr=? AND day=?""", [report_id, tags_repr, day]): qs.append( insert( 'mqe.report_instance_day', dict(report_id=report_id, tags_repr=tags_repr, day=day))) qs.append( bind( """UPDATE mqe.report_instance_count SET count=count+1 WHERE report_id=? AND tags_repr=?""", [report_id, tags_repr])) diskspace = self._compute_ri_diskspace(row) qs.append( bind( """UPDATE mqe.report_instance_diskspace SET bytes=bytes+? WHERE report_id=? AND tags_repr=?""", [diskspace, report_id, tags_repr])) ### queries for all tags qs.append( bind( """UPDATE mqe.report_instance_count_for_owner SET count=count+1 WHERE owner_id=?""", [owner_id])) if first_row: diskspace = self._compute_ri_diskspace(first_row) qs.append( bind( """UPDATE mqe.report_instance_diskspace_for_owner SET bytes=bytes+? WHERE owner_id=?""", [diskspace, owner_id])) # avoid reinserting the same tag multiple times tag_rows = c.cass.execute( """SELECT tag FROM mqe.report_tag WHERE report_id=? AND tag_prefix='' AND tag IN ?""", [report_id, tags]) tags_from_rows = {row['tag'] for row in tag_rows} for tag in tags: if tag in tags_from_rows: continue for p in util.iter_prefixes(tag, include_empty=True): qs.append( insert('mqe.report_tag', dict(report_id=report_id, tag_prefix=p, tag=tag))) c.cass.execute_parallel(qs) return postprocess_tags(first_row)