def update_containers_status(): global launcher_table try: logging.info("updating containers status") for container in list_containers(): logging.info("checking {} status".format(container["_id"])) status = "starting" for line in subprocess.check_output(['docker', 'ps','-a']).decode('utf-8').strip().split('\n')[1:]: if container["_id"] and "Exited" in line: status = "stopped" try: content = requests.get("https://127.0.0.1:{}".format(container["port"]),verify=False).content if "Zeppelin" in content: status = "running" if "HTTP ERROR: 503" in content: status = "error" except Exception as e: msg = traceback.format_exc() logging.error(msg) launcher_table.update(_id=container["_id"],mutation={"$put":{'status':status}}) logging.info("{} status : {}".format(container["_id"],status)) except Exception as e: logging.info( traceback.format_exc()) if "STATUS_TOKEN_EXPIRED" in traceback.format_exc(): logging.info("Resetting connection") connection = ConnectionFactory().get_connection(connection_str=connection_str) launcher_table = connection.get_or_create_store(LAUNCHER_TABLE)
def test_check_and_replace_false_condition(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/check-replace-test-store1'): document_store = connection.get_store( store_path='/check-replace-test-store1') else: document_store = connection.create_store( store_path='/check-replace-test-store1') before_action = document_store.find_by_id('id06') self.assertEqual(before_action, { '_id': 'id06', 'new_field': 123, 'new_array': [1, 2, 3] }) new_document = connection.new_document().set_id('id06').set( 'false_field', 321).set('false_array', [5, 5, 5]) condition = OJAIQueryCondition().or_().is_('test_int', QueryOp.GREATER_OR_EQUAL, 60)\ .is_('test_str', QueryOp.EQUAL, 'falsestr')\ .close().build() document_store.check_and_replace(new_document, condition) after_action = document_store.find_by_id('id06') self.assertEqual(after_action, { '_id': 'id06', 'new_field': 123, 'new_array': [1, 2, 3] })
def test_find_by_id_as_dict(self): connection = ConnectionFactory.get_connection(connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/find-by-id-test-store1'): document_store = connection.get_store(store_path='/find-by-id-test-store1') else: document_store = connection.create_store(store_path='/find-by-id-test-store1') self.assertTrue(connection.is_store_exists('/find-by-id-test-store1')) doc = document_store.find_by_id('121212') document = OJAIDocument().set_id("121212") \ .set('test_int', 123) \ .set('first.test_int', 1235) \ .set('first.test_long', 123456789) \ .set('first.test_time', OTime(timestamp=1518689532)) \ .set('first.test_date', ODate(days_since_epoch=3456)) \ .set('first.test_bool', True) \ .set('first.test_bool_false', False) \ .set('first.test_invalid', ODate(days_since_epoch=3457)) \ .set('first.test_str', 'strstr') \ .set('first.test_dict', {'a': 1, 'b': 2}) \ .set('first.test_dict2', {}) \ .set('first.test_list', [1, 2, 'str', False, ODate(days_since_epoch=3457)]) self.assertEqual(doc, document.as_dictionary())
def test_update_document_append(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path=UpdateTest.store_name): document_store = connection.get_store( store_path=UpdateTest.store_name) else: document_store = connection.create_store( store_path=UpdateTest.store_name) self.assertEqual( { '_id': 'id09', 'test_int': 51, 'test_str': 'strstr', 'test_list': [5, 6] }, document_store.find_by_id('id09')) mutation = OJAIDocumentMutation().append('test_list', [{ 'name': 'Jo' }, 7, 8]) document_store.update('id09', mutation) self.assertEqual( { '_id': 'id09', 'test_int': 51, 'test_str': 'strstr', 'test_list': [5, 6, { 'name': 'Jo' }, 7, 8] }, document_store.find_by_id('id09'))
def test_insert(self): dict_stream = [{'_id': "id001", 'test_int': 51, 'test_str': 'strstr'}, {'_id': 'id002', 'mystr': 'str', 'test_int': 51, 'test_str': 'strstr'}, {'_id': 'id003', 'test_int': 51, 'test_otime': OTime(timestamp=1518689532), 'test_str': 'strstr'}, {'_id': 'id004', 'test_int': 51, 'test_timestamp': OTimestamp(millis_since_epoch=29877132000), 'test_str': 'strstr'}, {'_id': 'id005', 'test_int': 51, 'test_bool': True, 'test_str': 'strstr'}, {'_id': 'id006', 'test_int': 51, 'test_str': 'strstr'}, {'_id': 'id007', 'test_int': 51, 'test_str': 'strstr'}, {'_id': 'id008', 'test_int': 51, 'test_str': 'strstr', 'test_dict': {'test_int': 5}}, {'_id': 'id009', 'test_int': 51, 'test_str': 'strstr', 'test_list': [5, 6]}, {'_id': 'id010', 'test_int': 51, 'test_str': 'strstr', 'test_null': None}] connection = ConnectionFactory.get_connection(connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) # should raise an error if exit is not 0 if connection.is_store_exists(store_path='/test-store6'): document_store = connection.get_store(store_path='/test-store6') else: document_store = connection.create_store(store_path='/test-store6') check_store = connection.is_store_exists(store_path='/test-store6') self.assertTrue(check_store) self.assertTrue(isinstance(document_store, OJAIDocumentStore)) for doc in dict_stream: document = connection.new_document(dictionary=doc) # print('Insert document with ID: ' + str(document.get_id())) document_store.insert(doc=document) # drop_store = connection.delete_store(store_path='/test-store6') self.assertTrue(drop_store)
def test_update_document_put(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path=UpdateTest.store_name): document_store = connection.get_store( store_path=UpdateTest.store_name) else: document_store = connection.create_store( store_path=UpdateTest.store_name) self.assertEqual({ '_id': 'id06', 'test_int': 51, 'test_str': 'strstr' }, document_store.find_by_id('id06')) mutation = OJAIDocumentMutation().set_or_replace('test_int', 50) document_store.update('id06', mutation) self.assertEqual({ '_id': 'id06', 'test_int': 50, 'test_str': 'strstr' }, document_store.find_by_id('id06')) document_store.update('id06', {"$put": {"a.x": 1}}) self.assertEqual( { '_id': 'id06', 'test_int': 50, 'test_str': 'strstr', 'a': { 'x': 1 } }, document_store.find_by_id('id06'))
def test_update_document_merge(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path=UpdateTest.store_name): document_store = connection.get_store( store_path=UpdateTest.store_name) else: document_store = connection.create_store( store_path=UpdateTest.store_name) self.assertEqual( { '_id': 'id11', 'test_int': 51, 'test_str': 'strstr', 'test_dict': { 'test_int': 5 } }, document_store.find_by_id('id11')) mutation = OJAIDocumentMutation().merge('test_dict', { 'd': 55, 'g': 'text' }) document_store.update('id11', mutation) self.assertEqual( { '_id': 'id11', 'test_int': 51, 'test_str': 'strstr', 'test_dict': { 'test_int': 5, 'd': 55, 'g': 'text' } }, document_store.find_by_id('id11'))
def test_check_and_delete(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/check-delete-test-store1'): document_store = connection.get_store( store_path='/check-delete-test-store1') else: document_store = connection.create_store( store_path='/check-delete-test-store1') for doc in DICT_STREAM: document_store.insert_or_replace(doc=connection.new_document( dictionary=doc)) before_action = document_store.find_by_id('id06') self.assertEqual(before_action, { '_id': 'id06', 'test_int': 51, 'test_str': 'strstr' }) document_store.check_and_delete( 'id06', OJAIQueryCondition().is_('test_int', QueryOp.LESS_OR_EQUAL, 52).close().build()) after_action = document_store.find_by_id('id06') self.assertEqual(after_action, {})
def test_check_and_delete_false_condition(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/check-delete-test-store1'): document_store = connection.get_store( store_path='/check-delete-test-store1') else: document_store = connection.create_store( store_path='/check-delete-test-store1') before_action = document_store.find_by_id('id07') self.assertEqual(before_action, { '_id': 'id07', 'test_int': 51, 'test_str': 'strstr' }) document_store.check_and_delete( 'id06', OJAIQueryCondition().is_('test_int', QueryOp.LESS_OR_EQUAL, 50).close().build()) after_action = document_store.find_by_id('id07') self.assertEqual(after_action, { '_id': 'id07', 'test_int': 51, 'test_str': 'strstr' })
def test_check_and_delete_dict_condition(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/check-delete-test-store1'): document_store = connection.get_store( store_path='/check-delete-test-store1') else: document_store = connection.create_store( store_path='/check-delete-test-store1') before_action = document_store.find_by_id('id09') self.assertEqual( before_action, { '_id': 'id09', 'test_int': 51, 'test_str': 'strstr', 'test_list': [5, 6] }) document_store.check_and_delete('id09', {'$eq': {'test_list': [5, 6]}}) after_action = document_store.find_by_id('id09') self.assertEqual(after_action, {}) document_store.check_and_delete('id08', {'$eq': { 'test_dict.test_int': 5 }}) additional_check = document_store.find_by_id('id08') self.assertEqual(additional_check, {})
def test_simple_find(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/find-test-store1'): document_store = connection.get_store( store_path='/find-test-store1') else: document_store = connection.create_store( store_path='/find-test-store1') document = connection.new_document( dictionary={ '_id': 'id008', 'test_int': 51, 'test_str': 'strstr', 'test_dict': { 'test_int': 5 }, 'test_list': [5, 6], 'test_null': None }) document_store.insert_or_replace(doc=document) query = OJAIQuery().select([ '_id', 'test_int', 'test_str', 'test_dict', 'test_list', 'test_null' ]).build() self.assertTrue(connection.is_store_exists('/find-test-store1')) doc_stream = document_store.find(query) for doc in doc_stream: self.assertEqual(doc, document.as_dictionary())
def test_find_on_empty_table(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/find-test-store2'): document_store = connection.get_store( store_path='/find-test-store2') else: document_store = connection.create_store( store_path='/find-test-store2') query = OJAIQuery().select(['_id', 'test_int', 'test_str', 'test_dict', 'test_list', 'test_null'])\ .build() self.assertTrue(connection.is_store_exists('/find-test-store2')) doc_stream = document_store.find(query) size = 0 for doc in doc_stream: size += 1 print(doc) self.assertEqual(size, 0)
def put(doc): singleton = MapRLoader.getInstance() connection = ConnectionFactory.get_connection( connection_str=singleton.connectionStr) store = connection.get_store(singleton.store) doc = store.insertOrReplace(doc) connection.close() return doc
def get(name): singleton = MapRLoader.getInstance() connection = ConnectionFactory.get_connection( connection_str=singleton.connectionStr) store = connection.get_store(singleton.store) config = Config(store.find_by_id(name)) connection.close() return config
def test_connection(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) before_create = connection.is_store_exists(store_path='/test-store1') self.assertFalse(before_create) store = connection.create_store(store_path='/test-store1') self.assertTrue(isinstance(store, OJAIDocumentStore)) after_create = connection.is_store_exists(store_path='/test-store1') self.assertTrue(after_create) delete_response = connection.delete_store(store_path='/test-store1') self.assertTrue(delete_response)
def queryResultsByUser(self, user): connection = ConnectionFactory.get_connection(self.connection_string) document_store = connection.get_store( store_path='/apps/course_results') #query_dict = {"$select":["RaceEntries.List[1].EventCourseID"],"$where":{"$like":{"RaceEntries.List[].DisplayName":"sargon%benjamin"}}} #query_dict = {"$select":["CourseID","CourseName","RaceID"],"$where":{"$matches":{"RaceEntries.List[].DisplayName":"(?i)"+user}}} #query_dict = {"$select":["CourseID","CourseName","RaceID","RaceEntries.List[].DisplayName"],"$where":{"$like":{"RaceEntries.List[].DisplayName":"sargon%"}}} query_dict = { "$select": [ "event_id", "RaceID", "CourseName", "CoursePattern", "DisplayName", "TicksString" ], "$where": { "$like": { "DisplayName": user } } } start = time.time() query_result = document_store.find(query_dict, options=self.options) iterations = 0 raceEntries = 0 raceIds = list() #courseIds = list() print(query_result.get_query_plan()) for item in query_result: iterations += 1 #print (item.as_dictionary()) row = item.as_dictionary() #courseId = row['CourseID'] #courseIds.append(courseId) courseName = row['CourseName'] raceIds.append(row['RaceID']) #racers = row['RaceEntries']['List'] print("Race : " + str(row['RaceID']) + " CoursePattern: " + str(row['CoursePattern']) + " " + courseName + " Time : " + str(row['TicksString']) + " event_id: " + str(row['event_id'])) #print("Race : " + str(row['RaceID']) + " Course: " + str(courseId) + " " + courseName + " with " + str(len(racers))) #raceEntries+=len(racers) #for racer in racers: # if racer['DisplayName'].lower() == 'sargon benjamin': # print("Race : " + str(row['RaceID']) + " Course: " + str(courseId) + " " + courseName + " with " + str(len(racers))) # print ("Found match with " + str(racer)) end = time.time() print("Duration = " + str(end - start)) print("iterations is " + str(iterations)) connection.close()
def test_delete_id(self): connection = ConnectionFactory.get_connection(connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/delete-test-store1'): document_store = connection.get_store(store_path='/delete-test-store1') else: document_store = connection.create_store(store_path='/delete-test-store1') document_store.delete(_id='id09') self.assertEqual(document_store.find_by_id('id09'), {})
def test_check_and_update(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path=UpdateTest.store_name): document_store = connection.get_store( store_path=UpdateTest.store_name) else: document_store = connection.create_store( store_path=UpdateTest.store_name) self.assertEqual( { '_id': 'id02', 'mystr': 'str', 'test_int': 51, 'test_str': 'strstr' }, document_store.find_by_id('id02')) mutation = OJAIDocumentMutation().set_or_replace( 'new_field', { 'd': 55, 'g': 'text' }) from mapr.ojai.ojai_query.OJAIQueryCondition import OJAIQueryCondition false_condition = OJAIQueryCondition().equals_( 'test_str', 'rtsrts').close().build() self.assertFalse( document_store.check_and_update('id02', mutation=mutation, query_condition=false_condition)) self.assertEqual( { '_id': 'id02', 'mystr': 'str', 'test_int': 51, 'test_str': 'strstr' }, document_store.find_by_id('id02')) true_condition = OJAIQueryCondition().equals_( 'test_str', 'strstr').close().build() document_store.check_and_update('id02', mutation=mutation, query_condition=true_condition) self.assertEqual( { '_id': 'id02', 'mystr': 'str', 'test_int': 51, 'test_str': 'strstr', 'new_field': { 'd': 55, 'g': 'text' } }, document_store.find_by_id('id02'))
def test_replace_error(self): connection = ConnectionFactory.get_connection(connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/test-store9'): document_store = connection.get_store(store_path='/test-store9') else: document_store = connection.create_store(store_path='/test-store9') doc_dict = {'_id': 'HtYrSv', 'long_field': 55} doc = connection.new_document(dictionary=doc_dict) with self.assertRaises(DocumentNotFoundError): document_store.replace(doc=doc) connection.delete_store(store_path='/test-store9')
def get_db_connection(): global connection global stop_threads global connection_thread_active connection_thread_active = True i = 0 while not stop_threads: i += 1 if i == 600 : logging.info("resetting db connection") connection = ConnectionFactory().get_connection(connection_str=connection_str) i = 0 time.sleep(1) connection_thread_active = False
def test_huge_operations(self): connection = ConnectionFactory.get_connection(CONNECTION_STR) if connection.is_store_exists(store_path='/long-run-test-store'): connection.delete_store(store_path='/long-run-test-store') document_store = connection.create_store(store_path='/long-run-test-store') document_list = [] print('Start generation.') for i in range(0, 10000): document_list.append(connection.new_document(dictionary={'_id': 'id00%s' % i, 'test_int': i, 'test_str': 'strstr', 'test_dict': {'test_int': i}, 'test_list': [5, 6], 'test_null': None})) print('Finish generation.') print('List size {0}'.format(len(document_list))) print('Start insert_or_replace') document_store.insert_or_replace(doc_stream=document_list) print('Finish insert_or_replace') print('Start replace') document_store.replace(doc_stream=document_list) print('Finish replace') print('Start find') doc_stream = document_store.find() print('Finish loop.') stream_size = 0 print('Start loop through document stream.') for _ in doc_stream: stream_size += 1 self.assertEqual(stream_size, 10000) print('Finish loop through document stream.') mutation = OJAIDocumentMutation().set_or_replace('test_str', 'new_string') print('Start update through loop.') for i in range(0, 10000): document_store.update('id00{0}'.format(i), mutation) print('Finish update through loop.') print('Start find after update.') doc_stream = document_store.find() print('Finsih find after update.') stream_size = 0 print('Start loop through document stream after update.') for doc in doc_stream: stream_size += 1 self.assertEqual(doc['test_str'], 'new_string') print('Finish loop through document stream after update.') self.assertEqual(stream_size, 10000)
def test_delete_document_stream(self): connection = ConnectionFactory.get_connection(connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/delete-test-store1'): document_store = connection.get_store(store_path='/delete-test-store1') else: document_store = connection.create_store(store_path='/delete-test-store1') doc_stream = [] for i in range(1, 5): doc_stream.append(OJAIDocument().from_dict(DICT_STREAM[i])) document_store.delete(doc_stream=doc_stream)
def sample(): # Create connection from connection_url # Cannot share connection for processes, # so need to create connection for each process. connection = ConnectionFactory().get_connection( connection_str=connection_str) # Get a store and assign it as a DocumentStore object store = connection.get_or_create_store('/tmp/store_name') # Insert 15 documents, represented as Python dictionaries, # into DocumentStore for i in range(15): store.insert_or_replace(doc={'_id': str(i), 'name': 'Greg'}) # Create DocumentMutation object using the OJAIConnection object mutation = connection.new_mutation() # Set mutation value mutation.set_or_replace(field_path='name', value='T') # Update 15 Document in store for i in range(15): store.update(_id=str(i), mutation=mutation)
def test_find_all(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/find-test-store4'): document_store = connection.get_store( store_path='/find-test-store4') else: document_store = connection.create_store( store_path='/find-test-store4') doc_stream = document_store.find() stream_size = 0 for _ in doc_stream: stream_size += 1 self.assertEqual(stream_size, 9)
def test_find_by_id_empty_response(self): connection = ConnectionFactory.get_connection(connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/find-by-id-test-store1'): document_store = connection.get_store(store_path='/find-by-id-test-store1') else: document_store = connection.create_store(store_path='/find-by-id-test-store1') self.assertTrue(connection.is_store_exists('/find-by-id-test-store1')) doc_as_dict = document_store.find_by_id('id9999') self.assertEqual(doc_as_dict, {}) doc_as_object = document_store.find_by_id('id9999', results_as_document=True) self.assertEqual(doc_as_object.as_dictionary(), {})
def test_insert_or_replace(self): connection = ConnectionFactory.get_connection(connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/test-store5'): connection.delete_store(store_path='/test-store5') document_store = connection.create_store(store_path='/test-store5') check_store = connection.is_store_exists(store_path='/test-store5') self.assertTrue(check_store) self.assertTrue(isinstance(document_store, OJAIDocumentStore)) for doc in DICT_STREAM: document = connection.new_document(dictionary=doc) # print('Insert document with ID: ' + str(document.get_id())) document_store.insert_or_replace(doc=document)
def test_delete_document(self): connection = ConnectionFactory.get_connection(connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) if connection.is_store_exists(store_path='/delete-test-store1'): document_store = connection.get_store(store_path='/delete-test-store1') else: document_store = connection.create_store(store_path='/delete-test-store1') document = None for doc in DICT_STREAM: document = connection.new_document(dictionary=doc) document_store.insert_or_replace(doc=document) document_store.delete(doc=document) self.assertEqual(document_store.find_by_id('id10'), {})
def test_find_timeout_error(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) options = { 'ojai.mapr.query.include-query-plan': True, 'ojai.mapr.query.timeout-milliseconds': 2147483648, 'ojai.mapr.query.result-as-document': False } if connection.is_store_exists(store_path='/find-test-store2'): document_store = connection.get_store( store_path='/find-test-store2') else: document_store = connection.create_store( store_path='/find-test-store2') with self.assertRaises(IllegalArgumentError): document_store.find(options=options)
def test_find_table_not_found(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) query = OJAIQuery().select(['_id', 'test_int', 'test_str', 'test_dict', 'test_list', 'test_null'])\ .build() connection.delete_store('/find-test-store3') self.assertFalse(connection.is_store_exists('/find-test-store3')) with self.assertRaises(StoreNotFoundError): document_store = connection.get_store( store_path='/find-test-store3') query_result = document_store.find(query) for doc in query_result: print(doc)
def test_create_table_error(self): connection = ConnectionFactory.get_connection( connection_str=CONNECTION_STR, options=CONNECTION_OPTIONS) connection.delete_store(store_path='/test-store2') before_create = connection.is_store_exists(store_path='/test-store2') self.assertFalse(before_create) store = connection.create_store(store_path='/test-store2') self.assertTrue(isinstance(store, OJAIDocumentStore)) self.assertTrue(connection.is_store_exists(store_path='/test-store2')) # connection.delete_store(store_path='/test-store2') with self.assertRaises(StoreAlreadyExistsError): connection.create_store(store_path='/test-store2') after_create = connection.is_store_exists(store_path='/test-store2') self.assertTrue(after_create) delete_response = connection.delete_store(store_path='/test-store2') self.assertTrue(delete_response)