def main(): gclient.create_dir_if_not_exist("/galaxy/aa-d/test_from_python") print(gclient.list_dirs_in_dir("/galaxy/aa-d/")) print(gclient.list_dirs_in_dir("/home/pslx/Downloads")) print(gclient.get_attr("/home/pslx/Downloads")) print(gclient.get_attr("/galaxy/aa-d/test_from_python")) gclient.create_file_if_not_exist("/galaxy/aa-d/test_from_python/test.txt") gclient_ext.cp_file("/galaxy/aa-d/test_from_python/test.txt", "/galaxy/aa-d/test_from_python/test1.txt") gclient_ext.mv_file("/galaxy/aa-d/test_from_python/test.txt", "/galaxy/aa-d/test_from_python/test3.txt") print(gclient.list_dirs_in_dir_recursive("/galaxy/aa-d/test_from_python")) print(gclient.list_files_in_dir_recursive("/galaxy/aa-d/test_from_python")) t = time.time() gclient.read("/galaxy/aa-d/large_test.txt") print(time.time() - t) data = gclient.read_multiple( ["/galaxy/aa-d/test_from_python/test1.txt", "/galaxy/aa-d/test3.txt"]) for key, val in data.items(): print(key, val) data = gclient_ext.read_txts( ["/galaxy/aa-d/test_from_python/test1.txt", "/galaxy/aa-d/test3.txt"]) for key, val in data.items(): print(key, val) print(gclient.list_cells()) print(gclient.read('/galaxy/aa-d/test.pb')) print(gclient_ext.read_txt('/galaxy/aa-d/test.pb')) message = TestMessage() message.name = "test" gclient_ext.write_proto_message('/galaxy/aa-d/test1.pb', message) print(gclient_ext.read_proto_message('/galaxy/aa-d/test1.pb', TestMessage)) print(gclient.check_health("aa")) gclient.write_multiple(path_data_map={ '/galaxy/aa-d/test_from_python/test2.txt': '123', '/galaxy/aa-d/test_from_python/test4.txt': '234', }, mode='a') gclient_ext.write_proto_messages( path_data_map={ '/galaxy/aa-d/test_from_python/test2.pb': message, '/galaxy/aa-d/test_from_python/test4.pb': message, }) print( gclient_ext.read_proto_messages(paths=[ '/galaxy/aa-d/test_from_python/test2.pb', '/galaxy/aa-d/test_from_python/test4.pb' ], message_type=TestMessage)) print(gclient.create_file_if_not_exist("/LOCAL/test/test.txt")) print(gclient.create_file_if_not_exist("/galaxy/aa-d/test/test.txt"))
def get_containers_info(): containers_info = [] existing_containers = {} all_proto_files = set() if not FileUtil.is_local_path(backend_folder): all_cells = [''] else: all_cells = gclient.list_cells() for cell_name in all_cells: folder = FileUtil.convert_local_to_cell_path( path=backend_folder, cell=cell_name) proto_files = FileUtil.list_files_in_dir(folder) all_proto_files = all_proto_files.union(set(proto_files)) for proto_file in all_proto_files: storage = ProtoTableStorage() storage.initialize_from_file( file_name=proto_file ) raw_data = storage.read_all() if not raw_data: continue key = sorted(raw_data.keys())[-1] val = raw_data[key] result_proto = ProtoUtil.any_to_message( message_type=ContainerBackendValue, any_message=val ) ttl = result_proto.ttl if ttl > 0 and result_proto.updated_time and TimezoneUtil.cur_time_in_pst() - TimezoneUtil.cur_time_from_str( result_proto.updated_time) >= datetime.timedelta(days=ttl): FileUtil.remove_file(storage.get_file_name()) else: container_info = { 'container_name': result_proto.container_name, 'status': ProtoUtil.get_name_by_value( enum_type=Status, value=result_proto.container_status), 'updated_time': result_proto.updated_time, 'mode': ProtoUtil.get_name_by_value(enum_type=ModeType, value=result_proto.mode), 'data_model': ProtoUtil.get_name_by_value( enum_type=DataModelType, value=result_proto.data_model), 'run_cell': result_proto.run_cell, 'snapshot_cell': result_proto.snapshot_cell, } if container_info['container_name'] not in existing_containers: existing_containers[container_info['container_name']] = container_info['updated_time'] containers_info.append(container_info) else: if container_info['updated_time'] >= existing_containers[container_info['container_name']]: containers_info.append(container_info) return containers_info
def _check_health(): for cell in gclient.list_cells(): logger.info("Checking health for cell " + cell + '.') health_json = gclient.check_health(cell) try: health = json.loads(health_json) if not health['healthy']: logger.error('Cell ' + cell + ' not healthy!') else: logger.info('Cell ' + cell + ' healthy.') except Exception as err: logger.error('Checking health for cell ' + cell + ' failed with error ' + str(err) + '.')
def get(self, p=''): p = p.strip() logger.info('Getting GET request from ip [' + request.remote_addr + '].') if p and p[0] != '/': p = '/' + p total = {'size': 0, 'dir': 0, 'file': 0} contents = [] if p == ROOT or not p: cells = gclient.list_cells() for cell in cells: path_name = os.path.join(ROOT, cell + '-d') try: attr = json.loads(gclient.get_attr(path_name)) except Exception as err: logger.error('Getting attribute of ' + path_name + ' failed with error ' + str(err) + '.') continue info = { 'name': path_name, 'mtime': int(attr['mtime']), 'type': 'dir', 'size': 0 } total['dir'] += 1 contents.append(info) page = render_template('index.html', path=ROOT, contents=contents, total=total, hide_dotfile="yes") res = make_response(page, 200) else: if gclient.dir_or_die(p): contents = [] total = {'size': 0, 'dir': 0, 'file': 0} for filename, statbuf in gclient_ext.list_all_in_dir( p).items(): if filename in ignored or filename[0] == '.': continue stat_res = json.loads(statbuf) info = {} info['name'] = filename info['mtime'] = int(stat_res['mtime']) ft = get_type(int(stat_res['mode'])) info['type'] = ft total[ft] += 1 sz = int(stat_res['size']) if 'size' in stat_res else 0 info['size'] = sz total['size'] += sz contents.append(info) page = render_template('index.html', path=p, contents=contents, total=total, hide_dotfile="yes") res = make_response(page, 200) elif gclient.file_or_die(p): res = file_response(p) else: res = make_response('Not found', 404) return res
help='The port for the viewer') parser.add_argument('--cell', dest='cell', default='bb', help='The cell to store the log') parser.add_argument('--debug', dest='debug', action='store_true', help='Whether to use debug mode') parser.add_argument('--health_check_interval', dest='health_check_interval', default=60, help='"Interval for checking server health') args = parser.parse_args() assert args.username is not None and args.password is not None and args.port is not None, 'Invalid arguments' assert args.cell in gclient.list_cells(), "Invalid cell" ROOT = '/galaxy/' APP_NAME = 'galaxy_viewer' LOG_DIR = '/LOCAL/ttl=7d/logs/' + APP_NAME + '/' logger = glogging.get_logger(APP_NAME, LOG_DIR) galaxy_viewer = Flask(APP_NAME, static_url_path='/assets', static_folder='assets') galaxy_viewer.config.update(SECRET_KEY=APP_NAME) galaxy_viewer.config.update(SESSION_COOKIE_NAME=APP_NAME + '_cookie') ignored = [ '.bzr', '$RECYCLE.BIN', '.DAV', '.DS_Store', '.git', '.hg', '.htaccess',