def _write_to_store(self, article_id, data): """Write a (compressed) article to store. Article ids is turned into something filesystem safe here. """ safe_id = article_id.replace('/', '_') safe_id += '.html.gz' BaseClient._write_to_store(self, safe_id, data)
def __init__(self, client_id, base_url, store_dir=None): """DiggClient Constructor.""" # Parent class constructor BaseClient.__init__(self, client_id, base_url, store_dir) # Informing the version of our ArticleRetriver self.headers["client-arver'"] = articleretriever_version # Registering Command Handlers self.handlers['ARTICLE'] = self.article
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals, absolute_import import requests from . import utils from django.test import override_settings from ci.tests import utils as test_utils from client import JobGetter, BaseClient from mock import patch from ci.tests import DBTester BaseClient.setup_logger() @override_settings(INSTALLED_GITSERVERS=[test_utils.github_config()]) class Tests(DBTester.DBTester): def create_getter(self): self.client_info = utils.default_client_info() getter = JobGetter.JobGetter(self.client_info) return getter @patch.object(requests, 'get') def test_get_possible_jobs(self, mock_get): g = self.create_getter() # test the non error operation job_response = {"jobs": "jobs"}
def run(self, exit_if=None): """ Main client loop. Polls the server for jobs and runs them. Loads the proper environment for each config. Inputs: exit_if: Optional function with a single parameter (which is passed as self; this client) that returns a bool. If the function returns True, exit the poll loop at that time (checked at the end of the poll loop). Used in testing. Returns: None """ logger.info('Starting client {}'.format(self.get_client_info('client_name'))) logger.info('Build root: {}'.format(self.get_build_root())) if exit_if is not None and (not callable(exit_if) or len(signature(exit_if).parameters) != 1): raise BaseClient.ClientException('exit_if must be callable with a single parameter (the client)') # Deprecated environment setting; you should start the client with the vars set instead if hasattr(settings, 'ENVIRONMENT') and settings.ENVIRONMENT is not None: logger.info('DEPRECATED: Set environment variables manually instead of using settings.ENVIRONMENT') for k, v in settings.ENVIRONMENT.items(): self.set_environment(k, v) logger.info('Available configs: {}'.format(' '.join([config for config in self.get_client_info("build_configs")]))) while True: if self.get_client_info('manage_build_root') and self.build_root_exists(): logger.warning("BUILD_ROOT {} already exists at beginning of poll loop; removing" .format(self.get_build_root())) self.remove_build_root() ran_job = False for server in settings.SERVERS: if self.cancel_signal.triggered or self.graceful_signal.triggered or self.runner_error: break try: if self.check_server(server): ran_job = True except Exception: logger.debug("Error: %s" % traceback.format_exc()) break if self.cancel_signal.triggered or self.graceful_signal.triggered: logger.info("Received signal...exiting") break if self.runner_error: logger.info("Error in runner...exiting") break if exit_if is not None: should_exit = exit_if(self) if type(should_exit) != bool: raise BaseClient.ClientException('exit_if must return type bool') if should_exit: break if not ran_job: time.sleep(self.get_client_info('poll')) if self.get_client_info('manage_build_root') and self.build_root_exists(): logger.warning("BUILD_ROOT {} still exists after exiting poll loop; removing" .format(self.get_build_root())) self.remove_build_root()
def commandline_client(args): parser = argparse.ArgumentParser() parser.add_argument('--client', dest='client', type=int, help='The number of the client.', required=True) parser.add_argument('--daemon', dest='daemon', choices=['start', 'stop', 'restart', 'none'], help="Start a UNIX daemon.", required=True) parser.add_argument( "--configs", dest='configs', nargs='+', help="The configurations this client supports (eg 'linux-gnu')") parser.add_argument( "--env", dest='env', nargs=2, action='append', help="Sets a client environment variable (example: VAR_NAME VALUE)") parser.add_argument("--build-root", type=str, dest='build_root', help="Sets the build root") parser.add_argument( "--user-client-suffix", action='store_true', dest='user_client_suffix', help= 'Adds the user to client name as a suffix, i.e, sets the name to <hostname>_<user>_<client number>' ) parsed = parser.parse_args(args) home = os.environ.get("CIVET_HOME", os.path.join(os.environ["HOME"], "civet")) log_dir = '{}/logs'.format(home) client_name = socket.gethostname() if parsed.user_client_suffix: client_name += '_{}'.format(pwd.getpwuid(os.getuid())[0]) client_name += '_{}'.format(parsed.client) client_info = { "url": "", "client_name": client_name, "server": "", "servers": [], "ssl_verify": False, "ssl_cert": "", "log_file": "", "log_dir": log_dir, "build_key": "", "single_shot": False, "poll": 60, "daemon_cmd": parsed.daemon, "request_timeout": 120, "update_step_time": 30, "server_update_timeout": 5, # This needs to be bigger than update_step_time so that # the ping message doesn't become the default message "server_update_interval": 50, "max_output_size": 5 * 1024 * 1024 } c = INLClient.INLClient(client_info) if parsed.daemon == 'start' or parsed.daemon == 'restart' or platform.system( ) == "Windows": if not parsed.configs: raise BaseClient.ClientException('--configs must be provided') if parsed.build_root: build_root = parsed.build_root else: build_root = '{}/build_{}'.format(home, parsed.client) for config in parsed.configs: c.add_config(config) c.set_environment('BUILD_ROOT', build_root) c.set_environment('CIVET_HOME', home) c.set_environment('CIVET_CLIENT_NUMBER', parsed.client) if parsed.env: for var, value in parsed.env: c.set_environment(var, value) return c, parsed.daemon
def client(): return BaseClient('http://127.0.0.1')
def create_inl_client(log_dir=None, log_file=None): client_info = default_client_info() BaseClient.setup_logger() # logger on stdout return INLClient.INLClient(client_info)