def _write_to_store(self, article_id, data):
"""Write a (compressed) article to store.
Article ids is turned into something filesystem safe here.
"""
safe_id = article_id.replace('/', '_')
safe_id += '.html.gz'
BaseClient._write_to_store(self, safe_id, data)
def __init__(self, client_id, base_url, store_dir=None):
"""DiggClient Constructor."""
# Parent class constructor
BaseClient.__init__(self, client_id, base_url, store_dir)
# Informing the version of our ArticleRetriver
self.headers["client-arver'"] = articleretriever_version
# Registering Command Handlers
self.handlers['ARTICLE'] = self.article
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import unicode_literals, absolute_import
import requests
from . import utils
from django.test import override_settings
from ci.tests import utils as test_utils
from client import JobGetter, BaseClient
from mock import patch
from ci.tests import DBTester
BaseClient.setup_logger()
@override_settings(INSTALLED_GITSERVERS=[test_utils.github_config()])
class Tests(DBTester.DBTester):
def create_getter(self):
self.client_info = utils.default_client_info()
getter = JobGetter.JobGetter(self.client_info)
return getter
@patch.object(requests, 'get')
def test_get_possible_jobs(self, mock_get):
g = self.create_getter()
# test the non error operation
job_response = {"jobs": "jobs"}
def run(self, exit_if=None):
"""
Main client loop. Polls the server for jobs and runs them.
Loads the proper environment for each config.
Inputs:
exit_if: Optional function with a single parameter (which is passed as self; this client)
that returns a bool. If the function returns True, exit the poll loop at that
time (checked at the end of the poll loop). Used in testing.
Returns:
None
"""
logger.info('Starting client {}'.format(self.get_client_info('client_name')))
logger.info('Build root: {}'.format(self.get_build_root()))
if exit_if is not None and (not callable(exit_if) or len(signature(exit_if).parameters) != 1):
raise BaseClient.ClientException('exit_if must be callable with a single parameter (the client)')
# Deprecated environment setting; you should start the client with the vars set instead
if hasattr(settings, 'ENVIRONMENT') and settings.ENVIRONMENT is not None:
logger.info('DEPRECATED: Set environment variables manually instead of using settings.ENVIRONMENT')
for k, v in settings.ENVIRONMENT.items():
self.set_environment(k, v)
logger.info('Available configs: {}'.format(' '.join([config for config in self.get_client_info("build_configs")])))
while True:
if self.get_client_info('manage_build_root') and self.build_root_exists():
logger.warning("BUILD_ROOT {} already exists at beginning of poll loop; removing"
.format(self.get_build_root()))
self.remove_build_root()
ran_job = False
for server in settings.SERVERS:
if self.cancel_signal.triggered or self.graceful_signal.triggered or self.runner_error:
break
try:
if self.check_server(server):
ran_job = True
except Exception:
logger.debug("Error: %s" % traceback.format_exc())
break
if self.cancel_signal.triggered or self.graceful_signal.triggered:
logger.info("Received signal...exiting")
break
if self.runner_error:
logger.info("Error in runner...exiting")
break
if exit_if is not None:
should_exit = exit_if(self)
if type(should_exit) != bool:
raise BaseClient.ClientException('exit_if must return type bool')
if should_exit:
break
if not ran_job:
time.sleep(self.get_client_info('poll'))
if self.get_client_info('manage_build_root') and self.build_root_exists():
logger.warning("BUILD_ROOT {} still exists after exiting poll loop; removing"
.format(self.get_build_root()))
self.remove_build_root()
def commandline_client(args):
parser = argparse.ArgumentParser()
parser.add_argument('--client',
dest='client',
type=int,
help='The number of the client.',
required=True)
parser.add_argument('--daemon',
dest='daemon',
choices=['start', 'stop', 'restart', 'none'],
help="Start a UNIX daemon.",
required=True)
parser.add_argument(
"--configs",
dest='configs',
nargs='+',
help="The configurations this client supports (eg 'linux-gnu')")
parser.add_argument(
"--env",
dest='env',
nargs=2,
action='append',
help="Sets a client environment variable (example: VAR_NAME VALUE)")
parser.add_argument("--build-root",
type=str,
dest='build_root',
help="Sets the build root")
parser.add_argument(
"--user-client-suffix",
action='store_true',
dest='user_client_suffix',
help=
'Adds the user to client name as a suffix, i.e, sets the name to <hostname>_<user>_<client number>'
)
parsed = parser.parse_args(args)
home = os.environ.get("CIVET_HOME",
os.path.join(os.environ["HOME"], "civet"))
log_dir = '{}/logs'.format(home)
client_name = socket.gethostname()
if parsed.user_client_suffix:
client_name += '_{}'.format(pwd.getpwuid(os.getuid())[0])
client_name += '_{}'.format(parsed.client)
client_info = {
"url": "",
"client_name": client_name,
"server": "",
"servers": [],
"ssl_verify": False,
"ssl_cert": "",
"log_file": "",
"log_dir": log_dir,
"build_key": "",
"single_shot": False,
"poll": 60,
"daemon_cmd": parsed.daemon,
"request_timeout": 120,
"update_step_time": 30,
"server_update_timeout": 5,
# This needs to be bigger than update_step_time so that
# the ping message doesn't become the default message
"server_update_interval": 50,
"max_output_size": 5 * 1024 * 1024
}
c = INLClient.INLClient(client_info)
if parsed.daemon == 'start' or parsed.daemon == 'restart' or platform.system(
) == "Windows":
if not parsed.configs:
raise BaseClient.ClientException('--configs must be provided')
if parsed.build_root:
build_root = parsed.build_root
else:
build_root = '{}/build_{}'.format(home, parsed.client)
for config in parsed.configs:
c.add_config(config)
c.set_environment('BUILD_ROOT', build_root)
c.set_environment('CIVET_HOME', home)
c.set_environment('CIVET_CLIENT_NUMBER', parsed.client)
if parsed.env:
for var, value in parsed.env:
c.set_environment(var, value)
return c, parsed.daemon
def client():
return BaseClient('http://127.0.0.1')
def create_inl_client(log_dir=None, log_file=None):
client_info = default_client_info()
BaseClient.setup_logger() # logger on stdout
return INLClient.INLClient(client_info)
