def request_file_from_upstream(self):
# 不存在本地缓存，也不存在lock文件，向上游请求下载
try:
yield semaphore.acquire() # 文件下载临界区，防止AsyncHTTPClient资源耗尽
self.temp_file = open(self.file_path, 'wb')
http_client = AsyncHTTPClient()
sep = '' if config['upstream'].endswith('/') else '/'
url = '{upstream}{sep}file/{filename}'.format(
upstream=config['upstream'], sep=sep, filename=self.file_name)
response = yield http_client.fetch(
url,
validate_cert=False,
streaming_callback=self.on_file_chunk,
connect_timeout=config.get('file_service_connect_timeout',
3600.0),
request_timeout=config.get('file_service_request_timeout',
3600.0))
self.generate_response(response)
except Exception as exc:
logger.error('#%d Error while fetching %s: %s',
id(self.request),
self.file_name,
exc,
exc_info=True)
self.send_error(500, message=exc)
finally:
yield semaphore.release()
self.close_file_resource()
def download_pkg(self, filename):
yield self.reporter.log_ok('Begin to download package '
'{} ...'.format(filename))
down_url = self.download_url + '?filename=' + filename
try:
response = yield self.client.fetch(
down_url,
connect_timeout=config.get('file_service_connect_timeout',
3600.0),
request_timeout=config.get('file_service_request_timeout',
3600.0),
validate_cert=False)
if response.code == 200:
if not nfs.exists(PKG_CACHE_DIR):
os.makedirs(PKG_CACHE_DIR)
nfs.copy(response.body, nfs.join(PKG_CACHE_DIR, filename))
yield self.reporter.log_ok(
'Download package {} success'.format(filename))
else:
raise MessageError(
'Download package {} failed, reason: {}!'.format(
filename, response.body))
except HTTPError as e:
raise MessageError(
'Download package {} failed, reason: {}, {}!'.format(
filename, e, e.message))
def get_first_report():
platform = pf.get_platform()
ips = [{
'ip': info.ipv4,
'netmask': info.netmask
} for info in sysutil.network_info()]
physical_memory = round(sysutil.physical_memory() / (1024 * 1024 * 1024.0),
2)
disk = round(
sum(part.total
for part in sysutil.disk_info()) / (1024 * 1024 * 1024.0), 2)
first_report = {
'id': config['id'],
'network_domain': config['network_domain'],
'version': config['version'],
'system': platform.system,
'dist': platform.dist,
'dist_version': platform.version,
'arch': platform.cpu,
'kernel': platform.kernel,
'ips': ips,
'ip': config.get('ip') or sysutil.main_ip(),
'home': settings.ROOT_DIR,
'is_virtual': sysutil.is_virtual_machine(),
'node_type': sysutil.node_type(),
'physical_cpu_count': sysutil.physical_cpu_count(),
'physical_memory': physical_memory,
'disk': disk
}
logger.debug(first_report)
return first_report
def admin(request):
token = request.args.get(b'token')
if token and token[0] == config.get('admintoken', '').encode():
# auth
return "you're in!"
else:
raise ValueError('invalid security token')
def eventloop(self):
while True:
task_id = None
task_ins = None
try:
for task_id, task_ins in task.running_tasks.items():
if task_ins.is_stopped():
result = ''
if task_ins.exit_code != 0 and task_ins.exc_info:
result = task_ins.exc_info
logger.error('Task <%d> exited by code %d: %s',
task_id, task_ins.exit_code, result)
if (task_ins.action
not in TaskSyncScheduler.DO_NOT_RESULT):
message = TaskResultMessage(
task_id=task_id,
result=result,
exit_code=task_ins.exit_code,
seq=-1,
is_aborted=task_ins.is_aborted)
result = yield self._transfer.send(message)
if not result:
yield cache.put_message(message)
task.running_tasks.pop(task_id, None)
except Exception as exc:
logger.error('Error while syncing tasks: %s', exc)
logger.info('Current task id: %s', task_id)
logger.info('Current task instance: %s', task_ins)
yield gen.sleep(config.get('task_sync_interval', 5))
def __init__(self, modules, io_loop=None, reload_config=True):
self.io_loop = io_loop if io_loop else ioloop.IOLoop.current()
self.client = AsyncHTTPClient(io_loop=self.io_loop)
self.circle_client = AsyncCircleClient()
self.modules = modules
self.reload_config = reload_config
self.download_url = 'http://127.0.0.1:{}/file' \
''.format(config.get('framework_port', '16600'))
def init_application():
app = web.Application([(r'/task/log', TaskLogHandler),
(r'/task/result', TaskResultHandler),
(r'/file', FileHandler)])
logger.info('Framework listen on port %d', config['framework_port'])
app.listen(config['framework_port'],
address='127.0.0.1',
max_body_size=config.get('framework_max_body_size',
1024 * 1024))
def wait_for_file_complete(self):
logger.info('#%d File lock exists, waiting for complete: %s',
id(self.request), self.file_path)
lock_watch_interval = config.get('file_service_lock_watch_interval',
5.0)
current_timeout = 0.0
request_timeout = config.get('file_service_request_timeout', 3600.0)
while current_timeout < request_timeout:
yield gen.sleep(lock_watch_interval)
current_timeout += lock_watch_interval
if not nfs.exists(self.lock_file) and nfs.exists(self.file_path):
self.write(self.file_path) # 文件缓存完毕，返回本地缓存文件的路径
return
else:
logger.info('#%d Waiting for file complete: %s',
id(self.request), self.file_path)
# 等待文件缓存超时
self.send_error(504, message='Waiting for file complete timeout')
def __init__(self):
self._cursor = self.get_cursor()
url = urlparse.urljoin(config['upstream'], 'dispatcher/message')
self._base_url = utils.update_query_params(url, {
'tenant': config['tenant'],
'id': config['id']
})
self._retry_times = 0
self._retry_interval = config.get('message_fetch_retry_interval', 5)
self._http_client = httpclient.AsyncHTTPClient(io_loop=get_io_loop())
from tornado import gen
# coding: utf-8
import requests
from urlparse import urljoin
from tornado import gen
from tornado.queues import Queue
from tornado.escape import json_encode
from tornado.httpclient import AsyncHTTPClient
from tornado.ioloop import IOLoop
from framework.config import config
from framework.utils import update_query_params
from framework.message.models import (TaskLogMessage, TaskResultMessage)
TASK_URL = 'http://127.0.0.1:{}/task/'.format(
config.get('framework_port', '16600'))
class EngineType(object):
__slots__ = ('REQUESTS', 'TORNADO')
REQUESTS = 1
TORNADO = 2
@classmethod
def types_str(cls):
return ', '.join('{}.{}'.format(cls.__name__, t)
for t in cls.__slots__)
class LogType(object):
__slots__ = ('LOG', 'RESULT')
LOG = 1
import os
import nfs
import os
import logging
import time
from tornado import web, gen
from tornado.locks import Semaphore
from tornado.httpclient import AsyncHTTPClient
from framework import settings
from framework.config import config
MAX_BODY_SIZE = 4 * 1024.0 * 1024.0 * 1024.0 # 4GB
GMT_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
AsyncHTTPClient.configure(None, max_body_size=MAX_BODY_SIZE)
logger = logging.getLogger('default')
semaphore = Semaphore(config.get('file_service_semaphore', 5))
class FileHandler(web.RequestHandler):
@gen.coroutine
def get(self):
self.file_name = self.get_argument('filename') # type: str
self.space_dir = nfs.join(settings.REPO_DIR,
settings.REPO_ANT_SPACENAME)
if not nfs.exists(self.space_dir):
nfs.makedirs(self.space_dir)
self.file_path = nfs.join(self.space_dir, self.file_name)
lock_file_name = nfs.extsep + self.file_name + nfs.extsep + 'lock'
self.lock_file = nfs.join(self.space_dir, lock_file_name)
logger.info('#%d Request file: %s', id(self.request), self.file_name)