def deploy_all(self): # active nodes nodes = db_manager.list('nodes', {'status': NodeStatus.ONLINE}) # all spiders spiders = db_manager.list('spiders', {'cmd': {'$exists': True}}) # iterate all nodes for node in nodes: node_id = node['_id'] for spider in spiders: spider_id = spider['_id'] spider_src = spider['src'] output_file_name = '%s_%s.zip' % (datetime.now().strftime( '%Y%m%d%H%M%S'), str(random())[2:12]) output_file_path = os.path.join(PROJECT_TMP_FOLDER, output_file_name) # zip source folder to zip file zip_file(source_dir=spider_src, output_filename=output_file_path) # upload to api files = {'file': open(output_file_path, 'rb')} r = requests.post( 'http://%s:%s/api/spiders/%s/deploy_file?node_id=%s' % ( node.get('ip'), node.get('port'), spider_id, node_id, ), files=files) return {'status': 'ok', 'message': 'success'}
def deploy(self, id: str) -> (dict, tuple): """ Submit HTTP requests to deploy the given spider to all nodes. :param id: :return: """ spider = db_manager.get('spiders', id=id) nodes = db_manager.list('nodes', {'status': NodeStatus.ONLINE}) for node in nodes: node_id = node['_id'] output_file_name = '%s_%s.zip' % ( datetime.now().strftime('%Y%m%d%H%M%S'), str(random())[2:12]) output_file_path = os.path.join(PROJECT_TMP_FOLDER, output_file_name) # zip source folder to zip file zip_file(source_dir=spider['src'], output_filename=output_file_path) # upload to api files = {'file': open(output_file_path, 'rb')} r = requests.post( 'http://%s:%s/api/spiders/%s/deploy_file?node_id=%s' % ( node.get('ip'), node.get('port'), id, node_id, ), files=files) # TODO: checkpoint for errors return {'code': 200, 'status': 'ok', 'message': 'deploy success'}
def deploy(self, id): spider = db_manager.get('spiders', id=id) nodes = db_manager.list('nodes', {}) for node in nodes: node_id = node['_id'] output_file_name = '%s_%s.zip' % ( datetime.now().strftime('%Y%m%d%H%M%S'), str(random())[2:12]) output_file_path = os.path.join(PROJECT_TMP_FOLDER, output_file_name) # zip source folder to zip file zip_file(source_dir=spider['src'], output_filename=output_file_path) # upload to api files = {'file': open(output_file_path, 'rb')} r = requests.post( 'http://%s:%s/api/spiders/%s/deploy_file?node_id=%s' % ( node.get('ip'), node.get('port'), id, node_id, ), files=files) return {'code': 200, 'status': 'ok', 'message': 'deploy success'}