class RpcContext(object):
def __init__(self, pool, config):
self.pool = weakref.proxy(pool)
self.proxy = ClusterRpcProxy(config, context_data=pool.context_data, timeout=pool.timeout)
self.rpc = self.proxy.start()
def stop(self):
self.proxy.stop()
self.proxy = None
self.rpc = None
def __enter__(self):
return self.rpc
def __exit__(self, exc_type, exc_value, traceback, **kwargs):
try:
if exc_type == RuntimeError and (
exc_value == "This consumer has been stopped, and can no longer be used"
or exc_value == "This consumer has been disconnected, and can no longer be used"):
self.pool._clear()
self.pool._reload() # reload all worker
self.stop()
elif exc_type == ConnectionError: # maybe check for RpcTimeout, as well
# self.pool._clear()
self.pool._reload(1) # reload atmost 1 worker
self.stop()
else:
self.pool._put_back(self)
except ReferenceError: # pragma: no cover
# We're detached from the parent, so this context
# is going to silently die.
self.stop()
class NfvoRpcClient(Nfvo):
def __init__(self, config, nfvo_agent_name):
self.config = config
self.nfvo_agent_name = nfvo_agent_name
self.cluster_rpc_c = ClusterRpcProxy(config)
self.cluster_rpc = self.cluster_rpc_c.start()
def deploy_instance(self, ctx, args):
agent = getattr(self.cluster_rpc, self.nfvo_agent_name)
return agent.deploy_instance.call_async(ctx, args)
def exec_action(self, ctx, args):
agent = getattr(self.cluster_rpc, self.nfvo_agent_name)
return agent.exec_action.call_async(ctx, args)
def exec_custom_action(self, ctx, args):
agent = getattr(self.cluster_rpc, self.nfvo_agent_name)
return agent.exec_custom_action.call_async(ctx, args)
def delete_instance(self, ctx, args):
agent = getattr(self.cluster_rpc, self.nfvo_agent_name)
return agent.delete_instance.call_async(ctx, args)
def stop(self):
self.cluster_rpc_c.stop()
def checked():
if request.method == 'POST':
school = request.form.get("school")
institution = request.form.get("institution")
name = request.form.get("name")
title = request.form.get("title")
sex = request.form.get("sex")
email = request.form.get("email")
tel = request.form.get("tel")
birthyear = request.form.get("birthyear")
fields = request.form.get("fields")
with ClusterRpcProxy(CONFIG) as rpc:
institution1 = rpc.teacher_update.get_institutionId(
school, institution)
teacher_id = rpc.teacher_update.get_teacher_id(name, institution1[0],
institution1[1])
if teacher_id:
rpc.teacher_update.update_teacher_info(title, sex, email, tel,
birthyear, fields,
teacher_id)
else:
rpc.teacher_update.insert_teacher_newinfo(name, title, sex,
institution1[0],
institution1[1], email,
tel, birthyear, fields)
with ClusterRpcProxy(CONFIG) as rpc:
rpc.teacher_update.delete_teacher_info(name, title, sex, school,
institution, email, tel,
birthyear, fields)
teacherinfo = rpc.teacher_update.get_info()
return render_template("check_info.html", teacherinfo=teacherinfo)
def compute():
if request.method == 'POST':
schoolName = request.form.get('schoolName')
institutionName = request.form.get('institutionName')
if institutionName:
with ClusterRpcProxy(CONFIG) as rpc:
institution_Id = rpc.test.get_institutionId(
schoolName, institutionName)
try:
teacher_name = rpc.test.get_academicianName(institution_Id[0])
for i in teacher_name:
i[0] = i[0].lstrip("['").rstrip("']")
if len(teacher_name) == 0:
teacher_name = ["", ""]
except BaseException as e:
teacher_name = ["", ""]
return render_template("outcome.html", teacher_name=teacher_name)
else:
with ClusterRpcProxy(CONFIG) as rpc:
school_Id = rpc.test.get_schoolId(schoolName)
try:
teacher = rpc.test.get_teacher_name_and_insId(school_Id)
for i in teacher:
institutionName = rpc.test.get_institution_name(i[1])
i[1] = institutionName[0].lstrip("['").rstrip("']")
if len(teacher) == 0:
teacher = [["", "", ""]]
except BaseException as e:
teacher = [["", "", ""]]
return render_template("outcome.html", teacher=teacher)
def __init__(self, settings=None):
if settings is None:
settings = {}
super(TelegramRunner, self).__init__(settings)
nameko_settings = settings['nameko']
""" :type : dict """
telegram_settings = settings['telegram']
""" :type : dict """
if self._prePath is not None:
telegram_settings.setdefault('path_prefix', self._prePath)
if self._prePath is not None:
nameko_settings.setdefault('path_prefix', self._prePath)
self.dispatcher = event_dispatcher(nameko_settings)
self.telegram = TelegramClient(telegram_settings)
self.telegram.get_user_external = self._rpc_service_user_get_authorized
self.service = StandaloneTelegramService()
self.service.dispatch_intent = self._dispatch_intent
self.service.telegram = self.telegram
nameko_settings['service_name'] = self.service.name
nameko_settings['service'] = self.service
nameko_settings['allowed_functions'] = self.service.allowed
self.listener = RPCListener(nameko_settings)
self._cluster_proxy = ClusterRpcProxy(
nameko_settings, timeout=nameko_settings.get('rpc_timeout', None)
)
self._proxy = None
self._done = threading.Event()
self._polling_timeout = settings.get('polling_interval', 2.0)
def __init__(self, pool, config):
self._pool = weakref.proxy(pool)
self._proxy = ClusterRpcProxy(config,
context_data=copy.deepcopy(
pool.context_data),
timeout=pool.timeout)
self._rpc = None
self._enable_rpc_call = False
class AuthProxy:
def __init__(self, token, config):
self.token = token
self.config = config
self.proxy = ClusterRpcProxy(
config, context_data={'miso_auth_token': self.token})
self.rpc = self.proxy.start()
def stop(self):
self.proxy.stop()
def index():
if request.method == 'POST':
#First, we need to get the selected effect from the request
selectedItem = request.form.get('effectHolder')
print('Selected Effect is: ' + selectedItem)
#Next, we need to get the blob file
if 'blob' in request.files:
file = request.files['blob']
file.save(
os.path.join(application.config['UPLOAD_FOLDER'],
'download.ogg'))
else:
print('No file found!')
#Then, we need to process the file depending on the selected effect
if selectedItem == 'reverbSmallRoom':
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.SigProc.reverbSmallRoom()
elif selectedItem == 'reverbCaveEffect':
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.SigProc.reverbReflectiveCave()
elif selectedItem == 'reverbConcertHall':
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.SigProc.reverbConcertHall()
elif selectedItem == 'miscReverseSong':
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.SigProc.miscReverseSong()
elif selectedItem == 'phaserDefault':
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.SigProc.phaserDefault()
elif selectedItem == 'phaserSpaceEffect':
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.SigProc.phaserSpaceEffect()
elif selectedItem == 'phaserSubtle':
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.SigProc.phaserSubtle()
elif selectedItem == 'miscSpeedUp2x':
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.SigProc.miscSpeedUp2x()
elif selectedItem == 'miscSlowDownHalf':
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.SigProc.miscSlowDownHalf()
elif selectedItem == 'miscReset':
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.SigProc.miscSlowDownHalf()
#Then, return the file
return send_file(
os.path.join(application.config['UPLOAD_FOLDER'], 'processed.ogg'))
elif request.method == 'GET':
return render_template('index.html', title='Testing')
class AsyncEvalServiceInvoker:
def __init__(self, conf):
self.rpc_proxy = ClusterRpcProxy(conf)
self.proxy = self.rpc_proxy.start()
def eval(self, method: EvalMethod,
inputs: List[DatahubTarget]) -> List[DatahubTarget]:
result = self.proxy.evaluation_service.eval(method, inputs)
print(result)
def stop(self):
self.rpc_proxy.stop()
def init_cloud_framework():
service_logger.info("%s %s %s ",
sys._getframe().f_code.co_name,
"RUNNING_PUBLIC_CLOUD_LIST:-",
RUNNING_PUBLIC_CLOUD_LIST)
service_logger.info("%s %s %s ",
sys._getframe().f_code.co_name,
"RUNNING_PRIVATE_CLOUD_LIST:-",
RUNNING_PRIVATE_CLOUD_LIST)
db_data = get_clouds()
with ClusterRpcProxy(CONFIG) as rpc:
rpc.initHandler.dispatchUpdateCloudTable(payload=db_data)
with ClusterRpcProxy(CONFIG) as rpc:
rpc.ClusterRegistryService.registerCloudInstance(
configDict=cloud_config)
def heavyprocess():
request_json = request.get_json(force=True)
process_wm = "Please wait the download, you'll check this endpoint GET - queue/:queueName for get results"
with ClusterRpcProxy(CONFIG) as rpc:
process = rpc.service_a.post_message.call_async(request_json)
queue_name = str(process.reply_event.queue_consumer.queue.name).strip()
return jsonify({'result': process_wm, 'queueName': queue_name}), 200
def callRpc(tweet):
"""
Post tweet to the microservice for processing
"""
with ClusterRpcProxy({"AMQP_URI":
os.environ.get(f"{ENV_AMQP_URI}")}) as rpc:
rpc.scraper_microservice.on_tweet(tweet)
def date_preprocess():
"""
Micro Service This API is made with Flask, Flasgger and Nameko
---
parameters:
- name: body
in: body
required: true
schema:
id: data
properties:
studyUID:
type: string
responses:
200:
description: Please wait the calculation, you'll receive an email with results
"""
studyuid = request.json.get('studyUID')
msg = "Please wait the calculation..."
with ClusterRpcProxy(CONFIG) as rpc:
print("receiving the request studyUID: %s" % studyuid)
result = rpc.service_dispatch.dispatching_load.call_async(studyuid)
print("put the requet into queue, result %s" % studyuid)
return msg, 200
def get(self, job_listing_id):
print("Request for job listing with id: " + job_listing_id)
es = Elasticsearch(hosts=["elastic"])
if (es.exists(index='joblistings',
doc_type='job-listing',
id=job_listing_id)):
print('Found the document in ElasticSearch')
doc = es.get(index='joblistings',
doc_type='job-listing',
id=job_listing_id)
return doc['_source']
print('Not found in ElasticSearch, trying a scrape')
with ClusterRpcProxy(CONFIG) as rpc:
listing = rpc.stack_overflow_job_listings_scraping_microservice.get_job_listing_info(
job_listing_id)
print(
"Microservice returned with a result - storing in ElasticSearch"
)
es.index(index='joblistings',
doc_type='job-listing',
id=job_listing_id,
body=listing)
return listing
def authen(authen: Authen):
with ClusterRpcProxy(broker_cfg) as rpc:
success = rpc.otp.authen(authen.email, authen.otp)
if success == 1:
rpc.otp.delete.call_async(authen.email)
return {'results': success}
def test_cluster_proxy_dict_access(container_factory, rabbit_manager,
rabbit_config):
container = container_factory(FooService, rabbit_config)
container.start()
with ClusterRpcProxy(rabbit_config) as proxy:
assert proxy['foobar'].spam(ham=3) == 3
def get_ticket():
"""Gets ticket by `ticket_id`
"""
ticket_id = request.args.get("ticket_id")
with ClusterRpcProxy(CONFIG) as rpc:
product = rpc.ticketService.get(ticket_id)
return Response(TicketSchema().dumps(product), mimetype='application/json')
def collect():
duration = request.json.get("duration")
query = request.json.get("query")
translate = request.json.get("translate")
with ClusterRpcProxy(CONFIG) as rpc:
rpc.collector.collect.call_async(duration, query, translate)
return "Success"
def home():
with ClusterRpcProxy(config) as cluster_rpc:
movies = cluster_rpc.show_time_service.get('Die Hard')
return render_template(
'./home.html',
value=movies,
)
def email():
"""
Micro Service Based Mail API
This API is made with Flask, Flasgger and Nameko
---
parameters:
- name: body
in: body
required: true
schema:
id: data
properties:
email:
type: string
subject:
type: string
msg:
type: string
responses:
200:
description: Email message has been submitted
"""
email = request.json.get('email')
subject = request.json.get('subject')
msg = request.json.get('msg')
with ClusterRpcProxy(CONFIG) as rpc:
rpc.mail.send(email, subject, msg)
return msg, 200
def post(self):
'''Upload a file and receive a token or uuid for further file retrieving after processing'''
# if request.content_type.startswith("multipart/form-data"):
args = upload_parser.parse_args()
audiofile = args['file'] #busca key
original_name = secure_filename(audiofile.filename)
log.info('An audio file uploaded with the name: %s', original_name)
infofile = fleep.get(audiofile.read(128))
log.info('file extension is: %s', infofile.extension)
log.info('file mimetype is: %s', infofile.mime)
log.info('file type is: %s', infofile.type)
if infofile.type_matches("audio") and (infofile.mime_matches("audio/mpeg") or infofile.mime_matches('audio/wav')) \
and (infofile.extension_matches('wav') or infofile.extension_matches('mp3')):
#Rename file before calling rpc method
generated_uuid = str(uuid.uuid4())
log.info('generated uuid: %s', generated_uuid)
audiofile.filename = f"{Path(original_name).stem.replace(' ','_')}_{generated_uuid}{Path(original_name).suffix}"
log.info('filename %s will be sent to queue', audiofile.filename)
try:
with ClusterRpcProxy(CONFIG) as rpc:
objResponse = { "uuid": f"{generated_uuid}","model": "Demucs","status" : "Queued" }
result = rpc.remote_call_demucs_service.call_demucs.call_async(audiofile.read(), audiofile.filename)
resp = Response(json.dumps(objResponse),200,headers={ "Content-Type" : "application/json" })
return resp
except (ConnectionRefusedError):
log.error('An error ocurred because call rpc method is not available')
api.abort(code=500, message="An internal server error ocurred. Unable to reach Demucs service")
except (OSError, socket.gaierror):
log.error('An error ocurred because was unable to connect to AMQP broker')
api.abort(code=500, message="An internal server error ocurred. Demucs service unavailable")
else:
log.info('Attemped to upload a file that is not mp3 or wav')
api.abort(code=415, message="File not valid.")
def compute():
"""
Micro Service Based Compute API, which also uses Mail and Customer Micro services via RPC communication
This API is made with Flask, Flasgger and Nameko
---
parameters:
- name: body
in: body
required: true
schema:
id: data
properties:
changeto:
type: string
enum:
- dolar
- euro
value:
type: integer
customerid:
type: integer
responses:
200:
description: Please wait the calculation, you'll receive an email with results
"""
changeto = request.json.get('changeto')
value = request.json.get('value')
customerid = request.json.get('customerid')
msg = "Please wait the calculation, you'll receive an email with results"
with ClusterRpcProxy(CONFIG) as rpc:
result = rpc.compute.compute.call_async(changeto, value, customerid)
return msg, 200
def create_ticket():
"""Create a new ticket - ticket data is posted as json
Example request ::
{
"id": "the_odyssey",
"title": "The Odyssey"
}
The response contains the new product ID in a json document ::
{"id": "1234"}
"""
schema = TicketSchema()
try:
# load input data through a schema (for validation)
# Note - this may raise `ValueError` for invalid json,
# or `ValidationError` if data is invalid.
ticket_data = schema.loads(request.get_data(as_text=True))
except ValueError as exc:
raise BadRequest("Invalid json: {}".format(exc))
# Create the product
with ClusterRpcProxy(CONFIG) as rpc:
rpc.ticketService.create(ticket_data)
return Response(json.dumps({'id': ticket_data['ticket_id']}),
mimetype='application/json')
def get(self, num_page=5, limit=5):
""" returns a list of brands """
with ClusterRpcProxy(CONFIG_RPC) as rpc:
response_data = rpc.query_brands.list(num_page, limit)
return Response(response=response_data,
status=200,
mimetype='application/json')
def get_otp(email: Email):
with ClusterRpcProxy(broker_cfg) as rpc:
otp = rpc.otp.create(email.email)
if otp != 0:
rpc.email_otp.send.call_async(otp, email.email)
return {'results': str(otp)}
def get(self, id):
""" returns a single brand item"""
with ClusterRpcProxy(CONFIG_RPC) as rpc:
response_data = rpc.query_brands.get(id)
return Response(response=response_data,
status=200,
mimetype='application/json')#json.loads(response_data)
def call_async_rpc(self, service_name, task_name):
"""
RPC Assinc call
"""
with ClusterRpcProxy(self.CONFIG) as cluster_rpc:
method = getattr(cluster_rpc[service_name], task_name)
method.call_async()
def get(self, name):
"""
This examples uses FlaskRESTful Resource
It works also with swag_from, schemas and spec_dict
---
parameters:
- in: path
name: username
type: string
required: true
responses:
200:
description: A single user item
schema:
id: User
properties:
name:
type: string
description: The name of the user
default: Chris Franklin
"""
with ClusterRpcProxy(CONFIG) as rpc:
# asynchronously spawning and email notification
# rpc.mail.send.async(email, subject, msg)
# synchronously spawning the player task
result = rpc.player.get(name)
# TODO: handle errors here
return result, 200
def teacher_info(school_name, institution_name):
with ClusterRpcProxy(CONFIG) as rpc:
# 学校名 => 学校ID
info = rpc.school.get_id_by_school_name(school_name, institution_name)
if len(info) == 0:
return "学校名或者学院名输入有误"
school_id = info['school_id']
institution_id = info['institution_id']
# 获取该学院的带有头衔的所有老师
result = rpc.school.get_famous_teachers_by_school(school_id, institution_id)
# 转换成字典
teachers = json.loads(result)
# 获取学院id对应的teacher_id和学科id
result = rpc.school.get_teacher_ids_by_institution_id(institution_id)
teachers_dis_mapping = json.loads(result)
# 按学科划分老师
dis_teachers = {}
for dis in teachers_dis_mapping:
discipline_id = dis['discipline_id']
teacher_id = str(dis['teacher_id'])
if discipline_id not in dis_teachers:
dis_teachers[discipline_id] = []
if teacher_id in teachers:
dis_teachers[discipline_id].append(teachers[teacher_id])
# 学校信息
result = rpc.school.get_discipline_by_institution(institution_id)
disciplines = json.loads(result)
return render_template("teachers.html",
disciplines=disciplines,
dis_teachers=dis_teachers)
def playlists():
"""
Micro Service Based Weather and Spotify API
This API is made with Flask and Nameko
---
parameters:
- name: zipcode
in: path
required: true
schema:
type: integer
description: location ZipCode
responses:
200:
description: Location data
"""
with ClusterRpcProxy(CONFIG) as rpc:
# Get the Spotify Authrization Token from header
# and OWM AppId from the header
openwm_appid = request.headers.get('X-OPENWM-APPID')
spotify_token = request.headers.get('X-SPOTIFY-TOKEN')
# Consuming Nameko service
# Here we pass the OpenWeatherMap AppId
# and the Spotify JWT
result = rpc.playlists.get_playlists(openwm_appid, spotify_token,
request.args)
return jsonify(result), 200
def make_nameko_helper(config):
"""Create a fake module that provides some convenient access to nameko
standalone functionality for interactive shell usage.
"""
module = ModuleType('nameko')
module.__doc__ = """Nameko shell helper for making rpc calls and dispatching
events.
Usage:
>>> n.rpc.service.method()
"reply"
>>> n.dispatch_event('service', 'event_type', 'event_data')
"""
proxy = ClusterRpcProxy(config)
module.rpc = proxy.start()
module.dispatch_event = event_dispatcher(config)
module.config = config
module.disconnect = proxy.stop
return module
def _get_nameko_connection(self):
proxy = ClusterRpcProxy(
self._config,
timeout=self._config.get('RPC_TIMEOUT', None)
)
return proxy.start()
