class HttpService:
name = "grab_client"
config = Config()
# Hello User!
@http('GET', '/')
def get_method0(self, request):
print("\n")
ip = self.config.get('IP')
print("\n")
print("IP: " + str(ip))
print("Calling Method 1 with python version: ", sys.version_info[:])
print("\n")
sys.stdout.flush()
# Format output for client
msg = '''Hello User!'''
return msg
# Trailing slash required to avoid 301 redirect which fails. Not resolved.
@http('GET', '/grab/<string:cmd>/')
def get_method1(self, request, cmd):
ip = self.config.get('IP')
print("\n")
print("IP: " + ip)
print("\n")
cip = "http://" + ip + "/" + cmd
print("\n")
print("IP: " + cip)
print("\n")
with urllib.request.urlopen(cip) as response:
html = response.read()
html = html.decode("utf8")
response.close()
return html
class ParserService:
name = 'parser'
inferer = utilities.LanguageInferer()
config = Config()
@rpc
def is_supported(self, language):
return language in self.config['LANGUAGES']
@rpc
def is_parsable(self, name):
language = self.inferer.infer(name)
return language is not None
@rpc
def get_functions(self, name, contents):
functions = None
language = self.inferer.infer(name)
if language is None:
logger.debug('%s is in an unsupported language', name)
else:
parser = get_parser(language)
functions = parser.get_functions(name, contents)
if functions is not None:
functions = FunctionSchema(many=True).dump(functions)
return functions
class ChatService:
name = "chat"
player_rpc = RpcProxy("player")
config = Config()
@rpc
def validate_message(self, sender, content):
content_empty = len(content) == 0
if content_empty:
return None
cleaned_content = html.escape(content)
return cleaned_content
@rpc
def create_room(self, room_name: str, creator: str):
creator_exists = self.player_rpc.get_player_by_username(
creator) is not None
secret = self.config.get("SECRET", "secret")
hash_string = f"{secret};{str(time.time())};{creator};{room_name}"
signature = hashlib.sha256(hash_string.encode()).hexdigest()
if not creator_exists:
return None
return json.dumps({"room": {"code": signature, "name": room_name}})
class KeywordService:
name = 'keyword'
config = Config()
project_rpc = RpcProxy('project')
repository_rpc = RpcProxy('repository')
@rpc
def collect(self, project, **options):
logger.debug(project)
project = ProjectSchema().load(self.project_rpc.get(project))
if project.language.lower() not in self.config['KEYWORDS']:
raise LanguageNotSupported(f'{project.language} not supported')
keywords = self.config['KEYWORDS'].get(project.language.lower())
keywrd = Keywrd(keywords=keywords)
commits = self.repository_rpc.get_commits(project.name)
chunks = utilities.chunk(commits, size=round(len(commits) * 0.01))
pool = GreenPool(os.cpu_count())
arguments = [(project, c, self.repository_rpc) for c in chunks]
keyword = list()
for patches in pool.starmap(_get_patches, arguments):
patches = PatchSchema(many=True).load(patches)
keyword.extend(keywrd.get(patches))
return KeywordSchema(many=True).dump(keyword)
class Service:
name = "test_x"
config = Config()
@rpc
def foo(self):
return self.config.get("FOO")
class HttpService:
name = "grab_client"
config = Config()
# Hello User!
@http('GET', '/')
def get_method0(self, request):
print("\n")
ip = self.config.get('IP')
print("\n")
print("IP: " + str(ip))
print("Calling Method 1 with python version: ", sys.version_info[:])
print("\n")
sys.stdout.flush()
# Format output for client
msg = '''
Hello User!
'''
return msg
@http('GET', '/<string:fname>')
def get_method1(self, request, fname):
print("\n")
ip = self.config.get('IP')
cip = "http://" + ip + "/" + fname
with urllib.request.urlopen(cip) as response:
html = response.read()
html = html.decode("utf8") + "\n"
response.close()
return html
class AccessService:
name = 'access'
player_rpc = RpcProxy("player")
player_rest = PlayerRESTProvider()
config = Config()
@rpc
def login(self, username, password):
player = self.player_rpc.get_player(username, password)
if player is None:
return None
jwt_secret = self.config.get("JWT_SECRET", "secret")
token = jwt.encode(
{
'username': player["username"],
"exp": time.time() + 60
},
jwt_secret,
algorithm='HS256')
return json.dumps({"jwt": token.decode(), "user": player})
@rpc
def signup(self, username, password, country):
return self.player_rpc.create_player(username, password, country, 1000)
class HttpService:
name = "connect_to_client"
config = Config()
# Hello User!
@http('GET', '/')
def get_method0(self, request):
print("\n")
ip = self.config.get('IP')
sys.stdout.flush()
msg = '''Hello User!!!!\n'''
return msg
@http('GET', '/<string:cmd>/<string:file>')
def get_method(self, request, cmd, file):
ip = self.config.get('IP')
#ip = '10.1.1.17'
cip = "http://" + ip + "/" + cmd + "/" + file
print("IP: " + cip, ip)
print("\n")
with urllib.request.urlopen(cip) as response:
html = response.read()
html = html.decode("utf8")
response.close()
return html
class FunctionChurnService:
name = 'functionchurn'
config = Config()
redis = Redis()
parser_rpc = RpcProxy('parser')
project_rpc = RpcProxy('project')
repository_rpc = RpcProxy('repository')
@rpc
def collect(self, project, **options):
logger.debug(project)
functionchurn = list()
project = ProjectSchema().load(self.project_rpc.get(project))
if self.parser_rpc.is_supported(project.language):
changes = self._get_changes(project)
helper = Helper(project, self.repository_rpc, self.parser_rpc,
self.redis)
functionchurn = helper.collect(changes)
return FunctionChurnSchema(many=True).dump(functionchurn)
def _get_changes(self, project):
changes = self.repository_rpc.get_changes(project.name)
return ChangesSchema(many=True).load(changes)
class ImageService:
name = "image_service"
config = Config()
image_host = Cloudinary()
@rpc
def new_image(self, image):
image_id = uuid4().hex
self.image_host.upload_image(image, image_id)
return image_id
@rpc
def download_image(self, image_id):
return self.image_host.download_image(image_id)
@http('POST', '/image')
def post_image(self, request):
image = request.files['data']
image_id = self.new_image(image)
return create_json_response({'image_id': image_id})
@http('GET', '/image/<image_id>')
def get_image(self, request, image_id):
try:
raw_image = self.download_image(image_id)
except ImageNotFound as err:
return 404, str(err)
else:
return create_image_response(raw_image)
class EventDispatchWithCidMixin(object):
"""
Example usage:
MyClass(EventDispatchWithCidMixin):
...
# Dispatch event
self.dispatch_event("user_created", {"email": email, "uuid": uuid})
"""
logger = logging.getLogger(__name__)
# Nameko Config is a simple dependency provider
config = Config()
# Nameko event dispatcher
dispatch = EventDispatcher()
def dispatch_event(self, event_name: str, event_data: dict):
""" Dispatch event """
self.logger.debug("Dispatching event: %s, event data: %s", event_name, json.dumps(event_data))
# Get the correlation ID if it exists, otherwise create one
cid = locals.get_cid() or str(uuid4())
event_data["cid"] = cid
self.dispatch(event_name, event_data)
class CrawlerService(object):
name = 'url_crawler'
storage = RedisStorage()
logger = LoggingDependency()
dispatch = EventDispatcher()
config = Config()
@event_handler('http_server', 'url_to_check')
@event_handler('timer', 'url_to_check')
def check_url(self, url_group_frequency):
url, group, frequency = url_group_frequency
log(('Checking {url} for group {group} and frequency "{frequency}"'.
format(url=url, group=group, frequency=frequency)))
if not validators.url(url):
logging.error('Error with {url}: not a URL'.format(url=url))
return
self.storage.store_url(url)
if group:
self.storage.store_group(url, group)
if frequency:
self.storage.store_frequency(url, group, frequency)
head_timeout = self.config.get('CRAWLER_HEAD_TIMEOUT', HEAD_TIMEOUT)
get_timeout = self.config.get('CRAWLER_GET_TIMEOUT', GET_TIMEOUT)
no_head_domains = self.config.get('HEAD_DOMAINS_BLACKLIST', [])
try:
domain = urlparse(url).netloc
head_offend = domain in no_head_domains
if not head_offend:
try:
response = session.head(url,
allow_redirects=True,
timeout=head_timeout)
except requests.exceptions.ReadTimeout:
# simulate 404 to trigger GET request below
response = FakeResponse(status_code=404, headers={})
# Double check for servers not dealing properly with HEAD.
if head_offend or response.status_code in (404, 405):
log('Checking {url} with a GET'.format(url=url))
response = session.get(url,
allow_redirects=True,
timeout=get_timeout,
stream=True)
response.close()
except (requests.exceptions.ConnectionError,
requests.exceptions.ReadTimeout):
response = FakeResponse(status_code=503,
headers={},
url=url,
history=[])
except Exception as e:
logging.error('Error with {url}: {e}'.format(url=url, e=e))
return
finally:
self.storage.remove_check_flag(url)
metadata = self.storage.store_metadata(url, response)
self.dispatch('url_crawled', metadata)
class EmailSender(object):
name = "email_sender"
config = Config()
@event_handler("payments", "payment_received")
def send_email(self, payload):
"""Send an email to a client to inform them they have received a payment."""
email_text = self.create_email(payload)
key = self.config.get('api_key')
sandbox = self.config.get('sandbox')
recipient = '{0} <{1}>'.format(payload['payee']['name'], payload['payee']['email'])
request_url = 'https://api.mailgun.net/v3/{0}/messages'.format(sandbox)
request = requests.post(request_url, auth=('api', key), data={
'from': 'Mailgun Sandbox <mailgun@{}>'.format(sandbox),
'to': recipient,
'subject': 'Payment received',
'text': email_text
})
status_code = request.status_code
if status_code == 200:
logging.info("Sent the following email:\n%s", email_text)
else:
logging.critical(
"Request Status Code: %s. Failed to send the following email:\n%s",
status_code, email_text
)
return status_code
@staticmethod
def create_email(payload):
"""Generate body of the email to send."""
payee = payload['payee']['name']
amount = payload['payment']['amount']
currency = payload['payment']['currency']
client = payload['client']['name']
email = payload['client']['email']
email_text = "Dear {payee},\n\n\
You have received a payment of {amount} {currency} from {client} ({email}).\n\
\n\
Yours,\n\
student.com\n".format(
payee=payee,
amount=amount,
currency=currency,
client=client,
email=email,
)
return email_text
class PingService:
"""
Ping service
"""
name = "ping"
config = Config()
producer = KafkaProducer(metrics_num_samples=1)
debug = False
def _on_send_error(self, excp):
logging.error('Error sending message', exc_info=excp)
# handle exception
# Note: implement a retry logic, or a dead letter queue, ...
def send_message(self, message_id, size):
msg = b'\0' * size
future = self.producer.send("pong", value=msg)
future.add_errback(self._on_send_error)
@log_stats
def handle_message(self, message):
# Expensive computation of around 0.025 seconds
#for _ in range(1, 1000000):
# pass
if self.debug:
logging.info('#' * 30)
logging.info(message)
logging.info('#' * 30)
def compute_consumer_stats(self, total_time, num_messages, size):
stats = {
'num_messages': num_messages,
'size': size,
'total time': total_time,
'byte rate (MB/s)': size * num_messages / total_time / 1000000,
'message rate (msg/s)': num_messages / total_time
}
return stats
@rpc
def send(self, num_messages, size=10):
start = time.time()
for i in range(num_messages):
self.send_message(message_id=i, size=size)
self.producer.flush()
end = time.time()
stats = self.compute_consumer_stats(end - start, num_messages, size)
logging.info(f'Sent messages stats: {stats}')
return json.dumps(stats)
@consume("ping", group_id="ping-group")
def consume_ping(self, message):
# Your message handler
self.handle_message(message)
class FileContentsService:
name = "getFileContents_service"
config = Config()
@http('GET', '/getFileContents/<string:file_name>')
def get_method(self, request, file_name):
print("hello front" + file_name)
ip = self.config.get('IP')
backendURL = "http://" + ip + "/getFileContents/" + file_name
content = urllib.request.urlopen(backendURL).read()
return json.dumps(
{'value': json.loads(content.decode('utf8'))['value']})
class ComplexityService:
name = 'nesting'
config = Config()
understand_rpc = RpcProxy('understand')
@rpc
def collect(self, project, **options):
logger.debug(project)
metrics = self.understand_rpc.get_metrics(project, METRICS)
metrics = MetricsSchema(many=True).load(metrics)
return NestingSchema(many=True).dump(_transform(metrics))
class OffenderService:
name = 'offender'
config = Config()
@rpc
def collect(self, project, **options):
logger.debug(project)
offenders = get_offenders(project)
return OffenderSchema(many=True).dump(offenders)
class OwnershipService:
name = 'ownership'
config = Config()
repository_rpc = RpcProxy('repository')
@rpc
def collect(self, project, **options):
logger.debug(project)
commits = self.repository_rpc.get_commits(project)
commits = CommitSchema(many=True).load(commits)
ownerships = get_ownership(commits)
return OwnershipSchema(many=True).dump(ownerships)
class Service:
name = "test_config"
config = Config()
@property
def foo_enabled(self):
return self.config.get('FOO_FEATURE_ENABLED', False)
@http('GET', '/foo')
def foo(self, request):
if not self.foo_enabled:
return 403, "FeatureNotEnabled"
return 'fool'
class ContributionService:
name = 'contribution'
config = Config()
repository_rpc = RpcProxy('repository')
@rpc
def collect(self, project, **options):
logger.debug(project)
changes = self.repository_rpc.get_changes(project)
changes = ChangesSchema(many=True).load(changes)
contribution = get_contribution(changes, **options)
return ContributionSchema(many=True).dump(contribution)
class PastChangesService:
name = 'pastchanges'
config = Config()
repository_rpc = RpcProxy('repository')
@rpc
def collect(self, project, **options):
logger.debug(project)
changes = self.repository_rpc.get_changes(project)
changes = ChangesSchema(many=True).load(changes)
pastchanges = _get_pastchanges(changes)
return PastChangesSchema(many=True).dump(pastchanges)
class WebhookService(object):
name = 'webhook_dispatcher'
storage = RedisStorage()
logger = LoggingDependency()
config = Config()
def _send(self, url, metadata):
"""POST metadata to url"""
try:
response = requests.post(url,
json={'data': metadata},
timeout=TIMEOUT)
except (requests.Timeout, requests.RequestException) as e:
raise WebhookUnreachableException('Unreachable',
url,
503,
original_exception=e)
if response.status_code < 200 or response.status_code >= 400:
raise WebhookUnreachableException('Unreachable', url,
response.status_code)
log('Successfully called webhook {url}'.format(url=url))
@event_handler('url_crawler', 'url_crawled')
def send_response(self, metadata):
"""Call a webhook with checked url results"""
url = metadata.get('checked-url')
callback_urls = self.storage.get_webhooks_for_url(url)
if not callback_urls:
return
for callback_url in callback_urls:
log(('Calling webhook url {callback_url} for checked url {url}'.
format(callback_url=callback_url, url=url)))
try:
send = retry(self._send,
for_exceptions=WebhookUnreachableException,
max_attempts=self.config.get(
'WEBHOOK_NB_RETRY', NB_RETRY),
delay=self.config.get('WEBHOOK_DELAY_INTERVAL',
DELAY_INTERVAL),
backoff=self.config.get('WEBHOOK_BACKOFF_FACTOR',
BACKOFF_FACTOR))
send(callback_url, metadata)
except WebhookUnreachableException as e:
logging.error(
('Webhook unreachable: {url} - {code} ({detail})'.format(
url=callback_url,
code=e.status,
detail=e.original_exception)))
class HttpService:
name = "connect_to_client"
config = Config()
# Hello User!
@http('GET', '/')
def get_method0(self, request):
print("\n")
ip = self.config.get('IP')
print("\n")
print("IP: " + str(ip))
print("Calling Method 1 with python version: ", sys.version_info[:])
print("\n")
sys.stdout.flush()
# Format output for client
msg = '''Hello User!!!!\n'''
return msg
# Trailing slash required to avoid 301 redirect which fails. Not resolved.
# @http('GET', '/grab/<string:cmd>/')
# def get_method(self, request, cmd):
# #ip = config.get('IP')
# ip = '10.1.1.17'
# cip = "http://"+ip +":8000" +"/" + cmd
# print("IP: " + cip, ip)
# print("\n")
# with urllib.request.urlopen(cip) as response:
# html = response.read()
# html = html.decode("utf8")
# response.close()
# return html
@http('GET', '/<string:cmd>/<string:file>')
def get_method(self, request, cmd, file):
#ip = config.get('IP')
ip = '10.1.1.17'
cip = "http://" + ip + ":8000" + "/" + cmd + "/" + file
print("IP: " + cip, ip)
print("\n")
with urllib.request.urlopen(cip) as response:
html = response.read()
html = html.decode("utf8")
response.close()
return html
class ProjectService:
name = 'project'
config = Config()
database = Database(DeclarativeBase)
@rpc
def get(self, project):
logger.debug(project)
_project = None
with self.database.get_session() as session:
_project = session.query(Project).get(project)
if _project is None:
raise NotFound('{} not found'.format(project))
return ProjectSchema().dump(_project)
class HttpService:
name = "files_client"
config = Config()
# Hello User!
@http('GET', '/')
def get_method0(self, request):
print("\n")
ip = '10.0.1.16'
print("\n")
print("IP: " + str(ip))
print("Calling Method 1 with python version: ", sys.version_info[:])
print("\n")
sys.stdout.flush()
# Format output for client
msg = '''
Hello, guys!
##########################################
Files available:
##########################################
URL: 10.0.1.16/files/
a.txt
b.txt
c.txt
d.txt
'''
return msg
@http('GET', '/files/')
def grabpage(self, request):
content = """function available\n1.a.txt \n2.b.txt \n3.c.txt \n4.d.txt \n"""
return content
# Trailing slash required to avoid 301 redirect which fails. Not resolved.
@http('GET', '/files/<string:filename>/')
def get_method1(self, request, filename):
backEndIp = "10.0.1.17"
CONFIG = {'AMQP_URI': "amqp://*****:*****@{}:5672".format(backEndIp)}
filename = str(filename)
with ClusterRpcProxy(CONFIG) as rpc:
data = rpc.files_service.getfiles(filename)
return data
class HunkService:
name = 'hunk'
config = Config()
repository_rpc = RpcProxy('repository')
@rpc
def collect(self, project, **options):
logger.debug(project)
hunks = list()
commits = self.repository_rpc.get_commits(project)
for chunk in utilities.chunk(commits, size=round(len(commits) * 0.05)):
patches = self.repository_rpc.get_patches(project, chunk)
patches = PatchSchema(many=True).load(patches)
hunks.extend(get_hunks(patches))
patches.clear()
return HunkSchema(many=True).dump(hunks)
class ChurnService:
name = 'churn'
config = Config()
parser_rpc = RpcProxy('parser')
project_rpc = RpcProxy('project')
repository_rpc = RpcProxy('repository')
@rpc
def collect(self, project, **options):
logger.debug(project)
project = ProjectSchema().load(self.project_rpc.get(project))
deltas = self._get_deltas(project)
return ChurnSchema(many=True).dump(_get_churn(deltas))
def _get_deltas(self, project):
deltas = self.repository_rpc.get_deltas(project.name)
return DeltasSchema(many=True).load(deltas)
class AccessService:
name = 'access'
player_rpc = RpcProxy("player")
player_rest = PlayerRESTProvider()
auth = RpcProxy("access")
config = Config()
@rpc
def login(self, username, password):
player = self.player_rpc.get_player(username, password)
if player is None:
return None
jwt_secret = self.config.get("JWT_SECRET", "secret")
token = jwt.encode(
{
'username': player["username"],
"exp": time.time() + 60
},
jwt_secret,
algorithm='HS256')
return json.dumps({"jwt": token.decode(), "user": player})
@rpc
def signup(self, username, password):
data = {
"username": username,
"password": password,
"country": "Chile",
"elo": 1000
}
req = self.player_rest.post("/player", data)
if req["status_code"] == 200:
return True
return False
class UnderstandService:
name = 'understand'
config = Config()
project_rpc = RpcProxy('project')
repository_rpc = RpcProxy('repository')
@rpc
def get_metrics(self, project, metrics):
logger.debug(project)
project = ProjectSchema().load(self.project_rpc.get(project))
udb = self._get_udb(project)
return MetricsSchema(many=True).dump(_get_metrics(udb, metrics))
def _get_udb(self, project):
path = self.repository_rpc.get_path(project.name)
version = self.repository_rpc.get_version(project.name)
name = f'{project.name}@{version}.udb'
root = self.config['UNDERSTAND_ROOT']
udb = UDB(root, name, project.language, path)
return udb
class HttpService:
name = "grab_client"
config = Config()
# Hello User!
@http('GET', '/')
def get_IP(self, request):
print("\n")
ip = self.config.get('IP')
print("\n")
print("IP: " + str(ip))
print ("Calling Method 1 with python version: ",sys.version_info[:])
print("\n")
sys.stdout.flush()
# Format output for client
msg="ip=" + str(ip) + '\n'
return msg
# Trailing slash required to avoid 301 redirect which fails. Not resolved.
@http('GET', '/grab/<string:cmd>/')
# def get_HTTP(self, request, cmd):
def get_method1 (self, request, cmd):
# print("\n")
# print("IP: " + cip)
# print("\n")
# with urllib.request.urlopen(cip) as response:
# html = response.read()
# html = html.decode("utf8")
# response.close()
cip = "http://"+BACKEND_IP + ":8000/" + cmd
response = requests.get (cip)
html = requests.text
print (cmd, ": ", html)
return html