class DeploymentController():
def __init__(self):
self.PATH_INSTANCES = "../../../instances/"
self.ROOT_SERVICE_DIR_NAME = "linkehub_api_root"
self.ROOT_SERVICE_NAME = "linkehub-api-root"
self.ROOT_SERVICE_DOCKER_IMG = "linkehub-api-root:0.1"
self.PREFIX_SERVICE_PATH_NAME = "linkehub_api_i"
self.PREFIX_SERVICE_NAME = "linkehub-api-i"
self.PREFIX_SERVICE_DOCKER_IMG = "linkehub-api-i{0}:0.1"
self.logger = Logger()
self.constUtils = ConstantUtils()
self.networkUtils = NetworkingUtils()
'''
Try to deploy N copies of the root microservice and if it succeeds
it returns a list with the urls of the running servers
'''
def deployNCopiesRootInstance(self, startIdx, endIdx):
response = {
"success": False,
"msg": "Failed to deploy copies of the root microservice",
"running_instances": []
}
try:
if not startIdx or not endIdx:
response[
"msg"] = "Could not deploy instances, one or more input parameters are invalid."
else:
for instanceNum in range(int(startIdx), int(endIdx)):
# Deploy a new copy of the root service to Heroku
workingDir = os.getcwd()
rootServicePath = "{0}{1}".format(
self.PATH_INSTANCES, self.ROOT_SERVICE_DIR_NAME)
copyServicePath = "{0}{1}{2}".format(
self.PATH_INSTANCES, self.PREFIX_SERVICE_PATH_NAME,
instanceNum)
copyServiceName = "{0}{1}".format(self.PREFIX_SERVICE_NAME,
instanceNum)
copyServiceUrl = "{0}{1}{2}".format(
self.constUtils.HTTPS_PREFIX, copyServiceName,
self.constUtils.POSTFIX_HEROKU_APPS_URL)
rootServiceRepoUrl = "{0}{1}{2}{3}".format(
self.constUtils.HTTPS_PREFIX,
self.constUtils.PREFIX_HEROKU_APPS_GIT_REPO,
self.ROOT_SERVICE_NAME, self.constUtils.GIT_POSTFIX)
copyServiceRepoUrl = "{0}{1}{2}{3}".format(
self.constUtils.HTTPS_PREFIX,
self.constUtils.PREFIX_HEROKU_APPS_GIT_REPO,
copyServiceName, self.constUtils.GIT_POSTFIX)
dockerImgNewService = self.PREFIX_SERVICE_DOCKER_IMG.format(
instanceNum)
print(
"\nCreating new instance of the root service: instance name {0} ..."
.format(copyServiceName))
print("workingDir: {0}".format(workingDir))
print("rootServicePath: {0}".format(rootServicePath))
print("copyServicePath: {0}".format(copyServicePath))
print("copyServiceName: {0}".format(copyServiceName))
print("copyServiceUrl: {0}".format(copyServiceUrl))
print("rootServiceRepoUrl: {0}".format(rootServiceRepoUrl))
status = call("rm -rf {0} ".format(copyServicePath),
shell=True)
print("\nstatus cmd delete copy folder: {0} \n".format(
status))
status = call("cp -fr {0} {1} ".format(
rootServicePath, copyServicePath),
shell=True)
print(
"\nstatus cmd copy contents of root service to new service: {0} \n"
.format(status))
status = call("git remote rm heroku",
cwd=copyServicePath,
shell=True)
print(
"\nstatus cmd remove remote of the source service: {0} \n"
.format(status))
status = call(
"heroku apps:create {0} ".format(copyServiceName),
cwd=copyServicePath,
shell=True)
print("\nstatus cmd create heroku app: {0} \n".format(
status))
status = call(
"git remote add heroku {0}".format(copyServiceRepoUrl),
cwd=copyServicePath,
shell=True)
print("\nstatus cmd add remote of the new service: {0} \n".
format(status))
status = call("git remote -v ",
cwd=copyServicePath,
shell=True)
print("\nstatus cmd git remote -v: {0} \n".format(status))
status = call("docker image build -t {0} . ".format(
dockerImgNewService),
cwd=copyServicePath,
shell=True)
print("\nstatus cmd build docker image: {0} \n".format(
status))
status = call(
"heroku container:push web --app {0} ".format(
copyServiceName),
cwd=copyServicePath,
shell=True)
print(
"\nstatus cmd push container to heroku: {0} \n".format(
status))
status = call(
"heroku container:release web --app {0} ".format(
copyServiceName),
cwd=copyServicePath,
shell=True)
print(
"\nstatus cmd release container to production: {0} \n".
format(status))
# Append the url of the instance to the list of instances
response["running_instances"].append(copyServiceUrl)
time.sleep(60)
# Fetch a successful response
response["success"] = True
response["msg"] = "The deployment script was executed"
response["created_at"] = self.logger.get_utc_iso_timestamp()
except Exception as err:
print("Failed to deployNCopiesRootInstance {0}".format(err))
return json.dumps(response)
class ScrapingController():
def __init__(self):
self.TAG = "ScrapingController"
self.logger = Logger()
self.netUtils = NetworkingUtils()
self.constUtils = ConstantUtils()
self.strUtils = StringUtils()
self.authController = AuthController()
self.gitController = GithubController()
self.idxNextInstance = 0
self.locations = [
"rio de janeiro", "belo horizonte", "manaus", "porto alegre",
"curitiba", "fortaleza", "recife", "sao paulo", "são paulo",
"berlin", "london", "new york", "san francisco", "tokyo",
"amsterdam", "ottawa", "zurich", "abu dhabi"
]
## ------------- ##
## Data scraping ##
## ------------- ##
'''
Scrap the basic profile info of Github users from a given city
'''
def scrapBasicProfileGithubUsers(self, username, password, location,
initialPage, numPages):
response = {
"success": False,
"msg":
"Failed to scrap basic profile info of Github users from the given location",
"instances": "",
"last_page": "",
"num_successes": 0,
"num_fails": 0,
"failed_pages": [],
"started_at": self.logger.get_utc_iso_timestamp(),
"finished_at": ""
}
try:
if not username or not password or not location or not initialPage or not numPages:
response["msg"] = "{0}. {1}".format(
response["msg"], "Invalid input parameters.")
else:
token = self.authController.login(username, password)
location = self.strUtils.getCleanedJsonVal(location)
if token != "":
print(
"\nRequesting basic Github profiles of users from location: {0} ..."
.format(location))
self.netUtils.updateListRemainingRequestsGithubAPI()
for currentPage in range(int(initialPage), int(numPages)):
# Hold the process until we have more requests, if needed
self.netUtils.waitRequestGithubApiIfNeeded()
apiInstance = self.netUtils.getInstanceForRequestToGithubAPI(
)
try:
# Make a request to the Github API and verify if the limit of requests per hour has been exceeded
connection = http.client.HTTPSConnection(
apiInstance.getBaseUrl())
headers = self.netUtils.getRequestHeaders(
self.constUtils.HEADERS_TYPE_AUTH_TOKEN, token)
endpoint = "/get_github_users_from_location/?store_in_db={0}&location={1}&page_number={2}".format(
True, urllib.parse.quote(location),
currentPage)
connection.request("GET",
endpoint,
headers=headers)
print(
"\nGET list of users from a location and store on DB \nurl: {0}{1}"
.format(apiInstance.getBaseUrl(), endpoint))
res = connection.getresponse()
data = res.read()
githubApiResponse = json.loads(
data.decode(self.constUtils.UTF8_DECODER))
apiInstance.remainingCallsGithub -= 1
# Process the response
succesfulResponse = False
if githubApiResponse is not None:
if "success" in githubApiResponse:
if githubApiResponse["success"]:
succesfulResponse = True
response["num_successes"] += 1
else:
print(githubApiResponse["msg"])
if not succesfulResponse:
response["num_fails"] += 1
response["failed_pages"].append(currentPage)
except Exception as e:
print(
"{0} Failed to process a Github user profile: {1}"
.format(self.TAG, e))
print("\n{0}:{1} Done! \n".format(
self.TAG, self.logger.get_utc_iso_timestamp()))
# Fetch a successful response
response["success"] = True
response[
"msg"] = "The script scrapBasicProfileGithubUsers executed correctly"
response[
"instances"] = self.netUtils.getSerializableApiInstances(
)
response["last_page"] = currentPage
response[
"finished_at"] = self.logger.get_utc_iso_timestamp()
else:
response["msg"] = "{0}. {1}".format(
response["msg"], "Wrong username or password.")
except ValueError as e:
print("{0} Failed to scrapBasicProfileGithubUsers {1}".format(
self.TAG, e))
return json.dumps(response)
'''
Scrap repositories and skills of Github users from a location
'''
def scrapGithubUsersRepositoriesSkills(self, username, password, location):
response = {
"success": False,
"msg":
"Failed to scrap the repositories and skills of Github users from a location",
"instances": "",
"num_successes": 0,
"failed_user_ids": [],
"num_fails": 0,
"started_at": self.logger.get_utc_iso_timestamp(),
"finished_at": ""
}
try:
if not username or not password:
response["msg"] = "{0}. {1}".format(
response["msg"], "Invalid input parameters.")
else:
# Make a request to the Linkehub database and return all the Github user ids from a location
token = self.authController.login(username, password)
location = self.strUtils.getCleanedJsonVal(location)
if token != "":
print(
"\nRequesting repositories and skills of Github users from : {0} ..."
.format(location))
self.netUtils.updateListRemainingRequestsGithubAPI()
try:
# Request a list of userIds from the service Database
userIds = self.gitController.getGithubUserIdsFromLocation(
token, location)
# Request the list of repositories and skills associated to a Github user
for githubUserId in userIds:
try:
githubUserId = self.strUtils.getCleanedJsonVal(
githubUserId)
# Hold the process until we have more requests, if needed
self.netUtils.waitRequestGithubApiIfNeeded()
apiInstance = self.netUtils.getInstanceForRequestToGithubAPI(
)
print(
"{0} Number of remaining requests to the Github API: {1}"
.format(
self.logger.get_utc_iso_timestamp(),
self.netUtils.
getNumRemaningRequestToGithub()))
if apiInstance.remainingCallsGithub > 0:
# This endpoint requests the repositories from the Github API, process the response and
# stores the list of repos and skills in the service Database
connection = http.client.HTTPSConnection(
apiInstance.getBaseUrl())
headers = self.netUtils.getRequestHeaders(
self.constUtils.
HEADERS_TYPE_AUTH_TOKEN, token)
endpoint = "/scrap_user_repositories_skills_from_github/?githubUserId={0}&location={1}".format(
urllib.parse.quote(githubUserId),
urllib.parse.quote(location))
connection.request("GET",
endpoint,
headers=headers)
print(
"\nGET repositories and skills of the Github user: {0}"
.format(githubUserId))
print("url: {0}{1}".format(
apiInstance.getBaseUrl(), endpoint))
res = connection.getresponse()
data = res.read()
githubUserReposSkillsResponse = json.loads(
data.decode(
self.constUtils.UTF8_DECODER))
apiInstance.remainingCallsGithub -= 1
# Process the response
successfulResponse = False
if githubUserReposSkillsResponse is not None:
if "success" in githubUserReposSkillsResponse:
if githubUserReposSkillsResponse[
"success"]:
response["num_successes"] += 1
successfulResponse = True
elif "msg" in githubUserReposSkillsResponse:
print(
githubUserReposSkillsResponse[
"msg"])
if not successfulResponse:
response["num_fails"] += 1
response["failed_user_ids"].append(
githubUserId)
except Exception as e:
print(
"{0} Failed to process the repositories and skills of the user: {1} \ncause: {2}"
.format(self.TAG, githubUserId, e))
print("\n{0}:{1} Done! \n".format(
self.TAG, self.logger.get_utc_iso_timestamp()))
except Exception as e:
print(
"{0} Failed to process the list of repositories and skills of Github users from: {1} \ncause:{2}"
.format(self.TAG, location, e))
else:
response["msg"] = "{0}. {1}".format(
response["msg"], "Wrong username or password.")
# Fetch a successful response
response["success"] = True
response[
"msg"] = "The script scrapGithubUsersRepositoriesSkills executed correctly"
response["instances"] = self.netUtils.getSerializableApiInstances()
response["finished_at"] = self.logger.get_utc_iso_timestamp()
except ValueError as e:
print(
"{0} Failed to scrapGithubUsersRepositoriesSkills: {1}".format(
self.TAG, e))
return json.dumps(response)
'''
Scrap list of commits of Github users from a location, the users are gathered from the Linkehub database
and them complemented with Github data
'''
def scrapCommitsCodeSamplesGithubUsersFromLocation(self, username,
password, location,
skill):
response = {
"success": False,
"msg":
"Failed to scrap the commits and code samples of the Github users from the location",
"instances": "",
"num_successes": 0,
"num_fails": 0,
"started_at": self.logger.get_utc_iso_timestamp(),
"finished_at": ""
}
try:
if not username or not password or not location or not skill:
response["msg"] = "{0}. {1}".format(
response["msg"], "Invalid input parameters.")
else:
token = self.authController.login(username, password)
location = self.strUtils.getCleanedJsonVal(location)
skill = self.strUtils.getCleanedJsonVal(skill)
if token != "":
print(
"\nRequesting commits and code samples of Github users from : {0} ..."
.format(location))
self.netUtils.updateListRemainingRequestsGithubAPI()
# Request a list of userIds from the service Database
userIds = self.gitController.getGithubUserIdsFromLocation(
token, location)
# Request the list of repositories and skills associated to a Github user
for githubUserId in userIds:
try:
# Hold the process until we have more requests, if needed
self.netUtils.waitRequestGithubApiIfNeeded()
apiInstance = self.netUtils.getInstanceForRequestToGithubAPI(
)
print(
"Number of remaining requests to the Github API: {0}"
.format(self.netUtils.
getNumRemaningRequestToGithub()))
if apiInstance.remainingCallsGithub > 0:
# This endpoint requests the repositories from the Github API, process the response and
# stores the list of repos and skills in the service Database
connection = http.client.HTTPSConnection(
apiInstance.getBaseUrl())
headers = self.netUtils.getRequestHeaders(
self.constUtils.HEADERS_TYPE_AUTH_TOKEN,
token)
endpoint = "/scrap_user_commits_language_github/?githubUserId={0}&language={1}".format(
urllib.parse.quote(githubUserId),
urllib.parse.quote(skill),
)
connection.request("GET",
endpoint,
headers=headers)
print(
"\nGET repositories and skills of the Github user: {0}"
.format(githubUserId))
print("url: {0}{1}".format(
apiInstance.getBaseUrl(), endpoint))
res = connection.getresponse()
data = res.read()
githubUserReposSkillsResponse = json.loads(
data.decode(self.constUtils.UTF8_DECODER))
apiInstance.remainingCallsGithub -= 1
# Process the response
if githubUserReposSkillsResponse is not None:
if "success" in githubUserReposSkillsResponse:
if githubUserReposSkillsResponse[
"success"]:
response["num_successes"] += 1
else:
if "msg" in githubUserReposSkillsResponse:
print(
githubUserReposSkillsResponse[
"msg"])
response["num_fails"] += 1
else:
response["num_fails"] += 1
else:
response["num_fails"] += 1
except Exception as e:
print(
"{0} Failed to process the repositories and skills of the user: {1} \ncause: {2}"
.format(self.TAG, githubUserId, e))
print("\n{0}:{1} Done! \n".format(
self.TAG, self.logger.get_utc_iso_timestamp()))
# Fetch a successful response
response["success"] = True
response[
"msg"] = "The script scrapCommitsCodeSamplesGithubUsersFromLocation executed correctly"
response[
"instances"] = self.netUtils.getSerializableApiInstances(
)
response[
"finished_at"] = self.logger.get_utc_iso_timestamp()
else:
response["msg"] = "{0}. {1}".format(
response["msg"], "Wrong username or password.")
except ValueError as e:
print(
"{0} Failed to scrapCommitsCodeSamplesGithubUsersFromLocation: {1}"
.format(self.TAG, e))
return json.dumps(response)
class PingController():
def __init__(self):
self.TAG = "PingController"
self.netUtils = NetworkingUtils()
self.logUtils = Logger()
self.session = FuturesSession()
'''
Make a request to verify if all the listed services are up and running
'''
def pingServices(self, key):
response = {"msg": "Failed to ping services"}
try:
if key == self.netUtils.key:
for service in self.netUtils.services:
self.session.get(
service.url,
background_callback=self.parseAsyncResponse)
service.status = False
service.msg = ""
else:
response["msg"] = "{0}{1}".format(response["msg"],
"Invalid key.")
except urllib.error.HTTPError as httpe:
print("{0}: Failed to pingService: {1}".format(self.TAG, httpe))
except urllib.error.URLError as urle:
print("{0}: Failed to pingService: {1}".format(self.TAG, urle))
return "The pingServices process finished at {0}".format(
datetime.datetime.utcnow())
'''
Process the response of the async request that was sent to the urls
'''
def parseAsyncResponse(self, session, response):
if response is not None:
if response.url is not None and response.content is not None:
service = self.netUtils.getServiceByUrl(response.url)
if service is not None:
service.msg = response.content.decode(
self.netUtils.UTF8_DECODER)
if self.netUtils.MANDATORY_TERM_SUCCESS_STATUS_RESPONSE in service.msg:
service.status = True
service.verifiedAt = self.logUtils.get_utc_iso_timestamp()
print("\n{0}".format(json.dumps(
service.getSerializable())))
'''
Return a list o the services with information about their current status and the last
time this controller tried to call them
'''
def returnStatusServices(self, key):
response = {
"msg": "Failed to return the status of the services",
"status_services": []
}
try:
if key == self.netUtils.key:
print("The key is valid. Starting to make requests ...")
for service in self.netUtils.services:
response["status_services"].append(
service.getSerializable())
response[
"msg"] = "Here is the status of the services listed in the configuration file."
else:
response["msg"] = "{0}. {1}".format(response["msg"],
"Invalid key.")
except Exception as e:
print("{0}: Failed to returnStatusServices: {1}".format(
self.TAG, e))
return json.dumps(response)
