def main():
initialize_log()
config_params = parse_config_params()
# Initialize server and start server loop
server = Server(config_params["port"], config_params["listen_backlog"])
server.run()
def main():
initialize_log()
config_params = parse_config_params("config.json")
new_repos_queue = Queue()
repos_search_queue = Queue()
lock = Lock()
# Initialize server and start server loop
server_download = Server(config_params["download_port"], config_params["listen_backlog"])
server_request = Server(config_params["query_port"], config_params["listen_backlog"])
threads = []
threads.append(Thread(query_controller, (server_request, repos_search_queue, REPOSITORIES_FILE ,lock)))
threads.append(Thread(grep_files_controller, (config_params["host"], config_params["grep_results_port"], repos_search_queue, REPOSITORIES_FOLDER)))
threads.append(Thread(download_controller, (server_download, new_repos_queue)))
threads.append(Thread(update_repositories_controller, (REPOSITORIES_FILE, new_repos_queue, lock)))
threads.append(Thread(monitor, (new_repos_queue, repos_search_queue, config_params["monitor_time"])))
threads.append(Thread(garbage_collector, (REPOSITORIES_FOLDER, REPOSITORIES_FILE, config_params["g_c_time"], lock)))
for thread in threads:
thread.start()
while True:
time.sleep(1)
killed = monitor_threads_controller(threads)
if killed:
break
os._exit(0)
def main():
initialize_log()
config_params = parse_config_params("config.json")
server_downloads = Server(config_params["download_port"],
config_params["listen_backlog"])
server_query = Server(config_params["query_port"],
config_params["listen_backlog"])
server_responses = Server(config_params["grep_results_port"],
config_params["listen_backlog"])
queues = [Queue() for _ in range(config_params["listen_backlog"])]
threads = []
threads.append(
Thread(download_controller, (server_downloads, config_params["host"],
config_params["client_download_port"])))
for i in range(config_params["listen_backlog"]):
threads.append(
Thread(query_controller,
(server_query, config_params["host"],
config_params["client_query_port"], queues[i], i)))
threads.append(Thread(grep_results_controller, (server_responses, queues)))
for thread in threads:
thread.start()
while True:
time.sleep(1)
killed = monitor_threads_controller(threads)
if killed:
break
os._exit(0)
def main():
import os
initialize_log()
config_params = parse_config_params()
# Initialize server and start server loop
server = Server(config_params["port"])
server.run()
def main():
config_params = initialize_config()
initialize_log(config_params["logging_level"])
# Log config parameters at the beginning of the program to verify the configuration
# of the component
logging.debug("Server configuration: {}".format(config_params))
# Initialize server and start server loop
server = Server(config_params["port"], config_params["listen_backlog"])
server.run()
def send_mail(title):
try:
server = Server(SMTP_SERVER, PASSWORD, FROM_ADDR)
if server.connet():
mail = Mail(server, TO_ADDRS)
mail.mail_title = title
mail.mail_content = '股票提醒消息来了'
mail.send_mail()
server.close()
print('mail send out!')
except Exception as e:
print(e)
print('mail send failed!')
def main():
import os
initialize_log()
config_params = parse_config_params()
hostname = socket.gethostname()
IP = socket.gethostbyname(hostname)
logging.info("my hostname is: {}. other ip is: {}".format(
IP, config_params["cord_ip"]))
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((config_params["cord_ip"], int(config_params["cord_port"])))
s.sendall(("sc" + config_params["ID"]).encode('utf-8'))
s.close()
# Initialize server and start server loop
server = Server(config_params["port"])
server.run()
def main():
initialize_log()
config_params = parse_config_params()
try:
os.mkdir("datavolume1/server")
except FileExistsError:
logging.info("path server/ already exists")
os.chdir("datavolume1/server")
# docs de manager.dict(): https://docs.python.org/3/library/multiprocessing.html#sharing-state-between-processes
manager = Manager()
# the following variable will be set to False by the server when a termination msg is received.
# The backup requester will run as long as this bool indicates the server should be kept running.
keep_server_running = Value(c_bool, True)
# Nodes manager uses this queue to send the nodes
# which need to be backed up to the Backup requester
admin_to_nodes_manager_msgs_queue = Queue()
# Used to signal that a backup is needed
node_to_backup_queue_from_node_manager_to_backup_requester = Queue()
logger_queue = Queue()
# Initialize server and start server loop
server = Server(config_params["port"], config_params["listen_backlog"],
keep_server_running, admin_to_nodes_manager_msgs_queue,
logger_queue)
node_manager = NodeManager(
keep_server_running, admin_to_nodes_manager_msgs_queue,
node_to_backup_queue_from_node_manager_to_backup_requester,
logger_queue)
backup_requester = BackupRequester(
node_to_backup_queue_from_node_manager_to_backup_requester,
logger_queue)
logger = Logger(logger_queue, "logger.log")
p1 = Process(target=server.run)
p2 = Process(target=backup_requester.run)
p3 = Process(target=node_manager.run)
p4 = Process(target=logger.run)
p1.start()
p2.start()
p3.start()
p4.start()
p1.join()
p2.join()
p3.join()
p4.join()
def getProjectServers(self, prj_id):
connection = self.db_engine.connect()
servers = []
try:
# retrieve the amount of resources in terms of cores and memory
QUERY = """select a.uuid, a.vcpus, a.memory_mb, a.root_gb, \
a.vm_state, a.user_data from nova.instances as a WHERE a.project_id=\
'%(project_id)s'and a.vm_state in ('active', 'building', 'error') and \
a.deleted_at is NULL and a.terminated_at is NULL""" % {"project_id": prj_id}
LOG.debug("getProjectServers query: %s" % QUERY)
result = connection.execute(QUERY)
for row in result.fetchall():
flavor = Flavor()
flavor.setVCPUs(row[1])
flavor.setMemory(row[2])
flavor.setStorage(row[3])
server = Server()
server.setId(row[0])
server.setState(row[4])
server.setUserData(row[5])
server.setFlavor(flavor)
QUERY = """select `key`, value from nova.instance_metadata \
where instance_uuid='%(id)s' and deleted_at is NULL""" % {"id": server.getId()}
LOG.debug("getProjectServers query: %s" % QUERY)
result = connection.execute(QUERY)
metadata = {}
for row in result.fetchall():
metadata[row[0]] = row[1]
server.setMetadata(metadata)
server.setType()
servers.append(server)
except SQLAlchemyError as ex:
raise SynergyError(ex.message)
finally:
connection.close()
return servers
def getServer(self, id, detail=False):
try:
response_data = self.getResource("servers/" + id, "GET")
except requests.exceptions.HTTPError as ex:
raise SynergyError("error on retrieving the server info (id=%r)"
": %s" % (id, ex.response.json()))
server = None
if response_data:
server_data = response_data["server"]
server = Server()
server.setId(server_data["id"])
server.setName(server_data["name"])
server.setKeyName(server_data["key_name"])
server.setMetadata(server_data["metadata"])
server.setUserData(server_data.get("OS-EXT-SRV-ATTR:user_data",
None))
server.setType()
server.setState(server_data["OS-EXT-STS:vm_state"])
server.setUserId(server_data["user_id"])
server.setProjectId(server_data["tenant_id"])
server.setCreatedAt(server_data["created"])
server.setUpdatedAt(server_data.get("updated", None))
server.setLaunchedAt(
server_data.get("OS-SRV-USG:launched_at", None))
server.setTerminatedAt(
server_data.get("OS-SRV-USG:terminated_at", None))
if detail:
server.setFlavor(self.getFlavor(server_data["flavor"]["id"]))
return server
def getServers(self, detail=False, status=None):
params = {}
if status:
params["status"] = status
url = "servers/detail"
try:
response_data = self.getResource(url, "GET", params)
except requests.exceptions.HTTPError as ex:
response = ex.response.json()
raise SynergyError("error on retrieving the servers list"
": %s" % (id, response))
servers = []
if response_data:
for server_data in response_data["servers"]:
server = Server()
server.setId(server_data["id"])
server.setName(server_data["name"])
server.setKeyName(server_data["key_name"])
server.setMetadata(server_data["metadata"])
server.setUserData(server_data.get("OS-EXT-SRV-ATTR:user_data",
None))
server.setType()
server.setState(server_data["OS-EXT-STS:vm_state"])
server.setUserId(server_data["user_id"])
server.setProjectId(server_data["tenant_id"])
server.setCreatedAt(server_data["created"])
server.setUpdatedAt(server_data.get("updated", None))
server.setLaunchedAt(
server_data.get("OS-SRV-USG:launched_at", None))
server.setTerminatedAt(
server_data.get("OS-SRV-USG:terminated_at", None))
if detail:
server.setFlavor(self.getFlavor(
server_data["flavor"]["id"]))
servers.append(server)
return servers
def _makeServer(self, server_info):
if not server_info:
return
flavor = Flavor()
flavor.setMemory(server_info["memory_mb"])
flavor.setVCPUs(server_info["vcpus"])
flavor.setStorage(server_info["root_gb"])
if "instance_type" in server_info:
flavor.setName(server_info["instance_type"])
server = Server()
server.setFlavor(flavor)
server.setUserId(server_info["user_id"])
server.setMetadata(server_info["metadata"])
server.setDeletedAt(server_info["deleted_at"])
server.setTerminatedAt(server_info["terminated_at"])
server.setType()
if "host" in server_info:
server.setHost(server_info["host"])
if "uuid" in server_info:
server.setId(server_info["uuid"])
elif "instance_id" in server_info:
server.setId(server_info["instance_id"])
if "project_id" in server_info:
server.setProjectId(server_info["project_id"])
elif "tenant_id" in server_info:
server.setProjectId(server_info["tenant_id"])
if "vm_state" in server_info:
server.setState(server_info["vm_state"])
elif "state" in server_info:
server.setState(server_info["state"])
return server
def getProjectServers(self, prj_id):
connection = self.db_engine.connect()
servers = []
try:
# retrieve the amount of resources in terms of cores and memory
QUERY = """select a.uuid, a.vcpus, a.memory_mb, a.root_gb, \
a.vm_state from nova.instances as a WHERE a.project_id='%(project_id)s' \
and a.vm_state in ('active', 'building', 'error') and a.deleted_at is NULL \
and a.terminated_at is NULL""" % {
"project_id": prj_id
}
LOG.debug("getProjectServers query: %s" % QUERY)
result = connection.execute(QUERY)
for row in result.fetchall():
flavor = Flavor()
flavor.setVCPUs(row[1])
flavor.setMemory(row[2])
flavor.setStorage(row[3])
server = Server()
server.setId(row[0])
server.setState(row[4])
server.setFlavor(flavor)
QUERY = """select `key`, value from nova.instance_metadata \
where instance_uuid='%(id)s' and deleted_at is NULL""" % {
"id": server.getId()
}
LOG.debug("getProjectServers query: %s" % QUERY)
result = connection.execute(QUERY)
metadata = {}
for row in result.fetchall():
metadata[row[0]] = row[1]
server.setMetadata(metadata)
servers.append(server)
except SQLAlchemyError as ex:
raise Exception(ex.message)
finally:
connection.close()
return servers
def getServer(self, id, detail=False):
try:
response_data = self.getResource("servers/" + id, "GET")
except requests.exceptions.HTTPError as ex:
raise Exception("error on retrieving the server info (id=%r)"
": %s" % (id, ex.response.json()))
server = None
if response_data:
server_data = response_data["server"]
server = Server()
server.setId(server_data["id"])
server.setName(server_data["name"])
server.setKeyName(server_data["key_name"])
server.setMetadata(server_data["metadata"])
server.setState(server_data["OS-EXT-STS:vm_state"])
server.setUserId(server_data["user_id"])
server.setProjectId(server_data["tenant_id"])
server.setCreatedAt(server_data["created"])
server.setUpdatedAt(server_data.get("updated", None))
server.setLaunchedAt(
server_data.get("OS-SRV-USG:launched_at", None))
server.setTerminatedAt(
server_data.get("OS-SRV-USG:terminated_at", None))
if "user_data" in server_data:
user_data = server_data["user_data"]
server.setUserData(utils.decodeBase64(user_data))
if detail:
server.setFlavor(self.getFlavor(server_data["flavor"]["id"]))
return server
def getServers(self, detail=False, status=None):
params = {}
if status:
params["status"] = status
url = "servers/detail"
try:
response_data = self.getResource(url, "GET", params)
except requests.exceptions.HTTPError as ex:
response = ex.response.json()
raise Exception("error on retrieving the servers list"
": %s" % (id, response))
servers = []
if response_data:
for server_data in response_data["servers"]:
server = Server()
server.setId(server_data["id"])
server.setName(server_data["name"])
server.setKeyName(server_data["key_name"])
server.setMetadata(server_data["metadata"])
server.setState(server_data["OS-EXT-STS:vm_state"])
server.setUserId(server_data["user_id"])
server.setProjectId(server_data["tenant_id"])
server.setCreatedAt(server_data["created"])
server.setUpdatedAt(server_data.get("updated", None))
server.setLaunchedAt(
server_data.get("OS-SRV-USG:launched_at", None))
server.setTerminatedAt(
server_data.get("OS-SRV-USG:terminated_at", None))
if "user_data" in server_data:
user_data = server_data["user_data"]
server.setUserData(utils.decodeBase64(user_data))
if detail:
server.setFlavor(
self.getFlavor(server_data["flavor"]["id"]))
servers.append(server)
return servers
def getExpiredServers(self, prj_id, server_ids, TTL):
servers = []
connection = self.db_engine.connect()
try:
# retrieve all expired instances for the specified
# project and expiration time
ids = ""
if server_ids:
ids = "uuid in ('%s') and " % "', '".join(server_ids)
QUERY = """select uuid, vcpus, memory_mb, root_gb, \
vm_state from nova.instances where project_id = \
'%(project_id)s' and deleted_at is NULL and (vm_state='error' or \
(%(server_ids)s vm_state='active' and terminated_at is NULL \
and timestampdiff(minute, launched_at, utc_timestamp()) >= %(expiration)s))\
""" % {
"project_id": prj_id,
"server_ids": ids,
"expiration": TTL
}
LOG.debug("getExpiredServers query: %s" % QUERY)
result = connection.execute(QUERY)
for row in result.fetchall():
flavor = Flavor()
flavor.setVCPUs(row[1])
flavor.setMemory(row[2])
flavor.setStorage(row[3])
server = Server()
server.setId(row[0])
server.setState(row[4])
server.setFlavor(flavor)
QUERY = """select `key`, value from nova.instance_metadata \
where instance_uuid='%(id)s' and deleted_at is NULL""" % {
"id": server.getId()
}
LOG.debug("getExpiredServers query: %s" % QUERY)
result = connection.execute(QUERY)
metadata = {}
for row in result.fetchall():
metadata[row[0]] = row[1]
server.setMetadata(metadata)
servers.append(server)
except SQLAlchemyError as ex:
raise Exception(ex.message)
finally:
connection.close()
return servers
def _makeServer(self, server_info):
if not server_info:
return
flavor = Flavor()
flavor.setMemory(server_info["memory_mb"])
flavor.setVCPUs(server_info["vcpus"])
flavor.setStorage(server_info["root_gb"])
if "instance_type" in server_info:
flavor.setName(server_info["instance_type"])
server = Server()
server.setFlavor(flavor)
server.setUserId(server_info["user_id"])
server.setMetadata(server_info["metadata"])
server.setDeletedAt(server_info["deleted_at"])
server.setTerminatedAt(server_info["terminated_at"])
if "host" in server_info:
server.setHost(server_info["host"])
if "uuid" in server_info:
server.setId(server_info["uuid"])
elif "instance_id" in server_info:
server.setId(server_info["instance_id"])
if "project_id" in server_info:
server.setProjectId(server_info["project_id"])
elif "tenant_id" in server_info:
server.setProjectId(server_info["tenant_id"])
if "vm_state" in server_info:
server.setState(server_info["vm_state"])
elif "state" in server_info:
server.setState(server_info["state"])
return server
def getExpiredServers(self, prj_id, server_ids, TTL):
servers = []
connection = self.db_engine.connect()
try:
# retrieve all expired instances for the specified
# project and expiration time
ids = ""
if server_ids:
ids = "uuid in ('%s') and " % "', '".join(server_ids)
QUERY = """select uuid, vcpus, memory_mb, root_gb, \
vm_state, user_data from nova.instances where project_id = \
'%(project_id)s' and deleted_at is NULL and (vm_state='error' or \
(%(server_ids)s vm_state='active' and terminated_at is NULL \
and timestampdiff(minute, launched_at, utc_timestamp()) >= %(expiration)s))\
""" % {"project_id": prj_id, "server_ids": ids, "expiration": TTL}
LOG.debug("getExpiredServers query: %s" % QUERY)
result = connection.execute(QUERY)
for row in result.fetchall():
flavor = Flavor()
flavor.setVCPUs(row[1])
flavor.setMemory(row[2])
flavor.setStorage(row[3])
server = Server()
server.setId(row[0])
server.setState(row[4])
server.setUserData(row[5])
server.setFlavor(flavor)
QUERY = """select `key`, value from nova.instance_metadata \
where instance_uuid='%(id)s' and deleted_at is NULL""" % {"id": server.getId()}
LOG.debug("getExpiredServers query: %s" % QUERY)
result = connection.execute(QUERY)
metadata = {}
for row in result.fetchall():
metadata[row[0]] = row[1]
server.setMetadata(metadata)
server.setType()
servers.append(server)
except SQLAlchemyError as ex:
raise SynergyError(ex.message)
finally:
connection.close()
return servers
