def get_topological_order_by_relations(self, relationships):
topsort_input = self.__prepare_topsort_input(relationships)
topological_order = list(toposort_flatten(topsort_input))
dep_breaks = ConfigReader().get_dependency_breaks()
tables_to_order = ConfigReader().get_all_tables()
tables_to_delete = list()
for r in relationships:
p = r['parent_table_name']
c = r['child_table_name']
dep_break_found = False
for dep_break in dep_breaks:
if p == dep_break['parent'] and c == dep_break['child']:
dep_break_found = True
break
if dep_break_found == True:
continue
if tables_to_order is not None and (p not in tables_to_order
or c not in tables_to_order):
continue
if c in topological_order:
tables_to_delete.append(r)
return list(reversed(tables_to_delete))
def __prepare_topsort_input(self, relationships):
dep_breaks = ConfigReader().get_dependency_breaks()
tables_to_order = ConfigReader().get_all_tables()
deps = dict()
for r in relationships:
p = r['parent_table_name']
c = r['child_table_name']
#break circ dependency
dep_break_found = False
for dep_break in dep_breaks:
if p == dep_break['parent'] and c == dep_break['child']:
dep_break_found = True
break
if dep_break_found == True:
continue
if tables_to_order is not None and len(tables_to_order) > 0 and (
p not in tables_to_order or c not in tables_to_order):
continue
if p in deps:
deps[p].add(c)
else:
deps[p] = set()
deps[p].add(c)
return deps
def send_email_to_student(self, recepient_email, message, cust_name,
responseId, lstCases):
try:
self.config_reader = ConfigReader()
self.configuration = self.config_reader.read_config()
# instance of MIMEMultipart
self.msg = MIMEMultipart()
# storing the senders email address
self.msg['From'] = self.configuration['SENDER_EMAIL']
# storing the receivers email address
self.msg['To'] = ",".join(recepient_email)
# storing the subject
self.msg['Subject'] = self.configuration['EMAIL_SUBJECT']
# string to store the body of the mail
#body = "This will contain attachment"
imgUrl = os.path.join(os.path.abspath('static/images'),
responseId + ".png")
self.get_cases_bar(responseId, lstCases, imgUrl)
fp = open(imgUrl, 'rb')
msgImage = MIMEImage(fp.read())
fp.close()
# Define the image's ID as referenced above
msgImage.add_header('Content-ID', '<image1>')
self.msg.attach(msgImage)
body = message
body = body.replace('cust_name', cust_name)
# attach the body with the msg instance
self.msg.attach(MIMEText(body, 'html'))
# instance of MIMEBase and named as p
self.p = MIMEBase('application', 'octet-stream')
# creates SMTP session
self.smtp = smtplib.SMTP('smtp.gmail.com', 587)
# start TLS for security
self.smtp.starttls()
# Authentication
self.smtp.login(self.msg['From'], self.configuration['PASSWORD'])
# Converts the Multipart msg into a string
self.text = self.msg.as_string()
# sending the mail
self.smtp.sendmail(self.msg['From'], recepient_email, self.text)
# terminating the session
self.smtp.quit()
except Exception as e:
print('the exception is ' + str(e))
def main():
cr = ConfigReader()
config = cr.read_config_as_dict(conf.CONFIG_PATH)
# create the log, so other module can use it
#Logger(config['logger']['path'], config['logger']['level'])
Logger(None, 7)
# create the pool, so other module can use it
DbPool(4, config['db']['host'], config['db']['port'], config['db']['user'],
config['db']['password'], config['db']['db_name'])
az_t = AnalyzeThread()
smt = StreamMonitorThread(config['get_stream_api_url'],
config['read_token'],
config['monitor_cdn_in_api'], az_t,
config['extra_streams'])
all_threads["analyze_thread"] = az_t
all_threads["stream_monitor"] = smt
az_t.start()
smt.start()
az_t.join()
smt.join()
def news(req):
sessionID = req.get("session")
session = re.compile("sessions/(.*)")
sessionID = session.findall(sessionID)[0]
result = req.get("queryResult")
user_says = result.get("queryText")
try:
config_reader = ConfigReader()
configuration = config_reader.read_config()
url = "http://newsapi.org/v2/top-headlines?country=in&category=health&apiKey=" + \
configuration['NEWS_API']
res = requests.get(url)
jsonRes = res.json()
articles = jsonRes["articles"]
news = list()
for i in range(len(articles)):
title = articles[i]["title"]
author = articles[i]["author"]
news_final = str(i + 1) + ". " + \
str(title) + " - " + str(author)
news.append(news_final)
fulfillmentText = "\n".join(news)
bot_says = fulfillmentText
return {"fulfillmentText": fulfillmentText}
except HTTPError as http_err:
print(f"HTTP error occurred: {http_err}")
except Exception as err:
print(f"Other error occurred: {err}")
def __init__(self, config=None, debug=False):
self.req_fields = ['title', 'descriprion', 'link']
self.opt_for_feed = { 'language': "", \
'subtitle': "", \
'updated': "", \
'updated_parsed': "", }
self.opt_for_items = { 'published': "", \
'published_parsed': "", \
'language': "", \
'term': { 'field': 'tags', \
'sub_field': 'term'},\
'summary': "" }
self.debug = debug
# Read config parametrs
config_reader = ConfigReader()
config_reader.read(config)
self.rss_urls = config_reader.url_names
self.freq = config_reader.freq
# Fields that are needed to be without html tags
self.text_extr_fields = set()
self.__set_text_extract_fields__()
self.cut_html = CutHTML()
self.cut_html.reset()
# Connect to db
self.db_connector = DBConnector(debug=self.debug)
# News agent name and time mark
self.news_agent_name = ""
# + 0000
self.time_mark = None
def webhook():
config_reader = ConfigReader()
configuration = config_reader.read_config()
req = request.get_json(silent=True, force=True)
print("Request:")
print(json.dumps(req, indent=4))
intent_check = req.get("queryResult").get("intent").get("displayName")
if intent_check == "CountryCases_menu":
res = getCountryName(req)
elif intent_check == "StateCases_menu":
res = getStateName(req)
elif intent_check == "MyAreaCases_menu":
res = getUserDetails(req)
elif intent_check == "GlobalCases_menu":
res = globalCases(req)
elif intent_check == "indiaCases_menu":
res = indiaCases(req)
elif intent_check == "News":
res = news(req)
elif intent_check == "Helpline_Menu":
res = helpLine(req)
res = json.dumps(res, indent=4)
r = make_response(res)
r.headers['Content-Type'] = 'application/json'
return r
def __init__(self, config=None, debug=False):
self.req_fields = ['title', 'descriprion', 'link']
self.opt_for_feed = { 'language': "", \
'subtitle': "", \
'updated': "", \
'updated_parsed': "", }
self.opt_for_items = { 'published': "", \
'published_parsed': "", \
'language': "", \
'term': { 'field': 'tags', \
'sub_field': 'term'},\
'summary': "" }
self.debug = debug
# Read config parametrs
config_reader = ConfigReader()
config_reader.read(config)
self.rss_urls = config_reader.url_names
self.freq = config_reader.freq
# Fields that are needed to be without html tags
self.text_extr_fields = set()
self.__set_text_extract_fields__()
self.cut_html = CutHTML()
self.cut_html.reset()
# Connect to db
self.db_connector = DBConnector(debug=self.debug)
# News agent name and time mark
self.news_agent_name = ""
# + 0000
self.time_mark = None
def send_email_to_botuser_local(recepient_email, message, id):
try:
config_reader = ConfigReader()
configuration = config_reader.read_config()
# instance of MIMEMultipart
msg = MIMEMultipart()
# storing the senders email address
msg['From'] = configuration['SENDER_EMAIL']
# storing the receivers email address
msg['To'] = ",".join(recepient_email)
# storing the subject
msg['Subject'] = configuration['EMAIL_SUBJECT']
# string to store the body of the mail
body = message
# attach the body with the msg instance
msg.attach(MIMEText(body, 'text'))
filename = "report" + str(id) + ".jpg"
attachment = open("LocalReport/" + filename, 'rb')
# instance of MIMEBase and named as p
p = MIMEBase('application', 'octet-stream')
# To change the payload into encoded form
p.set_payload((attachment).read())
# encode into base64
encoders.encode_base64(p)
p.add_header('Content-Disposition',
"attachment; filename= %s" % filename)
# attach the instance 'p' to instance 'msg'
msg.attach(p)
# creates SMTP session
smtp = smtplib.SMTP('smtp.gmail.com', 587)
# start TLS for security
smtp.starttls()
# Authentication
smtp.login(msg['From'], configuration['PASSWORD'])
# Converts the Multipart msg into a string
text = msg.as_string()
# sending the mail
smtp.sendmail(msg['From'], recepient_email, text)
print('email send')
# terminating the session
smtp.quit()
except Exception as e:
print('the exception is ' + str(e))
def run_test(instances):
"""
adb
-s
192.168.56.101:5555
shell
am
instrument
-w
-e
class
com.example.android.notepad.NotePadTest com.example.android.notepad.test/android.test.InstrumentationTestRunner
:return:
"""
for instance in instances:
adb = config.config_file_reader('adb')
directory = config.config_file_reader('directory')
#print(adb)
apk = config.config_file_reader('apk')
#print(directory + apk)
test_class = config.config_file_reader('test_class')
test_method = config.config_file_reader('test_method')
#print(test_method)
print(subprocess.check_output([adb, '-s', 'emulator-'+instance.port, 'shell', 'am','instrument', '-w', '-r', '-e', 'debug', 'false','-e','class', test_class, test_method]))
def send_email_to_student(self, recepient_email, message):
# print("recepient_email",recepient_email)
# print("message",message)
try:
self.config_reader=ConfigReader()
self.configuration=self.config_reader.read_config()
# instance of MIMEMultipart
self.msg = MIMEMultipart()
# storing the senders email address
self.msg['From'] = self.configuration['SENDER_EMAIL']
# storing the receivers email address
self.msg['To'] = ",".join(recepient_email)
# storing the subject
self.msg['Subject'] = self.configuration['EMAIL_SUBJECT']
# string to store the body of the mail
#body = "This will contain attachment"
body=message
# attach the body with the msg instance
self.msg.attach(MIMEText(body, 'html'))
# instance of MIMEBase and named as p
self.p = MIMEBase('application', 'octet-stream')
# creates SMTP session
self.smtp = smtplib.SMTP('smtp.gmail.com', 587)
# start TLS for security
self.smtp.starttls()
print("sender_email xx",self.msg['From'])
print("pass xx",self.configuration['PASSWORD'])
# Authentication
self.smtp.login(self.msg['From'], self.configuration['PASSWORD'])
# Converts the Multipart msg into a string
self.text = self.msg.as_string()
# sending the mail
self.smtp.sendmail(self.msg['From'] , recepient_email, self.text)
# terminating the session
self.smtp.quit()
except Exception as e:
print('the exception is email xx '+str(e))
def sendEmailToSupportForRecording(self, name, email, batch, day):
self.config_reader = ConfigReader()
self.configuration = self.config_reader.read_config()
# instance of MIMEMultipart
self.msg = MIMEMultipart()
# storing the senders email address
self.msg['From'] = "iNeuron Megatron Chatbot <*****@*****.**>"
# storing the receivers email address
self.msg['To'] = "*****@*****.**"
# storing the subject
self.msg['Subject'] = "Recording Uploading query"
body = """We have a query that recording is not uploaded in dashboard. Here is the details:-
Batch:- {}
Day:- {}
Customer-name:- {}
Customer-email:- {}
Please have a look into this.
Regards,
Team iNeuron
""".format(batch,day,name,email)
# attach the body with the msg instance
self.msg.attach(MIMEText(body, 'html'))
# instance of MIMEBase and named as p
self.p = MIMEBase('application', 'octet-stream')
# creates SMTP session
self.smtp = smtplib.SMTP('smtp.gmail.com', 587)
# start TLS for security
self.smtp.starttls()
# Authentication
self.smtp.login(
self.configuration['SENDER_EMAIL'], self.configuration['PASSWORD'])
# Converts the Multipart msg into a string
self.text = self.msg.as_string()
# sending the mail
self.smtp.sendmail(
self.configuration['SENDER_EMAIL'], self.msg['To'], self.text)
print("Email sent to support team!")
# terminating the session
self.smtp.quit()
return "Email sent to support team!"
def __init__(self):
reader = ConfigReader()
db_connection = reader.get_value("db_connection")
self.conn_string = '{db_engine}{connector}://{user}:{password}@{server}/{database}'.format(
db_engine=db_connection['db_engine'],
connector=db_connection['connector'],
user=db_connection['user'],
password=db_connection['password'],
server=db_connection['server'],
database=db_connection['database'])
def sendEmailToSupport(self, name, email, user_query):
self.config_reader = ConfigReader()
self.configuration = self.config_reader.read_config()
# instance of MIMEMultipart
self.msg = MIMEMultipart()
# storing the senders email address
self.msg['From'] = "iNeuron Megatron Chatbot <*****@*****.**>"
# storing the receivers email address
self.msg['To'] = "*****@*****.**"
# storing the subject
self.msg['Subject'] = "Customer query"
body = """We have a customer query, which we were not able to answer as of now. Here is the query:-
{}\n
Customer-name:- {}\n
Customer-email:- {}\n
Please have a look into this.
\n\n\n\n
Regards,
Team iNeuron
""".format(user_query,name,email)
# attach the body with the msg instance
self.msg.attach(MIMEText(body, 'html'))
# instance of MIMEBase and named as p
self.p = MIMEBase('application', 'octet-stream')
# creates SMTP session
self.smtp = smtplib.SMTP('smtp.gmail.com', 587)
# start TLS for security
self.smtp.starttls()
# Authentication
self.smtp.login(
self.configuration['SENDER_EMAIL'], self.configuration['PASSWORD'])
# Converts the Multipart msg into a string
self.text = self.msg.as_string()
# sending the mail
self.smtp.sendmail(
self.configuration['SENDER_EMAIL'], self.msg['To'], self.text)
print("Email sent to support team!")
# terminating the session
self.smtp.quit()
return "Email sent to support team!"
def send_email_to_support(self, cust_name, cust_email, cust_contact,
course_name, body):
try:
self.config_reader = ConfigReader()
self.configuration = self.config_reader.read_config()
# instance of MIMEMultipart
self.msg = MIMEMultipart()
# storing the senders email address
self.msg['From'] = self.configuration['SENDER_EMAIL']
# storing the subject
self.msg['Subject'] = self.configuration[
'SALES_TEAM_EMAIL_SUBJECT']
# string to store the body of the mail
# body = "This will contain attachment"
body = body.replace('cust_name', cust_name)
body = body.replace('cust_contact', cust_contact)
body = body.replace('cust_email', cust_email)
body = body.replace('course_name', course_name)
# attach the body with the msg instance
self.msg.attach(MIMEText(body, 'html'))
# instance of MIMEBase and named as p
self.p = MIMEBase('application', 'octet-stream')
# creates SMTP session
self.smtp = smtplib.SMTP('smtp.gmail.com', 587)
# start TLS for security
self.smtp.starttls()
# Authentication
self.smtp.login(self.msg['From'], self.configuration['PASSWORD'])
# Converts the Multipart msg into a string
self.text = self.msg.as_string()
# sending the mail
self.support_team_email = self.configuration['SALES_TEAM_EMAIL']
self.smtp.sendmail(self.msg['From'], self.support_team_email,
self.text)
# terminating the session
self.smtp.quit()
except Exception as e:
print('the exception is ' + str(e))
def __init__(self):
self.logger = logger
config_reader = ConfigReader("config.ini")
config = config_reader.load_config()
log_level = config['log_level']
log_file = config['log_file']
console_debug = config['console_debug']
if console_debug.strip() is "True":
console_debug = True
else:
console_debug = False
self.create_logger(log_level, log_file, console_debug=console_debug)
def __get_passthrough_tables(self, order):
passthrough_tables = ConfigReader().get_passthrough_tables()
passthrough_threshold = ConfigReader().get_passthrough_threshold()
for o in order:
for t in o:
c = database_helper.get_table_count(t, self.schema,
self.__source_conn)
if c <= passthrough_threshold:
passthrough_tables.append(t)
#an explicitly marked passthrough table canhave under 100 rows in which case it'll appear in final list twice
return list(set(passthrough_tables))
def test_get(self, whole_config):
# Setup
desired_conf = whole_config
# Exercise
cr = ConfigReader('data_test')
actual_conf = cr.svt_config
# Verify
assert desired_conf == actual_conf
assert cr.get('sql_alchemy') == desired_conf['sql_alchemy']
assert cr.get('sql_alchemy',
'host') == desired_conf['sql_alchemy']['host']
assert cr.get('sql_alchemy',
'port') == desired_conf['sql_alchemy']['port']
def setUp(self):
self.config = ConfigReader("""
<root>
<person>
<name>fake</name>
<age>15</age>
</person>
<person>
<name>test</name>
<age>43</age>
</person>
</root>
""")
def sendEmailDistrict(self, name, email, district, confirmedCases, body):
try:
self.config_reader = ConfigReader()
self.configuration = self.config_reader.read_config()
# instance of MIMEMultipart
self.msg = MIMEMultipart()
# storing the senders email address
self.msg['From'] = "WeBlaze <*****@*****.**>"
# storing the receivers email address
self.msg['To'] = email
# storing the subject
self.msg['Subject'] = self.configuration['EMAIL_SUBJECT']
# string to store the body of the mail
body = body.replace("name", name)
body = body.replace("district", district)
body = body.replace("num_cases", confirmedCases)
# attach the body with the msg instance
self.msg.attach(MIMEText(body, 'html'))
# instance of MIMEBase and named as p
self.p = MIMEBase('application', 'octet-stream')
# creates SMTP session
self.smtp = smtplib.SMTP('smtp.gmail.com', 587)
# start TLS for security
self.smtp.starttls()
# Authentication
self.smtp.login(self.configuration['SENDER_EMAIL'],
self.configuration['PASSWORD'])
# Converts the Multipart msg into a string
self.text = self.msg.as_string()
# sending the mail
self.smtp.sendmail(self.configuration['SENDER_EMAIL'], email,
self.text)
print("Email sent to district user!")
# terminating the session
self.smtp.quit()
except Exception as e:
print('the exception is ' + str(e))
def send_email(self, cust_name,recepient_email, message, fulfillmentText):
try:
self.config_reader=ConfigReader()
self.configuration=self.config_reader.read_config()
# instance of MIMEMultipart
self.msg = MIMEMultipart()
# storing the senders email address
self.msg['From'] = self.configuration['SENDER_EMAIL']
# storing the receivers email address
#self.msg['To'] = ",".join(recepient_email)
self.msg['To'] = recepient_email
# storing the subject
self.msg['Subject'] = self.configuration['EMAIL_SUBJECT']
# string to store the body of the mail
body = message
body = body.replace('cust_name', cust_name)
body = body.replace('fulfillmentText', fulfillmentText)
# attach the body with the msg instance
self.msg.attach(MIMEText(body, 'html'))
# instance of MIMEBase and named as p
self.p = MIMEBase('application', 'octet-stream')
# creates SMTP session
self.smtp = smtplib.SMTP('smtp.gmail.com', 587)
# start TLS for security
self.smtp.starttls()
# Authentication
self.smtp.login(self.msg['From'], self.configuration['PASSWORD'])
# Converts the Multipart msg into a string
self.text = self.msg.as_string()
# sending the mail
self.smtp.sendmail(self.msg['From'] , recepient_email, self.text)
# terminating the session
self.smtp.quit()
except Exception as e:
print('the exception is '+str(e))
def test_occupy_home_or_next_to_home():
# add two invalid files: "occupy_home_file.txt" and
# "occupy_next_to_home_file.txt" in "invalid_files"
path = "./invalid_files/occupy_home_file.txt"
try:
config_reader = ConfigReader(path)
return print("test_occupy_home_or_next_to_home failed!")
except ValueError:
return True
path = "./invalid_files/occupy_next_to_home_file.txt"
try:
config_reader = ConfigReader(path)
return print("test_occupy_home_or_next_to_home failed!")
except ValueError:
return True
def get_recs_from_recserver(self, users, nr_recs, algo, items):
config_reader = ConfigReader()
base_url = config_reader.get_value("rec_server_url")
#is_a_rec_request = True if algo == 'popular' or algo == 'topn' else False
recs = []
for userId in users:
# if is_a_rec_request:
url = f'{base_url}/algorithms/{algo}/recommendations?user_id={userId}&num_recs={nr_recs}'
# else:
# url = f'{base_url}/algorithms/{algo}/predictions?user_id={userId}&items={items}'
r = requests.get(url)
data = r.json()
#recs.append({'user': userId, 'recs': data['recommendations'] if is_a_rec_request else data['predictions']})
recs.append({'user': userId, 'recs': data['recommendations']})
return recs
def webhook():
config_reader = ConfigReader()
configuration = config_reader.read_config()
client = MongoClient(
"mongodb+srv://your_username:"******"MONGO_PASSWORD"] + "@cluster0-p5lkb.mongodb.net/dialogflow?retryWrites=true&w=majority")
db = client.dialogflow
req = request.get_json(silent=True, force=True)
print("Request:")
print(json.dumps(req, indent=4))
intent_check = req.get("queryResult").get("intent").get("displayName")
if (intent_check == "AboutCorona" or
intent_check == "CountryCases" or
intent_check == "CovidMap" or
intent_check == "CovidTest" or
intent_check == "Fallback" or
intent_check == "Goodbye" or
intent_check == "Menu" or
intent_check == "MyAreaCases" or
intent_check == "MythBuster" or
intent_check == "Precaution" or
intent_check == "QuarantineTips" or
intent_check == "StateCases" or
intent_check == "Symptoms" or
intent_check == "Welcome"):
res = saveToDb(req, db)
elif intent_check == "GetCountryName":
res = getCountryName(req, db)
elif intent_check == "GetStateName":
res = getStateName(req, db)
elif intent_check == "GetUserDetails":
res = getUserDetails(req, db)
elif intent_check == "GlobalCases":
res = globalCases(req, db)
elif intent_check == "IndiaCases":
res = indiaCases(req, db)
elif intent_check == "News":
res = news(req, db)
res = json.dumps(res, indent=4)
r = make_response(res)
r.headers['Content-Type'] = 'application/json'
return r
def test_file_not_found():
invalid_path = "./invalid_files/not_found.txt"
try:
config_reader = ConfigReader(invalid_path)
return print("test_file_not_found failed!")
except FileNotFoundError:
return True
class WeatherInformation():
def __init__(self):
self.config_reader = ConfigReader()
self.configuration = self.config_reader.read_config()
self.owmapikey = self.configuration['WEATHER_API_KEY']
self.owm = pyowm.OWM(self.owmapikey)
def get_weather_info(self,city):
self.city=city
observation = self.owm.weather_at_place(city)
w = observation.get_weather()
latlon_res = observation.get_location()
lat = str(latlon_res.get_lat())
lon = str(latlon_res.get_lon())
wind_res = w.get_wind()
wind_speed = str(wind_res.get('speed'))
humidity = str(w.get_humidity())
celsius_result = w.get_temperature('celsius')
temp_min_celsius = str(celsius_result.get('temp_min'))
temp_max_celsius = str(celsius_result.get('temp_max'))
fahrenheit_result = w.get_temperature('fahrenheit')
temp_min_fahrenheit = str(fahrenheit_result.get('temp_min'))
temp_max_fahrenheit = str(fahrenheit_result.get('temp_max'))
self.bot_says = "Today the weather in " + city +" is :\n Maximum Temperature :"+temp_max_celsius+ " Degree Celsius"+".\n Minimum Temperature :"+temp_min_celsius+ " Degree Celsius" +": \n" + "Humidity :" + humidity + "%"
return self.bot_says
def test_non_integer():
path = "./invalid_files/non_integer_file.txt"
try:
config_reader = ConfigReader(path)
return print("test_non_integer failed!")
except ValueError:
return True
def test_frame_out_of_range():
path = "./invalid_files/format_out_of_range_file.txt"
try:
config_reader = ConfigReader(path)
return print("test_frame_out_of_range failed!")
except SyntaxError:
return True
def test_format_error():
path = "./invalid_files/format_error_file.txt"
try:
config_reader = ConfigReader(path)
return print("test_format_error failed!")
except SyntaxError:
return True
def test_list_config_files(self, desired_config_files):
# Setup
config_directory = os.path.join(os.getcwd(), 'data_test')
# Exercise
actual_config_files = ConfigReader.list_config_files(config_directory)
# Verify
assert desired_config_files == actual_config_files
def __init__(self):
self.sleep_time = 60
self.config_file_path = "config.json"
self.config = ConfigReader(self.config_file_path)
self.checker = MovieChecker(self.config)
self.sender = EmailSender(self.config)
def __init__(self):
self.stats = {}
self.log_stream = []
self.containers = []
self.monitor = None
self.log_collector = None
log_mgr = LogManager()
# Read config file and load the dictionary
config_reader = ConfigReader("config.ini")
self.config = config_reader.load_config()
logger.info("Loaded configurations")
self._load_stats_monitor()
#self._load_log_collector()
self.containers = self.list_all_containers()
def create(self):
if self.use_existing_dump == True:
pass
else:
tt = ConfigReader().get_all_tables()
tables = ' '.join(['-t ' + t for t in tt])
cur_path = os.getcwd()
pg_dump_path = get_pg_bin_path()
if pg_dump_path != '':
os.chdir(pg_dump_path)
pg_dumpsql_path = os.path.join(self.output_path, 'schema_dump.sql')
os.system(
'pg_dump --dbname=postgresql://{0}:{1}@{2}:{3}/{4} > {5} --schema-only --no-owner --no-privileges {6}'
.format(self.source_connection_info['user_name'],
self.source_connection_info['password'],
self.source_connection_info['host'],
self.source_connection_info['port'],
self.source_connection_info['db_name'],
pg_dumpsql_path, tables))
os.chdir(cur_path)
self.__filter_commands(self.output_path)
self.destination_psql_client.run(
os.path.join(self.output_path, 'dump_create.sql'),
self.create_output_path, self.create_error_path, True, True)
def process_job_gather(self, ifn):
"""Process a job config file, allocate the tasks."""
print "proccess job", ifn
VRNUM = 80
jobid = random.randint(0,65535)
config = ConfigReader(ifn)
l = self.get_idle_nodes()
m = 0
# Liang: allocation strategy is here, needs improvement
for vrid, vrd in config.vrdetail.items():
m += 1
n = min(int(m/VRNUM), len(l)-1)
# Inform a node to intialize a job
if (m-1) % VRNUM == 0:
msg = "jobc%i|%s" % (jobid, config.vrset2msg(config.vrouters))
self.reliable_send(msg, (l[n][1], BPORT))
msg = "ctvr%i|%s" % (jobid, config.vrd2msg(vrd))
self.reliable_send(msg, (l[n][1], BPORT))
if config.has_upperapp(vrid):
msg = "ctap%i|%s" % (jobid, config.apd2msg(config.appdetail[vrid]))
self.reliable_send(msg, (l[n][1], BPORT))
pass
def main():
cr = ConfigReader()
config = cr.read_config_as_dict(conf.CONFIG_PATH)
# create the log, so other module can use it
#Logger(config['logger']['path'], config['logger']['level'])
Logger(None, 7)
# create the pool, so other module can use it
DbPool(4, config['db']['host'], config['db']['port'], config['db']['user'], config['db']['password'], config['db']['db_name'])
az_t = AnalyzeThread()
smt = StreamMonitorThread(config['get_stream_api_url'], config['read_token'], config['monitor_cdn_in_api'], az_t, config['extra_streams'])
all_threads["analyze_thread"] = az_t
all_threads["stream_monitor"] = smt
az_t.start()
smt.start()
az_t.join()
smt.join()
class TestConfigReader(unittest.TestCase):
def setUp(self):
self.config = ConfigReader("""
<root>
<person>
<name>山田</name>
<age>15</age>
</person>
<person>
<name>佐藤</name>
<age>43</age>
</person>
</root>
""")
def test_get_names(self):
self.assertEqual(self.config.get_names(), ['山田', '佐藤'])
def test_get_ages(self):
self.assertEqual(self.config.get_ages(), ['15', '43'])
def setUp(self):
self.config = ConfigReader("""
<root>
<person>
<name>山田</name>
<age>15</age>
</person>
<person>
<name>佐藤</name>
<age>43</age>
</person>
</root>
""")
def __init__(self):
self.config_reader = ConfigReader('../config.ini')
self.clock = 0
# Config information
self.current_server_id = int(self.config_reader.getConfiguration("CurrentServer", "sid"))
self.total_nodes = int(self.config_reader.getConfiguration("GeneralConfig", "nodes"))
self.peers = self.config_reader.get_peer_servers(self.current_server_id, self.total_nodes)
# log: Each list corresponds to each node
self.log = list()
self.timeTable = [[0 for i in range(self.total_nodes)] for i in range(self.total_nodes)]
# Blogs at this server
self.blogs = list()
# New server socket to listen to requests from Client/Peers
self.serverSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.serverSock.bind((self.config_reader.get_ip_and_port_of_server("Server" + str(self.current_server_id))))
self.serverSock.listen(128)
class Server(object):
def __init__(self):
self.config_reader = ConfigReader('../config.ini')
self.clock = 0
# Config information
self.current_server_id = int(self.config_reader.getConfiguration("CurrentServer", "sid"))
self.total_nodes = int(self.config_reader.getConfiguration("GeneralConfig", "nodes"))
self.peers = self.config_reader.get_peer_servers(self.current_server_id, self.total_nodes)
# log: Each list corresponds to each node
self.log = list()
self.timeTable = [[0 for i in range(self.total_nodes)] for i in range(self.total_nodes)]
# Blogs at this server
self.blogs = list()
# New server socket to listen to requests from Client/Peers
self.serverSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.serverSock.bind((self.config_reader.get_ip_and_port_of_server("Server" + str(self.current_server_id))))
self.serverSock.listen(128)
# Crux of server functionality. Main logic goes here
def requestHandler(self, server_socket, address):
print "New Thread..."
msg = receiveObject(server_socket)
if POST == msg.operation:
# Increment clock and Insert in respective log
self.clock += 1
self.log.append((self.current_server_id, self.clock, msg.blog))
self.timeTable[self.current_server_id][self.current_server_id] += 1
self.blogs.append(msg.blog)
self.print_data(msg, POST)
elif LOOKUP == msg.operation:
sendObject(server_socket, self.blogs)
self.print_data(msg, LOOKUP)
# Client sends the message sync so that the given server syncs with the given parameter
elif SYNC == msg.operation:
# all the entries where hasRec is false
filtered_log = self.filter_log(self.log, int(msg.server_to_sync))
message = Message(operation="server_sync", timeTable=self.timeTable, logs=filtered_log,
sent_server_id=self.current_server_id)
sock_to_other = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock_to_other.connect(self.config_reader.get_ip_and_port_of_server("Server" + msg.server_to_sync))
sendObject(sock_to_other, message)
sock_to_other.close()
print "Filtered Log: %s" % filtered_log
self.print_data(msg,SYNC)
# Received from a server. Update the logs and sync servers to complete replication
elif SERVER_SYNC == msg.operation:
sent_server = msg.sent_server_id
other_log = msg.logs
other_time_table = msg.timeTable
self.add_new_events_to_log_and_blog(other_time_table, other_log)
self.update_max_elements(other_time_table)
self.update_rows(other_time_table, sent_server)
self.print_data(msg,SERVER_SYNC)
else:
print "Received an unknown operation %s" % msg.operation
server_socket.close()
def start(self):
# Start listening for incoming requests
while True:
connection, address = self.serverSock.accept()
threading.Thread(target=self.requestHandler, args=(connection, address)).start()
def filter_log(self, log_to_be_filtered, server_to_sync):
log_to_send = []
# Get the max clock values that I know that server to sync knows about blogs in other servers
known_clocks = list()
for i in range(self.total_nodes):
known_clocks.append(self.timeTable[server_to_sync][i])
log_to_send = [logEntry for logEntry in log_to_be_filtered if logEntry[1] > known_clocks[logEntry[0]]]
return log_to_send
def update_max_elements(self, other_time_table):
for i in range(0, self.total_nodes):
for j in range(0, self.total_nodes):
self.timeTable[i][j] = max(self.timeTable[i][j], other_time_table[i][j])
def update_rows(self, other_time_table, sent_server):
for i in range(0, self.total_nodes):
self.timeTable[self.current_server_id][i] = max(self.timeTable[self.current_server_id][i],
other_time_table[sent_server][i])
def add_new_events_to_log_and_blog(self, other_time_table, other_log):
# Add to Log
log_to_append = self.filter_log(other_log, self.current_server_id)
print "Log to append: %s" %log_to_append
self.log.extend(log_to_append)
# Add to Dictionary
for i in range(0, len(log_to_append)):
self.blogs.append(log_to_append[i][2])
def print_data(self, msg, operation):
print "****************%s Operation***************\nLog: %s\nTT: %s\nMsg %s\nClock: %s\n" % (
operation, self.log, self.timeTable, msg, self.clock)
class Client(object):
def __init__(self):
self.config_reader = ConfigReader("client_config.ini")
self.total_nodes = self.config_reader.get_total_nodes()
self.servers = self.config_reader.get_peers(self.total_nodes)
self.client_id = 999
def post(self, msg,server_id):
server_ip, server_port = self.config_reader.get_server_port_ip(server_id, self.servers)
return_value = None
try:
connection = rpyc.connect(server_ip, server_port, config = {"allow_public_attrs" : True})
return_value = connection.root.postRPC(blog=msg,client_id=self.client_id)
except Exception as details:
print "\n"
print "Server down..."
return return_value
def lookup(self, server_id):
server_ip, server_port = self.config_reader.get_server_port_ip(server_id, self.servers)
return_value = None
try:
connection = rpyc.connect(server_ip, server_port, config = {"allow_public_attrs" : True})
return_value = connection.root.lookupRPC()
print "\nBlogs: "
print "\n"
for blog in return_value:
prefix = blog.split()[0]
if prefix != "JOINT_CONFIGURATION" and prefix!="NEW_CONFIGURATION":
print blog
except Exception as details:
print "\n"
print "Server down..."
def change_config_of_network(self, list_of_changes,server_id):
server_ip, server_port = self.config_reader.get_server_port_ip(server_id, self.servers)
return_value = None
try:
connection = rpyc.connect(server_ip, server_port, config = {"allow_public_attrs" : True})
return_value = connection.root.config_changeRPC(list_of_config_changes =list_of_changes, client_id=self.client_id)
except Exception as details:
print details
print "\n"
print "Server down..."
if return_value:
self.change_config_of_client(list_of_changes)
return return_value
def change_config_of_client(self, list_of_changes):
for config in list_of_changes:
if config[0] == "ADD":
self.servers.append((config[1],config[2],int(config[3])))
def start_console(self):
print "\n"
while True:
command = raw_input("\nOperations allowed: \n\nPOST <server id> message\nLOOKUP <server id>\nCONFIG <server_to_send_to> \nEnter 0 to EXIT\n\nEnter operation: ")
command_parse = command.split()
if command_parse[0] == "POST":
server_id = int(command_parse[1])
msg = command_parse[2]
try:
return_value = self.post(msg,server_id)
if return_value:
print "\nBlog successfully posted..."
except Exception as details:
print details
elif command_parse[0] == "LOOKUP":
server_id = int(command_parse[1])
self.lookup(server_id)
elif command_parse[0] == "CONFIG":
config_change_list = list() #Follow the format above
server_id = int(command_parse[1])
while True:
config_input = raw_input("Config Change Selected\n Please type ADD <server_id> <server_ip> <server_port> \n REMOVE <server_id> \n DONE\n")
config_command = config_input.split()
add_or_remove = config_command[0]
if "ADD" == add_or_remove or "REMOVE" == add_or_remove:
config_change_list.append(config_command)
elif "DONE" == add_or_remove:
if config_change_list:
break
else:
print "Empty config change found!"
break
else:
print "Wrong command. Exiting program."
sys.exit(0)
return_value = self.change_config_of_network(config_change_list,server_id)
print return_value
elif int(command) == 0:
sys.exit(0)
else:
print "Unknown command"
def __init__(self):
self.config_reader = ConfigReader("client_config.ini")
self.total_nodes = self.config_reader.get_total_nodes()
self.servers = self.config_reader.get_peers(self.total_nodes)
self.client_id = 999
'''
Created on 2011-12-24
@author: dista
'''
import web
import conf
import json
from config_reader import ConfigReader
from common import json_handler
cr = ConfigReader()
config = cr.read_config_as_dict(conf.CONFIG_PATH)
urls = (
"/samples/.*", "samples"
)
class samples:
def GET(self):
path_info_arr = web.ctx.env['PATH_INFO'].split('/')
offset, page_size = (int(i) for i in path_info_arr[4].split('-'))
keyword, sort_by, sort_order, sort_by_status_first = path_info_arr[5:]
sort_by_status_first = int(sort_by_status_first)
if sort_order.upper() not in ["ASC", "DESC"]:
raise badrequest()
db = web.database(dbn='mysql', db='tvie_production2', host='10.33.0.57', user='******', pw='tvierocks')