def __check_user_input(self, is_compound):
log_enter_func('CreateCssRuleSet', '__check_user_input',
{'is_compound': is_compound})
user_input_ok = True
if self.__selector_var.get() == '':
log_error('CreateCssRuleSet', '__check_user_input',
'Selector not set!', None)
messagebox.showerror('Input Error', 'Selector not set!')
user_input_ok = False
if is_compound and self.__selector_sec_part_var.get() == '':
log_error('CreateCssRuleSet', '__check_user_input',
'Selector second part not set!', None)
user_input_ok = False
log_leave_func('CreateCssRuleSet', '__check_user_input', user_input_ok)
return user_input_ok
def add_gtfs_to_db(file):
dbname = _filename_to_dbname(file)
db.create_db(dbname)
dao = db.access_direct_dao(dbname)
errormsg = None
try:
log_trace("Creating dataset...")
dataset_id = db.create_dataset(dbname)
log_trace("Loading GTFS...")
dao.load_gtfs(file)
new_agencies = dao.agencies()
log_trace("Calculating lat/lng of agencies...")
lat, lng = db.get_random_mean_lat_lng(dbname)
log_trace("Updating agencies...")
old_dataset_ids = db.update_agencies(new_agencies, dataset_id, lat, lng)
for old_id in old_dataset_ids:
log_trace("Deleting old dataset...")
db.delete_dataset(old_id)
calculate_urban(dbname)
log_trace("Filling population database...")
calculate_population(dbname, dataset_id)
log_trace("Setting success...")
db.set_done(dataset_id)
log_trace("Done")
except Exception as e:
log_error(e)
db.set_failed(dataset_id)
db.drop_database(dbname)
# TODO delete all created stuff and put db in old state
raise Exception("Error loading gtfs zip file: {0}".format(str(e)))
return dbname
def __on_ok(self):
log_enter_func('CreateCssRuleSet', '__on_ok')
selector_type = self.__selector_type_var.get()
log_set_var('CreateCssRuleSet', '__on_ok', 'selector_type',
selector_type)
is_compound = self.__is_compound(selector_type)
log_set_var('CreateCssRuleSet', '__on_ok', 'is_compound', is_compound)
if not self.__check_user_input(is_compound):
log_error('CreateCssRuleSet', '__on_ok', 'user input check failed')
log_leave_func('CreateCssRuleSet', '__on_ok')
return
selector_element = self.__get_selector_element(selector_type)
log_set_var('CreateCssRuleSet', '__on_ok', 'selector_element',
selector_element)
selector_specifier = self.__get_selector_specifier(
selector_type, is_compound)
log_set_var('CreateCssRuleSet', '__on_ok', 'selector_specifier',
selector_specifier)
self.__dialog.destroy()
self.__observer.on_create_css_rule_set_closed(
(self.__SELECTOR_TYPE_MAP[selector_type], selector_element,
selector_specifier, is_compound,
self.__SELECTOR_TYPE_SEP.get(selector_type)))
log_leave_func('CreateCssRuleSet', '__on_ok')
def parse_msg(raw_msg):
"""
Given a raw IRC message that is either a whisper or a channel message, parse out useful information.
:param raw_msg: str - The IRC raw message that includes the type PRIVMSG or WHISPER
:return: tuple<str, str, str, bool, bool> - A tuple of user info:
(Display_Name, channel/whisper target, message, is_mod, is_sub)
"""
try:
raw_msg_tokens = raw_msg.split(maxsplit=4)
display_name, is_mod, is_sub = TwitchBot.parse_tags(raw_msg_tokens[0][1:])
# If we fail to get the display name for whatever reason, get it from the raw IRC message
if not display_name:
display_name = raw_msg_tokens[1].split('!')[0][1:]
target_name = raw_msg_tokens[3]
# Channel messages have channel name starting with '#', whispers have no symbol
if target_name.startswith('#'):
target_name = target_name[1:]
msg = raw_msg_tokens[4][1:]
return display_name, target_name, msg, is_mod, is_sub
except Exception as e:
log_error('Unable to parse message "{}"'.format(raw_msg), e)
return None
def get_athlete(name_athlete):
"""
Get info of athlete
:param name_athlete: str
:return:
"""
if not name_athlete:
return 'nhập tên vận động viên'
connection = Connection().get_connection()
try:
cursor = connection.cursor() # prepare an object cursor
query = """
SELECT * FROM athlete WHERE name='{}';
""".format(name_athlete.strip().lower())
log_debug(query)
cursor.execute(query)
result = cursor.fetchall()
result = answer_athele(name_athlete, result)
# cursor.close()
# del cursor
# connection.close()
return result
except MySQLError as ex:
connection.close()
log_error("Can't get if of athlete with name - {}: {}".format(name_athlete, ex))
return []
def get_chatters(channel_name):
"""
Gets a list of chatters in a broadcaster's chatroom, sorted by their permissions in alphabetical order.
{
"_links": {},
"chatter_count": 4,
"chatters": {
"moderators": [
"mod_name",
"broadcaster_name"
],
"staff": [],
"admins": [],
"global_mods": [],
"viewers": [
"other_viewer",
"viewer_1"
]
}
}
:param channel_name: str - The name of the channel you want the chatters from
:return: dict - A dictionary of the above form, namely moderators and viewers
"""
try:
with urllib.request.urlopen(
settings.CHATTERS_URL.format(
channel_name.lower())) as chatters_response:
chatters = chatters_response.read().decode(encoding='UTF-8')
return json.loads(chatters)
except Exception as e:
log_error('Failed to get chatters dict', e)
return {}
def handle_msg(self, raw_msg):
"""
Given an arbitrary IRC message, handle it as necessary.
:param raw_msg: str - The IRC raw message
"""
super().handle_msg(raw_msg)
lower_msg = raw_msg.lower()
if lower_msg in [
':tmi.twitch.tv notice * :error logging in',
':tmi.twitch.tv notice * :login unsuccessful'
]:
raise RuntimeError(
'Failed to login, most likely invalid login credentials.')
raw_msg_tokens = raw_msg.split()
if len(raw_msg_tokens) < 3:
return
try:
if raw_msg_tokens[2] == 'PRIVMSG':
self.handle_channel_msg(raw_msg)
elif raw_msg_tokens[2] == 'WHISPER':
self.handle_whisper(raw_msg)
except Exception as e:
log_error('IRC message handler error', e)
def parse_msg(raw_msg):
"""
Given a raw IRC message that is either a whisper or a channel message, parse out useful information.
:param raw_msg: str - The IRC raw message that includes the type PRIVMSG or WHISPER
:return: tuple<str, str, str, bool, bool> - A tuple of user info:
(Display_Name, channel/whisper target, message, is_mod, is_sub)
"""
try:
raw_msg_tokens = raw_msg.split(maxsplit=4)
display_name, is_mod, is_sub = TwitchBot.parse_tags(
raw_msg_tokens[0][1:])
# If we fail to get the display name for whatever reason, get it from the raw IRC message
if not display_name:
display_name = raw_msg_tokens[1].split('!')[0][1:]
target_name = raw_msg_tokens[3]
# Channel messages have channel name starting with '#', whispers have no symbol
if target_name.startswith('#'):
target_name = target_name[1:]
msg = raw_msg_tokens[4][1:]
return display_name, target_name, msg, is_mod, is_sub
except Exception as e:
log_error('Unable to parse message "{}"'.format(raw_msg), e)
return None
def check_exists_create(fb_id, page_id):
"""
Check user is exists, if not create new one
:param fb_id: str
:param page_id: str
:return:
"""
connection = Connection().get_connection()
try:
cursor = connection.cursor() # prepare an object cursor
query = """
SELECT * FROM follower WHERE fb_id={};
""".format(fb_id)
log_debug(query)
cursor.execute(query)
result = cursor.fetchone()
if not result:
query = """
INSERT INTO follower (fb_id, page_id)
VALUES ('{}', '{}');
""".format(fb_id, page_id)
log_debug(query)
cursor.execute(query)
cursor.close()
del cursor
connection.commit()
# connection.close()
except MySQLError as ex:
# connection.close()
log_error("Can't check exists then create one: {}".format(ex))
def user_send_mess():
try:
data = request.get_json()
data["source"] = "zalo"
chatbot.handle(data)
except Exception as ex:
log_error(str(ex))
return "ok", 200
def webhook_handle():
try:
data = request.get_json()
data["source"] = "facebook"
chatbot.handle(data)
except Exception as ex:
log_error(str(ex))
return "ok", 200
def handler(*args, **kwargs):
try:
return fn(*args, **kwargs)
except ValidationError as e:
return ErrorMessage(str(e))
except Exception as e:
log_error(str(e))
traceback.print_exc(file=sys.stdout)
return ErrorMessage('Internal error.')
def run_owm_collect(self):
'''run_owm_collect'''
try:
owm_data = self.owm_data_collector.collect_default()
if owm_data is not None:
self.data_service_client.send_owm(owm_data)
#owm_data = None
except Exception as err:
logger.log_error(
'Failed to process OpenWeatherMap data in main loop', err)
def collect(self, location_id):
'''Collects transformed data from OpenWeatherMap'''
try:
data = self.get_data(location_id)
self._validator.validate(data)
transformed_data = self._transformer.transform_data(data)
return transformed_data
except Exception as err:
logger.log_error('Failed to collect OWM data:', err)
return None
def run_wu_collect(self, wu_location):
'''run_wu_collect'''
try:
if wu_location is not None:
wu_data = self.wu_data_collector.collect(wu_location)
if wu_data is not None:
self.data_service_client.send_wu(wu_data)
#wu_data = None
except Exception as err:
logger.log_error(
'Failed to process WeatherUnderground data in main loop', err)
def _load_yaml_file(yaml_file):
""" load yaml file and check file content format
"""
with io.open(yaml_file, 'r', encoding='utf-8') as stream:
yaml_content = None
try:
yaml_content = yaml.load(stream)
except yaml.YAMLError as ex:
logger.log_error(str(ex))
#raise
return yaml_content
def collect(self, location):
'''collect'''
try:
location_query = self.create_location_query(location)
data = self.get_data(location_query)
wu_validate(data)
transformed_data = self._transformer.transform_data(
data, location['_id'])
return transformed_data
except Exception as err:
logger.log_error('Failed to collect WU data', err)
return None
def call_service(functionToRun, responseType, **kwargs):
start = get_time()
response = ""
try:
response = jsonify({ responseType: functionToRun(**kwargs) })
except Exception as e:
log_error(e)
response = jsonify({ 'error': str(e) }), 400
finally:
end = get_time()
params = "| GET endpoint = /"+responseType+" | "+str(kwargs)
log_performance(start, end, params, "performance.log")
return response
def _load_json_file(json_file):
""" load json file and check file content format
"""
with io.open(json_file, encoding='utf-8') as data_file:
try:
json_content = json.load(data_file)
except json.JSONDecodeError:
err_msg = u"JSONDecodeError: JSON file format error: {}".format(
json_file)
logger.log_error(err_msg)
#raise exceptions.FileFormatError(err_msg)
return json_content
def get_moderators(channel_name):
"""
Gets a list of moderators present in a broadcaster's chatroom in alphabetical order.
channel_name: str - The name of the channel you want the mods from
:return: list<str> - A list of alphabetized mods
"""
try:
chatters = ChannelManager.get_chatters(channel_name)
return chatters['chatters']['moderators']
except Exception as e:
log_error('Failed to get online mod list', e)
return []
def transmit_message_and_get_response(address, path, message_bytes=None):
"""
:param path:
:param address: '<host>:<port>' (str)
:param message_bytes: message to transmit
:return:
"""
response = requests.post('http://' + address + '/' + path, data=message_bytes)
if not response.ok:
logger.log_error('post request status is not ok!')
return response.content
def get_viewers(channel_name):
"""
Gets a list of viewers present in a broadcaster's chatroom
channel_name: str - The name of the channel you want the mods from
:return: list<str> - A list of viewers (not alphabetized necessarily)
"""
try:
chatters = ChannelManager.get_chatters(channel_name)
viewers = []
for _, categorized_viewers in chatters['chatters'].items():
viewers.extend(categorized_viewers)
return viewers
except Exception as e:
log_error('Failed to get online viewers list', e)
return []
def send_wu(data):
'''send_wu'''
# Skipping unique validation, let's always save the data
#if not self.validate_unique_wu(data):
# return None
dump = json.dumps(data)
try:
hdrs = {'Content-type': 'application/json'}
response = requests.post(serviceconfig.URL_DATA_SERVICE_WEATHER2,
headers=hdrs,
data=dump)
logger.log_info(response)
except Exception as err:
logger.log_error('Failed to send WU weather data', err)
def command(self, command, args=None):
"""Invokes a command with a bazel binary.
Args:
command: A string specifying the bazel command to invoke.
args: An optional list of strings representing additional arguments to the
bazel command.
Returns:
A dict containing collected metrics (wall, cpu, system times and
optionally memory), the exit_status of the Bazel invocation, and the
start datetime (in UTC).
Returns None instead if the command equals 'shutdown'.
"""
args = args or []
logger.log('Executing Bazel command: bazel %s %s %s' %
(' '.join(self._startup_options), command, ' '.join(args)))
result = dict()
result['started_at'] = datetime.datetime.utcnow()
before_times = self._get_times()
dev_null = open(os.devnull, 'w')
exit_status = 0
try:
subprocess.check_call([self._bazel_binary_path] +
self._startup_options + [command] + args,
stdout=dev_null,
stderr=dev_null)
except subprocess.CalledProcessError as e:
exit_status = e.returncode
logger.log_error('Bazel command failed with exit code %s' %
e.returncode)
if command == 'shutdown':
return None
after_times = self._get_times()
for kind in ['wall', 'cpu', 'system']:
result[kind] = after_times[kind] - before_times[kind]
result['exit_status'] = exit_status
# We do a number of runs here to reduce the noise in the data.
result['memory'] = min([self._get_heap_size() for _ in range(5)])
return result
def run_bot():
"""
Simply starts the bot, and restarts on crash.
"""
try:
# Create the bot
bot = QuestBot(settings.BOT_NAME, settings.BROADCASTER_NAME, settings.BOT_OAUTH)
bot.connect()
bot.run()
except Exception as e2:
log_error('Bot crashed', e2)
# If the bot crashes for whatever reason, restart it
if settings.AUTO_RESTART_ON_CRASH:
time.sleep(5)
run_bot()
pause()
def validate_unique_owm(data_dict):
'''Validates that data service doesn't already contain the OWM data'''
try:
filters = {'locid': data_dict['locid'], 'time': data_dict['time']}
payload = {'where': json.dumps(filters)}
response = requests.get(serviceconfig.URL_DATA_SERVICE_WEATHER,
params=payload)
if len(response.json()['_items']) == 0:
return True
else:
logger.log_info(
'OWM values already stored - locid={}, time={}'.format(
filters['locid'], filters['time']))
except Exception as err:
logger.log_error('Failed to validate unique OWM weather data', err)
return False
def handle_msg(self, raw_msg):
"""
Given an arbitrary IRC message, handle it as necessary.
:param raw_msg: str - The IRC raw message
"""
super().handle_msg(raw_msg)
lower_msg = raw_msg.lower()
if lower_msg in [':tmi.twitch.tv notice * :error logging in', ':tmi.twitch.tv notice * :login unsuccessful']:
raise RuntimeError('Failed to login, most likely invalid login credentials.')
raw_msg_tokens = raw_msg.split()
if len(raw_msg_tokens) < 3:
return
try:
if raw_msg_tokens[2] == 'PRIVMSG':
self.handle_channel_msg(raw_msg)
elif raw_msg_tokens[2] == 'WHISPER':
self.handle_whisper(raw_msg)
except Exception as e:
log_error('IRC message handler error', e)
def execute_command(self, display_name, full_command):
"""
Given a command, try to execute it if it matches any of the given patterns.
:param display_name: str - The user executing the command
:param full_command: str - The entire message
"""
split_command = full_command.split(maxsplit=1)
command = split_command[0].lower()
params = split_command[1] if len(split_command) == 2 else ''
try:
if command in self.exact_match_commands:
self.exact_match_commands[command](display_name)
if command in self.starts_with_commands:
self.starts_with_commands[command](display_name, params)
# Recursively run the commands of all children
for child in self.children:
child.execute_command(display_name, full_command)
except Exception as e:
log_error('Error executing command {}'.format(command), e)
def execute_command(self, display_name, full_command):
"""
Given a command, try to execute it if it matches any of the given patterns.
:param display_name: str - The user executing the command
:param full_command: str - The entire message
"""
split_command = full_command.split(maxsplit=1)
command = split_command[0].lower()
params = split_command[1] if len(split_command) == 2 else ''
try:
if command in self.exact_match_commands:
self.exact_match_commands[command](display_name)
if command in self.starts_with_commands:
self.starts_with_commands[command](display_name, params)
# Recursively run the commands of all children
for child in self.children:
child.execute_command(display_name, full_command)
except Exception as e:
log_error('Error executing command {}'.format(command), e)
def get_ranking():
"""
Get ranking
:return:
"""
connection = Connection().get_connection()
try:
cursor = connection.cursor() # prepare an object cursor
query = """
SELECT * FROM ranking ORDER BY position ASC;
"""
log_debug(query)
cursor.execute(query)
result = cursor.fetchall()
result = answer_ranking(result)
# cursor.close()
# del cursor
# connection.close()
return result
except MySQLError as ex:
connection.close()
log_error("Can't get ranking: {}".format(ex))
return []
def upload_file_thread(filename):
try:
database_name = upload_gtfs.add_gtfs_to_db(filename)
except Exception as e:
log_error(e)
return error(str(e))
bot.run()
except Exception as e2:
log_error('Bot crashed', e2)
# If the bot crashes for whatever reason, restart it
if settings.AUTO_RESTART_ON_CRASH:
time.sleep(5)
run_bot()
pause()
def clear_temp_files():
"""
Delete all files that don't need to persist across runs.
"""
for filename in settings.FILES_TO_CLEAR_ON_LOAD:
if os.path.isfile(filename):
os.remove(filename)
if __name__ == '__main__':
clear_temp_files()
try:
try_update()
except Exception as e:
log_error('Update failed', e)
log('Continuing execution as normal!')
settings.load_settings_file()
run_bot()