def _create_bucket(self, bucket_name):
try:
self.client.make_bucket(bucket_name)
except minio.error.BucketAlreadyOwnedByYou as err:
logger.warning(err)
except minio.error.ResponseError as err:
logger.error(err)
def get_country_and_continent_from_location(loc_string, user_name,
website_name):
"""
finds user location (country and continent) from his location description
because the api sometimes is unavailable (for several reasons) the function works in this form:
1. try first time - in a case of a failure, log a warning and try again (with sleep time for not immediately
try again (could lead to problem again)
2. if there is a second error, log an ERROR message with the user details for scraping his location individually
later (this feature will be completed in milestone 3)
:param loc_string: user location description (str)
:return: country and continent (str, str) or (None, None)
"""
country, continent = None, None # initiate the returned variables
if not re.search(
config.GMT_REGEX, loc_string
): # handle "GMT {-8:00}" - time zone location inputted
try:
country, continent = GeoLocation.geolocator_process(loc_string)
except GeocoderUnavailable:
logger.warning(
config.GeocoderUnavailable_WARNING_STRING.format(
user_name, website_name, loc_string))
time.sleep(config.SLEEP_TIME_FOR_LOCATIONS_API)
try:
country, continent = GeoLocation.geolocator_process(
loc_string)
except GeocoderUnavailable:
logger.error(
config.GeocoderUnavailable_ERROR_STRING.format(
user_name, website_name, loc_string))
return country, continent
def __init__(self, _id):
"""init method """
_funct = Functs["x2"]
d = {
"_id": _id,
"year": 0,
"funct": _funct,
"objective": _funct["objective"],
"interval": _funct["interval"],
"winning_threshold": 0.005,
"seed_parents": 12,
"kill_rate": 0.333,
"demography": 1,
"average_child_numb": 0.5,
"kill_before_reproduce": 0,
"social_system": "normal",
"current_population": [],
"original_population": [],
"_round": 7,
}
logger.warning(str(d))
super().__init__(**d)
def reputation_hist(self):
"""
user reputation for years: [2017, 2018, 2019 ,2020]
will evaluate years only for users registered before 2017
in case that users is registered more then 4 years ago and there is a problem,
log a warning for checking manually the issue
:return: reputation for each year (dict) of dict of None's
"""
if self._member_since < UserScraper.threshold_date:
soup_activity = Website.create_soup(self._url + "?tab=topactivity")
source_data = soup_activity.find("div", {
"id": "top-cards"
}).contents[3].string
numbers = re.search(config.REPUTATION_REGEX, source_data).group(1)
reputation_numbers = ast.literal_eval(numbers)
try:
return {
f"reputation_{config.REPUTATION_YEARS[i]}":
reputation_numbers[UserScraper.year_indexes[i]]
for i in range(len(config.REPUTATION_YEARS))
}
except IndexError:
logger.warning(
f"website {self._website_name} user {self._name}"
f" is member since more than 4 years but have reputation plot of month"
)
return {
f"reputation_{config.REPUTATION_YEARS[i]}": None
for i in range(len(config.REPUTATION_YEARS))
}
def create_default_config_file(self):
shutil.copy(default_file_path + "/default_config_file.json",
config_file_path)
message = "Config file '%s' created.\n" % config_file_path
message += "Please, set a valid iam role in the file field 'role' before the first execution."
logger.warning(message)
utils.finish_successful_execution()
def _load():
"""rebuild and return an Object from json data in redis"""
d = session["algo_dict"][session["current_algo"]]
logger.warning(d)
obj = _HandleAlgo(d)
return obj
def __init__(self, id):
"""init method """
d = {
"_id": id,
"year": 0,
"funct": Functs["x2"],
"objective": "min",
"interval": [-100, 100],
"winning_threshold": 0.005,
"seed_parents": 12,
"kill_rate": 0.33,
"demography": 1,
"dimension": "1D",
"level": "easy",
"average_child_numb": 0.5,
"kill_before_reproduce": 1,
"social_system": "normal",
"current_population": [],
"learning_curve": [],
"original_population": [],
"_round": 7,
"is_won": 0,
"won_year": -1,
"kill_number": -1,
"saved_people": -1,
"new_people_number": -1,
"new_people_composition": {},
}
logger.warning(str(d))
super().__init__(**d)
def get_action(self, board, temp=1e-3, return_prob=0):
sensible_moves = board.availables
# the pi vector returned by MCTS as in the alphaGo Zero paper
move_probs = np.zeros(board.width * board.height)
if len(sensible_moves) > 0:
actions, probs = self.mcts.get_move_probs(board, temp)
move_probs[actions] = probs
move = np.random.choice(actions, p=probs)
if self._is_selfplay:
# update the root node and reuse the search tree
self.mcts.update_with_move(move)
else:
# reset the root node
self.mcts.update_with_move(-1)
# location = board.move_to_location(move)
# print("AI move: %d,%d\n" % (location[0], location[1]))
if return_prob:
return move, move_probs
else:
return move
else:
logger.warning("[!] Game board is full!")
def geolocator_process(loc_string):
"""
user location will be determent by the geo-locator library. it receives the users location as written in the web
converts it to latitude and longitude then it will be called again to convert the latitude and longitude to
a global unique name of country and continent.
"""
country, continent = None, None # initiate the returned variables
loc = geolocator.geocode(loc_string)
if loc:
lat, lon = loc.latitude, loc.longitude
time.sleep(config.SLEEP_TIME_FOR_LOCATIONS_API)
try:
new_loc = geolocator.reverse([lat, lon], language='en')
country = new_loc.raw["address"]["country"]
continent = config.continents_dict[
country_alpha2_to_continent_code(
country_name_to_country_alpha2(country))]
except TypeError:
logger.warning(
config.USER_PROBLEMATIC_COUNTRY.format(loc_string))
except KeyError:
if country in config.KNOWN_COUNTRIES:
country, continent = config.KNOWN_COUNTRIES[country]
finally:
time.sleep(config.SLEEP_TIME_FOR_LOCATIONS_API)
return country, continent
def delete_log_group(self, log_group_name):
try:
# Delete the cloudwatch log group
return self.get_client().delete_log_group(logGroupName=log_group_name)
except ClientError as ce:
if ce.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Cannot delete log group '%s'. Group not found." % log_group_name)
else:
logger.error("Error deleting the cloudwatch log",
"Error deleting the cloudwatch log: %s" % ce)
def on_epoch_end(self, epoch, logs=None):
with self.file_writer.as_default():
for key, value in logs.items():
try:
tensorflow.summary.scalar(key,
value,
step=epoch * self.multiplier)
except Exception as e:
logger.warning(str(e))
self.file_writer.flush()
def subscriber_handler(data, context, board):
if "data" in data:
results_json = base64.b64decode(data["data"]).decode("utf-8")
logger.warning("results data: {}".format(results_json))
# results_json = data["data"].decode("utf-8")
urls = json.loads(results_json)
s = consume_urls_parallel(urls)
board = board.lower()
insert_to_gcs(board, s)
def wrapper(*args, **kwargs):
for attempt in range(MAX_AUTO_RECONNECT_ATTEMPTS):
try:
return mongo_op_func(*args, **kwargs)
except pymongo.errors.AutoReconnect as e:
wait_t = 0.5 * pow(2, attempt) # exponential back off
logger.warning(
"PyMongo auto-reconnecting... %s. Waiting %.1f seconds.",
str(e), wait_t)
time.sleep(wait_t)
def get_bucket_files(self, bucket_name, prefix_key):
file_list = []
if self.client.find_bucket(bucket_name):
if prefix_key is None:
prefix_key = ''
result = self.client.list_files(bucket_name, key=prefix_key)
if 'Contents' in result:
for info in result['Contents']:
file_list += [info['Key']]
else:
logger.warning("Bucket '{0}' not found".format(bucket_name))
return file_list
def create_log_group(self, log_group_name, tags):
try:
logger.debug("Creating cloudwatch log group.")
return self.get_client().create_log_group(logGroupName=log_group_name, tags=tags)
except ClientError as ce:
if ce.response['Error']['Code'] == 'ResourceAlreadyExistsException':
logger.warning("Using existent log group '%s'" % log_group_name)
pass
else:
logger.error("Error creating log groups.",
"Error creating log groups: %s" % ce)
utils.finish_failed_execution()
def run(self):
times = deque(maxlen=10)
schedule = lr_schedule()
mean_iter_time = 0
# run the training pipeline
try:
for i in range(self.game_batch_num):
start = time()
self.collect_selfplay_data(self.play_batch_size)
mcts_time = time() - start
logger.info("iter: {}, episode_len:{}, mcts time: {:.2f}, mean time: {:.2f}".format(i + 1,
self.episode_len,
mcts_time,
mean_iter_time))
if len(self.states_buffer) > self.batch_size:
loss, entropy = self.policy_update(learning_rate=schedule(i))
times.append(time() - start)
mean_iter_time = sum(times) / len(times)
if (i + 1) % self.save_freq == 0 and not self.debug:
self.policy_value_net.save_model(os.path.join(self.save_dir, 'policy_{}.model'.format(i + 1)))
# check the performance of the current model,
if (i+1) % self.check_freq == 0:
logger.info("current self-play batch: {}, evaluating...".format(i+1))
win_ratio = self.policy_evaluate()
if win_ratio > self.best_win_ratio:
logger.info("Found new best policy, saving")
self.best_win_ratio = win_ratio
# update the best_policy
self.policy_value_net.save_model(os.path.join(self.save_dir, 'best_policy.model'))
if (self.best_win_ratio == 1.0 and
self.pure_mcts_playout_num < 5000):
self.pure_mcts_playout_num += 1000
self.best_win_ratio = 0.0
except KeyboardInterrupt:
logger.warning('Got keyboard interrupt, saving and quiting')
try:
if not self.debug:
self.policy_value_net.save_model(os.path.join(self.save_dir, 'current_policy.model'))
except:
logger.error("[!] Error while saving policy net on keyboard interrupt, quiting")
def insert(self, collection_name, documents):
try:
collection = self.mongo_db[collection_name]
if isinstance(documents, list):
_id = collection.insert_many(documents,
ordered=False).inserted_ids
else:
_id = collection.insert_one(documents).inserted_id
return _id
except pymongo.errors.BulkWriteError:
logger.warning('Duplicated records.')
except Exception as e:
print('Exception while inserting to MongoDB', e)
logger.debug(
'Exception while inserting to MongoDB as exception: {}'.format(
e))
def folder_exists(self, folder_name):
folder_name = '{0}/'.format(folder_name.strip('/ '))
url = 'https://{0}{1}{2}?children'.format(self.oneprovider_host,
self.cdmi_path,
self.onedata_space)
headers = {**self.cdmi_version_header, **self.onedata_auth_header}
try:
r = requests.get(url, headers=headers)
if r.status_code == 200:
if folder_name in r.json()['children']:
return True
except Exception as e:
logger.warning(
f'Cannot check if folder "{folder_name}" exists. Error: {e}')
return False
return False
def delete_folder(self, folder_name):
url = 'https://{0}{1}{2}/{3}/'.format(self.oneprovider_host,
self.cdmi_path,
self.onedata_space, folder_name)
headers = {**self.cdmi_version_header, **self.onedata_auth_header}
try:
r = requests.delete(url, headers=headers)
if r.status_code == 204:
logger.info(
f'Folder "{folder_name}" deleted successfully in space "{self.onedata_space}"'
)
else:
raise Exception(r.status_code)
except Exception as e:
logger.warning(
f'Unable to delete folder "{folder_name}". Error: {e}')
def time_interval_limit(self, event_times, event_list, seconds_limit, tag):
unqualified_time = 0
unqualified_event_list = []
if event_times > 1:
for index in range(0, event_times - 1):
job1_occure_time = event_list[index].occur_seconds
job2_occure_time = event_list[index + 1].occur_seconds
if (job2_occure_time - job1_occure_time) < seconds_limit:
unqualified_time += 1
logger.warning("（不合格）", tag, ", 两次事件 dump分别为：\n",
event_list[index].start_dump_str, "\n",
event_list[index + 1].start_dump_str)
self.unqualified_event_str = self.unqualified_event_str + "（不合格）" + tag + " 两次事件 dump分别为：\n" + event_list[
index].start_dump_str, "\n" + event_list[
index + 1].start_dump_str
unqualified_event_list.append(
event_list[index].start_dump_str)
unqualified_event_list.append(event_list[index +
1].start_dump_str)
def get_deployment_envvars(self, name, namespace):
deployments_path = self.deployments_path.format(namespace)
url = 'https://{0}:{1}{2}/{3}'.format(self.kubernetes_service_host,
self.kubernetes_service_port,
deployments_path,
name)
try:
r = requests.get(url,
verify=self.cert_verify,
headers=self.auth_header)
if r.status_code != 200:
raise Exception(f'Error reading deployment {name} - {str(r.status_code)}\n{str(r.content)}')
deploy = r.json()
if len(deploy['spec']['template']['spec']['containers']) > 1:
logger.warning('The function have more than one container. Getting environment variables from container 0')
container_info = deploy['spec']['template']['spec']['containers'][0]
envvars = container_info['env'] if 'env' in container_info else []
return envvars
except Exception as e:
logger.error(e)
return []