def handle_event(event: dict, channel: str, channel_id: str, message: str,
sc: SlackClient, logger: logging.Logger) -> None:
pretty_event = pformat(event)
logger.debug(f"Event received:\n{pretty_event}")
subtype = event.get('subtype')
user = event.get('user')
if subtype in ('group_join', 'channel_join') and user:
# We will use the event's channel ID to send a response and refer to
# users by their display_name in accordance with new guidelines.
# https://api.slack.com/changelog/2017-09-the-one-about-usernames
event_channel_id = event.get('channel')
user_profile = event.get('user_profile')
username = user_profile.get('display_name')
user_mention = f"<@{user}>"
message = message.replace('{user}', user_mention)
if event_channel_id == channel_id:
try:
sc.rtm_send_message(event_channel_id, message)
logger.info(f"Welcomed {username} to #{channel}")
except AttributeError:
logger.error(f"Couldn't send message to #{channel}")
def execute(self, compile_base_path: str, timeout: Optional[int], logger: Logger):
detector_invocation = ["java"] + self.detector.java_options + ["-jar", _quote(self.detector.jar_path)]
command = detector_invocation + self._get_detector_arguments(self.version.get_compile(compile_base_path))
command = " ".join(command)
start = time.time()
try:
Shell.exec(command, logger=logger, timeout=timeout)
self.result = Result.success
except CommandFailedError as e:
logger.error("Detector failed: %s", e)
self.result = Result.error
message = str(e)
message_lines = str.splitlines(message)
if len(message_lines) > 5000:
self.message = "\n".join(message_lines[0:500]) + "\n" + "\n".join(message_lines[-4500:])
else:
self.message = message
except TimeoutError:
logger.error("Detector took longer than the maximum of %s seconds", timeout)
self.result = Result.timeout
finally:
end = time.time()
runtime = end - start
self.runtime = runtime
logger.info("Run took {0:.2f} seconds.".format(runtime))
self.save()
def spinner(text: str, logger: Logger, quiet=False, debug=False):
'''Decoration for long running processes.
:param text: Message to output
:param logger: Logger to capture the error if it occurs
:param quiet: If ``True``, messages will be hidden
:param debug: If ``True``, show full tracebacks
'''
# pylint: disable=broad-except
try:
logger.info(text)
if not quiet:
print(text)
yield
if not quiet:
print('Done\n')
except Exception as exception:
exception_traceback = format_exc()
logger.error(exception_traceback)
if not quiet:
if debug:
print(exception_traceback)
else:
print(str(exception))
def is_satisfied(requirement: Requirement, logger: logging.Logger) -> bool:
try:
requirement.check()
logger.debug("Requirement '%s' satisfied", requirement.description)
return True
except Exception as e:
logger.error("Requirement '%s' not satisfied: %s", requirement.description, e)
return False
def error(
cls,
msg: str,
logger: logging.Logger = _default_logger
) -> "SOEFException": # pragma: no cover
"""Construct exception and write log."""
logger.error(msg)
return cls(msg)
def solution(infile: str, log: logging.Logger) -> None:
try:
with open(infile, "r") as input:
input_lines = input.read().splitlines()
except (IOError, FileNotFoundError) as exception:
log.error(f"Error opening input file: {exception}")
else:
print(f"Number of lines in this file are: {len(input_lines)}")
def solution_a(infile: str, log: logging.Logger) -> None:
try:
with open(infile, "r") as in_file:
lines = in_file.readlines()
except (IOError, FileNotFoundError) as exception:
log.error(f"Error opening file: {exception}")
else:
print(f"Last line in file {infile} is {lines[-1]}")
def are_int(num1: int, num2: int, logger: Logger) -> bool:
"""Check whether the numbers are both integers."""
if _is_int(num1) and _is_int(num2):
logger.debug("both args are integer")
return True
else:
logger.error("both args must be integers")
return False
def sample_logging(logger: logging.Logger):
"""Sample function on how logging works"""
# Add messages now!
logger.debug("A debug message")
logger.info("An info message")
logger.warning("Something is not right.")
logger.error("A Major error has happened.")
logger.critical("Fatal error. Cannot continue")
def parse_sv_residuals(dfSat: pd.DataFrame, logger: logging.Logger) -> dict:
"""
parse_sv_residuals parses the observed resiudals of the satellites
"""
cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')
logger.info('{func:s}: parses observed resiudals of satellites'.format(func=cFuncName))
# determine the list of satellites observed
obsSVs = np.sort(dfSat.SV.unique())
logger.info('{func:s}: observed SVs (#{nrsats:02d}):\n{sats!s}'.format(func=cFuncName, nrsats=len(obsSVs), sats=obsSVs))
# determine statistics for each SV
dSVList = {}
dSVList['#total'] = len(obsSVs)
nrGAL = 0
nrGPS = 0
GALList = []
GPSList = []
dGALsv = {}
dGPSsv = {}
for i, sv in enumerate(obsSVs):
# do some statistics on this sv
dSV = {}
dSV['count'] = int(dfSat.PRres[dfSat['SV'] == sv].count())
dSV['PRmean'] = dfSat.PRres[dfSat['SV'] == sv].mean()
dSV['PRmedian'] = dfSat.PRres[dfSat['SV'] == sv].median()
dSV['PRstd'] = dfSat.PRres[dfSat['SV'] == sv].std()
s = dfSat.PRres[dfSat['SV'] == sv].between(-2, +2, inclusive=True)
dSV['PRlt2'] = int(s.sum())
if dSV['count']:
dSV['PRlt2%'] = dSV['PRlt2'] / dSV['count'] * 100
else:
dSV['PRlt2%'] = 0
if sv.startswith('E'):
nrGAL += 1
GALList.append(sv)
dGALsv[sv] = dSV
elif sv.startswith('G'):
nrGPS += 1
GPSList.append(sv)
dGPSsv[sv] = dSV
else:
logger.error('{func:s}: erroneous satellite {sv:s} found'.format(func=cFuncName, sv=colored(sv, 'red')))
logger.info(' {sv:s}: #Obs = {obs:6d} PRres = {prmean:+6.3f} +- {prstd:6.3f}, {prlt2p:6.2f} (#{prlt2:5d}) within [-2, +2]'.format(sv=sv, obs=dSV['count'], prmean=dSV['PRmean'], prstd=dSV['PRstd'], prlt2p=dSV['PRlt2%'], prlt2=dSV['PRlt2']))
dSVList['#GPS'] = nrGPS
dSVList['#GAL'] = nrGAL
dSVList['GALList'] = GALList
dSVList['GPSList'] = GPSList
dSVList['GALSVs'] = dGALsv
dSVList['GPSSVs'] = dGPSsv
return dSVList
def download_url(
url: str, dest: str, logger: Logger,
show_progress: bool = SHOW_PROGRESS,
chunk_size: int= 1024 * 1024,
timeout: int= 5, pbar: Callable = None,
retries=5
) -> Optional[str]:
"""
Function to download the object from given url
(Inspired from FastAI)
:param url: The url to download the file from
:param logger: The Logger Object to use for Logging
:param chunk_size: chunk_size to read from url
:param timeout: Timeout for URL read
:param retries: Number of Retries for URL read
:param pbar: Parent Progress Bar to use
:param dest: The destination to download the url
:param show_progress: Whether to show the progress of downloading url or not
:return: Returns the destination of the downloaded file if successful
"""
s = requests.Session()
s.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
# additional line to identify as a firefox browser, see fastai/#2438
s.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:71.0) Gecko/20100101 Firefox/71.0'})
u = s.get(url, stream=True, timeout=timeout)
try:
file_size = int(u.headers["Content-Length"])
except Exception as e:
show_progress = False
try:
logger.warning(f"Writing downloaded data to {dest}")
with open(dest, 'wb') as f:
nbytes = 0
if show_progress:
pbar = progress_bar(range(file_size), leave=False, parent=pbar)
pbar.update(0)
for chunk in u.iter_content(chunk_size=chunk_size):
nbytes += len(chunk)
if show_progress:
pbar.update(nbytes)
f.write(chunk)
if show_progress:
pbar.comment("\n")
logger.info("Download is Complete.")
return dest
except requests.exceptions.ConnectionError as e:
filename, data_dir = os.path.split(dest)
logger.warning(f"\n Download of {url} has failed after {retries} retries\n"
f" Fix the download manually:\n"
f"$ mkdir -p {data_dir}\n"
f"$ cd {data_dir}\n"
f"$ wget -c {url}\n"
f"$ tar xf {filename}\n"
f" And re-run your code once the download is successful\n")
logger.error(e)
raise e
def report(
self,
analyze_pass=True,
analyze_fail=False,
analyze_all=False,
show_errors=True,
logger: Logger = default_log,
queue: Queue = None,
):
self.analyze(analyze_pass, analyze_fail, analyze_all, queue)
errors_detected = False
logger.info("REPORT STATISTICS")
logger.info("")
for name, log in self.log.items():
if log["fails"]:
printlog = logger.error
errors_detected = True
else:
printlog = logger.info
space = " " * (30 - len(name) if len(name) < 35 else 0)
analyze_msg = [f'{name}:{space}']
if analyze_pass:
analyze_msg.append(
f' max: {round(log["max"], 3) if log["max"] else None}s'
f' min: {round(log["min"], 3) if log["min"] else None}s'
f' avg: {round(log["avg"], 3) if log["avg"] else None}s'
f' med: {round(log["med"], 3) if log["med"] else None}s')
if analyze_fail:
analyze_msg.append(
f' max-err: {round(log["max-err"], 3) if log["max-err"] else None}s'
f' min-err: {round(log["min-err"], 3) if log["min-err"] else None}s'
f' avg-err: {round(log["avg-err"], 3) if log["avg-err"] else None}s'
f' med-err: {round(log["med-err"], 3) if log["med-err"] else None}s'
)
if analyze_all:
analyze_msg.append(
f' max-all: {round(max(log["all"]), 3)}s'
f' min-all: {round(min(log["all"]), 3)}s'
f' avg-all: {round(sum(log["all"]) / len(log["all"]), 3)}s'
f' med-all: {round(median(log["all"]), 3)}s')
analyze_msg.append(f' ok: {log["ok"]}'
f' total: {log["total"]}'
f' err: {log["err"]}')
printlog(''.join(analyze_msg))
if errors_detected and show_errors:
logger.info("")
logger.info("REPORT ERRORS")
logger.info("")
for name, log in self.log.items():
if log["fail"]:
logger.error(
f'{Fore.RED}{name} with {len(log["fail"])} errors:\n')
for e, n in log["fail"].items():
logger.error(
f'{Fore.RED}{e}, this error occurred {len(n)} times\n'
)
return self
def download_from_google_drive(shareable_url: str,
file_name: str,
log: logging.Logger,
download_path: str = downloads) -> Tuple:
"""Downloads file from the shareable url.
Downloads file from shareable url and saves it in downloads folder.
Args:
shareable_url: Url of the file.
file_name: Filename for the downloaded file.
log: Logger object for logging the status.
download_path: Path (default: ./downloads/) for saving file.
Returns:
Boolean value if the file is downloaded or not.
Raises:
ResponseError: If any unexpected response/error occurs.
ResponseNotChunked: If the response is not sending correct `chunks`.
Notes:
This function is capable of downloading files from Google Drive iff
these files are shareable using 'Anyone with the link' link sharing
option.
"""
# You can find the reference code here:
# https://stackoverflow.com/a/39225272
try:
file_id = shareable_url.split('https://drive.google.com/open?id=')[1]
session = requests.Session()
response = session.get(dev.DRIVE_DOWNLOAD_URL,
params={'id': file_id},
stream=True)
token = fetch_confirm_token(response)
if token:
response = session.get(dev.DRIVE_DOWNLOAD_URL,
params={
'id': file_id,
'confirm': token
},
stream=True)
# Write file to the disk.
with open(os.path.join(download_path, f'{file_name}.mp4'),
'wb') as file:
for chunk in response.iter_content(dev.CHUNK_SIZE):
if chunk:
file.write(chunk)
log.info(f'File "{file_name}.mp4" downloaded from Google Drive.')
if fz(os.path.join(download_path, f'{file_name}.mp4')).endswith('KB'):
log.error('Unusable file downloaded since file size is in KBs.')
return None, '[w] Unusable file downloaded.'
return True, os.path.join(download_path, f'{file_name}.mp4')
except (RequestError, RequestException):
log.error(
'File download from Google Drive failed because of poor network '
'connectivity.')
return None, '[e] Error while downloading file'
def add_sensor_alert(self,
node_id: int,
sensor_id: int,
state: int,
optional_data: Optional[Dict[str, Any]],
change_state: bool,
has_latest_data: bool,
data_type: int,
sensor_data: Any,
logger: logging.Logger = None) -> bool:
"""
Adds Sensor Alert to processing queue.
:param node_id:
:param sensor_id:
:param state:
:param optional_data:
:param change_state:
:param has_latest_data:
:param data_type:
:param sensor_data:
:param logger:
:return: Success or Failure
"""
if logger is None:
logger = self._logger
sensor_alert = SensorAlert()
sensor_alert.sensorId = sensor_id
sensor_alert.nodeId = node_id
sensor_alert.timeReceived = int(time.time())
sensor_alert.state = state
sensor_alert.changeState = change_state
sensor_alert.hasLatestData = has_latest_data
sensor_alert.dataType = data_type
sensor_alert.sensorData = sensor_data
sensor_alert.hasOptionalData = False
sensor_alert.optionalData = optional_data
if optional_data:
sensor_alert.hasOptionalData = True
sensor = self._storage.getSensorById(sensor_id, logger)
if sensor is None:
logger.error("[%s]: Not able to get sensor %d from database." % (self._log_tag, sensor_id))
return False
sensor_alert.description = sensor.description
sensor_alert.alertDelay = sensor.alertDelay
sensor_alert.alertLevels = sensor.alertLevels
with self._sensor_alert_queue_lock:
self._sensor_alert_queue.append(sensor_alert)
self._sensor_alert_event.set()
return True
def start_daemon(judge_logger: logging.Logger):
"""
Start a daemon process which is running the .
:param judge_config:
:param judge_logger:
:return: None
"""
pid_file_path = os.path.join(os.getcwd(), judge_config.RUN['pid_file'])
# pid = os.fork()
# if pid > 0:
# sys.exit(0)
#
# os.chdir('/')
# os.setsid()
# os.umask(0)
#
# pid = os.fork()
# if pid > 0:
# sys.exit(0)
if os.path.exists(pid_file_path):
print('Judged daemon has being running.')
judge_logger.error('Judged daemon has being running.')
exit(0)
try:
(_path, _) = os.path.split(pid_file_path)
if not os.path.exists(_path):
os.mkdir(_path)
pid_file = open(pid_file_path, mode='w+')
print('Judge daemon(pid=%d) start successfully.' % os.getpid())
judge_logger.info('Judge daemon(pid=%d) start successfully.' % os.getpid())
pid_file.write('%d' % os.getpid())
pid_file.close()
except Exception as e:
print(e)
#redirect stdio
sys.stdout.flush()
sys.stderr.flush()
si = open(os.devnull, 'r')
# so = open(os.devnull, 'a+')
# se = open(os.devnull, 'a+')
os.dup2(si.fileno(), sys.stdin.fileno())
# os.dup2(so.fileno(), sys.stdout.fileno())
# os.dup2(se.fileno(), sys.stderr.fileno())
# signal.signal(signal.SIGKILL, exit_clean)
main_loop(judge_logger)
try:
os.remove(pid_file_path)
except Exception as e:
judge_logger.error(e)
exit(0)
def _ensure_dir_exist(path: str, logger: logging.Logger):
try:
os.makedirs(path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
logger.error(e)
sys.exit(1)
def validate_iteration(*, operation, iteration_idx, iteration_data,
logger: logging.Logger):
if 'iterations' in operation['mapping']:
for prop, meta_info in operation['mapping']['iterations'].items():
if meta_info.get('required') and prop not in iteration_data:
logger.error(
f'Iteration #{iteration_idx + 1} does not contain required property "{prop}"'
)
return True
def unhash_country_code(hashed_code: str,
log: logging.Logger = None) -> Optional[str]:
"""Return unhashed country code."""
log = _log(__file__) if log is None else log
try:
return dict(map(reversed, h_country.items()))[hashed_code]
except (KeyError, ValueError):
log.error('KeyError or ValueError was raised.')
return None
def solution(dir: str, log: logging.Logger) -> None:
try:
for dir_path, dir_names, file_names in os.walk(dir):
for filename in file_names:
filepath = os.path.join(dir_path, filename)
filesize = os.path.getsize(filepath)
print(f"{filepath}: {filesize} B")
except OSError as exception:
log.error(f"Error checking directory contents: {exception}")
def run_expertise(self, config: dict, logger: logging.Logger):
try:
update_status(config, JobStatus.RUN_EXPERTISE)
execute_expertise(config=config.to_json())
update_status(config, JobStatus.COMPLETED)
except Exception as exc:
# Write error, clean up working directory and store log
logger.error(f"Error in job: {config.job_id}, {str(exc)}")
update_status(config, JobStatus.ERROR, str(exc))
def run_coalescer(cfg: dict,
tables: List[str],
period: str,
run_once: bool,
logger: Logger,
no_sqpoller: bool = False) -> None:
"""Run the coalescer.
Runs it once and returns or periodically depending on the
value of run_once. It also writes out the coalescer records
as a parquet file.
:param cfg: dict, the Suzieq config file read in
:param tables: List[str], list of table names to coalesce
:param period: str, the string of how periodically the poller runs,
Examples are '1h', '1d' etc.
:param run_once: bool, True if you want the poller to run just once
:param logger: logging.Logger, the logger to write logs to
:param no_sqpoller: bool, write records even when there's no sqpoller rec
:returns: Nothing
:rtype: none
"""
try:
schemas = Schema(cfg['schema-directory'])
except Exception as ex:
logger.error(f'Aborting. Unable to load schema: {str(ex)}')
print(f'ERROR: Aborting. Unable to load schema: {str(ex)}')
sys.exit(1)
coalescer_schema = SchemaForTable('sqCoalescer', schemas)
pqdb = get_sqdb_engine(cfg, 'sqCoalescer', None, logger)
if not run_once:
now = datetime.now()
nextrun = parse(period, settings={'PREFER_DATES_FROM': 'future'})
sleep_time = (nextrun - now).seconds
logger.info(f'Got sleep time of {sleep_time} secs')
while True:
try:
stats = do_coalesce(cfg, tables, period, logger, no_sqpoller)
except Exception:
logger.exception('Coalescer aborted. Continuing')
# Write the selftats
df = pd.DataFrame([asdict(x) for x in stats])
if not df.empty:
df['sqvers'] = coalescer_schema.version
df['version'] = SUZIEQ_VERSION
df['active'] = True
df['namespace'] = ''
pqdb.write('sqCoalescer', 'pandas', df, True,
coalescer_schema.get_arrow_schema(), None)
if run_once:
break
sleep(sleep_time)
def log_exception(logger: Logger, exception: BaseException):
"""
Log an exception to the provided logger. Special handling rules for DDNSWolf custom
exceptions will be applied.
"""
if isinstance(exception, DDNSWolfUserException):
logger.error(str(exception))
else:
logger.error(str(exception), exc_info=exception)
def validate_aqi_sample_df(df: DataFrame, log: Logger = None) -> DataFrame:
"""Validates sampled AQI values. Prints error if invalid values are found
and returns the dataframe where invalid AQI values are replaced with np.nan.
"""
if not validate_aqi_samples(list(df['aqi']), log):
if log: log.error('AQI sampling failed')
df['aqi'] = [get_valid_aqi_or_nan(aqi) for aqi in df['aqi']]
return df
def check_for_cycles(graph: nx.DiGraph, logger: Logger):
valid, cycles = detect_cycles(graph)
if not valid:
logger.error("Views dependency graph is not a DAG. Cycles detected:")
for cycle in cycles:
logger.error(sorted(cycle))
raise NotADAGError(
f"Graph in is not a DAG. Number of cycles: {len(cycles)}. "
f"First cycle: {cycles[0]}")
def stored(json_data: dict, log: logging.Logger):
try:
_status, _file = None, None
scheduled = json_data.get('schedule_download', False)
if scheduled:
scheduled_time = f'{json_data["start_date"]} {json_data["start_time"]}:00'
sleep_interval = datetime_to_utc(scheduled_time,
json_data["camera_timezone"],
'%Y-%m-%d %H:%M:%S')
sleep_interval = datetime.strptime(sleep_interval,
'%Y-%m-%d %H:%M:%S')
sleep_interval -= now()
if sleep_interval.seconds <= 0:
log.error('Scheduled time has passed already.')
return None
log.info(
'Video is scheduled for downloading, the process will suspend '
f'for {seconds_to_datetime(int(sleep_interval.seconds))}.')
time.sleep(1.0 + sleep_interval.seconds)
log.info('Initiating video download...')
if json_data.get('access_type', None) == 'GCP':
log.info('Downloading file via Google Drive...')
_status, _file = download_from_google_drive(
json_data['g_url'], json_data['stored_filename'], log)
elif json_data.get('access_type', None) == 'Microsoft':
log.info('Downloading file via Microsoft Azure...')
_status, _file = download_from_azure(
json_data['azure_account_name'],
json_data['azure_account_key'],
json_data['azure_container_name'],
json_data['azure_blob_name'], json_data['stored_filename'],
log)
elif json_data.get('access_type', None) == 'FTP':
log.info('Downloading file via FTP...')
if json_data.get("earthcam_download", False):
log.info('Downloading EarthCam file(s)...')
_status, _file = earthcam_specific_download(
json_data['p_name'], json_data['p_pass'],
json_data['p_ip'], json_data['point_access'],
json_data['earthcam_start_date'],
json_data['earthcam_start_time'],
json_data['earthcam_end_time'], log,
json_data['stored_filename'])
else:
_status, _file = batch_download_from_ftp(
json_data['p_name'], json_data['p_pass'],
json_data['p_ip'], json_data['point_access'], log)
elif json_data.get('access_type', None) == 'S3':
log.info('Downloading file via Amazon S3 storage...')
_status, _file = access_file(json_data['s3_access_key'],
json_data['s3_secret_key'],
json_data['s3_url'],
json_data['stored_filename'], log,
json_data['s3_bucket_name'])
return _status, _file
except Exception as error:
log.exception(error)
def _log_subprocess_err(self, out: Any, logger: logging.Logger) -> None:
"""Logger err message from subprocess
Args:
out (Any): stdout type
logger (Type): Simulator Logger running currently
"""
for line in iter(out.readline, b''):
logger.error(line.decode())
def run(self, name: str, logger: Logger) -> Any:
try:
loader = self.__loader__(self.save_folder, name)
logger.debug(f"[Input] read input file. {self.__name__}: {name}")
res = loader.load(*self.extra_args)
except Exception as e:
logger.error(f"[Exception] input: {self.__name__}, case: {name}")
raise e
return res
def exit_not_json(r: Response, logger: logging.Logger) -> None:
t = r.headers.get("content-type", "")
if not t.startswith("application/json"):
logger.error(
("Expected content-type 'application/json' " "actual '{}'").format(
t
)
)
sys.exit(EXIT_CODE_ZMF_NOK)
def log_model_grads(named_params: TorchNamedParams, logger: Logger) -> None:
"""
records the gradient values of the parameters in the model
Arguments:
named_params - Generator[str, torch.nn.parameter.Parameter]: output of model.named_parameters()
"""
for name, params in named_params:
if params.requires_grad:
logger.error(f"log_model_grads: {name}: {params.grad}")
def trinity_boot(args: Namespace, trinity_config: TrinityConfig,
extra_kwargs: Dict[str, Any], plugin_manager: PluginManager,
listener: logging.handlers.QueueListener, event_bus: EventBus,
main_endpoint: Endpoint, logger: logging.Logger) -> None:
# start the listener thread to handle logs produced by other processes in
# the local logger.
listener.start()
event_bus.start()
# First initialize the database process.
database_server_process = ctx.Process(
name="DB",
target=run_database_process,
args=(
trinity_config,
LevelDB,
),
kwargs=extra_kwargs,
)
# start the processes
database_server_process.start()
logger.info("Started DB server process (pid=%d)",
database_server_process.pid)
# networking process needs the IPC socket file provided by the database process
try:
wait_for_ipc(trinity_config.database_ipc_path)
except TimeoutError as e:
logger.error("Timeout waiting for database to start. Exiting...")
kill_process_gracefully(database_server_process, logger)
ArgumentParser().error(message="Timed out waiting for database start")
def kill_trinity_with_reason(reason: str) -> None:
kill_trinity_gracefully(logger, (database_server_process, ),
plugin_manager,
main_endpoint,
event_bus,
reason=reason)
main_endpoint.subscribe(ShutdownRequest,
lambda ev: kill_trinity_with_reason(ev.reason))
plugin_manager.prepare(args, trinity_config, extra_kwargs)
kill_trinity_with_reason("No beacon support yet. SOON!")
try:
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGTERM,
lambda: kill_trinity_with_reason("SIGTERM"))
loop.run_forever()
loop.close()
except KeyboardInterrupt:
kill_trinity_with_reason("CTRL+C / Keyboard Interrupt")
def broadcast(
logger: Logger,
dbsession: Session,
network: str,
ethereum_node_url: Union[str, Web3],
ethereum_private_key: str,
ethereum_gas_limit: Optional[str],
ethereum_gas_price: Optional[str],
commit=True,
):
"""Issue out a new Ethereum token."""
check_good_private_key(ethereum_private_key)
web3 = create_web3(ethereum_node_url)
service = EthereumStoredTXService(network, dbsession, web3,
ethereum_private_key, ethereum_gas_price,
ethereum_gas_limit, BroadcastAccount,
PreparedTransaction)
service.ensure_accounts_in_sync()
pending_broadcasts = service.get_pending_broadcasts()
logger.info("Pending %d transactions for broadcasting in network %s",
pending_broadcasts.count(), network)
if pending_broadcasts.count() == 0:
logger.info(
"No new transactions to broadcast. Use sto tx-update command to see tx status."
)
return []
account = Account.privateKeyToAccount(ethereum_private_key)
balance = web3.eth.getBalance(account.address)
logger.info("Our address %s has ETH balance of %f for operations",
account.address, from_wei(balance, "ether"))
txs = list(pending_broadcasts)
# https://stackoverflow.com/questions/41985993/tqdm-show-progress-for-a-generator-i-know-the-length-of
for tx in tqdm(txs, total=pending_broadcasts.count()):
try:
service.broadcast(tx)
# logger.info("Broadcasted %s", tx.txid)
except Exception as e:
logger.exception(e)
logger.error("Failed to broadcast transaction %s: %s", tx.txid,
tx.human_readable_description)
raise e
if commit:
dbsession.commit() # Try to minimise file system sync issues
return txs
def addQueueLogHandler(tracer: logging.Logger, ctx) -> None:
# Provide access to custom (payload-specific) fields
oldFactory = logging.getLogRecordFactory()
def recordFactory(name,
level,
pathname,
lineno,
msg,
args,
exc_info,
func=None,
sinfo=None,
**kwargs):
record = oldFactory(name,
level,
pathname,
lineno,
msg,
args,
exc_info,
func=func,
sinfo=sinfo,
kwargs=kwargs)
record.sapmonid = ctx.sapmonId
record.payloadversion = PAYLOAD_VERSION
return record
tracer.info("adding storage queue log handler")
try:
queueName = STORAGE_QUEUE_NAMING_CONVENTION % ctx.sapmonId
storageAccount = AzureStorageAccount(
tracer, ctx.sapmonId, ctx.msiClientId,
ctx.vmInstance["subscriptionId"],
ctx.vmInstance["resourceGroupName"])
storageKey = tracing.getAccessKeys(tracer, ctx)
queueStorageLogHandler = QueueStorageHandler(
account_name=storageAccount.accountName,
account_key=storageKey,
protocol="https",
queue=queueName)
queueStorageLogHandler.level = DEFAULT_QUEUE_TRACE_LEVEL
jsonFormatter = JsonFormatter(
tracing.config["formatters"]["json"]["fieldMapping"])
queueStorageLogHandler.setFormatter(jsonFormatter)
logging.setLogRecordFactory(recordFactory)
except Exception as e:
tracer.error(
"could not add handler for the storage queue logging (%s) " %
e)
return
queueStorageLogHandler.level = DEFAULT_QUEUE_TRACE_LEVEL
tracer.addHandler(queueStorageLogHandler)
return
def install_knime_pkgs(kpath: str,
kver: str,
logger: Logger = getLogger(__name__)) -> int:
"""
Install KNIME packages needed to execute RetroPath2.0 workflow.
Parameters
----------
kpath : str
Path that contains KNIME executable.
kver : str
Version of KNIME installed.
logger : Logger
The logger object.
Returns
-------
int Return code.
"""
StreamHandler.terminator = ""
logger.info(' |- Checking KNIME packages...')
logger.debug(' + kpath: ' + kpath)
logger.debug(' + kver: ' + kver)
args = \
' -application org.eclipse.equinox.p2.director' \
+ ' -nosplash -consolelog' \
+ ' -r http://update.knime.org/community-contributions/trunk,' \
+ 'http://update.knime.com/community-contributions/trusted/'+kver[:3]+',' \
+ 'http://update.knime.com/analytics-platform/'+kver[:3] \
+ ' -i org.knime.features.chem.types.feature.group,' \
+ 'org.knime.features.datageneration.feature.group,' \
+ 'org.knime.features.python.feature.group,' \
+ 'org.rdkit.knime.feature.feature.group' \
+ ' -bundlepool ' + kpath + ' -d ' + kpath
if ' ' in kpath:
cmd = '"'+os_path.join(kpath, 'knime')+'"' \
+ args
else:
cmd = os_path.join(kpath, 'knime') \
+ args
try:
printout = open(devnull, 'wb') if logger.level > 10 else None
CPE = run(cmd.split(), stdout=printout, stderr=printout,
shell=False) # nosec
logger.debug(CPE)
StreamHandler.terminator = "\n"
logger.info(' OK')
return CPE.returncode
except OSError as e:
logger.error(e)
return -1
def write_validation_result_to_log(self, log: logging.Logger):
"""Writes a the validators response to log provided.
Default behavior is to just write str representation
to log.
Args:
log (logging.Logger): A python log
table_resp (dict): A dictionary that will be written as
"""
log.error(str(self.response), extra={"context": "VALIDATION"})
async def send_dumplings_from_queue_to_hub(
kitchen_name: str,
hub: str,
dumpling_queue: multiprocessing.Queue,
kitchen_info: dict,
log: logging.Logger,
):
"""
Grabs dumplings from the dumpling queue and sends them to ``nd-hub``.
:param kitchen_name: The name of the kitchen.
:param hub: The address where ``nd-hub`` is receiving dumplings.
:param dumpling_queue: Queue to grab dumplings from.
:param kitchen_info: Dict describing the kitchen.
:param log: Logger.
"""
hub_ws = 'ws://{0}'.format(hub)
log.info("{0}: Connecting to the dumpling hub at {1}".format(
kitchen_name, hub_ws)
)
try:
websocket = await websockets.connect(hub_ws)
except OSError as e:
log.error(
"{0}: There was a problem with the dumpling hub connection. "
"Is nd-hub available?".format(kitchen_name))
log.error("{0}: {1}".format(kitchen_name, e))
return
try:
# Register our kitchen information with the dumpling hub.
await websocket.send(json.dumps(kitchen_info))
# Send dumplings to the hub when they come in from the chefs.
while True:
dumpling = dumpling_queue.get()
await websocket.send(dumpling)
except asyncio.CancelledError:
log.warning(
"{0}: Connection to dumpling hub cancelled; closing...".format(
kitchen_name))
try:
await websocket.close(*ND_CLOSE_MSGS['conn_cancelled'])
except websockets.exceptions.InvalidState:
pass
except websockets.exceptions.ConnectionClosed as e:
log.warning("{0}: Lost connection to dumpling hub: {1}".format(
kitchen_name, e))
except OSError as e:
log.exception(
"{0}: Error talking to dumpling hub: {1}".format(kitchen_name, e)
)
def __init__(self, logger: logging.Logger, port: int = 19132, interface: str = "0.0.0.0"):
self.logger = logger
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
try:
self.socket.bind((interface, port))
except Exception as e:
logger.error("FAILED TO BIND TO PORT! Perhaps another server is running on the port?")
logger.error(str(e))
finally:
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.socket.setblocking(False) # Non-blocking
async def log_unhandled_exc(logger: logging.Logger):
"""Log and suppress any unhandled exception.
This can be used as the outermost layer of a handler, so that unhandled
errors are logged explicitly, instead of being left to the "Task exception
was never retrieved" handler.
"""
try:
yield
except asyncio.CancelledError:
raise
except Exception:
logger.error('Unexpected error', exc_info=True)
def run(sc: SlackClient, channel: str, message: str, retries: int,
logger: logging.Logger) -> None:
if sc.rtm_connect():
logger.info("Connected to Slack")
channel_id = find_channel_id(channel, sc)
logger.debug(f"Found channel ID {channel_id} for #{channel}")
logger.info(f"Listening for joins in #{channel}")
retry_count = 0
backoff = 0.5
while True:
try:
# Handle dem events!
for event in sc.rtm_read():
handle_event(event, channel, channel_id, message, sc, logger)
# Reset exponential backoff retry strategy every time we
# successfully loop. Failure would have happened in rtm_read()
retry_count = 0
time.sleep(0.5)
# This is necessary to handle an error caused by a bug in Slack's
# Python client. For more information see
# https://github.com/slackhq/python-slackclient/issues/127
#
# The TimeoutError could be more elegantly resolved by making a PR
# to the websocket-client library and letting them coerce that
# exception to a WebSocketTimeoutException.
except (websocket.WebSocketConnectionClosedException, TimeoutError):
logger.error("Lost connection to Slack, reconnecting...")
if not sc.rtm_connect():
logger.info("Failed to reconnect to Slack")
if retry_count >= retries:
sys.exit(bail(
'fatal',
'red',
"Too many failed reconnect attempts, shutting down")
)
time.sleep((backoff ** 2) / 4)
else:
logger.info("Reconnected to Slack")
retry_count += 1
else:
sys.exit(bail('fatal', 'red', "Couldn't connect to Slack"))
def __init__(self, msg: str, code: str=None, exception: Exception=None, logger: logging.Logger=None):
self.code = code or RegovarException.code
self.msg = msg or RegovarException.msg
self.id = str(uuid.uuid4())
self.date = datetime.datetime.utcnow().timestamp()
self.log = "ERROR {} [{}] {}".format(self.code, self.id, self.msg)
if logger:
logger.error(self.log)
if exception and not isinstance(exception, RegovarException):
# To avoid to log multiple time the same exception when chaining try/catch
logger.exception(exception)
else:
err(self.log, exception)
def main_loop(judge_logger: logging.Logger):
"""
A loop to check if there is
:param logger:
:return:
"""
try:
db_coon = JudgeDBConnection(judge_config)
while True:
time.sleep(3)
db_coon.has_new_run()
judge_logger.debug('Judge daemon runs for 3s.')
except Exception as e:
judge_logger.error(e)
return
def parse(acacia_ltl_text: str, acacia_part_text: str, logger: Logger) -> (list, list, dict):
acacia_ltl_text = _add_spec_unit_if_necessary(acacia_ltl_text)
input_signals, output_signals = _parse_part(acacia_part_text)
data_by_unit = dict(acacia_parser.parse(acacia_ltl_text, lexer=acacia_lexer))
known_signals = input_signals + output_signals
for (assumptions, guarantees) in data_by_unit.values():
error = check_unknown_signals_in_properties(assumptions + guarantees, known_signals)
if error:
logger.error(error)
return None
return input_signals, output_signals, data_by_unit
def parse_ltl(anzu_text, logger:Logger) -> dict:
""" Return {section:data} or None in case of error """
section_name_to_data = dict(anzu_parser.parse(anzu_text, lexer=anzu_lexer))
input_signals = section_name_to_data[ANZU_INPUT_VARIABLES]
output_signals = section_name_to_data[ANZU_OUTPUT_VARIABLES]
known_signals = input_signals + output_signals
for asts in (section_name_to_data[ANZU_ENV_FAIRNESS],
section_name_to_data[ANZU_ENV_INITIAL],
section_name_to_data[ANZU_ENV_TRANSITIONS],
section_name_to_data[ANZU_SYS_FAIRNESS],
section_name_to_data[ANZU_SYS_INITIAL],
section_name_to_data[ANZU_SYS_TRANSITIONS]):
error = check_unknown_signals_in_properties(asts, known_signals)
if error:
logger.error(error)
return None
return section_name_to_data
def attempt_database_upgrade(oLogHandler=None):
"""Attempt to upgrade the database, going via a temporary memory copy."""
oTempConn = connectionForURI("sqlite:///:memory:")
oLogger = Logger('attempt upgrade')
if oLogHandler:
oLogger.addHandler(oLogHandler)
(bOK, aMessages) = create_memory_copy(oTempConn, oLogHandler)
if bOK:
oLogger.info("Copied database to memory, performing upgrade.")
if len(aMessages) > 0:
oLogger.info("Messages reported: %s", aMessages)
(bOK, aMessages) = create_final_copy(oTempConn, oLogHandler)
if bOK:
oLogger.info("Everything seems to have gone OK")
if len(aMessages) > 0:
oLogger.info("Messages reported %s", aMessages)
return True
else:
oLogger.critical("Unable to perform upgrade.")
if len(aMessages) > 0:
oLogger.error("Errors reported: %s", aMessages)
oLogger.critical("!!YOUR DATABASE MAY BE CORRUPTED!!")
else:
oLogger.error("Unable to create memory copy. Database not upgraded.")
if len(aMessages) > 0:
oLogger.error("Errors reported %s", aMessages)
return False
def begin(filename=None, failopen=False):
if not filename:
unique = os.environ['LOGNAME']
cmd = os.path.basename(sys.argv[0])
filename = "/tmp/%s-%s.lock" % (unique, cmd)
if os.path.exists(filename):
log.warn("Lockfile found!")
f = open(filename, "r")
pid = None
try:
pid = int(f.read())
except ValueError:
pass
f.close()
if not pid:
log.error("Invalid lockfile contents.")
else:
try:
os.getpgid(pid)
log.error("Aborting! Previous process ({pid}) is still alive. Remove lockfile manually if in error: {path}".format(pid=pid, path=filename))
sys.exit(1)
except OSError:
if failopen:
log.fatal("Aborting until stale lockfile is investigated: {path}".format(path=filename))
sys.exit(1)
log.error("Lockfile is stale.")
log.info("Removing old lockfile.")
os.unlink(filename)
f = open(filename, "w")
f.write(str(os.getpid()))
f.close()
global lockfile
lockfile = filename
print "usage: %s threadPoolSize numberOfQueries fileWithUrls" % (sys.argv[0])
exit(1)
maxThreadCount = 5
if (len(sys.argv) >= 2):
maxThreadCount = int(sys.argv[1])
maxQueryCount = 20
if (len(sys.argv) >= 3):
maxQueryCount = int(sys.argv[2])
urlFile = os.path.dirname(sys.argv[0]) + "/testUrls.txt"
if (len(sys.argv) >= 4):
urlFile = sys.argv[3]
if (not os.path.exists(urlFile)):
Logger.error(logger, "%s does not exist!" % (urlFile))
exit(1)
totalFetchTime = 0.0
totalRequestsCompleted = 0
urls = []
queue = Queue.Queue()
class ThreadUrl(threading.Thread):
"""Threaded Url Grab"""
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
class DBSession(scoped_session):
def __init__(self, user, password, db_host, db_name,
logger_name, need_echo=False):
"""Create db session. Return engine and Base class.
:param user:
db username;
:param password:
db user password;
:param db_host:
db host;
:param db_name:
db name;
:param need_echo:
flag: show or not sql statement.
"""
self.base = Base
engine = "postgresql+psycopg2://{user}:{password}@{db_host}/{db_name}"
engine = engine.format(**vars())
self.engine = create_engine(engine, convert_unicode=True,
echo=need_echo)
self.base.metadata.create_all(self.engine)
maker = sessionmaker(autocommit=False, autoflush=False,
bind=self.engine)
self.logger = Logger(logger_name, level=INFO)
super().__init__(maker)
def __call__(self, **kw):
"""Redefine __call__ in sql alchemy to add get_one_or_log function."""
registry = super().__call__(**kw)
registry.get_one_or_log = self.get_one_or_log
return registry
def get_one_or_log(self, query, message, need_log=True):
"""Get one data from db or write error to log file.
:param query:
sql_alchemy query object;
:param message:
text or error message which must write to log;
:param need_log:
flag to write to log file or not.
"""
result = None
# get one result
try:
result = query.one()
# many object returned
except MultipleResultsFound:
if need_log:
msg = "It is a lot of identical {0}.".format(message)
self.logger.error(msg)
# no object returned
except NoResultFound:
if need_log:
msg = "Such {0} doesn't exist.".format(message)
self.logger.error(msg)
return result
class Printer:
"""
Printer utility, can display depending on 4 mods : debug, info, warning and error
Will be writen again later with a proper logger implementation
Bear with it for now !
"""
__metaclass__ = Singleton
DEBUG = 0
INFO = 1
WARNING = 2
ERROR = 3
def __init__(self):
self.level = self.INFO
self.logger = None
def setLogger(self, filepath, level):
""" Define logger """
if not os.path.isdir(os.path.dirname(filepath)):
raise Exception("Unknown directory " + os.path.dirname(filepath))
## Why ? well ... https://docs.python.org/2/library/logging.html#levels
logLevel = 10 if not level else int(level) * 10
handler = FileHandler(filepath)
formatter = Formatter('%(asctime)s - %(levelname)-8s - %(message)s')
handler.setFormatter(formatter)
self.logger = Logger('main')
self.logger.addHandler(handler)
self.logger.setLevel(logLevel)
def debug(self, origin, message):
""" print a debug message """
if self.logger:
self.logger.debug(message, {'origin' : origin})
if self.level > self.DEBUG:
return
print '[DEBUG][' + str(datetime.datetime.now()) + '][' + origin + '] ' + message
def info(self, origin, message):
""" print an info message """
if self.logger:
self.logger.info(message, {'origin' : origin})
if self.level > self.INFO:
return
print Color.INFO + '[INFO][' + str(datetime.datetime.now()) + '][' + origin + '] ' + message + Color.ENDC
def warning(self, origin, message):
""" print a warning message """
if self.logger:
self.logger.warning(message, {'origin' : origin})
if self.level > self.WARNING:
return
print Color.WARNING + '[WARNING][' + str(datetime.datetime.now()) + '][' + origin + '] ' + message + Color.ENDC
def error(self, origin, message):
""" print an error message """
if self.logger:
self.logger.error(message, {'origin' : origin})
if self.level > self.ERROR:
return
print Color.FAIL + '[ERROR][' + str(datetime.datetime.now()) + '][' + origin + '] ' + message + Color.ENDC
import lxml.html
import lxml.etree
from logging import Logger
logger = Logger(__name__)
def parseargs():
parser = argparse.ArgumentParser(description='Split a file on <split name="foo">')
parser.add_argument('file', type=pathlib.Path)
parser.add_argument('--out', type=pathlib.Path, default=None, help="Defaults to same location as source file")
return parser.parse_args()
args = parseargs()
if not args.file.exists():
logger.error('File `{}` does not exist'.format(args.file))
exit(1)
if args.out is None:
args.out = args.file.parent
if not args.out.exists():
args.out.mkdir(parents=True)
XML = '.xml'
HTML = '.html'
modes = {XML, HTML}
mode = args.file.suffix
def error(self, msg, *args, **kwargs):
msg = "%s# %s" % (MyLogger.indent, msg)
Logger.error(self, msg, *args, **kwargs)
def run_compiler(
name: str,
args: List[str],
logger: logging.Logger,
) -> int:
"""
Execute the original vbsp, vvis or vrad.
The provided logger will be given the output from the compiler.
The process exit code is returned.
"""
logger.info("Calling original {}...", name.upper())
logger.info('Args: {}', ', '.join(map(repr, args)))
buf_out = bytearray()
buf_err = bytearray()
comp_name = get_compiler_name(name)
# On Windows, calling this will pop open a console window. This suppresses
# that.
if sys.platform == 'win32':
startup_info = subprocess.STARTUPINFO()
startup_info.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startup_info.wShowWindow = subprocess.SW_HIDE
else:
startup_info = None
with subprocess.Popen(
args=[comp_name] + args,
executable=comp_name,
universal_newlines=False,
bufsize=0, # No buffering at all.
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
startupinfo=startup_info,
) as proc:
# Loop reading data from the subprocess, until it's dead.
stdout = proc.stdout # type: io.FileIO
stderr = proc.stderr # type: io.FileIO
while proc.poll() is None: # Loop until dead.
buf_out.extend(stdout.read(64))
buf_err.extend(stderr.read(64))
if b'\r' in buf_err:
buf_err = buf_err.replace(b'\r\n', b'\n').replace(b'\r', b'\n')
if b'\r' in buf_out:
buf_out = buf_out.replace(b'\r\n', b'\n').replace(b'\r', b'\n')
try:
newline_off = buf_err.index(b'\n')
except ValueError:
pass
else:
# Discard any invalid ASCII - we don't really care.
logger.error(buf_err[:newline_off].decode('ascii', 'ignore'))
buf_err = buf_err[newline_off+1:]
try:
newline_off = buf_out.index(b'\n')
except ValueError:
pass
else:
# Discard any invalid ASCII - we don't really care.
logger.info(buf_out[:newline_off].decode('ascii', 'ignore'))
buf_out = buf_out[newline_off+1:]
return proc.returncode
def debugError(_logger: logging.Logger, msg: Any) -> None:
"""Log error messages."""
if pyppeteer.DEBUG:
_logger.error(msg)
else:
_logger.debug(msg)
def error(self, msg, *args, **kwargs):
kwargs = self.configKeys(**kwargs)
return OriginalLogger.error(self, msg, *args, **kwargs)
class AbstractKnot(Process):
'''
Ein abstrakter Knoten Prozess. Uebernimmt die einfachsten Aufgaben: Konfigurationsdatei lesen, Port oeffnen,
Nachrichten senden
'''
def __init__(self, ID, connections_filename):
super(AbstractKnot, self).__init__()
prctl.set_proctitle(self._name + '-' + str(ID))
self._ID = str(ID)
self.__connections_filename = connections_filename
self._ips_and_ports = None
self.__ip = None
self.__port = None
self._listeningSocket = None
self._neighbours = {}
self._system_random = SystemRandom()
#init logging
self.logger = Logger(__name__ + '-' + str(ID))
formatter = logging.Formatter('%(name)s %(levelname)s %(asctime)s: %(message)s')
filehandler = FileHandler('./Logging/' + self._name + '-' + str(ID) + '.log', 'w')
filehandler.setFormatter(formatter)
filehandler.setLevel(logging.NOTSET)
for hdlr in self.logger.handlers: # remove all old handlers
self.logger.removeHandler(hdlr)
self.logger.addHandler(filehandler)
def getID(self):
return self._ID
def info(self):
'''
Der Knoten loggt Informationen ueber sich selbst.
'''
info_message = 'module name:' + __name__ + '\n'
if hasattr(os, 'getppid'):
info_message += 'parent process:' + str(os.getppid()) + '\n'
info_message += 'process id:' + str(os.getpid()) + '\n'
info_message += 'LocalKnot id:' + str(self._ID) + '\n\n'
self.logger.info(info_message)
def read_connections_file(self):
'''
Liest die Datei mit den Informationen ueber die Knoten die es ebenfalls im Netzwerk gibt und speichert
sie in ein dictionary.
Somit sind die IP und der Port der anderen Knoten mittels ihrerer ID abrufbar.
Bei der Datei, handelt es sich um eine Json Datei, die mit der richtigen Bibliothek einfach geladen werden kann.
'''
json_data = open(self.__connections_filename)
self._ips_and_ports = json.load(json_data)
json_data.close()
def open_port(self):
'''
Oeffnet einen Socket auf dem eigenen Serverport auf dem der Prozess hoert und loescht sich selbst aus dem
dictionary mit den anderen Knoten.
'''
self.__ip = self._ips_and_ports[self._ID]["ip"]
self.__port = self._ips_and_ports[self._ID]["port"]
del self._ips_and_ports[self._ID]
#self._listeningSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._listeningSocket = socket.socket()
self._listeningSocket.bind((self.__ip, self.__port))
self._listeningSocket.listen(1000)
def read_connections_and_open_port(self):
'''
Fasst die Methoden read_connections_file() und open_port() zusammen. Erst in Ueebung 2 benoetigt.
'''
self.read_connections_file()
self.open_port()
def receive_messages(self):
'''
* Empfaengt eine Nachricht auf dem Port auf dem der Prozess hoert.
* Deserialisiert die Nachricht
* loggt die Nachricht
* verarbeitet die Nachricht in der abstrakten Methode process_received_message()
'''
connection, addr = self._listeningSocket.accept()
data = connection.recv(1024)
if data:
message = cPickle.loads(data)
self.logger.info("empfangen: " + message.printToString())
self.process_received_message(connection, message)
def return_received_message(self):
'''
* Empfaengt eine Nachricht auf dem Port auf dem der Prozess hoert.
* Deserialisiert die Nachricht
* loggt die Nachricht
* verarbeitet die Nachricht in der abstrakten Methode process_received_message()
'''
connection, addr = self._listeningSocket.accept()
data = connection.recv(1024)
if data:
message = cPickle.loads(data)
self.logger.info("empfangen: " + message.printToString())
return connection, message
@abstractmethod
def process_received_message(self, connection, message):
pass
def send_message_to_id(self, message, ID):
'''
Sendet eine Nachricht an den Prozess mit der uebergebenen ID.
* Oeffnet einen Socket
* ermittelt die IP und den Port des Empfaengers
* serialisiert die Nachricht und sendet sie an den Empfaenger
Das Ergebnis wird geloggt.
'''
for i in range(3):
try:
#sender = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sender = socket.socket()
receiver = self._ips_and_ports[str(ID)]
sender.connect((receiver["ip"], receiver["port"]))
sender.sendall(cPickle.dumps(message))
self.logger.info("gesendet an: " + str(ID) + " Message: " + message.printToString())
return sender
except:
self.logger.error("Error while sending message to " + str(ID), exc_info=1)
return None
def send_message_over_socket(self, socket1, message):
try:
socket1.sendall(cPickle.dumps(message))
self.logger.info("gesendet an: " + str(socket1) + " Message: " + message.printToString())
return True
except:
self.logger.error("Error while sending message to " + str(socket1), exc_info=1)
return False
def choose_new_neighbours(self, amount_neighbours):
'''
Waehle per Zufall amount_neighbours -viele Nachbarn. Wird in der Uebung 1 einmal aufgerufen,
in der Uebung 2 mehrmals. Die IDs der neuen Nachbarn werden geloggt.
'''
self._neighbours.clear()
ips_and_ports_copy = self._ips_and_ports.copy()
for i in range(amount_neighbours):
random_index = self._system_random.randint(0, len(ips_and_ports_copy) - 1)
ID = ips_and_ports_copy.keys()[random_index]
self._neighbours[ID] = self._ips_and_ports[ID]
del ips_and_ports_copy[ID]
self.logger.info("Neue Nachbarn: " + str(self._neighbours))
def get_valid_chefs(
kitchen_name: str,
chef_modules: List[str],
chefs_requested: Union[List[str], bool],
log: logging.Logger,
) -> Dict:
"""
Retrieves the names of all valid DumplingChef subclasses for later
instantiation. Valid chefs are all the classes in ``chef_modules`` which
subclass DumplingChef and are included in our list of ``chefs_requested``.
They also need to have their ``assignable_to_kitchen`` attribute set to
True.
:param kitchen_name: Kitchen name (for logging purposes).
:param chef_modules: List of modules to look for chefs in.
:param chefs_requested: List of requested chef names (True means all chefs
are requested).
:param log: Logger to log to.
:return: Dict of valid DumpingChef subclasses. Keys are the Python module
names and the values are a list of valid chef class names in each
module.
"""
valid_chefs = {}
chef_info = netdumplings.DumplingKitchen.get_chefs_in_modules(chef_modules)
# TODO: chefs_seen could be a set.
chefs_seen = []
# Find all the valid chefs.
for chef_module in chef_info:
import_error = chef_info[chef_module]['import_error']
if import_error:
log.error('Problem with {}: {}'.format(chef_module, import_error))
continue
chef_class_names = chef_info[chef_module]['chef_classes']
is_py_file = chef_info[chef_module]['is_py_file']
if is_py_file:
spec = importlib.util.spec_from_file_location('chefs', chef_module)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
else:
# TODO: Investigate replacing __import__ with
# importlib.import_module
mod = __import__(chef_module, fromlist=chef_class_names)
for chef_class_name in chef_class_names:
chefs_seen.append(chef_class_name)
klass = getattr(mod, chef_class_name)
if not klass.assignable_to_kitchen:
log.warning("{0}: Chef {1} is marked as unassignable".format(
kitchen_name, chef_class_name))
continue
# A chefs_requested value of True means all chefs.
if chefs_requested is True or chef_class_name in chefs_requested:
try:
valid_chefs[chef_module].append(chef_class_name)
except KeyError:
valid_chefs[chef_module] = [chef_class_name]
# Warn about any requested chefs which were not found.
if chefs_requested is not True:
for chef_not_found in [chef for chef in chefs_requested
if chef not in chefs_seen]:
log.warning("{0}: Chef {1} not found".format(
kitchen_name, chef_not_found))
return valid_chefs