def __save_conversation(self, communications):
LOG.info('conversation: communications: ' + str(len(communications)))
communication_ids = []
conversation_participants = {}
last_message_time = datetime.datetime.min
conversation_matched_keywords = {}
for c in communications:
r = db.Communications.Add(
communication_log_id=str(self.communication_log['_id']),
from_=c['from'],
to=c['to'],
message=c['message'],
message_time=c['message_time'],
matched_keywords=c['matched_keywords'],
source=self.communication_log['source'],
enterprise_id=self.communication_log['enterprise_id'],
)
for k in c['matched_keywords']:
conversation_matched_keywords[k] = 1
communication_ids.append(str(r.inserted_id))
conversation_participants[c['from']['name']] = 1
for p in c['to']:
conversation_participants[p['name']] = 1
if last_message_time < c['message_time']:
last_message_time = c['message_time']
db.Conversations.Add(
communication_log_id=str(self.communication_log['_id']),
communication_ids=communication_ids,
matched_keywords=conversation_matched_keywords.keys(),
participants=list(conversation_participants.keys()),
last_message_time=last_message_time,
source=self.communication_log['source'],
enterprise_id=self.communication_log['enterprise_id'],
)
def on_modified(self, event):
path = event.src_path
dest_path = path.replace(".less", ".css")
cmd = 'lessc "%s" "%s"' % (path, dest_path)
status = os.system(cmd)
LOG.info(cmd)
LOG.info("Status : %d" % status)
def log_org_repo_contributors(message: Dict[str, Any]) -> None:
""" Audit github organization repository contributors """
org = os.environ["GITHUB_ORG"]
repo = message.get("repo", None)
audit_id = message.get("audit_id")
if audit_id:
LOG.info({
"action": "Audit organization org repo contributors",
"org": org,
"repo": repo,
"audit_id": audit_id,
})
if repo:
repo_name = repo["name"]
members = github_api.get_github_org_repo_contributors(
org, repo_name)
for member in members:
event = make_audit_event(
type="OrganizationRepoContributor",
org=org,
member=member,
repository=repo,
audit_id=audit_id,
)
LOG.info(event)
else:
raise IncompleteAuditError(audit_id=audit_id,
source=message,
message="Repo not specified")
def send_health_monitoring_data_to_splunk(payload_to_send):
""" Send the Health Monitoring payload to Splunk Cloud HEC """
try:
splunk_hec_token = get_splunk_hec_token(SPLUNK_HEC_SSM_PARAMETER,
AWS_REGION)
splunk_hec_endpoint = (
"https://http-inputs-gds.splunkcloud.com/services/collector")
headers = {"Authorization": "Splunk " + splunk_hec_token}
response = requests.post(splunk_hec_endpoint,
payload_to_send,
headers=headers,
verify=False)
if response.status_code != 200:
LOG.debug(
"Received a non 200 HTTP status code from Splunk Cloud HEC")
LOG.debug("Response code: %s: message: %s", response.status_code,
response.text)
elif response.status_code == 200:
LOG.info("Successful: message: %s", response.text)
except (ValueError, KeyError):
LOG.error("Failed to send health monitoring data to Splunk Cloud HEC")
def ebs_usage(connect, volumes):
"""
Checks for no usage in 24 hours
"""
if type(volumes) is not list:
raise TypeError("Not a list")
unused_volumes = []
if volumes and volumes is not None:
try:
for ebs in volumes:
response = connect.cloudwatch.get_metric_statistics(Namespace='AWS/EBS',
MetricName='VolumeReadBytes',
StartTime=datetime.now() - timedelta(days=1),
EndTime=datetime.now(),
Period=86400, Statistics=['Average'],
Dimensions=[{'Name':'VolumeId','Value':ebs.id}])
if 'Datapoints' in response and not response['Datapoints']:
LOG.info("INFO: {0} is not active".format(ebs.id))
unused_volumes.append(ebs)
except Exception, err:
LOG.error("ERROR: {0}".format(err))
def audit_ebs(connect, volumes, days=None):
"""
Creates list of volumes older than X days
Defaults to 1 week
"""
if type(volumes) is not list:
raise TypeError("Not a list")
# Sets days if not specified
if not days:
date = str(datetime.now() - timedelta(days=7))
else:
date = str(datetime.now() - timedelta(days=days))
audit_tag = [{'Key':'audit','Value':str(datetime.now())}]
unused_ebs=[]
for ebs in volumes:
try:
# Gets tags for volume
for attribute in ebs.tags:
# Searches for audit tag
if attribute.get('Key') == 'audit':
# Compares audit dates
if attribute.get('Value') < date:
LOG.info("INFO: {0} ready for deletion" .format(ebs.id))
unused_ebs.append(ebs.id)
# Tags volume if missing audit tag
else:
LOG.info("INFO: Audit tag added to {0}" .format(ebs.id))
ebs.create_tags(Tags=audit_tag)
except Exception, err:
LOG.error("ERROR: {0}".format(err))
def main():
queue = get_aws_queue(os.environ['AWS_SQS_QUEUE_URL'])
try:
handle_queue(queue)
except KeyboardInterrupt:
LOG.info("Quitting.")
def acceptBluetoothConnection(self):
try:
self.clientSocket, clientInfo = self.serverSocket.accept()
LOG.info("Accepted bluetooth connection from %s", clientInfo)
except (bluetooth.BluetoothError, SystemExit, KeyboardInterrupt):
LOG.error("Failed to accept bluetooth connection ... ",
exc_info=True)
def main():
queue = get_aws_queue(os.environ['AWS_SQS_QUEUE_URL'])
try:
handle_queue(queue)
except KeyboardInterrupt:
LOG.info("Quitting.")
def _add(self, resource_type, _id, contents):
try:
res = self.request('post', f'/{resource_type}/add', json=contents)
res.raise_for_status()
LOG.info(f'UPDATED {resource_type}/{_id}')
except HTTPError as her:
LOG.info(f'Failed to UPDATE {resource_type}/{_id} : {her}')
def on_moved(self, event):
super(ChangeHandler, self).on_moved(event)
if event.is_directory:
return
LOG.info("Moved %s: from %s to %s", 'file', event.src_path,
event.dest_path)
self.helper.dispatch_change(Event.CHANGE, event.dest_path)
def Execute(parse_source, parse_state):
LOG.info('STARTED: ' + os.path.basename(__file__))
LOG.info('parse_source: \'' + str(parse_source) + '\'')
if not parse_state:
parse_state = db.CommunicationLogs.States.DOWNLOADED #production mode: parsing newly downloaded logs
LOG.info('parse_state: \'' + str(parse_state) + '\'')
communication_logs = db.CommunicationLogs.GetByStateSource(
state=parse_state,
source=parse_source,
)
LOG.debug(communication_logs)
for communication_log in communication_logs:
try:
file_path = settings.DOWNLOAD_DIR + '/' + communication_log[
'source'] + '/' + communication_log['file_name']
if not os.path.exists(file_path):
LOG.error('Does not exists: ' + file_path)
continue
LOG.info('Parsing: ' + file_path)
parsers = imp.load_source(
'parsers', 'parsers/' + communication_log['source'] + '.py')
#class_ = getattr(parsers, settings.COMMUNICATION_LOG_SOURCES[communication_log['source']]['ParserClass'])
parser_class = getattr(parsers, communication_log['source'])
parser = parser_class()
parser.Parse(communication_log)
except:
LOG.exception(sys.exc_info()[0])
LOG.info('COMPLETED')
def process_message(raw_message):
header = raw_message['header']
if not validate_header(header):
return True # Effectively drop the message
decoded = TrainMovementsMessage(raw_message['body'])
if (decoded.event_type == EventType.arrival and
decoded.status == VariationStatus.late and
decoded.location.is_public_station and
decoded.operating_company and
decoded.operating_company.is_delay_repay_eligible(
decoded.minutes_late)):
LOG.info('{} {} arrival at {} ({}) - eligible for '
'compensation from {}: {}'.format(
decoded.actual_datetime,
decoded.early_late_description,
decoded.location.name,
decoded.location.three_alpha,
decoded.operating_company,
str(decoded)))
else:
LOG.debug('Dropping {} {} {} message'.format(
decoded.status, decoded.event_type,
decoded.early_late_description))
return True
def closeBluetoothSocket(self):
try:
self.clientSocket.close()
self.serverSocket.close()
LOG.info("Bluetooth sockets successfully closed ...")
except bluetooth.BluetoothError:
LOG.error("Failed to close the bluetooth sockets ", exc_info=True)
def count_all_topic_messages(self, topic): # individual messages
subtotals = []
self.get_consumer(True, topic)
while not self.consumer.poll():
try:
self.consumer.seek_to_beginning()
break
except AssertionError:
LOG.info(f'{topic} waiting for consumer poll...')
sleep(0.25)
try:
while True:
messages = self.consumer.poll_and_deserialize(1000, 1000)
if not messages:
return sum(subtotals)
parts = [i for i in messages.keys()]
for part in parts:
bundles = messages.get(part)
for bundle in bundles:
_msgs = bundle.get('messages')
subtotals.append(sum([1 for m in _msgs]))
except Exception as err:
raise err
finally:
self.consumer.close()
def _delete(self, resource_type, _id):
try:
res = self.request('get', f'/{resource_type}/delete?id={_id}')
res.raise_for_status()
LOG.info(f'DELETED {resource_type}/{_id}')
except HTTPError:
LOG.error(f'Failed to DELETE {resource_type}/{_id}')
def process_message(raw_message):
header = raw_message['header']
if not validate_header(header):
return True # Effectively drop the message
decoded = TrainMovementsMessage(raw_message['body'])
if (decoded.event_type == EventType.arrival
and decoded.status == VariationStatus.late
and decoded.location.is_public_station
and decoded.operating_company
and decoded.operating_company.is_delay_repay_eligible(
decoded.minutes_late)):
LOG.info('{} {} arrival at {} ({}) - eligible for '
'compensation from {}: {}'.format(
decoded.actual_datetime, decoded.early_late_description,
decoded.location.name, decoded.location.three_alpha,
decoded.operating_company, str(decoded)))
else:
LOG.debug('Dropping {} {} {} message'.format(
decoded.status, decoded.event_type,
decoded.early_late_description))
return True
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not TimeoutError:
LOG.info(f'exit before alarm.')
else:
LOG.info(f'TIMEOUT')
signal.alarm(0)
if self.suppress and exc_type is TimeoutError:
return True
def db_connect(db_path=DEFAULT_PATH):
try:
con = sqlite3.connect(db_path)
return con
except Exception as e:
LOG.info(e)
LOG.info(traceback.format_exc())
return None
def increment_message_counter(self, num_bytes):
self.sent_message_count += 1
self.sent_bytes += num_bytes
if self.sent_message_count % LOG_EVERY_N_MESSAGES == 0:
LOG.info('Sent {} messages, ~{:.3f} MB'.format(
self.sent_message_count,
self.sent_bytes / (1024 * 1024)))
def _run_lola(self, lola_file_name, formula):
"""
Run LoLA for a certain file and formula
"""
LOG.info("Running LoLA in temporal file for formula:")
LOG.info("'{0}'".format(formula))
command = ["lola", lola_file_name, "--formula={0}".format(formula)]
(ret, _, stderr) = run(command)
return check_result(stderr)
def getBluetoothSocket(self):
try:
self.serverSocket = bluetooth.BluetoothSocket(bluetooth.RFCOMM)
LOG.info(
"Bluetooth server socket successfully created for RFCOMM service..."
)
except (bluetooth.BluetoothError, SystemExit, KeyboardInterrupt):
LOG.error("Failed to create the bluetooth server socket ",
exc_info=True)
def __init__(self, phi_1, phi_2):
if not isinstance(phi_1, CTLFormula):
err_message = "Phi provided is not an CTL formula"
raise CTLException(err_message)
if not isinstance(phi_2, CTLFormula):
err_message = "Phi provided is not an CTL formula"
raise CTLException(err_message)
self._phi_1 = phi_1
self._phi_2 = phi_2
LOG.info("New negated exist until predicate created")
def model_checking(self, m_0, formula):
"""
Perform model checking of the petri net for a certain marking and
formula using lola.
"""
lola_file_name = self._create_lola_file(m_0)
result = self._run_lola(lola_file_name, formula.print_lola())
LOG.info("Model Checking result: \"{0}\"".format(result))
os.remove(lola_file_name)
LOG.info("Removing LoLA temporal file")
return result
def print_ebml_element(self, size, id, name, type_, value):
#if type_ != ebml.BINARY:
# LOG.info('value: %s' % (value))
#else:
# LOG.info('value: %s' % ('<BINARY>'))
if type_ != ebml.BINARY:
LOG.info('name:%s, size:%d, id:%s, type_:%s, value: %s' %
(name, size, hex(id), type_, value))
else:
LOG.info('name:%s, size:%d, id:%s, type_:%s, value: %s' %
(name, size, hex(id), type_, '<BINARY>'))
def _create_lola_file(self, m_0):
"""
Create a LoLA file of the model.
"""
LOG.info("Creating LoLA temporal file")
lola_file_name = "__tmp_lola_model_.lola"
file_object = open(lola_file_name, "wb")
file_object.write(self.export_lola(m_0));
file_object.close()
LOG.info("LoLA temporal file created")
return lola_file_name
def handle_change(self, change: Event, _file_type: FileType, _id, _contents):
_resource = _file_type.name.lower()
LOG.debug(f'helper handling {change} on {_resource}/{_id}')
if change is Event.DELETE and _file_type is not FileType.BPMN:
self._delete(_file_type, _id)
elif change is Event.DELETE and _file_type is FileType.BPMN:
LOG.info('We cannot _yet_ automatically delete BPMNs from the brokers')
elif _file_type is FileType.BPMN:
LOG.debug('Working on adding BPMN')
self.add_bpmn(_id, _contents)
else:
self._update(_resource, _id, _contents)
def __init__(
self,
stream_name,
time_span_between_frames_in_secs=-1,
frame_queue_max_length=10,
save_frames2directory='./_frames',
catch_frames=True,
reconnect_max_count=3, #ignored when refreshing connection due to AWS kinesis time limit
):
try:
LOG.info('Parser starting for %s' % stream_name)
self.lock = threading.Lock()
self.frames_lock = threading.Lock()
self.tags_lock = threading.Lock()
self.disposing = False
self.stream_name = stream_name
self.TimeSpanBetweenFramesInSecs = time_span_between_frames_in_secs
self.FrameQueueMaxLength = frame_queue_max_length
if save_frames2directory == True:
save_frames2directory = './_frames'
self.frame_directory = save_frames2directory
if self.frame_directory:
if os.path.exists(self.frame_directory):
import shutil
shutil.rmtree(self.frame_directory)
os.makedirs(self.frame_directory)
self.catch_frames = catch_frames
self.reconnect_max_count = reconnect_max_count
self.next_frame_time = 0.0
with self.frames_lock:
self.Frames = []
with self.tags_lock:
self.tags_line = []
self.last_packet_tags = None
self.last_frame_id = 0
self.connection_attempts_count = 0
self.connection_renewals_count = 0
self.kinesis_stream = None
self.libav_input_descriptor = None
self.kinesis_stream_reader_thread = None
self.libav_parser_thread = None
self.kinesis_stream_pipe = r'/tmp/AwsKinesisParserFifo'
self.starter(False)
except:
LOG.exception(sys.exc_info()[0])
finally:
pass
def create_temperature_humidity_tb(con):
temperature_humidity_tb = '''
CREATE TABLE IF NOT EXISTS temperature_humidity (
id INTEGER NOT NULL PRIMARY KEY,
temperature REAL,
humidity REAL,
`timestamp` TEXT
)
'''
cur = con.cursor()
cur.execute(temperature_humidity_tb)
LOG.info('temperature_humidity table created.')
def load_file(self, file_name):
"""
Load a Petri Net model dumped into a JSON file with name 'file_name'.
"""
msg = "Loading Petri Net from file '{0}'".format(file_name)
LOG.info(msg)
with open(file_name) as in_file:
in_dict = json.load(in_file)
self._places = in_dict["P"]
self._transitions = in_dict["T"]
self._input = self._read_io_dict("I", in_dict["I"])
self._output = self._read_io_dict("O", in_dict["O"])
msg = "Loading completed"
LOG.info(msg)
def reachability_set(self, m_0):
"""
Get the reachability set of the model for marking 'm'.
"""
m_0 = self._fix_marking(m_0)
msg = "Getting reachability set from '{0}'".format(m_0)
LOG.info(msg)
reach_graph = {str(m_0): []}
open_set = [m_0]
closed_set = []
while len(open_set):
m = open_set.pop(0)
if m in closed_set:
continue
m_key = str(m)
msg = "Adding new marking to reachability set: '{0}'".format(m)
LOG.info(msg)
reach_graph[m_key] = self._get_succesors(m)
open_set.extend(reach_graph[m_key])
closed_set.append(m)
msg = "Reachability set calculated from: '{0}'".format(m_0)
LOG.info(msg)
msg = "Reachability set size is: '{0}'".format(len(reach_graph))
LOG.info(msg)
return reach_graph
def advertiseBluetoothService(self):
try:
bluetooth.advertise_service(
self.serverSocket,
self.serviceName,
service_id=self.uuid,
service_classes=[self.uuid, bluetooth.SERIAL_PORT_CLASS],
profiles=[bluetooth.SERIAL_PORT_PROFILE],
#protocols = [ OBEX_UUID ]
)
LOG.info("%s advertised successfully ...", self.serviceName)
except (bluetooth.BluetoothError, SystemExit, KeyboardInterrupt):
LOG.error("Failed to advertise bluetooth services ... ",
exc_info=True)
def delete_vol(connect, volumes):
"""
Deletes Volumes Passed
"""
if type(volumes) is not list:
raise TypeError("Not a list")
# Currently only printing id
for ebs in volumes:
try:
LOG.info("INFO: {0} would have been deleted" .format(ebs))
except Exception, err:
LOG.error("ERROR: {0}".format(err))
def __download_communication_logs(source):
try:
LOG.info('Checking: ' + source['Name'])
transport = paramiko.Transport(source['Host'], source['Port'])
transport.connect(username=source['User'],
password=source['Password'])
sftp = paramiko.SFTPClient.from_transport(transport)
sftp.chdir(source['RemoteDir'])
files = sftp.listdir('.')
for file in files:
try:
#lstat = sftp.lstat(file)
#LOG.debug(file, lstat)
#communication_log = db.CommunicationLogs.GetByKey(source['Name'], file, lstat.st_mtime)
communication_log = db.CommunicationLogs.GetByKey(
source['Name'], file)
if not reload_state: #production mode: only loading new logs, not reloading old ones
if communication_log:
continue
else: #testing mode: only reloading old logs, not loading new ones
if not communication_log:
continue
if communication_log['state'] != reload_state:
continue
db.CommunicationLogs.DeleteById(
communication_log['_id'])
db.Conversations.DeleteByCommunicationLogId(
communication_log['_id'])
db.Communications.DeleteByCommunicationLogId(
communication_log['_id'])
unique_file_name = file #+ '.' + str(lstat.st_mtime)
LOG.info('Loading: %s/%s', source['Name'],
unique_file_name)
directory = settings.DOWNLOAD_DIR + '/' + source['Name']
if not os.path.exists(directory):
os.makedirs(directory)
sftp.get(file, directory + '/' + unique_file_name)
db.CommunicationLogs.Add(
source=source['Name'],
remote_file_name=file,
#remote_file_modified_time = lstat.st_mtime,
file_name=unique_file_name,
enterprise_id=source['EnterpriseId'],
)
except:
LOG.exception(sys.exc_info()[0])
except:
LOG.exception(sys.exc_info()[0])
def topics(self):
while True:
topic_size = {}
self.get_consumer(quiet=True)
refresh_str = "-> Refresh Topics"
detailed_str = "-> Get Real Message Counts for Topics"
quit_str = "-> Exit KafkaViewer\n"
LOG.info(f'fetching topics')
bold('Fetching Topics')
topics = None
with timeout(5):
topics = sorted([i for i in self.consumer.topics() if i not in EXCLUDED_TOPICS])
LOG.info(f'found topics {topics}')
bold(f'Inspecting topics {topics}')
if not topics:
LOG.info(f'no topics found')
bold("No topics available")
for topic in topics:
LOG.info(f'inspecting topic: {topic}')
bold(f'Inspecting topic: {topic}')
try:
with timeout(5):
topic_size[topic] = self.get_topic_size(topic)
LOG.info(f'{topic} size: {topic_size[topic]}')
bold('...Success')
except TimeoutError:
topic_size[topic] = 'timed-out'
bold('...Failure!')
clear()
prompt_key = { "topic: %s {%s}" % (topic, topic_size[topic]) : topic for topic in topics }
prompts = sorted(prompt_key.keys())
prompts.extend([detailed_str, refresh_str, quit_str])
bold("Choose a Topic to View")
norm("-> topic {# of offsets in topic}\n")
topic = self.ask(prompts)
topic = prompt_key.get(topic) if topic in prompt_key else topic
if topic is quit_str:
return
elif topic is refresh_str:
clear()
continue
elif topic is detailed_str:
self.print_topic_sizes(topics)
continue
self.get_consumer(topic=topic)
self.consumer.seek_to_beginning()
self.show_topic()
def get_topic_size(self, topic): # offsets
self.get_consumer(True, topic)
while not self.consumer.poll():
try:
self.consumer.seek_to_end()
break
except AssertionError:
LOG.info(f'{topic} waiting for consumer poll...')
sleep(0.25)
partitions = [TopicPartition(topic, p) for p in self.consumer.partitions_for_topic(topic)]
end_offsets = self.consumer.end_offsets(partitions)
self.consumer.close(autocommit=False)
size = sum([i for i in end_offsets.values()])
return size
def _set_zb(self):
zeebe_url = os.environ.get('ZEEBE_ADDRESS')
if zeebe_url:
LOG.info(f'Using External ZB @ {zeebe_url}')
zb_config = ZeebeConfig(
url=zeebe_url,
client_id=os.environ.get('ZEEBE_CLIENT_ID'),
client_secret=os.environ.get('ZEEBE_CLIENT_SECRET'),
audience=os.environ.get('ZEEBE_AUDIENCE'),
token_url=os.environ.get('ZEEBE_AUTHORIZATION_SERVER_URL')
)
else:
LOG.info(f'No External ZB configured, using local @ {self._local_zb_url}')
zb_config = ZeebeConfig(url=self._local_zb_url)
self.zeebe_connection = ZeebeConnection(zb_config)
def save_file(self, file_name):
"""
Dump the Petri Net model into a JSON file with name 'file_name'.
"""
msg = "Dumping Petri Net into file '{0}'".format(file_name)
LOG.info(msg)
out_dict = {}
out_dict["P"] = self._places
out_dict["T"] = self._transitions
out_dict["I"] = self._get_io_dict("I")
out_dict["O"] = self._get_io_dict("O")
with open(file_name, "w+") as out_file:
json.dump(out_dict, out_file)
msg = "Dumping completed"
LOG.info(msg)
def add_transition(self, transition):
"""
Function to add transition to T set if not already in it. If
'transition' is not an string, this function will raise an exception.
"""
if not isinstance(transition, basestring):
err_message = "Transition provided is not an string"
raise PetriNetException(err_message)
if transition not in self._transitions:
self._transitions.append(transition)
for place in self._places:
self.change_input_flow(place, transition, 0)
self.change_output_flow(place, transition, 0)
LOG.info("Transition '{0}' added to T set".format(transition))
else:
LOG.info("Transition '{0}' already in T set".format(transition))
def publish_alert(message, topic_arn):
"""Publish alert message to SNS"""
try:
sns = boto3.client("sns")
sns_response = sns.publish(
TopicArn=topic_arn,
Message=message,
Subject="New report-phishing alert recorded",
MessageStructure="json",
)
LOG.debug("SNS Response: %s", sns_response)
LOG.info("Alert published to SNS")
return sns_response
except ClientError as err:
raise ServerError("Error publishing message to SNS") from err
def add_place(self, place):
"""
Function to add places to P set if not already in it. If 'place' is not
an string, this function will raise an exception.
"""
if not isinstance(place, basestring):
err_message = "Place provided is not an string"
raise PetriNetException(err_message)
if place not in self._places:
self._places.append(place)
for transition in self._transitions:
self.change_input_flow(place, transition, 0)
self.change_output_flow(place, transition, 0)
LOG.info("Place '{0}' added to P set".format(place))
else:
LOG.info("Place '{0}' already in P set".format(place))
def process_update_dashboard_event(lambda_invoke_event):
""" Receive and process Health Monitoring message """
try:
LOG.debug(lambda_invoke_event)
health_events = [
json.loads(event["Sns"]["Message"])
for event in lambda_invoke_event["Records"]
]
for health_monitoring_message in health_events:
LOG.info("Message: %s", str(health_monitoring_message))
payload_to_send = build_splunk_payload(health_monitoring_message)
send_health_monitoring_data_to_splunk(payload_to_send)
except (ValueError, KeyError, json.JSONDecodeError) as error:
LOG.error(
"Failed to build Splunk payload for health monitoring data: %s",
error)
def _set_consumer(self):
consumer_url = os.environ.get('CONSUMER_URL')
self.session = requests.Session()
if consumer_url:
LOG.info(f'Using external consumer @ {consumer_url}')
self._url = consumer_url
consumer_user = os.environ.get('CONSUMER_USER')
self.session.auth = (
consumer_user,
os.environ.get('CONSUMER_PASSWORD')
)
else:
LOG.info(f'Using default consumer @ {self._url}')
self.session.auth = (
self._default_consumer_user,
self._default_consumer_password
)
def run(command):
cmd_string = " ".join(command)
LOG.info("Runnin command \"{0}\"".format(cmd_string))
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret = process.wait()
stdout, stderr = process.communicate()
LOG.info("RC: \"{0}\"".format(ret))
LOG.info("STDOUT: \"{0}\"".format(stdout))
LOG.info("STDERR: \"{0}\"".format(stderr))
return (ret, stdout, stderr)
def _processZIP(self, zip_archive, converter_name):
LOG.info('Incoming request (%s, %d bytes)' % (converter_name, len(zip_archive)))
ts = time.time()
# temp directory handling
now = datetime.now().strftime('%Y%m%d%Z%H%M%S')
ident = '%s-%s' % (now, uuid.uuid4())
tempdir = os.path.join(self.temp_directory, ident)
os.makedirs(tempdir)
# store zip archive first
zip_temp = os.path.join(tempdir, 'input.zip')
file(zip_temp, 'wb').write(base64.decodestring(zip_archive))
ZF = zipfile.ZipFile(zip_temp, 'r')
for name in ZF.namelist():
destfile = os.path.join(tempdir, name)
if not os.path.exists(os.path.dirname(destfile)):
os.makedirs(os.path.dirname(destfile))
file(destfile, 'wb').write(ZF.read(name))
ZF.close()
# find HTML file
html_files = glob.glob(os.path.join(tempdir, '*.htm*'))
if not html_files:
raise IOError('Archive does not contain any html files')
if len(html_files) > 1:
raise RuntimeError('Archive contains more than one html file')
html_filename = html_files[0]
# inject BASE tag containing the full local path (required by PrinceXML)
self._inject_base_tag(html_filename)
result = self._convert(html_filename,
converter_name=converter_name)
output_filename = result['output_filename']
basename, ext = os.path.splitext(os.path.basename(output_filename))
# Generate result ZIP archive with base64-encoded result
zip_out = os.path.join(tempdir, '%s.zip' % ident)
ZF = zipfile.ZipFile(zip_out, 'w')
ZF.writestr('output%s' % ext, file(output_filename, 'rb').read())
ZF.writestr('conversion-output.txt', result['output'])
ZF.close()
LOG.info('Request end (%3.2lf seconds)' % (time.time() - ts))
return zip_out, output_filename
def valid_data(flat_info):
"""Check if extracted flat info valid. Return boolean."""
valid = True
description, postcode_area, bedrooms, price, website = flat_info
if not description:
LOG.warning(f'No description found: {website}. Skipping div')
valid = False
if postcode_area not in ('EH6', 'EH7', ''):
LOG.info(
f'{postcode_area} not a valid postcode: {website}. Skipping div')
valid = False
if bedrooms <= 0:
LOG.warning(
f'{bedrooms} not a valid num of bedrooms: {website}. Skipping div')
valid = False
if price <= 0:
LOG.warning(f'{price} not a valid price: {website}. Skipping div')
valid = False
return valid
def runcmd(cmd):
""" Execute a command using the subprocess module """
if win32:
cmd = cmd.replace('\\', '/')
s = Popen(cmd, shell=False)
s.wait()
return 0, ''
else:
if execute_mode == 'system':
status = os.system(cmd)
return status, ''
elif execute_mode == 'commands':
return commands.getstatusoutput(cmd)
elif execute_mode == 'process':
stdin = open('/dev/null')
stdout = stderr = PIPE
p = Popen(
cmd,
shell=True,
stdin=stdin,
stdout=stdout,
stderr=stderr,
)
status = p.wait()
stdout_ = p.stdout.read().strip()
stderr_ = p.stderr.read().strip()
if stdout_:
LOG.info(stdout_)
if stderr_:
LOG.info(stderr_)
return status, stdout_ + stderr_
else:
raise ValueError('Unknown value for $ZOPYX_CONVERT_EXECUTE_MODE')
def Dispose(self):
if self.disposing:
return
self.disposing = True
with self.lock:
LOG.info('Shutting down Parser...\r\n%s' %
'\r\n'.join(traceback.format_stack()))
self.run_kinesis_stream_reader = False
self.run_libav_parser = False
time.sleep(1)
if self.kinesis_stream:
try:
self.kinesis_stream.close()
except:
LOG.exception(sys.exc_info()[0])
self.kinesis_stream = None
if self.libav_input_descriptor:
try:
self.libav_input_descriptor.close()
except:
LOG.exception(sys.exc_info()[0])
self.libav_input_descriptor = None
self.libav_output_reader = None
if self.kinesis_stream_reader_thread:
self.kinesis_stream_reader_thread.join(3)
if self.kinesis_stream_reader_thread.isAlive():
raise Exception(
'kinesis_stream_reader_thread has not stopped!')
if self.libav_parser_thread:
self.libav_parser_thread.join(3)
if self.libav_parser_thread.isAlive():
raise Exception('libav_parser_thread has not stopped!')
LOG.info("Parser has been disposed.")
def runcmd(cmd):
""" Execute a command using the subprocess module """
if win32:
cmd = cmd.replace('\\', '/')
s = Popen(cmd, shell=False)
s.wait()
return 0, ''
else:
if execute_mode == 'system':
status = os.system(cmd)
return status, ''
elif execute_mode == 'commands':
return commands.getstatusoutput(cmd)
elif execute_mode == 'process':
stdin = open('/dev/null')
stdout = stderr = PIPE
p = Popen(cmd,
shell=True,
stdin=stdin,
stdout=stdout,
stderr=stderr,
)
status = p.wait()
stdout_ = p.stdout.read().strip()
stderr_ = p.stderr.read().strip()
if stdout_:
LOG.info(stdout_)
if stderr_:
LOG.info(stderr_)
return status, stdout_ + stderr_
else:
raise ValueError('Unknown value for $ZOPYX_CONVERT_EXECUTE_MODE')
def main():
username = os.environ['NR_DATAFEEDS_USERNAME']
password = os.environ['NR_DATAFEEDS_PASSWORD']
# See http://nrodwiki.rockshore.net/index.php/Train_Movements
datafeeds_hostname = 'datafeeds.networkrail.co.uk'
datafeeds_channel = 'TRAIN_MVT_ALL_TOC'
handler = UploadStompMessagesToAmazonSQS(os.environ['AWS_SQS_QUEUE_URL'])
conn = create_data_feed_connection(
datafeeds_hostname, username, password, datafeeds_channel, handler)
while True:
try:
time.sleep(1)
except KeyboardInterrupt:
LOG.info("Keyboard interrupt, quitting.")
break
conn.disconnect()
def change_output_flow(self, place, transition, value):
"""
Change the value of the output function for 'place' and 'transition'
equals to 'value':
O(place, transition) = value
Raise an exception if 'place' is not in P, 'transition' is not in T, or
'value' is not an integer value equal or greatter than 0.
"""
if place not in self._places:
err_message = "Place is not in P"
raise PetriNetException(err_message)
if transition not in self._transitions:
err_message = "Transition is not in T"
raise PetriNetException(err_message)
if not isinstance(value, int) or value < 0:
err_message = "The value is not a integer greater or equal than 0"
raise PetriNetException(err_message)
key_pair = (place, transition)
self._output[key_pair] = value
msg = "Value updated in O. O({0}, {1}) = {2}".format(place, transition,
value)
LOG.info(msg)
def export_lola(self, m_0):
"""
Export the Petri Net model as a LoLA file.
"""
msg = "Exporting Petri Net to LoLA"
LOG.info(msg)
place_list = self._get_place_list()
marking_list = self._get_marking_list(m_0)
transition_list = self._get_transition_list()
lola_file = LOLA_TEMPLATE.format(place_list, marking_list, transition_list)
msg = "Export completed"
LOG.info(msg)
msg = "LoLA file is:"
LOG.info(msg)
LOG.info(lola_file)
return lola_file
def start():
event_handler = MyHandler(patterns=["*.less"], ignore_directories=True)
observer = Observer()
path = os.path.dirname(__file__)
observer.schedule(event_handler, path=path, recursive=True)
observer.start()
LOG.info("-" * 40)
LOG.info("Started LESSCSS watchdog")
LOG.info("-" * 40)
def handle_queue(queue):
LOG.info("There are ~{} messages in the queue. Let's go!".format(
queue.attributes['ApproximateNumberOfMessages']))
params = {
'MaxNumberOfMessages': 10,
'VisibilityTimeout': 10,
'WaitTimeSeconds': 10,
}
count = 0
while True:
for sqs_message in queue.receive_messages(**params):
message = decode_sqs_message(sqs_message)
if process_message(message):
sqs_message.delete()
else:
LOG.info("Not sending ACK for this one")
count += 1
if count % LOG_EVERY_N_MESSAGES == 0:
LOG.info('Processed {} messages, ~{} messages in queue'.format(
count, queue.attributes['ApproximateNumberOfMessages']))
def __init__(self, place):
if not isinstance(place, basestring):
err_message = "Place provided is not an string"
raise CTLException(err_message)
self._place = place
LOG.info("New negated atomic proposition predicate created")
def registerResource(name, directory):
if not os.path.exists(directory):
raise IOError('Directory "%s" does not exit' % directory)
resources_registry[name] = directory
LOG.info('Registered resource directory "%s" as "%s"' % (directory, name))
def __init__(self, phi):
if not isinstance(phi, CTLFormula):
err_message = "Phi provided is not an CTL formula"
raise CTLException(err_message)
self._phi = phi
LOG.info("New negated exist globally predicate created")
def __init__(self):
self._operator = "TRUE"
LOG.info("New TRUE predicate created")
def __init__(self):
self._operator = "FALSE"
LOG.info("New FALSE predicate created")
def app(global_config, **kw):
""" This function returns a pyramid.router.Router object. It
is usually called by the PasteDeploy framework during ``paster
serve``"""
# paster app config callback
from zopyx.smartprintng.server.models import get_root
import zopyx.smartprintng.server
from models import root
from logger import LOG
import zopyx.convert2
if "mail_config" in global_config:
mail_config = os.path.abspath(global_config["mail_config"])
os.environ["EMAIL_CONFIG"] = mail_config
config = mail_util.setupMailer()
LOG.info("Using email configuration at %s" % mail_config)
LOG.info(config)
LOG.info("SmartPrintNG server started")
LOG.info("Temp directory: %s" % root.temp_directory)
LOG.info("Spool directory: %s" % root.spool_directory)
LOG.info("Available converters: %s" % ", ".join(zopyx.convert2.registry.availableConverters()))
if have_authentication:
LOG.info("Authentication module found - server requires authentication")
else:
LOG.info("No auth module found - serving for anonymous")
return make_app(get_root, zopyx.smartprintng.server, options=kw)
