def close(self):
def get_report_path(report_id):
return os.path.join(config.main.report_dir, report_id)
report_filename = get_report_path(self.report_id)
try:
with open(report_filename) as fd:
g = yaml.safe_load_all(fd)
report_details = g.next()
except IOError:
raise ReportNotFound
timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time']))
dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format(
timestamp=timestamp,
**report_details)
dst_path = os.path.join(config.main.archive_dir,
report_details['probe_cc'])
if not os.path.isdir(dst_path):
os.mkdir(dst_path)
dst_path = os.path.join(dst_path, dst_filename)
os.rename(report_filename, dst_path)
del config.reports[self.report_id]
def report_file_name(report_details):
timestamp = otime.timestamp(
datetime.fromtimestamp(
report_details['start_time']))
dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format(
timestamp=timestamp,
**report_details)
return dst_filename
def report_file_name(archive_dir, report_details):
timestamp = datetime.fromtimestamp(report_details['start_time'])
keys = dict(report_details.items(),
iso8601_timestamp=otime.timestamp(timestamp),
year=timestamp.strftime("%Y"),
month=timestamp.strftime("%m"),
day=timestamp.strftime("%d"),
hour=timestamp.strftime("%H"),
minute=timestamp.strftime("%M"),
second=timestamp.strftime("%S"))
report_file_template = "{probe_cc}/{test_name}-{iso8601_timestamp}-{probe_asn}-probe.yamloo"
if config.main.report_file_template:
report_file_template = config.main.report_file_template
dst_filename = os.path.join(archive_dir,
report_file_template.format(**keys))
dst_dir = os.path.dirname(dst_filename)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
return dst_filename
def report_file_name(archive_dir, report_details):
timestamp = datetime.fromtimestamp(report_details['start_time'])
keys = dict(
report_details.items(),
iso8601_timestamp=otime.timestamp(timestamp),
year=timestamp.strftime("%Y"),
month=timestamp.strftime("%m"),
day=timestamp.strftime("%d"),
hour=timestamp.strftime("%H"),
minute=timestamp.strftime("%M"),
second=timestamp.strftime("%S")
)
report_file_template = "{probe_cc}/{test_name}-{iso8601_timestamp}-{probe_asn}-probe.yamloo"
if config.main.report_file_template:
report_file_template = config.main.report_file_template
dst_filename = os.path.join(archive_dir, report_file_template.format(**keys))
dst_dir = os.path.dirname(dst_filename)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
return dst_filename
def close_report(report_id):
report_filename = get_report_path(report_id)
try:
with open(report_filename) as fd:
yaml_data = ''.join(fd.readline() for _ in range(12))
report_details = yaml.safe_load(yaml_data)
except IOError:
raise ReportNotFound
timestamp = otime.timestamp(
datetime.fromtimestamp(report_details['start_time']))
dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format(
timestamp=timestamp, **report_details)
dst_path = os.path.join(config.main.archive_dir,
report_details['probe_cc'])
if not os.path.isdir(dst_path):
os.mkdir(dst_path)
dst_path = os.path.join(dst_path, dst_filename)
os.rename(report_filename, dst_path)
def close_report(report_id):
report_filename = get_report_path(report_id)
try:
with open(report_filename) as fd:
yaml_data = ''.join(fd.readline() for _ in range(12))
report_details = yaml.safe_load(yaml_data)
except IOError:
raise ReportNotFound
timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time']))
dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format(
timestamp=timestamp,
**report_details)
dst_path = os.path.join(config.main.archive_dir,
report_details['probe_cc'])
if not os.path.isdir(dst_path):
os.mkdir(dst_path)
dst_path = os.path.join(dst_path, dst_filename)
os.rename(report_filename, dst_path)
def report_file_path(archive_dir, report_details,
report_id='no_report_id'):
if report_details.get("start_time"):
timestamp = datetime.fromtimestamp(report_details['start_time'])
elif report_details.get("test_start_time"):
timestamp = datetime.strptime(report_details['test_start_time'], "%Y-%m-%d %H:%M:%S")
else:
raise Exception("Could not find valid timestamp")
if report_details['format'] == 'json':
ext = 'json'
elif report_details['format'] == 'yaml':
ext = 'yaml'
else:
ext = 'invalid'
keys = dict(
report_details.items(),
iso8601_timestamp=otime.timestamp(timestamp),
year=timestamp.strftime("%Y"),
month=timestamp.strftime("%m"),
day=timestamp.strftime("%d"),
hour=timestamp.strftime("%H"),
minute=timestamp.strftime("%M"),
second=timestamp.strftime("%S"),
ext=ext,
report_id=report_id
)
report_file_template = "{iso8601_timestamp}-{test_name}-{report_id}-{probe_asn}-{probe_cc}-probe-0.2.0.{ext}"
if config.main.get('report_file_template'):
report_file_template = config.main['report_file_template']
dst_path = archive_dir.child(report_file_template.format(**keys))
try:
FilePath(dst_path.dirname()).makedirs()
except OSError as e:
if not e.errno == errno.EEXIST:
raise
return dst_path
print "Parsed this data %s" % report_data
software_name = report_data['software_name']
software_version = report_data['software_version']
test_name = report_data['test_name']
test_version = report_data['test_version']
probe_asn = report_data['probe_asn']
content = yaml.safe_load(report_data['content'])
content['backend_version'] = config.backend_version
serialized_content = yaml.dump(content)
content = "---\n" + serialized_content + '...\n'
if not probe_asn:
probe_asn = "AS0"
report_id = otime.timestamp() + '_' \
+ probe_asn + '_' \
+ randomStr(50)
# The report filename contains the timestamp of the report plus a
# random nonce
report_filename = os.path.join(config.main.report_dir, report_id)
response = {
'backend_version': config.backend_version,
'report_id': report_id
}
config.reports[report_id] = time.time()
reactor.callLater(config.main.stale_time, stale_check, report_id)
def post(self):
"""
Creates a new report with the input
* Request
{'software_name': 'XXX',
'software_version': 'XXX',
'test_name': 'XXX',
'test_version': 'XXX',
'probe_asn': 'XXX'
'content': 'XXX'
}
Optional:
'test_helper': 'XXX'
'client_ip': 'XXX'
(not implemented, nor in client, nor in backend)
The idea behind these two fields is that it would be interesting to
also collect how the request was observed from the collectors point
of view.
We use as a unique key the client_ip address and a time window. We
then need to tell the test_helper that is selected the client_ip
address and tell it to expect a connection from a probe in that time
window.
Once the test_helper sees a connection from that client_ip it will
store for the testing session the data that it receives.
When the probe completes the report (or the time window is over) the
final report will include also the data collected from the
collectors view point.
* Response
{'backend_version': 'XXX', 'report_id': 'XXX'}
"""
# Note: the request is being validated inside of parseNewReportRequest.
report_data = parseNewReportRequest(self.request.body)
log.debug("Parsed this data %s" % report_data)
software_name = str(report_data['software_name'])
software_version = str(report_data['software_version'])
probe_asn = str(report_data['probe_asn'])
probe_cc = str(report_data.get('probe_cc', 'ZZ'))
self.testName = str(report_data['test_name'])
self.testVersion = str(report_data['test_version'])
if self.policy_file:
try:
self.inputHashes = report_data['input_hashes']
except KeyError:
raise e.InputHashNotProvided
self.checkPolicy()
if 'content' in report_data:
content = yaml.safe_load(report_data['content'])
report_header = validate_report_header(content)
else:
content = {
'software_name': software_name,
'software_version': software_version,
'probe_asn': probe_asn,
'probe_cc': probe_cc,
'test_name': self.testName,
'test_version': self.testVersion,
'input_hashes': self.inputHashes,
'start_time': time.time()
}
content['backend_version'] = config.backend_version
report_header = yaml.dump(content)
content = "---\n" + report_header + '...\n'
if not probe_asn:
probe_asn = "AS0"
report_id = otime.timestamp() + '_' \
+ probe_asn + '_' \
+ randomStr(50)
# The report filename contains the timestamp of the report plus a
# random nonce
report_filename = os.path.join(self.report_dir, report_id)
response = {
'backend_version': config.backend_version,
'report_id': report_id
}
requested_helper = report_data.get('test_helper')
if requested_helper:
try:
response['test_helper_address'] = self.helpers[
requested_helper].address
except KeyError:
raise e.TestHelperNotFound
self.reports[report_id] = Report(report_id, self.stale_time,
self.report_dir, self.archive_dir,
self.reports)
self.writeToReport(report_filename, content)
self.write(response)
'probe_cc': probe_cc,
'test_name': self.testName,
'test_version': self.testVersion,
'input_hashes': self.inputHashes,
'start_time': time.time()
}
content['backend_version'] = config.backend_version
report_header = yaml.dump(content)
content = "---\n" + report_header + '...\n'
if not probe_asn:
probe_asn = "AS0"
report_id = otime.timestamp() + '_' \
+ probe_asn + '_' \
+ randomStr(50)
# The report filename contains the timestamp of the report plus a
# random nonce
report_filename = os.path.join(config.main.report_dir, report_id)
response = {
'backend_version': config.backend_version,
'report_id': report_id
}
try:
requested_helper = report_data['test_helper']
except KeyError:
def post(self):
"""
Creates a new report with the input
* Request
{'software_name': 'XXX',
'software_version': 'XXX',
'test_name': 'XXX',
'test_version': 'XXX',
'probe_asn': 'XXX'
'content': 'XXX'
}
Optional:
'test_helper': 'XXX'
'client_ip': 'XXX'
(not implemented, nor in client, nor in backend)
The idea behind these two fields is that it would be interesting to
also collect how the request was observed from the collectors point
of view.
We use as a unique key the client_ip address and a time window. We
then need to tell the test_helper that is selected the client_ip
address and tell it to expect a connection from a probe in that time
window.
Once the test_helper sees a connection from that client_ip it will
store for the testing session the data that it receives.
When the probe completes the report (or the time window is over) the
final report will include also the data collected from the
collectors view point.
* Response
{'backend_version': 'XXX', 'report_id': 'XXX'}
"""
# Note: the request is being validated inside of parseNewReportRequest.
report_data = parseNewReportRequest(self.request.body)
log.debug("Parsed this data %s" % report_data)
software_name = str(report_data['software_name'])
software_version = str(report_data['software_version'])
probe_asn = str(report_data['probe_asn'])
probe_cc = str(report_data.get('probe_cc', 'ZZ'))
self.testName = str(report_data['test_name'])
self.testVersion = str(report_data['test_version'])
if self.policy_file:
try:
self.inputHashes = report_data['input_hashes']
except KeyError:
raise e.InputHashNotProvided
self.checkPolicy()
if 'content' in report_data:
content = yaml.safe_load(report_data['content'])
report_header = validate_report_header(content)
else:
content = {
'software_name': software_name,
'software_version': software_version,
'probe_asn': probe_asn,
'probe_cc': probe_cc,
'test_name': self.testName,
'test_version': self.testVersion,
'input_hashes': self.inputHashes,
'start_time': time.time()
}
content['backend_version'] = config.backend_version
report_header = yaml.dump(content)
content = "---\n" + report_header + '...\n'
if not probe_asn:
probe_asn = "AS0"
report_id = otime.timestamp() + '_' \
+ probe_asn + '_' \
+ randomStr(50)
# The report filename contains the timestamp of the report plus a
# random nonce
report_filename = os.path.join(self.report_dir, report_id)
response = {
'backend_version': config.backend_version,
'report_id': report_id
}
requested_helper = report_data.get('test_helper')
if requested_helper:
try:
response['test_helper_address'] = self.helpers[
requested_helper].address
except KeyError:
raise e.TestHelperNotFound
self.reports[report_id] = Report(report_id,
self.stale_time,
self.report_dir,
self.archive_dir,
self.reports)
self.writeToReport(report_filename, content)
self.write(response)
def post(self):
"""
Creates a new report with the input
* Request
{'software_name': 'XXX',
'software_version': 'XXX',
'test_name': 'XXX',
'test_version': 'XXX',
'probe_asn': 'XXX'
'content': 'XXX'
}
Optional:
'test_helper': 'XXX'
'client_ip': 'XXX'
(not implemented, nor in client, nor in backend)
The idea behind these two fields is that it would be interesting to
also collect how the request was observed from the collectors point
of view.
We use as a unique key the client_ip address and a time window. We
then need to tell the test_helper that is selected the client_ip
address and tell it to expect a connection from a probe in that time
window.
Once the test_helper sees a connection from that client_ip it will
store for the testing session the data that it receives.
When the probe completes the report (or the time window is over) the
final report will include also the data collected from the
collectors view point.
* Response
{
'backend_version': 'XXX',
'report_id': 'XXX',
'supported_formats': ['yaml', 'json']
}
"""
# Note: the request is being validated inside of parseNewReportRequest.
report_data = parseNewReportRequest(self.request.body)
log.debug("Parsed this data %s" % report_data)
self.testName = str(report_data['test_name'])
self.testVersion = str(report_data['test_version'])
if self.policy_file:
try:
self.inputHashes = report_data['input_hashes']
except KeyError:
raise e.InputHashNotProvided
self.checkPolicy()
data = None
if report_data['format'] == 'yaml' and 'content' not in report_data:
content = {
'software_name': str(report_data['software_name']),
'software_version': str(report_data['software_version']),
'probe_asn': str(report_data['probe_asn']),
'probe_cc': str(report_data['probe_cc']),
'test_name': self.testName,
'test_version': self.testVersion,
'input_hashes': report_data.get('input_hashes', []),
'test_start_time': str(report_data['test_start_time']),
'data_format_version': str(report_data.get('data_format_version', '0.1.0'))
}
data = "---\n" + yaml.dump(content) + "...\n"
elif report_data['format'] == 'yaml' and 'content' in report_data:
header = yaml.safe_load(report_data['content'])
data = "---\n" + yaml.dump(validateHeader(header)) + "...\n"
report_id = otime.timestamp() + '_' \
+ report_data.get('probe_asn', 'AS0') + '_' \
+ randomStr(50)
# The report filename contains the timestamp of the report plus a
# random nonce
report_path = self.report_dir.child(report_id)
# We use this file to store the metadata associated with the report
# submission.
report_metadata_path = self.report_dir.child(report_id + METADATA_EXT)
response = {
'backend_version': config.backend_version,
'report_id': report_id,
'supported_formats': ['yaml', 'json']
}
requested_helper = report_data.get('test_helper')
if requested_helper:
try:
response['test_helper_address'] = self.helpers[
requested_helper].address
except KeyError:
raise e.TestHelperNotFound
with report_metadata_path.open('w') as f:
f.write(json_dumps(report_data))
f.write("\n")
report_path.touch()
if data is not None:
with report_path.open('w') as f:
f.write(data)
self.write(response)
