def lambda_handler(event, context):
s3 = boto3.resource("s3")
s3_client = boto3.client("s3")
print("Script starting at %s\n" % (get_timestamp()))
to_download = clamav.update_defs_from_s3(s3_client,
AV_DEFINITION_S3_BUCKET,
AV_DEFINITION_S3_PREFIX)
for download in to_download.values():
s3_path = download["s3_path"]
local_path = download["local_path"]
print("Downloading definition file %s from s3://%s" %
(local_path, s3_path))
s3.Bucket(AV_DEFINITION_S3_BUCKET).download_file(s3_path, local_path)
print("Downloading definition file %s complete!" % (local_path))
clamav.update_defs_from_freshclam(AV_DEFINITION_PATH, CLAMAVLIB_PATH)
# If main.cvd gets updated (very rare), we will need to force freshclam
# to download the compressed version to keep file sizes down.
# The existence of main.cud is the trigger to know this has happened.
if os.path.exists(os.path.join(AV_DEFINITION_PATH, "main.cud")):
os.remove(os.path.join(AV_DEFINITION_PATH, "main.cud"))
if os.path.exists(os.path.join(AV_DEFINITION_PATH, "main.cvd")):
os.remove(os.path.join(AV_DEFINITION_PATH, "main.cvd"))
clamav.update_defs_from_freshclam(AV_DEFINITION_PATH, CLAMAVLIB_PATH)
clamav.upload_defs_to_s3(s3_client, AV_DEFINITION_S3_BUCKET,
AV_DEFINITION_S3_PREFIX, AV_DEFINITION_PATH)
print("Script finished at %s\n" % get_timestamp())
def timeouthandler(self):
if self in connections_waiting:
connections_waiting.remove(self)
if not self.command in ["get_message", "scope_message"]:
print ">>> failed, wrong connection type in queue"
self.out_buffer += self.RESPONSE_TIMEOUT % get_timestamp()
else:
self.out_buffer += NOT_FOUND % (get_timestamp(), 0, '')
self.timeout = 0
def timeouthandler(self):
if self in connections_waiting:
connections_waiting.remove(self)
if not self.command in ["get_message", "scope_message"]:
print ">>> failed, wrong connection type in queue"
self.out_buffer += self.RESPONSE_TIMEOUT % get_timestamp()
else:
self.out_buffer += NOT_FOUND % (get_timestamp(), 0, '')
self.timeout = 0
def lambda_handler(event, context):
s3 = boto3.resource("s3", endpoint_url=S3_ENDPOINT)
s3_client = boto3.client("s3", endpoint_url=S3_ENDPOINT)
sns_client = boto3.client("sns", endpoint_url=SNS_ENDPOINT)
start_clamd(s3, s3_client)
# Get some environment variables
EVENT_SOURCE = os.getenv("EVENT_SOURCE", "S3")
start_time = get_timestamp()
logging.debug("Script starting at %s\n" % (start_time))
s3_object = event_object(event, event_source=EVENT_SOURCE)
if str_to_bool(AV_PROCESS_ORIGINAL_VERSION_ONLY):
verify_s3_object_version(s3, s3_object)
# Publish the start time of the scan
if AV_SCAN_START_SNS_ARN not in [None, ""]:
start_scan_time = get_timestamp()
sns_start_scan(sns_client, s3_object, AV_SCAN_START_SNS_ARN,
start_scan_time)
with tempfile.TemporaryDirectory(prefix=EFS_SCAN_FILE_PATH) as tmpdirname:
file_path = get_local_path(s3_object, tmpdirname)
create_dir(os.path.dirname(file_path))
s3_object.download_file(file_path)
scan_result, scan_signature = clamav.scan_file(file_path)
logging.info(
"Scan of s3://%s resulted in %s\n" %
(os.path.join(s3_object.bucket_name, s3_object.key), scan_result))
result_time = get_timestamp()
# Set the properties on the object with the scan results
if "AV_UPDATE_METADATA" in os.environ:
set_av_metadata(s3_object, scan_result, scan_signature,
result_time)
set_av_tags(s3_client, s3_object, scan_result, scan_signature,
result_time)
# Publish the scan results
if AV_STATUS_SNS_ARN not in [None, ""]:
sns_scan_results(
sns_client,
s3_object,
AV_STATUS_SNS_ARN,
scan_result,
scan_signature,
result_time,
)
stop_scan_time = get_timestamp()
logging.debug("Script finished at %s\n" % stop_scan_time)
def lambda_handler(event, context):
s3 = boto3.resource("s3")
sns_client = boto3.client("sns")
# Get some environment variables
EVENT_SOURCE = os.getenv("EVENT_SOURCE", "S3")
start_time = get_timestamp()
print("Script starting at %s\n" % (start_time))
s3_object = event_object(event, event_source=EVENT_SOURCE)
if str_to_bool(AV_PROCESS_ORIGINAL_VERSION_ONLY):
verify_s3_object_version(s3, s3_object)
# Publish the start time of the scan
if AV_SCAN_START_SNS_ARN not in [None, ""]:
start_scan_time = get_timestamp()
sns_start_scan(sns_client, s3_object, AV_SCAN_START_SNS_ARN,
start_scan_time)
file_path = get_local_path(s3_object, "/tmp")
create_dir(os.path.dirname(file_path))
try:
s3_object.download_file(file_path)
except OSError as e:
remove_file(file_path)
if e.errno == 28:
print("Ran out of disk space. Scan failed")
publish_results(s3_object, AV_STATUS_FAILED,
"File too large to scan")
return
else:
raise
download_clamav_databases()
scan_result, scan_signature = clamav.scan_file(file_path)
print("Scan of s3://%s resulted in %s\n" %
(os.path.join(s3_object.bucket_name, s3_object.key), scan_result))
publish_results(s3_object, scan_result, scan_signature)
# Delete downloaded file to free up room on re-usable lambda function container
remove_file(file_path)
if str_to_bool(
AV_DELETE_INFECTED_FILES) and scan_result == AV_STATUS_INFECTED:
delete_s3_object(s3_object)
stop_scan_time = get_timestamp()
print("Script finished at %s\n" % stop_scan_time)
def publish_results(s3_object, scan_result, scan_signature):
result_time = get_timestamp()
sns_client = boto3.client("sns")
s3_client = boto3.client("s3")
ENV = os.getenv("ENV", "")
# Set the properties on the object with the scan results
if "AV_UPDATE_METADATA" in os.environ:
set_av_metadata(s3_object, scan_result, scan_signature, result_time)
set_av_tags(s3_client, s3_object, scan_result, scan_signature, result_time)
# Publish the scan results
if AV_STATUS_SNS_ARN not in [None, ""]:
sns_scan_results(
sns_client,
s3_object,
AV_STATUS_SNS_ARN,
scan_result,
scan_signature,
result_time,
)
metrics.send(env=ENV,
bucket=s3_object.bucket_name,
key=s3_object.key,
status=scan_result)
def create_insertion(jcode):
d = ordereddict.OrderedDict(map(lambda x: [x, 0.0], get_keys()))
d['InvDate'] = datetime.date.today().strftime('%d/%m/%Y')
d['InvBillingPeriod'] = period.mmmmyyyy()
d['InvJobCode'] = str(jcode)
d['InvComments'] = "PMS " + common.get_timestamp()
return d
def thread(task):
drive = client.upload_file(task)
drive.finish_time = get_timestamp()
drive.spend_time = drive.finish_time - drive.start_time
if drive.status != 1:
print_error(os.path.basename(drive.filepath) + ' 上传失败')
client.save_task(drive)
def test_sns_start_scan(self):
sns_stubber = Stubber(self.sns_client)
s3_stubber_resource = Stubber(self.s3.meta.client)
sns_arn = "some_arn"
version_id = "version-id"
timestamp = get_timestamp()
message = {
"bucket": self.s3_bucket_name,
"key": self.s3_key_name,
"version": version_id,
AV_SCAN_START_METADATA: True,
AV_TIMESTAMP_METADATA: timestamp,
}
publish_response = {"MessageId": "message_id"}
publish_expected_params = {
"TargetArn": sns_arn,
"Message": json.dumps({"default": json.dumps(message)}),
"MessageStructure": "json",
}
sns_stubber.add_response("publish", publish_response, publish_expected_params)
head_object_response = {"VersionId": version_id}
head_object_expected_params = {
"Bucket": self.s3_bucket_name,
"Key": self.s3_key_name,
}
s3_stubber_resource.add_response(
"head_object", head_object_response, head_object_expected_params
)
with sns_stubber, s3_stubber_resource:
s3_obj = self.s3.Object(self.s3_bucket_name, self.s3_key_name)
sns_start_scan(self.sns_client, s3_obj, sns_arn, timestamp)
def sns_scan_results(sns_client, s3_object, sns_arn, scan_result,
scan_signature, timestamp):
# Don't publish if scan_result is CLEAN and CLEAN results should not be published
if scan_result == AV_STATUS_CLEAN and not str_to_bool(
AV_STATUS_SNS_PUBLISH_CLEAN):
return
# Don't publish if scan_result is INFECTED and INFECTED results should not be published
if scan_result == AV_STATUS_INFECTED and not str_to_bool(
AV_STATUS_SNS_PUBLISH_INFECTED):
return
message = {
"bucket": s3_object.bucket_name,
"key": s3_object.key,
"version": s3_object.version_id,
AV_SIGNATURE_METADATA: scan_signature,
AV_STATUS_METADATA: scan_result,
AV_TIMESTAMP_METADATA: get_timestamp(),
}
sns_client.publish(
TargetArn=sns_arn,
Message=json.dumps({"default": json.dumps(message)}),
MessageStructure="json",
MessageAttributes={
AV_STATUS_METADATA: {
"DataType": "String",
"StringValue": scan_result
},
AV_SIGNATURE_METADATA: {
"DataType": "String",
"StringValue": scan_signature,
},
},
)
def __init__(self, drive_id, root_path, chunk_size=10485760):
self.status = 0
self.create_time = 0
self.start_time = common.get_timestamp()
self.finish_time = 0
self.spend_time = 0
self.drive_id = drive_id
self.root_path = root_path
self.chunk_size = chunk_size
self.filepath = None
self.realpath = None
self.filename = None
self.hash = None
self.proof_code = None
self.part_info_list = []
self.part_upload_url_list = []
self.upload_id = 0
self.file_id = 0
self.part_number = 0
self.filesize = 0
self.headers = {
'authorization': DATA['access_token'],
'content-type': 'application/json;charset=UTF-8'
}
self.id = None
def return_service_list(self, serviceList):
content = SERVICE_LIST % "".join(
[SERVICE_ITEM % service.encode('utf-8')
for service in serviceList])
self.out_buffer += self.RESPONSE_SERVICELIST % (
get_timestamp(),
len(content),
content)
def thread(task):
LOCK_TOKEN_REFRESH.acquire()
LOCK_TOKEN_REFRESH.release()
drive = client.upload_file(task)
drive.finish_time = get_timestamp()
drive.spend_time = drive.finish_time - drive.start_time
if drive.status != 1:
print_error(os.path.basename(drive.filepath) + ' 上传失败')
client.save_task(drive)
def test_web_sock_13_high_load(self):
if self.headers.get("Upgrade") == "websocket":
self.del_channel()
self.timeout = 0
TestWebSocket13HighLoad(self.socket, self.headers, self.in_buffer,
self.path)
else:
self.out_buffer += BAD_REQUEST % get_timestamp()
self.timeout = 0
def quote(symbol, flags=None):
quote = common.StockQuote(symbol)
quote.complete_time = common.get_timestamp()
quote.valid = False
return quote
def get_stp_version(self):
content = scope.get_STP_version()
self.out_buffer += RESPONSE_OK_CONTENT % (
get_timestamp(),
'',
"text/plain",
len(content),
content)
self.timeout = 0
def stp_1_channel(self):
if self.headers.get("Upgrade") == "websocket":
self.del_channel()
self.timeout = 0
STPWebSocket(self.socket, self.headers, self.in_buffer, self.path,
self.context, scope.get_scope_connection())
else:
self.out_buffer += BAD_REQUEST % get_timestamp()
self.timeout = 0
def main(home_dir, experiments_dir, subsystem_dir, telemetry_script_dir):
"""
Home directory: Where config info for experiments, etc., is
Experiments directory: Where experiment implementations are
Both should be given as absolute directories
"""
time_str = get_timestamp()
if not check_file_exists(home_dir, 'config.json'):
print('Dashboard config (config.json) is missing in {}'.format(home_dir))
return 1
dash_config = read_json(home_dir, 'config.json')
# must expand all tildes in the config to avoid future errors
for path_field in ['tmp_data_dir', 'setup_dir', 'backup_dir']:
dash_config[path_field] = os.path.expanduser(dash_config[path_field])
tmp_data_dir = os.path.join(dash_config['tmp_data_dir'], 'benchmarks_' + time_str)
data_archive = os.path.join(dash_config['tmp_data_dir'], 'benchmarks_' + time_str + '_data.tar.gz')
setup_dir = dash_config['setup_dir']
backup_archive = os.path.join(dash_config['backup_dir'], 'dashboard_' + time_str + '.tar.gz')
idemp_mkdir(tmp_data_dir)
idemp_mkdir(os.path.dirname(backup_archive))
idemp_mkdir(setup_dir)
info = DashboardInfo(home_dir)
# make a backup of the previous dashboard files if they exist
if os.path.exists(home_dir):
subprocess.call(['tar', '-zcf', backup_archive, home_dir])
# directories whose contents should not change between runs of the dashboard
persistent_dirs = {info.exp_data,
info.exp_configs,
info.subsys_configs,
info.subsys_output}
all_dashboard_dirs = info.all_experiment_dirs() + info.all_subsystem_dirs()
# instantiate necessary dashboard dirs and clean any that should be empty
for dashboard_dir in all_dashboard_dirs:
if dashboard_dir not in persistent_dirs:
subprocess.call(['rm', '-rf', dashboard_dir])
idemp_mkdir(dashboard_dir)
randomize_exps = True
if 'randomize' in dash_config:
randomize_exps = dash_config['randomize']
telemetry_rate = dash_config.get('telemetry_rate', 15)
run_cpu_telemetry = dash_config.get('run_cpu_telemetry', False)
run_gpu_telemetry = dash_config.get('run_gpu_telemetry', False)
run_all_experiments(info, experiments_dir, setup_dir,
tmp_data_dir, data_archive,
time_str, telemetry_script_dir, run_cpu_telemetry=run_cpu_telemetry, run_gpu_telemetry=run_gpu_telemetry,
telemetry_interval=telemetry_rate, randomize=randomize_exps)
run_all_subsystems(info, subsystem_dir, time_str)
def quote(symbol, flags=None):
data = common.download_webpage(JsonReader._etnet_url % symbol)
# strip JSON from reply
r = re.search(r'\[(.*)\]', data, re.S | re.U)
if not r:
quote.complete_time = common.get_timestamp()
return quote
d = json.loads(r.groups()[0])
quote = common.StockQuote(symbol)
quote.quote = d['ask']
quote.low = d['low']
quote.high = d['high']
quote.volume = d['sharestraded']
quote.opening = None
quote.close_yest = d['prvClose']
# format as isotime string
SEHKTime = d['SEHKTime'].partition(' ')
quote.quote_time = SEHKTime[0] + 'T' + SEHKTime[1]
quote.complete_time = common.get_timestamp()
quote.valid = True
return quote
def test_sns_scan_results(self):
sns_stubber = Stubber(self.sns_client)
s3_stubber_resource = Stubber(self.s3.meta.client)
sns_arn = "some_arn"
version_id = "version-id"
scan_result = "CLEAN"
scan_signature = AV_SIGNATURE_OK
timestamp = get_timestamp()
message = {
"bucket": self.s3_bucket_name,
"key": self.s3_key_name,
"version": version_id,
AV_SIGNATURE_METADATA: scan_signature,
AV_STATUS_METADATA: scan_result,
AV_TIMESTAMP_METADATA: timestamp,
}
publish_response = {"MessageId": "message_id"}
publish_expected_params = {
"TargetArn": sns_arn,
"Message": json.dumps({"default": json.dumps(message)}),
"MessageAttributes": {
"av-status": {
"DataType": "String",
"StringValue": scan_result
},
"av-signature": {
"DataType": "String",
"StringValue": scan_signature
},
"bucket": {
"DataType": "String",
"StringValue": self.s3_bucket_name
},
"key": {
"DataType": "String",
"StringValue": self.s3_key_name
},
},
"MessageStructure": "json",
}
sns_stubber.add_response("publish", publish_response,
publish_expected_params)
head_object_response = {"VersionId": version_id}
head_object_expected_params = {
"Bucket": self.s3_bucket_name,
"Key": self.s3_key_name,
}
s3_stubber_resource.add_response("head_object", head_object_response,
head_object_expected_params)
with sns_stubber, s3_stubber_resource:
s3_obj = self.s3.Object(self.s3_bucket_name, self.s3_key_name)
sns_scan_results(self.sns_client, s3_obj, sns_arn, scan_result,
scan_signature, timestamp)
def lambda_handler(event, context):
s3 = boto3.resource("s3")
s3_client = boto3.client("s3")
start_time = datetime.utcnow()
print("Script starting at %s\n" %
(start_time.strftime("%Y/%m/%d %H:%M:%S UTC")))
shutil.rmtree(AV_DEFINITION_PATH, ignore_errors=True)
os.mkdir(AV_DEFINITION_PATH)
to_download = clamav.update_defs_from_s3(s3_client,
AV_DEFINITION_S3_BUCKET,
AV_DEFINITION_S3_PREFIX)
print("Skipping clamav definition download %s\n" % (get_timestamp()))
# for download in to_download.values():
# s3_path = download["s3_path"]
# local_path = download["local_path"]
# print("Downloading definition file %s from s3://%s" % (local_path, s3_path))
# s3.Bucket(AV_DEFINITION_S3_BUCKET).download_file(s3_path, local_path)
# print("Downloading definition file %s complete!" % (local_path))
retVal = clamav.update_defs_from_freshclam(AV_DEFINITION_PATH,
CLAMAVLIB_PATH)
if retVal != 0:
raise RuntimeError("clamAV update process returned %d" % (retVal))
# If main.cvd gets updated (very rare), we will need to force freshclam
# to download the compressed version to keep file sizes down.
# The existence of main.cud is the trigger to know this has happened.
if os.path.exists(os.path.join(AV_DEFINITION_PATH, "main.cud")):
os.remove(os.path.join(AV_DEFINITION_PATH, "main.cud"))
if os.path.exists(os.path.join(AV_DEFINITION_PATH, "main.cvd")):
os.remove(os.path.join(AV_DEFINITION_PATH, "main.cvd"))
retVal = clamav.update_defs_from_freshclam(AV_DEFINITION_PATH,
CLAMAVLIB_PATH)
if retVal != 0:
raise RuntimeError("Refresh clamAV update process returned %d" %
(retVal))
clamav.upload_defs_to_s3(s3_client, AV_DEFINITION_S3_BUCKET,
AV_DEFINITION_S3_PREFIX, AV_DEFINITION_PATH)
print("Script finished at %s\n" % get_timestamp())
def fix_missing_entries(scrip):
c.pr("I","Fixing Missing Entries For Scrip "+scrip,1)
uniq_dates = s.sql_array("SELECT DISTINCT CAST(`time` AS DATE) AS dateonly FROM `"+scrip+"`","dateonly")
for date in uniq_dates:
dp_req = fetch_dp_req(str(date),scrip)
db_dp = s.sql_hash(scrip,"timestamp","close","WHERE `time` BETWEEN '"+str(date)+" 09:16:00' AND '"+str(date)+" 15:30:00'")
dp_cur = len(db_dp)
dp_mis = (dp_req - dp_cur)
dp_map = {}
if dp_mis > 1:
c.pr("I","DATE --> "+str(date)+" DP REQ --> "+str(dp_req)+" DP CUR --> "+str(dp_cur)+" DP MIS --> "+str(dp_mis),1)
#Here We attempt to fix DP
dp_min = int(c.get_timestamp(str(date)+" 09:16:00"))
dp_max = int(c.get_timestamp(str(date)+" 15:30:00"))
#c.pr("I","DP MIN ---> "+str(dp_min)+" DP MAX ---> "+str(dp_max),1)
dp_chk = dp_min
ctr = 1
dp_last = 0
while dp_chk != (dp_max+60):
if not str(dp_chk) in db_dp:
#If MIN AND CHK Are Same
if dp_chk == dp_min:
c.pr("I",str(dp_chk)+" ---> MIN MISSING",1)
#exit()
else:
if str((dp_chk - 60)) in db_dp:
#Case Where Previous Data point exists
dp_prev = db_dp[str((dp_chk - 60))]['close']
#print(str(dp_chk)+" ---> PREV PRESENT"+" DP PREV ---> "+str(dp_prev))
dp_map[str(dp_chk)] = process_missing(dp_prev,dp_chk)
else:
#print(str(dp_chk)+" ---> PREV MISSISNG"+" DP PREV ---> "+str(dp_last))
if dp_last:
dp_prev = db_dp[str(dp_last)]['close']
dp_map[str(dp_chk)] = process_missing(dp_prev,dp_chk)
#print(str(dp_chk)+" ---> PREV MISSISNG"+" DP PREV ---> "+str(dp_prev))
else:
dp_last = dp_chk
dp_chk = (dp_chk+60)
if len(dp_map):
store_data(dp_map,scrip)
return
def ping_recv(self, msg):
"""Parse ping (without flag) and send back when necessary."""
seq = int(msg[0])
if seq == 0:
raw_packet = "1" + "1" + msg[1:] + get_timestamp()
to_write = self.cipher.encrypt(raw_packet) + self.split
self.send(to_write)
else:
time1 = parse_timestamp(msg[1:])
self.latency = int(time.time() * 1000) - time1
logging.debug("latency: %dms" % self.latency)
def ping_recv(self, msg):
"""Parse ping (without flag) and send back when necessary."""
seq = int(msg[0])
if seq == 0:
raw_packet = "1" + "1" + msg[1:] + get_timestamp()
to_write = self.cipher.encrypt(raw_packet) + self.split
self.send(to_write)
else:
time1 = parse_timestamp(msg[1:])
self.latency = int(time.time() * 1000) - time1
logging.debug("latency: %dms" % self.latency)
def print_config_info(self):
s = ''
for k in DATA['config'].keys():
s += "\n\t\t%s：%s" % (k, DATA['config'][k])
content = '''=================================================
阿里云盘上传工具启动成功
当前时间：%s%s
=================================================
''' % (date(get_timestamp()), s)
self.print(content, 'info')
def return_scope_message_STP_0(self, msg, sender):
""" return a message to the client"""
service, payload = msg
if self.debug:
pretty_print_XML("\nsend to client: %s" % service, payload,
self.debug_format)
self.out_buffer += self.SCOPE_MESSAGE_STP_0 % (
get_timestamp(), service, len(payload), payload)
self.timeout = 0
if not sender == self:
self.handle_write()
def test_web_sock_13(self):
if self.headers.get("Upgrade") == "websocket":
self.del_channel()
self.timeout = 0
TestWebSocket13(self.socket,
self.headers,
self.in_buffer,
self.path)
else:
self.out_buffer += BAD_REQUEST % get_timestamp()
self.timeout = 0
def execute(self):
for symbol in self.symbols:
redisdb.hmset(symbol, self.__get_quote(symbol))
redisdb.delete('WORKER-SYMBOLS')
redisdb.sadd('WORKER-SYMBOLS', *self.symbols)
redisdb.hmset('WORKER-LASTEXEC',
{
'time': common.get_timestamp(),
'QUOTE_PROVIDER': self.__quote_provider.__class__.__name__,
'DETAIL_PROVIDER': self.__detail_provider.__class__.__name__,
}
)
def snapshot(self):
"""store a markup snapshot"""
raw_data = self.raw_post_data
if raw_data:
name, data = raw_data.split(CRLF, 1)
f = open(name + ".xml", 'wb')
# f.write(pretty_dragonfly_snapshot(data))
data = data.replace("'=\"\"", "")
data = re.sub(r'<script(?:[^/>]|/[^>])*/>[ \r\n]*', '', data)
f.write(data.replace("'=\"\"", ""))
f.close()
self.out_buffer += self.RESPONSE_OK_OK % get_timestamp()
self.timeout = 0
def print_config_info(self):
s = ''
config__keys = DATA['config'].keys()
for k in config__keys:
if k in ['REFRESH_TOKEN', 'DRIVE_ID']: continue
s += "\n\t\t%s：%s" % (k, DATA['config'][k])
content = '''=================================================
阿里云盘上传工具启动成功
当前时间：%s%s
=================================================
''' % (date(get_timestamp()), s)
self.print(content, 'info')
def snapshot(self):
"""store a markup snapshot"""
raw_data = self.raw_post_data
if raw_data:
name, data = raw_data.split(CRLF, 1)
f = open(name + ".xml", 'wb')
# f.write(pretty_dragonfly_snapshot(data))
data = data.replace("'=\"\"", "")
data = re.sub(r'<script(?:[^/>]|/[^>])*/>[ \r\n]*', '', data)
f.write(data.replace("'=\"\"", ""))
f.close()
self.out_buffer += self.RESPONSE_OK_OK % get_timestamp()
self.timeout = 0
def stp_1_channel(self):
if self.headers.get("Upgrade") == "websocket":
self.del_channel()
self.timeout = 0
STPWebSocket(self.socket,
self.headers,
self.in_buffer,
self.path,
self.context,
scope.get_scope_connection())
else:
self.out_buffer += BAD_REQUEST % get_timestamp()
self.timeout = 0
def return_scope_message_STP_0(self, msg, sender):
""" return a message to the client"""
service, payload = msg
if self.debug:
pretty_print_XML("\nsend to client: %s" % service, payload, self.debug_format)
self.out_buffer += self.SCOPE_MESSAGE_STP_0 % (
get_timestamp(),
service,
len(payload),
payload)
self.timeout = 0
if not sender == self:
self.handle_write()
def savefile(self):
"""save file"""
raw_data = self.raw_post_data
file_name = self.arguments[0]
print file_name
if not os.path.exists("screenshots"):
os.mkdir("screenshots")
if raw_data:
f = open(os.path.join("screenshots", file_name), 'wb')
f.write(raw_data)
f.close()
self.out_buffer += self.RESPONSE_OK_OK % get_timestamp()
self.timeout = 0
def enable(self):
"""to enable a scope service"""
service = self.arguments[0]
if scope.services_enabled[service]:
print ">>> service is already enabled", service
else:
scope.send_command("*enable %s" % service)
scope.services_enabled[service] = True
if service.startswith('stp-'):
scope.set_STP_version(service)
self.out_buffer += self.RESPONSE_OK_OK % get_timestamp()
self.timeout = 0
def get_output_filename():
print "Specify a %s where to save the soundcard output %s." % ( color('filename','green'), color('(without .ogg)','green') )
print "You can also press %s to save the output %s." % ( color('ENTER','green'), color('to a temp. file','green') )
filename = raw_input( color("Filename: ", 'yellow', ['bold']) )
if len(filename) == 0:
filename = os.path.join( TMP_DIR, "record_%s.ogg" % common.get_timestamp() )
else:
filename = os.path.join(TMP_DIR, filename + ".ogg")
if os.path.exists(filename):
sys.stderr.write( "%s: Error: the file %s already exists.\n" % (sys.argv[0], filename) )
sys.exit(-2)
return filename
def savefile(self):
"""save file"""
raw_data = self.raw_post_data
file_name = self.arguments[0]
print file_name
if not os.path.exists("screenshots"):
os.mkdir("screenshots")
if raw_data:
f = open(os.path.join("screenshots", file_name), 'wb')
f.write(raw_data)
f.close()
self.out_buffer += self.RESPONSE_OK_OK % get_timestamp()
self.timeout = 0
def enable(self):
"""to enable a scope service"""
service = self.arguments[0]
if scope.services_enabled[service]:
print ">>> service is already enabled", service
else:
scope.send_command("*enable %s" % service)
scope.services_enabled[service] = True
if service.startswith('stp-'):
scope.set_STP_version(service)
self.out_buffer += self.RESPONSE_OK_OK % get_timestamp()
self.timeout = 0
def test_set_av_metadata(self):
scan_result = "CLEAN"
scan_signature = AV_SIGNATURE_OK
timestamp = get_timestamp()
s3_obj = self.s3.Object(self.s3_bucket_name, self.s3_key_name)
s3_stubber_resource = Stubber(self.s3.meta.client)
# First head call is done to get content type and meta data
head_object_response = {"ContentType": "content", "Metadata": {}}
head_object_expected_params = {
"Bucket": self.s3_bucket_name,
"Key": self.s3_key_name,
}
s3_stubber_resource.add_response("head_object", head_object_response,
head_object_expected_params)
# Next two calls are done when copy() is called
head_object_response_2 = {
"ContentType": "content",
"Metadata": {},
"ContentLength": 200,
}
head_object_expected_params_2 = {
"Bucket": self.s3_bucket_name,
"Key": self.s3_key_name,
}
s3_stubber_resource.add_response("head_object", head_object_response_2,
head_object_expected_params_2)
copy_object_response = {"VersionId": "version_id"}
copy_object_expected_params = {
"Bucket": self.s3_bucket_name,
"Key": self.s3_key_name,
"ContentType": "content",
"CopySource": {
"Bucket": self.s3_bucket_name,
"Key": self.s3_key_name
},
"Metadata": {
AV_SIGNATURE_METADATA: scan_signature,
AV_STATUS_METADATA: scan_result,
AV_TIMESTAMP_METADATA: timestamp,
},
"MetadataDirective": "REPLACE",
}
s3_stubber_resource.add_response("copy_object", copy_object_response,
copy_object_expected_params)
with s3_stubber_resource:
set_av_metadata(s3_obj, scan_result, scan_signature, timestamp)
def sns_scan_results(sns_client, s3_object, sns_arn, scan_result,
scan_signature, timestamp):
# Don't publish if scan_result is CLEAN and CLEAN results should not be published
if scan_result == AV_STATUS_CLEAN and not str_to_bool(
AV_STATUS_SNS_PUBLISH_CLEAN):
return
# Don't publish if scan_result is INFECTED and INFECTED results should not be published
if scan_result == AV_STATUS_INFECTED and not str_to_bool(
AV_STATUS_SNS_PUBLISH_INFECTED):
return
message = {
"bucket": s3_object.bucket_name,
"key": s3_object.key,
"version": s3_object.version_id,
AV_SIGNATURE_METADATA: scan_signature,
AV_STATUS_METADATA: scan_result,
AV_TIMESTAMP_METADATA: get_timestamp(),
}
sns_client.publish(
TargetArn=sns_arn,
Message=json.dumps({"default": json.dumps(message)}),
MessageStructure="json",
MessageAttributes={
AV_STATUS_METADATA: {
"DataType": "String",
"StringValue": scan_result
},
AV_SIGNATURE_METADATA: {
"DataType": "String",
"StringValue": scan_signature,
},
},
)
response = requests.post(
url=
"https://hooks.slack.com/services/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",
headers={
"Content-Type": "application/json; charset=utf-8",
},
data=json.dumps({
"username": "******",
"icon_emoji": ":ghost:",
"channel": "#symview",
"text": json.dumps(message)
}))
print('Response HTTP Status Code: {status_code}'.format(
status_code=response.status_code))
print('Response HTTP Response Body: {content}'.format(
content=response.content))
def test_set_av_tags(self):
scan_result = "CLEAN"
scan_signature = AV_SIGNATURE_OK
timestamp = get_timestamp()
tag_set = {
"TagSet": [
{
"Key": AV_SIGNATURE_METADATA,
"Value": scan_signature
},
{
"Key": AV_STATUS_METADATA,
"Value": scan_result
},
{
"Key": AV_TIMESTAMP_METADATA,
"Value": timestamp
},
]
}
s3_stubber = Stubber(self.s3_client)
get_object_tagging_response = tag_set
get_object_tagging_expected_params = {
"Bucket": self.s3_bucket_name,
"Key": self.s3_key_name,
}
s3_stubber.add_response(
"get_object_tagging",
get_object_tagging_response,
get_object_tagging_expected_params,
)
put_object_tagging_response = {}
put_object_tagging_expected_params = {
"Bucket": self.s3_bucket_name,
"Key": self.s3_key_name,
"Tagging": tag_set,
}
s3_stubber.add_response(
"put_object_tagging",
put_object_tagging_response,
put_object_tagging_expected_params,
)
with s3_stubber:
s3_obj = self.s3.Object(self.s3_bucket_name, self.s3_key_name)
set_av_tags(self.s3_client, s3_obj, scan_result, scan_signature,
timestamp)
def main(config_dir, home_dir, output_dir):
info = DashboardInfo(home_dir)
conf = read_config(config_dir)
data_dir = os.path.join(output_dir, 'data')
graph_dir = os.path.join(output_dir, 'graphs')
idemp_mkdir(data_dir)
idemp_mkdir(graph_dir)
timestamp = get_timestamp()
score_confs = conf['score_confs']
metrics = set(score_confs.keys())
metrics = metrics.intersection(set(SCORE_METRICS.keys()))
if not metrics:
write_status(output_dir, True, 'No scores to report')
return 0
score_data = {}
score_reports = {}
for metric in metrics:
score_metric = SCORE_METRICS[metric](score_confs[metric])
valid, msg = check_prerequisites(info, score_metric.prereq())
if not valid:
write_status(output_dir, False, msg)
return 1
score_data_dir = os.path.join(data_dir, metric)
score_graph_dir = os.path.join(graph_dir, metric)
idemp_mkdir(score_data_dir)
idemp_mkdir(score_graph_dir)
try:
report = process_score(info, score_metric, score_data_dir,
score_graph_dir, timestamp)
score_reports[metric] = report
except Exception as e:
write_status(
output_dir, False,
'Encountered exception while scoring {}:\n{}'.format(
metric, render_exception(e)))
return 1
report = {'title': 'Metric Scores', 'value': format_scores(score_reports)}
write_json(output_dir, 'report.json', report)
write_status(output_dir, True, 'success')
def upload_file(self, task):
save_task(task['id'], {'status': 2})
drive = AliyunDrive(DATA['config']['DRIVE_ID'],
DATA['config']['ROOT_PATH'],
DATA['config']['CHUNK_SIZE'])
# 加载任务队列
drive.load_task(task)
# 刷新token
if not os.path.exists(task['realpath']):
drive.status = -1
return drive
drive.load_file(task['filepath'], task['realpath'])
# 创建目录
LOCK.acquire()
try:
parent_folder_id = drive.get_parent_folder_id(drive.filepath)
finally:
LOCK.release()
# 断点续传
if DATA['config']['RESUME'] and DATA['config']['DRIVE_ID'] == task[
'drive_id']:
if 0 not in [
drive.drive_id,
drive.part_number,
drive.chunk_size,
] and not drive.file_id and not drive.upload_id:
# 获取上传地址
drive.part_upload_url_list = drive.get_upload_url()
# 上传
return self.__upload(drive)
# 创建上传
create_post_json = drive.create(parent_folder_id)
if 'rapid_upload' in create_post_json and create_post_json[
'rapid_upload']:
drive.finish_time = get_timestamp()
drive.spend_time = drive.finish_time - drive.start_time
self.print(
'【{filename}】秒传成功！消耗{s}秒'.format(filename=drive.filename,
s=drive.spend_time),
'success', drive.id)
drive.status = 1
return drive
# 上传
return self.__upload(drive)
def annotation(self, job):
'Return some RTF text for non-vanilla jobs'
self.para(3)
#self.add('Approved: ______________________________________')
#self.add(approver)
self.setFontSize(12)
self.add(common.get_timestamp())
annotations = []
if job['Weird']: annotations.append('Unorthodox')
if job['WIP']: annotations.append('WIP')
if len(annotations) > 0:
self.para(2)
text = 'Ann: ' + ' '.join(annotations)
self.add(text)
self.setFontSize(18)
def post_command(self):
"""send a command to scope"""
raw_data = self.raw_post_data
is_ok = False
if scope.version == "stp-1":
args = self.arguments
"""
message type: 1 = command, 2 = response, 3 = event, 4 = error
message TransportMessage
{
required string service = 1;
required uint32 commandID = 2;
required uint32 format = 3;
optional uint32 status = 4;
optional uint32 tag = 5;
required binary payload = 8;
}
/send-command/" + service + "/" + command_id + "/" + tag
"""
if self.is_timing:
command_times[args[2]] = (args[0], args[1], time() * 1000)
scope.send_command({
0: 1, # message type
1: args[0],
2: int(args[1]),
3: 1,
5: int(args[2]),
8: self.raw_post_data,
})
is_ok = True
else:
service = self.arguments[0]
if service in scope.services_enabled:
if not raw_data.startswith("<?xml") and \
not raw_data.startswith("STP/1"):
raw_data = XML_PRELUDE % raw_data
msg = "%s %s" % (service, raw_data.decode('UTF-8'))
scope.send_command(msg)
is_ok = True
else:
print "tried to send a command before %s was enabled" % service
self.out_buffer += (is_ok and
self.RESPONSE_OK_OK or
BAD_REQUEST) % get_timestamp()
self.timeout = 0
def return_scope_message_STP_1(self, msg, sender):
""" return a message to the client
message TransportMessage
{
required string service = 1;
required uint32 commandID = 2;
required uint32 format = 3;
optional uint32 status = 4;
optional uint32 tag = 5;
required binary payload = 8;
}
"""
if not msg[8]:
# workaround, status 204 does not work
msg[8] = ' '
if self.debug and (not self.debug_only_errors or msg[4] == MSG_TYPE_ERROR):
pretty_print("send to client:", msg,
self.debug_format, self.debug_format_payload, self.verbose_debug)
if self.is_timing:
tag = str(msg[5])
if tag in command_times:
item = command_times.pop(tag)
print item[0],
print MessageMap.get_cmd_name(item[0], item[1]),
print time() * 1000 - item[2]
self.out_buffer += self.SCOPE_MESSAGE_STP_1 % (
get_timestamp(),
msg[1], # service
msg[2], # command
msg[4], # status
msg[5], # tag
len(msg[8]),
msg[8], # payload
)
self.timeout = 0
if not sender == self:
self.handle_write()
def get_description(self, headers):
content = DEVICE_DESCRIPTION % (self.ip, self.stp_port, self.ip, self.http_port)
args = (common.get_timestamp(), "", "text/xml", len(content), content)
return common.RESPONSE_OK_CONTENT % args
def _is_die(self, worker):
timestamp = common.get_timestamp()
return timestamp - worker.get_heartbeat() > self.conf.DIE_THRESHOLD
