def test_isotime_iso():
iso_date = now_as_iso()
iso_format = re.compile(r'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z')
assert isinstance(iso_date, str)
assert iso_format.match(iso_date)
assert epoch_to_iso(iso_to_epoch(iso_date)) == iso_date
assert iso_date == epoch_to_iso(local_to_epoch(epoch_to_local(iso_to_epoch(iso_date))))
def main():
for day in range(31):
today = now_as_iso(24 * 60 * 60 * day)
query = "__expiry_ts__:[%s TO %s+1DAY]" % (today, today)
minutes_params = (
("rows", "0"),
("facet", "on"),
("facet.date", "__expiry_ts__"),
("facet.date.start", today),
("facet.date.end", today + "+1DAY"),
("facet.date.gap", "+1MINUTE"),
("facet.mincount", "1"),
)
res_minutes = datastore.direct_search("emptyresult", query, args=minutes_params)
minutes = res_minutes.get("facet_counts", {}).get("facet_dates", {}).get("__expiry_ts__", {})
for minute, minute_count in minutes.iteritems():
if minute in ['end', 'gap', 'start']:
continue
if minute_count > 0:
for x in datastore.stream_search('emptyresult', "__expiry_ts__:[%s TO %s+1MINUTE]" % (minute, minute)):
try:
created = epoch_to_iso(iso_to_epoch(today) - (15 * 24 * 60 * 60))
riak_key = x['_yz_rk']
path = os.path.join(directory, created[:10]) + '.index'
fh = get_filehandle(path)
fh.write(riak_key + "\n")
fh.flush()
except: # pylint: disable=W0702
logger.exception('Unhandled exception:')
def test_isotime_local():
local_date = now_as_local()
local_format = re.compile(r'[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}.*')
assert isinstance(local_date, str)
assert local_format.match(local_date)
assert epoch_to_local(local_to_epoch(local_date)) == local_date
assert local_date == epoch_to_local(iso_to_epoch(epoch_to_iso(local_to_epoch(local_date))))
def rescan(self, submission: Submission, results: Dict[str, Result],
file_infos: Dict[str, FileInfo], file_tree, errors: List[str],
rescan_services: List[str]):
"""
Rescan a submission started on another system.
"""
# Reset submission processing data
submission['times'].pop('completed')
submission['state'] = 'submitted'
# Set the list of service to rescan
submission['params']['services']['rescan'] = rescan_services
# Create the submission object
submission_obj = Submission(submission)
if len(submission_obj.files) == 0:
raise SubmissionException("No files found to submit.")
for f in submission_obj.files:
if not self.datastore.file.exists(f.sha256):
raise SubmissionException(
f"File {f.sha256} does not exist, cannot continue submission."
)
# Set the new expiry
if submission_obj.params.ttl:
submission_obj.expiry_ts = epoch_to_iso(now() +
submission_obj.params.ttl *
24 * 60 * 60)
# Clearing runtime_excluded on initial submit or resubmit
submission_obj.params.services.runtime_excluded = []
# Save the submission
self.datastore.submission.save(submission_obj.sid, submission_obj)
# Dispatch the submission
self.log.debug("Submission complete. Dispatching: %s",
submission_obj.sid)
self.dispatcher.dispatch_bundle(submission_obj, results, file_infos,
file_tree, errors)
return submission
def test_isotime_epoch():
epoch_date = now(200)
assert epoch_date == local_to_epoch(epoch_to_local(epoch_date))
assert epoch_date == iso_to_epoch(epoch_to_iso(epoch_date))
assert isinstance(epoch_date, float)
def submit(self,
submission_obj: SubmissionObject,
local_files: List = None,
completed_queue=None):
"""Submit several files in a single submission.
After this method runs, there should be no local copies of the file left.
"""
if local_files is None:
local_files = []
if len(submission_obj.files) == 0 and len(local_files) == 0:
raise SubmissionException("No files found to submit...")
if submission_obj.params.ttl:
expiry = epoch_to_iso(submission_obj.time.timestamp() +
submission_obj.params.ttl * 24 * 60 * 60)
else:
expiry = None
max_size = self.config.submission.max_file_size
for local_file in local_files:
# Upload/download, extract, analyze files
original_classification = str(submission_obj.params.classification)
file_hash, size, new_metadata = self._ready_file(
local_file, expiry, original_classification)
new_name = new_metadata.pop('name',
safe_str(os.path.basename(local_file)))
meta_classification = new_metadata.pop('classification',
original_classification)
if meta_classification != original_classification:
try:
submission_obj.params.classification = Classification.max_classification(
meta_classification, original_classification)
except InvalidClassification as ic:
raise SubmissionException(
"The classification found inside the cart file cannot be merged with "
f"the classification the file was submitted as: {str(ic)}"
)
submission_obj.metadata.update(**flatten(new_metadata))
# Check that after we have resolved exactly what to pass on, that it
# remains a valid target for scanning
if size > max_size and not submission_obj.params.ignore_size:
msg = "File too large (%d > %d). Submission failed" % (
size, max_size)
raise SubmissionException(msg)
elif size == 0:
msg = "File empty. Submission failed"
raise SubmissionException(msg)
submission_obj.files.append(
File({
'name': new_name,
'size': size,
'sha256': file_hash,
}))
# Clearing runtime_excluded on initial submit or resubmit
submission_obj.params.services.runtime_excluded = []
# We should now have all the information we need to construct a submission object
sub = Submission(
dict(
archive_ts=now_as_iso(
self.config.datastore.ilm.days_until_archive * 24 * 60 *
60),
classification=submission_obj.params.classification,
error_count=0,
errors=[],
expiry_ts=expiry,
file_count=len(submission_obj.files),
files=submission_obj.files,
max_score=0,
metadata=submission_obj.metadata,
params=submission_obj.params,
results=[],
sid=submission_obj.sid,
state='submitted',
scan_key=submission_obj.scan_key,
))
if self.config.ui.allow_malicious_hinting and submission_obj.params.malicious:
sub.verdict = {"malicious": [submission_obj.params.submitter]}
self.datastore.submission.save(sub.sid, sub)
self.log.debug("Submission complete. Dispatching: %s", sub.sid)
self.dispatcher.dispatch_submission(sub,
completed_queue=completed_queue)
return sub
def do_local_update(self) -> None:
old_update_time = self.get_local_update_time()
if not os.path.exists(UPDATER_DIR):
os.makedirs(UPDATER_DIR)
_, time_keeper = tempfile.mkstemp(prefix="time_keeper_", dir=UPDATER_DIR)
if self._service.update_config.generates_signatures:
output_directory = tempfile.mkdtemp(prefix="update_dir_", dir=UPDATER_DIR)
self.log.info("Setup service account.")
username = self.ensure_service_account()
self.log.info("Create temporary API key.")
with temporary_api_key(self.datastore, username) as api_key:
self.log.info(f"Connecting to Assemblyline API: {UI_SERVER}")
al_client = get_client(UI_SERVER, apikey=(username, api_key), verify=False)
# Check if new signatures have been added
self.log.info("Check for new signatures.")
if al_client.signature.update_available(
since=epoch_to_iso(old_update_time) or '', sig_type=self.updater_type)['update_available']:
self.log.info("An update is available for download from the datastore")
self.log.debug(f"{self.updater_type} update available since {epoch_to_iso(old_update_time) or ''}")
extracted_zip = False
attempt = 0
# Sometimes a zip file isn't always returned, will affect service's use of signature source. Patience..
while not extracted_zip and attempt < 5:
temp_zip_file = os.path.join(output_directory, 'temp.zip')
al_client.signature.download(
output=temp_zip_file,
query=f"type:{self.updater_type} AND (status:NOISY OR status:DEPLOYED)")
self.log.debug(f"Downloading update to {temp_zip_file}")
if os.path.exists(temp_zip_file) and os.path.getsize(temp_zip_file) > 0:
self.log.debug(f"File type ({os.path.getsize(temp_zip_file)}B): {zip_ident(temp_zip_file, 'unknown')}")
try:
with ZipFile(temp_zip_file, 'r') as zip_f:
zip_f.extractall(output_directory)
extracted_zip = True
self.log.info("Zip extracted.")
except BadZipFile:
attempt += 1
self.log.warning(f"[{attempt}/5] Bad zip. Trying again after 30s...")
time.sleep(30)
except Exception as e:
self.log.error(f'Problem while extracting signatures to disk: {e}')
break
os.remove(temp_zip_file)
if extracted_zip:
self.log.info("New ruleset successfully downloaded and ready to use")
self.serve_directory(output_directory, time_keeper)
else:
self.log.error("Signatures aren't saved to disk.")
shutil.rmtree(output_directory, ignore_errors=True)
if os.path.exists(time_keeper):
os.unlink(time_keeper)
else:
self.log.info("No signature updates available.")
shutil.rmtree(output_directory, ignore_errors=True)
if os.path.exists(time_keeper):
os.unlink(time_keeper)
else:
output_directory = self.prepare_output_directory()
self.serve_directory(output_directory, time_keeper)
def resubmit_for_dynamic(sha256, *args, **kwargs):
"""
Resubmit a file for dynamic analysis
Variables:
sha256 => Resource locator (SHA256)
Arguments (Optional):
copy_sid => Mimic the attributes of this SID.
name => Name of the file for the submission
Data Block:
None
Result example:
# Submission message object as a json dictionary
"""
user = kwargs['user']
quota_error = check_submission_quota(user)
if quota_error:
return make_api_response("", quota_error, 503)
file_info = STORAGE.file.get(sha256, as_obj=False)
if not file_info:
return make_api_response(
{},
f"File {sha256} cannot be found on the server therefore it cannot be resubmitted.",
status_code=404)
if not Classification.is_accessible(user['classification'],
file_info['classification']):
return make_api_response(
"",
"You are not allowed to re-submit a file that you don't have access to",
403)
submit_result = None
try:
copy_sid = request.args.get('copy_sid', None)
name = safe_str(request.args.get('name', sha256))
if copy_sid:
submission = STORAGE.submission.get(copy_sid, as_obj=False)
else:
submission = None
if submission:
if not Classification.is_accessible(user['classification'],
submission['classification']):
return make_api_response(
"",
"You are not allowed to re-submit a submission that you don't have access to",
403)
submission_params = submission['params']
submission_params['classification'] = submission['classification']
expiry = submission['expiry_ts']
else:
submission_params = ui_to_submission_params(
load_user_settings(user))
submission_params['classification'] = file_info['classification']
expiry = file_info['expiry_ts']
# Calculate original submit time
if submission_params['ttl'] and expiry:
submit_time = epoch_to_iso(
iso_to_epoch(expiry) - submission_params['ttl'] * 24 * 60 * 60)
else:
submit_time = None
if not FILESTORE.exists(sha256):
return make_api_response(
{},
"File %s cannot be found on the server therefore it cannot be resubmitted."
% sha256,
status_code=404)
files = [{'name': name, 'sha256': sha256, 'size': file_info['size']}]
submission_params['submitter'] = user['uname']
submission_params['quota_item'] = True
if 'priority' not in submission_params:
submission_params['priority'] = 500
submission_params[
'description'] = "Resubmit %s for Dynamic Analysis" % name
if "Dynamic Analysis" not in submission_params['services']['selected']:
submission_params['services']['selected'].append(
"Dynamic Analysis")
try:
submission_obj = Submission({
"files": files,
"params": submission_params,
"time": submit_time
})
except (ValueError, KeyError) as e:
return make_api_response("", err=str(e), status_code=400)
submit_result = SubmissionClient(
datastore=STORAGE,
filestore=FILESTORE,
config=config,
identify=IDENTIFY).submit(submission_obj)
submission_received(submission_obj)
return make_api_response(submit_result.as_primitives())
except SubmissionException as e:
return make_api_response("", err=str(e), status_code=400)
finally:
if submit_result is None:
decrement_submission_quota(user)
def resubmit_submission_for_analysis(sid, *args, **kwargs):
"""
Resubmit a submission for analysis with the exact same parameters as before
Variables:
sid => Submission ID to re-submit
Arguments:
None
Data Block:
None
Result example:
# Submission message object as a json dictionary
"""
user = kwargs['user']
quota_error = check_submission_quota(user)
if quota_error:
return make_api_response("", quota_error, 503)
submit_result = None
try:
submission = STORAGE.submission.get(sid, as_obj=False)
if submission:
if not Classification.is_accessible(user['classification'],
submission['classification']):
return make_api_response(
"",
"You are not allowed to re-submit a submission that you don't have access to",
403)
submission_params = submission['params']
submission_params['classification'] = submission['classification']
else:
return make_api_response({},
"Submission %s does not exists." % sid,
status_code=404)
submission_params['submitter'] = user['uname']
submission_params['quota_item'] = True
submission_params[
'description'] = "Resubmit %s for analysis" % ", ".join(
[x['name'] for x in submission["files"]])
# Calculate original submit time
if submission_params['ttl'] and submission['expiry_ts']:
submit_time = epoch_to_iso(
iso_to_epoch(submission['expiry_ts']) -
submission_params['ttl'] * 24 * 60 * 60)
else:
submit_time = None
try:
submission_obj = Submission({
"files": submission["files"],
"metadata": submission['metadata'],
"params": submission_params,
"time": submit_time
})
except (ValueError, KeyError) as e:
return make_api_response("", err=str(e), status_code=400)
submit_result = SubmissionClient(
datastore=STORAGE,
filestore=FILESTORE,
config=config,
identify=IDENTIFY).submit(submission_obj)
submission_received(submission_obj)
return make_api_response(submit_result.as_primitives())
except SubmissionException as e:
return make_api_response("", err=str(e), status_code=400)
finally:
if submit_result is None:
decrement_submission_quota(user)
def run_expiry_once(self):
now = now_as_iso()
reached_max = False
# Expire data
for collection in self.expirable_collections:
# Call heartbeat pre-dated by 5 minutes. If a collection takes more than
# 5 minutes to expire, this container could be seen as unhealthy. The down
# side is if it is stuck on something it will be more than 5 minutes before
# the container is restarted.
self.heartbeat(int(time.time() + 5 * 60))
# Start of expiry transaction
if self.apm_client:
self.apm_client.begin_transaction("Delete expired documents")
if self.config.core.expiry.batch_delete:
computed_date = epoch_to_iso(
dm(f"{now}||-{self.config.core.expiry.delay}h/d").
float_timestamp)
else:
computed_date = epoch_to_iso(
dm(f"{now}||-{self.config.core.expiry.delay}h").
float_timestamp)
delete_query = f"expiry_ts:[* TO {computed_date}]"
if self.config.core.expiry.delete_storage and collection.name in self.fs_hashmap:
file_delete = True
sort = ["expiry_ts asc", "id asc"]
else:
file_delete = False
sort = None
number_to_delete = collection.search(
delete_query,
rows=0,
as_obj=False,
use_archive=True,
sort=sort,
track_total_hits=EXPIRY_SIZE)['total']
if self.apm_client:
elasticapm.label(query=delete_query)
elasticapm.label(number_to_delete=number_to_delete)
self.log.info(f"Processing collection: {collection.name}")
if number_to_delete != 0:
if file_delete:
with elasticapm.capture_span(
name='FILESTORE [ThreadPoolExecutor] :: delete()',
labels={
"num_files": number_to_delete,
"query": delete_query
}):
# Delete associated files
with concurrent.futures.ThreadPoolExecutor(
self.config.core.expiry.workers,
thread_name_prefix="file_delete") as executor:
for item in collection.search(
delete_query,
fl='id',
rows=number_to_delete,
sort=sort,
use_archive=True,
as_obj=False)['items']:
executor.submit(
self.fs_hashmap[collection.name],
item['id'], computed_date)
self.log.info(
f' Deleted associated files from the '
f'{"cachestore" if "cache" in collection.name else "filestore"}...'
)
# Proceed with deletion
collection.delete_by_query(
delete_query,
workers=self.config.core.expiry.workers,
sort=sort,
max_docs=number_to_delete)
else:
# Proceed with deletion
collection.delete_by_query(
delete_query, workers=self.config.core.expiry.workers)
if number_to_delete == EXPIRY_SIZE:
reached_max = True
self.counter.increment(f'{collection.name}',
increment_by=number_to_delete)
self.log.info(
f" Deleted {number_to_delete} items from the datastore..."
)
else:
self.log.debug(" Nothing to delete in this collection.")
# End of expiry transaction
if self.apm_client:
self.apm_client.end_transaction(collection.name, 'deleted')
return reached_max
def ensure_iso(v):
try:
return epoch_to_iso(float(v))
except (TypeError, ValueError):
return v
