def worker(url, agent, vmhost, residential=False):
# Only useful when testing outside of Docker; Can be removed later
utils.kill_old_processes('chrome', age=config.OBSOLETE_PROCESS_AGE)
utils.delete_old_files(config.CHROME_BINARY_PATH, 'jsgraph.log',
config.OBSOLETE_PROCESS_AGE)
print "%s started. Domain: %s; Agent: %s" % (os.getpid(), url, agent)
tabs_opened = 0
log_id = "_".join((vmhost, socket.gethostname(), us_timestamp_str()))
error = False
adminer = None
try:
adminer = ad_miner.AdMiner(start_url=url,
log_id=log_id,
agent_name=agent)
tabs_opened = run_adminer(adminer)
except Exception as e:
error = True
print "Got exception: for %s" % (os.getpid())
print e
#import ipdb; ipdb.set_trace()
if adminer is not None:
utils.kill_processes_by_cmdline(
'chrome', adminer.log_id
) # Kill relevant chrome and chromedriver processes
adminer.cleanup()
print "Killed browser for a broken session: %s" % (log_id, )
# print "tabs opened:", tabs_opened
ads_opened = 0
if adminer is not None:
tabs_opened = adminer.bi.overall_tabs_opened
log_id = adminer.log_id
logger = logging.getLogger(log_id)
logger.info("END OF CRAWLER LOGS")
ad_objects = log_parser.process_session(log_id, logger)
if ad_objects is not None:
ads_opened = len(
[x for x in ad_objects if len(x.redirections) > 0])
# Sending logs:
if ads_opened > 0:
ship_logs(log_id, residential=residential)
print "**************NUMBER OF TABS OPENED HERE: %s*************" % (
tabs_opened, )
print "**************URL: %s, AGENT: %s, Log ID: %s*************" % (
url, agent, log_id)
downloads_path = os.path.join(config.MAIN_LOG_PATH, config.DOWNLOADS_DIR)
print "******* DOWNLOADS seen:", os.listdir(downloads_path)
return {
"log_id": log_id,
"tabs_opened": tabs_opened,
"error": error,
"ads_opened": ads_opened
}
def worker(url, agent, vmhost):
# Only useful when testing outside of Docker; Can be removed later
utils.kill_old_processes('chrome', age=config.OBSOLETE_PROCESS_AGE)
utils.delete_old_files(config.CHROME_BINARY_PATH, 'jsgraph.log',
config.OBSOLETE_PROCESS_AGE)
print "%s started. Domain: %s; Agent: %s" % (os.getpid(), url, agent)
tabs_opened = 0
log_id = "_".join((vmhost, socket.gethostname(), us_timestamp_str()))
error = False
adminer = None
try:
adminer = ad_miner.AdMiner(start_url=url,
log_id=log_id,
agent_name=agent)
tabs_opened = run_adminer(adminer)
except Exception as e:
error = True
print "Got exception: for %s" % (os.getpid())
print e
#import ipdb; ipdb.set_trace()
if adminer is not None:
utils.kill_processes_by_cmdline(
'chrome', adminer.log_id
) # Kill relevant chrome and chromedriver processes
adminer.cleanup()
print "Killed browser for a broken session: %s" % (log_id, )
image_hash, loaded_url, downloaded_files = get_milking_return_data(log_id)
file_hashes = get_downloaded_files_list()
loaded_sld = ""
if loaded_url:
ext = tldextract.extract(loaded_url)
loaded_sld = '.'.join(part for part in ext if part)
# Sending logs:
ship_logs(log_id, milking=True)
return {
"log_id": log_id,
"error": error,
"image_hash": image_hash,
"loaded_url": loaded_url,
"loaded_sld": loaded_sld,
"downloaded_files": downloaded_files,
"file_hashes": file_hashes
}
def publish_recent(src, dst, ext=".exe", keep_n_recent=KEEP_N_RECENT_FILES):
"""
Copy most recent file with 'ext' extension from src directory to dst directory
:param src: source directory
:param dst: destination directory
:param ext: file extension to be copied
:param keep_n_recent: indicates max number of files to keep, deletes oldest (if n > 0)
:return: void
"""
print "\n>>> Looking for most recent \'%s\' file in \'%s\'\n" % (ext, src)
lst_files = utils.get_files(src, ext)
if not lst_files:
print "\t No such file found, aborting."
return
latest_version = os.path.basename(max(lst_files, key=os.path.getctime))
print "\t Latest version available: \'%s\'\n" % latest_version
publish(os.path.join(src, latest_version), dst)
utils.delete_old_files(dst, ext, keep_n_recent)
utils.delete_old_files(src, ext, keep_n_recent)
def backup(self):
try:
column_value = {
'name': socket.gethostname(),
'percents_completed': 0,
'status': 1,
'start_backup_datetime': 'now()',
'finish_backup_datetime': 'NULL'
}
self.pk_row = self.db.insert(
self.config['db_name_record'], column_value)
query = (
u"UPDATE {0} SET database_storage_ip='{1}', storage_ip='{2}',"
" path_folders_pass='******', storage_destiny_path='{4}' WHERE id={5}"
).format(
self.config['db_name_record'],
self.db.get_ip(),
self.config['server_address'],
','.join(self.config['folders_to_pass']),
self.config['server_mount_folder'],
self.pk_row
)
self.db.query(query)
self.mount(self.config)
self.insert_config(
self.config['pg_user'], self.config['db_password'])
db_list = self.get_db_list(
self.config['pg_user'], self.config['host_machine'])
self.create_bkp_files(db_list, self.config)
msg = "Deleting old folders"
self.pk_log_row = self.db.insert(
self.config['db_name_log_record'], {
'backup_id': self.pk_row,
'log': msg,
'status': 1,
'log_datetime': 'now()'
}
)
folders_deleted = delete_old_files(
self.config['days_delete'],
get_last_folder_path(self.bkp_folder_path))
msg = "Old folders deleted: {0}".format(folders_deleted)
self.steps_done.append(True)
self.db.update(
self.config['db_name_log_record'], {
'id': self.pk_log_row,
'status': 2,
'log': msg
}
)
self.db.update(
self.config['db_name_record'], {
'id': self.pk_row,
'status': 1,
'percents_completed': self.count_percentage(),
'finish_backup_datetime': 'NULL'
}
)
self.email_context_success = self.email_context_success \
+ '- {0}\n'.format(
msg)
self.sync(self.config)
except KeyError as err:
err = "Error in {0}! Variable not found: ".format(
socket.gethostname()) + str(err)
print (err)
self.email_context_error = \
self.email_context_error + err + '\n'
except Exception as err:
self.treat_exception(err)
finally:
self.umount(self.config)
percentage = self.count_percentage()
if self.get_status() == 3:
percentage = 100
self.db.update(
self.config['db_name_record'], {
'id': self.pk_row,
'status': self.get_status(),
'percents_completed': percentage,
'finish_backup_datetime': 'now()'
}
)
self.db.close_conn()
email_ctx_error = self.email_context_error
email_ctx_success = self.email_context_success
if self.email_context_error == '':
email_ctx_error = self.email['success_msg']
if self.email_context_success == '':
email_ctx_success = self.email['error_msg']
email_context = self.email['email_context'].format(
email_ctx_success, email_ctx_error)
print(email_context)
if self.config['send_email_success']\
or self.email_context_error != '':
self.dispatch_email(email_context)
pass
logger = logging.getLogger()
fh = logging.FileHandler(log_filename)
logger.addHandler(fh)
logger.setLevel(logging.DEBUG)
logger.info('Loading config...')
print 'Loading config...'
env = simpy.Environment()
hierarchy, links = build_d3n(config, logger,env)
logger.info('Creating Enviroment...')
print 'Creating Enviroment...'
utils.delete_old_files(config)
logger.info('Parsing Trace File...')
print "Parsing Trace File..."
jobList=deque()
inputParser(config.get('Simulation', 'input'),jobList)
# Instantiate a thread pool with N worker threads
clientNum = int(config.get('Simulation', 'clientNum'))
nodeNum = int(config.get('Simulation', 'nodeNum'))
threadNum = int(config.get('Simulation', 'threadNum'))
shadow_window = int(config.get('Simulation', 'shadow_window'))
f_adapt = config.get('Simulation', 'adaptive_algorithm')
warmup = int(config.get('Simulation', 'warmup_time'))
a_time = int(config.get('Simulation', 'algorithm_time'))
def backup(self):
try:
column_value = {
'name': socket.gethostname(),
'percents_completed': 0,
'status': 1,
'start_backup_datetime': 'now()',
'finish_backup_datetime': 'NULL'
}
self.pk_row = self.db.insert(self.config['db_name_record'],
column_value)
# query = (
# u"UPDATE {0} SET database_storage_ip='{1}', storage_ip='{2}',"
# " path_folders_pass='******', storage_destiny_path='{4}' WHERE id={5}"
# ).format(
# self.config['db_name_record'],
# self.db.get_ip(),
# self.config['server_address'],
# ','.join(self.config['folders_to_pass']),
# self.config['server_mount_folder'],
# self.pk_row
# )
# self.db.query(query)
# self.mount(self.config)
self.insert_config(self.config['pg_user'],
self.config['db_password'])
db_list = self.get_db_list(self.config['pg_user'],
self.config['host_machine'])
self.create_bkp_files(db_list, self.config)
msg = "Deleting old folders"
self.pk_log_row = self.db.insert(
self.config['db_name_log_record'], {
'backup_id': self.pk_row,
'log': msg,
'status': 1,
'log_datetime': 'now()'
})
folders_deleted = delete_old_files(
self.config['days_delete'],
get_last_folder_path(self.bkp_folder_path))
msg = "Old folders deleted: {0}".format(folders_deleted)
self.steps_done.append(True)
self.db.update(self.config['db_name_log_record'], {
'id': self.pk_log_row,
'status': 2,
'log': msg
})
self.db.update(
self.config['db_name_record'], {
'id': self.pk_row,
'status': 1,
'percents_completed': self.count_percentage(),
'finish_backup_datetime': 'NULL'
})
self.email_context_success = self.email_context_success \
+ '- {0}\n'.format(
msg)
self.sync(self.config)
except KeyError as err:
err = "Error in {0}! Variable not found: ".format(
socket.gethostname()) + str(err)
print(err)
self.email_context_error = \
self.email_context_error + err + '\n'
except Exception as err:
self.treat_exception(err)
finally:
# self.umount(self.config)
percentage = self.count_percentage()
if self.get_status() == 3:
percentage = 100
self.db.update(
self.config['db_name_record'], {
'id': self.pk_row,
'status': self.get_status(),
'percents_completed': percentage,
'finish_backup_datetime': 'now()'
})
self.db.close_conn()
email_ctx_error = self.email_context_error
email_ctx_success = self.email_context_success
if self.email_context_error == '':
email_ctx_error = self.email['success_msg']
if self.email_context_success == '':
email_ctx_success = self.email['error_msg']
email_context = self.email['email_context'].format(
email_ctx_success, email_ctx_error)
print(email_context)
if self.config['send_email_success']\
or self.email_context_error != '':
self.dispatch_email(email_context)