async def start_server(server_section: str):
socket = config.get(server_section, "socket", fallback=None)
app = main_app
method = "unknown"
if server_section == "APIserver":
app = api_app
if socket:
await start_socket(app, socket)
method = socket
else:
host = config.get(server_section, "host")
port = config.getint(server_section, "port")
environ = os.environ.get("PA_BASEPATH", "/")
if config.getboolean(server_section, "ssl"):
try:
chain = config.get(server_section, "ssl_fullchain")
key = config.get(server_section, "ssl_privkey")
except configparser.NoOptionError:
logger.critical(
"SSL CONFIGURATION IS NOT CORRECTLY CONFIGURED. ABORTING LAUNCH."
)
sys.exit(2)
await start_https(app, host, port, chain, key)
method = f"https://{host}:{port}{environ}"
else:
await start_http(app, host, port)
method = f"http://{host}:{port}{environ}"
print(f"======== Starting {server_section} on {method} ========")
def _check_if_downloaded(self, file_path, expected_size):
file_name = os.path.basename(file_path)
downloaded_size = os.path.getsize(file_path)
if isinstance(expected_size, basestring):
logger.debug("expected_size is instance of basestring.")
try:
expected_size = int(expected_size)
except:
logger.error("Failed to convert expected_size to int.")
if downloaded_size == expected_size:
logger.debug("{0} IS the right size: {1} == {2}".format(
file_name, downloaded_size, expected_size))
return True
else:
logger.critical("{0} is NOT the right size: {1} != {2}".format(
file_name, downloaded_size, expected_size))
logger.debug("types: {0} and {1}".format(type(downloaded_size),
type(expected_size)))
return False
def country_fix(self):
logger.info(self.line)
logger.info("Fixing Country Data")
self.cursor.execute(self.attackers_query)
result = self.cursor.fetchall()
country = open('country.txt', 'w+')
self.count = 0
for x in result:
try:
# ip_info = self.gi.record_by_name(x[1])
ip_info = self.gi.city(x[0])
print(x[0] + " -> " + ip_info.country.name)
country_name = ip_info.country.name
self.count = self.count + 1
# country_name = ip_info['country_name'].replace("'", "")
# city_name = ip_info['city'] or None
query = self.update_query % (country_name, x[0])
logger.critical(query)
try:
self.cursor.execute(query)
self.db.commit()
except Exception as e:
pass
except Exception as e:
# ip = x[1]
# logger.info( ip,str(e)
country.write(x[0] + '\n')
def _create_op_handler(self):
plat = systeminfo.code()
if plat == 'darwin':
from operationhandler.machandler import MacOpHandler
return MacOpHandler()
elif plat == 'linux':
distro = platform.linux_distribution()[0].lower()
# List to check RedHat derived distros that use yum.
_redhat = 'red hat enterprise linux server'
_rpm_distros = ['fedora', 'centos', 'centos linux']
_debian_distros = ['debian', 'ubuntu', 'linuxmint']
if distro == _redhat:
from operationhandler.rhelhandler import RhelOpHandler
logger.debug('Using RhelOpHandler.')
return RhelOpHandler()
if distro in _rpm_distros:
from operationhandler.rpmhandler import RpmOpHandler
logger.debug('Using RpmOpHandler.')
return RpmOpHandler()
elif distro in _debian_distros:
from operationhandler.debhandler import DebianHandler
logger.debug('Using DebianHandler.')
return DebianHandler()
else:
logger.critical(
"Current platform '%s' isn't supported. Ignoring operations." %
plat)
return None
def _check_if_downloaded(self, file_path, expected_size):
file_name = os.path.basename(file_path)
downloaded_size = os.path.getsize(file_path)
if isinstance(expected_size, basestring):
logger.debug("expected_size is instance of basestring.")
try:
expected_size = int(expected_size)
except:
logger.error("Failed to convert expected_size to int.")
if downloaded_size == expected_size:
logger.debug(
"{0} IS the right size: {1} == {2}"
.format(file_name, downloaded_size, expected_size)
)
return True
else:
logger.critical(
"{0} is NOT the right size: {1} != {2}"
.format(file_name, downloaded_size, expected_size)
)
logger.debug(
"types: {0} and {1}".format(
type(downloaded_size), type(expected_size)
)
)
return False
def validate_victim(self):
""" Sends information about the current target. """
if not self.target.is_empty:
self.available_targets = self.data_fusion.validate_victim(self.target.info.id,
valid=self.gui_result.victimValid,
verified=self.target.is_verified())
else:
log.critical('Reached data fusion validation without target.')
def __init__(self, parent_pool, *args, **kwargs):
# super(Connection, self).__init__(*args, **kwargs)
self.time_limit = CONNECTTION_LIFETIME
self.created_time = time.time()
try:
self.connection = MySQLdb.connect(user=USER, passwd=passw, db=DB_NAME)
except Exception, e:
logger.critical('Error during connection to database > %s' % e)
raise
def __init__(self, parent_pool, *args, **kwargs):
# super(Connection, self).__init__(*args, **kwargs)
self.time_limit = CONNECTTION_LIFETIME
self.created_time = time.time()
try:
self.connection = MySQLdb.connect(user=USER,
passwd=passw,
db=DB_NAME)
except Exception, e:
logger.critical('Error during connection to database > %s' % e)
raise
def migrate(session):
try:
current_version = session.query(db.InternalMeta.version)[0].version
except IndexError:
logger.critical("Fatal error: Database version information is absent!")
return
if current_version > db.VERSION:
logger.warning("Error: Invalid database version: existing={}, latest={}."
.format(current_version, db.VERSION))
elif current_version < db.VERSION:
logger.warning("Outdated database version: existing={}, latest={}. Attempting migration."
.format(current_version, db.VERSION))
def start_https(host, port, chain, key):
import ssl
ctx = ssl.SSLContext()
try:
ctx.load_cert_chain(chain, key)
except FileNotFoundError:
logger.critical("SSL FILES ARE NOT FOUND. ABORTING LAUNCH.")
sys.exit(2)
web.run_app(
app,
host=host,
port=port,
ssl_context=ctx,
)
async def start_https(app: web.Application, host, port, chain, key):
import ssl
ctx = ssl.SSLContext()
try:
ctx.load_cert_chain(chain, key)
except FileNotFoundError:
logger.critical("SSL FILES ARE NOT FOUND. ABORTING LAUNCH.")
sys.exit(2)
await setup_runner(
app,
web.TCPSite,
host=host,
port=port,
ssl_context=ctx,
)
def api_call(self, action, **data):
url = self._api_url(action)
resp = yield from aiohttp.request('POST', url, data=data)
if resp.status == 200:
return (yield from resp.json())
elif resp.status == 502: # telegram nginx bad gateway
logger.warning(
'telegram returns a 502 error from action {}'.format(action)
)
yield from asyncio.sleep(10)
else:
api_return = yield from resp.text()
logger.critical(
'could not process the return of telegram api: {}'.format(
api_return
)
)
def _major_failure(self, operation, exception):
if operation:
root = {}
root[OperationKey.Operation] = operation.type
root[OperationKey.OperationId] = operation.id
root[OperationKey.AgentId] = settings.AgentId
root['error'] = str(exception)
operation.raw_result = json.dumps(root)
self.add_to_result_queue(operation)
else:
# TODO: Should we send something back to server?
logger.critical("Operation is empty in _major_failure")
def _get_server_addresses():
server_hostname = _config.get(_app_settings_section, 'ServerHostname')
server_ip_address = _config.get(_app_settings_section, 'ServerIpAddress')
if server_hostname == '' and server_ip_address == '':
logger.critical(
"No valid hostname or ip address was given for the RV Server.")
logger.critical("Please edit agent.config "
"(in the TopPatch/agent directory) to correct it.")
sys.exit(1)
else:
return server_hostname, server_ip_address
def _major_failure(self, operation, exception):
if operation:
root = {}
root[OperationKey.Operation] = operation.type
root[OperationKey.OperationId] = operation.id
root[OperationKey.AgentId] = settings.AgentId
root['error'] = str(exception)
operation.raw_result = json.dumps(root)
self.add_to_result_queue(operation)
else:
# TODO: Should we send something back to server?
logger.critical("Operation is empty in _major_failure")
def _yum_vars():
"""Gets the yum variables used for normal operation.
Such as:
- $basearch
- $arch
- $releasever
These variables are crucial for normal yum operation. With out them,
forget about it!
Returns:
- A dict with the following keys:
- releasever
- basearch
- arch
"""
# Crucial variables to get
var_command = [
'/usr/bin/python', '-c',
'import yum; yb = yum.YumBase(); print yb.conf.yumvar'
]
try:
process = subprocess.Popen(var_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, stderr = process.communicate()
return ast.literal_eval(output.splitlines()[1])
except Exception as e:
logger.critical(
"Could not find yum vars. ($basearch, $releasever, etc)")
logger.critical("Exiting agent now. Please contact support.")
logger.exception(e)
sys.exit(-1)
def _yum_vars():
"""Gets the yum variables used for normal operation.
Such as:
- $basearch
- $arch
- $releasever
These variables are crucial for normal yum operation. With out them,
forget about it!
Returns:
- A dict with the following keys:
- releasever
- basearch
- arch
"""
# Crucial variables to get
var_command = [
'/usr/bin/python', '-c',
'import yum; yb = yum.YumBase(); print yb.conf.yumvar']
try:
process = subprocess.Popen(var_command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, stderr = process.communicate()
return ast.literal_eval(output.splitlines()[1])
except Exception as e:
logger.critical(
"Could not find yum vars. ($basearch, $releasever, etc)"
)
logger.critical("Exiting agent now. Please contact support.")
logger.exception(e)
sys.exit(-1)
def load(self):
""" Loads the configuration file and sets up the FSM accordingly. """
try:
# Read the configuration file.
with open(self.config) as file_handler:
data = yaml.load(file_handler)
except IOError:
log.critical('Could not read configuration file.')
sys.exit(1)
try:
states = data[self.strategy]['states']
except KeyError:
log.critical('%s is not a valid strategy.', self.strategy)
sys.exit(1)
# Setting up the FSM
self.machine = Machine(model=self)
# Get all the states for the given strategy.
self.states = [state['name'] for state in states]
# Set up states tasks.
for state in states:
self.machine.add_states(state['name'],
on_enter=state['tasks'],
on_exit=state['clean'])
# Create the transition table.
self.transitions = []
for state in states:
for transition in state['transitions']:
self.machine.add_transition(
transition['trigger'],
state['name'],
transition['to'],
before=transition['before'],
after=transition['after'],
conditions=transition['conditions'])
# Sets up the initial state
self.machine.set_state(self.states[0])
log.debug('FSM has been loaded.')
def _get_server_addresses():
server_hostname = _config.get(_app_settings_section, 'ServerHostname')
server_ip_address = _config.get(_app_settings_section, 'ServerIpAddress')
if server_hostname == '' and server_ip_address == '':
logger.critical(
"No valid hostname or ip address was given for the RV Server."
)
logger.critical(
"Please edit agent.config "
"(in the TopPatch/agent directory) to correct it."
)
sys.exit(1)
else:
return server_hostname, server_ip_address
def attackers_state(self):
logger.info(self.line)
logger.info("Attacker State Report Generation")
worksheet = self.workbook.add_worksheet(self.__attacker_info)
worksheet.write('A1', 'ip', self.bold)
worksheet.write('B1', 'Saba lab', self.bold)
worksheet.write('C1', 'Abusedb lab', self.bold)
worksheet.write('D1', 'BlockServers lab', self.bold)
query = self.attackers_state_query % (self.from_date, self.to_date)
self.cursor.execute(query)
result = self.cursor.fetchall()
i = 0
for ip in result:
logger.critical("get Info For : " + str(ip[0]))
i = i + 1
worksheet.write(i, 0, self.ensure_unicode(ip[0]))
if self.find_in_lab_db(ip[0]):
worksheet.write(i, 1, "1")
else:
worksheet.write(i, 1, "0")
try:
if self.find_over_internet_abusedb(ip[0]):
worksheet.write(i, 2, "1")
else:
worksheet.write(i, 2, "0")
except:
worksheet.write(i, 2, "0")
try:
blocked_ip = self.find_over_internet_blockservers_lab(ip[0])
except:
blocked_ip = None
if blocked_ip is None:
worksheet.write(i, 3, "0")
else:
worksheet.write(i, 3,
blocked_ip['blockedservers']['blocked_count'])
class Parameters:
program_dir = Path(abspath(__file__)).parents[1]
config_dir = 'config'
config_name = 'default_config.json'
config_path = join(program_dir, config_dir, config_name)
#try to load the config file
try:
#load json file here
config_dict = load_json_file(config_path)
except FileNotFoundError:
logger.critical(f'config file was not found at: {config_path}')
exit(1)
except Exception as e:
logger.critical(f'failed to load config file: {e}')
exit(1)
if is_valid_config(config_dict):
#set all parameters here
root_dir = Path(abspath(__file__)).parents[1]
whiskybroker_base_url = config_dict['whiskybroker base url']
email_address = config_dict['email address']
data_dir = join(root_dir, config_dict['data dir'])
logging_level, status = get_logging_level_from_parameter(
config_dict['general logging level'])
#TODO: maybe change the below logging to debug
logging.info(f'logging level parsing status: {status}')
logger.info(f'setting general logging level to: {logging_level}')
logger.setLevel(logging_level)
else:
#TODO: check why the config dict was not valid, and return any missing keys/wrong values
logger.critical(f'config file contained something wrong...')
exit(1)
def _create_op_handler(self):
plat = systeminfo.code()
if plat == 'darwin':
from operationhandler.machandler import MacOpHandler
return MacOpHandler()
elif plat == 'linux':
distro = platform.linux_distribution()[0].lower()
# List to check RedHat derived distros that use yum.
_redhat = 'red hat enterprise linux server'
_rpm_distros = ['fedora', 'centos', 'centos linux']
_debian_distros = ['debian', 'ubuntu', 'linuxmint']
if distro == _redhat:
from operationhandler.rhelhandler import RhelOpHandler
logger.debug('Using RhelOpHandler.')
return RhelOpHandler()
if distro in _rpm_distros:
from operationhandler.rpmhandler import RpmOpHandler
logger.debug('Using RpmOpHandler.')
return RpmOpHandler()
elif distro in _debian_distros:
from operationhandler.debhandler import DebianHandler
logger.debug('Using DebianHandler.')
return DebianHandler()
else:
logger.critical(
"Current platform '%s' isn't supported. Ignoring operations." %
plat
)
return None
def create_reverse_tunnel(local_port, host_port, server=None, ssh_port=None):
ssh_output = os.path.join(settings.TempDirectory, 'ssh_output')
if not ssh_port:
ssh_port = 22
try:
if os.path.exists(ssh_output):
os.remove(ssh_output)
# Unfortunately ssh ignores stdin when running in the backgroud
# (option -f) so we use StrictHostKeyChecking=no just on first
# connection only. If it changes afterwards an error occurs.
# And for some reason can't read from stdout with stdout.read()
# but it writes to file just fine so we read that file instead.
with open(ssh_output, 'w') as dev_null:
process = subprocess.Popen([
'/usr/bin/ssh', '-oStrictHostKeyChecking=no', '-i', tunnel_key,
'-fnNR',
'%s:localhost:%s' % (host_port, local_port),
'toppatch@%s' % server,
'-p %s' % ssh_port
],
stdout=dev_null,
stderr=dev_null)
time.sleep(5)
with open(ssh_output, 'r') as dev_null:
output = dev_null.read()
for line in output.splitlines():
if 'WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!' in line:
process.kill()
logger.critical("Failed to verify RV ssh host public key.")
logger.critical(output)
return (
False, "Server failed to verify. It's possible"
" the ssh host keys changed or, worse, someone could"
" be eavesdropping via man-in-the-middle attack."
" Terminating ssh tunnel for security reasons. Please"
" investigate agent log for further details.")
elif ('Warning: remote port forwarding failed for listen port'
in line):
process.kill()
logger.critical("Unable to connect to server port."
" Port %s might be in use?" % host_port)
logger.critical(output)
return (
False,
"Agent was unable to connect to the specified port. It"
" might be in use by another process. Detailed Error: "
" %s" % output)
elif 'Permission denied (publickey)' in line:
process.kill()
logger.critical("Failed to verify RV ssh host public key.")
logger.critical(output)
return (False, "Error with the public key: %s" % output)
elif ('ssh: connect to host' in line
and 'port 22: Connection refused' in line):
process.kill()
msg = (
"Failed to connect to ssh. Is the ssh server running"
" and listening on port 22?")
logger.critical(msg)
logger.critical(output)
return (False, msg)
elif (("Warning: Identity file %s not accessible" % tunnel_key)
in line):
process.kill()
msg = (
"Failed to connect to ssh. Private key was not found.")
logger.critical(msg)
logger.critical(output)
return (False, msg)
_write_ssh_details(host_port, local_port)
return True, ''
except Exception as e:
logger.error('Could not create ssh tunnel.')
logger.exception(e)
return False, 'Unable to create ssh tunnel. %e' % str(e)
def softwareupdate(self, update_name, proc_niceness):
# Need to wrap call to /usr/sbin/softwareupdate with a utility
# that makes softwareupdate think it is connected to a tty-like
# device so its output is unbuffered so we can get progress info
# '-v' (verbose) option is available on OSX > 10.5
cmd = [
'nice', '-n',
CpuPriority.niceness_to_string(proc_niceness), self.ptyexec_cmd,
self.softwareupdate_cmd, '-v', '-i', update_name
]
logger.debug("Running softwareupdate: " + str(cmd))
success = 'false'
error = ''
if not os.path.exists(self.ptyexec_cmd):
raise PtyExecMissingException(settings.BinPath)
try:
job = launchd.Job(cmd)
job.start()
except launchd.LaunchdJobException as e:
error_message = 'Error with launchd job (%s): %s' % (cmd, str(e))
logger.error(error_message)
logger.critical('Skipping softwareupdate run.')
return 'false', error_message
while True:
output = job.stdout.readline()
if not output:
if job.returncode() is not None:
break
else:
# no data, but we're still running
# sleep a bit before checking for more output
time.sleep(2)
continue
# Checking output to verify results.
output = output.decode('UTF-8').strip()
if output.startswith('Installed '):
# 10.6 / 10.7 / 10.8 Successful install of package name.
success = 'true'
error = ''
break
elif output.startswith('Done with'):
success = 'true'
error = ''
break
# elif output.startswith('Done '):
# # 10.5. Successful install of package name.
# install_successful = True
elif 'No such update' in output:
# 10.8 When a package cannot be found.
success = 'false'
error = "Update not found."
break
elif output.startswith('Error '):
# 10.8 Error statement
# Note: Checking for updates doesn't display the
# 'Error' string when connection is down.
if "Internet connection appears to be offline" in output:
error = "Could not download files."
else:
error = output
success = 'false'
break
elif 'restart immediately' in output:
# Ignore if output is indicating a restart. Exact line:
# "You have installed one or more updates that requires that
# you restart your computer. Please restart immediately."
continue
elif output.startswith('Package failed'):
success = 'false'
error = output
logger.debug(error)
break
elif (output == '' or output.startswith('Progress')
or output.startswith('Done')
or output.startswith('Running package')
or output.startswith('Copyright')
or output.startswith('Software Update Tool')
or output.startswith('Downloading')
or output.startswith('Moving items into place')
or output.startswith('Writing package receipts')
or output.startswith('Removing old files')
or output.startswith('Registering updated components')
or output.startswith('Waiting for other installations')
or output.startswith('Writing files')
or output.startswith('Cleaning up')
or output.startswith('Registering updated applications')
or output.startswith('About')
or output.startswith('Less than a minute')):
# Output to ignore
continue
elif (output.startswith('Checking packages')
or output.startswith('Installing') or
output.startswith('Optimizing system for installed software')
or output.startswith('Waiting to install')
or output.startswith('Validating packages')
or output.startswith('Finding available software')
or output.startswith('Downloaded')):
# Output to display
logger.debug('softwareupdate: ' + output)
else:
success = 'false'
error = "softwareupdate (unknown): " + output
logger.debug(error)
# return_code = job.returncode()
# if return_code == 0:
# # get SoftwareUpdate's LastResultCode
# su_path = '/Library/Preferences/com.apple.SoftwareUpdate.plist'
# su_prefs = plist.convert_and_read_plist(su_path)
# last_result_code = su_prefs['LastResultCode'] or 0
# if last_result_code > 2:
# return_code = last_result_code
logger.debug("Done with softwareupdate.")
return success, error
port=port,
ssl_context=ctx,
)
def start_socket(sock):
web.run_app(app, path=sock)
if __name__ == "__main__":
socket = config.get("Webserver", "socket", fallback=None)
if socket:
start_socket(socket)
else:
host = config.get("Webserver", "host")
port = config.getint("Webserver", "port")
if config.getboolean("Webserver", "ssl"):
try:
chain = config.get("Webserver", "ssl_fullchain")
key = config.get("Webserver", "ssl_privkey")
except configparser.NoOptionError:
logger.critical(
"SSL CONFIGURATION IS NOT CORRECTLY CONFIGURED. ABORTING LAUNCH."
)
sys.exit(2)
start_https(host, port, chain, key)
else:
start_http(host, port)
def create_reverse_tunnel(local_port, host_port, server=None, ssh_port=None):
ssh_output = os.path.join(settings.TempDirectory, 'ssh_output')
if not ssh_port:
ssh_port = 22
try:
if os.path.exists(ssh_output):
os.remove(ssh_output)
# Unfortunately ssh ignores stdin when running in the backgroud
# (option -f) so we use StrictHostKeyChecking=no just on first
# connection only. If it changes afterwards an error occurs.
# And for some reason can't read from stdout with stdout.read()
# but it writes to file just fine so we read that file instead.
with open(ssh_output, 'w') as dev_null:
process = subprocess.Popen(
[
'/usr/bin/ssh', '-oStrictHostKeyChecking=no', '-i',
tunnel_key, '-fnNR',
'%s:localhost:%s' %
(
host_port,
local_port
),
'toppatch@%s' % server,
'-p %s' % ssh_port
],
stdout=dev_null,
stderr=dev_null
)
time.sleep(5)
with open(ssh_output, 'r') as dev_null:
output = dev_null.read()
for line in output.splitlines():
if 'WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!' in line:
process.kill()
logger.critical("Failed to verify RV ssh host public key.")
logger.critical(output)
return (
False,
"Server failed to verify. It's possible"
" the ssh host keys changed or, worse, someone could"
" be eavesdropping via man-in-the-middle attack."
" Terminating ssh tunnel for security reasons. Please"
" investigate agent log for further details."
)
elif (
'Warning: remote port forwarding failed for listen port' in
line
):
process.kill()
logger.critical("Unable to connect to server port."
" Port %s might be in use?" % host_port)
logger.critical(output)
return (
False,
"Agent was unable to connect to the specified port. It"
" might be in use by another process. Detailed Error: "
" %s" % output
)
elif 'Permission denied (publickey)' in line:
process.kill()
logger.critical("Failed to verify RV ssh host public key.")
logger.critical(output)
return (
False,
"Error with the public key: %s" % output
)
elif (
'ssh: connect to host' in line
and 'port 22: Connection refused' in line
):
process.kill()
msg = (
"Failed to connect to ssh. Is the ssh server running"
" and listening on port 22?"
)
logger.critical(msg)
logger.critical(output)
return (
False,
msg
)
elif (
("Warning: Identity file %s not accessible" % tunnel_key)
in line
):
process.kill()
msg = (
"Failed to connect to ssh. Private key was not found."
)
logger.critical(msg)
logger.critical(output)
return (
False,
msg
)
_write_ssh_details(host_port, local_port)
return True, ''
except Exception as e:
logger.error('Could not create ssh tunnel.')
logger.exception(e)
return False, 'Unable to create ssh tunnel. %e' % str(e)
def softwareupdate(self, update_name, proc_niceness):
# Need to wrap call to /usr/sbin/softwareupdate with a utility
# that makes softwareupdate think it is connected to a tty-like
# device so its output is unbuffered so we can get progress info
# '-v' (verbose) option is available on OSX > 10.5
cmd = [
"nice",
"-n",
CpuPriority.niceness_to_string(proc_niceness),
self.ptyexec_cmd,
self.softwareupdate_cmd,
"-v",
"-i",
update_name,
]
logger.debug("Running softwareupdate: " + str(cmd))
success = "false"
error = ""
if not os.path.exists(self.ptyexec_cmd):
raise PtyExecMissingException(settings.BinPath)
try:
job = launchd.Job(cmd)
job.start()
except launchd.LaunchdJobException as e:
error_message = "Error with launchd job (%s): %s" % (cmd, str(e))
logger.error(error_message)
logger.critical("Skipping softwareupdate run.")
return "false", error_message
while True:
output = job.stdout.readline()
if not output:
if job.returncode() is not None:
break
else:
# no data, but we're still running
# sleep a bit before checking for more output
time.sleep(2)
continue
# Checking output to verify results.
output = output.decode("UTF-8").strip()
if output.startswith("Installed "):
# 10.6 / 10.7 / 10.8 Successful install of package name.
success = "true"
error = ""
break
elif output.startswith("Done with"):
success = "true"
error = ""
break
# elif output.startswith('Done '):
# # 10.5. Successful install of package name.
# install_successful = True
elif "No such update" in output:
# 10.8 When a package cannot be found.
success = "false"
error = "Update not found."
break
elif output.startswith("Error "):
# 10.8 Error statement
# Note: Checking for updates doesn't display the
# 'Error' string when connection is down.
if "Internet connection appears to be offline" in output:
error = "Could not download files."
else:
error = output
success = "false"
break
elif "restart immediately" in output:
# Ignore if output is indicating a restart. Exact line:
# "You have installed one or more updates that requires that
# you restart your computer. Please restart immediately."
continue
elif output.startswith("Package failed"):
success = "false"
error = output
logger.debug(error)
break
elif (
output == ""
or output.startswith("Progress")
or output.startswith("Done")
or output.startswith("Running package")
or output.startswith("Copyright")
or output.startswith("Software Update Tool")
or output.startswith("Downloading")
or output.startswith("Moving items into place")
or output.startswith("Writing package receipts")
or output.startswith("Removing old files")
or output.startswith("Registering updated components")
or output.startswith("Waiting for other installations")
or output.startswith("Writing files")
or output.startswith("Cleaning up")
or output.startswith("Registering updated applications")
or output.startswith("About")
or output.startswith("Less than a minute")
):
# Output to ignore
continue
elif (
output.startswith("Checking packages")
or output.startswith("Installing")
or output.startswith("Optimizing system for installed software")
or output.startswith("Waiting to install")
or output.startswith("Validating packages")
or output.startswith("Finding available software")
or output.startswith("Downloaded")
):
# Output to display
logger.debug("softwareupdate: " + output)
else:
success = "false"
error = "softwareupdate (unknown): " + output
logger.debug(error)
# return_code = job.returncode()
# if return_code == 0:
# # get SoftwareUpdate's LastResultCode
# su_path = '/Library/Preferences/com.apple.SoftwareUpdate.plist'
# su_prefs = plist.convert_and_read_plist(su_path)
# last_result_code = su_prefs['LastResultCode'] or 0
# if last_result_code > 2:
# return_code = last_result_code
logger.debug("Done with softwareupdate.")
return success, error
'Will start to update article from [{}] now'.format(csv_filepath))
parser = ArticleParser.from_csv(csv_filepath)
parser.fetch_artices()
# parser.articles_to_csv()
if __name__ == '__main__':
logger.info('####### STARTING COINTELEGRAPH SCRAPER #######')
choices = {'step1': step1, 'step2': step2, 'update': update}
parser = ArgumentParser(description='Parse data from cointelegraph.com')
parser.add_argument('step', choices=choices, help='which step to run')
parser.add_argument('--date', metavar='DATE', type=str, help='DATE limit')
parser.add_argument('--filepath',
metavar='FILEPATH',
type=str,
help='CSV file')
args = parser.parse_args()
function = choices[args.step]
date, csv_filepath = args.date, args.filepath
try:
if function == update and validate_csv_filepath_arg(csv_filepath):
function(csv_filepath)
elif function in [step1, step2] and validate_date_arg(date):
function(date)
except Exception:
logger.critical('Failed unexpectedly. Check logs.')
logger.exception('Exception occured')
