def scan(self):
try:
scan_text = "masscan -p {ports} -iL {target} -oL {output} --excludefile {exclude} --randomize-hosts --rate={rate}".format( \
ports=self.ports, target=self.target_file, \
output=self.result_file, rate=self.rate, exclude=self.exclude)
os.system(scan_text)
except Exception, ex:
logger.exception(ex.message)
def readdirAsync(self, path, depth=2, threadPool=pool):
try:
foo = json.loads(self.disk.list_files(path))
except Exception as s:
logger.exception(s)
files = ['.', '..']
abs_files = []
if 'errno' in foo:
logger.error(
"maybe token is not right, try re login http://pan.baidu.com in Chrome"
)
if "list" not in foo:
# logger.info("no list")
return
for file in foo['list']:
if file['server_filename'].startswith("."):
continue
files.append(file['server_filename'])
abs_files.append(file['path'])
# logger.debug(file['path'])
file_num = len(abs_files)
group = int(math.ceil(file_num / 100.0))
# logger.debug(f"group: {group}")
# logger.debug(f"abs_files: {abs_files}")
for i in range(group):
obj = [f for n, f in enumerate(abs_files) if n % group == i] #一组数据
while 1:
try:
ret = json.loads(self.disk.meta(obj))
# logger.debug(f'{ret}')
break
except Exception as e:
logger.info(ret)
logger.exception(e)
for file_info in ret['info']:
# logger.debug(file_info)
self._add_file_to_buffer(file_info['path'], file_info)
if depth > 0:
depth -= 1
if file_info['isdir']:
if file_info['path'] not in self.traversed_folder:
self.traversed_folder[path] = True
threadPool.submit(self.readdirAsync,
file_info['path'], depth,
threadPool)
self.dir_buffer[path] = files
def open(self, path, flags):
if path in self.writing_files:
return 0
# method does not have thread race problem, open by one thread only
try:
if path not in self.downloading_files:
url = self.disk.getRestUrl(path)
x = Task(url, path, self.disk)
x.start()
self.downloading_files[path] = x
except Baidu8Secs as e:
logger.exception(e)
except Exception as e:
logger.exception(e)
return 0
def analysis(self):
result_fp = open(self.result_file, 'r')
result_json = result_fp.readlines()
result_fp.close()
del result_json[0]
del result_json[-1]
open_list = {}
for res in result_json:
try:
# masscan can't use '.', replace with '_'
ip = res.split()[3].replace('.', '_')
port = res.split()[2]
if ip in open_list:
open_list[ip].append(port)
else:
open_list[ip] = [port]
except Exception as ex:
logger.exception("Analysis result error!" + ex.message)
return open_list
def find_assets(self):
# NewAssets = []
today_data = None
yesterday_data = None
try:
today = date.today()
today_str = str(today)
# today_str = '2018-03-14'
tmp_data = mongo[Config.MONGO_C_MASSCAN].find_one({'time':today_str})
if tmp_data:
today_data = tmp_data['data']
except Exception as ex:
logger.exception("Get today data error!" + ex.message)
sys.exit(1)
try:
yesterday = today - timedelta(1)
yesterday_str = str(yesterday) # yyy-mm-dd
tmp_data = mongo[Config.MONGO_C_MASSCAN].find_one({'time':yesterday_str})
if tmp_data:
yesterday_data = tmp_data['data']
except Exception as ex:
logger.exception("Get yestoday data error " + ex.message)
sys.exit(1)
# first compare with enquipment ips then yesterday data
self.get_company_assets()
try:
for ip in today_data:
if ip not in self.all_enq_ips:
self.not_in_enq.append(ip)
if not yesterday_data:
continue
elif ip not in yesterday_data.keys():
self.not_in_all.append(ip)
except Exception as ex:
logger.exception("Compare Error!" + ex.message)
sys.exit(1)
return self.not_in_enq, self.not_in_all
def loop_run():
try:
run_events_loop(redirector_event)
except:
logger.exception('error in rederector')
os._exit(1)
logger.exception('error in rederector')
os._exit(1)
if __name__ == '__main__':
logger.info('rederector starting')
sys.excepthook = except_hook
event_loop_thread = threading.Thread(target=loop_run)
event_loop_thread.start()
logger.info('rederector started')
try:
while True:
line = sys.stdin.readline()
if len(line) <= 1:
logger.debug('rederector received blank line')
logger.info('rederector stopping')
redirector.log_statistic()
logger.info('rederector stoped')
os._exit(0)
url = redirector.redirect(line)
sys.stdout.write(url)
sys.stdout.flush()
except KeyboardInterrupt:
os._exit(0)
except:
logger.exception('error in rederector')
os._exit(1)
def mutate(self, info, id, **kwargs):
errors = []
name = kwargs.get('name', None)
permissions = kwargs.get('permissions', None)
# Get the Project
project = Synapse.client().get(id)
if name:
project.name = name
project = Synapse.client().store(project)
# An empty list means remove all permissions.
# A null list means don't remove any permissions.
if permissions is not None:
for permission in permissions:
try:
principal_id = permission['principal_id']
access = permission['access']
access_type = getattr(Synapse, '{0}_PERMS'.format(access))
# Only add permissions, do not update permissions.
current_perms = Synapse.client().getPermissions(
project, principal_id)
if not current_perms:
try:
Synapse.client().setPermissions(
project,
principal_id,
accessType=access_type,
warn_if_inherits=False)
except Exception as syn_ex:
logger.exception(
'Error setting permission: {0} - {1}'.format(
permission, syn_ex))
if 'a foreign key constraint fails' in str(syn_ex):
errors.append(
'User or Team ID: {0} does not exist.'.
format(principal_id))
else:
errors.append(
'Error setting permission for User or Team ID: {0}'
.format(principal_id))
except Exception as ex:
logger.exception(
'Error creating project permissions: {0} - {1}'.format(
permission, ex))
errors.append('Error creating project permissions.')
# Remove permissions
try:
new_principal_ids = [
int(p['principal_id']) for p in permissions
]
acl = Synapse.client()._getACL(project)
current_principal_ids = [
int(r['principalId']) for r in acl['resourceAccess']
]
for current_principal_id in current_principal_ids:
if current_principal_id == int(project.createdBy):
continue
if current_principal_id not in new_principal_ids:
Synapse.client().setPermissions(project,
current_principal_id,
accessType=None,
warn_if_inherits=False)
except Exception as ex:
logger.exception(
'Error removing project permissions: {0}'.format(ex))
errors.append('Error removing project permissions.')
# Get the latest version of the project.
project = Synapse.client().get(id)
updated_syn_project = SynProject.from_project(project)
return UpdateSynProject(syn_project=updated_syn_project,
errors=(errors if errors else None))
def get_company_assets(self):
try:
# get DAS_SVR_Info ip
das = [] # [[out1, out2, in1, in2], ]
das_ips = [] # [ip, ip, ip]
query_das = 'select OuterIP1, OuterIP2, InnerIP1, InnerIP2 from DAS_SVR_Info'
result = self.sql_query(query_das)
for server in result:
single_server = []
for des, ip in server.items():
if ip:
single_server.append(ip)
das_ips.append(ip)
if single_server:
das.append(single_server)
if das_ips:
self.all_enq_ips += das_ips
except Exception as ex:
logger.exception('Get DAS_SVR_Info Error:' + ex.message)
try:
# get SP_Firewall_Info Ip
query_firewall = 'select Ip from SP_Firewall_Info'
result = self.sql_query(query_firewall)
# embed()
firewall = [ ip['Ip'] for ip in result if ip['Ip'] ]
if firewall:
self.all_enq_ips += firewall
except Exception as ex:
logger.exception('Get SP_Firewall_Info Error:' + ex.message)
try:
# get SP_Storage_Info MainIP
query_storage = 'select MainIP from SP_Storage_Info'
result = self.sql_query(query_storage)
if result:
storage = [ ip['MainIP'] for ip in result if ip['MainIP'] ]
if storage:
self.all_enq_ips += storage
except Exception as ex:
logger.exception('Get SP_Storage_Info Error:' + ex.message)
try:
# get SP_Switch_Info Ip
query_switch = 'select Ip from SP_Switch_Info'
result = self.sql_query(query_switch)
if result:
switch = [ ip['Ip'] for ip in result if ip['Ip'] ]
if switch:
self.all_enq_ips += switch
except Exception as ex:
logger.exception('Get SP_Switch_Info Error:' + ex.message)
# save all equipment platform ips to masscan exclude
fp = open('data/masscan/exclute.txt', 'w')
unique_ips = set(self.all_enq_ips)
for ip in unique_ips:
fp.write(ip + '\n')
fp.close()
return set(unique_ips)
def loop_run():
try:
run_events_loop(logger_event)
except:
logger.exception('error in squid3 logger')
os._exit(1)
logger.debug('squid3 logger caught siglan {0}'.format(signum))
if __name__ == '__main__':
logger.info('squid3 logger starting')
sys.excepthook = except_hook
signal.signal(signal.SIGTERM, sig_handler)
event_loop_thread = threading.Thread(target=loop_run)
event_loop_thread.start()
logger.info('squid3 logger started')
try:
while True:
line = sys.stdin.readline()
cmd_type = line[0]
if cmd_type == 'L':
squid3_logger.log(line)
elif cmd_type == 'F':
squid3_logger.flush()
except (IOError, KeyboardInterrupt):
logger.info('squid3 logger stopping')
squid3_logger.log_statistic()
logger.info('squid3 logger stoped')
os._exit(0)
except:
logger.exception('error in squid3 logger')
os._exit(1)
def mutate(self, info, name, **kwargs):
errors = []
permissions = kwargs.get('permissions', None)
annotations = kwargs.get('annotations', None)
wiki = kwargs.get('wiki', None)
folders = kwargs.get('folders', None)
posts = kwargs.get('posts', None)
# Check if a project with the same name already exists.
project_name_taken = Synapse.client().findEntityId(name) is not None
if project_name_taken:
raise ValueError(
'Another Synapse project with the name: {0} already exists.'.
format(name))
# Build the annotations
project_annotations = {}
if annotations:
for annotation in annotations:
project_annotations[annotation['key']] = annotation['value']
# Create the Project
project = Synapse.client().store(
Project(name=name, annotations=project_annotations))
# Add the permissions
if permissions:
for permission in permissions:
try:
principal_id = permission.get('principal_id')
access = permission.get('access')
access_type = getattr(Synapse, '{0}_PERMS'.format(access))
try:
Synapse.client().setPermissions(project,
principal_id,
accessType=access_type,
warn_if_inherits=False)
except Exception as syn_ex:
logger.exception(
'Error setting permission: {0} - {1}'.format(
permission, syn_ex))
if 'a foreign key constraint fails' in str(syn_ex):
errors.append(
'User or Team ID: {0} does not exist.'.format(
principal_id))
else:
errors.append(
'Error setting permission for User or Team ID: {0}'
.format(principal_id))
except Exception as ex:
logger.exception(
'Error creating project permissions: {0} - {1}'.format(
permission, ex))
errors.append('Error creating project permissions.')
# Add the the folders
if folders:
for folder_path in folders:
folder_path_parts = list(filter(None, folder_path.split('/')))
parent = project
for folder_name in folder_path_parts:
try:
parent = Synapse.client().store(
Folder(name=folder_name, parent=parent))
except Exception as syn_ex:
logger.exception(
'Error creating project folder: {0} - {1}'.format(
folder_name, syn_ex))
errors.append(
'Error creating project folder: {0}.'.format(
folder_name))
# Add the posts
if posts:
try:
forum_id = Synapse.client().restGET(
'/project/{0}/forum'.format(project.id)).get('id')
for post in posts:
try:
body = {
'forumId': forum_id,
'title': post.get('title'),
'messageMarkdown': post.get('message_markdown')
}
Synapse.client().restPOST('/thread',
body=json.dumps(body))
except Exception as syn_ex:
logger.exception(
'Error creating project post: {0} - {1}'.format(
post, syn_ex))
errors.append(
'Error creating project post: {0}.'.format(
post.get('title', None)))
except Exception as ex:
logger.exception(
'Error creating project posts: {1} - {0}'.format(
posts, ex))
errors.append('Error creating projects posts.')
# Add the wiki
if wiki:
try:
Synapse.client().store(
Wiki(title=wiki.title,
markdown=wiki.markdown,
owner=project))
except Exception as syn_ex:
logger.exception(
'Error creating project wiki: {0} - {1}'.format(
wiki, syn_ex))
errors.append('Error creating project wiki.')
new_syn_project = SynProject.from_project(project)
return CreateSynProject(syn_project=new_syn_project,
errors=(errors if errors else None))