def initConfig(self):
config = self.config
idType = self.__getInput("What do you want to integrate (ORGANIZATION/PROJECT)", ["ORGANIZATION", "PROJECT"])
config.setIdType(idType)
id = self.__getInput("Enter your " + idType + " Id ", None)
config.setId(id)
util = Util(config)
if idType == "ORGANIZATION":
self.printProjectList(util.getProjectList(), "\nProjects")
serviceAccountProjectId = self.__getInput("Enter the projectId where you want to create the Service Account", None)
config.setServiceAccountProjectId(serviceAccountProjectId)
else:
config.setServiceAccountProjectId(id)
self.printAPI("\nAPIs to be enabled")
self.printProjectList(util.getProjectList(), "\nProjects")
enableApi = self.getYesNoPrompt("Do You want to enable APIs in the projects(yes/no)")
config.setEnableApi(enableApi)
self.printRole("\nRoles Required")
modifyIamPolicy = self.getYesNoPrompt("Do you want to modify "+ config.getIdType() + " IAM Policy(yes/no)")
config.setSetIAMPolicy(modifyIamPolicy);
self.review(util)
flag = self.getYesNoPrompt("\nDo You Want to continue(yes/no)")
if not flag:
exit(1)
def __init__(self, min_instances, max_instances, warmuptime, prefix,
healthEndpoint):
self.prefix = prefix
self.min_instances = min_instances
self.max_instances = max_instances
self.desired_instances = min_instances
self.warmuptime = warmuptime
self.list_instances = []
self.stopped_instances = []
self.random = str(random.randint(1, 9999))
self.security_group, self.security_group_id = self.create_security_group(
)
self._is_scaling = False
self._lock = threading.Lock()
self.instance_management_thread = threading.Thread(
target=self.instance_management)
self.watch_usage_thread = threading.Thread(target=self.watch_usage)
self.healthEndpoint = healthEndpoint
self.CpuUtilizeMaxMetric = CpuUtilizeMetric(60, 5, "ScaleOut")
self.CpuUtilizeMinMetric = CpuUtilizeMetric(30, 10, "ScaleIn")
self.config = Util().get_config()
self.AcceptableScaleOutBacklogMetric = AcceptableBacklogMetric(
40, 40, self.config.get('dev', 'REQUESTS_SQS'), "ScaleOut")
self.AcceptableScaleInBacklogMetric = AcceptableBacklogMetric(
40, 40, self.config.get('dev', 'REQUESTS_SQS'), "ScaleIn")
self.ScaleInMetric = [self.AcceptableScaleInBacklogMetric]
self.ScaleOutMetric = [self.AcceptableScaleOutBacklogMetric]
self.__init()
def create_app():
app = Flask(__name__)
with app.app_context():
config = Util().get_config()
request_sqs = config.get('dev', 'REQUESTS_SQS')
response_sqs = config.get('dev', 'RESPONSE_SQS')
sqsworker = SQSWorker(60, request_sqs, response_sqs, 360, 1)
thread = threading.Thread(target=sqsworker.listener)
sqsworker.recurrentDeleteMessageJob.start()
thread.start()
@app.route('/health', methods=['GET'])
def get_health():
result = {'Response': 'Done'}
return make_response(jsonify(result), 200)
@app.route('/averageResponseTime', methods=['GET'])
def get_averageResponseTime():
result = {'Response': 'Done'}
return make_response(jsonify(result), 200)
@app.route('/stop', methods=['GET'])
def stop_listner():
sqsworker.stopListner = True
result = {'Response': 'Done'}
return make_response(jsonify(result), 200)
return app
def run(self):
self.util = Util(self.config)
util = self.util
appManager = AppManager(self.config, util)
api_success_list, api_error_list, service_account, key, setIamPolicyStatus = appManager.run()
if api_success_list and len(api_success_list) != 0:
self.printProjectList(api_success_list, "Successfully Enabled APIs in following projects")
if api_error_list and len(api_error_list) != 0:
self.printProjectErrorList(api_error_list, "Error Enabling APIs in following projects")
if self.config.getSetIAMPolicy():
self.printIamPolicyStatus(setIamPolicyStatus, "IAM Policy Set Status")
self.printInterationData(service_account, key)
if key:
try:
path = os.getcwd() +"/credentials.txt"
self.writeToFile(service_account, key, path)
logging.info("Copy Of Credentials written to file: " + path)
except:
logging.exception("Could not write data to file ")
def __init__(self):
super(DCWrapper, self).__init__()
self.util = Util()
self.store = Store()
self.sleep_time = 5 # 没有任务休眠时间
self.sending_queue_max = 8000 # sending_queue 最大值
self.send_url_to_sended_queue()
def __init__(self):
self.util = Util()
super(MiSpider, self).__init__()
self.extractor = MiExtractor()
self.keywords = self.util.get_keywords()
self.phone_url = 'http://app.mi.com/searchAll?keywords={}&typeall=phone&page={}'
self.pad_url = 'http://app.mi.com/searchAll?keywords={}&typeall=pad&page={}'
def __init__(self):
util = Util()
self.cookies = util.get_cookies()
self.headers = util.get_headers()
self.proxies = util.get_proxies()
print(self.headers)
print(self.cookies)
def delegate_check():
c = Config()
u = Util(c)
tasks = get_tasks(c,u)
# Async Loop
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(asyncio.wait(tasks))
finally:
loop.close()
def validate(self):
self.util = Util(self.config)
config = self.config
if config.getIdType() == "ORGANIZATION" and not config.getId().isdigit():
raise Exception("Invalid org id")
# Validate service account project id
for project in self.util.getProjectList():
if project['projectId'] == config.getServiceAccountProjectId():
return True
raise Exception("Project Id is not valid: " + config.getServiceAccountProjectId())
class BasePage:
_base_url = ""
util = Util()
def __init__(self, driver: WebDriver = None, reuse=False):
if driver is None:
if reuse:
self.util.reuse_chrome()
else:
self.util.open_chrome_gui_browser()
self.util.driver.implicitly_wait(5)
else:
self.util.driver = driver
if self._base_url != "":
self.util.driver.get(self._base_url)
def __init__(self, listenerTimeOut, requestQueueName, responseQueueName,
visibilityTimeOut, numberofRequestFetch):
self.processedReqeuests = queue.Queue()
self.listenerTimeOut = listenerTimeOut
self.responseListenerTimeOut = 5
self.requestQueueName = requestQueueName
self.responseQueueName = responseQueueName
self.visibilityTimeOut = visibilityTimeOut
self.numberOfRequestFetched = numberofRequestFetch
self.messageReceiverTimeout = 10
self.requestsQueueUrl = None
self.responseQueueUrl = None
self.stopListner = False
self.avrageResponseTime = 40
self.__init_queue_url()
self.recurrentDeleteMessageJob = RepeatedTimer(30, self.delete_message)
self.config = Util().get_config()
self.darknetTargets = [
'/home/ubuntu/distributedSurvellienceSystem/darknet',
'/home/ubuntu/distributedSurvellienceSystem/darknet2',
'/home/ubuntu/distributedSurvellienceSystem/darknet3'
]
self.responseDict = defaultdict()
def __init__(self):
self.util = Util()
self.writed_content = list()
def __init__(self):
self.util = Util()
super(TencentSpider, self).__init__()
self.extractor = TencentExtractor()
self.keywords = self.util.get_keywords()
self.base_url = 'https://sj.qq.com/myapp/searchAjax.htm?kw={}&pns={}&sid=0'
def lambda_handler(event, context):
util1 = Util()
print(util1.get_current_time_utc())
def __init__(self):
self.util = Util()
super(HWSpider, self).__init__()
self.extractor = HWExtractor()
self.keywords = self.util.get_keywords()
self.url = 'http://app.hicloud.com/search/{}/{}'
# remove non-accepted transactions from being marked as completed
if len(for_removal) > 0:
for i in for_removal:
print("Removing RowId: ", i)
unique_rowid.remove(i)
snekdb.processStagedPayment(unique_rowid)
# payment run complete
print('Payment Run Completed!')
# sleep 10 minutes between tx blasts
time.sleep(600)
else:
time.sleep(150)
if __name__ == '__main__':
data = Config()
network = Network(data.network)
u = Util(data.network)
snekdb = SnekDB(data.database_user, data.network, data.delegate)
client = u.get_client(network.api_port)
build_network()
#get dot path for load_env and load
dot = u.core + '/.env'
load_dotenv(dot)
share()
def scan(self):
bruteforce_list = set()
search_list = set()
if is_windows:
subdomains_queue = list()
else:
subdomains_queue = multiprocessing.Manager().list()
# Check Bruteforce Status
# if self.scan_flags.BruteForce or self.scan_flags.BruteForce is None:
# self.scan_flags.BruteForce = True
# Check Takeover Status
# if self.scan_flags.TakeoverCheck or self.scan_flags.TakeoverCheck is None:
# self.scan_flags.TakeoverCheck = True
# Validate domain
domain_check = re.compile(
"^(http|https)?[a-zA-Z0-9]+([\-.][a-zA-Z0-9]+)*\.[a-zA-Z]{2,}$")
if not domain_check.match(self.domain):
if not self.scan_flags.Silent:
print(self.logger.R + "Error: Please enter a valid domain" +
self.logger.W)
return []
if not self.domain.startswith(
'http://') and not self.domain.startswith('https://'):
self.domain = 'http://' + self.domain
parsed_domain = urlparse.urlparse(self.domain)
if not self.scan_flags.Silent:
print(self.logger.B + "[-] Enumerating subdomains now for %s" %
parsed_domain.netloc + self.logger.W)
if self.scan_flags.Verbose and not self.scan_flags.Silent:
print(
self.logger.Y +
"[-] verbosity is enabled, will show the subdomains results in realtime"
+ self.logger.W)
chosenEnums = []
if self.scan_flags.Engines is None:
chosenEnums = Engines.supported_engines.values()
else:
engines = self.scan_flags.Engines.split(',')
for engine in engines:
if engine.lower() in Engines.supported_engines:
chosenEnums.append(
Engines.supported_engines[engine.lower()])
# Start the engines enumeration
enums = [
enum(self.domain, [],
q=subdomains_queue,
silent=self.scan_flags.Silent,
logger=self.logger) for enum in chosenEnums
]
for enum in enums:
enum.start()
for enum in enums:
enum.join()
subdomains = set(subdomains_queue)
for subdomain in subdomains:
search_list.add(subdomain)
if self.scan_flags.BruteForce:
if not self.scan_flags.Silent:
print(self.logger.G +
"[-] Starting bruteforce module now using subbrute.." +
self.logger.W)
record_type = False
path_to_file = os.path.dirname(os.path.realpath(__file__))
subs = os.path.join(path_to_file, 'subbrute', 'names.txt')
resolvers = os.path.join(path_to_file, 'subbrute', 'resolvers.txt')
process_count = self.scan_flags.ThreadCount
output = False
json_output = False
bruteforce_list = subbrute.print_target(parsed_domain.netloc,
record_type, subs,
resolvers, process_count,
output, json_output,
search_list,
self.scan_flags.Verbose)
subdomains = search_list.union(bruteforce_list)
if subdomains:
subdomains = sorted(subdomains, key=Util().subdomain_sorting_key)
if self.scan_flags.SaveFile:
print("%s[-] Saving results to file: %s%s%s%s" %
(self.logger.Y, self.logger.W, self.logger.R,
self.scan_flags.SaveFile, self.logger.W))
Util().write_file(self.scan_flags.SaveFile, subdomains)
if not self.scan_flags.Silent:
print(self.logger.Y + "[-] Total Unique Subdomains Found: %s" %
len(subdomains) + self.logger.W)
if self.scan_flags.TakeoverCheck:
print(
self.logger.G +
"[-] Checking for subdomains pointing to unregistered services"
+ self.logger.W)
for subdomain in subdomains:
if self.scan_flags.Verbose:
print(self.logger.G + "[-] Checking " + subdomain +
self.logger.W)
services = Util().get_url_signatures("http://" + subdomain)
if len(services) > 0:
for service in services:
print(self.logger.Y +
"[-] Found unregistered service \"" +
service + "\" on subdomain " + subdomain +
self.logger.W)
if self.scan_flags.Ports:
if not self.scan_flags.Silent:
print(
self.logger.G +
"[-] Starting port scan for the following ports: %s%s" %
(self.logger.Y, self.scan_flags.Ports) + self.logger.W)
ports = self.scan_flags.Ports.split(',')
pscan = PortScanner(subdomains, ports)
pscan.run()
elif not self.scan_flags.Silent:
for subdomain in subdomains:
print(self.logger.G + subdomain + self.logger.W)
return subdomains
def create_app():
app = Flask(__name__, instance_relative_config=True)
with app.app_context():
config = Util().get_config()
sqsworker = SQSWorker(60, "request.fifo", "response.fifo", 240, 1)
_set = set()
pollerThread = threading.Thread(target=sqsworker.response_poller)
pollerThread.start()
@app.route("/")
def get_default():
url = config.get('dev', 'VIDEO_URL')
remotefile = urllib.request.urlopen(url)
blah = remotefile.info()['Content-Disposition']
_, params = cgi.parse_header(blah)
filename = params["filename"]
uploadFlag = True
lock.acquire()
if filename in _set:
uploadFlag = False
lock.release()
d = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
filepath = os.path.join(d, "data", filename)
bucket_name = config.get('dev', 'BUCKET_NAME')
lock.acquire()
if uploadFlag:
result = urllib.request.urlretrieve(url, filepath)
filepath = result[0]
upload_to_s3(bucket_name, filepath, filename)
_set.add(filename)
lock.release()
#thread = threading.Thread(target=upload_to_s3, args=(bucket_name, filename, filepath))
#Upload the video to s3
print(filepath, filename)
message_attribute = {
'Filename': {
'DataType': 'String',
'StringValue': filename
}
}
message_body = "Filename" + filename
print(filename)
groupid = "distributedSurvellienceSystem"
messageId = sqsworker.send_message_sqs(filename, message_attribute,
message_body, groupid)
response = ResponseObject()
#response.status = "Success"
response.messageId = messageId
response.videoId = filename
thread = threading.Thread(target=wait_func, args=(messageId, ))
thread.start()
thread.join()
resp = sqsworker.responseDict[messageId]
response.response = resp.strip()
response.status = "Success"
#_set.remove(filename)
return "[" + response.videoId + "," + response.response + "]"
@app.route("/getMessageId")
def getMessageId():
url = config.get('dev', 'VIDEO_URL')
remotefile = urllib.request.urlopen(url)
blah = remotefile.info()['Content-Disposition']
_, params = cgi.parse_header(blah)
filename = params["filename"]
d = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
filepath = os.path.join(d, "data", filename)
result = urllib.request.urlretrieve(url, filepath)
filepath = result[0]
bucket_name = config.get('dev', 'BUCKET_NAME')
#thread = threading.Thread(target=upload_to_s3, args=(bucket_name, filename, filepath))
#Upload the video to s3
print(filepath, filename)
upload_to_s3(bucket_name, filepath, filename)
message_attribute = {
'Filename': {
'DataType': 'String',
'StringValue': filename
}
}
message_body = "Filename" + filename
print(filename)
groupid = "distributedSurvellienceSystem"
messageId = sqsworker.send_message_sqs(filename, message_attribute,
message_body, groupid)
response = ResponseObject()
response.status = "Success"
response.messageId = messageId
response.videoId = filename
return json.dumps(response.__dict__)
@app.route("/getResponse")
def getResponse():
response = ResponseObject()
print(sqsworker.responseDict)
messageId = request.args.get('messageId')
print(messageId)
if messageId in sqsworker.responseDict:
resp = sqsworker.responseDict[messageId]
response.response = resp
response.status = "Success"
response.messageId = messageId
print('Found')
print("SQS ", sqsworker.responseDict)
return json.dumps(response.__dict__)
def upload_to_s3(bukcet_name, filepath, filename):
s3 = boto3.resource('s3')
bucket = s3.Bucket(bukcet_name)
bucket.upload_file(filepath, filename)
#s3 = boto3.client("s3")
#s3.upload_file(key,bukcet_name,content)
def wait_func(messageId):
while True:
if messageId in sqsworker.responseDict:
break
time.sleep(2)
class ResponseObject:
def __init__(self):
self.status = "Failure"
self.response = None
self.messageId = None
self.videoId = None
return app
