def login():
if request.method == "POST":
user = request.form["nm"]
main(user) # loc_map leave main(user)
return render_template("login.html")
else:
return render_template("login.html")
def index():
if request.method == 'POST':
try:
twitter.main(request.form['name'])
return redirect(url_for('map'))
except:
return render_template('error.html')
else:
return render_template('index.html')
def view():
ascii_banner = pyfiglet.figlet_format("Scrapper - An OSINT Tool")
print(ascii_banner)
print("\n Enter 1: For Instagram Scrapper >>>>>>>>\n")
print("\n Enter 2: For Twitter Scrapper >>>>>>>>\n")
i = input(">>>")
if int(i) == 1:
print(
"\n!!!!!!! The User_ID To Be Scraped Should Be Either Public Or Is Your Connect\n"
)
from insta import Instagram
elif int(i) == 2:
ta.main()
else:
print(" Invalid Option ")
print "keyword cannot be blank"
keyword = input_keyword()
elif "." in keyword:
sys.stdout.write(
"Please enter the keyword without specifying .com,.or,etc..")
raw_input()
keyword = input_keyword()
else:
break
create_queries(keyword)
twitter.main(keyword)
google.gscrapper(keyword)
def get_text(url):
r = requests.get(url, verify=False)
soup = BeautifulSoup(r.content, "html.parser")
# delete unwanted tags:
for s in soup(['figure', 'script', 'style', 'table']):
s.decompose()
article_soup = [
s.get_text(separator="\n", strip=True)
for s in soup.find_all(['p', {
'class': 'mol-para-with-font'
def post():
time = datetime.datetime.today().strftime("%H:%M:%S")
message = ""
if request.method == 'POST':
result = []
if 'you_d' in request.form:
title = "Vtuber-日間youtube動画ランキング！！"
r = ranking.main("day")
for i in range(10):
result += ["{}位".format(i + 1), r[i]]
elif 'you_w' in request.form:
title = "Vtuber週間youtube動画ランキング！！"
r = ranking.main("week")
for i in range(30):
result += ["{}位".format(i + 1), r[i]]
elif 'you_m' in request.form:
title = "Vtuber月間youtube動画ランキング！！"
r = ranking.main("month")
for i in range(30):
result += ["{}位".format(i + 1), r[i]]
elif 'nico_w' in request.form:
title = "Vtuber週間ニコニコ動画ランキング！！"
r = nico.main("week")
for i in range(30):
result += ["{}位".format(i + 1), r[i]]
elif 'nico_m' in request.form:
title = "Vtuber月間ニコニコ動画ランキング！！"
r = nico.main("month")
for i in range(30):
result += ["{}位".format(i + 1), r[i]]
elif 'nico_3m' in request.form:
title = "Vtuber四半期ニコニコ動画ランキング！！"
r = nico.main("3m")
for i in range(30):
result += ["{}位".format(i + 1), r[i]]
elif 'sou_w' in request.form:
title = "Vtuber週間総合動画ランキング！！"
r = sougou.main("week")
for i in range(30):
t = r[i][0] + ", " + str(r[i][1]) + "ポイント"
result += ["{}位".format(i + 1), t]
elif 'sou_m' in request.form:
title = "Vtuber月間総合動画ランキング！！"
r = sougou.main("month")
for i in range(30):
t = r[i][0] + ", " + str(r[i][1]) + "ポイント"
result += ["{}位".format(i + 1), t]
elif 'next_w' in request.form:
title = "次世代Vtuber週間youtube動画ランキング！！"
r = nextr.main("week")
for i in range(30):
result += ["{}位".format(i + 1), r[i]]
elif 'next_m' in request.form:
title = "次世代Vtuber月間youtube動画ランキング！！"
r = nextr.main("month")
for i in range(30):
result += ["{}位".format(i + 1), r[i]]
elif 'tw' in request.form:
title = "twitter活発度ランキング！！"
r = tw.main()
for i in range(30):
t = r[i][0] + ", " + str(int(r[i][1])) + "ポイント"
result += ["{}位".format(i + 1), t]
return render_template('index.html',
result=result,
title=title,
time=time)
else:
return redirect(url_for('index'))
def run(self):
main()
def main():
"""main function. Called after run."""
# if argument is empty or -h, print help message
if (len(sys.argv) is 1) or ('-h' in sys.argv):
print HELP_MESSAGE
return
# if argument contains -r, print the list of contents in text file.
if '-r' in sys.argv:
prev_data = read_list()
for prevhash in prev_data.keys():
print prevhash
print '\t.' + prev_data[prevhash][0][len(IMAGES_DIR):]+\
'\t' + str(prev_data[prevhash][1])
return
prev_data = {}
if not '-i' in sys.argv:
prev_data = read_list()
if '-v' in sys.argv:
print '\nPrevious file list:'
for prevhash in prev_data.keys():
print prevhash
print '\t.' + prev_data[prevhash][0][len(IMAGES_DIR):] +\
'\t' + str(prev_data[prevhash][1])
file_list = []
file_list = images_list(IMAGES_DIR, TARGET_EXT)
file_data = {}
for filepath in file_list:
sha256 = hashlib.sha256()
f = open(filepath, 'rb')
for block in iter(lambda: f.read(65536), b''):
sha256.update(block)
f.close()
file_data[sha256.hexdigest()] = (filepath, TIMESTAMP)
# if argument contains -f, print the list of current files.
if ('-v' in sys.argv) or ('-f' in sys.argv):
if '-v' in sys.argv:
print '\nCurrent file list:'
for filehash in file_data.keys():
print filehash
print '\t.' + file_data[filehash][0][len(IMAGES_DIR):] +\
'\t' + str(file_data[filehash][1])
if '-f' in sys.argv:
return
# prev_data : Data from previous run or empty data
# file_data : Data from the current file system
# new_data : Data that will be saved after the run
"""
Possible cases
In prev_data & In file_data
if path are same
prevhash : (prevpath, prevtime)
if not same
*Maybe file is moved. Doesn't affect program.
prevhash : (prevpath, prevtime)
In prev_data & Not in file_data
*File is erased or modified. Add its path to search it later.
append to missing_prevhash
Not in prev_data & In file_data
*New file is found. Add it to new_data.
filehash : (filepath, filetime)
In file_data & file path is matched
filehash : (filepath, prevtime)
"""
new_data = {}
missing_prevhash = {}
for prevhash in prev_data.keys():
if file_data.has_key(prevhash):
new_data[prevhash] = (file_data[prevhash][0],
prev_data[prevhash][1])
del file_data[prevhash]
else:
missing_prevhash[prev_data[prevhash][0]] = prev_data[prevhash][1]
for filehash in file_data.keys():
filepath = file_data[filehash][0]
if missing_prevhash.has_key(filepath):
new_data[filehash] = (filepath, missing_prevhash[filepath])
else:
new_data[filehash] = (filepath, file_data[filehash][1])
if ('-v' in sys.argv) or ('-n' in sys.argv):
print '\nUpdated file list:'
for newhash in new_data.keys():
print newhash
print '\t.' + new_data[newhash][0][len(IMAGES_DIR):] +\
'\t' + str(new_data[newhash][1])
if '-n' in sys.argv:
return
new_sorted = []
new_sorted = time_based_sort(new_data)
sorted_keys = sorted(new_sorted.keys(), key=int, reverse=True)
if ('-v' in sys.argv) or ('-N' in sys.argv):
if '-v' in sys.argv:
print '\nIntermediate file contents:'
for newtime in sorted_keys:
print newtime
for newtuple in new_sorted[newtime]:
print newtuple[0] + '\t' + newtuple[1]
if '-N' in sys.argv:
return
# new_data : filehash -> (filepath, timestamp)
# new_sorted : timestamp -> [(filehash, filepath), ...]
# sorted_keys : [timestamp, ...], most recent comes first
#Choose one image, randomly
target_img_name = ''
random.seed(TIMESTAMP)
if len(sorted_keys) == 1:
target_img = random.choice(new_sorted[sorted_keys[0]])
target_img_name = target_img[1]
new_sorted[sorted_keys[0]].remove(target_img)
new_sorted[str(int(sorted_keys[0]) - 1)] = [
target_img,
]
if len(new_sorted[sorted_keys[0]]) == 0:
del new_sorted[sorted_keys[0]]
elif len(sorted_keys) == 0:
pass
else:
if len(new_sorted[sorted_keys[0]]) == 1:
target_img = new_sorted[sorted_keys[0]][0]
target_img_name = target_img[1]
del new_sorted[sorted_keys[0]]
else:
target_img = random.choice(new_sorted[sorted_keys[0]])
target_img_name = target_img[1]
new_sorted[sorted_keys[0]].remove(target_img)
new_sorted[sorted_keys[-1]].append(target_img)
sorted_keys = [] #To prevent referring legacy data
if '-v' in sys.argv:
print 'result'
print target_img_name
#Resize and copy the chosen image to target dir.
new_img_name = ''
img = Image.open(target_img_name)
basewidth = 1000
ratio = (basewidth / float(img.size[0]))
hsize = int((float(img.size[1]) * float(ratio)))
img_new = img.resize((basewidth, hsize), Image.ANTIALIAS)
if not target_img_name == '':
if os.path.isfile(TARGET_DIR + '/' + TARGET_FILE + '.png'):
try:
os.remove(TARGET_DIR + '/' + TARGET_FILE + '.png')
except OSError as e:
print e
if os.path.isfile(TARGET_DIR + '/' + TARGET_FILE + '.jpg'):
try:
os.remove(TARGET_DIR + '/' + TARGET_FILE + '.jpg')
except OSError as e:
print e
if target_img_name[-4:] == '.png':
new_img_name = TARGET_DIR + '/' + TARGET_FILE + '.png'
img_new.save(new_img_name)
else:
new_img_name = TARGET_DIR + '/' + TARGET_FILE + '.jpg'
img_new.save(new_img_name)
#Update the imglist.txt file
write_list(new_sorted)
#Post on twitter
if '--no-twitter' in sys.argv:
return
if '-i' in sys.argv:
twitter.main()
twitter.post_on_twitter(new_img_name, TWEET_CONTENT)
return
def twitter(self, options):
twitter.main()
return True