def apply(self):
try:
if self.subject is None:
self.delete_instance()
return False
except Bot.DoesNotExist:
self.delete_instance()
return False
if self.action == 'category':
from models import Category
try:
cat = Category.get(Category.id == self.value)
self.subject.category = cat
except Category.DoesNotExist:
raise AttributeError("Category to change to does not exist.")
elif self.action == 'name':
self.subject.name = self.value
elif self.action == 'username':
self.subject.username = self.value
elif self.action == 'description':
self.subject.description = self.value
elif self.action == 'extra':
self.subject.extra = self.value
elif self.action == 'country':
if self._value == 'None' or self._value is None:
self.subject.country = None
else:
from models import Country
try:
con = Country.get(id=self._value)
self.subject.country = con
except Country.DoesNotExist:
raise AttributeError(
"Country to change to does not exist.")
elif self.action == 'inlinequeries':
self.subject.inlinequeries = bool(self.value)
elif self.action == 'official':
self.subject.official = bool(self.value)
elif self.action == 'offline':
self.subject.offline = bool(self.value)
elif self.action == 'spam':
self.subject.spam = bool(self.value)
elif self.action == 'add_keyword':
kw_obj = Keyword(name=self.value, entity=self.subject)
kw_obj.save()
elif self.action == 'remove_keyword':
try:
kw_obj = Keyword.get(name=self.value, entity=self.subject)
kw_obj.delete_instance()
except Keyword.DoesNotExist:
raise AttributeError(
"Keyword to disable does not exist anymore.")
self.subject.save()
self.executed = True
self.save()
return True
def add_keyword(request, room_title):
room = get_object_or_404(Room, title=room_title)
keyword_str = request.POST['keyword'].strip()
keywords = room.keyword_set.filter(name=keyword_str).all()
if len(keywords)==0:
keyword = Keyword(name=keyword_str, room=room)
keyword.save()
return HttpResponse('OK')
def keyword_addition(paper_id, keywords):
paper = Paper.objects.get(pk=paper_id)
print "Extracting from..", paper
#extract top 20
keywords = keywords[:20]
print keywords
count = 0
for items in keywords:
kw = Keyword(keyword= items[0], paper= paper)
kw.save()
count = count + 1
print count, " keywords stored"
def submit():
word = request.form["keyword"]
if not word:
return "<div>Please input keyword!</div>"
keywords = Keyword.objects.filter(word=word)
if keywords:
keyword = keywords[0]
else:
keyword = Keyword(word=word)
domain = whois.query("%s.com" % str(word))
keyword.website = {"exist": bool(domain)}
for platform in PLATFORMS:
keyword.__setattr__(platform, {"exist": fetch(platform, word)})
keyword.save()
platforms = dict((key, keyword.__getattribute__(key)) for key in PLATFORMS)
platforms["website"] = keyword.website
return render_template("result.html", platforms=platforms)
def add(request):
name=request.POST['name']
url=request.POST['url']
number=request.POST['number']
keywords=request.POST['words']
"""
If the website already exists, then we update it. Otherwise, we create
a new one.
"""
w1=Website.objects.filter(url=url)
"""
At this point, the previous entry for this website is deleted, rather than
updated. The idea is that it is faster to delete the old record, rather
than searching through each of its fields to check which ones need to be
updated and which do not.
"""
if w1!=[]:
w1.name=name
w1.number=number
w1.delete()
#Now we add the website to the database.
w1=Website(name=name, url=url, number=number)
w1.save()
"""Remove blank spaces from words and separate into separate entries in
a list, rather than just a string.
"""
keywords=stl(keywords)
"""
Add the keywords that describe this Website. If the keyword already
exists, then we find it and name it k1. Otherwise, we create a
new Keyword.
"""
for keyword in keywords:
try:
k1=Keyword.objects.get(name=keyword)
w1.words.add(k1)
except Keyword.DoesNotExist:
#Create the new Keyword and save it.
k1=Keyword(name=keyword)
k1.save()
#add this word to the Website
w1.words.add(k1)
def carga_keywords():
# aqui se va a abrir el archivo txt y se metera en la base de datos
contador = 0
try:
with open('keywords.txt') as file:
for line in file:
line = line.replace('\n', '').lower()
if line != "":
keywords = Keyword(line)
keywords.save()
# aqui voy a llamar a comprobar y me retorna la posicion
contador += 1
key = Keyword(line, 12)
key.update()
except FileNotFoundError:
print("No se encuentra Fichero keywords.txt")
return Keyword.get_all()
def dbInsertKeywords(keywordList, lazy=False):
keywordList = list(
filter(None, [strToKeyword(phrase) for phrase in keywordList]))
if lazy:
if DEBUG:
(print(
"dbInsertKeywords: Using Lazy DB check, no inserting or updating\n"
))
keywordObjects = []
for phrase in keywordList:
keyQuery = Keyword.objects(key=phrase)
if (keyQuery): keywordObjects.append(keyQuery[0])
if DEBUG: (print("dbInsertKeywords: Returning lazy DB query\n"))
return list(set(keywordObjects))
if DEBUG: (print("dbInsertKeywords: Taking keyword string list...\n"))
#static URL for keywordseverywhere bulk tool
if DEBUG: (print(""))
URL = "https://keywordseverywhere.com/ke/4/manual.php"
if DEBUG:
(print("dbInsertKeywords: Scraping for given keywords at %s...\n",
URL))
#setup browser options
chrome_options = Options()
chrome_options.add_argument("user-data-dir=/tmp/tarun")
download_dir = r"C:\Users\Oliver\Desktop\Misc Code\python\KC\temp"
preferences = {
"download.default_directory": download_dir,
"directory_upgrade": True,
"safebrowsing.enabled": True
}
chrome_options.add_experimental_option("prefs", preferences)
if DEBUG: (print("dbInsertKeywords: Getting webdriver for chrome...\n"))
driver = webdriver.Chrome(
executable_path=r'C:\Users\Oliver\dev\cfehome\chromedriver.exe',
options=chrome_options)
fileName = ''
keywordObjectReturnList = []
keywordLists = chunks(list(set(keywordList)), 2000)
for chunk in keywordLists:
driver.get(URL)
#find and enter to form
if DEBUG:
(print("dbInsertKeywords: Trying to find keyword form...\n"))
form = driver.find_element_by_id("keywords")
text = ", ".join(chunk)
if DEBUG:
(print(
"dbInsertKeywords: Copying to clipboard with pyperclip.copy()...\n"
))
pyperclip.copy(text)
if DEBUG: (print("dbInsertKeywords: Pasting with os Keys...\n"))
form.send_keys(Keys.CONTROL, 'v')
if DEBUG: (print("dbInsertKeywords: Finding submit button...\n"))
button = driver.find_element_by_id("submit")
if DEBUG: (print("dbInsertKeywords: Clicking submit button...\n"))
button.click()
time.sleep(2)
if DEBUG:
(print("dbInsertKeywords: Finding CSV download button...\n"))
CSVDownloadButton = driver.find_element_by_class_name("buttons-csv")
if DEBUG:
(print(
"dbInsertKeywords: Trying to click submit button until CSV button is displayed...\n"
))
while not CSVDownloadButton.is_displayed():
try:
button.click()
if DEBUG: (print("dbInsertKeywords: clicked [SUBMIT]...\n"))
except:
pass
time.sleep(1)
if DEBUG:
(print("dbInsertKeywords: Clicking CSV download button...\n"))
CSVDownloadButton.click()
time.sleep(1)
if DEBUG:
(print(
"dbInsertKeywords: Trying to find file that was downloaded...\n"
))
name = str.lower(chunk[0]).replace("&", "").replace(
"|", "").strip().replace(" ", "-")
if name == "":
name = "csv"
fileName = "C:\\Users\\Oliver\\Desktop\\Misc Code\\python\\KC\\temp\\" + name + ".csv"
if DEBUG: (print("dbInsertKeywords: Opening as CSV...\n"))
try:
csv_file = open(fileName)
except NameError:
print("No file found")
return []
reader = csv.reader(csv_file, delimiter=',')
titleRowSkip = False
if DEBUG: (print("dbInsertKeywords: Adding/updating keywords...\n"))
for row in reader:
if titleRowSkip:
phrase = row[1]
vol = row[5]
cpc = row[6]
competition = row[7]
keyQuery = Keyword.objects(key=phrase)
if (len(phrase) < 20):
if (not keyQuery):
kw = Keyword(key=phrase,
volume=vol.replace(",", ""),
cpc=cpc.replace("$", ""),
competition=competition)
kw.save()
keyQuery = kw
print("Saved New keyword:")
print(keyQuery.key)
else:
keyQuery.update_one(set__volume=vol.replace(",", ""))
keyQuery.update_one(set__cpc=cpc.replace("$", ""))
keyQuery.update_one(set__competition=competition)
keyQuery = keyQuery[0]
print("Updated keyword:")
print(keyQuery.key)
if (keyQuery): keywordObjectReturnList.append(keyQuery)
titleRowSkip = True
if DEBUG: (print("dbInsertKeywords: Closing CSV...\n"))
csv_file.close()
if DEBUG:
(print("dbInsertKeywords: Closing browser and deleting CSV file...\n")
)
driver.close()
os.remove(fileName)
if DEBUG: (print("dbInsertKeywords: Returning Keyword Object List...\n"))
return keywordObjectReturnList
