def create(self, validated_data):
"""
Create a new Content instance.
Note that lots of saving process is done in views.py and models.py.
"""
c = Content(**validated_data)
c.save()
return c
def testNewContentFromTestContentDir(self):
self.assertTrue(
os.path.isdir(TEST_CONTENT_DIR),
"Directory '%s' containing test files does not exist." %
TEST_CONTENT_DIR)
files = os.listdir(TEST_CONTENT_DIR)
print("\n\nFAILEJA", TEST_CONTENT_DIR, files)
self.assertGreater(
len(files), 0, "Directory '%s' containing test files is empty." %
TEST_CONTENT_DIR)
cnt = 0
for filename in files:
cnt += 1
c = Content(caption="New content #%d" % cnt)
full_path = os.path.join(TEST_CONTENT_DIR, str(filename))
c.set_file(str(filename), full_path)
c.set_fileinfo()
c.generate_thumbnail()
c.save()
maintype = c.mimetype.split("/")[0]
print("MIMETYYPPI", c.mimetype, c.preview)
if maintype in ["video", "audio"]:
ffp = content.filetools.FFProbe(c.file.path)
info = content.filetools.fileinfo(c.file.path)
print(info)
if ffp.is_video():
new_video, cmd_str, output = create_videoinstance(
c.file.path)
vi = Videoinstance(content=c)
vi.save()
vi.set_file(new_video, "webm")
vi.set_metadata(info)
vi.save()
print("%s %.1f sec %dx%d pix" %
(vi.mimetype, vi.duration, vi.width, vi.height))
if ffp.is_audio():
new_audio, cmd_str, output = create_audioinstance(
c.file.path)
ai = Audioinstance(content=c)
ai.save()
ai.set_file(new_audio, "ogg")
ai.set_metadata(info)
ai.save()
print("%s %.1f sec" % (ai.mimetype, ai.duration))
self.all_content.append(c)
# self.assertEqual(c.file.path, "sd", c.file.path)
# import time
# time.sleep(20)
self.assertEqual(Content.objects.count(), len(self.all_content),
"1 or more files failed")
def simple_upload(request):
"""
Handles uploaded files
"""
try:
if request.method == 'POST': # If the form has been submitted...
user = request.user if request.user.is_authenticated() else None
form = UploadForm(request.POST, request.FILES) # A form bound to the POST data
if form.is_valid(): # File was posted with form
c = None
kwargs = {}
kwargs['author'] = user.username.title() if user else 'Anonymous'
# Create a new Egg
e = Egg(**kwargs)
print kwargs
for filefield, tmpname in handle_uploaded_file(request):
print "HANDLING FILE:", filefield, tmpname
c = Content(**kwargs)
originalname = str(request.FILES["file"])
# c.user = request.user # Only authenticated users can use this view
c.set_file(originalname, tmpname) # Save uploaded file to filesystem
c.get_type_instance() # Create thumbnail if it is supported
c.save()
e.content = c
e.uid = c.uid
break # We save only the first file
if c:
c.user = user
c.save()
e.user = user
print "USER", user
if user:
response_status = 200 # Created
else:
response_status = 401 # Unauthorized
e.save()
# We can handle tags after egg has id (it is saved)
response = HttpResponse(status=response_status)
#response.status_code = 201
# FIXME: use reverse()
#return HttpResponseRedirect(reverse('egg api', args=[e.uid]))
response['Location'] = '/fourdnest/api/v1/egg/%s/' % e.uid
return response
else:
response = HttpResponse(status=204)
return response
else:
raise Http404
except Exception, err:
print err
raise
return HttpResponse("Server error", status=500)
def post(request):
if(request.method == 'POST'):
title = request.POST['title']
tags = request.POST['tags']
abstract = request.POST.get('abstract',"d")
image = request.POST['image']
links = request.POST['links']
date = datetime.datetime.now()
content = Content(title=title,tags=tags,abstract=abstract,image=image,links=links,date=date)
content.save()
return render(request,'content/home.html',{})
else:
return render(request,'content/publish.html',{})
def testNewContentSaving(self):
self.assertTrue(
os.path.isdir(TEST_CONTENT_DIR),
f"Directory '{TEST_CONTENT_DIR}' containing test files does not exist."
)
files = os.listdir(TEST_CONTENT_DIR)
self.assertGreater(
len(files), 0,
f"Directory '{TEST_CONTENT_DIR}' containing test files is empty.")
cnt = 0
for filename in files:
full_path = os.path.join(TEST_CONTENT_DIR, str(filename))
if os.path.isfile(full_path) is False:
print(f"Skip {full_path}, not a file")
continue
cnt += 1
c = Content(caption="New content #%d" % cnt)
c.set_file(str(filename), full_path)
c.set_fileinfo()
c.generate_thumbnail()
c.save()
maintype = c.mimetype.split("/")[0]
if maintype in ["video", "audio"]:
ffp = content.filetools.FFProbe(c.file.path)
info = content.filetools.fileinfo(c.file.path)
print(info)
if ffp.is_video():
new_video, cmd_str, output = create_videoinstance(
c.file.path)
vi = Videoinstance(content=c)
vi.save()
vi.set_file(new_video, "webm")
vi.set_metadata(info)
vi.save()
print(
f"{c.mimetype} {c.preview} {vi.mimetype} {vi.duration:.1f} sec {vi.width}x{vi.height} px"
)
if ffp.is_audio():
new_audio, cmd_str, output = create_audioinstance(
c.file.path)
ai = Audioinstance(content=c)
ai.save()
ai.set_file(new_audio, "ogg")
ai.set_metadata(info)
ai.save()
print(
f"{c.mimetype} {c.preview} {ai.mimetype} {ai.duration:.1f} sec"
)
self.all_content.append(c)
def testNewContentSaving(self):
self.assertTrue(os.path.isdir(TESTCONTENT_DIR), "Directory '%s' containing test files does not exist." % TESTCONTENT_DIR)
files = os.listdir(TESTCONTENT_DIR)
self.assertGreater(len(files), 0, "Directory '%s' containing test files is empty." % TESTCONTENT_DIR)
cnt = 0
for filename in files:
cnt += 1
c = Content(caption=u'New content #%d' % cnt)
full_path = os.path.join(TESTCONTENT_DIR, filename)
if not os.path.isfile(full_path):
continue
c.set_file(filename, full_path)
c.set_fileinfo()
c.generate_thumbnail()
c.save()
maintype = c.mimetype.split('/')[0]
print "MIMETYYPPI", c.mimetype, c.preview
if maintype in ['video', 'audio']:
ffp = content.filetools.FFProbe(c.file.path)
info = content.filetools.fileinfo(c.file.path)
print info
#finfo = get_ffmpeg_videoinfo(c.file.path)
#print finfo
if ffp.is_video():
new_video, cmd_str = create_videoinstance(c.file.path)
vi = Videoinstance(content=c)
vi.save()
vi.set_file(new_video, 'webm')
#info = get_videoinfo(get_ffmpeg_videoinfo(vi.file.path))
vi.set_metadata(info)
vi.save()
print u'%s %.1f sec %dx%d pix' % (vi.mimetype, vi.duration, vi.width, vi.height)
if ffp.is_audio():
new_audio, cmd_str = create_audioinstance(c.file.path)
ai = Audioinstance(content=c)
ai.save()
ai.set_file(new_audio, 'ogg')
#info = get_audioinfo(get_ffmpeg_videoinfo(ai.file.path))
#print info
ai.set_metadata(info)
ai.save()
print u'%s %.1f sec' % (ai.mimetype, ai.duration)
#print c.get_type_instance()
#print c.caption
self.all_content.append(c)
#self.assertEqual(c.file.path, "sd", c.file.path)
#import time
#time.sleep(20)
self.assertEqual(Content.objects.count(), len(self.all_content), "1 or more files failed")
def testNewContentFromTestContentDir(self):
self.assertTrue(os.path.isdir(TESTCONTENT_DIR), "Directory '%s' containing test files does not exist." % TESTCONTENT_DIR)
files = os.listdir(TESTCONTENT_DIR)
self.assertGreater(len(files), 0, "Directory '%s' containing test files is empty." % TESTCONTENT_DIR)
cnt = 0
for filename in files:
cnt += 1
c = Content(caption=u'New content #%d' % cnt)
full_path = os.path.join(TESTCONTENT_DIR, filename)
c.set_file(filename, full_path)
c.save()
#print c.get_type_instance()
#print c.caption
self.all_content.append(c)
#self.assertEqual(c.file.path, "sd", c.file.path)
self.assertEqual(Content.objects.count(), len(self.all_content), "1 or more files failed")
def test_fully_content_save ( self ):
""" 测试signal是否正确执行
"""
c1 = Content ( slug = "test_signal", body = "one" )
c1.save ( )
c2 = Content.objects.get ( slug = "test_signal" )
c2.body = "one\ntwo"
c2.save ( )
self.assertEqual ( 1, Content.objects.get ( slug = "test_signal" )\
.revisions )
revision1 = c2.changelog_set.all ()[0]
self.assertEqual ( 1, revision1.additions )
self.assertEqual ( 0, revision1.deletions )
self.assertEqual ( 1, revision1.revision )
self.assertEqual ( self.REVISION_DIFF, revision1.body )
def find_content(self):
content = Content.get_next()
text = ''
if content.text:
text += f'"{content.text}"\n'
text += f' - {content.creator_nickname} ({content.creator.name})\n'
date = content.create_date.strftime("%B %d, %Y")
text += f' {date}'
return content.upload.url
def addcontent(request):
if request.method == 'POST':
form = ContentForm(request.POST, request.FILES)
if form.is_valid():
current_user = request.user
data = Content()
data.user_id = current_user.id
data.title = form.cleaned_data['title']
data.keywords = form.cleaned_data['keywords']
data.description = form.cleaned_data['description']
data.thumbnail = form.cleaned_data['thumbnail']
data.type = form.cleaned_data['type']
data.slug = form.cleaned_data['slug']
data.detail = form.cleaned_data['detail']
data.status = 'False'
data.save()
messages.success(request, 'İçeriğin başarıyla eklendi.')
return HttpResponseRedirect('/user/contents')
else:
messages.warning(request, 'Content Form Hata:' + str(form.errors))
return HttpResponseRedirect('/user/addcontent')
else:
setting = Settings.objects.get(pk=1)
category = Category.objects.all()
form = ContentForm()
context = {
'setting': setting,
'category': category,
'form': form,
}
return render(request, 'user_addcontent.html', context)
def savefiles(msg, simulate):
"""
Extract parts from msg (which is an email.message_from_string(str) instance)
and send them to the database.
NOTES:
- uses only the first found email address to assume recipient
TODO stuff
- reject if From: is empty
"""
part_counter = 1
subject = get_subject(msg)
tos = get_recipient(msg)
#print tos
msg_id = msg.get('message-id', '')
froms = msg.get_all('from', [])
p = re.compile('([\w\.\-]+)@')
try: # May raise in some cases IndexError: list index out of range
matches = p.findall(froms[0])
sender_nick = matches[0].split(
".")[0].title() # Use all before first '.'
except:
print "ERROR: No From header %s" % (msg_id)
return False
if len(tos) == 0:
print "ERROR: No Tos found %s" % (msg_id)
return False
p = re.compile('([\w]+)\.([\w]+)@') # e.g. [email protected]
matches = p.findall(tos[0])
if len(matches) > 0:
username = matches[0][0].title()
key = matches[0][1].lower()
else:
print "ERROR: No user.authkey found from %s %s" % (tos[0], msg_id)
return False
#print "User, key:", username, key
# TODO: replace this with AuthTicket stuff
#from django.contrib.auth import authenticate
#user = authenticate(authtoken='qwerty123')
try:
user = User.objects.get(username=username.lower())
except User.DoesNotExist:
print "User.DoesNotExist !", username
log.warning("User.DoesNotExist: '%s'" % username)
return False
contentgroup = None
if user.albumitgroups.count() > 0:
contentgroup = user.albumitgroups.all()[0]
sourceorg = None
if user.sourceorgs.count() > 0:
sourceorg = user.sourceorgs.all()[0]
photographer_name = sender_nick
photographer = None
if sourceorg and sourceorg.photographers.count() > 0:
p = sourceorg.photographers.all()[0]
photographer_name = u'{} {}'.format(p.firstname, p.lastname)
photographer = p
#privacy = 'PRIVATE'
privacy = 'RESTRICTED'
if key.lower() == 'pub':
privacy = 'PUBLIC'
elif key.lower() == 'res':
privacy = 'RESTRICTED'
else:
log.warning("Privacy part not found: '%s'" % key)
parts_not_to_save = [
"multipart/mixed",
"multipart/alternative",
"multipart/related",
"text/plain",
]
if simulate: # Print lots of debug stuff
print u'=========\nMetadata:\n========='
print u'''Subject: %s\nUsername: %s\nFrom: %s\nTo: %s\nM-id: %s\n(%s)''' % (
subject, user, u','.join(froms), u','.join(tos), msg_id, privacy)
print u'=========\nParts:\n========='
saved_parts = 0
log.info("Walking through message parts")
for part in msg.walk():
part_content_type = part.get_content_type()
filename, filedata = handle_part(part)
if part_content_type in parts_not_to_save or filename is None:
# print "NOT SAVING", part_content_type
log_msg = "Not saving '%s', filename '%s'." % (part_content_type,
filename)
log.info(log_msg)
if simulate: print log_msg # Print lots of debug stuff
continue
#print filedata, type(filedata), len(filedata)
if filedata is None or len(filedata) == 0:
log_msg = "Not saving '%s', filename '%s', file has no data" % (
part_content_type, filename)
log.warning(log_msg)
if simulate:
print log_msg # Print lots of debug stuff
continue
log_msg = u'Saving: %s (%s)' % (filename, part_content_type)
log.info(log_msg)
if simulate:
print log_msg # Print lots of debug stuff
c = Content(
user=user,
privacy=privacy,
caption=subject,
author=photographer_name,
#author=sender_nick,
#group=contentgroup,
)
if simulate is False:
log.info("Saving file %s" % filename)
c.set_file(filename, filedata)
log.info("set_fileinfo()")
c.set_fileinfo()
log.info("c.generate_thumbnail()")
c.generate_thumbnail()
c.save()
saved_parts += 1
log.info("Saving really")
else:
log.info("Not saving, simulate %s" % simulate)
m = Metadata(
content=c,
sourceorg=sourceorg,
photographer=photographer,
group=contentgroup,
caption=subject,
author=photographer_name,
geometry=c.point,
)
m.save()
return saved_parts
def addContent(request):
url = request.META.get('HTTP_REFERER')
if request.method == 'POST':
form = DuyuruForm(request.POST, request.FILES)
if form.is_valid():
current_user = request.user
data = Content()
data.user_id = current_user.id
data.title = form.cleaned_data['title']
data.keywords = form.cleaned_data['keywords']
data.description = form.cleaned_data['description']
data.image = form.cleaned_data['image']
data.type = form.cleaned_data['type']
data.slug = form.cleaned_data['slug']
data.detail = form.cleaned_data['detail']
data.status = 'Hayır'
data.save()
messages.success(
request, 'Duyurunuz başarı ile gönderilmiştir.Teşekkür ederiz')
return HttpResponseRedirect(url)
else:
messages.warning(request, 'Form Hatalı!!')
return HttpResponseRedirect(url)
else:
menu = Menu.objects.all()
setting = Setting.objects.get(pk=1)
form = DuyuruForm()
context = {
'setting': setting,
'menu': menu,
'form': form,
}
return render(request, 'userDuyuruEkle.html', context)
def api_upload(request):
"""
Renders the upload form page.
"""
try:
if request.method == 'POST': # If the form has been submitted...
#for header in request.META.keys():
# if header.startswith('HTTP'):
# print header, request.META[header]
#print request.raw_post_data[:1000]
#if request.user.is_authenticated() is False:
# return HttpResponse(status=401)
form = UploadForm(request.POST, request.FILES) # A form bound to the POST data
#validate_authorization(request)
if form.is_valid(): # File was posted with form
c = None
data = dict(request.POST)
jsondata = request.POST.get('data')
if jsondata is None: # Temporary fix for ambiguous parameter name data / metadata
jsondata = request.POST.get('metadata')
if jsondata:
try: # data is a json string containing the same keys as multipart form
data = json.loads(jsondata)
except: # if it was not valid json, use normal post data
data = dict(request.POST)
#raise
SUPPORTED_FIELDS = ['title', 'caption', 'author']
kwargs = {}
for field in SUPPORTED_FIELDS:
kwargs[field] = data.get(field, '')
try:
kwargs['point'] = Point(float(data.get('lon')), float(data.get('lat')))
except:
kwargs['point'] = None
#raise
pass
# Create a new Egg
e = Egg(**kwargs)
print kwargs
response_status = 201 # Created
tags = data.get('tags')
if tags:
tags = [x.lower() for x in tags]
tag_str = ','.join(tags)
else:
tags = []
tag_str = ''
user = c = None
file_md5_sums = []
for filefield, tmpname in handle_uploaded_file(request):
print "HANDLING FILE:", filefield, tmpname
c = Content(**kwargs)
c.keywords = tag_str
originalname = str(request.FILES["file"])
# c.user = request.user # Only authenticated users can use this view
c.set_file(originalname, tmpname) # Save uploaded file to filesystem
digest_maker = hashlib.md5()
with open(c.file.path, 'rb') as f:
buf = f.read(4096)
while buf:
digest_maker.update(buf)
buf = f.read(4096)
file_md5_sums.append(digest_maker.hexdigest())
#print digest_maker.hexdigest()
c.get_type_instance() # Create thumbnail if it is supported
c.save()
# Copy coordinates from content (parsed while saving it if they existed)
e.content = c
if e.point is None and c.point:
e.point = c.point
e.uid = c.uid
break # We save only the first file
# Handle authorization after files are handled
user = validate_4dnest_authorization(request, file_md5_sums)
if c:
c.user = user
c.save()
e.user = user
print "USER", user
if user:
response_status = 201 # Created
else:
response_status = 401 # Unauthorized
e.save()
# We can handle tags after egg has id (it is saved)
#print "LOOPING TAGS"
for tagname in tags:
try:
tag = Tag.objects.get(name=tagname)
#print "Tag old:",
except Tag.DoesNotExist:
tag = Tag(name=tagname)
tag.save()
#print "Tag new:",
#print tagname, tag
e.tags.add(tag)
response = HttpResponse(status=response_status)
#response.status_code = 201
# FIXME: use reverse()
#return HttpResponseRedirect(reverse('egg api', args=[e.uid]))
response['Location'] = '/fourdnest/api/v1/egg/%s/' % e.uid
return response
else:
response = HttpResponse(status=204)
return response
else:
raise Http404
except Exception, err:
print err
raise
return HttpResponse("Server error", status=500)
def savefiles(msg):
"""
Extract parts from msg (which is an email.message_from_string(str) instance)
and send them to the plok-database.
NOTES:
- uses only the first found email address to assume recipient
"""
part_counter = 1
subject = get_subject(msg)
tos = get_recipient(msg)
msg_id = msg.get('message-id', '')
froms = msg.get_all('from', [])
p = re.compile('([\w\.\-]+)@')
try: # May raise in some cases IndexError: list index out of range
matches = p.findall(froms[0])
sender_nick = matches[0].split(".")[0].title() # Use all before first '.'
except:
print "ERROR: No From header %s" % (msg_id)
return False
print subject, tos, froms, msg_id
if len(tos) == 0:
print "ERROR: No Tos found %s" % (msg_id)
return False
p = re.compile('([\w]+)\.([\w]+)@') # e.g. [email protected]
matches = p.findall(tos[0])
if len(matches) > 0:
user = matches[0][0].title()
key = matches[0][1].lower()
else:
print "ERROR: No user.authkey found from %s %s" % (tos[0], msg_id)
return False
# TODO: replace this with AuthTicket stuff
from django.contrib.auth import authenticate
user = authenticate(authtoken='qwerty123')
print user
parts_not_to_save = ["multipart/mixed",
"multipart/alternative",
"multipart/related",
"text/plain",
]
saved_parts = 0
for part in msg.walk():
part_content_type = part.get_content_type()
if part_content_type in parts_not_to_save:
# print "NOT SAVING", part_content_type
continue
print part_content_type
filename, filedata = handle_part(part)
if filename is None:
print "No filename"
continue
c = Content(
user = user,
#privacy = legal_key_map[key],
caption = subject,
author = sender_nick,
)
c.set_file(filename, filedata)
c.get_type_instance()
c.save()
print c
saved_parts += 1
return saved_parts
def test_duplicates_raise_exception(self):
c = Content.objects.create(text="First")
with self.assertRaises(ValidationError):
c2 = Content(text="First")
c2.save()
c2.full_clean()
def addcontent(request):
if request.method == 'POST':
form = ContentForm(request.POST, request.FILES)
print("post girdi")
if form.is_valid():
print("valid girdi")
current_user = request.user
data = Content()
data.user_id = current_user.id
data.title = form.cleaned_data['title']
data.keywords = form.cleaned_data['keywords']
data.description = form.cleaned_data['description']
data.image = form.cleaned_data['image']
data.slug = form.cleaned_data['slug']
data.detail = form.cleaned_data['detail']
data.category_id = form.cleaned_data['category'].id
data.status = 'False'
data.save()
messages.success(request, 'Your Content Inserted Successfully')
return HttpResponseRedirect('/user/contents')
else:
print("hata girdi")
messages.success(request,
'Content Form Error : ' + str(form.errors))
return HttpResponseRedirect('/user/addcontent')
else:
category = Category.objects.all()
form = ContentForm()
context = {'category': category, 'form': form}
return render(request, 'user_addcontent.html', context)
def addcontent(request):
setting = Setting.objects.get(pk=1)
current_user = request.user
profile = UserProfile.objects.get(user_id=current_user.id)
if request.method == 'POST':
form = ContentForm(request.POST, request.FILES)
if form.is_valid():
current_user = request.user
data = Content()
data.user_id = current_user.id
data.title = form.cleaned_data['title']
data.keywords = form.cleaned_data['keywords']
data.description = form.cleaned_data['description']
data.image = form.cleaned_data['image']
data.type = form.cleaned_data['type']
data.slug = form.cleaned_data['slug']
data.detail = form.cleaned_data['detail']
data.status = 'False'
data.save()
messages.success(request, 'Your Content Inserted Succesfully')
return HttpResponseRedirect('/user/contents')
else:
messages.success(request,
'Content Form Error : ' + str(form.errors))
return HttpResponseRedirect('/user/addcontent')
else:
category = Categories.objects.all()
menu = Menu.objects.all()
form = ContentForm()
context = {
'category': category,
'menu': menu,
'form': form,
'profile': profile,
'setting': setting,
}
return render(request, 'user_addcontent.html', context)
def add_content(request):
if request.method == 'POST':
form = ContentForm(request.POST, request.FILES)
if form.is_valid():
current_user = request.user
data = Content()
data.user_id = current_user.id
data.category = form.cleaned_data['category']
data.title = form.cleaned_data['title']
data.keywords = form.cleaned_data['keywords']
data.description = form.cleaned_data['description']
data.image = form.cleaned_data['image']
data.slug = form.cleaned_data['slug']
data.detail = form.cleaned_data['detail']
data.status = 'False'
data.save()
messages.success(request, 'Your trip has been added.')
return HttpResponseRedirect('/user/contents')
else:
messages.error(request, 'Content Form Error: ', + str(form.errors))
return HttpResponseRedirect('/user/add_content')
else:
form = ContentForm()
context = {
'form': form
}
return render(request, 'add_content.html', context)
def addcontent(request):
if request.method == 'POST':
form = ContentForm(request.POST, request.FILES)
if form.is_valid():
current_user = request.user
data = Content()
data.user_id = current_user.id
data.title = form.cleaned_data['title']
data.keywords = form.cleaned_data['keywords']
data.description = form.cleaned_data['description']
data.image = form.cleaned_data['image']
data.type = form.cleaned_data['type']
data.slug = form.cleaned_data['slug']
data.detail = form.cleaned_data['detail']
data.status = 'False'
data.save() # veritabanına kaydet
messages.success(request, 'Your content insterted successfully!')
return HttpResponseRedirect('/user/contents')
else:
messages.error(request, 'Content Form Error :' + str(form.errors))
return HttpResponseRedirect('/user/addcontent')
else:
category = Category.objects.all()
menu = Menu.objects.all()
form = ContentForm()
return render(request, 'user_addcontent.html', {'form': form, 'category': category, 'menu': menu, })
def process_item(self, item, spider):
'''
Function that processes each item
after it has been scraped.
Its purpose is to create content
objects for each keyword on the page.
'''
stop_words = []
stop_words_obj_list = StopWord.objects.all()
for stop_word_obj in stop_words_obj_list:
stop_words.append(stop_word_obj.stopword.strip())
stop_words.append("")
temp_word_list = []
text = item["text"].encode("ascii","ignore")
text = text.lower().translate(None,"\r\"!\\/?,:|][*&£^`¬;()").translate(None,digits)
temp_word_list = text.split(" ")
word_list = [word.strip(" \'").rstrip("-.") for word in temp_word_list if word.rstrip("-.") not in stop_words and word.strip("\n \t") != ""]
page_text = ' '.join(word_list)
domains = Domain.objects.all()
domain_obj = None
for domain in domains:
temp_domain = domain.domain.replace("http://","").replace("www.","").rstrip("/")
if temp_domain in item["link"]:
domain_obj = domain
try:
page = Page.objects.get(domain = domain_obj, pageURL=item["link"])
except Page.DoesNotExist:
page = None
if page:
if page.text == page_text:
return item
else:
page.text = page_text
page.has_adscore = False
page.save()
else:
page = Page(domain=domain_obj, pageURL=item["link"], text=page_text)
page.save()
counter_list = Counter(word_list)
for k,v in counter_list.items():
if k.strip() == "" or k is None:
continue
try:
keyword = Keyword.objects.get(keyword=k)
except Keyword.DoesNotExist:
keyword = None
if keyword:
temp_content = Content(page=page, keyword=keyword, occurrence=v)
temp_content.save()
else:
temp_keyword = Keyword(keyword=k)
temp_keyword.save()
temp_content = Content(page=page, keyword=temp_keyword, occurrence=v)
temp_content.save()
return item
def handle(self, *args, **options):
starting_message_id = None
for i in range(350):
print(f'on batch {i}')
messages_url = f"{BASE_URL}groups/{GROUP_ID}/messages?token={TOKEN}&limit=100"
if starting_message_id:
messages_url += f"&before_id={starting_message_id}"
response = requests.get(messages_url)
if response.status_code != 200:
print(response)
continue
content = json.loads(response.content.decode())
message_list = content['response']['messages']
for message in message_list:
if Content.objects.filter(name=message['id'], media_url__isnull=False).exists():
print(f'seen message with id {message["id"]} before, exiting')
return
try:
if message['attachments'] and len(message['favorited_by']) > 1:
try:
member = Member.objects.get(groupme_id=message['user_id'])
except Member.DoesNotExist:
member = Member.objects.create(
groupme_id=message['user_id'],
name=message['name'],
avatar_url=message['avatar_url']
)
if not member.avatar_url and message['avatar_url']:
member.avatar_url = message['avatar_url']
member.save()
Nickname.objects.get_or_create(
nickname=message['name'],
member=member
)
attachment = message['attachments'][0]
url = attachment.get('url') or attachment.get('source_url') or attachment.get('preview_url')
print(url)
if url:
file_type = self.get_file_type(url)
if file_type:
if file_type == '.mp4':
kind = 'VIDEO'
else:
kind = 'IMAGE'
print(file_type)
file_name = '{}{}'.format(message['id'], file_type)
response = requests.get(url, stream=True)
if response.status_code != 200:
continue
lf = tempfile.NamedTemporaryFile()
for block in response.iter_content(1024 * 8):
if not block:
break
lf.write(block)
kwargs = {
"create_date": datetime.datetime.fromtimestamp(message['created_at']),
"creator": member,
"creator_nickname": message['name'],
"kind": kind,
"likes": len(message['favorited_by']),
"name": message['id'],
"text": self.strip_urls(message['text']),
"media_url": url
}
content = Content(**kwargs)
content.upload.save(file_name, files.File(lf))
else:
print(message)
print('found new url type, in above message')
elif not message['attachments'] and len(message['favorited_by']) >= 1:
try:
member = Member.objects.get(groupme_id=message['user_id'])
except Member.DoesNotExist:
member = Member.objects.create(
groupme_id=message['user_id'],
name=message['name'],
avatar_url=message['avatar_url']
)
if not member.avatar_url and message['avatar_url']:
member.avatar_url = message['avatar_url']
member.save()
Nickname.objects.get_or_create(
nickname=message['name'],
member=member
)
kwargs = {
"avatar_url": message['avatar_url'],
"creator": member,
"creator_nickname": message['name'],
"create_date": datetime.datetime.fromtimestamp(message['created_at']),
"kind": 'TEXT',
"likes": len(message['favorited_by']),
"name": message['id'],
"text": self.strip_urls(message['text'])
}
Content.objects.create(**kwargs)
except Exception as e:
print(e)
finally:
try:
message_list[message_list.index(message) + 1]
except IndexError:
starting_message_id = message['id']
print(f'new starting message id: {starting_message_id}')
def get_footer_introduction():
return Content.get_solo().footer_introduction
def addcontent(request):
if request.method == "POST":
form = ContentForm(request.POST, request.FILES)
if form.is_valid():
data = Content()
data.user_id = request.user.id
data.title = form.cleaned_data['title']
data.keywords = form.cleaned_data['keywords']
data.description = form.cleaned_data['description']
data.image = form.cleaned_data['image']
data.kind = form.cleaned_data['kind']
data.slug = form.cleaned_data['slug']
data.detail = form.cleaned_data['detail']
data.status = "False"
data.save()
messages.success(request, 'Content added!')
return HttpResponseRedirect('/user/contents')
else:
messages.warning(request, 'Content Form Error:' + str(form.errors))
return HttpResponseRedirect('/user/addcontent')
else:
setting = Setting.objects.first()
category = Category.objects.filter(status=True)
menu = Menu.objects.filter(status=True)
form = ContentForm()
context = {
'setting': setting,
'category': category,
'menu': menu,
'page': 'Add Content',
'form': form
}
return render(request, 'user_addcontent.html', context)
def addcontent(request):
if request.method == 'POST':
form = ContentForm(request.POST, request.FILES) # fileupload varsa request.FILES yapmazsak formumuz çalışmaz
if form.is_valid():
current_user = request.user
data = Content() # model ile bağlantı kur
data.user_id = current_user.id
data.title = form.cleaned_data['title']
data.keywords = form.cleaned_data['keywords']
data.description = form.cleaned_data['description']
data.image = form.cleaned_data['image']
data.type = form.cleaned_data['type']
data.slug = form.cleaned_data['slug']
data.detail = form.cleaned_data['detail']
data.status = 'False'
data.save() # veritabanına kaydet
messages.success(request, 'İçeriğiniz başarılı bir şekilde eklendi !')
return HttpResponseRedirect("/user/contents")
else:
messages.error(request, 'İçerik Form Hatası : ' + str(
form.errors)) # bunu buraya yazabilmemiz için stringe çevirmemiz lazım on yüzden str yazıyoruz
return HttpResponseRedirect('/user/addcontent')
else:
menu = Menu.objects.all()
form = ContentForm() # content form ile ilişki kuruyorum
context = {
'form': form,
'menu': menu
}
return render(request, 'user_addcontent.html', context)
def addcontent(request):
if request.method == 'POST':
form = ContentForm(request.POST, request.FILES)
if form.is_valid():
current_user = request.user
data = Content()
data.user_id = current_user.id
data.title = form.cleaned_data['title']
data.price = form.cleaned_data['price']
data.keywords = form.cleaned_data['keywords']
data.description = form.cleaned_data['description']
data.image = form.cleaned_data['image']
data.type = form.cleaned_data['type']
data.slug = form.cleaned_data['slug']
data.detail = form.cleaned_data['detail']
data.status = 'False'
data.save()
messages.success(request, "İçerik başarı ile eklendi")
return HttpResponseRedirect('/user/contents')
else:
messages.success(request, 'İçerik form hatası' + str(form.errors))
return HttpResponseRedirect('/user/addcontent')
else:
category = Category.objects.all()
menu = Menu.objects.all()
form = ContentForm()
context = {
'category': category,
'form': form,
'menu': menu,
}
return render(request, 'user_addcontent.html', context)
def addcontent(request):
setting = Setting.objects.get(pk=1)
last_posts = Post.objects.filter(status=True).order_by('-id')[:4]
if request.method == 'POST':
form = ContentForm(request.POST, request.FILES)
if form.is_valid():
current_user = request.user
data = Content()
data.user_id = current_user.id
data.type = form.cleaned_data['type']
data.title = form.cleaned_data['title']
data.keywords = form.cleaned_data['keywords']
data.description = form.cleaned_data['description']
data.image = form.cleaned_data['image']
data.slug = form.cleaned_data['slug']
data.detail = form.cleaned_data['detail']
data.status = 'False'
data.save()
messages.success(request, 'İçeriğiniz başarıyla kaydedildi.')
return HttpResponseRedirect('/user/contents')
else:
messages.error(request, 'Lütfen hatalı alanları kontrol ediniz.<br>'+ str(form.errors))
return HttpResponseRedirect('/user/contents')
else:
category = Category.objects.all()
menu = Menu.objects.all()
form = ContentForm()
context = {'setting': setting,
'menu': menu,
'category': category,
'form': form,
'last_posts': last_posts,
}
return render(request, 'user_addcontent.html', context)
