def showDownloadDash(request,cloudItem,t):
""" Displays the dashboard of the download """
data = {}
try:
ci = checkCloudItem(cloudItem,request.user.id)
at = checkAccessToken(t,ci)
data['showToken'] = True
data['credVerified'] = "Not started"
data['metaWait'] = "Not started"
data['downSize'] = "Not started"
data['fileWait'] = "Not started"
data['verificationWait'] = "Not started"
data['objID'] = ci.id
data['tokenID'] = at.id
data['form'] = TSCredentialsForm()
#button has been clicked
if request.method == "POST":
subForm = TSCredentialsForm(request.POST)
if subForm.is_valid():
down = Download.objects.get(tokenID=at)
down.threadStatus = constConfig.THREAD_CLICKED
down.save()
pwd = subForm.cleaned_data['pwd']
account = subForm.cleaned_data['uname']
#start the celery asynchronous task
download.delay(down,account,pwd)
else:
raise Exception("Invalid form")
#default check to start the periodically ajax function
try:
# check to start
down = Download.objects.get(tokenID=at)
except Download.DoesNotExist:
#store a new download that has not been clicked yet
down = Download(threadStatus=constConfig.THREAD_NOTCLICKED,tokenID=at,folder=sessionName(t))
down.save()
if down:
data['down'] = down
data['downStatus'] = down.threadStatus
data['downMessage'] = down.threadMessage
except Exception as e:
data['errors'] = formatException(e)
return render_to_response("dashboard/down.html", data, context_instance=RequestContext(request))
def home(request):
if request.method == 'POST':
form = VideoForm(request.POST)
if form.is_valid():
url = form.cleaned_data['url']
mail = form.cleaned_data['email']
from tasks import download
download.delay(url, mail)
#form = VideoForm()
else:
form = VideoForm()
return render_to_response('index.html', {'video': form}, context_instance=RequestContext(request))
def home(request):
if request.method == 'POST':
form = VideoForm(request.POST)
if form.is_valid():
url = form.cleaned_data['url']
mail = form.cleaned_data['email']
from tasks import download
download.delay(url, mail)
#form = VideoForm()
else:
form = VideoForm()
return render_to_response('index.html', {'video': form},
context_instance=RequestContext(request))
def index(request, response):
if request.method == "POST":
url = request.POST.get('url')
email = request.POST.get('email')
# song = Song(link=url)
# session.add(song)
# session.commit()
create_query(url, email)
download.delay(url, email)
response.text = "Вам будет отправлено письмо на email"
return response
data = render_template("index.html")
response.text = data
return response
def test():
c = Baidu_crawler()
for i in c.baidu_crawler():
if i:
for url in i:
result = download.delay(url)
if result.ready():
url, html = result.get()
print(url)
collections.insert(result.id, url, html)
time.sleep(5)
def test2():
n = 0
tasks = []
c = Baidu_crawler()
urls = c.baidu_crawler()
for url_list in urls:
if url_list:
for url in url_list:
result = download.delay(url)
print(result.task_id)
tasks.append(result.id)
print(len(tasks))
time.sleep(60)
for i in tasks:
if test3(i):
n += 1
print(n)
from tasks import download
from tqdm import tqdm
remaining = [s.strip() for s in open('remaining-videos.txt', 'r').readlines()]
for video in tqdm(remaining):
download.delay(video)
def process_records(records):
"""Process the provided records.
Uses a Celery task to download emails and extract plain text, then stores
the resulting data in the database for later use.
Arguments:
records (dict): Records retrieved from the State Department.
Returns:
None
"""
print "Processing records..."
start = time()
results = []
emails = records['Results']
for email in emails:
results.append(download.delay(email))
emails = []
for result in results:
data = result.get()
if data is not None:
sender = data['from']
recipient = data['to']
sent = data['sent']
subject = data['subject']
body = data['body']
pdf_path = data['pdf_path']
pdf_link = data['pdf_link']
pdf_posted = data['pdf_posted']
case_number = data['caseNumber']
document_class = data['documentClass']
document_id = data['document_id']
is_redacted = data['is_redacted']
email = Email(
sender=sender,
recipient=recipient,
sent=sent,
subject=subject,
body=body,
pdf_path=pdf_path,
pdf_link=pdf_link,
pdf_posted=pdf_posted,
case_number=case_number,
document_class=document_class,
document_id=document_id,
is_redacted=is_redacted,
)
emails.append(email)
# Bulk insert for improved performance.
session.add_all(emails)
session.commit()
end = time()
print "Complete in {} seconds.".format(end - start)
