def download():
leads = current_user.leads
idc = current_user.id
dnw = Download.query.filter_by(cliente=idc).order_by(
Download.id.desc()).first()
if dnw:
num = dnw.qtd + leads
qtm = dnw.qtm
if dnw.desc == False:
dnr = Download(cliente=idc,
data=datetime.now(),
qtd=num,
qtm=num,
desc=True)
else:
dnr = Download(cliente=idc, data=datetime.now(), qtm=num)
asc = Asc.query.filter(Asc.id.between(qtm, num))
else:
asc = Asc.query.filter(Asc.id.between('0', leads))
dnr = Download(cliente=idc, data=datetime.now(), qtm=leads, desc=True)
# Fazer download do arquivo em csv
with open('exportar.csv', 'w', newline='') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',')
csvwriter.writerow(["Nome", "Email", "Telefone"])
for p in asc:
csvwriter.writerow([p.nome, p.email, p.tele])
db.session.add(dnr)
db.session.commit()
return send_file('exportar.csv',
mimetype='text/csv',
attachment_filename='exportar.csv',
as_attachment=True,
cache_timeout=0)
def showDownloadDash(request,cloudItem,t):
""" Displays the dashboard of the download """
data = {}
try:
ci = checkCloudItem(cloudItem,request.user.id)
at = checkAccessToken(t,ci)
data['showToken'] = True
data['credVerified'] = "Not started"
data['metaWait'] = "Not started"
data['downSize'] = "Not started"
data['fileWait'] = "Not started"
data['verificationWait'] = "Not started"
data['objID'] = ci.id
data['tokenID'] = at.id
data['form'] = TSCredentialsForm()
#button has been clicked
if request.method == "POST":
subForm = TSCredentialsForm(request.POST)
if subForm.is_valid():
down = Download.objects.get(tokenID=at)
down.threadStatus = constConfig.THREAD_CLICKED
down.save()
pwd = subForm.cleaned_data['pwd']
account = subForm.cleaned_data['uname']
#start the celery asynchronous task
download.delay(down,account,pwd)
else:
raise Exception("Invalid form")
#default check to start the periodically ajax function
try:
# check to start
down = Download.objects.get(tokenID=at)
except Download.DoesNotExist:
#store a new download that has not been clicked yet
down = Download(threadStatus=constConfig.THREAD_NOTCLICKED,tokenID=at,folder=sessionName(t))
down.save()
if down:
data['down'] = down
data['downStatus'] = down.threadStatus
data['downMessage'] = down.threadMessage
except Exception as e:
data['errors'] = formatException(e)
return render_to_response("dashboard/down.html", data, context_instance=RequestContext(request))
def __init__(self, slug, file_path, timeout_hours=None, max_clicks=None):
"""creates a new dynamic link"""
self.new_link = Download()
self.new_link.slug = slug
self.new_link.file_path = file_path
if timeout_hours:
self.new_link.timeout_hours = timeout_hours
if max_clicks:
self.new_link.max_clicks = max_clicks
self.new_link.save()
def downloads_new():
download = Download()
form = DownloadForm(request.form, download)
if form.validate_on_submit():
f = request.files['url']
if f:
filename = secure_filename(f.filename)
f.save(os.path.join(neobug.config['UPLOAD_FOLDER'], filename))
form.populate_obj(download)
download.url = 'static/uploads/' + filename
download.save()
return redirect('/downloads')
return render_template('downloads_new.html',
form=form)
def download_file():
download_form = DelayForm()
if request.method == 'POST':
if download_form.validate_on_submit():
# Get validated data from form
delay = int(download_form.delay.data)
client_ip = request.remote_addr
time.sleep(delay)
# save user to database
record = Download(client_ip, "placeholder", delay)
db.session.add(record)
db.session.commit()
csv = '1,2,3\n4,5,6\n'
return Response(csv,
mimetype="text/csv",
headers={
"Content-disposition":
"attachment; filename=somefile.csv"
})
flash_errors(download_form)
return render_template('download.html', form=download_form)
def download_stop_handler(self, key):
print '**** Download stopped. Going to handle it ...', key
try:
dl_info = Download.get(Download.key == key)
dl_info.status = 'Finished'
dl_info.save()
except:
print 'Problem in updating download info'
del self.pool[key]
def __init__(self, slug, file_path, timeout_hours=None, max_clicks=None):
"""creates a new dynamic link"""
self.new_link = Download()
self.new_link.slug = slug
self.new_link.file_path = file_path
if timeout_hours:
self.new_link.timeout_hours = timeout_hours
if max_clicks:
self.new_link.max_clicks = max_clicks
self.new_link.save()
class DynamicLink:
"""
create and access dynamic links form outsite of this app
"""
def __init__(self, slug, file_path, timeout_hours=None, max_clicks=None):
"""creates a new dynamic link"""
self.new_link = Download()
self.new_link.slug = slug
self.new_link.file_path = file_path
if timeout_hours:
self.new_link.timeout_hours = timeout_hours
if max_clicks:
self.new_link.max_clicks = max_clicks
self.new_link.save()
def get_link_key(self):
"""get his unique key"""
return self.new_link.link_key
def get_link_url(self, request, langcode='lg'):
"""access his url"""
return file_link_url(request, self.new_link, langcode)
class DynamicLink:
"""
create and access dynamic links form outsite of this app
"""
def __init__(self, slug, file_path, timeout_hours=None, max_clicks=None):
"""creates a new dynamic link"""
self.new_link = Download()
self.new_link.slug = slug
self.new_link.file_path = file_path
if timeout_hours:
self.new_link.timeout_hours = timeout_hours
if max_clicks:
self.new_link.max_clicks = max_clicks
self.new_link.save()
def get_link_key(self):
"""get his unique key"""
return self.new_link.link_key
def get_link_url(self, request, langcode='lg'):
"""access his url"""
return file_link_url(request, self.new_link, langcode)
async def on_url(self, event: NewMessage.Event):
if self.youtube_manager.is_downloading(event.chat_id):
self.logger.debug('Already downloading for {}'.format(
event.chat_id))
await event.respond('Something already downloading. Please wait')
return
url: str = event.raw_text
self.logger.debug('Url received: {}'.format(url))
target_message = await event.respond('Checking...')
info = await self.youtube_manager.get_info(url)
if info is None:
self.logger.debug('Video was not found')
await event.respond('Video was not found', buttons=Button.clear())
return
download_id = random_string(16)
with use_session(self.session_cls, autocommit=True) as db_session:
download = Download(event.chat_id, download_id, url, info['title'],
info['duration'], json.dumps(info['formats']))
db_session.insert_or_replace(download)
self.logger.debug('Download object was created for {}'.format(
info['title'][:30]))
await event.client.delete_messages(
event.chat_id, [event.message.id, target_message.id])
await event.respond(
'<b>{}</b>\nChoose download type:'.format(info['title']),
buttons=[
Button.inline(
'Video',
data=f'type_{download_id}_{DownloadType.VIDEO.name}'),
Button.inline(
'Audio (default)',
data=f'type_{download_id}_{DownloadType.AUDIO_DEFAULT.name}'
),
Button.inline(
'Audio (custom)',
data=f'type_{download_id}_{DownloadType.AUDIO_CUSTOM.name}'
)
])
def run(self):
print 'Download Manager Started ...'
while True:
if self.stop_flag:
self.stop_downlods()
break
while not self.queue.empty():
req = self.queue.get()
dl = Download()
dl.link = req['link']
dl.username = req.get('username')
dl.password = req.get('password')
dl.save(force_insert=True)
print dl
if len(self.pool) < config.MAX_JOBS:
self.start_new_download(dl)
sleep(2)
def start_stream(self):
subreddit = self.reddit.subreddit(self.subreddit_name)
for submission in subreddit.stream.submissions():
result = self.download_submission(submission)
if result:
now = datetime.datetime.now()
download = Download(thing_id=submission.id,
title=submission.title,
url=submission.url,
path=result['output_dir'],
created_at=now,
updated_at=now,
downloaded_at=now)
self.db.session.add(download)
self.db.session.commit()
self.post_download_hook(self, result)
self.logger.debug("Download successful: {}".format(
repr(result)))
def start_download(download_link,
download_location,
file_name,
download_headers={}):
# check if file already exists
if ospath.exists(download_location + file_name):
return None
# generate uuid for identification
id = str(uuid.uuid4())
session = Session()
download = Download(id, download_link, download_location + file_name,
'CREATED_THREAD', '0 MB', '0 MB', datetime.now(),
datetime.now())
session.add(download)
session.commit()
Session.close()
executor.submit(download_file, id, download_link, download_headers,
download_location, file_name)
return id
def new_download():
if request.method == 'POST':
dl_url = request.form.get('dl_url').strip()
path = request.form.get('path').strip()
username = request.form.get('username')
password = request.form.get('password')
dl_account = request.form.get('dl_account')
dl = Download(url=dl_url, path=path)
if username:
dl.username = username
if password:
dl.password = password
if dl_account != 'None':
dl.account = DownloadAccount.select().where(
DownloadAccount.pk == int(dl_account))[0]
dl.save()
flash('New download added successfully.', 'alert-success')
return redirect(url_for('index'))
return render_template('new_download.html',
dl_accounts=DownloadAccount.select())
def index():
return render_template('index.html', downloads=Download.select())
def process_resource(self, resource_entry):
"""Transform each ReportToProcess into a proper Report."""
# Load variables from stored entity
t = resource_entry.t
gbifdatasetid = resource_entry.gbifdatasetid
event = resource_entry.resource
s = "Version: %s\n" % __version__
s += "Processing %s" % gbifdatasetid
logging.info(s)
# Extract useful information
number_of_records = event['records']
query_countries = [
QueryCountry(**x) for x in event['query_countries'].values()
]
query_dates = [
QueryDate(query_date=datetime.strptime(x['query_date'],
'%Y-%m-%d'),
times=x['times']) for x in event['query_dates'].values()
]
query_terms = [QueryTerms(**x) for x in event['query_terms'].values()]
# Build report ID
report_id = "|".join([self.period, gbifdatasetid])
# Build dataset key
dataset_key = ndb.Key("Dataset", gbifdatasetid)
# Build period key
period_key = ndb.Key("Period", self.period)
# QC
sum_query_countries = 0
for i in event['query_countries'].values():
sum_query_countries += i['times']
sum_query_dates = 0
for i in event['query_dates'].values():
sum_query_dates += i['times']
sum_query_terms = 0
for i in event['query_terms'].values():
sum_query_terms += i['times']
if sum_query_countries != sum_query_dates or \
sum_query_countries != sum_query_terms or \
sum_query_dates != sum_query_terms:
logging.warning("WARNING: lengths of query entities keys list"
"do not match:")
logging.warning("Query countries: %d" % sum_query_countries)
logging.warning("Query dates: %d" % sum_query_dates)
logging.warning("Query terms: %d" % sum_query_terms)
number_of_events = max([
sum_query_countries, sum_query_countries, sum_query_countries
])
else:
number_of_events = sum_query_countries
# Get existing or create new Report entity
s = "Version: %s\n" % __version__
s += "Retrieving existing report or creating new one"
logging.info(s)
report = Report.get_or_insert(
report_id,
parent=period_key,
created=datetime.today(),
reported_period=period_key,
reported_resource=dataset_key,
searches=Search(
events=0,
records=0,
query_countries=[],
query_dates=[],
query_terms=[],
# status="in progress"
),
downloads=Download(
events=0,
records=0,
query_countries=[],
query_dates=[],
query_terms=[],
# status="in progress"
),
stored=False,
issue_sent=False)
# Populate event data
s = "Version: %s\n" % __version__
s += "Storing %s data" % t
logging.info(s)
if t == 'search':
report.searches.records = number_of_records
report.searches.events = number_of_events
report.searches.query_countries = query_countries
report.searches.query_dates = query_dates
report.searches.query_terms = query_terms
elif t == 'download':
report.downloads.records = number_of_records
report.downloads.events = number_of_events
report.downloads.query_countries = query_countries
report.downloads.query_dates = query_dates
report.downloads.query_terms = query_terms
return report
def create_database():
Download.create_table()
DownloadAccount.create_table()
return redirect('/')
