def fetch(session, url):
try:
r = requests.get(url.address, allow_redirects=True)
except:
print("invalid url")
return
db.update_url(session, url.address, "last_fetched",
datetime.datetime.now())
# get files from db associated with this url
files = db.get_files_by_url(session, url.id)
last_modified = r.headers.get('Last-Modified')
for f in files:
# check head to see when file was last modified and if it matches a file in the db
if last_modified == f.last_modified:
return
# if we make it here, no file's last modified value matched, so we have a new file version
# get name and store
filename = get_filename_from_cd(r, url.address)
# print(r.content)
print(filename)
open("file_storage/" + filename, 'wb').write(r.content)
# add file info to db
db.add_file(session, filename, url.id, "file_storage" + filename, False,
last_modified)
# make call to verification apps
return
def post(message):
if "files" in message.body.keys():
labels = set()
for file in message.body["files"]:
label = animal.predict(file["url_private_download"])
if label is not None:
db.add_file(file["permalink"], label)
labels.add(label)
for l in labels:
message.react(animal.get_emoji(l))
def post(self):
room_id = self.get_argument('room', default='')
if room_id not in db.rooms:
self.write({
'result': False,
'msg': 'Invalid room id',
})
return
if 'file' not in self.request.files:
self.write({
'result': False,
'msg': 'File not provided',
})
return
upload_file = self.request.files['file'][0]
hsh = db.add_file(upload_file['filename'], upload_file['body'])
msg = '<a href="%s/getfile?file=%s">%s</a>' % (
'http://index.lv6.tw:8888',
hsh,
upload_file['filename'],
)
db.send_message(room_id, self.get_user(), msg)
self.write({
'result': True,
'msg': 'Send file success',
})
def post(self):
room_id = self.get_argument('room', default='')
if room_id not in db.rooms:
self.write({
'result': False,
'msg': 'Invalid room id',
})
return
if 'file' not in self.request.files:
self.write({
'result': False,
'msg': 'File not provided',
})
return
upload_file = self.request.files['file'][0]
hsh = db.add_file(upload_file['filename'], upload_file['body'])
msg = '<a href="%s/getfile?file=%s">%s</a>' % (
'http://index.lv6.tw:8888',
hsh,
upload_file['filename'],
)
db.send_message(room_id, self.get_user(), msg)
self.write({
'result': True,
'msg': 'Send file success',
})
def real_worker(self, which):
while True:
ti = self.queue.get()
start_time = time.time()
self.lock.acquire()
url = ti['url']
tag = ti['tag']
ti['start_time'] = time.time()
self.lock.release()
try:
# Upload the track for analysis
ti['status'] = 'transferring'
code, message, track = uploader(ti['url'], ti['tag'], ti['api_key'])
ti['message'] = message
if code == 0 and track:
trid = track['id']
ti['status'] = 'analyzing'
code, message = check_profile(trid, ti['api_key'])
ti['message'] = message
if code == 0:
ti['trid'] = trid
# Add the file to the database
db.add_file(trid, track['artist'], track['title'], tag, url)
ti['status'] = 'done'
else:
ti['status'] = 'error'
else:
ti['status'] = 'error'
except Exception as e:
ti['message'] = 'internal error'
ti['status'] = 'error'
error_log(ti['url'])
delta_time = time.time() - start_time
self.lock.acquire()
self.proc_count += 1
self.tot_proc_time += delta_time
self.avg_proc_time = self.tot_proc_time / self.proc_count
if ti['status'] == 'error':
self.proc_errors += 1
self.lock.release()
def put(lpath, spath):
"""Uploads file from lpath to spath"""
lpath = os.path.expanduser(lpath)
abs_path = navigate.get_abs_path(spath)
parent, name = navigate.split_path(abs_path)
up_parent, up_name = navigate.split_path(parent)
file_size = os.stat(lpath).st_size
if up_parent is not None and not db.directory_exists(up_parent, up_name):
print "Error: '" + parent + "' is not a valid directory."
elif db.file_exists(parent, name):
print "Error: '" + spath + "' already exists."
elif file_size > 2 * 1024 * 1024 * 1024 - ACCOUNT_BUFFER:
print "Error: individual files must be 2GB or smaller."
else:
dbox_path = '/' + name
access_token = accounts.get_useable_account(file_size)
client = dropbox.client.DropboxClient(access_token)
lfile = open(lpath)
client.put_file(dbox_path, lfile)
lfile.close()
db.add_file(access_token, parent, name)
def file_filter(cur_file: File, is_file): # true가 우리가 찾는 파일 or 그냥 폴더
if cur_file.name not in FILE_FILTER['SELECT_NAME']:
if ignore_regex.search(cur_file.name):
return False
if not is_file:
return True
if not extension_regex.search(cur_file.name):
return False
prev_file = db.get_file(cur_file.fullname, cur_file.repo_fullname)
if not prev_file: # db에 파일이 기록되지 않음
db.add_file(cur_file)
return True
prev_file = db.file_dict_to_obj(prev_file)
if cur_file.sha != prev_file.sha: # db에 파일이 기록됨 -> 업데이트 되었는가?
db.update_file(fullname=prev_file.fullname, sha=cur_file.sha)
return True
return False
def dropMimeData(self, data, action, row, column, parent):
new_file = []
for url in data.urls() or list():
file_path = url.toLocalFile()
file_name = os.path.split(file_path)[1]
if file_name.split('.')[-1] not in ["ma", "mb"]: continue
new = True
for f in db.file_list():
if file_path == f["source"]: new = False
if not new: continue
prefix, suffix = file_path.split("/DB/")
target_path = prefix + "/DB/Common/Mods/" + suffix
new_file.append({"name":file_name, "source":file_path, "target":target_path, "status":"wait"})
if not new_file: return False
db.add_file(*new_file)
self.dataChanged.emit(QtCore.QModelIndex(), QtCore.QModelIndex())
return True